思路:
数据切割。分批次,批量插入。每次批量插入1000条数据。如果万级数据,循环插入10次就完成了。根据实际情况调整。————如果数据量是十万级,百万级以上,可以考虑多线程、并行流快速处理。
private final static int SIZE = 1000;
public static void main(String[] args) throws Exception {
List<InfoDto> feeList = new ArrayList<>();
for (int i = 0; i < 10000; i++) {
InfoDto infoDto = new InfoDto();
infoDto.setName("name" + i);
infoDto.setBirthday("birthday" + i);
infoDto.setAge("" + i);
infoDto.setSex("" + i);
feeList.add(infoDto);
}
insertBigData.insertData(feeList);
}
public static void insertData(List list) {
if (list == null || list.size() < 1) {
return;
}
List<List> streamList = new ArrayList<>();
for (int i = 0; i < list.size(); i += SIZE) {
int j = Math.min((i + SIZE), list.size());
List subList = list.subList(i, j);
streamList.add(subList);
}
System.out.println("分割:");
for (List list1:streamList) {
insertFrozenBuckleList(list1);
}
}
public static Consumer<List> insertFrozenBuckleList(List<InfoDto> sourceList) {
Consumer<List> consumer = (List x) -> {
List<InfoDto> infoDtoList = (List<InfoDto>) x;
System.out.println("批量导入插入----"+x.get(0));
};
consumer.accept(sourceList);
return consumer;
}