java list和map切割分段的实现及多线程应用案例
作者：蚕1蚕2
这篇文章主要为大家介绍了java list和map切割分段的实现及多线程应用案例，有需要的朋友可以借鉴参考下，希望能够有所帮助，祝大家多多进步，早日升职加薪
map工具类
/**
* @author chengxianwei
* @title map工具类
* @date 2023/08/03
**/
public class MapUtil {
/**
* List拆分 (指定分组大小)
*
* @param chunkList 被分隔的数组
* @param chunkNum 每段的大小
* @param <T> List中的类型
* @return
*/
public static <T> List<List<T>> listChunk(List<T> chunkList, int chunkNum) {
if (chunkList == null || chunkNum <= 0) {
List<List<T>> t = new ArrayList<>();
t.add(chunkList);
return t;
}
Iterator<T> iterator = chunkList.iterator();
int i = 1;
List<List<T>> total = new ArrayList<>();
List<T> tem = new ArrayList<>();
while (iterator.hasNext()) {
T next = iterator.next();
tem.add(next);
if (i == chunkNum) {
total.add(tem);
tem = new ArrayList<>();
i = 0;
}
i++;
}
if (!ChargeEmptyUtil.isEmpty(tem)) {
total.add(tem);
}
return total;
}
/**
* Map拆分 (指定分组大小)
*
* @param chunkMap 被切段的map
* @param chunkNum 每段的大小
* @param <k> map的key类型
* @param <v> map的value类型 如果是自定义类型，则必须实现equals和hashCode方法
* @return
*/
public static <k, v> List<Map<k, v>> mapChunk(Map<k, v> chunkMap, int chunkNum) {
if (chunkMap == null || chunkNum <= 0) {
List<Map<k, v>> list = new ArrayList<>();
list.add(chunkMap);
return list;
}
Set<k> keySet = chunkMap.keySet();
Iterator<k> iterator = keySet.iterator();
int i = 1;
List<Map<k, v>> total = new ArrayList<>();
Map<k, v> tem = new HashMap<>();
while (iterator.hasNext()) {
k next = iterator.next();
tem.put(next, chunkMap.get(next));
if (i == chunkNum) {
total.add(tem);
tem = new HashMap<>();
i = 0;
}
i++;
}
if (!ChargeEmptyUtil.isEmpty(tem)) {
total.add(tem);
}
return total;
}
/**
* Map拆分 (指定分组大小)
*
* @param map Map
* @param chunkSize 每个分组的大小 (>=1)
* @param <K> Key
* @param <V> Value
* @return 子Map列表
*/
public static <K, V> List<Map<K, V>> splitByChunkSize(Map<K, V> map, int chunkSize) {
if (Objects.isNull(map) || map.isEmpty() || chunkSize < 1) {
//空map或者分组大小<1，无法拆分
return Collections.emptyList();
}
int mapSize = map.size(); //键值对总数
int groupSize = mapSize / chunkSize + (mapSize % chunkSize == 0 ? 0 : 1); //计算分组个数
List<Map<K, V>> list = Lists.newArrayListWithCapacity(groupSize); //子Map列表
if (chunkSize >= mapSize) { //只能分1组的情况
list.add(map);
return list;
}
int count = 0; //每个分组的组内计数
Map<K, V> subMap = Maps.newHashMapWithExpectedSize(chunkSize); //子Map
for (Map.Entry<K, V> entry : map.entrySet()) {
if (count < chunkSize) {
//给每个分组放chunkSize个键值对，最后一个分组可能会装不满
subMap.put(entry.getKey(), entry.getValue());
count++; //组内计数+1
} else {
//结束上一个分组
list.add(subMap); //当前分组装满了->加入列表
//开始下一个分组
subMap = Maps.newHashMapWithExpectedSize(chunkSize); //新的分组
subMap.put(entry.getKey(), entry.getValue()); //添加当前键值对
count = 1; //组内计数重置为1
}
}
list.add(subMap); //添加最后一个分组
return list;
}
/**
* Map拆分(指定分组个数)
*
* @param map Map
* @param groupSize 分组个数 (>=1)
* @param <K> Key
* @param <V> Value
* @return 子Map列表
*/
public static <K, V> List<Map<K, V>> splitByGroupSize(Map<K, V> map, int groupSize) {
if (Objects.isNull(map) || map.isEmpty() || groupSize < 1) {
//空map或者分组数<1，无法拆分
return Collections.emptyList();
}
List<Map<K, V>> list = Lists.newArrayListWithCapacity(groupSize);
if (groupSize == 1) { //只有1个分组的情况
list.add(map);
return list;
}
int mapSize = map.size(); //键值对总数
int chunkIndex = 0; //当前分组的下标，[0, groupSize-1]
int restCount = mapSize % groupSize; //平均后剩余的键值对数
int chunkSize0 = mapSize / groupSize; //每个分组键值对数量
int chunkSize1 = chunkSize0 + 1; //多分一个
int chunkSize = chunkIndex < restCount ? chunkSize1 : chunkSize0; //实际每组的大小（前面的部分分组可能会多分1个）
int count = 0; //每个分组的组内计数
Map<K, V> subMap = Maps.newHashMapWithExpectedSize(chunkSize);//子Map
for (Map.Entry<K, V> entry : map.entrySet()) {
if (count < chunkSize) {
//每个分组按实际分组大小（chunkSize）加入键值对
subMap.put(entry.getKey(), entry.getValue());
count++; //组内计数+1
} else {
//结束上一个分组
list.add(subMap); //当前分组装满了->加入列表
chunkIndex++; //分组个数+1
//开始下一个分组
chunkSize = chunkIndex < restCount ? chunkSize1 : chunkSize0; //重新计算分组大小
subMap = Maps.newHashMapWithExpectedSize(chunkSize); //新的分组
subMap.put(entry.getKey(), entry.getValue()); //添加当前键值对
count = 1; //组内计数重置为1
}
}
list.add(subMap); //添加最后一个分组
return list;
}
//测试
public static void main(String[] args) {
Map<String, String> map = new HashMap<>();
map.put("a", "1");
map.put("b", "2");
map.put("c", "3");
map.put("d", "4");
map.put("e", "5");
map.put("f", "6");
map.put("g", "7");
System.out.println("splitByChunkSize: " + splitByChunkSize(map, 3));
System.out.println("splitByGroupSize: " + splitByGroupSize(map, 3));
}
}应用案例
List分段
{
// 导出动态标题
List<String> titleList = new ArrayList<>();
// 上网电量和发电量
String[] monthKeyArr = {"powernetday", "powerday"};
// 获取导出电站基础数据
List<XczxPoverPhotovolSourceVO>[0] = new ArrayList<>();
// 保存成<consNo, <日电量数据列表>> 形式
ConcurrentHashMap<String, List<XczxPoverPhotovolSourceDayEleVO>> xczxPovertyPowerDayListMap = new ConcurrentHashMap<>();
...
...
...
// 多线程分段批量处理List列表
int threadSize = 2000;//每2000条开启一个任务
int dataSize = xczxPoverPhotovolSourceVOList[0].size();
int threadNum = dataSize / threadSize + 1;// 开启的任务数
boolean special = dataSize % threadSize ==0;// 表计整除 (threadNum多算了一次)
Callable<Integer> task = null;
List<XczxPoverPhotovolSourceVO> curList = null;
List<Callable<Integer>> tasks = new ArrayList<>();
ExecutorService exec = Executors.newCachedThreadPool();
CountDownLatch countDownLatch = new CountDownLatch(threadNum);
// 使用Collections将线程不安全的集合转成线程安全的集合
List<Map<String, Object>> dataList = Collections.synchronizedList(new ArrayList<>(dataSize));
for (int i = 0; i < threadNum; i++) {
if (i == threadNum - 1) {
if(special){
countDownLatch.countDown();
break;
}
curList = xczxPoverPhotovolSourceVOList[0].subList(threadSize * i, dataSize);
} else {
curList = xczxPoverPhotovolSourceVOList[0].subList(threadSize * i, threadSize * (i + 1));
}
final List<XczxPoverPhotovolSourceVO> list2 = curList;
task = new Callable<Integer>() {
@Override
public synchronized Integer call() throws Exception {
try {
XczxPoverPhotovolSourceVO next = null;
Date parallelInTime = null;
Map<String, Object> map = null;
List<Map<String, Object>> totalList = null;
Map<String, Object> totalMap = null;
List<Map<String, Object>> povertyList = null;
Map<String, Object> entityMap = null;
XczxPoverPhotovolSourceDayEleVO next2 = null;
XczxPoverPhotovolSourceDayEleVO curDayEle = null;
List<XczxPoverPhotovolSourceDayEleVO> list = null;
List<XczxPoverPhotovolSourceDayEleVO> listDay = new ArrayList<>();
// 迭代导出电站基础数据
Iterator<XczxPoverPhotovolSourceVO> iterator = list2.iterator();
while (iterator.hasNext()) {
next = iterator.next();
map = CommonUtils.transClassToMapLowCase(next);
// 合计 集合
totalList = new ArrayList<>();
totalMap = new HashMap<>();
// 合计
BigDecimal powerNetTotal = new BigDecimal(BigInteger.ZERO);
BigDecimal powerTotal = new BigDecimal(BigInteger.ZERO);
// 指定日期的 户号列表 17123
listDay = xczxPovertyPowerDayListMap.get(""+next.getId());
if (listDay != null && listDay.size() > 0) {
// 遍历一个户号 所有日期
for (int i = 0; i < titleList.size() - 1; i++) {
String date = titleList.get(i).substring(0, 4) + titleList.get(i).substring(5, 7) + titleList.get(i).substring(8, 10);
list = listDay.stream().filter(x -> date.equals(x.getPovertyDay())).collect(Collectors.toList());
povertyList = new ArrayList<>();
entityMap = new HashMap<>();
if (list != null && list.size() > 0) {
for (XczxPoverPhotovolSourceDayEleVO vo : list) {
entityMap.put(monthKeyArr[0] + titleList.get(i), new BigDecimal(vo.getPowerNetDay()));
entityMap.put(monthKeyArr[1] + titleList.get(i), new BigDecimal(vo.getPowerDay()));
powerNetTotal = powerNetTotal.add(new BigDecimal(vo.getPowerNetDay()));
powerTotal = powerTotal.add(new BigDecimal(vo.getPowerDay()));
povertyList.add(entityMap);
}
} else {
// 如果为空，填充0
entityMap.put(monthKeyArr[0] + titleList.get(i), "0");
entityMap.put(monthKeyArr[1] + titleList.get(i), "0");
povertyList.add(entityMap);
}
map.put(day + titleList.get(i), povertyList);
}
} else {
// 遍历一个户号 所有日期
for (int i = 0; i < titleList.size() - 1; i++) {
povertyList = new ArrayList<>();
entityMap = new HashMap<>();
// 如果为空，填充0
entityMap.put(monthKeyArr[0] + titleList.get(i), "0");
entityMap.put(monthKeyArr[1] + titleList.get(i), "0");
povertyList.add(entityMap);
povertyList.add(entityMap);
map.put(day + titleList.get(i), povertyList);
}
}
// 合计模型
totalMap.put(monthKeyArr[0] + "合计", powerNetTotal);
totalMap.put(monthKeyArr[1] + "合计", powerTotal);
totalList.add(totalMap);
map.put(day + "合计", totalList);
dataList.add(map);
map.put("xh", dataList.size());
}
} catch (Exception e) {
log.error("catch error",e);
} finally {
countDownLatch.countDown();
return 1;
}
}
};
tasks.add(task);
}
// 多线程并发
List<Future<Integer>> futureList = null;
try {
futureList = exec.invokeAll(tasks);
for(Future<Integer> future : futureList){
if(future!=null) {
while (!future.isDone());
}
}
countDownLatch.await();
} catch (Exception e) {
log.error("catch error",e);
} finally {
exec.shutdown();
}
...
...
...
}Map分段
{
...
...
...
// <户主身份证号, List<户主成员(含户主)>> 集合档案信息
Map<String, List<XczxArchives>> impListMap = new HashMap<>();
...
...
...
// 多线程分段批量处理Map集合
int threadSize = 2000;// 每2000条开启一个任务
int dataSize = impListMap.size();// map数据条数
int threadNum = dataSize / threadSize + 1;// 开启的任务数
List<Callable<Integer>> tasks = new ArrayList<>();
ExecutorService exec = Executors.newCachedThreadPool();
List<Map<String, List<XczxArchives>>> splitGroup = MapUtil.splitByChunkSize(impListMap, threadSize);
for (int i = 0; i < threadNum; i++) {
Map<String, List<XczxArchives>> curLists = splitGroup.get(i);
// 新建一个并发任务
Callable<Integer> task = new Callable<Integer>() {
@Override
public Integer call() throws Exception {
int count = 0;
// iterator遍历分段Map
Iterator<Map.Entry<String, List<XczxArchives>>> entryIterator = curLists.entrySet().iterator();
while (entryIterator.hasNext()) {
XczxArchives head = null;
Map.Entry<String, List<XczxArchives>> entry = entryIterator.next();
String key = entry.getKey();// 户主身份证号
List<XczxArchives> list = entry.getValue();// 户主及成员列表
Iterator<XczxArchives> dataIterator = list.iterator();
while (dataIterator.hasNext()) {
XczxArchives next = dataIterator.next();
if (key.equals(next.getIdCard())) {
head = next;
dataIterator.remove();
}
}
if (head != null) {
// 插入户主
head.setHeadName(null);
head.setHeadIdCard(null);
head.setHeadMobile(null);
head.setCreateTime(date);
head.setUpdateTime(date);
int result = xczxArchivesMapper.insert(head);
if (result == 1) {
count++;
if (list != null && list.size() > 0) {
for (XczxArchives next : list) {
next.setCreateTime(date);
next.setUpdateTime(date);
next.setHeadId(head.getArchivesId());
}
// 插入成员
int result2 = xczxArchivesMapper.batchInsert(list);
if (result2 > 0) {
count = count + result2;
}
}
}
}
}
return count;
}
};
tasks.add(task);
}
// 多线程并发
List<Future<Integer>> futureList = null;
try {
futureList = exec.invokeAll(tasks);
for (Future<Integer> future : futureList) {
if (future != null) {
while (!future.isDone());
successCount = successCount + future.get();
}
}
} catch (Exception e) {
log.error("catch error",e);
} finally {
exec.shutdown();
}
...
...
...
}以上就是java list和map切割分段的实现及多线程应用案例的详细内容，更多关于java list map多线程切割分段的资料请关注脚本之家其它相关文章！