java list和map切割分段的实现及多线程应用案例

/**
 * @author chengxianwei
 * @title map工具类
 * @date 2023/08/03
 **/
public class MapUtil {
    /**
     * List拆分 (指定分组大小)
     *
     * @param chunkList 被分隔的数组
     * @param chunkNum  每段的大小
     * @param <T>       List中的类型
     * @return
     */
    public static <T> List<List<T>> listChunk(List<T> chunkList, int chunkNum) {
        if (chunkList == null || chunkNum <= 0) {
            List<List<T>> t = new ArrayList<>();
            t.add(chunkList);
            return t;
        }
        Iterator<T> iterator = chunkList.iterator();
        int i = 1;
        List<List<T>> total = new ArrayList<>();
        List<T> tem = new ArrayList<>();
        while (iterator.hasNext()) {
            T next = iterator.next();
            tem.add(next);
            if (i == chunkNum) {
                total.add(tem);
                tem = new ArrayList<>();
                i = 0;
            }
            i++;
        }
        if (!ChargeEmptyUtil.isEmpty(tem)) {
            total.add(tem);
        }
        return total;
    }
    /**
     * Map拆分 (指定分组大小)
     *
     * @param chunkMap 被切段的map
     * @param chunkNum 每段的大小
     * @param <k>      map的key类型
     * @param <v>      map的value类型 如果是自定义类型，则必须实现equals和hashCode方法
     * @return
     */
    public static <k, v> List<Map<k, v>> mapChunk(Map<k, v> chunkMap, int chunkNum) {
        if (chunkMap == null || chunkNum <= 0) {
            List<Map<k, v>> list = new ArrayList<>();
            list.add(chunkMap);
            return list;
        }
        Set<k> keySet = chunkMap.keySet();
        Iterator<k> iterator = keySet.iterator();
        int i = 1;
        List<Map<k, v>> total = new ArrayList<>();
        Map<k, v> tem = new HashMap<>();
        while (iterator.hasNext()) {
            k next = iterator.next();
            tem.put(next, chunkMap.get(next));
            if (i == chunkNum) {
                total.add(tem);
                tem = new HashMap<>();
                i = 0;
            }
            i++;
        }
        if (!ChargeEmptyUtil.isEmpty(tem)) {
            total.add(tem);
        }
        return total;
    }
    /**
     * Map拆分 (指定分组大小)
     *
     * @param map       Map
     * @param chunkSize 每个分组的大小 (>=1)
     * @param <K>       Key
     * @param <V>       Value
     * @return 子Map列表
     */
    public static <K, V> List<Map<K, V>> splitByChunkSize(Map<K, V> map, int chunkSize) {
        if (Objects.isNull(map) || map.isEmpty() || chunkSize < 1) {
            //空map或者分组大小<1，无法拆分
            return Collections.emptyList();
        }
        int mapSize = map.size(); //键值对总数
        int groupSize = mapSize / chunkSize + (mapSize % chunkSize == 0 ? 0 : 1); //计算分组个数
        List<Map<K, V>> list = Lists.newArrayListWithCapacity(groupSize); //子Map列表
        if (chunkSize >= mapSize) { //只能分1组的情况
            list.add(map);
            return list;
        }
        int count = 0; //每个分组的组内计数
        Map<K, V> subMap = Maps.newHashMapWithExpectedSize(chunkSize); //子Map
        for (Map.Entry<K, V> entry : map.entrySet()) {
            if (count < chunkSize) {
                //给每个分组放chunkSize个键值对，最后一个分组可能会装不满
                subMap.put(entry.getKey(), entry.getValue());
                count++; //组内计数+1
            } else {
                //结束上一个分组
                list.add(subMap); //当前分组装满了->加入列表
                //开始下一个分组
                subMap = Maps.newHashMapWithExpectedSize(chunkSize); //新的分组
                subMap.put(entry.getKey(), entry.getValue()); //添加当前键值对
                count = 1; //组内计数重置为1
            }
        }
        list.add(subMap);  //添加最后一个分组
        return list;
    }
    /**
     * Map拆分(指定分组个数)
     *
     * @param map       Map
     * @param groupSize 分组个数 (>=1)
     * @param <K>       Key
     * @param <V>       Value
     * @return 子Map列表
     */
    public static <K, V> List<Map<K, V>> splitByGroupSize(Map<K, V> map, int groupSize) {
        if (Objects.isNull(map) || map.isEmpty() || groupSize < 1) {
            //空map或者分组数<1，无法拆分
            return Collections.emptyList();
        }
        List<Map<K, V>> list = Lists.newArrayListWithCapacity(groupSize);
        if (groupSize == 1) { //只有1个分组的情况
            list.add(map);
            return list;
        }
        int mapSize = map.size(); //键值对总数
        int chunkIndex = 0; //当前分组的下标，[0, groupSize-1]
        int restCount = mapSize % groupSize; //平均后剩余的键值对数
        int chunkSize0 = mapSize / groupSize; //每个分组键值对数量
        int chunkSize1 = chunkSize0 + 1; //多分一个
        int chunkSize = chunkIndex < restCount ? chunkSize1 : chunkSize0; //实际每组的大小（前面的部分分组可能会多分1个）
        int count = 0; //每个分组的组内计数
        Map<K, V> subMap = Maps.newHashMapWithExpectedSize(chunkSize);//子Map
        for (Map.Entry<K, V> entry : map.entrySet()) {
            if (count < chunkSize) {
                //每个分组按实际分组大小（chunkSize）加入键值对
                subMap.put(entry.getKey(), entry.getValue());
                count++; //组内计数+1
            } else {
                //结束上一个分组
                list.add(subMap); //当前分组装满了->加入列表
                chunkIndex++; //分组个数+1
                //开始下一个分组
                chunkSize = chunkIndex < restCount ? chunkSize1 : chunkSize0; //重新计算分组大小
                subMap = Maps.newHashMapWithExpectedSize(chunkSize); //新的分组
                subMap.put(entry.getKey(), entry.getValue()); //添加当前键值对
                count = 1; //组内计数重置为1
            }
        }
        list.add(subMap); //添加最后一个分组
        return list;
    }
    //测试
    public static void main(String[] args) {
        Map<String, String> map = new HashMap<>();
        map.put("a", "1");
        map.put("b", "2");
        map.put("c", "3");
        map.put("d", "4");
        map.put("e", "5");
        map.put("f", "6");
        map.put("g", "7");
        System.out.println("splitByChunkSize: " + splitByChunkSize(map, 3));
        System.out.println("splitByGroupSize: " + splitByGroupSize(map, 3));
    }
}

{
            // 导出动态标题
            List<String> titleList = new ArrayList<>();
            // 上网电量和发电量
            String[] monthKeyArr = {"powernetday", "powerday"};
            // 获取导出电站基础数据
            List<XczxPoverPhotovolSourceVO>[0] = new ArrayList<>();
            // 保存成<consNo, <日电量数据列表>> 形式
            ConcurrentHashMap<String, List<XczxPoverPhotovolSourceDayEleVO>> xczxPovertyPowerDayListMap = new ConcurrentHashMap<>();
            ...
            ...
            ...
            // 多线程分段批量处理List列表
            int threadSize = 2000;//每2000条开启一个任务
            int dataSize = xczxPoverPhotovolSourceVOList[0].size();
            int threadNum = dataSize / threadSize + 1;// 开启的任务数
            boolean special = dataSize % threadSize ==0;// 表计整除 (threadNum多算了一次)
            Callable<Integer> task = null;
            List<XczxPoverPhotovolSourceVO> curList = null;
            List<Callable<Integer>> tasks = new ArrayList<>();
            ExecutorService exec = Executors.newCachedThreadPool();
            CountDownLatch countDownLatch = new CountDownLatch(threadNum);
            // 使用Collections将线程不安全的集合转成线程安全的集合
            List<Map<String, Object>> dataList = Collections.synchronizedList(new ArrayList<>(dataSize));
            for (int i = 0; i < threadNum; i++) {
                if (i == threadNum - 1) {
                    if(special){
                        countDownLatch.countDown();
                        break;
                    }
                    curList = xczxPoverPhotovolSourceVOList[0].subList(threadSize * i, dataSize);
                } else {
                    curList = xczxPoverPhotovolSourceVOList[0].subList(threadSize * i, threadSize * (i + 1));
                }
                final List<XczxPoverPhotovolSourceVO> list2 = curList;
                task = new Callable<Integer>() {
                    @Override
                    public synchronized Integer call() throws Exception {
                        try {
                            XczxPoverPhotovolSourceVO next = null;
                            Date parallelInTime = null;
                            Map<String, Object> map = null;
                            List<Map<String, Object>> totalList = null;
                            Map<String, Object> totalMap = null;
                            List<Map<String, Object>> povertyList = null;
                            Map<String, Object> entityMap = null;
                            XczxPoverPhotovolSourceDayEleVO next2 = null;
                            XczxPoverPhotovolSourceDayEleVO curDayEle = null;
                            List<XczxPoverPhotovolSourceDayEleVO> list = null;
                            List<XczxPoverPhotovolSourceDayEleVO> listDay = new ArrayList<>();
                            // 迭代导出电站基础数据
                            Iterator<XczxPoverPhotovolSourceVO> iterator = list2.iterator();
                            while (iterator.hasNext()) {
                                next = iterator.next();
                                map = CommonUtils.transClassToMapLowCase(next);
                                // 合计 集合
                                totalList = new ArrayList<>();
                                totalMap = new HashMap<>();
                                // 合计
                                BigDecimal powerNetTotal = new BigDecimal(BigInteger.ZERO);
                                BigDecimal powerTotal = new BigDecimal(BigInteger.ZERO);
                                // 指定日期的 户号列表 17123
                                listDay = xczxPovertyPowerDayListMap.get(""+next.getId());
                                if (listDay != null && listDay.size() > 0) {
                                    // 遍历一个户号 所有日期
                                    for (int i = 0; i < titleList.size() - 1; i++) {
                                        String date = titleList.get(i).substring(0, 4) + titleList.get(i).substring(5, 7) + titleList.get(i).substring(8, 10);
                                        list = listDay.stream().filter(x -> date.equals(x.getPovertyDay())).collect(Collectors.toList());
                                        povertyList = new ArrayList<>();
                                        entityMap = new HashMap<>();
                                        if (list != null && list.size() > 0) {
                                            for (XczxPoverPhotovolSourceDayEleVO vo : list) {
                                                entityMap.put(monthKeyArr[0] + titleList.get(i), new BigDecimal(vo.getPowerNetDay()));
                                                entityMap.put(monthKeyArr[1] + titleList.get(i), new BigDecimal(vo.getPowerDay()));
                                                powerNetTotal = powerNetTotal.add(new BigDecimal(vo.getPowerNetDay()));
                                                powerTotal = powerTotal.add(new BigDecimal(vo.getPowerDay()));
                                                povertyList.add(entityMap);
                                            }
                                        } else {
                                            // 如果为空，填充0
                                            entityMap.put(monthKeyArr[0] + titleList.get(i), "0");
                                            entityMap.put(monthKeyArr[1] + titleList.get(i), "0");
                                            povertyList.add(entityMap);
                                        }
                                        map.put(day + titleList.get(i), povertyList);
                                    }
                                } else {
                                    // 遍历一个户号 所有日期
                                    for (int i = 0; i < titleList.size() - 1; i++) {
                                        povertyList = new ArrayList<>();
                                        entityMap = new HashMap<>();
                                        // 如果为空，填充0
                                        entityMap.put(monthKeyArr[0] + titleList.get(i), "0");
                                        entityMap.put(monthKeyArr[1] + titleList.get(i), "0");
                                        povertyList.add(entityMap);
                                        povertyList.add(entityMap);
                                        map.put(day + titleList.get(i), povertyList);
                                    }
                                }
                                // 合计模型
                                totalMap.put(monthKeyArr[0] + "合计", powerNetTotal);
                                totalMap.put(monthKeyArr[1] + "合计", powerTotal);
                                totalList.add(totalMap);
                                map.put(day + "合计", totalList);
                                dataList.add(map);
                                map.put("xh", dataList.size());
                            }
                        } catch (Exception e) {
                            log.error("catch error",e);
                        } finally {
                            countDownLatch.countDown();
                            return 1;
                        }
                    }
                };
                tasks.add(task);
            }
            // 多线程并发
            List<Future<Integer>> futureList = null;
            try {
                futureList = exec.invokeAll(tasks);
                for(Future<Integer> future : futureList){
                    if(future!=null) {
                        while (!future.isDone());
                    }
                }
                countDownLatch.await();
            } catch (Exception e) {
                log.error("catch error",e);
            } finally {
                exec.shutdown();
            }
            ...
            ...
            ...
}

{
           ...
           ...
           ...
           // <户主身份证号, List<户主成员(含户主)>> 集合档案信息
           Map<String, List<XczxArchives>> impListMap = new HashMap<>();
           ...
           ...
           ...
           // 多线程分段批量处理Map集合
           int threadSize = 2000;// 每2000条开启一个任务
           int dataSize = impListMap.size();// map数据条数
           int threadNum = dataSize / threadSize + 1;// 开启的任务数
           List<Callable<Integer>> tasks = new ArrayList<>();
           ExecutorService exec = Executors.newCachedThreadPool();
           List<Map<String, List<XczxArchives>>> splitGroup = MapUtil.splitByChunkSize(impListMap, threadSize);
           for (int i = 0; i < threadNum; i++) {
               Map<String, List<XczxArchives>> curLists = splitGroup.get(i);
               // 新建一个并发任务
               Callable<Integer> task = new Callable<Integer>() {
                   @Override
                   public Integer call() throws Exception {
                       int count = 0;
                       // iterator遍历分段Map
                       Iterator<Map.Entry<String, List<XczxArchives>>> entryIterator = curLists.entrySet().iterator();
                       while (entryIterator.hasNext()) {
                           XczxArchives head = null;
                           Map.Entry<String, List<XczxArchives>> entry = entryIterator.next();
                           String key = entry.getKey();// 户主身份证号
                           List<XczxArchives> list = entry.getValue();// 户主及成员列表
                           Iterator<XczxArchives> dataIterator = list.iterator();
                           while (dataIterator.hasNext()) {
                               XczxArchives next = dataIterator.next();
                               if (key.equals(next.getIdCard())) {
                                   head = next;
                                   dataIterator.remove();
                               }
                           }
                           if (head != null) {
                               // 插入户主
                               head.setHeadName(null);
                               head.setHeadIdCard(null);
                               head.setHeadMobile(null);
                               head.setCreateTime(date);
                               head.setUpdateTime(date);
                               int result = xczxArchivesMapper.insert(head);
                               if (result == 1) {
                                   count++;
                                   if (list != null && list.size() > 0) {
                                       for (XczxArchives next : list) {
                                           next.setCreateTime(date);
                                           next.setUpdateTime(date);
                                           next.setHeadId(head.getArchivesId());
                                       }
                                       // 插入成员
                                       int result2 = xczxArchivesMapper.batchInsert(list);
                                       if (result2 > 0) {
                                           count = count + result2;
                                       }
                                   }
                               }
                           }
                       }
                       return count;
                   }
               };
               tasks.add(task);
           }
           // 多线程并发
           List<Future<Integer>> futureList = null;
           try {
               futureList = exec.invokeAll(tasks);
               for (Future<Integer> future : futureList) {
                   if (future != null) {
                       while (!future.isDone());
                       successCount = successCount + future.get();
                   }
               }
           } catch (Exception e) {
               log.error("catch error",e);
           } finally {
               exec.shutdown();
           }
           ...
           ...
           ...
}