使用EasyExcel实现百万级别数据导出的代码示例

select * from t_user limit 500000 ; 

 select * from t_user limit 500000,500000 ; 

id|select_type|table |partitions|type|possible_keys|key|key_len|ref|rows   |filtered|Extra|
--+-----------+------+----------+----+-------------+---+-------+---+-------+--------+-----+
 1|SIMPLE     |t_user|          |ALL |             |   |       |   |2993040|   100.0|     |

select * from t_user where id > 500000 limit 500000 ; 

id|select_type|table |partitions|type |possible_keys|key    |key_len|ref|rows   |filtered|Extra      |
--+-----------+------+----------+-----+-------------+-------+-------+---+-------+--------+-----------+
 1|SIMPLE     |t_user|          |range|PRIMARY      |PRIMARY|8      |   |1496520|   100.0|Using where|

object header +pointer+id字段+name字段大小=8+8+4+10=30字节

public static void main(String[] args) {
        SpringApplication app = new SpringApplication(WebApplication.class);
        Environment env = app.run(args).getEnvironment();
        logger.info("启动成功！！");
        logger.info("地址: \thttp://127.0.0.1:{}", env.getProperty("server.port"));

        TUserMapper userMapper = SpringUtil.getBean(TUserMapper.class);
        //计算总的数据量
        int count = (int) userMapper.countByExample(null);


        //获取分页总数
        int queryCount = 50_0000;
        int pageCount = count % queryCount == 0 ? count / queryCount : count / queryCount + 1;

        //设置导出的文件名
        String fileName = "result.xlsx";
        //设置excel的sheet号码
        int sheetNo = 1;
        //设置第一个sheet的名字
        String sheetName = "sheet-" + sheetNo;


        long start = System.currentTimeMillis();
        // 创建writeSheet
        WriteSheet writeSheet = EasyExcel.writerSheet(sheetNo, sheetName).build();
        //记录每次分页查询的最大值
        Long maxId = null;

        //指定文件
        try (ExcelWriter excelWriter = EasyExcel.write(fileName, TUser.class).build()) {
            //写入每一页分页查询的数据
            for (int i = 1; i <= pageCount; i++) {
                // 分页去数据库查询数据 这里可以去数据库查询每一页的数据
                long queryStart = System.currentTimeMillis();
                TUserExample userExample = new TUserExample();


                //如果是第一次则直接进行分页查询，反之基于上一次分页查询的分页定位实际偏移量，筛选前n条数据以达到分页效果
                if (i == 1) {
                    PageHelper.startPage(i, queryCount, false);
                } else if (maxId != null) {
                    userExample.createCriteria().andIdGreaterThan(maxId);
                    PageHelper.startPage(0, queryCount, false);
                }


                List<TUser> userList = userMapper.selectByExample(userExample);
                //更新下一次分页查询用的id
                if (CollUtil.isNotEmpty(userList)) {
                    maxId = userList.get(userList.size() - 1).getId();
                }

                long queryEnd = System.currentTimeMillis();
                logger.info("数据大小:{},写入sheet位置:{}，耗时:{}", userList.size(), sheetName, queryEnd - queryStart);

                long writeStart = System.currentTimeMillis();
                excelWriter.write(userList, writeSheet);

                long writeEnd = System.currentTimeMillis();
                logger.info("本次写入耗时:{}", writeEnd - writeStart);

                //如果% 2 == 0，则说明一个sheet写入了50*2即100w的数据，需要创建新的sheet进行写入
                if (i % 2 == 0) {
                    sheetName = "sheet-" + (++sheetNo);
                    writeSheet = EasyExcel.writerSheet(sheetNo, sheetName).build();
                    logger.info("写满一个sheet,切换到下一个sheet:{}", sheetName);
                }
            }
        }
        long total = System.currentTimeMillis() - start;
        logger.info("导出结束，总耗时:{}", total);

    }

# 50w的读写耗时
com.sharkChili.webTemplate.config.WebApplication  :73   [32m                  [0;39m 数据大小:500000,写入sheet位置:sheet-1，耗时:4719
2023-12-03 10:13:58.675 INFO  com.sharkChili.webTemplate.config.WebApplication  :78   [32m                  [0;39m 本次写入耗时:2911
2023-12-03 10:14:02.517 INFO  com.sharkChili.webTemplate.config.WebApplication  :73   [32m                  [0;39m 数据大小:500000,写入sheet位置:sheet-1，耗时:3841
2023-12-03 10:14:04.860 INFO  com.sharkChili.webTemplate.config.WebApplication  :78   [32m                  [0;39m 本次写入耗时:2343