使用JavaScript进行高效处理CSV文件

npm install csv-parser fast-csv

const fs = require('fs');
const csv = require('csv-parser');

fs.createReadStream('data.csv') // 'data.csv' 是你的CSV文件路径
  .pipe(csv())
  .on('data', (row) => {
    console.log(row);
  })
  .on('end', () => {
    console.log('CSV文件读取完成');
  });

const fs = require('fs');
const fastcsv = require('fast-csv');

const data = [
  { name: 'Alice', age: 25, city: 'New York' },
  { name: 'Bob', age: 30, city: 'Los Angeles' },
  { name: 'Charlie', age: 35, city: 'Chicago' }
];

const ws = fs.createWriteStream('output.csv');

fastcsv
  .write(data, { headers: true })
  .pipe(ws);

ws.on('finish', () => {
  console.log('CSV文件写入完成');
})

npm install-parser fast-csv

const fs require('fs');
 csv = require('csv-parser');
const Transform } = require('streamconst fastcsv = require('fast-csvconst BATCH_SIZE 10000; // 设置批处理大小

let records = [];
let batchCounter = 0;

// 创建流器，用于处理每批次的数据
const transform = new Transform({
  objectMode: true,
  transform(chunk encoding, callback)    records.push(chunk);
    if (.length >= BATCH_SIZE) {
      this(records);
      records [];
    }
    callback();
  },
 flush(callback) {
 if (records.length > 0) {
      this.push(records);
    }
    callback();
  }
});

// 创建CSV读取
const readStream fs.createReadStream('large_data.csv').pipe(csv());

// 创建CSV写入流
const writeStream = fs.createWriteStream('processed_data.csv 处理每批的数据
transform.on('data', (batch) => {
  // 对数据进行处理，比如过滤、转换等
 batch = batch.map => ({
    ...,
    processedField: parseInt(record.some, 10) * 2 // 示例处理逻辑
 }));

  // 写处理后的数据
 fastcsv.write, { headers: !!batchCounter }).pipeStream, { end false });
  batchCounter++;
});

write.on('finish', => {
  console.log('CSV文件处理完成');
});

// 将读取流管道连接到转换流
readStream.pipe);

node process_csv.js