asp.net实现Postgresql快速写入/读取大量数据实例

CREATE TABLE public.mesh
(
 x integer NOT NULL,
 y integer NOT NULL,
 z integer,
 CONSTRAINT prim PRIMARY KEY (x, y)
)

copy mesh from 'd:/user.csv' csv

class Program
{
  static void Main(string[] args)
  {
    var list = GetData("D:\\user.csv");
    TimeCalc.LogStartTime();
    using (var sm = new SqlManipulation(@"Strings", SqlType.PostgresQL))
    {
      sm.Init();
      foreach (var n in list)
      {
        sm.ExcuteNonQuery($"insert into mesh(x,y,z) values({n.x},{n.y},{n.z})");
      }
    }
    TimeCalc.ShowTotalDuration();

    Console.ReadKey();
  }

  static List<(int x, int y, int z)> GetData(string filepath)
  {
    List<ValueTuple<int, int, int>> list = new List<(int, int, int)>();
    foreach (var n in File.ReadLines(filepath))
    {
      string[] x = n.Split(',');
      list.Add((Convert.ToInt32(x[0]), Convert.ToInt32(x[1]), Convert.ToInt32(x[2])));
    }
    return list;
  }
}

var bag = GetData("D:\\user.csv");
//使用时，直接执行stringbuilder的tostring方法。
List<StringBuilder> listbuilder = new List<StringBuilder>();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < bag.Count; i++)
{
  if (i % 100 == 0)
  {
    sb = new StringBuilder();
    listbuilder.Add(sb);
    sb.Append("insert into mesh(x,y,z) values");
    sb.Append($"({bag[i].x}, {bag[i].y}, {bag[i].z})");
  }
  else
    sb.Append($",({bag[i].x}, {bag[i].y}, {bag[i].z})");
}

static void Main(string[] args)
{
  var bag = GetData("D:\\user.csv");
  List<StringBuilder> listbuilder = new List<StringBuilder>();
  StringBuilder sb = new StringBuilder();
  for (int i = 0; i < bag.Count; i++)
  {
    if (i % 1000 == 0)
    {
      sb = new StringBuilder();
      listbuilder.Add(sb);
      //sb.Append("insert into mesh(x,y,z) values");
      sb.Append($"{bag[i].x}, {bag[i].y}, {bag[i].z}");
    }
    else
      sb.Append($",{bag[i].x}, {bag[i].y}, {bag[i].z}");
  }
  StringBuilder sbp = new StringBuilder();
  sbp.Append("PREPARE insertplan (");
  for (int i = 0; i < 1000; i++)
  {
    sbp.Append("int,int,int,");
  }
  sbp.Remove(sbp.Length - 1, 1);
  sbp.Append(") AS INSERT INTO mesh(x, y, z) values");
  for (int i = 0; i < 1000; i++)
  {
    sbp.Append($"(${i*3 + 1},${i* 3 + 2},${i*3+ 3}),");
  }
  sbp.Remove(sbp.Length - 1, 1);
  TimeCalc.LogStartTime();

  using (var sm = new SqlManipulation(@"string", SqlType.PostgresQL))
  {
    sm.Init();
    sm.ExcuteNonQuery(sbp.ToString());
    foreach (var n in listbuilder)
    {
      sm.ExcuteNonQuery($"EXECUTE insertplan({n.ToString()})");
    }
  }
  TimeCalc.ShowTotalDuration();

  Console.ReadKey();
}

var lists = new List<List<string>>();
var listt = new List<string>();
for (int i = 0; i < listbuilder.Count; i++)
{
  if (i % 1000 == 0)
  {
    listt = new List<string>();
    lists.Add(listt);
  }
  listt.Add(listbuilder[i].ToString());
}
TimeCalc.LogStartTime();
Parallel.ForEach(lists, (x) =>
{
  using (var sm = new SqlManipulation(@";string;MaxPoolSize=1000;", SqlType.PostgresQL))
  {
    sm.Init();
    foreach (var n in x)
    {
      sm.ExcuteNonQuery(n);
    }
  }
});
TimeCalc.ShowTotalDuration();

insert into mesh on conflict (x,y) do update set z = excluded.z

postgresql的copy方法提供stdout binary方式，可以指定一条查询进行输出，耗时53.20s。
public List<(int x, int y, int z)> BulkIQueryNpg()
{
  List<(int, int, int)> dict = new List<(int, int, int)>();
  using (var reader = ((NpgsqlConnection)_conn).BeginBinaryExport("COPY (select x,y,z from mesh) TO STDOUT (FORMAT BINARY)"))
  {
    while (reader.StartRow() != -1)
    {
      var x = reader.Read<int>(NpgsqlDbType.Integer);
      var y = reader.Read<int>(NpgsqlDbType.Integer);
      var z = reader.Read<int>(NpgsqlDbType.Integer);
      dict.Add((x, y, z));
    }
  }
  return dict;
}