C# 如何将通用源数组批量插入任意SQL表

我的应用程序使用实体框架。大多数时候我对此感觉很好，但当我需要进行批量插入时，我遇到了麻烦。我还没有找到一种方法让EF快速完成这些任务。因此，我需要一个超越EF的解决方案来进行更新

我想要一个方法，它接受连接字符串、目标表名和源数据的通用数组，并执行大容量插入。此外，我希望它将源数据的属性映射到特定的表字段，理想情况下不需要源对象中的属性来指定表字段

因此，对于这个目标：

public class Customer
{
    //property populated & used only in memory
    public string TempProperty { get; set; }

    //properties saved in the database
    public string Name { get; set; }
    public string Address { get; set; }
    public int Age { get; set; }
    public string Comments { get; set; }

    //various methods, constructors, etc.
}

---

我应该能够提供表名

Data.Customer

，方法应该映射

Customer.name->Data.Customers.name

，

Customer.Address->Data.Customers.Address

，等等。

我很惊讶我在任何地方都找不到类似的代码。也许这不是解决问题的最好办法？尽管有这种可能性，我还是想出了一个解决办法

我的开始大纲是这样的：

+------------+----------------------------------------------+
| Field Name |               Mapping Function               |
+------------+----------------------------------------------+
| Name       | Customer.Name -> Data.Customers.Name         |
| Address    | Customer.Address -> Data.Customers.Address   |
| Age        | Customer.Age -> Data.Customers.Age           |
| Comments   | Customer.Comments -> Data.Customers.Comments |
+------------+----------------------------------------------+

使用目标表和源对象类型使用反射创建映射对象

应用映射器，迭代源对象以生成插入数据

使用

System.data.SqlClient.SqlBulkCopy

对象将数据插入目标表

当我看到这样的大纲时，我会尝试用数据类型来重新表述它，因为实际上我所做的只是将我的输入（

T[]

）转换成

SqlBulkCopy

可以接受的内容。但要做到这一点，我需要决定如何映射字段

以下是我决定的：

var mapper = new List<Tuple<string, Func<object, string, object>>>();

事实上，它是一个列表，表示行。这就给我们留下了

元组

。这将创建一种字典（但索引），其中给定的字符串（字段名）映射到一个函数，当给定源对象

T

和源字段（例如

Address

）时，该函数将检索适当的值。如果没有为表字段找到相应的属性，我们将只返回null

验证输入值是否有效（连接有效、表存在等）后，我们将创建映射对象：

//get all the column names for the table to build mapping object
SqlCommand command = new SqlCommand($"SELECT TOP 1 * FROM {foundTableName}", conn);
SqlDataReader reader = command.ExecuteReader();

//build mapping object by iterating through rows and verifying that there is a match in the table
var mapper = new List<Tuple<string, Func<object, string, object>>>();
foreach (DataRow col in reader.GetSchemaTable().Rows)
{
    //get column information
    string columnName = col.Field<string>("ColumnName");
    PropertyInfo property = typeof(T).GetProperty(columnName);
    Func<object, string, object> map;
    if (property == null)
    {
        //check if it's nullable and exit if not
        bool nullable = col.Field<bool>("Is_Nullable");
        if (!nullable)
            return $"No corresponding property found for Non-nullable field '{columnName}'.";

        //if it's nullable, create mapping function
        map = new Func<object, string, object>((a, b) => null);
    }
    else
        map = new Func<object, string, object>((src, fld) => typeof(T).GetProperty(fld).GetValue(src));

    //add mapping object
    mapper.Add(new Tuple<string, Func<object, string, object>>(columnName, map));
}

然后通过写入数据来完成：

  //set up the bulk copy connection
  SqlBulkCopy sbc = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock | 
                                          SqlBulkCopyOptions.UseInternalTransaction, null);
  sbc.DestinationTableName = foundTableName;
  sbc.BatchSize = BATCH_SIZE;
  sbc.WriteToServer(rows);

就这样！工作起来很有魅力，而且运行速度太快，我懒得进行基准测试（EF花了几分钟运行这些导入）

我应该注意，我遗漏了很多错误检查代码，可能还会添加更多。但是，如果您想完整地了解我的课程，请看以下内容：

  using System;
  using System.Collections.Generic;
  using System.Data;
  using System.Data.Common;
  using System.Data.SqlClient;
  using System.Linq;
  using System.Reflection;

  namespace DatabaseUtilities
  {
      public static class RapidDataTools
      {
          const int SCHEMA_SCHEMA_NAME = 1;
          const int SCHEMA_TABLE_NAME = 2;
          const int BATCH_SIZE = 1000;

          /// <summary>
          /// Imports an array of data into a specified table. It does so by mapping object properties
          /// to table columns. Only properties with the same name as the column name will be copied;
          /// other columns will be left null. Non-nullable columns with no corresponding property will
          /// throw an error.
          /// </summary>
          /// <param name="connectionString"></param>
          /// <param name="destTableName">Qualified table name (e.g. Admin.Table)</param>
          /// <param name="sourceData"></param>
          /// <returns></returns>
          public static string Import<T>(string connectionString, string destTableName, T[] sourceData)
          {
              //get destination table qualified name
              string[] tableParts = destTableName.Split('.');
              if (tableParts.Count() != 2) return $"Invalid or unqualified destination table name: {destTableName}.";
              string destSchema = tableParts[0];
              string destTable = tableParts[1];

              //create the database connection
              SqlConnection conn = GetConnection(connectionString);
              if (conn == null) return "Invalid connection string.";

              //establish connection
              try { conn.Open(); }
              catch { return "Could not connect to database using provided connection string."; }

              //make sure the requested table exists
              string foundTableName = string.Empty;
              foreach (DataRow row in conn.GetSchema("Tables").Rows)
                  if (row[SCHEMA_SCHEMA_NAME].ToString().Equals(destSchema, StringComparison.CurrentCultureIgnoreCase) &&
                      row[SCHEMA_TABLE_NAME].ToString().Equals(destTable, StringComparison.CurrentCultureIgnoreCase))
                  {
                      foundTableName = $"{row[SCHEMA_SCHEMA_NAME]}.{row[SCHEMA_TABLE_NAME]}";
                      break;
                  }
              if (foundTableName == string.Empty) return $"Specified table '{destTableName}' could not be found in table.";

              //get all the column names for the table to build mapping object
              SqlCommand command = new SqlCommand($"SELECT TOP 1 * FROM {foundTableName}", conn);
              SqlDataReader reader = command.ExecuteReader();

              //build mapping object by iterating through rows and verifying that there is a match in the table
              var mapper = new List<Tuple<string, Func<object, string, object>>>();
              foreach (DataRow col in reader.GetSchemaTable().Rows)
              {
                  //get column information
                  string columnName = col.Field<string>("ColumnName");
                  PropertyInfo property = typeof(T).GetProperty(columnName);
                  Func<object, string, object> map;
                  if (property == null)
                  {
                      //check if it's nullable and exit if not
                      bool nullable = col.Field<bool>("Is_Nullable");
                      if (!nullable)
                          return $"No corresponding property found for Non-nullable field '{columnName}'.";

                      //if it's nullable, create mapping function
                      map = new Func<object, string, object>((a, b) => null);
                  }
                  else
                      map = new Func<object, string, object>((src, fld) => typeof(T).GetProperty(fld).GetValue(src));

                  //add mapping object
                  mapper.Add(new Tuple<string, Func<object, string, object>>(columnName, map));
              }

              //get all the data
              int dataCount = sourceData.Count();
              var rows = new DataRow[dataCount];
              DataTable destTableDT = new DataTable();
              destTableDT.Load(reader);
              for (int x = 0; x < dataCount; x++)
              {
                  var dataRow = destTableDT.NewRow();
                  dataRow.ItemArray = mapper.Select(m => m.Item2.Invoke(sourceData[x], m.Item1)).ToArray();
                  rows[x] = dataRow;
              }

              //close the old connection
              conn.Close();

              //set up the bulk copy connection
              SqlBulkCopy sbc = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock | SqlBulkCopyOptions.UseInternalTransaction, null);
              sbc.DestinationTableName = foundTableName;
              sbc.BatchSize = BATCH_SIZE;

              //establish connection
              try { conn.Open(); }
              catch { return "Failed to re-established connection to the database after reading data."; }

              //write data
              try { sbc.WriteToServer(rows); }
              catch (Exception ex) { return $"Batch write failed. Details: {ex.Message} - {ex.StackTrace}"; }

              //if we got here, everything worked!
              return string.Empty;
          }

          private static SqlConnection GetConnection(string connectionString)
          {
              DbConnectionStringBuilder csb = new DbConnectionStringBuilder();
              try { csb.ConnectionString = connectionString; }
              catch { return null; }
              return new SqlConnection(csb.ConnectionString);
          }
      }
  }

使用系统；
使用System.Collections.Generic；
使用系统数据；
使用System.Data.Common；
使用System.Data.SqlClient；
使用System.Linq；
运用系统反思；
命名空间数据库实用程序
{
公共静态类RapidDataTools
{
const int SCHEMA_SCHEMA_NAME=1；
const int SCHEMA_TABLE_NAME=2；
常数int批量大小=1000；
/// 
///将数据数组导入到指定的表中。它通过映射对象属性来实现
///复制到表列。仅复制与列名同名的属性；
///其他列将保留为null。没有相应属性的不可为null的列将保留为null
///抛出一个错误。
/// 
/// 
///限定表名称（例如管理表）
/// 
/// 
公共静态字符串导入（字符串连接字符串、字符串destTableName、T[]源数据）
{
//获取目标表限定名
字符串[]tableParts=destTableName.Split（'.'）；
if（tableParts.Count（）！=2）返回$“无效或不合格的目标表名：{destTableName}。”；
字符串destSchema=tableParts[0]；
字符串destable=tableParts[1]；
//创建数据库连接
SqlConnection conn=GetConnection（connectionString）；
if（conn==null）返回“无效连接字符串”；
//建立联系
试试{conn.Open（）；}
catch{return“无法使用提供的连接字符串连接到数据库。”；}
//确保请求的表存在
string foundTableName=string.Empty；
foreach（conn.GetSchema（“表”）.Rows中的数据行）
if（行[SCHEMA\u SCHEMA\u NAME].ToString（）.Equals（destSchema，StringComparison.CurrentCultureIgnoreCase）&&
行[SCHEMA_TABLE_NAME].ToString（）.Equals（destTable，StringComparison.CurrentCultureIgnoreCase））
{
foundTableName=$“{row[SCHEMA\u SCHEMA\u NAME]}.{row[SCHEMA\u TABLE\u NAME]}”；
打破
}
if（foundTableName==string.Empty）返回$“在表中找不到指定的表“{destTableName}”；
//获取要生成映射对象的表的所有列名
SqlCommand=newSQLCommand（$“从{foundTableName}中选择顶部1*”，conn）；
SqlDataReader=command.ExecuteReader（）；
//通过遍历行并验证表中是否存在匹配项来构建映射对象
var mapper=新列表（）；
foreach（reader.GetSchemaTable（）.Rows中的DataRow列）
{
//获取列信息
字符串columnName=列字段（“columnName”）；
PropertyInfo property=typeof（T）.GetProperty（columnName）；
Func图；
if（属性==null）
{
//检查它是否为空，如果不为空则退出
bool nullable=列字段（“Is_nullable”）；
如果（！nullable）
return$“没有对应的
  using System;
  using System.Collections.Generic;
  using System.Data;
  using System.Data.Common;
  using System.Data.SqlClient;
  using System.Linq;
  using System.Reflection;

  namespace DatabaseUtilities
  {
      public static class RapidDataTools
      {
          const int SCHEMA_SCHEMA_NAME = 1;
          const int SCHEMA_TABLE_NAME = 2;
          const int BATCH_SIZE = 1000;

          /// <summary>
          /// Imports an array of data into a specified table. It does so by mapping object properties
          /// to table columns. Only properties with the same name as the column name will be copied;
          /// other columns will be left null. Non-nullable columns with no corresponding property will
          /// throw an error.
          /// </summary>
          /// <param name="connectionString"></param>
          /// <param name="destTableName">Qualified table name (e.g. Admin.Table)</param>
          /// <param name="sourceData"></param>
          /// <returns></returns>
          public static string Import<T>(string connectionString, string destTableName, T[] sourceData)
          {
              //get destination table qualified name
              string[] tableParts = destTableName.Split('.');
              if (tableParts.Count() != 2) return $"Invalid or unqualified destination table name: {destTableName}.";
              string destSchema = tableParts[0];
              string destTable = tableParts[1];

              //create the database connection
              SqlConnection conn = GetConnection(connectionString);
              if (conn == null) return "Invalid connection string.";

              //establish connection
              try { conn.Open(); }
              catch { return "Could not connect to database using provided connection string."; }

              //make sure the requested table exists
              string foundTableName = string.Empty;
              foreach (DataRow row in conn.GetSchema("Tables").Rows)
                  if (row[SCHEMA_SCHEMA_NAME].ToString().Equals(destSchema, StringComparison.CurrentCultureIgnoreCase) &&
                      row[SCHEMA_TABLE_NAME].ToString().Equals(destTable, StringComparison.CurrentCultureIgnoreCase))
                  {
                      foundTableName = $"{row[SCHEMA_SCHEMA_NAME]}.{row[SCHEMA_TABLE_NAME]}";
                      break;
                  }
              if (foundTableName == string.Empty) return $"Specified table '{destTableName}' could not be found in table.";

              //get all the column names for the table to build mapping object
              SqlCommand command = new SqlCommand($"SELECT TOP 1 * FROM {foundTableName}", conn);
              SqlDataReader reader = command.ExecuteReader();

              //build mapping object by iterating through rows and verifying that there is a match in the table
              var mapper = new List<Tuple<string, Func<object, string, object>>>();
              foreach (DataRow col in reader.GetSchemaTable().Rows)
              {
                  //get column information
                  string columnName = col.Field<string>("ColumnName");
                  PropertyInfo property = typeof(T).GetProperty(columnName);
                  Func<object, string, object> map;
                  if (property == null)
                  {
                      //check if it's nullable and exit if not
                      bool nullable = col.Field<bool>("Is_Nullable");
                      if (!nullable)
                          return $"No corresponding property found for Non-nullable field '{columnName}'.";

                      //if it's nullable, create mapping function
                      map = new Func<object, string, object>((a, b) => null);
                  }
                  else
                      map = new Func<object, string, object>((src, fld) => typeof(T).GetProperty(fld).GetValue(src));

                  //add mapping object
                  mapper.Add(new Tuple<string, Func<object, string, object>>(columnName, map));
              }

              //get all the data
              int dataCount = sourceData.Count();
              var rows = new DataRow[dataCount];
              DataTable destTableDT = new DataTable();
              destTableDT.Load(reader);
              for (int x = 0; x < dataCount; x++)
              {
                  var dataRow = destTableDT.NewRow();
                  dataRow.ItemArray = mapper.Select(m => m.Item2.Invoke(sourceData[x], m.Item1)).ToArray();
                  rows[x] = dataRow;
              }

              //close the old connection
              conn.Close();

              //set up the bulk copy connection
              SqlBulkCopy sbc = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock | SqlBulkCopyOptions.UseInternalTransaction, null);
              sbc.DestinationTableName = foundTableName;
              sbc.BatchSize = BATCH_SIZE;

              //establish connection
              try { conn.Open(); }
              catch { return "Failed to re-established connection to the database after reading data."; }

              //write data
              try { sbc.WriteToServer(rows); }
              catch (Exception ex) { return $"Batch write failed. Details: {ex.Message} - {ex.StackTrace}"; }

              //if we got here, everything worked!
              return string.Empty;
          }

          private static SqlConnection GetConnection(string connectionString)
          {
              DbConnectionStringBuilder csb = new DbConnectionStringBuilder();
              try { csb.ConnectionString = connectionString; }
              catch { return null; }
              return new SqlConnection(csb.ConnectionString);
          }
      }
  }

using (var ctx = new EntitiesContext())
{
    ctx.BulkInsert(list);
}