C# 从db读取200MB文件会引发内存不足异常

我正在尝试查询数据库并提取excel文件，这些文件可能会大到100万行（~200MB），存储为

varbinary

，并将其传递给验证器

我们的构建服务器有6GB的内存和负载平衡的处理器，在运行时，CPU或内存几乎无法达到最大

然而，大约40秒后，进程抛出一个

OutOfMemoryException

这是堆栈跟踪：

System.OutOfMemoryException: Exception of type 'System.OutOfMemoryException' was thrown.
   at System.Data.SqlTypes.SqlBinary.get_Value()
   at System.Data.SqlClient.SqlBuffer.get_ByteArray()
   at System.Data.SqlClient.SqlBuffer.get_Value()
   at System.Data.SqlClient.SqlDataReader.GetValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData metaData)
   at System.Data.SqlClient.SqlDataReader.GetValueInternal(Int32 i)
   at System.Data.SqlClient.SqlDataReader.GetValue(Int32 i)
   at System.Data.SqlClient.SqlCommand.CompleteExecuteScalar(SqlDataReader ds, Boolean returnSqlValue)
   at System.Data.SqlClient.SqlCommand.ExecuteScalar()
   at eConfirmations.DataService.FileServices.FileDataService.GetFileContent(Guid fileId) in d:\w1\3\s\Source\eConfirmations.DataService\FileServices\FileDataService.cs:line 157
...
   at System.Data.SqlTypes.SqlBinary.get_Value()
   at System.Data.SqlClient.SqlBuffer.get_ByteArray()
   at System.Data.SqlClient.SqlBuffer.get_Value()
   at System.Data.SqlClient.SqlDataReader.GetValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData metaData)
   at System.Data.SqlClient.SqlDataReader.GetValueInternal(Int32 i)
   at System.Data.SqlClient.SqlDataReader.GetValue(Int32 i)
   at System.Data.SqlClient.SqlCommand.CompleteExecuteScalar(SqlDataReader ds, Boolean returnSqlValue)
   at System.Data.SqlClient.SqlCommand.ExecuteScalar()
   at eConfirmations.DataService.FileServices.FileDataService.GetFileContent(Guid fileId) in d:\w1\3\s\Source\eConfirmations.DataService\FileServices\FileDataService.cs:line 157

    private byte[] GetFileContent(Guid fileId)
    {
        byte[] content;
        string connectionString = ConfigurationManager.ConnectionStrings["eConfirmationsDatabase"].ConnectionString;

        using (SqlConnection sqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sqlConnection.CreateCommand())
            {
                sqlCommand.CommandTimeout = 300;
                sqlCommand.CommandText = $"SELECT Content FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlConnection.Open();
                content = sqlCommand.ExecuteScalar() as byte[];
                sqlConnection.Close();
                sqlCommand.Dispose();
            }
            sqlConnection.Dispose();
        }
        return content;
    }

    public File GetFileViaFileIdGuid(Guid fileId)
    {
        File file = new File();
        string connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString;
        using (var sourceSqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sourceSqlConnection.CreateCommand())
            {
                sqlCommand.CommandText = $"SELECT FileName, FileExtension, UploadedDateTime, DATALENGTH(Content) as [ContentLength] FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlCommand.CommandType = CommandType.Text;
                sqlCommand.CommandTimeout = 300;
                sourceSqlConnection.Open();

                var reader = sqlCommand.ExecuteReader();
                while (reader.Read())
                {
                    file.FileId = fileId;
                    file.FileExtension = reader["FileExtension"].ToString();
                    file.FileName = reader["FileName"].ToString();
                    file.UploadedDateTime = (DateTime)reader["UploadedDateTime"];
                    file.Content = new byte[Convert.ToInt32(reader["ContentLength"])];
                }

                reader.Close();
                sourceSqlConnection.Close();
            }
        }
        file.Content = GetFileContent(file.FileId, file.Content.Length);
        return file;
    }

    private byte[] GetFileContent(Guid fileId, int contentLength)
    {
        int outputSize = 1048576;
        int bufferSize = contentLength + outputSize;
        byte[] content = new byte[bufferSize];
        string connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString;

        using (SqlConnection sqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sqlConnection.CreateCommand())
            {
                sqlCommand.CommandTimeout = 300;
                sqlCommand.CommandText = $"SELECT Content FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlConnection.Open();
                using (SqlDataReader reader = sqlCommand.ExecuteReader(CommandBehavior.SequentialAccess))
                {

                    while (reader.Read())
                    {
                        int startIndex = 0;
                        long returnValue = reader.GetBytes(0, startIndex, content, startIndex, outputSize);
                        while (returnValue == outputSize)
                        {
                            startIndex += outputSize;
                            returnValue = reader.GetBytes(0, startIndex, content, startIndex, outputSize);
                        }
                    }
                }

                sqlConnection.Close();
            }
        }
        return content;
    }

下面是引发异常的my代码：

System.OutOfMemoryException: Exception of type 'System.OutOfMemoryException' was thrown.
   at System.Data.SqlTypes.SqlBinary.get_Value()
   at System.Data.SqlClient.SqlBuffer.get_ByteArray()
   at System.Data.SqlClient.SqlBuffer.get_Value()
   at System.Data.SqlClient.SqlDataReader.GetValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData metaData)
   at System.Data.SqlClient.SqlDataReader.GetValueInternal(Int32 i)
   at System.Data.SqlClient.SqlDataReader.GetValue(Int32 i)
   at System.Data.SqlClient.SqlCommand.CompleteExecuteScalar(SqlDataReader ds, Boolean returnSqlValue)
   at System.Data.SqlClient.SqlCommand.ExecuteScalar()
   at eConfirmations.DataService.FileServices.FileDataService.GetFileContent(Guid fileId) in d:\w1\3\s\Source\eConfirmations.DataService\FileServices\FileDataService.cs:line 157
...
   at System.Data.SqlTypes.SqlBinary.get_Value()
   at System.Data.SqlClient.SqlBuffer.get_ByteArray()
   at System.Data.SqlClient.SqlBuffer.get_Value()
   at System.Data.SqlClient.SqlDataReader.GetValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData metaData)
   at System.Data.SqlClient.SqlDataReader.GetValueInternal(Int32 i)
   at System.Data.SqlClient.SqlDataReader.GetValue(Int32 i)
   at System.Data.SqlClient.SqlCommand.CompleteExecuteScalar(SqlDataReader ds, Boolean returnSqlValue)
   at System.Data.SqlClient.SqlCommand.ExecuteScalar()
   at eConfirmations.DataService.FileServices.FileDataService.GetFileContent(Guid fileId) in d:\w1\3\s\Source\eConfirmations.DataService\FileServices\FileDataService.cs:line 157

    private byte[] GetFileContent(Guid fileId)
    {
        byte[] content;
        string connectionString = ConfigurationManager.ConnectionStrings["eConfirmationsDatabase"].ConnectionString;

        using (SqlConnection sqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sqlConnection.CreateCommand())
            {
                sqlCommand.CommandTimeout = 300;
                sqlCommand.CommandText = $"SELECT Content FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlConnection.Open();
                content = sqlCommand.ExecuteScalar() as byte[];
                sqlConnection.Close();
                sqlCommand.Dispose();
            }
            sqlConnection.Dispose();
        }
        return content;
    }

    public File GetFileViaFileIdGuid(Guid fileId)
    {
        File file = new File();
        string connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString;
        using (var sourceSqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sourceSqlConnection.CreateCommand())
            {
                sqlCommand.CommandText = $"SELECT FileName, FileExtension, UploadedDateTime, DATALENGTH(Content) as [ContentLength] FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlCommand.CommandType = CommandType.Text;
                sqlCommand.CommandTimeout = 300;
                sourceSqlConnection.Open();

                var reader = sqlCommand.ExecuteReader();
                while (reader.Read())
                {
                    file.FileId = fileId;
                    file.FileExtension = reader["FileExtension"].ToString();
                    file.FileName = reader["FileName"].ToString();
                    file.UploadedDateTime = (DateTime)reader["UploadedDateTime"];
                    file.Content = new byte[Convert.ToInt32(reader["ContentLength"])];
                }

                reader.Close();
                sourceSqlConnection.Close();
            }
        }
        file.Content = GetFileContent(file.FileId, file.Content.Length);
        return file;
    }

    private byte[] GetFileContent(Guid fileId, int contentLength)
    {
        int outputSize = 1048576;
        int bufferSize = contentLength + outputSize;
        byte[] content = new byte[bufferSize];
        string connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString;

        using (SqlConnection sqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sqlConnection.CreateCommand())
            {
                sqlCommand.CommandTimeout = 300;
                sqlCommand.CommandText = $"SELECT Content FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlConnection.Open();
                using (SqlDataReader reader = sqlCommand.ExecuteReader(CommandBehavior.SequentialAccess))
                {

                    while (reader.Read())
                    {
                        int startIndex = 0;
                        long returnValue = reader.GetBytes(0, startIndex, content, startIndex, outputSize);
                        while (returnValue == outputSize)
                        {
                            startIndex += outputSize;
                            returnValue = reader.GetBytes(0, startIndex, content, startIndex, outputSize);
                        }
                    }
                }

                sqlConnection.Close();
            }
        }
        return content;
    }

---

有没有一种更有效的方法可以收回此数据，或者我们可以更新生成服务器上的设置以避免此错误？

好的，下面是发生的情况：

因为它是在32位构建上运行的，所以最大内存分配是2GB，但我仍然没有接近这个阈值

根据这一点，与我的情况非常相似，.NET framework将内存中的对象限制为

256MB

因此，即使我的文件只有200MB，

byte[]

s和

memoryStream

也会以2的幂展开，直到达到所需的256MB。当它们展开时，它们会创建一个大小合适的新实例，并将旧数据复制到新实例上，有效地将内存使用量乘以3，从而导致异常

，但我使用静态字节[]来代替文件流，该字节[]使用预先初始化为数据大小

这是我的最终解决方案：

System.OutOfMemoryException: Exception of type 'System.OutOfMemoryException' was thrown.
   at System.Data.SqlTypes.SqlBinary.get_Value()
   at System.Data.SqlClient.SqlBuffer.get_ByteArray()
   at System.Data.SqlClient.SqlBuffer.get_Value()
   at System.Data.SqlClient.SqlDataReader.GetValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData metaData)
   at System.Data.SqlClient.SqlDataReader.GetValueInternal(Int32 i)
   at System.Data.SqlClient.SqlDataReader.GetValue(Int32 i)
   at System.Data.SqlClient.SqlCommand.CompleteExecuteScalar(SqlDataReader ds, Boolean returnSqlValue)
   at System.Data.SqlClient.SqlCommand.ExecuteScalar()
   at eConfirmations.DataService.FileServices.FileDataService.GetFileContent(Guid fileId) in d:\w1\3\s\Source\eConfirmations.DataService\FileServices\FileDataService.cs:line 157
...
   at System.Data.SqlTypes.SqlBinary.get_Value()
   at System.Data.SqlClient.SqlBuffer.get_ByteArray()
   at System.Data.SqlClient.SqlBuffer.get_Value()
   at System.Data.SqlClient.SqlDataReader.GetValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData metaData)
   at System.Data.SqlClient.SqlDataReader.GetValueInternal(Int32 i)
   at System.Data.SqlClient.SqlDataReader.GetValue(Int32 i)
   at System.Data.SqlClient.SqlCommand.CompleteExecuteScalar(SqlDataReader ds, Boolean returnSqlValue)
   at System.Data.SqlClient.SqlCommand.ExecuteScalar()
   at eConfirmations.DataService.FileServices.FileDataService.GetFileContent(Guid fileId) in d:\w1\3\s\Source\eConfirmations.DataService\FileServices\FileDataService.cs:line 157

    private byte[] GetFileContent(Guid fileId)
    {
        byte[] content;
        string connectionString = ConfigurationManager.ConnectionStrings["eConfirmationsDatabase"].ConnectionString;

        using (SqlConnection sqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sqlConnection.CreateCommand())
            {
                sqlCommand.CommandTimeout = 300;
                sqlCommand.CommandText = $"SELECT Content FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlConnection.Open();
                content = sqlCommand.ExecuteScalar() as byte[];
                sqlConnection.Close();
                sqlCommand.Dispose();
            }
            sqlConnection.Dispose();
        }
        return content;
    }

    public File GetFileViaFileIdGuid(Guid fileId)
    {
        File file = new File();
        string connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString;
        using (var sourceSqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sourceSqlConnection.CreateCommand())
            {
                sqlCommand.CommandText = $"SELECT FileName, FileExtension, UploadedDateTime, DATALENGTH(Content) as [ContentLength] FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlCommand.CommandType = CommandType.Text;
                sqlCommand.CommandTimeout = 300;
                sourceSqlConnection.Open();

                var reader = sqlCommand.ExecuteReader();
                while (reader.Read())
                {
                    file.FileId = fileId;
                    file.FileExtension = reader["FileExtension"].ToString();
                    file.FileName = reader["FileName"].ToString();
                    file.UploadedDateTime = (DateTime)reader["UploadedDateTime"];
                    file.Content = new byte[Convert.ToInt32(reader["ContentLength"])];
                }

                reader.Close();
                sourceSqlConnection.Close();
            }
        }
        file.Content = GetFileContent(file.FileId, file.Content.Length);
        return file;
    }

    private byte[] GetFileContent(Guid fileId, int contentLength)
    {
        int outputSize = 1048576;
        int bufferSize = contentLength + outputSize;
        byte[] content = new byte[bufferSize];
        string connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString;

        using (SqlConnection sqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sqlConnection.CreateCommand())
            {
                sqlCommand.CommandTimeout = 300;
                sqlCommand.CommandText = $"SELECT Content FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlConnection.Open();
                using (SqlDataReader reader = sqlCommand.ExecuteReader(CommandBehavior.SequentialAccess))
                {

                    while (reader.Read())
                    {
                        int startIndex = 0;
                        long returnValue = reader.GetBytes(0, startIndex, content, startIndex, outputSize);
                        while (returnValue == outputSize)
                        {
                            startIndex += outputSize;
                            returnValue = reader.GetBytes(0, startIndex, content, startIndex, outputSize);
                        }
                    }
                }

                sqlConnection.Close();
            }
        }
        return content;
    }

并获取内容：

System.OutOfMemoryException: Exception of type 'System.OutOfMemoryException' was thrown.
   at System.Data.SqlTypes.SqlBinary.get_Value()
   at System.Data.SqlClient.SqlBuffer.get_ByteArray()
   at System.Data.SqlClient.SqlBuffer.get_Value()
   at System.Data.SqlClient.SqlDataReader.GetValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData metaData)
   at System.Data.SqlClient.SqlDataReader.GetValueInternal(Int32 i)
   at System.Data.SqlClient.SqlDataReader.GetValue(Int32 i)
   at System.Data.SqlClient.SqlCommand.CompleteExecuteScalar(SqlDataReader ds, Boolean returnSqlValue)
   at System.Data.SqlClient.SqlCommand.ExecuteScalar()
   at eConfirmations.DataService.FileServices.FileDataService.GetFileContent(Guid fileId) in d:\w1\3\s\Source\eConfirmations.DataService\FileServices\FileDataService.cs:line 157
...
   at System.Data.SqlTypes.SqlBinary.get_Value()
   at System.Data.SqlClient.SqlBuffer.get_ByteArray()
   at System.Data.SqlClient.SqlBuffer.get_Value()
   at System.Data.SqlClient.SqlDataReader.GetValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData metaData)
   at System.Data.SqlClient.SqlDataReader.GetValueInternal(Int32 i)
   at System.Data.SqlClient.SqlDataReader.GetValue(Int32 i)
   at System.Data.SqlClient.SqlCommand.CompleteExecuteScalar(SqlDataReader ds, Boolean returnSqlValue)
   at System.Data.SqlClient.SqlCommand.ExecuteScalar()
   at eConfirmations.DataService.FileServices.FileDataService.GetFileContent(Guid fileId) in d:\w1\3\s\Source\eConfirmations.DataService\FileServices\FileDataService.cs:line 157

    private byte[] GetFileContent(Guid fileId)
    {
        byte[] content;
        string connectionString = ConfigurationManager.ConnectionStrings["eConfirmationsDatabase"].ConnectionString;

        using (SqlConnection sqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sqlConnection.CreateCommand())
            {
                sqlCommand.CommandTimeout = 300;
                sqlCommand.CommandText = $"SELECT Content FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlConnection.Open();
                content = sqlCommand.ExecuteScalar() as byte[];
                sqlConnection.Close();
                sqlCommand.Dispose();
            }
            sqlConnection.Dispose();
        }
        return content;
    }

    public File GetFileViaFileIdGuid(Guid fileId)
    {
        File file = new File();
        string connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString;
        using (var sourceSqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sourceSqlConnection.CreateCommand())
            {
                sqlCommand.CommandText = $"SELECT FileName, FileExtension, UploadedDateTime, DATALENGTH(Content) as [ContentLength] FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlCommand.CommandType = CommandType.Text;
                sqlCommand.CommandTimeout = 300;
                sourceSqlConnection.Open();

                var reader = sqlCommand.ExecuteReader();
                while (reader.Read())
                {
                    file.FileId = fileId;
                    file.FileExtension = reader["FileExtension"].ToString();
                    file.FileName = reader["FileName"].ToString();
                    file.UploadedDateTime = (DateTime)reader["UploadedDateTime"];
                    file.Content = new byte[Convert.ToInt32(reader["ContentLength"])];
                }

                reader.Close();
                sourceSqlConnection.Close();
            }
        }
        file.Content = GetFileContent(file.FileId, file.Content.Length);
        return file;
    }

    private byte[] GetFileContent(Guid fileId, int contentLength)
    {
        int outputSize = 1048576;
        int bufferSize = contentLength + outputSize;
        byte[] content = new byte[bufferSize];
        string connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString;

        using (SqlConnection sqlConnection = new SqlConnection(connectionString))
        {
            using (SqlCommand sqlCommand = sqlConnection.CreateCommand())
            {
                sqlCommand.CommandTimeout = 300;
                sqlCommand.CommandText = $"SELECT Content FROM dbo.[File] WHERE FileId = '{fileId}'";
                sqlConnection.Open();
                using (SqlDataReader reader = sqlCommand.ExecuteReader(CommandBehavior.SequentialAccess))
                {

                    while (reader.Read())
                    {
                        int startIndex = 0;
                        long returnValue = reader.GetBytes(0, startIndex, content, startIndex, outputSize);
                        while (returnValue == outputSize)
                        {
                            startIndex += outputSize;
                            returnValue = reader.GetBytes(0, startIndex, content, startIndex, outputSize);
                        }
                    }
                }

                sqlConnection.Close();
            }
        }
        return content;
    }

---

C#环境有多少堆空间可用？@Carcigenicate.或带有SequentialAccess标志的SqlDataReader将允许您对数据进行流式处理。看起来其他一些人在同一点上有gor神秘的OutOfMemoryException。例如，请参见。我找到的所有搜索结果都没有解决方案。我怀疑对结果的大小有人为的限制。顺便说一句，你知道你是双重处理，实际上是三重处理，对吗？

GetBytes

中的第一个参数不应该是0而不是1？如果你的验证器在一个单独的进程中运行，你可以将文件直接流到磁盘，或者通过管道将流传输到另一个进程。这可以让你把文件分成小块。@BerinLoritsch不幸的是，没有。我正在使用

Aspose.Cells

在内存中创建一个excel文件，如果它没有完整的内容，它将失败。这是您之所以要构建32位应用程序的原因，因为

Aspose.Cells

库？作为一个有点相关的数据点，我发现最大的32位C#应用程序在遇到内存问题之前最终达到了1.4GB左右。请记住，您还必须有空间进行垃圾收集，并且有多个GC桶——如果其中任何一个装满了垃圾，您将得到一个

OutOfMemoryException

。最后，如果您使用WinForms并尝试绘制小于1像素的图形，您也会得到一个

OutOfMemoryException

。这是我遇到的最令人惊讶的事情。在处理记忆问题时。