C# 在.NET中处理大型文件 问题
我需要能够使用C#保存和读取非常大的数据结构。结构本身比较简单,；它是一个非常长的数组，由一个恒定大小的简单结构组成 为了清楚起见，举个例子：C# 在.NET中处理大型文件 问题,c#,.net,asynchronous,io,io-completion-ports,C#,.net,Asynchronous,Io,Io Completion Ports,我需要能够使用C#保存和读取非常大的数据结构。结构本身比较简单,；它是一个非常长的数组，由一个恒定大小的简单结构组成 为了清楚起见，举个例子： struct st { UInt32 a; UInt16 b; //etc. } completion ports st[] data = new st[1024*1024*100] 我希望能够保存和加载这些文件的速度和效率尽可能快 总方向 到目前为止，我的想法是将数据切割成段，当然，从概念上讲，将这些段分配给任务，然后将它们异步写入文件中。File
struct st {
UInt32 a;
UInt16 b;
//etc.
} completion ports
st[] data = new st[1024*1024*100]
MemoryStream.MemoryStream（byte[]）苏。。。。这个问题毫无意义。您最大的瓶颈将是IO，它必须以独占方式访问文件。实际的字节处理速度会很快——您也可以直接将其写入文件（注意，
FileStreamBufferedStreamasyncBiaryWriterBinaryReader不安全的Write: 2012ms
Read: 1089ms
File: 838,860,804 bytes
据我所知，读取或写入文件的最快方法是一个仅向前的进程。否则，除了强制读/写之外，磁盘还必须在文件中来回移动 当然，这并不意味着不能在多个并发线程中处理数据
如果段足够大，磁盘移动的开销可能不会引起注意。[EDIT]我更新了这篇文章，以包含一个完整的可编译示例，并解决@Daniel在下面的评论中提出的问题。因此，此代码不再使用任何“危险”方法，并且没有代码分析警告。[/编辑] 如果您的结构只包含可blittable类型，那么有一种方法可以稍微加快速度 您可以使用封送处理将数据直接读取到数组中，而无需制作其他副本，例如（完整的可编译示例）： 消费线程将从队列中删除内容，如下所示：
public void ProcessDataFromQueue<T>(BlockingCollection<T[]> queue) where T : struct
{
foreach (var array in queue.GetConsumingEnumerable())
{
// Do something with 'array'
}
}
public void ProcessDataFromQueue（BlockingCollection队列），其中T:struct
{
foreach（队列中的var数组。getConsuminegumerable（））
{
//使用“数组”执行某些操作
}
}
不安全的memcpyusing System;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using Microsoft.Win32.SafeHandles;
namespace ConsoleApplication1
{
internal class Program
{
struct TestStruct // Mutable for brevity; real structs should be immutable.
{
public byte ByteValue;
public short ShortValue;
public int IntValue;
public long LongValue;
public float FloatValue;
public double DoubleValue;
}
static void Main()
{
var array = new TestStruct[10];
for (byte i = 0; i < array.Length; ++i)
{
array[i].ByteValue = i;
array[i].ShortValue = i;
array[i].IntValue = i;
array[i].LongValue = i;
array[i].FloatValue = i;
array[i].DoubleValue = i;
}
Directory.CreateDirectory("C:\\TEST");
using (var output = new FileStream(@"C:\TEST\TEST.BIN", FileMode.Create))
FastWrite(output, array, 0, array.Length);
using (var input = new FileStream(@"C:\TEST\TEST.BIN", FileMode.Open))
array = FastRead<TestStruct>(input, array.Length);
for (byte i = 0; i < array.Length; ++i)
{
Trace.Assert(array[i].ByteValue == i);
Trace.Assert(array[i].ShortValue == i);
Trace.Assert(array[i].IntValue == i);
Trace.Assert(array[i].LongValue == i);
Trace.Assert(array[i].FloatValue == i);
Trace.Assert(array[i].DoubleValue == i);
}
}
/// <summary>
/// Writes a part of an array to a file stream as quickly as possible,
/// without making any additional copies of the data.
/// </summary>
/// <typeparam name="T">The type of the array elements.</typeparam>
/// <param name="fs">The file stream to which to write.</param>
/// <param name="array">The array containing the data to write.</param>
/// <param name="offset">The offset of the start of the data in the array to write.</param>
/// <param name="count">The number of array elements to write.</param>
/// <exception cref="IOException">Thrown on error. See inner exception for <see cref="Win32Exception"/></exception>
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2004:RemoveCallsToGCKeepAlive")]
public static void FastWrite<T>(FileStream fs, T[] array, int offset, int count) where T: struct
{
int sizeOfT = Marshal.SizeOf(typeof(T));
GCHandle gcHandle = GCHandle.Alloc(array, GCHandleType.Pinned);
try
{
uint bytesWritten;
uint bytesToWrite = (uint)(count * sizeOfT);
if
(
!WriteFile
(
fs.SafeFileHandle,
new IntPtr(gcHandle.AddrOfPinnedObject().ToInt64() + (offset*sizeOfT)),
bytesToWrite,
out bytesWritten,
IntPtr.Zero
)
)
{
throw new IOException("Unable to write file.", new Win32Exception(Marshal.GetLastWin32Error()));
}
Debug.Assert(bytesWritten == bytesToWrite);
}
finally
{
gcHandle.Free();
}
}
/// <summary>
/// Reads array data from a file stream as quickly as possible,
/// without making any additional copies of the data.
/// </summary>
/// <typeparam name="T">The type of the array elements.</typeparam>
/// <param name="fs">The file stream from which to read.</param>
/// <param name="count">The number of elements to read.</param>
/// <returns>
/// The array of elements that was read. This may be less than the number that was
/// requested if the end of the file was reached. It may even be empty.
/// NOTE: There may still be data left in the file, even if not all the requested
/// elements were returned - this happens if the number of bytes remaining in the
/// file is less than the size of the array elements.
/// </returns>
/// <exception cref="IOException">Thrown on error. See inner exception for <see cref="Win32Exception"/></exception>
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2004:RemoveCallsToGCKeepAlive")]
public static T[] FastRead<T>(FileStream fs, int count) where T: struct
{
int sizeOfT = Marshal.SizeOf(typeof(T));
long bytesRemaining = fs.Length - fs.Position;
long wantedBytes = count * sizeOfT;
long bytesAvailable = Math.Min(bytesRemaining, wantedBytes);
long availableValues = bytesAvailable / sizeOfT;
long bytesToRead = (availableValues * sizeOfT);
if ((bytesRemaining < wantedBytes) && ((bytesRemaining - bytesToRead) > 0))
{
Debug.WriteLine("Requested data exceeds available data and partial data remains in the file.", "Dmr.Common.IO.Arrays.FastRead(fs,count)");
}
T[] result = new T[availableValues];
if (availableValues == 0)
return result;
GCHandle gcHandle = GCHandle.Alloc(result, GCHandleType.Pinned);
try
{
uint bytesRead;
if
(
!ReadFile
(
fs.SafeFileHandle,
gcHandle.AddrOfPinnedObject(),
(uint)bytesToRead,
out bytesRead,
IntPtr.Zero
)
)
{
throw new IOException("Unable to read file.", new Win32Exception(Marshal.GetLastWin32Error()));
}
Debug.Assert(bytesRead == bytesToRead);
}
finally
{
gcHandle.Free();
}
return result;
}
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Interoperability", "CA1415:DeclarePInvokesCorrectly")]
[DllImport("kernel32.dll", SetLastError=true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool WriteFile
(
SafeFileHandle hFile,
IntPtr lpBuffer,
uint nNumberOfBytesToWrite,
out uint lpNumberOfBytesWritten,
IntPtr lpOverlapped
);
/// <summary>See the Windows API documentation for details.</summary>
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Interoperability", "CA1415:DeclarePInvokesCorrectly")]
[DllImport("kernel32.dll", SetLastError=true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool ReadFile
(
SafeFileHandle hFile,
IntPtr lpBuffer,
uint nNumberOfBytesToRead,
out uint lpNumberOfBytesRead,
IntPtr lpOverlapped
);
}
}
public void ReadIntoQueue<T>(FileStream fs, BlockingCollection<T[]> queue, int blockSize) where T: struct
{
while (true)
{
var data = FastRead<T>(fs, blockSize);
if (data.Length == 0)
{
queue.CompleteAdding();
break;
}
queue.Add(data);
}
}
public void ProcessDataFromQueue<T>(BlockingCollection<T[]> queue) where T : struct
{
foreach (var array in queue.GetConsumingEnumerable())
{
// Do something with 'array'
}
}