C# 为什么在使用Windows ReadFile读入数据后解析数据时性能会如此缓慢?
我正在对几种在C#Net中执行文件IO的方法进行性能测试。我执行的两个任务是将文本文件全部读入缓冲区,然后解析该文件。数据是带有标题记录的制表符分隔的记录,但这并不重要 测试中唯一不同的是从文件中读取数据的方法。解析代码是相同的。但是,一种文件读取方法(直接调用Windows API ReadFile)比另一种方法(file.ReadAllBytes)快1.4倍,但之后使用ReadFile读取的文件的解析时间比使用file.ReadAllBytes读取数据时长2.5倍 调用ReadFile涉及固定和取消固定内存和不安全代码。这是性能下降的原因吗?有办法绕过它吗?调用ReadFile的更好方法?这是我的测试代码循环。我没有展示解析代码,但它的所有工作都局限于扫描IO缓冲区、查找制表符和CRLF分隔符以及创建字符串。它本身没有不安全的代码,也没有对字符串类、编码类和泛型列表以外的外部方法的任何调用C# 为什么在使用Windows ReadFile读入数据后解析数据时性能会如此缓慢?,c#,.net,io,unsafe,C#,.net,Io,Unsafe,我正在对几种在C#Net中执行文件IO的方法进行性能测试。我执行的两个任务是将文本文件全部读入缓冲区,然后解析该文件。数据是带有标题记录的制表符分隔的记录,但这并不重要 测试中唯一不同的是从文件中读取数据的方法。解析代码是相同的。但是,一种文件读取方法(直接调用Windows API ReadFile)比另一种方法(file.ReadAllBytes)快1.4倍,但之后使用ReadFile读取的文件的解析时间比使用file.ReadAllBytes读取数据时长2.5倍 调用ReadFile涉及固
private void mctlReadFileButton_Click(object sender, EventArgs e)
{
//MessageBox.Show(Separator, "Record Delimiter");
var timer = new Stopwatch();
var timer2 = new Stopwatch();
int bufferSize = 128 * 1024 * 1024;
byte[] ByteBuf = new byte[bufferSize];
WinFileIO WFIO = null;
double readTimeTotal = 0;
double parseTimeTotal = 0;
int loopCount = RepeatCount;
mctlTimeTextBox.Text = "Running...";
mctlTimeToParseTextBox.Text = "Running...";
mctlTimeTextBox.Refresh();
mctlTimeToParseTextBox.Refresh();
for (var i = 0; i < loopCount; i++)
{
timer.Start();
switch (IOMethod)
{
case "Windows API":
WFIO = new WinFileIO(ByteBuf);
WFIO.OpenForReading(FileName);
WFIO.ReadBlocks(bufferSize);
WFIO.Close();
break;
case "File.ReadAllBytes":
ByteBuf = File.ReadAllBytes(FileName);
break;
default:
break;
}
timer.Stop();
readTimeTotal += (double)timer.ElapsedMilliseconds / 1000.0;
timer.Reset();
timer2.Start();
if (ParseFile)
{
var parser = new ColumnSelector(new List<String>()
{
"AcctID", "Policy#", "Location ID", "TIV", "LOSS Gross", "State"
},
ByteBuf
);
var columns = parser.Select();
}
if (WFIO != null)
WFIO.Dispose();
timer2.Stop();
parseTimeTotal += (double)timer2.ElapsedMilliseconds / 1000.0;
timer2.Reset();
}
var timeMsg = "" + loopCount + " times. " + readTimeTotal + " sec";
mctlTimeTextBox.Text = timeMsg;
mctlTimeTextBox.Refresh();
timeMsg = "" + loopCount + " times. " + parseTimeTotal + " sec";
mctlTimeToParseTextBox.Text = timeMsg;
mctlTimeToParseTextBox.Refresh();
}
private void mctlReadFileButton\u单击(对象发送方,事件参数e)
{
//Show(分隔符,“记录分隔符”);
var timer=新秒表();
var timer2=新秒表();
int bufferSize=128*1024*1024;
byte[]ByteBuf=新字节[bufferSize];
WinFileIO WFIO=null;
double readTimeTotal=0;
double parseTimeTotal=0;
int loopCount=RepeatCount;
mctlTimeTextBox.Text=“正在运行…”;
mctlTimeToParseTextBox.Text=“正在运行…”;
mctlTimeTextBox.Refresh();
mctlTimeToParseTextBox.Refresh();
对于(变量i=0;i
对Windows ReadFile函数的调用依赖于Robert G.Bryan编写的一些示例代码。这是他的WinFileIO课程。我不相信我以任何方式修改了它
using System;
using System.ComponentModel;
using System.Runtime.InteropServices;
namespace Win32FileIO
{
unsafe public class WinFileIO : IDisposable
{
// This class provides the capability to utilize the ReadFile and Writefile windows IO functions. These functions
// are the most efficient way to perform file I/O from C# or even C++. The constructor with the buffer and buffer
// size should usually be called to init this class. PinBuffer is provided as an alternative. The reason for this
// is because a pointer needs to be obtained before the ReadFile or WriteFile functions are called.
//
// Error handling - In each public function of this class where an error can occur, an ApplicationException is
// thrown with the Win32Exception message info if an error is detected. If no exception is thrown, then a normal
// return is considered success.
//
// This code is not thread safe. Thread control primitives need to be added if running this in a multi-threaded
// environment.
//
// The recommended and fastest function for reading from a file is to call the ReadBlocks method.
// The recommended and fastest function for writing to a file is to call the WriteBlocks method.
//
// License and disclaimer:
// This software is free to use by any individual or entity for any endeavor for profit or not.
// Even though this code has been tested and automated unit tests are provided, there is no gaurantee that
// it will run correctly with your system or environment. I am not responsible for any failure and you agree
// that you accept any and all risk for using this software.
//
//
// Written by Robert G. Bryan in Feb, 2011.
//
// Constants required to handle file I/O:
private const uint GENERIC_READ = 0x80000000;
private const uint GENERIC_WRITE = 0x40000000;
private const uint OPEN_EXISTING = 3;
private const uint CREATE_ALWAYS = 2;
private const int BlockSize = 65536;
//
private GCHandle gchBuf; // Handle to GCHandle object used to pin the I/O buffer in memory.
private System.IntPtr pHandle; // Handle to the file to be read from or written to.
private void* pBuffer; // Pointer to the buffer used to perform I/O.
// Define the Windows system functions that are called by this class via COM Interop:
[System.Runtime.InteropServices.DllImport("kernel32", SetLastError = true)]
static extern unsafe System.IntPtr CreateFile
(
string FileName, // file name
uint DesiredAccess, // access mode
uint ShareMode, // share mode
uint SecurityAttributes, // Security Attributes
uint CreationDisposition, // how to create
uint FlagsAndAttributes, // file attributes
int hTemplateFile // handle to template file
);
[System.Runtime.InteropServices.DllImport("kernel32", SetLastError = true)]
static extern unsafe bool ReadFile
(
System.IntPtr hFile, // handle to file
void* pBuffer, // data buffer
int NumberOfBytesToRead, // number of bytes to read
int* pNumberOfBytesRead, // number of bytes read
int Overlapped // overlapped buffer which is used for async I/O. Not used here.
);
[System.Runtime.InteropServices.DllImport("kernel32", SetLastError = true)]
static extern unsafe bool WriteFile
(
IntPtr handle, // handle to file
void* pBuffer, // data buffer
int NumberOfBytesToWrite, // Number of bytes to write.
int* pNumberOfBytesWritten,// Number of bytes that were written..
int Overlapped // Overlapped buffer which is used for async I/O. Not used here.
);
[System.Runtime.InteropServices.DllImport("kernel32", SetLastError = true)]
static extern unsafe bool CloseHandle
(
System.IntPtr hObject // handle to object
);
public WinFileIO()
{
pHandle = IntPtr.Zero;
}
public WinFileIO(Array Buffer)
{
// This constructor is provided so that the buffer can be pinned in memory.
// Cleanup must be called in order to unpin the buffer.
PinBuffer(Buffer);
pHandle = IntPtr.Zero;
}
protected void Dispose(bool disposing)
{
// This function frees up the unmanaged resources of this class.
Close();
UnpinBuffer();
}
public void Dispose()
{
// This method should be called to clean everything up.
Dispose(true);
// Tell the GC not to finalize since clean up has already been done.
GC.SuppressFinalize(this);
}
~WinFileIO()
{
// Finalizer gets called by the garbage collector if the user did not call Dispose.
Dispose(false);
}
public void PinBuffer(Array Buffer)
{
// This function must be called to pin the buffer in memory before any file I/O is done.
// This shows how to pin a buffer in memory for an extended period of time without using
// the "Fixed" statement. Pinning a buffer in memory can take some cycles, so this technique
// is helpful when doing quite a bit of file I/O.
//
// Make sure we don't leak memory if this function was called before and the UnPinBuffer was not called.
UnpinBuffer();
gchBuf = GCHandle.Alloc(Buffer, GCHandleType.Pinned);
IntPtr pAddr = Marshal.UnsafeAddrOfPinnedArrayElement(Buffer, 0);
// pBuffer is the pointer used for all of the I/O functions in this class.
pBuffer = (void*)pAddr.ToPointer();
}
public void UnpinBuffer()
{
// This function unpins the buffer and needs to be called before a new buffer is pinned or
// when disposing of this object. It does not need to be called directly since the code in Dispose
// or PinBuffer will automatically call this function.
if (gchBuf.IsAllocated)
gchBuf.Free();
}
public void OpenForReading(string FileName)
{
// This function uses the Windows API CreateFile function to open an existing file.
// A return value of true indicates success.
Close();
pHandle = CreateFile(FileName, GENERIC_READ, 0, 0, OPEN_EXISTING, 0, 0);
if (pHandle == System.IntPtr.Zero)
{
Win32Exception WE = new Win32Exception();
ApplicationException AE = new ApplicationException("WinFileIO:OpenForReading - Could not open file " +
FileName + " - " + WE.Message);
throw AE;
}
}
public void OpenForWriting(string FileName)
{
// This function uses the Windows API CreateFile function to open an existing file.
// If the file exists, it will be overwritten.
Close();
pHandle = CreateFile(FileName, GENERIC_WRITE, 0, 0, CREATE_ALWAYS, 0, 0);
if (pHandle == System.IntPtr.Zero)
{
Win32Exception WE = new Win32Exception();
ApplicationException AE = new ApplicationException("WinFileIO:OpenForWriting - Could not open file " +
FileName + " - " + WE.Message);
throw AE;
}
}
public int Read(int BytesToRead)
{
// This function reads in a file up to BytesToRead using the Windows API function ReadFile. The return value
// is the number of bytes read.
int BytesRead = 0;
if (!ReadFile(pHandle, pBuffer, BytesToRead, &BytesRead, 0))
{
Win32Exception WE = new Win32Exception();
ApplicationException AE = new ApplicationException("WinFileIO:Read - Error occurred reading a file. - " +
WE.Message);
throw AE;
}
return BytesRead;
}
public int ReadUntilEOF()
{
// This function reads in chunks at a time instead of the entire file. Make sure the file is <= 2GB.
// Also, if the buffer is not large enough to read the file, then an ApplicationException will be thrown.
// No check is made to see if the buffer is large enough to hold the file. If this is needed, then
// use the ReadBlocks function below.
int BytesReadInBlock = 0, BytesRead = 0;
byte* pBuf = (byte*)pBuffer;
// Do until there are no more bytes to read or the buffer is full.
for (; ; )
{
if (!ReadFile(pHandle, pBuf, BlockSize, &BytesReadInBlock, 0))
{
// This is an error condition. The error msg can be obtained by creating a Win32Exception and
// using the Message property to obtain a description of the error that was encountered.
Win32Exception WE = new Win32Exception();
ApplicationException AE = new ApplicationException("WinFileIO:ReadUntilEOF - Error occurred reading a file. - "
+ WE.Message);
throw AE;
}
if (BytesReadInBlock == 0)
break;
BytesRead += BytesReadInBlock;
pBuf += BytesReadInBlock;
}
return BytesRead;
}
public int ReadBlocks(int BytesToRead)
{
// This function reads a total of BytesToRead at a time. There is a limit of 2gb per call.
int BytesReadInBlock = 0, BytesRead = 0, BlockByteSize;
byte* pBuf = (byte*)pBuffer;
// Do until there are no more bytes to read or the buffer is full.
do
{
BlockByteSize = Math.Min(BlockSize, BytesToRead - BytesRead);
if (!ReadFile(pHandle, pBuf, BlockByteSize, &BytesReadInBlock, 0))
{
Win32Exception WE = new Win32Exception();
ApplicationException AE = new ApplicationException("WinFileIO:ReadBytes - Error occurred reading a file. - "
+ WE.Message);
throw AE;
}
if (BytesReadInBlock == 0)
break;
BytesRead += BytesReadInBlock;
pBuf += BytesReadInBlock;
} while (BytesRead < BytesToRead);
return BytesRead;
}
public int Write(int BytesToWrite)
{
// Writes out the file in one swoop using the Windows WriteFile function.
int NumberOfBytesWritten;
if (!WriteFile(pHandle, pBuffer, BytesToWrite, &NumberOfBytesWritten, 0))
{
Win32Exception WE = new Win32Exception();
ApplicationException AE = new ApplicationException("WinFileIO:Write - Error occurred writing a file. - " +
WE.Message);
throw AE;
}
return NumberOfBytesWritten;
}
public int WriteBlocks(int NumBytesToWrite)
{
// This function writes out chunks at a time instead of the entire file. This is the fastest write function,
// perhaps because the block size is an even multiple of the sector size.
int BytesWritten = 0, BytesToWrite, RemainingBytes, BytesOutput = 0;
byte* pBuf = (byte*)pBuffer;
RemainingBytes = NumBytesToWrite;
// Do until there are no more bytes to write.
do
{
BytesToWrite = Math.Min(RemainingBytes, BlockSize);
if (!WriteFile(pHandle, pBuf, BytesToWrite, &BytesWritten, 0))
{
// This is an error condition. The error msg can be obtained by creating a Win32Exception and
// using the Message property to obtain a description of the error that was encountered.
Win32Exception WE = new Win32Exception();
ApplicationException AE = new ApplicationException("WinFileIO:WriteBlocks - Error occurred writing a file. - "
+ WE.Message);
throw AE;
}
pBuf += BytesToWrite;
BytesOutput += BytesToWrite;
RemainingBytes -= BytesToWrite;
} while (RemainingBytes > 0);
return BytesOutput;
}
public bool Close()
{
// This function closes the file handle.
bool Success = true;
if (pHandle != IntPtr.Zero)
{
Success = CloseHandle(pHandle);
pHandle = IntPtr.Zero;
}
return Success;
}
}
}
使用系统;
使用系统组件模型;
使用System.Runtime.InteropServices;
名称空间Win32FileIO
{
不安全的公共类WinFileIO:IDisposable
{
//此类提供了利用ReadFile和Writefile windows IO函数的功能
//是从C++或C++中执行文件I/O的最有效方法。
//通常应调用size来初始化此类。PinBuffer是作为替代方案提供的。原因是
//是因为在调用ReadFile或WriteFile函数之前需要获取指针。
//
//错误处理-在此类中可能发生错误的每个公共函数中,都会出现ApplicationException
//如果检测到错误,则引发Win32Exception消息信息。如果未引发异常,则正常
//返回被认为是成功。
//
//此代码不是线程安全的。如果在多线程环境中运行此代码,则需要添加线程控制原语
//环境。
//
//从文件中读取的推荐且最快的函数是调用ReadBlocks方法。
//写入文件的建议和最快功能是调用WriteBlocks方法。
//
//许可证和免责声明:
//本软件可由任何个人或实体免费使用,无论是否出于盈利目的。
//即使该代码已经过测试并提供了自动单元测试,也没有一个gaurantee
//它将在您的系统或环境中正确运行。我不对任何故障负责,您也同意
//您接受使用本软件的任何和所有风险。
//
//
//罗伯特·G·布莱恩于2011年2月撰写。
//
//处理文件I/O所需的常量:
专用consuint GENERIC_READ=0x8000000;
专用consunt GENERIC_WRITE=0x40000000;
私人建筑的开放性=3;
私有consuint CREATE_始终=2;
private const int BlockSize=65536;
//
专用GCHandle gchBuf;//用于在内存中固定I/O缓冲区的GCHandle对象的句柄。
private System.IntPtr pHandle;//要读取或写入的文件的句柄。
private void*pBuffer;//指向用于执行I/O的缓冲区的指针。
//定义此类通过COM互操作调用的Windows系统函数:
[System.Runtime.InteropServices.DllImport(“内核32”,