C# 使用CsvHelper同时从zip存档读取多个文件
我正在做的是下载一个zip档案,然后把档案中的每一个文件读到列表中。当我同步编程时,它会成功,但需要永远 我决定尝试使用任务读取不同线程上的每个文件。当我这样做时,我得到以下信息: 找不到中心目录记录的结尾 我编写了一个类来处理下载和提取:C# 使用CsvHelper同时从zip存档读取多个文件,c#,.net,multithreading,zip,gtfs,C#,.net,Multithreading,Zip,Gtfs,我正在做的是下载一个zip档案,然后把档案中的每一个文件读到列表中。当我同步编程时,它会成功,但需要永远 我决定尝试使用任务读取不同线程上的每个文件。当我这样做时,我得到以下信息: 找不到中心目录记录的结尾 我编写了一个类来处理下载和提取: public class GtfsFileDownloader { public string FileLocation { get; set; } public string FileName { get; set; } publi
public class GtfsFileDownloader
{
public string FileLocation { get; set; }
public string FileName { get; set; }
public MemoryStream ZipStream { get; set; }
public GtfsFileDownloader(string loc, string nm)
{
FileLocation = loc;
FileName = nm;
}
public void DownloadZip()
{
ZipStream = new MemoryStream(new WebClient().DownloadData(FileLocation + FileName));
}
public List<T> GetFileContents<T, Q>(string fileName) where Q: ClassMap
{
var retList = new List<T>();
var entry = new ZipArchive(ZipStream).Entries.SingleOrDefault(x => x.FullName == fileName);
if(entry != null)
{
using (var reader = new StreamReader(entry.Open()))
{
using (var csv = new CsvReader(reader))
{
csv.Configuration.HeaderValidated = null;
csv.Configuration.MissingFieldFound = null;
csv.Configuration.RegisterClassMap<Q>();
try
{
retList = csv.GetRecords<T>().ToList();
}
catch(CsvHelperException ex)
{
throw new System.Exception(ex.Message);
}
}
}
}
return retList;
}
}
公共类GtfsFileDownloader
{
公共字符串文件位置{get;set;}
公共字符串文件名{get;set;}
公共内存流ZipStream{get;set;}
公共GtfsFileDownloader(字符串loc,字符串nm)
{
FileLocation=loc;
文件名=nm;
}
public void DownloadZip()
{
ZipStream=newmemoryStream(newWebClient().DownloadData(FileLocation+FileName));
}
公共列表GetFileContents(字符串文件名),其中Q:ClassMap
{
var retList=新列表();
var entry=new-ZipArchive(ZipStream).Entries.SingleOrDefault(x=>x.FullName==fileName);
if(条目!=null)
{
使用(var reader=newstreamreader(entry.Open()))
{
使用(var csv=新的CsvReader(读卡器))
{
csv.Configuration.HeaderValidated=null;
csv.Configuration.MissingFieldFound=null;
csv.Configuration.RegisterClassMap();
尝试
{
retList=csv.GetRecords().ToList();
}
捕获(CsvHelperException ex)
{
抛出新系统异常(例如消息);
}
}
}
}
返回列表;
}
}
以下是主要代码:
var downloader = new GtfsFileDownloader(agency.GtfsZipUrlDirectory, agency.GtfsZipUrlFileName);
downloader.DownloadZip();
var agencyInfo = new List<DbAgency>();
var stopInfo = new List<DbStop>();
var routeInfo = new List<DbRoute>();
var tripInfo = new List<DbTrip>();
var stopTimeInfo = new List<DbStopTime>();
var calendarInfo = new List<DbCalendar>();
var fareAttributeInfo = new List<DbFareAttribute>();
var shapeInfo = new List<DbShape>();
var frequencyInfo = new List<DbFrequency>();
var transferInfo = new List<DbTransfer>();
var pathwayInfo = new List<DbPathway>();
var levelInfo = new List<DbLevel>();
var feedInfoInfo = new List<DbFeedInfo>();
var tasks = new List<Task>();
tasks.Add(new Task(() => { agencyInfo = downloader.GetFileContents<DbAgency, AgencyMap>("agencies.txt"); }));
tasks.Add(new Task(() => { stopInfo = downloader.GetFileContents<DbStop, StopMap>("stops.txt"); }));
tasks.Add(new Task(() => { routeInfo = downloader.GetFileContents<DbRoute, RouteMap>("routes.txt"); }));
tasks.Add(new Task(() => { tripInfo = downloader.GetFileContents<DbTrip, TripMap>("trips.txt"); }));
tasks.Add(new Task(() => { stopTimeInfo = downloader.GetFileContents<DbStopTime, StopTimeMap>("stop_times.txt"); }));
tasks.Add(new Task(() => { calendarInfo = downloader.GetFileContents<DbCalendar, CalendarMap>("calendar.txt"); }));
tasks.Add(new Task(() => { fareAttributeInfo = downloader.GetFileContents<DbFareAttribute, FareAttributeMap>("fare_attributes.txt"); }));
tasks.Add(new Task(() => { shapeInfo = downloader.GetFileContents<DbShape, ShapeMap>("shapes.txt"); }));
tasks.Add(new Task(() => { frequencyInfo = downloader.GetFileContents<DbFrequency, FrequencyMap>("frequencies.txt"); }));
tasks.Add(new Task(() => { transferInfo = downloader.GetFileContents<DbTransfer, TransferMap>("transfers.txt"); }));
tasks.Add(new Task(() => { pathwayInfo = downloader.GetFileContents<DbPathway, PathwayMap>("pathways.txt"); }));
tasks.Add(new Task(() => { levelInfo = downloader.GetFileContents<DbLevel, LevelMap>("levels.txt"); }));
tasks.Add(new Task(() => { feedInfoInfo = downloader.GetFileContents<DbFeedInfo, FeedInfoMap>("feed_info.txt"); }));
foreach(Task t in tasks)
{
t.Start();
}
Task.WaitAll(tasks.ToArray());
var downloader=新的GtfsFileDownloader(agency.GtfsZipUrlDirectory,agency.GtfsZipUrlFileName);
downloader.DownloadZip();
var agencyInfo=新列表();
var stopInfo=新列表();
var routeInfo=新列表();
var tripInfo=新列表();
var stopTimeInfo=新列表();
var calendarInfo=新列表();
var fareAttributeInfo=新列表();
var shapeInfo=新列表();
var frequencyInfo=新列表();
var transferInfo=新列表();
var pathwayInfo=新列表();
var levelInfo=新列表();
var feedInfoInfo=新列表();
var tasks=新列表();
添加(新任务(()=>{agencyInfo=downloader.GetFileContents(“agents.txt”);});
添加(新任务(()=>{stopInfo=downloader.GetFileContents(“stops.txt”);});
添加(新任务(()=>{routeInfo=downloader.GetFileContents(“routes.txt”);});
添加(新任务(()=>{tripInfo=downloader.GetFileContents(“trips.txt”);});
添加(新任务(()=>{stopTimeInfo=downloader.GetFileContents(“stop_times.txt”);});
添加(新任务(()=>{calendarInfo=downloader.GetFileContents(“calendar.txt”);});
添加(新任务(()=>{fareAttributeInfo=downloader.GetFileContents(“fare_attributes.txt”);});
添加(新任务(()=>{shapeInfo=downloader.GetFileContents(“shapes.txt”);});
添加(新任务(()=>{frequencyInfo=downloader.GetFileContents(“frequencies.txt”);});
添加(新任务(()=>{transferInfo=downloader.GetFileContents(“transfers.txt”);});
添加(新任务(()=>{pathwayInfo=downloader.GetFileContents(“paths.txt”);});
添加(新任务(()=>{levelInfo=downloader.GetFileContents(“levels.txt”);});
添加(新任务(()=>{feedInfoInfo=downloader.GetFileContents(“feed_info.txt”);});
foreach(任务中的任务t)
{
t、 Start();
}
Task.WaitAll(tasks.ToArray());
我假设我在多线程方面做得不正确(我在多线程方面没有太多经验)。就像我提到的,如果我取出任务内容并单线程运行它,它不会抛出上面的错误。尝试为每个
任务制作内存流
:
public class GtfsFileDownloader
{
public string FileLocation { get; set; }
public string FileName { get; set; }
public byte[] ZipBytes { get; set; }
public GtfsFileDownloader(string loc, string nm)
{
FileLocation = loc;
FileName = nm;
}
public void DownloadZip()
{
ZipBytes = new WebClient().DownloadData(FileLocation + FileName);
}
public List<T> GetFileContents<T, Q>(string fileName) where Q: ClassMap
{
var retList = new List<T>();
using (var ZipStream = new MemoryStream(ZipBytes)) {
var entry = new ZipArchive(ZipStream).Entries.SingleOrDefault(x => x.FullName == fileName);
if(entry != null)
{
using (var reader = new StreamReader(entry.Open()))
{
using (var csv = new CsvReader(reader))
{
csv.Configuration.HeaderValidated = null;
csv.Configuration.MissingFieldFound = null;
csv.Configuration.RegisterClassMap<Q>();
try
{
retList = csv.GetRecords<T>().ToList();
}
catch(CsvHelperException ex)
{
throw new System.Exception(ex.Message);
}
}
}
}
}
return retList;
}
}
公共类GtfsFileDownloader
{
公共字符串文件位置{get;set;}
公共字符串文件名{get;set;}
公共字节[]ZipBytes{get;set;}
公共GtfsFileDownloader(字符串loc,字符串nm)
{
FileLocation=loc;
文件名=nm;
}
public void DownloadZip()
{
ZipBytes=new WebClient()。下载数据(文件位置+文件名);
}
公共列表GetFileContents(字符串文件名),其中Q:ClassMap
{
var retList=新列表();
使用(var ZipStream=新内存流(ZipBytes)){
var entry=new-ZipArchive(ZipStream).Entries.SingleOrDefault(x=>x.FullName==fileName);
if(条目!=null)
{
使用