C# Parallel.ForEach（…）系统内存不足异常
我已经研究过这个问题，但没有给出多少解决方案。我对使用Parallel.ForEach非常陌生，所以我正试图弄清楚到底发生了什么 诊断工具的上限为1023次重复（我知道这是x86到x64的arch限制，但我想提供两种格式的程序。）我也不觉得任何程序都应该达到这个阈值。当我在x64中编译程序时，我的MaxDegree大约为1.1-1.4GB。为了进行测试，我在每个Parallel.ForEach迭代结束时运行GC.Collection（）（我知道这不是一个好的实践，我只是尝试在这一点上进行故障排除。） 以下是我看到的： 现在，如果我尝试使用分区器方法，例如：C# Parallel.ForEach（…）系统内存不足异常,c#,parallel-processing,garbage-collection,C#,Parallel Processing,Garbage Collection,我已经研究过这个问题，但没有给出多少解决方案。我对使用Parallel.ForEach非常陌生，所以我正试图弄清楚到底发生了什么 诊断工具的上限为1023次重复（我知道这是x86到x64的arch限制，但我想提供两种格式的程序。）我也不觉得任何程序都应该达到这个阈值。当我在x64中编译程序时，我的MaxDegree大约为1.1-1.4GB。为了进行测试，我在每个Parallel.ForEach迭代结束时运行GC.Collection（）（我知道这不是一个好的实践，我只是尝试在这一点上进行故障排除
var checkforfinished=Parallel.ForEach（Partitioner.Create（0，lstBackupUsers.Items.Count），lstBackupUsers.Items.Cast（），opts，name=>Parallel.ForEachvar checkforfinished=Parallel.ForEach（Partitioner.Create（0，lstBackupUsers.Items.Count），lstBackupUsers.Items，opts，name=>并行。ForEach- 用户单击“开始备份”按钮，如屏幕截图所示
- 按钮开始一个单独的函数，检查用户选择从中获取用户名列表的方法（LDAP或平面文本文件）
- 然后，该函数发送一个bgw_dowork（）请求
- 在DoWork请求中，它看起来像是以下内容的摘要：
- 检查初步报表（例如bgw.cancellationpending）
- 继续从configurationmanager.appsettings获取一些设置
- 开始“complex”Parallel.ForEach命令，该命令读取列表框记录行，ForEach行执行很长的命令列表，以完成一个用户的操作
- 整个程序，尤其是bgw_dowork，大量使用Google的v3驱动API以用户身份登录，获取其他功能记录的文件，这些功能准备备份用户目录（单独的功能以用户身份登录，记录其文件（和文件ID）及其目录/子目录）bgw_dowork执行实际下载功能的一部分，然后调用其他功能在文件下载后完成移动
private void bgW_DoWork（对象发送方，DoWorkEventArgs e）
{
{
尝试
{
txtFile.ReadOnly=true；
btnStart.Text=“取消备份”；
var-appSettings=ConfigurationManager.appSettings；
字符串checkreplace=ConfigurationManager.AppSettings[“checkreplace”]；
字符串userfile=txtFile.Text；
int计数器=0；
int arraycount=0；
如果（bgW.取消待定）
{
e、 取消=真；
stripLabel.Text=“操作已取消！”；
}
其他的
{
对于（int z=0；z>=计数器；z++）
{
如果（bgW.取消待定）
{
e、 取消=真；
stripLabel.Text=“操作已取消！”；
打破
}
其他的
{
double totalresource=int.Parse（ConfigurationManager.AppSettings[“多线程”]）；
totalresource=（totalresource/100）；
//var opts=new ParallelOptions{maxdegreeofpparallelism=Convert.ToInt32（Math.天花（（Environment.ProcessorCount*totalresource）*1.0））；
var opts=newparalleloptions{maxdegreeofpparallelism=2}；
var part=Partitioner.Create（1100）；
//foreach（lstBackupUsers.Items中的ListViewItem名称）
var checkforfinished=Parallel.ForEach（lstBackupUsers.Items.Cast（），name=>
{
尝试
{
字符串名称=名称。子项[0]。文本；
lstBackupUsers.Items[arraycount].Selected=true；
lstBackupUsers.Items[arraycount].BackColor=Color.CornflowerBlue；
arraycount++；
stripLabel.Text=“”；
Console.WriteLine（“选择用户：+names.ToString（））；
txtLog.Text+=“正在选择用户：”+names.ToString（）+Environment.NewLine；
txtCurrentUser.Text=names.ToString（）；
//定义请求的参数。
字符串user=names.ToString（）；
//检查目录是否存在，如果不存在则创建。
字符串savelocation=ConfigurationManager.AppSettings[“savelocation”]+用户+“\\”；
if（File.Exists（savelocation+“.deltalog.tok”））
Delete（savelocation+“.deltalog.tok”）；
FileInfo testdir=新的FileInfo（保存位置）；
testdir.Directory.Create（）；
字符串savedStartPageToken=“”；
弗吉尼亚州
private void bgW_DoWork(object sender, DoWorkEventArgs e)
{
{
try
{
txtFile.ReadOnly = true;
btnStart.Text = "Cancel Backup";
var appSettings = ConfigurationManager.AppSettings;
string checkreplace = ConfigurationManager.AppSettings["checkreplace"];
string userfile = txtFile.Text;
int counter = 0;
int arraycount = 0;
if (bgW.CancellationPending)
{
e.Cancel = true;
stripLabel.Text = "Operation was canceled!";
}
else
{
for (int z = 0; z >= counter; z++)
{
if (bgW.CancellationPending)
{
e.Cancel = true;
stripLabel.Text = "Operation was canceled!";
break;
}
else
{
double totalresource = int.Parse(ConfigurationManager.AppSettings["multithread"]);
totalresource = (totalresource / 100);
//var opts = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Math.Ceiling((Environment.ProcessorCount * totalresource) * 1.0)) };
var opts = new ParallelOptions { MaxDegreeOfParallelism = 2 };
var part = Partitioner.Create(1, 100);
//foreach (ListViewItem name in lstBackupUsers.Items)
var checkforfinished = Parallel.ForEach(lstBackupUsers.Items.Cast<ListViewItem>(), name =>
{
try
{
string names = name.SubItems[0].Text;
lstBackupUsers.Items[arraycount].Selected = true;
lstBackupUsers.Items[arraycount].BackColor = Color.CornflowerBlue;
arraycount++;
stripLabel.Text = "";
Console.WriteLine("Selecting user: " + names.ToString());
txtLog.Text += "Selecting user: " + names.ToString() + Environment.NewLine;
txtCurrentUser.Text = names.ToString();
// Define parameters of request.
string user = names.ToString();
// Check if directory exists, create if not.
string savelocation = ConfigurationManager.AppSettings["savelocation"] + user + "\\";
if (File.Exists(savelocation + ".deltalog.tok"))
File.Delete(savelocation + ".deltalog.tok");
FileInfo testdir = new FileInfo(savelocation);
testdir.Directory.Create();
string savedStartPageToken = "";
var start = CreateService.BuildService(user).Changes.GetStartPageToken().Execute();
// This token is set by Google, it defines changes made and
// increments the token value automatically.
// The following reads the current token file (if it exists)
if (File.Exists(savelocation + ".currenttoken.tok"))
{
StreamReader curtokenfile = new StreamReader(savelocation + ".currenttoken.tok");
savedStartPageToken = curtokenfile.ReadLine().ToString();
curtokenfile.Dispose();
}
else
{
// Token record didn't exist. Create a generic file, start at "1st" token
// In reality, I have no idea what token to start at, but 1 seems to be safe.
Console.Write("Creating new token file.\n");
//txtLog.Text += ("Creating new token file.\n" + Environment.NewLine);
StreamWriter sw = new StreamWriter(savelocation + ".currenttoken.tok");
sw.Write(1);
sw.Dispose();
savedStartPageToken = "1";
}
string pageToken = savedStartPageToken;
int gtoken = int.Parse(start.StartPageTokenValue);
int mytoken = int.Parse(savedStartPageToken);
txtPrevToken.Text = pageToken.ToString();
txtCurrentToken.Text = gtoken.ToString();
if (gtoken <= 10)
{
Console.WriteLine("Nothing to save!\n");
//txtLog.Text += ("User has nothing to save!" + Environment.NewLine);
}
else
{
if (pageToken == start.StartPageTokenValue)
{
Console.WriteLine("No file changes found for " + user + "\n");
//txtLog.Text += ("No file changes found! Please wait while I tidy up." + Environment.NewLine);
}
else
{
// .deltalog.tok is where we will place our records for changed files
Console.WriteLine("Changes detected. Making notes while we go through these.");
lblProgresslbl.Text = "Scanning Drive directory.";
// Damnit Google, why did you change how the change fields work?
if (savedStartPageToken == "1")
{
statusStripLabel1.Text = "Recording folder list ...";
txtLog.Text = "Recording folder list ..." + Environment.NewLine;
exfunctions.RecordFolderList(savedStartPageToken, pageToken, user, savelocation);
statusStripLabel1.Text = "Recording new/changed files ... This may take a bit!";
txtLog.Text += Environment.NewLine + "Recording new/changed list for: " + user;
exfunctions.ChangesFileList(savedStartPageToken, pageToken, user, savelocation);
}
else
{
//proUserclass = proUser;
statusStripLabel1.Text = "Recording new/changed files ... This may take a bit!";
txtLog.Text += Environment.NewLine + "Recording new/changed list for: " + user + Environment.NewLine;
exfunctions.ChangesFileList(savedStartPageToken, pageToken, user, savelocation);
}
// Get all our files for the user. Max page size is 1k
// after that, we have to use Google's next page token
// to let us get more files.
StreamWriter logFile = new StreamWriter(savelocation + ".recent.log");
string[] deltafiles = File.ReadAllLines(savelocation + ".deltalog.tok");
int totalfiles = deltafiles.Count();
int cnttototal = 0;
Console.WriteLine("\nFiles to backup:\n");
if (deltafiles == null)
{
return;
}
else
{
double damn = ((gtoken - double.Parse(txtPrevToken.Text)));
damn = Math.Round((damn / totalfiles));
if (damn <= 0)
damn = 1;
foreach (var file in deltafiles)
{
try
{
if (bgW.CancellationPending)
{
stripLabel.Text = "Backup canceled!";
e.Cancel = true;
break;
}
DateTime dt = DateTime.Now;
string[] foldervalues = File.ReadAllLines(savelocation + "folderlog.txt");
cnttototal++;
bgW.ReportProgress(cnttototal);
proUser.Maximum = int.Parse(txtCurrentToken.Text);
stripLabel.Text = "File " + cnttototal + " of " + totalfiles;
double? mathisfun;
mathisfun = ((100 * cnttototal) / totalfiles);
if (mathisfun <= 0)
mathisfun = 1;
double mathToken = double.Parse(txtPrevToken.Text);
mathToken = Math.Round((damn + mathToken));
// Bring our token up to date for next run
txtPrevToken.Text = mathToken.ToString();
File.WriteAllText(savelocation + ".currenttoken.tok", mathToken.ToString());
int proval = int.Parse(txtPrevToken.Text);
int nowval = int.Parse(txtCurrentToken.Text);
if (proval >= nowval)
proval = nowval;
proUser.Value = (proval);
lblProgresslbl.Text = ("Current progress: " + mathisfun.ToString() + "% completed.");
// Our file is a CSV. Column 1 = file ID, Column 2 = File name
var values = file.Split(',');
string fileId = values[0];
string fileName = values[1];
string mimetype = values[2];
mimetype = mimetype.Replace(",", "_");
string folder = values[3];
string ext = null;
int folderfilelen = foldervalues.Count();
fileName = GetSafeFilename(fileName);
Console.WriteLine("Filename: " + values[1]);
logFile.WriteLine("ID: " + values[0] + " - Filename: " + values[1]);
logFile.Flush();
// Things get sloppy here. The reason we're checking MimeTypes
// is because we have to export the files from Google's format
// to a format that is readable by a desktop computer program
// So for example, the google-apps.spreadsheet will become an MS Excel file.
switch (mimetype)
{
(switch statement here removed due to body length issues for this post.)
}
if (ext.Contains(".doc") || ext.Contains(".xls"))
{
string whatami = null;
if (ext.Contains(".xls"))
{
whatami = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
}
else if (ext.Contains(".doc"))
{
whatami = "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
}
else if (ext.Contains(".ppt"))
{
whatami = "application/vnd.openxmlformats-officedocument.presentationml.presentation";
}
if (fileName.Contains(".mov") || ext == ".ggl" || fileName.Contains(".mp4"))
{
txtLog.Text += Environment.NewLine + "Skipping file.";
return;
}
var requestfileid = CreateService.BuildService(user).Files.Export(fileId, whatami);
statusStripLabel1.Text = (savelocation + fileName + ext);
txtCurrentUser.Text = user;
string dest1 = Path.Combine(savelocation, fileName + ext);
var stream1 = new System.IO.FileStream(dest1, FileMode.OpenOrCreate, FileAccess.ReadWrite);
scrolltobtm();
requestfileid.MediaDownloader.ProgressChanged +=
(IDownloadProgress progress) =>
{
switch (progress.Status)
{
case DownloadStatus.Downloading:
{
Console.WriteLine(progress.BytesDownloaded);
logFile.WriteLine("Downloading: " + progress.BytesDownloaded);
txtLog.Text += ("Downloading ... " + progress.BytesDownloaded + Environment.NewLine);
scrolltobtm();
logFile.Flush();
break;
}
case DownloadStatus.Completed:
{
Console.WriteLine("Download complete.");
logFile.WriteLine("[" + user + "] Download complete for: " + requestfileid.ToString());
txtLog.Text += ("[" + user + "] Download complete for: " + fileName + Environment.NewLine);
logFile.Flush();
break;
}
case DownloadStatus.Failed:
{
Console.WriteLine("Download failed.");
logFile.WriteLine("Download failed.");
logFile.Flush();
break;
}
}
};
scrolltobtm();
GC.Collect();
GC.WaitForPendingFinalizers();
requestfileid.Download(stream1);
stream1.Close();
stream1.Dispose();
}
else
{
scrolltobtm();
var requestfileid = CreateService.BuildService(user).Files.Get(fileId);
//Generate the name of the file, and create it as such on the local filesystem.
statusStripLabel1.Text = (savelocation + fileName + ext);
string dest1 = Path.Combine(savelocation, fileName + ext);
var stream1 = new System.IO.FileStream(dest1, FileMode.OpenOrCreate, FileAccess.ReadWrite);
requestfileid.MediaDownloader.ProgressChanged +=
(IDownloadProgress progress) =>
{
switch (progress.Status)
{
case DownloadStatus.Downloading:
{
Console.WriteLine(progress.BytesDownloaded);
logFile.WriteLine("Downloading: " + progress.BytesDownloaded);
txtLog.Text += ("Downloading ... " + progress.BytesDownloaded + Environment.NewLine);
scrolltobtm();
logFile.Flush();
break;
}
case DownloadStatus.Completed:
{
Console.WriteLine("Download complete.");
logFile.WriteLine("Download complete for: " + requestfileid.ToString());
txtLog.Text += (Environment.NewLine + "[" + user + "] Download complete for: " + fileName + Environment.NewLine);
logFile.Flush();
break;
}
case DownloadStatus.Failed:
{
Console.WriteLine("Download failed.");
logFile.WriteLine("Download failed.");
logFile.Flush();
break;
}
}
};
scrolltobtm();
GC.Collect();
GC.WaitForPendingFinalizers();
requestfileid.Download(stream1);
stream1.Close();
stream1.Dispose();
}
}
catch (Google.GoogleApiException ex)
{
Console.Write("\nInfo: ---> " + ex.Message.ToString() + "\n");
}
}
}
exfunctions.MoveFiles(savelocation);
Console.WriteLine("\n\n\tBackup completed for selected user!");
txtLog.Text += ("\n\nBackup completed for selected user.\n\n");
statusStripLabel1.Text = "";
//logFile.Close();
//logFile.Dispose();
}
}
}
catch (Google.GoogleApiException ex)
{
Console.WriteLine("Info: " + ex.Message.ToString());
}
}
);
if (checkforfinished.IsCompleted == true)
{
MessageBox.Show("Parallel.ForEach() Finished!");
Console.WriteLine("Parallel.ForEach() Finished!");
}
else
{
MessageBox.Show("Parallel.ForEach() not completed!");
Console.WriteLine("Parallel.ForEach() not completed!");
}
}
}
}
}
catch (Google.GoogleApiException ex)
{
Console.WriteLine("Info: " + ex.Message.ToString());
}
}
}