C# e(256.ToList()); dates.RemoveAll(x=>x.Count()==0); prices.RemoveAll(x=>x.Count()==0); 看,停; WriteLine(“重新设置数据的范围:{0}s”,watch.eassed.TotalSeconds); watch.Restart(); var exceldatadefined=新字典(); foreach(日期中的IEnumerable日期列表) { 十进制数; IEnumerable datelist1=日期列表; IEnumerable价目表= 价格[dates.IndexOf(datelist1)]。选择(值=>值0米)。其中( content=>decimal.TryParse(content.ToString(),out num)); 字典格言= Zip(pricelist,(k,v)=>new{k,v}).ToDictionary( x=>(字符串)x.k,x=>decimal.Parse(x.v.ToString()); 如果(!ExcelDataDefined.ContainsKey(产品[dates.IndexOf(datelist1)]) { 添加(产品[dates.IndexOf(datelist1)],dict); } } 看,停; WriteLine(“将数据压缩到:{0}s”,watch.eassed.TotalSeconds); 返回精练的数据; } 每个人都需要一个应用程序。3,5秒重新排列,3秒压缩数据 Benchmark_AsParallel需要应用程序。12秒重新排列,0005秒压缩数据 Benchmark_ForEach需要应用程序。16秒重新排列,0.05秒压缩数据。
为什么会这样?我认为Asparell是最快的,因为它是并行执行的,而不是顺序执行的。我该怎么优化呢?在Benchmark\u AsParallel和Benchmark\u ForEach中,您在Benchmark\u ForEach n中执行2n操作。为了实现并行计算,您必须有多个处理器或内核,否则您只是在线程池中排队等待CPU。也就是说,单核机器上的aspallel是顺序的,加上线程池和线程上下文切换的开销。即使在双核机器上,您也可能无法同时获得两个核,因为许多其他东西都在同一台机器上运行C# e(256.ToList()); dates.RemoveAll(x=>x.Count()==0); prices.RemoveAll(x=>x.Count()==0); 看,停; WriteLine(“重新设置数据的范围:{0}s”,watch.eassed.TotalSeconds); watch.Restart(); var exceldatadefined=新字典(); foreach(日期中的IEnumerable日期列表) { 十进制数; IEnumerable datelist1=日期列表; IEnumerable价目表= 价格[dates.IndexOf(datelist1)]。选择(值=>值0米)。其中( content=>decimal.TryParse(content.ToString(),out num)); 字典格言= Zip(pricelist,(k,v)=>new{k,v}).ToDictionary( x=>(字符串)x.k,x=>decimal.Parse(x.v.ToString()); 如果(!ExcelDataDefined.ContainsKey(产品[dates.IndexOf(datelist1)]) { 添加(产品[dates.IndexOf(datelist1)],dict); } } 看,停; WriteLine(“将数据压缩到:{0}s”,watch.eassed.TotalSeconds); 返回精练的数据; } 每个人都需要一个应用程序。3,5秒重新排列,3秒压缩数据 Benchmark_AsParallel需要应用程序。12秒重新排列,0005秒压缩数据 Benchmark_ForEach需要应用程序。16秒重新排列,0.05秒压缩数据。,c#,linq,performance,loops,parallel-processing,C#,Linq,Performance,Loops,Parallel Processing,为什么会这样?我认为Asparell是最快的,因为它是并行执行的,而不是顺序执行的。我该怎么优化呢?在Benchmark\u AsParallel和Benchmark\u ForEach中,您在Benchmark\u ForEach n中执行2n操作。为了实现并行计算,您必须有多个处理器或内核,否则您只是在线程池中排队等待CPU。也就是说,单核机器上的aspallel是顺序的,加上线程池和线程上下文切换的开销。即使在双核机器上,您也可能无法同时获得两个核,因为许多其他东西都在同一台机器上运行 真
真正的
.aspallel()
只有在具有阻塞操作(I/O)的长时间运行任务的情况下才有用,操作系统可以挂起阻塞线程并让另一个线程运行。创建附加线程和管理每个线程的工作负载会带来开销。如果您的工作量有限,那么创建额外线程的开销、线程之间的任务切换、线程之间的工作窃取和重新分配等可能会超过您首先通过并行工作获得的收益。您可能需要分析应用程序,以确定在使用单个进程运行时是否真的受到CPU的限制。如果不是,最好是保持单线程,这样瓶颈就变成了IO,而IO不容易并行化
另外两条建议:使用AsOrdered和TakeWhile会导致性能下降,因为它们都需要同步回原始线程。考虑分析而不需要排序,看看它是否提供了性能改进。
也可以考虑使用并发字典而不是标准泛型字典来避免添加项目时的并发问题。
我测量了FrEACH中的每个重排操作和AsSimulink方法之间的时间。第一个操作在第二个操作花费0.05s1的所有时间内都会消耗。)如果……(字段)%2==1,则不需要测试,并且可以用“else”替换,因为%2将创建零或一。2.)您使用的“IEnumerable”有多大ExcelDataRaw.Count()返回65535个元素(excel文件中的最大行数)。excelDataRaw.SelectMany(x=>x.Values).Count()返回16645890个元素,其中包含大量我无法忽略的空值。它基本上是一个包含254列和65535行的网格,其中每个网格点都将其列头作为keyYes连接到它,我在公司的pc上只有两个内核,我想我的输入数据太小了,所以AsParallel会增加很多开销。如果你是CPU限制的,而不是IO限制的(假设你有多个内核),AsParallel实际上是最好的。如果您是IO绑定的,最好使用异步策略。您应该使用异步来不阻塞工作线程,即为每个工作线程创建任务链,但您仍然希望使用AsParell扇出工作线程,这样他们就可以从池中获取线程来启动链 product1 | unnamedcol2 | product2 | unnamedcol4 | product3 | unnamedcol6 | ------------------------------------------------------------------------------- @1foo | 1.10 | @1foo | 0.3 | @1foo | 0.3 @2foo | 1.00 | @2foo | 2 | @2foo | @3foo | 1.52 | @3foo | 2.53 | @3foo | @4foo | 1.47 | | | @4foo | 1.31 @5foo | 1.49 | | | @5foo | 1.31IEnumerable<IDictionary<string, object>> excelDataRaw =
conn.Query(string.Format("select * from {0}", table)).Cast<IDictionary<string, object>>();
var excelDataRefined = new List<IDictionary<string, IDictionary<string, decimal>>>();
excelDataRefined.Add(new Dictionary<string, IDictionary<string, decimal>>());
excelDataRefined[0].Add( "product", new Dictionary<string, decimal>());
excelDataRefined[0]["product"].Add("@1foo", 1.1m);
private static Dictionary<string, IDictionary<string, decimal>> Benchmark_foreach(IEnumerable<IDictionary<string, object>> excelDataRaw)
{
Console.WriteLine("1. Using foreach");
var watch = new Stopwatch();
watch.Start();
List<string> headers = excelDataRaw.Select(dictionary => dictionary.Keys).First().ToList();
bool isEven = false;
List<string> products = headers.Where(h => isEven = !isEven).ToList();
var dates = new List<IEnumerable<object>>();
var prices = new List<IEnumerable<object>>();
foreach (string field in headers)
{
string product1 = field;
if (headers.IndexOf(field) % 2 == 0)
{
dates.Add(
excelDataRaw.AsParallel().AsOrdered().Select(col => col[product1]).Where(row => row != null));
}
if (headers.IndexOf(field) % 2 == 1)
{
prices.Add(
excelDataRaw.AsParallel().AsOrdered().Select(col => col[product1] ?? 0m).Take(dates.Last().Count()));
}
}
watch.Stop();
Console.WriteLine("Rearange the data in: {0}s", watch.Elapsed.TotalSeconds);
watch.Restart();
var excelDataRefined = new Dictionary<string, IDictionary<string, decimal>>();
foreach (IEnumerable<object> datelist in dates)
{
decimal num;
IEnumerable<object> datelist1 = datelist;
IEnumerable<object> pricelist =
prices[dates.IndexOf(datelist1)].Select(value => value ?? 0m).Where(
content => decimal.TryParse(content.ToString(), out num));
Dictionary<string, decimal> dict =
datelist1.Zip(pricelist, (k, v) => new { k, v }).ToDictionary(
x => (string)x.k, x => decimal.Parse(x.v.ToString()));
if (!excelDataRefined.ContainsKey(products[dates.IndexOf(datelist1)]))
{
excelDataRefined.Add(products[dates.IndexOf(datelist1)], dict);
}
}
watch.Stop();
Console.WriteLine("Zipped the data in: {0}s", watch.Elapsed.TotalSeconds);
return excelDataRefined;
}
private static Dictionary<string, IDictionary<string, decimal>> Benchmark_AsParallel(IEnumerable<IDictionary<string, object>> excelDataRaw)
{
Console.WriteLine("2. Using AsParallel().AsOrdered().ForAll");
var watch = new Stopwatch();
watch.Start();
List<string> headers = excelDataRaw.Select(dictionary => dictionary.Keys).First().ToList();
bool isEven = false;
List<string> products = headers.Where(h => isEven = !isEven).ToList();
var dates = new List<IEnumerable<object>>();
var prices = new List<IEnumerable<object>>();
headers.AsParallel().AsOrdered().ForAll(
field =>
dates.Add(
excelDataRaw.AsParallel().AsOrdered().TakeWhile(x => headers.IndexOf(field) % 2 == 0).Select(
col => col[field]).Where(row => row != null).ToList()));
headers.AsParallel().AsOrdered().ForAll(
field =>
prices.Add(
excelDataRaw.AsParallel().AsOrdered().TakeWhile(x => headers.IndexOf(field) % 2 == 1).Select(
col => col[field] ?? 0m).Take(256).ToList()));
dates.RemoveAll(x => x.Count() == 0);
prices.RemoveAll(x => x.Count() == 0);
watch.Stop();
Console.WriteLine("Rearange the data in: {0}s", watch.Elapsed.TotalSeconds);
watch.Restart();
var excelDataRefined = new Dictionary<string, IDictionary<string, decimal>>();
foreach (IEnumerable<object> datelist in dates)
{
decimal num;
IEnumerable<object> datelist1 = datelist;
IEnumerable<object> pricelist =
prices[dates.IndexOf(datelist1)].Select(value => value ?? 0m).Where(
content => decimal.TryParse(content.ToString(), out num));
Dictionary<string, decimal> dict =
datelist1.Zip(pricelist, (k, v) => new { k, v }).ToDictionary(
x => (string)x.k, x => decimal.Parse(x.v.ToString()));
if (!excelDataRefined.ContainsKey(products[dates.IndexOf(datelist1)]))
{
excelDataRefined.Add(products[dates.IndexOf(datelist1)], dict);
}
}
watch.Stop();
Console.WriteLine("Zipped the data in: {0}s", watch.Elapsed.TotalSeconds);
return excelDataRefined;
}
private static Dictionary<string, IDictionary<string, decimal>> Benchmark_ForEach(IEnumerable<IDictionary<string, object>> excelDataRaw)
{
Console.WriteLine("3. Using ForEach");
var watch = new Stopwatch();
watch.Start();
List<string> headers = excelDataRaw.Select(dictionary => dictionary.Keys).First().ToList();
bool isEven = false;
List<string> products = headers.Where(h => isEven = !isEven).ToList();
var dates = new List<IEnumerable<object>>();
var prices = new List<IEnumerable<object>>();
headers.ForEach(
field =>
dates.Add(
excelDataRaw.TakeWhile(x => headers.IndexOf(field) % 2 == 0).Select(col => col[field]).Where(
row => row != null).ToList()));
headers.ForEach(
field =>
prices.Add(
excelDataRaw.TakeWhile(x => headers.IndexOf(field) % 2 == 1).Select(col => col[field] ?? 0m).
Take(256).ToList()));
dates.RemoveAll(x => x.Count() == 0);
prices.RemoveAll(x => x.Count() == 0);
watch.Stop();
Console.WriteLine("Rearange the data in: {0}s", watch.Elapsed.TotalSeconds);
watch.Restart();
var excelDataRefined = new Dictionary<string, IDictionary<string, decimal>>();
foreach (IEnumerable<object> datelist in dates)
{
decimal num;
IEnumerable<object> datelist1 = datelist;
IEnumerable<object> pricelist =
prices[dates.IndexOf(datelist1)].Select(value => value ?? 0m).Where(
content => decimal.TryParse(content.ToString(), out num));
Dictionary<string, decimal> dict =
datelist1.Zip(pricelist, (k, v) => new { k, v }).ToDictionary(
x => (string)x.k, x => decimal.Parse(x.v.ToString()));
if (!excelDataRefined.ContainsKey(products[dates.IndexOf(datelist1)]))
{
excelDataRefined.Add(products[dates.IndexOf(datelist1)], dict);
}
}
watch.Stop();
Console.WriteLine("Zipped the data in: {0}s", watch.Elapsed.TotalSeconds);
return excelDataRefined;
}