C# 如何在运行时动态更改batchblock的批大小?
我在tpl数据流中有一个批处理块,并且有几个目标块链接到批处理块。但是,目标块的数量会动态变化,因此批次的大小也会随之变化。问题是批大小必须在batchblock初始化时提供,我看不到以后调整它的方法。有什么办法可以解决这个问题吗?是否只有这样才能取消链接(处理到batchblock和来自batchblock的所有链接),使用新的批大小重新初始化批块,然后再次链接?我可以这样做,但如何确保旧批次和新批次不会完全混淆 例如,如果我有两个转换块流到批处理块,现在有一个额外的转换块,并且希望将批大小增加到3,那么如何确保在增加之前处理所有以前的批以确保同步行为?关键是,所有变换块都得到完全相同的项,并且这些变换块的输出应该以仅对匹配相同输入的输出进行批处理的方式进行批处理 下面是我希望的示例: 转换块的常量整数流: 1,2,3,[批量增加的点]、4,5 让transform块输出它们得到的结果,如1=>1 因此,batchblock的输出应如下所示: [1,1],[2,2],[3,3],[批量变化],[4,4,4],[5,5,5] 以下是我当前的代码:C# 如何在运行时动态更改batchblock的批大小?,c#,concurrency,task-parallel-library,tpl-dataflow,batching,C#,Concurrency,Task Parallel Library,Tpl Dataflow,Batching,我在tpl数据流中有一个批处理块,并且有几个目标块链接到批处理块。但是,目标块的数量会动态变化,因此批次的大小也会随之变化。问题是批大小必须在batchblock初始化时提供,我看不到以后调整它的方法。有什么办法可以解决这个问题吗?是否只有这样才能取消链接(处理到batchblock和来自batchblock的所有链接),使用新的批大小重新初始化批块,然后再次链接?我可以这样做,但如何确保旧批次和新批次不会完全混淆 例如,如果我有两个转换块流到批处理块,现在有一个额外的转换块,并且希望将批大小增
public class Test
{
private Stopwatch watch;
private BroadcastBlock<List<InputObject>> tempBCB;
private BatchBlock<Tuple<List<InputObject>, Dictionary<int, IntermediateObject>>> batchBlock;
private TransformBlock<Tuple<List<InputObject>, Dictionary<int, IntermediateObject>>[], List<FinalObject>> transformBlock;
private ActionBlock<List<FinalObject>> justToFlushTransformBlock;
private CoreLogic core1;
private CoreLogic core2;
public Test()
{
tempBCB = new BroadcastBlock<List<InputObject>>(input => input);
//here batch size = 2
batchBlock = new BatchBlock<Tuple<List<InputObject>,Dictionary<int,IntermediateObject>>>(2, new GroupingDataflowBlockOptions { Greedy = false });
transformBlock = new TransformBlock<Tuple<List<InputObject>,Dictionary<int,IntermediateObject>>[],List<FinalObject>>(array =>
{
List<InputObject> inputObjects = array[0].Item1;
List<FinalObject> ret = inputObjects.ConvertAll(x => new FinalObject(x));
foreach (var tuple in array)
{
//iterate over each individual object
foreach (var dictionary in tuple.Item2)
{
ret[dictionary.Key].outputList.Add(dictionary.Value);
}
}
return ret;
}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
justToFlushTransformBlock = new ActionBlock<List<FinalObject>>(list =>
{
//just in order to accept items from the transformBlock output queue
});
//Generate 2 CoreLogic objects
core1 = new CoreLogic();
core2 = new CoreLogic();
//linking
tempBCB.LinkTo(core1.transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
tempBCB.LinkTo(core2.transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
core1.transformBlock.LinkTo(batchBlock);
core2.transformBlock.LinkTo(batchBlock);
batchBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock.LinkTo(justToFlushTransformBlock, new DataflowLinkOptions { PropagateCompletion = true });
}
public void Start()
{
const int numberChunks = 30;
watch = new Stopwatch();
watch.Start();
for (int j = 1; j <= numberChunks; j++)
{
int collectionSize = 10000 * j;
List<InputObject> collection = new List<InputObject>(collectionSize);
for (int i = 0; i < collectionSize; i++)
{
collection.Add(new InputObject(i));
}
tempBCB.Post(collection);
}
tempBCB.Complete();
Task.WhenAll(core1.transformBlock.Completion, core2.transformBlock.Completion).ContinueWith(_ =>
{
batchBlock.Complete();
});
transformBlock.Completion.Wait();
watch.Stop();
Console.WriteLine("Elapsed time (in milliseconds): " + watch.ElapsedMilliseconds);
Console.ReadLine();
}
}
public class CoreLogic
{
private Random rand;
public TransformBlock<List<InputObject>, Tuple<List<InputObject>, Dictionary<int, IntermediateObject>>> transformBlock;
public CoreLogic()
{
const int numberIntermediateObjects = 10000;
transformBlock = new TransformBlock<List<InputObject>, Tuple<List<InputObject>, Dictionary<int, IntermediateObject>>>(input =>
{
//please ignore the fact that `input` is not utilized here, the point is to generate a collection of IntermediateObject and return
Dictionary<int, IntermediateObject> ret = new Dictionary<int, IntermediateObject>();
for (int i = 0; i < numberIntermediateObjects; i++)
{
IntermediateObject value = new IntermediateObject(i);
ret.Add(i, value);
}
var tuple = new Tuple<List<InputObject>, Dictionary<int, IntermediateObject>>(input, ret);
return tuple;
});
}
}
public class InputObject : ICloneable
{
public int value1 { get; private set; }
public InputObject(int value)
{
this.value1 = value;
}
object ICloneable.Clone()
{
return Clone();
}
public InputObject Clone()
{
return (InputObject)this.MemberwiseClone();
}
}
public class IntermediateObject
{
public int value1 { get; private set; }
public IntermediateObject(int value)
{
this.value1 = value;
}
}
public class FinalObject
{
public InputObject input { get; private set; }
public List<IntermediateObject> outputList;
public FinalObject(InputObject input)
{
this.input = input;
this.outputList = new List<IntermediateObject>();
}
}
public static class Cloning
{
public static List<TValue> CloneListCloneValues<TValue>(List<TValue> original) where TValue : ICloneable
{
List<TValue> ret = new List<TValue>(original.Count);
foreach (TValue entry in original)
{
ret.Add((TValue)entry.Clone());
}
return ret;
}
}
公共类测试
{
私人秒表;
专用广播块tempbcbcb;
专用批块批块;
私有转换块转换块;
私有操作块justtoflusionblock;
私有corelogiccore1;
私有corelogiccore2;
公开考试()
{
tempBCB=新广播块(输入=>输入);
//此处批量大小=2
batchBlock=new batchBlock(2,new GroupingDataflowBlockOptions{贪婪=false});
transformBlock=新transformBlock(数组=>
{
列表inputObjects=array[0]。项1;
List ret=inputObject.ConvertAll(x=>newfinalobject(x));
foreach(数组中的变量元组)
{
//迭代每个单独的对象
foreach(tuple.Item2中的var字典)
{
ret[dictionary.Key].outputList.Add(dictionary.Value);
}
}
返回ret;
},新的ExecutionDataflowBlockOptions{MaxDegreeOfParallelism=DataflowBlockOptions.Unbounded});
justToFlushTransformBlock=新操作块(列表=>
{
//只是为了接受transformBlock输出队列中的项目
});
//生成2个CoreLogic对象
core1=新的CoreLogic();
core2=新的CoreLogic();
//连接
LinkTo(core1.transformBlock,新数据流链接选项{PropagateCompletion=true});
LinkTo(core2.transformBlock,新数据流链接选项{PropagateCompletion=true});
core1.transformBlock.LinkTo(批块);
core2.transformBlock.LinkTo(批块);
LinkTo(transformBlock,新的DataflowLinkOptions{PropagateCompletion=true});
LinkTo(justToFlushTransformBlock,新数据流链接选项{PropagateCompletion=true});
}
公开作废开始()
{
常数int numberChunks=30;
手表=新秒表();
watch.Start();
对于(int j=1;j
{
batchBlock.Complete();
});
transformBlock.Completion.Wait();
看,停;
Console.WriteLine(“运行时间(毫秒):”+watch.elapsedmillesons);
Console.ReadLine();
}
}
公共类核心逻辑
{
私有随机兰德;
公共转换块转换块;
公共核心逻辑()
{
常数int numberntermediateobjects=10000;
transformBlock=新transformBlock(输入=>
{
//请忽略以下事实:此处未使用'input',重点是生成一个IntermediateObject和return的集合
字典ret=新字典();
对于(int i=0;i
我也想知道这个问题的答案,但与此同时,难道不可能有两个不同大小的批块放在一起,然后在它们之间切换吗?我要试试看。或者另一种方法是完成现有的批处理,并在继续中使用新的大小重新实例化新批处理,并将其分配给同一个变量。@Dimitri,恐怕解决方案会更复杂一些。你不能只吃几批