C# .net Rx：按顺序批处理消息
我试图使用Rx实现一个异步工作流，但我似乎做得完全错误 我想做的是：C# .net Rx：按顺序批处理消息,c#,.net,wpf,system.reactive,C#,.net,Wpf,System.reactive,我试图使用Rx实现一个异步工作流，但我似乎做得完全错误 我想做的是： From an undefined asynchronous stream of un-parsed message strings (i.e. an IObservable<string>) parse the message strings asynchronously, but preserve their order. (IObservable<Message>) Batch up parsed
From an undefined asynchronous stream of un-parsed message strings (i.e. an IObservable<string>)
parse the message strings asynchronously, but preserve their order. (IObservable<Message>)
Batch up parsed Messages in groups of 100 or so (IObservable<IEnumerable<Message>>)
Send each batch, when complete, to the UI thread to be processed. Batches must arrive in the same order they were started.
private IObservable<IEnumerable<Message>> messageSource;
public IObservable<IEnumerable<Message>> MessageSource { get { return messageSource; } }
/// <summary>
/// Sub-classes of MessageProviderBase provide this IEnumerable to
/// generate unparsed message strings synchronously
/// </summary>
protected abstract IEnumerable<string> UnparsedMessages { get; }
public MessageProviderBase()
{
// individual parsed messages as a PLINQ query
var parsedMessages = from unparsedMessage in UnparsedMessages.AsParallel().AsOrdered()
select ParseMessage(unparsedMessage);
// convert the above PLINQ query to an observable, buffering up to 100 messages at a time
var batchedMessages
= parsedMessages.ToObservable().BufferWithTimeOrCount(TimeSpan.FromMilliseconds(200), 100);
// ISSUE #1:
// batchedMessages seems to call OnNext before all of the messages in its buffer are parsed.
// If you convert the IObservable<Message> it generates to an enumerable, it blocks
// when you try to enumerate it.
// Convert each batch to an IEnumerable
// ISSUE #2: Even if the following Rx query were to run asynchronously (it doesn't now, see the above comment),
// it could still deliver messages out of order. Only, instead of delivering individual
// messages out of order, the message batches themselves could arrive out of order.
messageSource = from messageBatch in batchedMessages
select messageBatch.ToEnumerable().ToList();
}
IObservable<string> UnparsedMessages = ...;
Func<string, Message> ParseMessage = ...;
public static IObservable<U> SelectAsync<T, U>(this IObservable<T> source,
Func<T, U> selector)
{
var subject = new Subject<U>();
var queue = new Queue<System.Threading.Tasks.Task<U>>();
var completing = false;
var subscription = (IDisposable)null;
Action<Exception> onError = ex =>
{
queue.Clear();
subject.OnError(ex);
subscription.Dispose();
};
Action dequeue = () =>
{
lock (queue)
{
var error = false;
while (queue.Count > 0 && queue.Peek().IsCompleted)
{
var task = queue.Dequeue();
if (task.Exception != null)
{
error = true;
onError(task.Exception);
break;
}
else
{
subject.OnNext(task.Result);
}
}
if (!error && completing && queue.Count == 0)
{
subject.OnCompleted();
subscription.Dispose();
}
}
};
Action<T> enqueue = t =>
{
if (!completing)
{
var task = new System.Threading.Tasks.Task<U>(() => selector(t));
queue.Enqueue(task);
task.ContinueWith(tu => dequeue());
task.Start();
}
};
subscription = source.Subscribe(
t => { lock(queue) enqueue(t); },
x => { lock(queue) onError(x); },
() => { lock(queue) completing = true; });
return subject.AsObservable();
}
var observableOfU = observableOfT.ForkSelect(funcOfT2U);
享受吧 我下面的答案在某种程度上基于Enigmativity的代码，但修复了一些与完成相关的争用条件，还添加了对取消和自定义调度程序的支持，这将使单元测试变得更加容易
public static IObservable<U> Fork<T, U>(this IObservable<T> source,
Func<T, U> selector)
{
return source.Fork<T, U>(selector, Scheduler.TaskPool);
}
public static IObservable<U> Fork<T, U>(this IObservable<T> source,
Func<T, U> selector, IScheduler scheduler)
{
return Observable.CreateWithDisposable<U>(observer =>
{
var runningTasks = new CompositeDisposable();
var lockGate = new object();
var queue = new Queue<ForkTask<U>>();
var completing = false;
var subscription = new MutableDisposable();
Action<Exception> onError = ex =>
{
lock(lockGate)
{
queue.Clear();
observer.OnError(ex);
}
};
Action dequeue = () =>
{
lock (lockGate)
{
var error = false;
while (queue.Count > 0 && queue.Peek().Completed)
{
var task = queue.Dequeue();
observer.OnNext(task.Value);
}
if (completing && queue.Count == 0)
{
observer.OnCompleted();
}
}
};
Action onCompleted = () =>
{
lock (lockGate)
{
completing = true;
dequeue();
}
};
Action<T> enqueue = t =>
{
var cancellation = new MutableDisposable();
var task = new ForkTask<U>();
lock(lockGate)
{
runningTasks.Add(cancellation);
queue.Enqueue(task);
}
cancellation.Disposable = scheduler.Schedule(() =>
{
try
{
task.Value = selector(t);
lock(lockGate)
{
task.Completed = true;
runningTasks.Remove(cancellation);
dequeue();
}
}
catch(Exception ex)
{
onError(ex);
}
});
};
return new CompositeDisposable(runningTasks,
source.AsObservable().Subscribe(
t => { enqueue(t); },
x => { onError(x); },
() => { onCompleted(); }
));
});
}
private class ForkTask<T>
{
public T Value = default(T);
public bool Completed = false;
}
看起来不错！我建议删除主题并将代码包装在Observable.CreateWithDisposable中，以避免出现争用情况。然后，您还可以返回订阅，它将自动在completed/OnError上处理，因此您也可以删除该代码。您也可以放弃所有的方法顺序保护，例如onCompleted after onError，转而订阅source.AsObservable，因为这将强制您执行顺序。此外，当将订阅分配给局部变量，然后使用来自观察者的该变量时，我建议使用一个可变的一次性配置来防止可能导致NullReferenceException的竞争条件，最后，很抱歉，如果您使用的是.NET 4，则可以通过Disposable支持CancellationTokenSource。使用CompositeDisposable创建源订阅并与之结合，以允许在出现错误时取消其他任务。我不确定我是否理解使用Subject而不是Observable.CreateWithDisposable的竞争条件。你能详细说明一下吗？也许能举个例子？谢谢另外，上述代码中没有使用ConcurrentQueue的具体原因是什么？这将在.Net 4上运行。@Jeremy-任务可能在主题返回之前完成，而CreateWithDisposable将为您处理。至于ConcurrentQueue，您从来没有提到过.NET4，所以我猜Enigmativity不想在他的示例中要求它。不幸的是，这个解决方案不起作用。即使我让它编译Tuple是不可变的，而且看起来里面有一些scala，返回的IObservable也不会提供任何结果。我相信它在自己的线程上已经死锁了。@Jeremy-你说得对，到处都是编译错误-对此很抱歉。我已经修复了它，并将元组依赖项替换为自定义类。然而，我还不能重现死锁，所以我已经包含了一个随机任务长度的示例用法。我还使用Scheduler.Immediate对其进行了测试，以确保在任务立即完成时它能够正常工作。看起来死锁是由于我在工作流早期误用了BufferWithTimeOrCount操作符造成的。
public static IObservable<U> SelectMany<T, U>(this IObservable<T> source, Func<T, Task<U>> selector)
{
return source.ForkSelect<T, U>(selector);
}
public static IObservable<V> SelectMany<T, U, V>(this IObservable<T> source, Func<T, Task<U>> taskSelector, Func<T, U, V> resultSelector)
{
if (source == null) throw new ArgumentNullException("source");
if (taskSelector == null) throw new ArgumentNullException("taskSelector");
if (resultSelector == null) throw new ArgumentNullException("resultSelector");
return source.Zip(source.ForkSelect<T, U>(taskSelector), (t, u) => resultSelector(t, u));
}
var observableOfU = observableOfT.ForkSelect(funcOfT2U);
var observableOfU = observableOfT.ForkSelect(funcOfT2TaskOfU);
var observableOfU =
from t in observableOfT
from u in funcOfT2TaskOfU(t)
select u;
public static IObservable<U> Fork<T, U>(this IObservable<T> source,
Func<T, U> selector)
{
return source.Fork<T, U>(selector, Scheduler.TaskPool);
}
public static IObservable<U> Fork<T, U>(this IObservable<T> source,
Func<T, U> selector, IScheduler scheduler)
{
return Observable.CreateWithDisposable<U>(observer =>
{
var runningTasks = new CompositeDisposable();
var lockGate = new object();
var queue = new Queue<ForkTask<U>>();
var completing = false;
var subscription = new MutableDisposable();
Action<Exception> onError = ex =>
{
lock(lockGate)
{
queue.Clear();
observer.OnError(ex);
}
};
Action dequeue = () =>
{
lock (lockGate)
{
var error = false;
while (queue.Count > 0 && queue.Peek().Completed)
{
var task = queue.Dequeue();
observer.OnNext(task.Value);
}
if (completing && queue.Count == 0)
{
observer.OnCompleted();
}
}
};
Action onCompleted = () =>
{
lock (lockGate)
{
completing = true;
dequeue();
}
};
Action<T> enqueue = t =>
{
var cancellation = new MutableDisposable();
var task = new ForkTask<U>();
lock(lockGate)
{
runningTasks.Add(cancellation);
queue.Enqueue(task);
}
cancellation.Disposable = scheduler.Schedule(() =>
{
try
{
task.Value = selector(t);
lock(lockGate)
{
task.Completed = true;
runningTasks.Remove(cancellation);
dequeue();
}
}
catch(Exception ex)
{
onError(ex);
}
});
};
return new CompositeDisposable(runningTasks,
source.AsObservable().Subscribe(
t => { enqueue(t); },
x => { onError(x); },
() => { onCompleted(); }
));
});
}
private class ForkTask<T>
{
public T Value = default(T);
public bool Completed = false;
}
AutoResetEvent are = new AutoResetEvent(false);
Random rand = new Random();
Observable.Range(0, 5)
.Fork(i =>
{
int delay = rand.Next(50, 500);
Thread.Sleep(delay);
return i + 1;
})
.Subscribe(
i => Console.WriteLine(i),
() => are.Set()
);
are.WaitOne();
Console.ReadLine();