C# 如何使用ReactiveExtensions Range和WithAsync每隔N页批写入数据库

我有以下功能，这是工作得很好。。。然而，考虑到它生成的数据量，我需要每隔10页左右对数据库进行批量写入

var start = DateTime.Now;

IList<AggV2> list = null;

var lastRan = DateTime.UtcNow.AddMonths(-6); // get dummy date 6 months ago
var daysToRun = (DateTime.UtcNow - lastRan).Days;

try
{
    IObservable<IList<AggV2>> query =
        Observable
            .Using(
                () => new HttpClient(),
                hc =>
                    from day in
                        Observable
                            .Range(1, daysToRun)
                            .Select(day =>
                                Observable
                                    .FromAsync(ct => PolygonWebApi.GetGroupedDailyBarsAsync(hc, this.apiKey, Locale.US, Market.Stocks, lastRan.AddDays(day), false, ct))
                                    .Select(r =>
                                    {
                                        this.logger.LogInformation("got {0} records for {1}", r.Results.Count(), lastRan.AddDays(day));
                                        return new TickersResponseWithDay(lastRan.AddDays(day), r);
                                    }))
                            .Merge(MaxConcurrentDownloads)
                    from tv2 in day.AggregateResponse.Results
                    select tv2)
            .ToList();

    list = await query.ToTask(cancellationToken);
}
catch (OperationCanceledException) { }
catch (Exception e) { this.logger.LogError(e, e.Message); }

var duration = DateTime.Now - start;

if (cancellationToken.IsCancellationRequested)
    this.logger.LogInformation("{0} cancelled after {1}, database not updated", this.GetType().Name, duration.Humanize());
else
{
    this.logger.LogInformation("{0} downloaded {1} tickers in {2}, saving to database...", this.GetType().Name, list.Count, duration.Humanize());
    await SaveTickersToDatabaseAsync(list, cts.Token);
}

var start=DateTime.Now；
IList list=null；
var lastRan=DateTime.UtcNow.AddMonths（-6）；//6个月前的假约会
var daysToRun=（DateTime.UtcNow-lastRan）.Days；
尝试
{
可观测查询=
可观察
.使用(
（）=>新的HttpClient（），
hc=>
从天而降
可观察
.范围（1，daysToRun）
.选择（日期=>
可观察
.FromAsync（ct=>PolygonWebApi.GetGroupedDailybarAsync（hc，this.apiKey，Locale.US，Market.Stocks，lastRan.AddDays（day），false，ct））
.选择（r=>
{
this.logger.LogInformation（“获取了{1}的{0}条记录”，r.Results.Count（），lastRan.AddDays（day））；
返回新的TickerResponseWithDay（lastRan.AddDays（day），r）；
}))
.Merge（MaxConcurrentDownloads）
来自tv2 in day.aggregatereresponse.Results
选择tv2）
.ToList（）；
list=wait query.ToTask（cancellationToken）；
}
捕获（操作取消异常）{}
catch（异常e）{this.logger.LogError（e，e.Message）；}
var duration=DateTime.Now-start；
if（cancellationToken.IsCancellationRequested）
this.logger.LogInformation（“{1}之后取消了{0}，数据库未更新”，this.GetType（）.Name，duration.Humanize（））；
其他的
{
this.logger.LogInformation（“{0}下载了{2}中的{1}标记，保存到数据库…”，this.GetType（）.Name，list.Count，duration.Humanize（））；
等待SaveTickersToDatabaseAsync（列表，cts.Token）；
}

我希望每隔10页调用

SaveTickersToDatabaseAsync（list，cancellationToken）

，而不是获取所有数据然后写入

我还需要能够在设置

cancellationToken

的任何时候退出应用程序

---

是否可以将上述批处理和取消要求结合起来？

使用

.Buffer（10）

非常简单。但是，通过将

SaveTickersToDatabaseAsync

推入查询（这是正确的做法），您将使整个错误处理和日志记录在方法末尾变得越来越不相关。我建议尝试删除它，并尝试将其全部放在查询中

下面是代码需要的样子：

IObservable<IList<Unit>> query =
    Observable
        .Using(
            () => new HttpClient(),
            hc =>
                from day in
                    Observable
                        .Range(1, daysToRun)
                        .Select(day =>
                            Observable
                                .FromAsync(ct => PolygonWebApi.GetGroupedDailyBarsAsync(hc, this.apiKey, Locale.US, Market.Stocks, lastRan.AddDays(day), false, ct))
                                .Do(r => this.logger.LogInformation("got {0} records for {1}", r.Results.Count(), lastRan.AddDays(day)))
                                .Select(r => new TickersResponseWithDay(lastRan.AddDays(day), r)))
                        .Merge(MaxConcurrentDownloads)
                from tv2 in day.AggregateResponse.Results
                select tv2)
        .Buffer(10)
        .SelectMany(xs => Observable.FromAsync(ct => SaveTickersToDatabaseAsync(xs, ct)))
        .ToList();

IList<Unit> list = await query.ToTask(cancellationToken);

这是干净和独立的，这是Rx的做事方式

IObservable<Unit> query =
    Observable
        .Defer(() =>
        {
            var lastRan = DateTime.UtcNow.AddMonths(-6); // get dummy date 6 months ago
            var daysToRun = (DateTime.UtcNow - lastRan).Days;
            
            return
                Observable
                    .Using(
                        () => new HttpClient(),
                        hc =>
                            from day in
                                Observable
                                    .Range(1, daysToRun)
                                    .Select(day =>
                                        Observable
                                            .FromAsync(ct => PolygonWebApi.GetGroupedDailyBarsAsync(hc, this.apiKey, Locale.US, Market.Stocks, lastRan.AddDays(day), false, ct))
                                            .Do(r => this.logger.LogInformation("got {0} records for {1}", r.Results.Count(), lastRan.AddDays(day)))
                                            .Select(r => new TickersResponseWithDay(lastRan.AddDays(day), r)))
                                    .Merge(MaxConcurrentDownloads)
                            from tv2 in day.AggregateResponse.Results
                            select tv2)
                    .Buffer(10)
                    .SelectMany(xs => Observable.FromAsync(ct => SaveTickersToDatabaseAsync(xs, ct)));
        });

IDisposable subscription =
    query
        .Subscribe(
            x => { /* each call to `SaveTickersToDatabaseAsync` runs this code */  },
            ex => { /* an exception? then end here */ },
            () => { /* successfully completed */ });