C# 使用DynamoDB和AWSSDK.net的BatchWriteItem

我当前在使用DynamoDB调用BatchWriteItem请求时遇到问题。我得到以下错误

<强> >无法将数据保存到DB。超出了配置表的吞吐量水平。请考虑使用可更新的API增加您的配置级别“< /强>

”。 配置：

吞吐量：读取=5；写入=5

这是我的设想： 我需要一个接一个地向同一个表执行两组数据写入请求 -使用聚合数据（932KB：分为63KB的数据块，约为15个数据块） -非聚合数据（940 KB：分为63KB的数据块，约为15个数据块）

在写之前，我根据散列键和范围键值删除dynamo db表中的所有项（对于聚合数据：删除16项）

代码如下：

public void PutItems(string tableName, string id, string message, bool aggregated)
    {
         if (aggregated)
             id = "a_" + id;
         else
             id = "r_" + id;
        List<WriteRequest> writeRequests = new List<WriteRequest>();
        List<WriteRequest> DeleteWriteRequest = new List<WriteRequest>();
        var itemsToDelete = GetItemsToDelete(tableName, id);
        if (itemsToDelete.Count > 0)
        {
            foreach (Tuple<string, int> item in itemsToDelete)
            {

                DeleteRequest deleteRequest = new DeleteRequest
                {
                    Key = new Key
                    {
                        HashKeyElement = new AttributeValue { S = item.Item1 },
                        RangeKeyElement = new AttributeValue { N = item.Item2.ToString() }
                    }
                };
                WriteRequest deleteReq = new WriteRequest();
                deleteReq.DeleteRequest = deleteRequest;
                DeleteWriteRequest.Add(deleteReq);
            }
            logger.Info(this.GetType().ToString(), string.Format(".....Deleting Old Data..........."));
            foreach (IEnumerable<WriteRequest> ls in LinqExtensions.Partition(DeleteWriteRequest, SplitLevel))
            {
                BatchWriteItemRequest delWriteRequest = new BatchWriteItemRequest();
                delWriteRequest.WithRequestItems(new KeyValuePair<string, List<WriteRequest>>(tableName, ls.ToList()));

                CallBatchWriteTillCompletion(delWriteRequest);
            }
            logger.Info(this.GetType().ToString(), string.Format(".....Delete Complete!..........")); 
        }

        int MaxLength = 64512; //64KB = 65536 Bytes ; 60KB = 61440 Bytes
        logger.Info(this.GetType().ToString(), string.Format("Message Size : {0}", message.Length)); 
        var str = SplitToChunks(message, MaxLength).ToList();
        for (int i = 0; i < str.Count; i++)
        {



            PutRequest putRequest = new PutRequest
            {
                Item = new Dictionary<string, AttributeValue>()
                    {
                        {"Received", new AttributeValue {S = id}},
                        {"SequenceNum" , new AttributeValue {N = i.ToString()}},
                        {"Message", new AttributeValue {S = str[i]}}
                    }
            };
            WriteRequest request = new WriteRequest();
            request.PutRequest = putRequest;
            writeRequests.Add(request);
        }

        logger.Info(this.GetType().ToString(), string.Format(".....Writing Data..........."));
        foreach (IEnumerable<WriteRequest> ls in LinqExtensions.Partition(writeRequests, SplitLevel))
        {
                System.Threading.Thread.Sleep(1000);
                BatchWriteItemRequest writeRequest = new BatchWriteItemRequest();
                writeRequest.WithRequestItems(new KeyValuePair<string, List<WriteRequest>>(tableName, ls.ToList()));
                CallBatchWriteTillCompletion(writeRequest);


        }
        logger.Info(this.GetType().ToString(), string.Format(".....Write Complete!.........."));
    }


    private void CallBatchWriteTillCompletion(BatchWriteItemRequest request)
    {
        BatchWriteItemResponse response;

        int callCount = 0;
        do
        {
            if (callCount > 0)
            {
                System.Threading.Thread.Sleep(1000);
            }
            logger.Info(this.GetType().ToString(), string.Format("Making Request"));
            response = Instance.Client.BatchWriteItem(request);
            callCount++;

            // Check the response.
            var result = response.BatchWriteItemResult;
            var responses = result.Responses;
            var unprocessed = result.UnprocessedItems;

            logger.Info(this.GetType().ToString(), string.Format("Response"));
            foreach (var resp in responses)
            {

                logger.Info(this.GetType().ToString(), string.Format("{0} - {1}", resp.Key, resp.Value.ConsumedCapacityUnits));
            }

            logger.Info(this.GetType().ToString(), string.Format("Unprocessed"));
            foreach (var unp in unprocessed)
            {

                logger.Info(this.GetType().ToString(), string.Format("{0} - {1}", unp.Key, unp.Value.Count));

            }

            // For the next iteration, the request will have unprocessed items.
            request.RequestItems = unprocessed;

        } while (response.BatchWriteItemResult.UnprocessedItems.Count > 0);

        logger.Info(this.GetType().ToString(), string.Format("Total # of batch write API calls made: {0}", callCount));

    }



    //This method is use to split a string message of smaller chunks of 64KB.
    private static IEnumerable<String> SplitToChunks(String str, int maxLength)
    {
            for (int index = 0; index < str.Length; index += maxLength)
            {
                yield return str.Substring(index, Math.Min(maxLength, str.Length - index));
            }          
    }

此数字（大致）转换为每秒KB： 吞吐量：写入=5

如果您试图写入一个63KB的项目，则需要写入该项目，然后等待约13秒，然后再尝试写入。您已设置每秒5 KB，然后在单个突发中使用了63 KB。在接下来的13秒钟内，您向表中写入的任何请求都可能被阻止

根据时间戳判断，客户机正在通过计算错误和重试不可见地处理所提供的数据；例如，在这里您可以看到请求过程中经过了10秒：

2013-03-06 21:22:08.9687 INFO     Making Request
2013-03-06 21:22:18.0156 INFO     Response

最终，在一系列较小的请求之后，桌子上出现了一个大请求：

2013-03-06 21:23:41.9531 INFO     Key: TrafficConditionsMessages_dev - Consumed Capacity Units: 320

下一个请求耗时太长，以至于您的客户决定是时候向您公开错误了：

2013-03-06 21:23:42.4218 INFO     Making Request
2013-03-06 21:23:50.5468 ERROR    Unable to save data to DB. The ...

解决方案是提高您的配置吞吐量，根据返回的ConsumedCapacity延长睡眠时间，或者减少数据写入量。

此数字（大致）转换为每秒KB： 吞吐量：写入=5

如果您试图写入一个63KB的项目，则需要写入该项目，然后等待约13秒，然后再尝试写入。您已设置每秒5 KB，然后在单个突发中使用了63 KB。在接下来的13秒钟内，您向表中写入的任何请求都可能被阻止

根据时间戳判断，客户机正在通过计算错误和重试不可见地处理所提供的数据；例如，在这里您可以看到请求过程中经过了10秒：

2013-03-06 21:22:08.9687 INFO     Making Request
2013-03-06 21:22:18.0156 INFO     Response

最终，在一系列较小的请求之后，桌子上出现了一个大请求：

2013-03-06 21:23:41.9531 INFO     Key: TrafficConditionsMessages_dev - Consumed Capacity Units: 320

下一个请求耗时太长，以至于您的客户决定是时候向您公开错误了：

2013-03-06 21:23:42.4218 INFO     Making Request
2013-03-06 21:23:50.5468 ERROR    Unable to save data to DB. The ...

解决方案是要么提高您的配置吞吐量，根据返回的ConsumedCapacity睡眠更长时间，要么写更少的数据