Google bigquery 带有templateSuffix的TableData.insertAll-经常出现503错误

我们正在使用带有templateSuffix的TableData.insertAll，并且在使用模式中经常遇到503错误

我们根据两条信息设置templateSuffix—插入事件的名称和插入事件的数据。例如“NewPlayer20160712”。表ID设置为“事件”

在大多数情况下，这会如预期的那样工作，但通常会失败并返回错误。大约每200个插入中就有1个会失败，这对于预期的行为来说似乎太频繁了

我们的事件摄取服务的核心如下所示：

//Handle all rows in rowsBySuffix
async.mapLimit(Object.keys(rowsBySuffix), 5, function(suffix) {
    //Construct request for suffix
    var request = {
        projectId: "tactile-analytics",
        datasetId: "discoducksdev",
        tableId: "events",
        resource: {
            "kind": "bigquery#tableDataInsertAllRequest",
            "skipInvalidRows": true,
            "ignoreUnknownValues": true,
            "templateSuffix": suffix, // E.g. NewPlayer20160712
            "rows": rowsBySuffix[suffix]
        },
        auth: jwt // valid google.auth.JWT instance
    };

    //Insert all rows into BigQuery
    var cb = arguments[arguments.length-1];
    bigquery.tabledata.insertAll(request, function(err, result) {
        if(err) {
            console.log("Error insertAll. err=" + JSON.stringify(err) + ", request.resource=" + JSON.stringify(request.resource));
        }
        cb(err, result);
    });
}, arguments[arguments.length-1]);

{
   "code": 503,
   "errors": [
      {
         "domain": "global",
         "reason": "backendError",
         "message": "Error encountered during execution. Retrying may solve the problem."
      }
   ]
}

 {
   "kind": "bigquery#tableDataInsertAllRequest",
   "skipInvalidRows": true,
   "ignoreUnknownValues": true,
   "templateSuffix": "GameStarted20160618",
   "rows": [
      {
         "insertId": "1f4786eaccd1c16d7ce865fea4c7af89",
         "json": {
            "eventName": "gameStarted",
            "eventSchemaHash": "unique-schema-hash-value",
            "eventTimestamp": 1466264556,
            "userId": "f769dc78-3210-4fd5-a2b0-ca4c48447578",
            "sessionId": "821f8f40-ed08-49ff-b6ac-9a1b8194286b",
            "platform": "WEBPLAYER",
            "versionName": "1.0.0",
            "versionCode": 12345,
            "ts_param1": "2016-06-04 00:00",
            "ts_param2": "2014-01-01 00:00",
            "i_param0": 598,
            "i_param1": 491,
            "i_param2": 206,
            "i_param3": 412,
            "i_param4": 590,
            "i_param5": 842,
            "f_param0": 5945.442,
            "f_param1": 1623.4111,
            "f_param2": 147.04747,
            "f_param3": 6448.521,
            "b_param0": true,
            "b_param1": false,
            "b_param2": true,
            "b_param3": true,
            "s_param0": "Im guesior ti asorne usse siorst apedir eamighte rel kin.",
            "s_param1": "Whe autiorne awayst pon, lecurt mun.",
            "eventHash": "1f4786eaccd1c16d7ce865fea4c7af89",
            "collectTimestamp": "1468346812",
            "eventDate": "2016-06-18"
         }
      }
   ]
}

典型错误如下所示：

//Handle all rows in rowsBySuffix
async.mapLimit(Object.keys(rowsBySuffix), 5, function(suffix) {
    //Construct request for suffix
    var request = {
        projectId: "tactile-analytics",
        datasetId: "discoducksdev",
        tableId: "events",
        resource: {
            "kind": "bigquery#tableDataInsertAllRequest",
            "skipInvalidRows": true,
            "ignoreUnknownValues": true,
            "templateSuffix": suffix, // E.g. NewPlayer20160712
            "rows": rowsBySuffix[suffix]
        },
        auth: jwt // valid google.auth.JWT instance
    };

    //Insert all rows into BigQuery
    var cb = arguments[arguments.length-1];
    bigquery.tabledata.insertAll(request, function(err, result) {
        if(err) {
            console.log("Error insertAll. err=" + JSON.stringify(err) + ", request.resource=" + JSON.stringify(request.resource));
        }
        cb(err, result);
    });
}, arguments[arguments.length-1]);

{
   "code": 503,
   "errors": [
      {
         "domain": "global",
         "reason": "backendError",
         "message": "Error encountered during execution. Retrying may solve the problem."
      }
   ]
}

 {
   "kind": "bigquery#tableDataInsertAllRequest",
   "skipInvalidRows": true,
   "ignoreUnknownValues": true,
   "templateSuffix": "GameStarted20160618",
   "rows": [
      {
         "insertId": "1f4786eaccd1c16d7ce865fea4c7af89",
         "json": {
            "eventName": "gameStarted",
            "eventSchemaHash": "unique-schema-hash-value",
            "eventTimestamp": 1466264556,
            "userId": "f769dc78-3210-4fd5-a2b0-ca4c48447578",
            "sessionId": "821f8f40-ed08-49ff-b6ac-9a1b8194286b",
            "platform": "WEBPLAYER",
            "versionName": "1.0.0",
            "versionCode": 12345,
            "ts_param1": "2016-06-04 00:00",
            "ts_param2": "2014-01-01 00:00",
            "i_param0": 598,
            "i_param1": 491,
            "i_param2": 206,
            "i_param3": 412,
            "i_param4": 590,
            "i_param5": 842,
            "f_param0": 5945.442,
            "f_param1": 1623.4111,
            "f_param2": 147.04747,
            "f_param3": 6448.521,
            "b_param0": true,
            "b_param1": false,
            "b_param2": true,
            "b_param3": true,
            "s_param0": "Im guesior ti asorne usse siorst apedir eamighte rel kin.",
            "s_param1": "Whe autiorne awayst pon, lecurt mun.",
            "eventHash": "1f4786eaccd1c16d7ce865fea4c7af89",
            "collectTimestamp": "1468346812",
            "eventDate": "2016-06-18"
         }
      }
   ]
}

insertAll失败的资源部分如下所示：

//Handle all rows in rowsBySuffix
async.mapLimit(Object.keys(rowsBySuffix), 5, function(suffix) {
    //Construct request for suffix
    var request = {
        projectId: "tactile-analytics",
        datasetId: "discoducksdev",
        tableId: "events",
        resource: {
            "kind": "bigquery#tableDataInsertAllRequest",
            "skipInvalidRows": true,
            "ignoreUnknownValues": true,
            "templateSuffix": suffix, // E.g. NewPlayer20160712
            "rows": rowsBySuffix[suffix]
        },
        auth: jwt // valid google.auth.JWT instance
    };

    //Insert all rows into BigQuery
    var cb = arguments[arguments.length-1];
    bigquery.tabledata.insertAll(request, function(err, result) {
        if(err) {
            console.log("Error insertAll. err=" + JSON.stringify(err) + ", request.resource=" + JSON.stringify(request.resource));
        }
        cb(err, result);
    });
}, arguments[arguments.length-1]);

{
   "code": 503,
   "errors": [
      {
         "domain": "global",
         "reason": "backendError",
         "message": "Error encountered during execution. Retrying may solve the problem."
      }
   ]
}

 {
   "kind": "bigquery#tableDataInsertAllRequest",
   "skipInvalidRows": true,
   "ignoreUnknownValues": true,
   "templateSuffix": "GameStarted20160618",
   "rows": [
      {
         "insertId": "1f4786eaccd1c16d7ce865fea4c7af89",
         "json": {
            "eventName": "gameStarted",
            "eventSchemaHash": "unique-schema-hash-value",
            "eventTimestamp": 1466264556,
            "userId": "f769dc78-3210-4fd5-a2b0-ca4c48447578",
            "sessionId": "821f8f40-ed08-49ff-b6ac-9a1b8194286b",
            "platform": "WEBPLAYER",
            "versionName": "1.0.0",
            "versionCode": 12345,
            "ts_param1": "2016-06-04 00:00",
            "ts_param2": "2014-01-01 00:00",
            "i_param0": 598,
            "i_param1": 491,
            "i_param2": 206,
            "i_param3": 412,
            "i_param4": 590,
            "i_param5": 842,
            "f_param0": 5945.442,
            "f_param1": 1623.4111,
            "f_param2": 147.04747,
            "f_param3": 6448.521,
            "b_param0": true,
            "b_param1": false,
            "b_param2": true,
            "b_param3": true,
            "s_param0": "Im guesior ti asorne usse siorst apedir eamighte rel kin.",
            "s_param1": "Whe autiorne awayst pon, lecurt mun.",
            "eventHash": "1f4786eaccd1c16d7ce865fea4c7af89",
            "collectTimestamp": "1468346812",
            "eventDate": "2016-06-18"
         }
      }
   ]
}

我们注意到，如果我们避免在后缀中包含事件名称（例如NewPlayer部分），而只是将日期作为后缀，那么我们永远不会遇到这些错误

---

有什么方法可以使它可靠地工作吗？

后端错误会发生，我们通常从10000个请求中看到5个。我们只需重试，就有了更稳定的速率，我们可以提供一个可重构的用例，我们可以在Bigquery问题跟踪器上设置一个记录单。这样，如果我们的项目有问题，就可以进行调查