Node.js nodej在promise.all()中延迟每个承诺
我正在尝试更新一个不久前创建的工具,该工具使用nodejs(我不是JS开发人员,所以我正在尝试将代码拼凑在一起),我在最后一个障碍上遇到了困难 新功能将采用swagger.json定义,使用JS的Node.js nodej在promise.all()中延迟每个承诺,node.js,aws-api-gateway,aws-cli,aws-sdk-js,Node.js,Aws Api Gateway,Aws Cli,Aws Sdk Js,我正在尝试更新一个不久前创建的工具,该工具使用nodejs(我不是JS开发人员,所以我正在尝试将代码拼凑在一起),我在最后一个障碍上遇到了困难 新功能将采用swagger.json定义,使用JS的'AWS-sdk'sdk将端点与AWS服务上匹配的API网关进行比较,然后相应地更新网关 代码在一个小的定义文件(大约15个端点)上运行良好,但只要我给它一个更大的定义文件,我就会收到大量的TooManyRequestsException错误 我知道这是因为我对API网关服务的调用太快,需要延迟/暂停。
'AWS-sdk'
sdk将端点与AWS服务上匹配的API网关进行比较,然后相应地更新网关
代码在一个小的定义文件(大约15个端点)上运行良好,但只要我给它一个更大的定义文件,我就会收到大量的TooManyRequestsException
错误
我知道这是因为我对API网关服务的调用太快,需要延迟/暂停。这就是我被困的地方
我试着加上
- 返回每个承诺的延迟()
- 在每个承诺中运行setTimeout()
- 向Promise.all和Promise.mapSeries添加延迟
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath));
循环完成后,我运行以下操作:
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
我在地图系列上也尝试过同样的方法(运气不好)
看起来(getMethodResponse
promise)中的函数是立即运行的,因此,无论我添加了什么类型的延迟,它们都只是执行。我怀疑的是,我需要(getMethodResponse
)返回一个函数,然后使用mapSeries,但我也不能让它工作
我试过的代码:
将getMethodResponse
包装在以下内容中:
return function(value){}
然后将其添加到循环后(以及循环内-无差异):
还尝试了许多其他建议:
有什么建议吗
编辑
根据请求,一些额外的代码试图指出问题
当前正在使用一小部分端点(在Swagger文件中)的代码:
结果是这样的(每个端点都是一样的,有很多端点):
错误:TooManyRequestsException:请求太多
X
错误:TooManyRequestsException:请求太多
X
错误:TooManyRequestsException:请求太多
AWS SDK在每个承诺中被调用3次,其函数为(从getMethodResponse()函数启动):
典型的AWS SDK文档指出,这是进行太多连续调用(太快)时的典型行为。我过去也遇到过类似的问题,只需在调用的代码中添加一个.delay(500)就可以解决
比如:
return apigateway.updateModelAsync(updateModelParams)
.tap(() => logger.verbose(`Updated model ${updatedModel.name}`))
.tap(() => bar.tick())
.delay(500)
编辑#2
我想以彻底的名义,包括我的整个.js
文件
'use strict';
const AWS = require('aws-sdk');
let apigateway, lambda;
const Promise = require('bluebird');
const R = require('ramda');
const logger = require('../logger');
const config = require('../config/default');
const helpers = require('../library/helpers');
const winston = require('winston');
const request = require('request');
const _ = require('lodash');
const region = 'ap-southeast-2';
const methodLib = require('../aws/methods');
const emitter = require('../library/emitter');
emitter.on('updateRegion', (region) => {
region = region;
AWS.config.update({ region: region });
apigateway = new AWS.APIGateway({ apiVersion: '2015-07-09' });
Promise.promisifyAll(apigateway);
});
function getSwaggerFromHttp(externalUrl) {
return new Promise((resolve, reject) => {
request.get({
url: externalUrl,
header: {
"content-type": "application/json"
}
}, (err, res, body) => {
if (err) {
winston.error(err);
reject(err);
}
let result = JSON.parse(body);
resolve(result);
})
});
}
/*
Deletes a method response
*/
function deleteMethodResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteMethodResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Method response ${statusCode} deleted for path: ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Method Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Deletes an integration response
*/
function deleteIntegrationResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteIntegrationResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Integration response ${statusCode} deleted for path ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Integration Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Get Resource
*/
function getMethodResponse(httpMethod, statusCode, apiName, resourcePath) {
let params = {
httpMethod: httpMethod.toUpperCase(),
resourceId: '',
restApiId: ''
}
return getResourceDetails(apiName, resourcePath)
.error((e) => {
logger.unimportant('Error: ' + e.stack)
})
.then((result) => {
//Only run the comparrison of models if the resourceId (from the url passed in) is found within the AWS Gateway
if (result) {
params.resourceId = result.resourceId
params.restApiId = result.apiId
var awsMethodResponses = [];
try {
apigateway.getMethodAsync(params, function (err, data) {
if (err) {
if (err.statusCode == 404) {
return console.log(`Method ${params.httpMethod} not found on resource path: ${resourcePath} (resourceId: ${params.resourceId})`); // an error occurred
}
console.log(err, err.stack); // an error occurred
}
else {
if (data) {
_.each(data.methodResponses, function (value, key) {
if (key >= 200 && key <= 204) {
awsMethodResponses.push(key)
}
});
awsMethodResponses = _.pull(awsMethodResponses, statusCode); //List of items not found within the Gateway - to be removed.
_.each(awsMethodResponses, function (value, key) {
if (data.methodResponses[value].responseModels) {
var existingModel = data.methodResponses[value].responseModels['application/json']; //Check if there is currently a model attached to the resource / method about to be deleted
methodLib.updateResponseAssociation(params.httpMethod, params.resourceId, params.restApiId, statusCode, existingModel); //Associate this model to the same resource / method, under the new response status
}
deleteMethodResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
deleteIntegrationResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
})
}
}
})
.catch(err => {
console.log(`Error: ${err}`);
});
}
catch (e) {
console.log(`getMethodAsync failed, Error: ${e}`);
}
}
})
};
function getResourceDetails(apiName, resourcePath) {
let resourceExpr = new RegExp(resourcePath + '$', 'i');
let result = {
apiId: '',
resourceId: '',
path: ''
}
return helpers.apiByName(apiName, AWS.config.region)
.delay(1200)
.then(apiId => {
result.apiId = apiId;
let resourceParams = {
restApiId: apiId,
limit: config.awsGetResourceLimit,
};
return apigateway.getResourcesAsync(resourceParams)
})
.then(R.prop('items'))
.filter(R.pipe(R.prop('path'), R.test(resourceExpr)))
.tap(helpers.handleNotFound('resource'))
.then(R.head)
.then([R.prop('path'), R.prop('id')])
.then(returnedObj => {
if (returnedObj.id) {
result.path = returnedObj.path;
result.resourceId = returnedObj.id;
logger.unimportant(`ApiId: ${result.apiId} | ResourceId: ${result.resourceId} | Path: ${result.path}`);
return result;
}
})
.catch(err => {
console.log(`Error: ${err} on API: ${apiName} Resource: ${resourcePath}`);
});
};
function delay(t) {
return new Promise(function(resolve) {
setTimeout(resolve, t)
});
}
module.exports = (apiName, externalUrl) => {
return getSwaggerFromHttp(externalUrl)
.then((swagger) => {
let paths = swagger.paths;
let resourcePath = '';
let resourceMethod = '';
let promises = [];
_.each(paths, function (value, key) {
resourcePath = key;
_.each(value, function (value, key) {
resourceMethod = key;
let statusList = [];
_.each(value.responses, function (value, key) {
if (key >= 200 && key <= 204) {
statusList.push(key)
}
});
_.each(statusList, function (value, key) { //Only for 200-201 range
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath))
});
});
});
//Working with small set
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
})
.catch((err) => {
winston.error(err);
});
};
“严格使用”;
const AWS=require('AWS-sdk');
让阿皮盖特,拉姆达;
const Promise=require(‘蓝鸟’);
常数R=需要('ramda');
const logger=require(“../logger”);
const config=require('../config/default');
常量helpers=require('../library/helpers');
const winston=require('winston');
const request=require('request');
const=require('lodash');
常量区域='ap-东南-2';
const methodLib=require('../aws/methods');
const emitter=require('../library/emitter');
发射器.on('updateRegion',(region)=>{
区域=区域;
AWS.config.update({region:region});
apigateway=新的AWS.apigateway({apiVersion:'2015-07-09'});
promisifyAll(apigateway);
});
函数getSwaggerFromHttp(externalUrl){
返回新承诺((解决、拒绝)=>{
请求({
url:externalUrl,
标题:{
“内容类型”:“应用程序/json”
}
},(错误、恢复、正文)=>{
如果(错误){
错误;
拒绝(错误);
}
让result=JSON.parse(body);
决心(结果);
})
});
}
/*
删除方法响应
*/
函数deleteMethodResponse(httpMethod、resourceId、restApiId、statusCode、resourcePath){
让methodResponseParams={
httpMethod:httpMethod,
resourceId:resourceId,
restApiId:restApiId,
状态代码:状态代码
};
返回apigateway.deleteMethodResponseAsync(methodResponseParams)
.延迟(1200)
.tap(()=>logger.verbose(`methodresponse${statusCode}已删除路径:${resourcePath}`)
.错误((e)=>{
return console.log(`Error deleting Method Response${httpMethod}在资源路径${resourcePath}(resourceId:${resourceId})上找不到)`;//发生错误
logger.error('error:'+e.stack)
});
}
/*
删除集成响应
*/
函数deleteIntegrationResponse(httpMethod、resourceId、restApiId、statusCode、resourcePath){
让methodResponseParams={
httpMethod:httpMethod,
resourceId:resourceId,
restApiId:restApiId,
状态代码:状态代码
};
返回apigateway.deleteIntegrationResponseAsync(methodResponseParams)
.延迟(1200)
.tap(()=>logger.verbose(`Integration response${statusCode}已删除路径${resourcePath}`)
.错误((e)=>{
return console.log(`Error deleting Integration Response${httpMethod}未在资源路径${resourcePath}(resourceId:${resourceId})`)上找到;//发生错误
logger.error('error:'+e.stack)
});
}
/*
获取资源
*/
函数getMethodResponse(httpMethod、statusCode、apiName、resourcePath){
设params={
httpMethod:httpMethod.toUpperCase(),
资源ID:“”,
restApiId:“”
}
返回getResourceDetails(apiName、resourcePath)
.错误((e)=>{
logger.nonimportant('错误:'+e.stack)
})
。然后((结果)=>{
//仅当resourceId(来自传入的url)为fo时,才运行模型的比较
Promise.map(promises, function() {
// Promise.map awaits for returned promises as well.
console.log('X');
},{concurrency: 5})
.then(function() {
return console.log("y");
});
apigateway.getRestApisAsync()
return apigateway.getResourcesAsync(resourceParams)
apigateway.getMethodAsync(params, function (err, data) {}
return apigateway.updateModelAsync(updateModelParams)
.tap(() => logger.verbose(`Updated model ${updatedModel.name}`))
.tap(() => bar.tick())
.delay(500)
'use strict';
const AWS = require('aws-sdk');
let apigateway, lambda;
const Promise = require('bluebird');
const R = require('ramda');
const logger = require('../logger');
const config = require('../config/default');
const helpers = require('../library/helpers');
const winston = require('winston');
const request = require('request');
const _ = require('lodash');
const region = 'ap-southeast-2';
const methodLib = require('../aws/methods');
const emitter = require('../library/emitter');
emitter.on('updateRegion', (region) => {
region = region;
AWS.config.update({ region: region });
apigateway = new AWS.APIGateway({ apiVersion: '2015-07-09' });
Promise.promisifyAll(apigateway);
});
function getSwaggerFromHttp(externalUrl) {
return new Promise((resolve, reject) => {
request.get({
url: externalUrl,
header: {
"content-type": "application/json"
}
}, (err, res, body) => {
if (err) {
winston.error(err);
reject(err);
}
let result = JSON.parse(body);
resolve(result);
})
});
}
/*
Deletes a method response
*/
function deleteMethodResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteMethodResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Method response ${statusCode} deleted for path: ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Method Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Deletes an integration response
*/
function deleteIntegrationResponse(httpMethod, resourceId, restApiId, statusCode, resourcePath) {
let methodResponseParams = {
httpMethod: httpMethod,
resourceId: resourceId,
restApiId: restApiId,
statusCode: statusCode
};
return apigateway.deleteIntegrationResponseAsync(methodResponseParams)
.delay(1200)
.tap(() => logger.verbose(`Integration response ${statusCode} deleted for path ${resourcePath}`))
.error((e) => {
return console.log(`Error deleting Integration Response ${httpMethod} not found on resource path: ${resourcePath} (resourceId: ${resourceId})`); // an error occurred
logger.error('Error: ' + e.stack)
});
}
/*
Get Resource
*/
function getMethodResponse(httpMethod, statusCode, apiName, resourcePath) {
let params = {
httpMethod: httpMethod.toUpperCase(),
resourceId: '',
restApiId: ''
}
return getResourceDetails(apiName, resourcePath)
.error((e) => {
logger.unimportant('Error: ' + e.stack)
})
.then((result) => {
//Only run the comparrison of models if the resourceId (from the url passed in) is found within the AWS Gateway
if (result) {
params.resourceId = result.resourceId
params.restApiId = result.apiId
var awsMethodResponses = [];
try {
apigateway.getMethodAsync(params, function (err, data) {
if (err) {
if (err.statusCode == 404) {
return console.log(`Method ${params.httpMethod} not found on resource path: ${resourcePath} (resourceId: ${params.resourceId})`); // an error occurred
}
console.log(err, err.stack); // an error occurred
}
else {
if (data) {
_.each(data.methodResponses, function (value, key) {
if (key >= 200 && key <= 204) {
awsMethodResponses.push(key)
}
});
awsMethodResponses = _.pull(awsMethodResponses, statusCode); //List of items not found within the Gateway - to be removed.
_.each(awsMethodResponses, function (value, key) {
if (data.methodResponses[value].responseModels) {
var existingModel = data.methodResponses[value].responseModels['application/json']; //Check if there is currently a model attached to the resource / method about to be deleted
methodLib.updateResponseAssociation(params.httpMethod, params.resourceId, params.restApiId, statusCode, existingModel); //Associate this model to the same resource / method, under the new response status
}
deleteMethodResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
deleteIntegrationResponse(params.httpMethod, params.resourceId, params.restApiId, value, resourcePath)
.delay(1200)
.done();
})
}
}
})
.catch(err => {
console.log(`Error: ${err}`);
});
}
catch (e) {
console.log(`getMethodAsync failed, Error: ${e}`);
}
}
})
};
function getResourceDetails(apiName, resourcePath) {
let resourceExpr = new RegExp(resourcePath + '$', 'i');
let result = {
apiId: '',
resourceId: '',
path: ''
}
return helpers.apiByName(apiName, AWS.config.region)
.delay(1200)
.then(apiId => {
result.apiId = apiId;
let resourceParams = {
restApiId: apiId,
limit: config.awsGetResourceLimit,
};
return apigateway.getResourcesAsync(resourceParams)
})
.then(R.prop('items'))
.filter(R.pipe(R.prop('path'), R.test(resourceExpr)))
.tap(helpers.handleNotFound('resource'))
.then(R.head)
.then([R.prop('path'), R.prop('id')])
.then(returnedObj => {
if (returnedObj.id) {
result.path = returnedObj.path;
result.resourceId = returnedObj.id;
logger.unimportant(`ApiId: ${result.apiId} | ResourceId: ${result.resourceId} | Path: ${result.path}`);
return result;
}
})
.catch(err => {
console.log(`Error: ${err} on API: ${apiName} Resource: ${resourcePath}`);
});
};
function delay(t) {
return new Promise(function(resolve) {
setTimeout(resolve, t)
});
}
module.exports = (apiName, externalUrl) => {
return getSwaggerFromHttp(externalUrl)
.then((swagger) => {
let paths = swagger.paths;
let resourcePath = '';
let resourceMethod = '';
let promises = [];
_.each(paths, function (value, key) {
resourcePath = key;
_.each(value, function (value, key) {
resourceMethod = key;
let statusList = [];
_.each(value.responses, function (value, key) {
if (key >= 200 && key <= 204) {
statusList.push(key)
}
});
_.each(statusList, function (value, key) { //Only for 200-201 range
promises.push(getMethodResponse(resourceMethod, value, apiName, resourcePath))
});
});
});
//Working with small set
return Promise.all(promises)
.catch((err) => {
winston.error(err);
})
})
.catch((err) => {
winston.error(err);
});
};
const rp = require('request-promise');
let urls = [...]; // large array of URLs to process
Promise.map(urls, function(url) {
return rp(url).then(function(data) {
// process scraped data here
return someValue;
});
}, {concurrency: 20}).then(function(results) {
// process array of results here
}).catch(function(err) {
// error here
});
Promise.map(promises, function() {
...
});