Aws lambda 来自lambda的aws sdk js runJobFlow未发送启动EMR群集的请求
我试图从一个lambda启动一个EMR集群,该lambda由S3 bucket中的事件触发 触发器工作正常,然后我使用Aws lambda 来自lambda的aws sdk js runJobFlow未发送启动EMR群集的请求,aws-lambda,aws-sdk,amazon-emr,aws-sdk-js,aws-sdk-nodejs,Aws Lambda,Aws Sdk,Amazon Emr,Aws Sdk Js,Aws Sdk Nodejs,我试图从一个lambda启动一个EMR集群,该lambda由S3 bucket中的事件触发 触发器工作正常,然后我使用runJobFlow创建了一个AWSRequest,但是EMR控制台中没有事件,集群也没有启动 包括创建的AWSRequest在内的所有内容都会记录到CloudWatch中,但不会记录任何错误 这根本没用 代码如下: const aws = require('aws-sdk'); const emr = new aws.EMR({ apiVersion: '2009-
runJobFlow
创建了一个AWSRequest,但是EMR控制台中没有事件,集群也没有启动
包括创建的AWSRequest在内的所有内容都会记录到CloudWatch中,但不会记录任何错误
这根本没用
代码如下:
const aws = require('aws-sdk');
const emr = new aws.EMR({
apiVersion: '2009-03-31',
region: 'us-east-1'
});
const emrClusterConfig = (s3_input_path, s3_output_path) => {
const ret = {
Name:`cluster-for-job`,
ServiceRole: 'EMR_DefaultRole',
JobFlowRole: 'EMR_EC2_DefaultRole',
VisibleToAllUsers: true,
ScaleDownBehavior: 'TERMINATE_AT_TASK_COMPLETION',
LogUri: 's3n://log-uri/elasticmapreduce/',
ReleaseLabel: 'emr-5.29.0',
Instances:{
InstanceGroups: [
{
Name: 'Master Instance Group',
Market: 'ON_DEMAND',
InstanceRole: 'MASTER',
InstanceType: 'm5.xlarge',
InstanceCount: 1,
EbsConfiguration: {
EbsBlockDeviceConfigs: [
{
VolumeSpecification: {
SizeInGB: 32,
VolumeType: 'gp2',
},
VolumesPerInstance: 2
},
]
},
},
{
Name: 'Core Instance Group',
{... similar to master ...}
}
],
Ec2KeyName: 'my-keys',
Ec2SubnetId: 'my-subnet-id',
EmrManagedSlaveSecurityGroup:'sg-slave-security-group',
EmrManagedMasterSecurityGroup:'sg-master-security-group',
KeepJobFlowAliveWhenNoSteps: false,
TerminationProtected: false
},
Applications:[
{
'Name': 'Spark'
},
],
Configurations:[{
"Classification":"spark",
"Properties":{}
}],
Steps:[{
'Name': 'step',
'ActionOnFailure': 'TERMINATE_CLUSTER',
'HadoopJarStep': {
'Jar': 's3n://elasticmapreduce/libs/script-runner/script-runner.jar',
'Args': [
"/usr/bin/spark-submit", "--deploy-mode", "cluster",
's3://path-to-a-very-humble.jar', s3_input_path, s3_output_path
]
}
}],
}
return ret
}
exports.handler = async (event, context) => {
const record = event.Records[0];
const eventName = record.eventName;
if(eventName === 'ObjectCreated:Put' || eventName === 'ObjectCreated:Post' || eventName === 'ObjectCreated:CompleteMultipartUpload' || eventName === 'ObjectCreated:Copy'){
const s3_inputPath = 's3n://in-bucket/key';
const s3_outputPath = 's3n://out-bucket/key';
try{
const cluster_config = emrClusterConfig(s3_inputPath,s3_outputPath);
const AWS_EMRJobRequest = emr.runJobFlow(cluster_config)
AWS_EMRJobRequest
.on('success', function(response){ console.log("success => " + response)})
.on('error', function(response){ console.log("error => " + response)})
.on('complete', function(response){ console.log("complete => " + response)})
.send( function(err, data){
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
context.done(null,'λ Completed');
});
console.log('Finished Launching EMR cluster: ', AWS_EMRJobRequest)
}
catch(err){
console.log(err);
}
}
else{
console.log(`:: not interested in event ${eventName}`);
}
context.done(null, 'λ Completed');
};
我以前手动设置过这些集群,它们工作得很好。我从AWS CLI导出中的信息复制了群集配置,以匹配现有群集的设置
这没有任何作用,只是在“已完成启动EMR群集”的末尾记录请求obj,但什么也没有发生。aws在收到响应之前终止函数,因为AWSRequest以异步方式发送请求。由于您使用的是异步处理程序,因此可以使用AWS.Request.promise。这会立即启动服务调用,并返回一个承诺,该承诺要么用response data属性实现,要么用response error属性拒绝
let AWS_EMRJobRequest = emr.runJobFlow(cluster_config);
return AWS_EMRJobRequest.promise();
有关更多信息,请参阅