如何设置amazon ami';使用java代码进行hadoop配置
我想设置这个配置如何设置amazon ami';使用java代码进行hadoop配置,java,hadoop,apache-pig,amazon-emr,Java,Hadoop,Apache Pig,Amazon Emr,我想设置这个配置textinputformat.record.delimiter=到hadoop 现在,我使用以下代码在ami上运行pig脚本。有人知道如何使用以下代码设置此配置吗 代码: 您需要做的是创建BootstrapActionConfig并将其添加到正在创建的RunJobFlowRequest,然后将自定义hadoop配置添加到集群中 以下是我在编辑代码后为您编写的完整代码: import java.util.ArrayList; 导入java.util.List; 导入com.ama
textinputformat.record.delimiter=代码>到hadoop
现在,我使用以下代码在ami上运行pig脚本。有人知道如何使用以下代码设置此配置吗
代码:
您需要做的是创建BootstrapActionConfig
并将其添加到正在创建的RunJobFlowRequest
,然后将自定义hadoop配置添加到集群中
以下是我在编辑代码后为您编写的完整代码:
import java.util.ArrayList;
导入java.util.List;
导入com.amazonaws.auth.AWSCredentials;
导入com.amazonaws.auth.BasicAWSCredentials;
导入com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClient;
导入com.amazonaws.services.elasticmapreduce.model.bootstrapacationconfig;
导入com.amazonaws.services.elasticmapreduce.model.JobFlowInstancesConfig;
导入com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest;
导入com.amazonaws.services.elasticmapreduce.model.RunJobFlowResult;
导入com.amazonaws.services.elasticmapreduce.model.scriptbootstrapacationconfig;
导入com.amazonaws.services.elasticmapreduce.model.StepConfig;
导入com.amazonaws.services.elasticmapreduce.util.StepFactory;
/**
*
*@作者阿马尔
*
*/
公共类RunEMRJobFlow{
私有静态最终字符串CONFIG_HADOOP_BOOTSTRAP_ACTION=“s3://elasticmapreduce/BOOTSTRAP actions/configure HADOOP”;
公共静态void main(字符串[]args){
字符串accessKey=“”;
字符串secretKey=“”;
AWSCredentials credentials=新的基本WSCredentials(accessKey、secretKey);
AmazonElasticMapReduceClient emr=新AmazonElasticMapReduceClient(凭据);
StepFactory StepFactory=新的StepFactory();
StepConfig enabledebugging=new StepConfig().withName(“启用调试”)
.withActionOnFailure(“终止作业流”)。withHadoopJarStep(stepFactory.newEnableDebuggingStep());
StepConfig installHive=new StepConfig().withName(“安装配置单元”).withActionOnFailure(“终止作业流”)
.withHadoopJarStep(stepFactory.newInstallHiveStep());
List setMapperArgs=new ArrayList();
setMapperArgs.add(“-s”);
setMapperArgs.add(“textinputformat.record.delimiter=;”;
BootstrapActionConfig映射程序bootstrapconfig=createBootstrapAction(“设置Hadoop配置”,
配置\u HADOOP\u引导\u操作,setMapperArgs);
RunJobFlowRequest=新的RunJobFlowRequest()
.withBootstrapActions(MapperBootstrapConfig)
.withName(“蜂巢互动”)
.with步骤(启用调试、安装配置单元)
.withLogUri(“s3://myawsbucket/”)
.举例说明(
新的JobFlowInstanceConfig().withEc2KeyName(“keypair”).withHadoopVersion(“0.20”)
.withInstanceCount(5)。WithKeepJobFlowAliveWhenNostps(true)
.使用MasterInstanceType(“m1.small”)。使用LaveInstanceType(“m1.small”);
RunJobFlowResult=emr.runJobFlow(请求);
}
私有静态BootstrapActionConfig createBootstrapAction(字符串bootstrapName、字符串bootstrapPath、,
列表参数){
ScriptBootstrapActionConfig bootstrapScriptConfig=new-ScriptBootstrapActionConfig();
bootstrappscriptconfig.setPath(bootstrapPath);
如果(args!=null){
bootstrapScriptConfig.setArgs(args);
}
BootstrapActionConfig bootstrapConfig=new BootstrapActionConfig();
bootstrapConfig.setName(bootstrapName);
bootstrapConfig.setscriptbootstrapacation(bootstrapScriptConfig);
返回bootstrapConfig;
}
}
StepConfig installPig = new StepConfig()
.withName("Install Pig")
.withActionOnFailure(ActionOnFailure.TERMINATE_JOB_FLOW.name())
.withHadoopJarStep(stepFactory.newInstallPigStep());
// [Configure pig script][1]
String[] scriptArgs = new String[] { "-p", input, "-p", output };
StepConfig runPigLatinScript = new StepConfig()
.withName("Run Pig Script") .withActionOnFailure(ActionOnFailure.CANCEL_AND_WAIT.name())
.withHadoopJarStep(stepFactory.newRunPigScriptStep("s3://pig/script.pig", scriptArgs));
// Configure JobFlow [R1][2], [R3][3]
//
//
RunJobFlowRequest request = new RunJobFlowRequest()
.withName(jobFlowName)
.withSteps(installPig, runPigLatinScript)
.withLogUri(logUri)
.withAmiVersion("2.3.2")
.withInstances(new JobFlowInstancesConfig()
.withEc2KeyName(this.ec2KeyName)
.withInstanceCount(this.count)
.withKeepJobFlowAliveWhenNoSteps(false)
.withMasterInstanceType(this.masterType)
.withSlaveInstanceType(this.slaveType));
// Run JobFlow
RunJobFlowResult runJobFlowResult = this.amazonEmrClient.runJobFlow(request);
import java.util.ArrayList;
import java.util.List;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClient;
import com.amazonaws.services.elasticmapreduce.model.BootstrapActionConfig;
import com.amazonaws.services.elasticmapreduce.model.JobFlowInstancesConfig;
import com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest;
import com.amazonaws.services.elasticmapreduce.model.RunJobFlowResult;
import com.amazonaws.services.elasticmapreduce.model.ScriptBootstrapActionConfig;
import com.amazonaws.services.elasticmapreduce.model.StepConfig;
import com.amazonaws.services.elasticmapreduce.util.StepFactory;
/**
*
* @author amar
*
*/
public class RunEMRJobFlow {
private static final String CONFIG_HADOOP_BOOTSTRAP_ACTION = "s3://elasticmapreduce/bootstrap-actions/configure-hadoop";
public static void main(String[] args) {
String accessKey = "";
String secretKey = "";
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonElasticMapReduceClient emr = new AmazonElasticMapReduceClient(credentials);
StepFactory stepFactory = new StepFactory();
StepConfig enabledebugging = new StepConfig().withName("Enable debugging")
.withActionOnFailure("TERMINATE_JOB_FLOW").withHadoopJarStep(stepFactory.newEnableDebuggingStep());
StepConfig installHive = new StepConfig().withName("Install Hive").withActionOnFailure("TERMINATE_JOB_FLOW")
.withHadoopJarStep(stepFactory.newInstallHiveStep());
List<String> setMappersArgs = new ArrayList<String>();
setMappersArgs.add("-s");
setMappersArgs.add("textinputformat.record.delimiter=;");
BootstrapActionConfig mappersBootstrapConfig = createBootstrapAction("Set Hadoop Config",
CONFIG_HADOOP_BOOTSTRAP_ACTION, setMappersArgs);
RunJobFlowRequest request = new RunJobFlowRequest()
.withBootstrapActions(mappersBootstrapConfig)
.withName("Hive Interactive")
.withSteps(enabledebugging, installHive)
.withLogUri("s3://myawsbucket/")
.withInstances(
new JobFlowInstancesConfig().withEc2KeyName("keypair").withHadoopVersion("0.20")
.withInstanceCount(5).withKeepJobFlowAliveWhenNoSteps(true)
.withMasterInstanceType("m1.small").withSlaveInstanceType("m1.small"));
RunJobFlowResult result = emr.runJobFlow(request);
}
private static BootstrapActionConfig createBootstrapAction(String bootstrapName, String bootstrapPath,
List<String> args) {
ScriptBootstrapActionConfig bootstrapScriptConfig = new ScriptBootstrapActionConfig();
bootstrapScriptConfig.setPath(bootstrapPath);
if (args != null) {
bootstrapScriptConfig.setArgs(args);
}
BootstrapActionConfig bootstrapConfig = new BootstrapActionConfig();
bootstrapConfig.setName(bootstrapName);
bootstrapConfig.setScriptBootstrapAction(bootstrapScriptConfig);
return bootstrapConfig;
}
}