Warning: file_get_contents(/data/phpspider/zhask/data//catemap/9/java/379.json): failed to open stream: No such file or directory in /data/phpspider/zhask/libs/function.php on line 167

Warning: Invalid argument supplied for foreach() in /data/phpspider/zhask/libs/tag.function.php on line 1116

Notice: Undefined index: in /data/phpspider/zhask/libs/function.php on line 180

Warning: array_chunk() expects parameter 1 to be array, null given in /data/phpspider/zhask/libs/function.php on line 181
如何设置amazon ami';使用java代码进行hadoop配置_Java_Hadoop_Apache Pig_Amazon Emr - Fatal编程技术网

如何设置amazon ami';使用java代码进行hadoop配置

如何设置amazon ami';使用java代码进行hadoop配置,java,hadoop,apache-pig,amazon-emr,Java,Hadoop,Apache Pig,Amazon Emr,我想设置这个配置textinputformat.record.delimiter=到hadoop 现在,我使用以下代码在ami上运行pig脚本。有人知道如何使用以下代码设置此配置吗 代码: 您需要做的是创建BootstrapActionConfig并将其添加到正在创建的RunJobFlowRequest,然后将自定义hadoop配置添加到集群中 以下是我在编辑代码后为您编写的完整代码: import java.util.ArrayList; 导入java.util.List; 导入com.ama

我想设置这个配置
textinputformat.record.delimiter=到hadoop

现在,我使用以下代码在ami上运行pig脚本。有人知道如何使用以下代码设置此配置吗

代码:


您需要做的是创建
BootstrapActionConfig
并将其添加到正在创建的
RunJobFlowRequest
,然后将自定义hadoop配置添加到集群中

以下是我在编辑代码后为您编写的完整代码:

import java.util.ArrayList;
导入java.util.List;
导入com.amazonaws.auth.AWSCredentials;
导入com.amazonaws.auth.BasicAWSCredentials;
导入com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClient;
导入com.amazonaws.services.elasticmapreduce.model.bootstrapacationconfig;
导入com.amazonaws.services.elasticmapreduce.model.JobFlowInstancesConfig;
导入com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest;
导入com.amazonaws.services.elasticmapreduce.model.RunJobFlowResult;
导入com.amazonaws.services.elasticmapreduce.model.scriptbootstrapacationconfig;
导入com.amazonaws.services.elasticmapreduce.model.StepConfig;
导入com.amazonaws.services.elasticmapreduce.util.StepFactory;
/**
* 
*@作者阿马尔
* 
*/
公共类RunEMRJobFlow{
私有静态最终字符串CONFIG_HADOOP_BOOTSTRAP_ACTION=“s3://elasticmapreduce/BOOTSTRAP actions/configure HADOOP”;
公共静态void main(字符串[]args){
字符串accessKey=“”;
字符串secretKey=“”;
AWSCredentials credentials=新的基本WSCredentials(accessKey、secretKey);
AmazonElasticMapReduceClient emr=新AmazonElasticMapReduceClient(凭据);
StepFactory StepFactory=新的StepFactory();
StepConfig enabledebugging=new StepConfig().withName(“启用调试”)
.withActionOnFailure(“终止作业流”)。withHadoopJarStep(stepFactory.newEnableDebuggingStep());
StepConfig installHive=new StepConfig().withName(“安装配置单元”).withActionOnFailure(“终止作业流”)
.withHadoopJarStep(stepFactory.newInstallHiveStep());
List setMapperArgs=new ArrayList();
setMapperArgs.add(“-s”);
setMapperArgs.add(“textinputformat.record.delimiter=;”;
BootstrapActionConfig映射程序bootstrapconfig=createBootstrapAction(“设置Hadoop配置”,
配置\u HADOOP\u引导\u操作,setMapperArgs);
RunJobFlowRequest=新的RunJobFlowRequest()
.withBootstrapActions(MapperBootstrapConfig)
.withName(“蜂巢互动”)
.with步骤(启用调试、安装配置单元)
.withLogUri(“s3://myawsbucket/”)
.举例说明(
新的JobFlowInstanceConfig().withEc2KeyName(“keypair”).withHadoopVersion(“0.20”)
.withInstanceCount(5)。WithKeepJobFlowAliveWhenNostps(true)
.使用MasterInstanceType(“m1.small”)。使用LaveInstanceType(“m1.small”);
RunJobFlowResult=emr.runJobFlow(请求);
}
私有静态BootstrapActionConfig createBootstrapAction(字符串bootstrapName、字符串bootstrapPath、,
列表参数){
ScriptBootstrapActionConfig bootstrapScriptConfig=new-ScriptBootstrapActionConfig();
bootstrappscriptconfig.setPath(bootstrapPath);
如果(args!=null){
bootstrapScriptConfig.setArgs(args);
}
BootstrapActionConfig bootstrapConfig=new BootstrapActionConfig();
bootstrapConfig.setName(bootstrapName);
bootstrapConfig.setscriptbootstrapacation(bootstrapScriptConfig);
返回bootstrapConfig;
}
}
StepConfig installPig = new StepConfig()
.withName("Install Pig")
.withActionOnFailure(ActionOnFailure.TERMINATE_JOB_FLOW.name())
.withHadoopJarStep(stepFactory.newInstallPigStep());

// [Configure pig script][1]

String[] scriptArgs = new String[] { "-p", input, "-p", output };
StepConfig runPigLatinScript = new StepConfig()
.withName("Run Pig Script")             .withActionOnFailure(ActionOnFailure.CANCEL_AND_WAIT.name())
.withHadoopJarStep(stepFactory.newRunPigScriptStep("s3://pig/script.pig", scriptArgs));

// Configure JobFlow [R1][2], [R3][3]
//
//

RunJobFlowRequest request = new RunJobFlowRequest()
.withName(jobFlowName)
.withSteps(installPig, runPigLatinScript)
.withLogUri(logUri)
.withAmiVersion("2.3.2")
.withInstances(new JobFlowInstancesConfig()
            .withEc2KeyName(this.ec2KeyName)
            .withInstanceCount(this.count)
            .withKeepJobFlowAliveWhenNoSteps(false)
            .withMasterInstanceType(this.masterType)
            .withSlaveInstanceType(this.slaveType));
// Run JobFlow
RunJobFlowResult runJobFlowResult = this.amazonEmrClient.runJobFlow(request);
import java.util.ArrayList;
import java.util.List;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClient;
import com.amazonaws.services.elasticmapreduce.model.BootstrapActionConfig;
import com.amazonaws.services.elasticmapreduce.model.JobFlowInstancesConfig;
import com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest;
import com.amazonaws.services.elasticmapreduce.model.RunJobFlowResult;
import com.amazonaws.services.elasticmapreduce.model.ScriptBootstrapActionConfig;
import com.amazonaws.services.elasticmapreduce.model.StepConfig;
import com.amazonaws.services.elasticmapreduce.util.StepFactory;

/**
 * 
 * @author amar
 * 
 */
public class RunEMRJobFlow {

    private static final String CONFIG_HADOOP_BOOTSTRAP_ACTION = "s3://elasticmapreduce/bootstrap-actions/configure-hadoop";

    public static void main(String[] args) {

        String accessKey = "";
        String secretKey = "";
        AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
        AmazonElasticMapReduceClient emr = new AmazonElasticMapReduceClient(credentials);

        StepFactory stepFactory = new StepFactory();

        StepConfig enabledebugging = new StepConfig().withName("Enable debugging")
                .withActionOnFailure("TERMINATE_JOB_FLOW").withHadoopJarStep(stepFactory.newEnableDebuggingStep());

        StepConfig installHive = new StepConfig().withName("Install Hive").withActionOnFailure("TERMINATE_JOB_FLOW")
                .withHadoopJarStep(stepFactory.newInstallHiveStep());
        List<String> setMappersArgs = new ArrayList<String>();
        setMappersArgs.add("-s");
        setMappersArgs.add("textinputformat.record.delimiter=;");

        BootstrapActionConfig mappersBootstrapConfig = createBootstrapAction("Set Hadoop Config",
                CONFIG_HADOOP_BOOTSTRAP_ACTION, setMappersArgs);

        RunJobFlowRequest request = new RunJobFlowRequest()
                .withBootstrapActions(mappersBootstrapConfig)
                .withName("Hive Interactive")
                .withSteps(enabledebugging, installHive)
                .withLogUri("s3://myawsbucket/")
                .withInstances(
                        new JobFlowInstancesConfig().withEc2KeyName("keypair").withHadoopVersion("0.20")
                                .withInstanceCount(5).withKeepJobFlowAliveWhenNoSteps(true)
                                .withMasterInstanceType("m1.small").withSlaveInstanceType("m1.small"));

        RunJobFlowResult result = emr.runJobFlow(request);
    }

    private static BootstrapActionConfig createBootstrapAction(String bootstrapName, String bootstrapPath,
            List<String> args) {

        ScriptBootstrapActionConfig bootstrapScriptConfig = new ScriptBootstrapActionConfig();
        bootstrapScriptConfig.setPath(bootstrapPath);

        if (args != null) {
            bootstrapScriptConfig.setArgs(args);
        }

        BootstrapActionConfig bootstrapConfig = new BootstrapActionConfig();
        bootstrapConfig.setName(bootstrapName);
        bootstrapConfig.setScriptBootstrapAction(bootstrapScriptConfig);

        return bootstrapConfig;
    }

}