Eclipse 在Mapreduce中进行作业链接时,如何解决chainmapper不适用于参数错误?
我正在使用Hadoop1.2.1,EclipseJuno。我试图在一个Mapreduce作业中链接三个map任务。在eclipse中编写Mapreduce代码时,我遇到了一个错误,比如chainmapper不适用于参数,并且我无法设置inputpath。下面是我的mapreduce代码Eclipse 在Mapreduce中进行作业链接时,如何解决chainmapper不适用于参数错误?,eclipse,hadoop,hadoop-streaming,hadoop2,hadoop-partitioning,Eclipse,Hadoop,Hadoop Streaming,Hadoop2,Hadoop Partitioning,我正在使用Hadoop1.2.1,EclipseJuno。我试图在一个Mapreduce作业中链接三个map任务。在eclipse中编写Mapreduce代码时,我遇到了一个错误,比如chainmapper不适用于参数,并且我无法设置inputpath。下面是我的mapreduce代码 package org.myorg; import java.io.IOException; import java.net.URI; import java.nio.file.FileSystem; imp
package org.myorg;
import java.io.IOException;
import java.net.URI;
import java.nio.file.FileSystem;
import java.util.StringTokenizer;
import javax.security.auth.login.Configuration;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.MapRunnable;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.ChainMapper;
import org.apache.hadoop.mapred.lib.ChainReducer;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.net.StaticMapping;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class Recommand extends Configured implements Tool {
public static class IdIndexMapper extends MapReduceBase implements Mapper<LongWritable, Text, Text, Text>{
public void map(LongWritable key, Text val, OutputCollector<Text, Text> output,Reporter reporter)throws IOException{
String[] ids;
String ln=val.toString();
ids=ln.split("\t");
output.collect(new Text(ids[0]),new Text(ids[1]));
}
}
public static class FtrMapper extends MapReduceBase implements Mapper<Text, Text, Text, Text>{
public void map(Text key, Text val, OutputCollector<Text, Text>output, Reporter reporter) throws IOException{
String[] str;
String lne=val.toString();
while(lne.contains("M1024")){
str=lne.split(",");
String[] str1=new String[str.length];
for(int i=0;i<str.length;i++){
if(str[i]=="M1024"){ //hre need to give id which we need to split;
continue;
}
str1[i]=str[i];
output.collect(key,new Text(str1[i]));
// System.out.println("str1 out:"+str[i]);
}
}
}
}
public static class CntMapper extends MapReduceBase implements Mapper<Text, Text, Text, IntWritable>{
private final static IntWritable one=new IntWritable(1);
private Text word=new Text();
public void map(Text key, Text val, OutputCollector<Text, IntWritable>output, Reporter reporter)throws IOException{
String line = val.toString();
StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
word.set(tokenizer.nextToken());
output.collect(word, one);
}
}
}
public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable>{
public void reduce(Text key, Iterable<IntWritable>values, OutputCollector<Text, IntWritable>output, Reporter reporter)throws IOException{
int sum=0;
for(IntWritable val:values){
sum+=val.get();
}
output.collect(key,new IntWritable(sum));
}
}
static int printUsage() {
System.out.println("recommand ");
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
public int run(String[] args) throws Exception {
JobConf conf = new JobConf(getConf(), Recommand.class);
conf.setJobName("wordcount");
if (args.length != 2) {
System.out.println("ERROR: Wrong number of parameters: " +
args.length + " instead of 2.");
return printUsage();
}
FileInputFormat.setInputPaths(conf, args[0]);
FileOutputFormat.setOutputPath(conf, new Path(args[1]));
conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(TextOutputFormat.class);
JobConf mapAConf = new JobConf(false);
ChainMapper.addMapper(conf, IdIndexMapper.class, LongWritable.class, Text.class, Text.class, Text.class, true, mapAConf);
JobConf mapBConf = new JobConf(false);
ChainMapper.addMapper(conf, FtrMapper.class, Text.class, Text.class, Text.class, Text.class, true, mapBConf);
JobConf mapCConf = new JobConf(false);
ChainMapper.addMapper(conf, CntMapper.class, Text.class, Text.class, Text.class, IntWritable.class, true, mapBConf);
JobConf reduceConf = new JobConf(false);
ChainReducer.setReducer(conf, Reduce.class, Text.class, IntWritable.class, Text.class, IntWritable.class, true, reduceConf);
JobClient.runJob(conf);
return 0;
}
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new org.apache.hadoop.conf.Configuration(), Recommand(), args);
System.exit(res);
}
}
package org.myorg;
导入java.io.IOException;
导入java.net.URI;
导入java.nio.file.FileSystem;
导入java.util.StringTokenizer;
导入javax.security.auth.login.Configuration;
导入org.apache.hadoop.classification.InterfaceAudience.Private;
导入org.apache.hadoop.conf.Configured;
导入org.apache.hadoop.fs.Path;
导入org.apache.hadoop.io.*;
导入org.apache.hadoop.mapred.FileOutputFormat;
导入org.apache.hadoop.mapred.JobClient;
导入org.apache.hadoop.mapred.JobConf;
导入org.apache.hadoop.mapred.MapReduceBase;
导入org.apache.hadoop.mapred.MapRunnable;
导入org.apache.hadoop.mapred.OutputCollector;
导入org.apache.hadoop.mapred.Reducer;
导入org.apache.hadoop.mapred.Reporter;
导入org.apache.hadoop.mapred.TextInputFormat;
导入org.apache.hadoop.mapred.TextOutputFormat;
导入org.apache.hadoop.mapred.lib.ChainMapper;
导入org.apache.hadoop.mapred.lib.ChainReducer;
导入org.apache.hadoop.mapreduce.Job;
导入org.apache.hadoop.mapreduce.Mapper;
导入org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
导入org.apache.hadoop.net.StaticMapping;
导入org.apache.hadoop.util.Tool;
导入org.apache.hadoop.util.ToolRunner;
公共类建议扩展配置的实现工具{
公共静态类IdIndexMapper扩展MapReduceBase实现Mapper{
公共void映射(LongWritable键、文本val、OutputCollector输出、Reporter报告器)引发IOException{
字符串[]id;
字符串ln=val.toString();
ids=ln.split(“\t”);
collect(新文本(ids[0])、新文本(ids[1]);
}
}
公共静态类FtrMapper扩展MapReduceBase实现映射器{
公共无效映射(文本键、文本值、OutputCollectoroutput、Reporter)引发IOException{
字符串[]str;
字符串lne=val.toString();
而(lne.包含(“M1024”)){
str=lne.split(“,”);
字符串[]str1=新字符串[str.length];
对于(int i=0;i确保以下内容以避免此错误
这两个类都扩展了Mapper
类
使用的ChainMapper
类来自正确的API,以适用于代码的API为准
org.apache.hadoop.mapreduce.lib.chain.ChainMapper
或importorg.apache.hadoop.mapred.lib.ChainMapper
您能发布错误消息吗?第一条错误消息出现在“addmapper”上,即“addmapper方法(JobConf,ClassTry)尝试这篇文章的答案(可能重复):是的,我尝试了与你给出的相同的内容。当我在eclipse中尝试该代码时,它显示了我所说的错误,在ChainMapper.addMapper的行上。我不知道如何解决它们。我不这么认为……上面提到的帖子的答案是importsorg.apache.hadoop.mapred.Mapper;
而不是org.apache.hadoop.mapreduce.Mapper;
,就像你一样。你是吗那是什么?