Java 如何将减速器设置为emmit<;文本,IntWritable>;和一个映射器来接收<;文本,IntWritable>;?
我正在用mapreduce在hadoop上开发一些代码,它使用两个映射器和两个还原器。 我被告知使用SequenceFileInputFormat和SequenceFileOutputFormat使第一个减速机的输出和第二个映射器的输入协同工作。 问题是我遇到了一个错误,在谷歌搜索了很多次之后,我不知道为什么 错误:Java 如何将减速器设置为emmit<;文本,IntWritable>;和一个映射器来接收<;文本,IntWritable>;?,java,hadoop,mapreduce,Java,Hadoop,Mapreduce,我正在用mapreduce在hadoop上开发一些代码,它使用两个映射器和两个还原器。 我被告知使用SequenceFileInputFormat和SequenceFileOutputFormat使第一个减速机的输出和第二个映射器的输入协同工作。 问题是我遇到了一个错误,在谷歌搜索了很多次之后,我不知道为什么 错误: package casoTaxis; import java.io.IOException; import java.util.StringTokenizer; import
package casoTaxis;
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
public class Eje1{
public static class MapperJob1 extends Mapper<Object, Text, Text, IntWritable> {
//El metodo map recibe un conjunto clave-valor, lo procesa y lo vuelca en un contexto.adasdadada
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
Text hackLicense; IntWritable totalAmount; //salidas
StringTokenizer itr = new StringTokenizer(value.toString(), ",");
itr.nextToken();
hackLicense = new Text(itr.nextToken());
for(int i=2; i<itr.countTokens(); i++) itr.nextToken();
totalAmount = new IntWritable( Integer.parseInt(itr.nextToken()) );
context.write(hackLicense, totalAmount);
}
}
public static class ReducerJob1 extends Reducer<Text, IntWritable, Text, IntWritable> { //No encontre una clase InpuFormat que sea Text, IntWritable
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
}
}
public static class MapperJob2 extends Mapper<Text, IntWritable, Text, IntWritable> {
//El metodo map recibe un conjunto clave-valor, lo procesa y lo vuelca en un contexto.adasdadada
public void map(Text key, IntWritable value, Context context) throws IOException, InterruptedException {
context.write(key, value);
}
}
public static class ReducerJob2 extends Reducer<Text, IntWritable, Text, Text> {
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int max = 0;
for (IntWritable val : values) {
int maxVal = val.get();
if( maxVal>max ) max = maxVal;
}
String licencia = "Conductor con licencia = " + key;
String recaudacion = "Recaudacion = " + max;
context.write(new Text(licencia), new Text(recaudacion));
}
}
public static void main(String[] args) throws Exception {
Configuration conf1 = new Configuration();
Configuration conf2 = new Configuration();
//conf2.set("mapreduce.input.keyvaluelinerecordreader.key.value.separator", " ");
Job job1 = Job.getInstance(conf1, "Eje1-Job1");
Job job2 = Job.getInstance(conf2, "Eje1-Job2");
job1.setJarByClass(Eje1.class);
job2.setJarByClass(Eje1.class);
job1.setMapperClass(MapperJob1.class);
job2.setMapperClass(MapperJob2.class);
job1.setReducerClass(ReducerJob1.class);
job2.setReducerClass(ReducerJob2.class);
job1.setMapOutputKeyClass(Text.class);
job1.setMapOutputValueClass(IntWritable.class);
job1.setOutputKeyClass(Text.class);
job1.setOutputValueClass(IntWritable.class);
job2.setMapOutputKeyClass(Text.class);
job2.setMapOutputKeyClass(IntWritable.class);
job2.setOutputKeyClass(Text.class);
job2.setOutputValueClass(Text.class);
job1.setOutputFormatClass(SequenceFileOutputFormat.class);
job2.setInputFormatClass(SequenceFileInputFormat.class);///asdasdads
FileInputFormat.addInputPath(job1, new Path(args[0]));
FileOutputFormat.setOutputPath(job1, pathIntermedio);
FileInputFormat.addInputPath(job2, pathIntermedio);
FileOutputFormat.setOutputPath(job2, new Path(args[1]));
job1.waitForCompletion(true);
System.exit(job2.waitForCompletion(true) ? 0 : 1);
}
private static final Path pathIntermedio = new Path("intermediate_output");
}
java.lang.Exception:java.io.IOException:map中的键类型不匹配:预期的org.apache.hadoop.io.可写的,收到的org.apache.hadoop.io.文本
映射中的键类型不匹配:应为
org.apache.hadoop.io.intwriteable,已收到org.apache.hadoop.io.文本
代码:
package casoTaxis;
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
public class Eje1{
public static class MapperJob1 extends Mapper<Object, Text, Text, IntWritable> {
//El metodo map recibe un conjunto clave-valor, lo procesa y lo vuelca en un contexto.adasdadada
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
Text hackLicense; IntWritable totalAmount; //salidas
StringTokenizer itr = new StringTokenizer(value.toString(), ",");
itr.nextToken();
hackLicense = new Text(itr.nextToken());
for(int i=2; i<itr.countTokens(); i++) itr.nextToken();
totalAmount = new IntWritable( Integer.parseInt(itr.nextToken()) );
context.write(hackLicense, totalAmount);
}
}
public static class ReducerJob1 extends Reducer<Text, IntWritable, Text, IntWritable> { //No encontre una clase InpuFormat que sea Text, IntWritable
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
}
}
public static class MapperJob2 extends Mapper<Text, IntWritable, Text, IntWritable> {
//El metodo map recibe un conjunto clave-valor, lo procesa y lo vuelca en un contexto.adasdadada
public void map(Text key, IntWritable value, Context context) throws IOException, InterruptedException {
context.write(key, value);
}
}
public static class ReducerJob2 extends Reducer<Text, IntWritable, Text, Text> {
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int max = 0;
for (IntWritable val : values) {
int maxVal = val.get();
if( maxVal>max ) max = maxVal;
}
String licencia = "Conductor con licencia = " + key;
String recaudacion = "Recaudacion = " + max;
context.write(new Text(licencia), new Text(recaudacion));
}
}
public static void main(String[] args) throws Exception {
Configuration conf1 = new Configuration();
Configuration conf2 = new Configuration();
//conf2.set("mapreduce.input.keyvaluelinerecordreader.key.value.separator", " ");
Job job1 = Job.getInstance(conf1, "Eje1-Job1");
Job job2 = Job.getInstance(conf2, "Eje1-Job2");
job1.setJarByClass(Eje1.class);
job2.setJarByClass(Eje1.class);
job1.setMapperClass(MapperJob1.class);
job2.setMapperClass(MapperJob2.class);
job1.setReducerClass(ReducerJob1.class);
job2.setReducerClass(ReducerJob2.class);
job1.setMapOutputKeyClass(Text.class);
job1.setMapOutputValueClass(IntWritable.class);
job1.setOutputKeyClass(Text.class);
job1.setOutputValueClass(IntWritable.class);
job2.setMapOutputKeyClass(Text.class);
job2.setMapOutputKeyClass(IntWritable.class);
job2.setOutputKeyClass(Text.class);
job2.setOutputValueClass(Text.class);
job1.setOutputFormatClass(SequenceFileOutputFormat.class);
job2.setInputFormatClass(SequenceFileInputFormat.class);///asdasdads
FileInputFormat.addInputPath(job1, new Path(args[0]));
FileOutputFormat.setOutputPath(job1, pathIntermedio);
FileInputFormat.addInputPath(job2, pathIntermedio);
FileOutputFormat.setOutputPath(job2, new Path(args[1]));
job1.waitForCompletion(true);
System.exit(job2.waitForCompletion(true) ? 0 : 1);
}
private static final Path pathIntermedio = new Path("intermediate_output");
}
packasotaxis;
导入java.io.IOException;
导入java.util.StringTokenizer;
导入org.apache.hadoop.conf.Configuration;
导入org.apache.hadoop.fs.Path;
导入org.apache.hadoop.io.IntWritable;
导入org.apache.hadoop.io.Text;
导入org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
导入org.apache.hadoop.mapreduce.Job;
导入org.apache.hadoop.mapreduce.Mapper;
导入org.apache.hadoop.mapreduce.Reducer;
导入org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
导入org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
导入org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
公共类Eje1{
公共静态类MapperJob1扩展了Mapper{
//这幅地图展现了我们的勇气,我们的过程和经验
公共void映射(对象键、文本值、上下文上下文)引发IOException、InterruptedException{
Text-hackLicense;IntWritable totalAmount;//salidas
StringTokenizer itr=新的StringTokenizer(value.toString(),“,”);
itr.nextToken();
hackLicense=新文本(itr.nextToken());
对于(inti=2;imax)max=maxVal;
}
字符串licencia=“导线con licencia=“+键;
字符串recaudacion=“recaudacion=“+max;
编写(新文本(许可证)、新文本(再许可证));
}
}
公共静态void main(字符串[]args)引发异常{
Configuration conf1=新配置();
Configuration conf2=新配置();
//conf2.set(“mapreduce.input.keyvaluelinerecordreader.key.value.separator”,”;
Job job1=Job.getInstance(conf1,“Eje1-job1”);
Job job2=Job.getInstance(conf2,“Eje1-job2”);
job1.setJarByClass(Eje1.class);
job2.setJarByClass(Eje1.class);
job1.setMapperClass(MapperJob1.class);
job2.setMapperClass(MapperJob2.class);
job1.setReducerClass(ReducerJob1.class);
job2.setReducerClass(ReducerJob2.class);
job1.setMapOutputKeyClass(Text.class);
job1.setMapOutputValueClass(IntWritable.class);
job1.setOutputKeyClass(Text.class);
job1.setOutputValueClass(IntWritable.class);
job2.setMapOutputKeyClass(Text.class);
job2.setMapOutputKeyClass(IntWritable.class);
job2.setOutputKeyClass(Text.class);
job2.setOutputValueClass(Text.class);
job1.setOutputFormatClass(SequenceFileOutputFormat.class);
job2.setInputFormatClass(SequenceFileInputFormat.class);///asdasdad
addInputPath(作业1,新路径(args[0]);
setOutputPath(job1,pathIntermedio);
addInputPath(作业2,路径中介);
setOutputPath(作业2,新路径(args[1]);
作业1.等待完成(真);
系统退出(作业2.等待完成(真)?0:1;
}
私有静态最终路径pathIntermedio=新路径(“中间_输出”);
}
我为什么会收到此错误?有没有更好的方法来实现这一点?错误在于行
job2.setMapOutputKeyClass(Text.class);
job2.setMapOutputKeyClass(IntWritable.class);
其中第二个应该是:
job2.setMapOutputValueClass(IntWritable.class);