Warning: file_get_contents(/data/phpspider/zhask/data//catemap/9/java/367.json): failed to open stream: No such file or directory in /data/phpspider/zhask/libs/function.php on line 167

Warning: Invalid argument supplied for foreach() in /data/phpspider/zhask/libs/tag.function.php on line 1116

Notice: Undefined index: in /data/phpspider/zhask/libs/function.php on line 180

Warning: array_chunk() expects parameter 1 to be array, null given in /data/phpspider/zhask/libs/function.php on line 181
Java Mapreduce错误:输出地图数据而不是Reduce_Java_Hadoop_Mapreduce - Fatal编程技术网

Java Mapreduce错误:输出地图数据而不是Reduce

Java Mapreduce错误:输出地图数据而不是Reduce,java,hadoop,mapreduce,Java,Hadoop,Mapreduce,MapReduce从映射器而不是reducer输出键值对,有什么可能的原因吗 我正在使用Hadoop运行MapReduceJava程序。我已将映射器设置为输出值为8或9的IntWritables。我已经将减速器的输出值设置为5或17 我最终得到以下结果: AAKASH LAKSHMANAN 9 AALIYAH HARRISON 9 AARON CARMACK 9 AARON CRAIG 9 AARON GOLD 9 AARON LAWSON 9 AARON LEVINSON

MapReduce从映射器而不是reducer输出键值对,有什么可能的原因吗

我正在使用Hadoop运行MapReduceJava程序。我已将映射器设置为输出值为8或9的IntWritables。我已经将减速器的输出值设置为5或17

我最终得到以下结果:

AAKASH LAKSHMANAN   9
AALIYAH HARRISON    9
AARON CARMACK   9
AARON CRAIG 9
AARON GOLD  9
AARON LAWSON    9
AARON LEVINSON  9
AARON NAZIR 9

.....
很明显,映射器函数中的值。输出数据中的键也没有组合

完整代码如下:

//Task 3: List of people who visited at least once in both 2009 and 2010.

package org.myorg;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.*;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class Task3 {

public static class Map
       extends Mapper<LongWritable, Text, Text, IntWritable> {

    protected void setup(Context context) throws IOException, InterruptedException {
    }

    public void map(LongWritable key,
                    Text value,
                    Context context) throws IOException {
        try{

          String line = value.toString();

          String[] lineArray = line.split(",");
          String yearRaw;
          String name;
          name = lineArray[1] + " " + lineArray[0];
          yearRaw = lineArray[10];

          String[] year1Arr = yearRaw.split("[ /]");
          int y1, y2;

          if (year1Arr.length==4 && year1Arr[2] != null) {
             y1 = Integer.parseInt(year1Arr[2]);
          } else {
            return;
          }

          String year2Raw = lineArray[11];
          String[] year2Arr = year2Raw.split("[ /]");
          if (year2Arr.length > 3 && year2Arr[2] != null) {
             y2 = Integer.parseInt(year2Arr[2]);
          } else {
            return;
          }

          if ((y1 == 2009) || (y2 == 2009)) {
             context.write(new Text(name), new IntWritable(8));
          }

          if ((y1 == 2010) || (y2 == 2010)) {
             context.write(new Text(name), new IntWritable(9));
          }
        } catch(InterruptedException e) {
          System.out.println("Interrupted Exception");
        }
    }

    protected void cleanup(Context context) throws IOException, InterruptedException {
    }
}


public static class Reduce
              extends Reducer<Text, IntWritable, Text, IntWritable> {

    protected void setup(Context context) throws IOException, InterruptedException {
    }
    public void reduce(Text key,
                       Iterator<IntWritable> values,
                       Context context) throws IOException {

        try {
          int y1 = 0;
          int y2 = 0;

          IntWritable value;

          while (values.hasNext()) {
              value = values.next();

              if (value.get() == 8) {
                 y1 += 1;
              } else if (value.get() == 9) {
                y2 += 1;
              }

          }

          if ((y1 + y2) > 1) {
             context.write(key, new IntWritable(5));
          } else {
            context.write(key, new IntWritable(17));
          }

        }catch (InterruptedException e) {
            System.out.println("Interrupted Exception");
        }
    }

    protected void cleanup(Context context) throws IOException, InterruptedException {
    }
}



public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: task3 <in> <out>");
        System.exit(2);
    }
    // Creates a MapReduce job and links it to our class
    Job job = Job.getInstance(conf);
    job.setJarByClass(Task3.class);
    conf.set("mapreduce.output.textoutputformat.separator", ",");

    // Selects mapper/combiner/reducer
    job.setMapperClass(Map.class);
    //job.setCombinerClass(Reduce.class);
    job.setReducerClass(Reduce.class);

    // This says that (k1, v1) should be read from text files
    // and that (k3, v3) should be written to text files
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    // The paths of these input/output are from application arguments
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    // Finally, run the job!
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}
//任务3:2009年和2010年至少访问过一次的人员列表。
包org.myorg;
导入java.io.DataInput;
导入java.io.DataOutput;
导入java.io.IOException;
导入java.util.*;
导入org.apache.hadoop.conf.Configuration;
导入org.apache.hadoop.fs.Path;
导入org.apache.hadoop.io.IntWritable;
导入org.apache.hadoop.io.Text;
导入org.apache.hadoop.io.LongWritable;
导入org.apache.hadoop.io.writeable;
导入org.apache.hadoop.mapreduce.Job;
导入org.apache.hadoop.mapreduce.Mapper;
导入org.apache.hadoop.mapreduce.Reducer;
导入org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
导入org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
导入org.apache.hadoop.util.GenericOptionsParser;
公开课任务3{
公共静态类映射
扩展映射器{
受保护的无效设置(上下文上下文)引发IOException、InterruptedException{
}
公共无效映射(可长写密钥,
文本值,
上下文)抛出IOException{
试一试{
字符串行=value.toString();
String[]lineArray=line.split(“,”);
细绳年画;
字符串名;
name=lineArray[1]+“”+lineArray[0];
yearRaw=线性阵列[10];
字符串[]year1Arr=yearRaw.split([/]);
int y1,y2;
如果(year1Arr.length==4&&year1Arr[2]!=null){
y1=整数.parseInt(year1Arr[2]);
}否则{
返回;
}
字符串year2Raw=lineArray[11];
字符串[]year2Arr=year2Raw.split([/]);
如果(year2Arr.length>3&&year2Arr[2]!=null){
y2=整数.parseInt(year2Arr[2]);
}否则{
返回;
}
如果((y1==2009)| |(y2==2009)){
write(新文本(名称),新intwriteable(8));
}
如果((y1==2010)| |(y2==2010)){
write(新文本(名称),新intwriteable(9));
}
}捕捉(中断异常e){
System.out.println(“中断异常”);
}
}
受保护的无效清理(上下文上下文)引发IOException、InterruptedException{
}
}
公共静态类减少
伸缩减速机{
受保护的无效设置(上下文上下文)引发IOException、InterruptedException{
}
public void reduce(文本键,
迭代器值,
上下文)抛出IOException{
试一试{
int y1=0;
int y2=0;
可写值;
while(values.hasNext()){
value=values.next();
if(value.get()==8){
y1+=1;
}else if(value.get()==9){
y2+=1;
}
}
如果((y1+y2)>1){
write(key,新的intwriteable(5));
}否则{
write(key,新的intwriteable(17));
}
}捕捉(中断异常e){
System.out.println(“中断异常”);
}
}
受保护的无效清理(上下文上下文)引发IOException、InterruptedException{
}
}
公共静态void main(字符串[]args)引发异常{
Configuration conf=新配置();
String[]otherArgs=新的GenericOptionsParser(conf,args);
if(otherArgs.length!=2){
System.err.println(“用法:task3”);
系统出口(2);
}
//创建MapReduce作业并将其链接到我们的类
Job Job=Job.getInstance(conf);
job.setJarByClass(Task3.class);
conf.set(“mapreduce.output.textoutputformat.separator”,“,”);
//选择映射器/合并器/还原器
job.setMapperClass(Map.class);
//job.setCombinerClass(Reduce.class);
job.setReducerClass(Reduce.class);
//这意味着(k1,v1)应该从文本文件中读取
//(k3,v3)应该写入文本文件
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
//这些输入/输出的路径来自应用程序参数
addInputPath(作业,新路径(其他参数[0]);
setOutputPath(作业,新路径(其他参数[1]);
//最后,运行作业!
系统退出(作业等待完成(真)?0:1;
}

}

//job.setCombinerClass(Reduce.class)?我刚刚试过,但是我仍然得到映射器值作为我的结果。“字数”对你有用吗?我发现它应该是“迭代器”,而不是reduce函数中的“iterable”,因为我使用的是新的api。谢谢你的帮助!例如,使用reduce和map的@Override装饰器(decorator)可以很方便地确保您实际上正在重写您认为是的方法。