Warning: file_get_contents(/data/phpspider/zhask/data//catemap/9/java/392.json): failed to open stream: No such file or directory in /data/phpspider/zhask/libs/function.php on line 167

Warning: Invalid argument supplied for foreach() in /data/phpspider/zhask/libs/tag.function.php on line 1116

Notice: Undefined index: in /data/phpspider/zhask/libs/function.php on line 180

Warning: array_chunk() expects parameter 1 to be array, null given in /data/phpspider/zhask/libs/function.php on line 181
Java orPlatformA+=intValueOfValue; System.out.println(“键='A'”+“值:”+intValueOfValue); System.out.println(“计数器A:+反格式A+”| | |计数器B:+反格式B+”| |--| |”); } else if(keyOfValue.equals(“B”){//如果值来自平台B 反形态B+=intValueOfValue; System.out.println(“键='B'”+“值:”+intValueOfValue); System.out.println(“计数器A:+反格式A+”| | |计数器B:+反格式B+”| |--| |”); } 否则{ //错误 System.out.println(“不等于A或B”); }*/ } 总和+=intValueOfValue; } 写(键,新文本(总和)); } } 公共静态void main(字符串[]args)引发异常{ 如果(参数长度!=3){ System.err.println(“用法:>”; 系统出口(0); } Configuration conf=新配置(); String[]files=new-GenericOptionsParser(conf,args).getRemainingArgs(); 路径输入1=新路径(文件[0]); 路径输入2=新路径(文件[1]); 路径输出=新路径(文件[2]); //如果输出已经存在->删除它 FileSystem fs=FileSystem.get(conf); 如果(fs.存在(输出)){ fs.delete(输出,true); } Job Job=Job.getInstance(conf,“字数”); job.setJarByClass(WordCount.class); setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); addInputPath(作业、input1、TextInputFormat.class); addInputPath(作业、input2、TextInputFormat.class); setOutputPath(作业,输出); 系统退出(作业等待完成(真)?0:1; } }_Java_Hadoop_Mapreduce - Fatal编程技术网

Java orPlatformA+=intValueOfValue; System.out.println(“键='A'”+“值:”+intValueOfValue); System.out.println(“计数器A:+反格式A+”| | |计数器B:+反格式B+”| |--| |”); } else if(keyOfValue.equals(“B”){//如果值来自平台B 反形态B+=intValueOfValue; System.out.println(“键='B'”+“值:”+intValueOfValue); System.out.println(“计数器A:+反格式A+”| | |计数器B:+反格式B+”| |--| |”); } 否则{ //错误 System.out.println(“不等于A或B”); }*/ } 总和+=intValueOfValue; } 写(键,新文本(总和)); } } 公共静态void main(字符串[]args)引发异常{ 如果(参数长度!=3){ System.err.println(“用法:>”; 系统出口(0); } Configuration conf=新配置(); String[]files=new-GenericOptionsParser(conf,args).getRemainingArgs(); 路径输入1=新路径(文件[0]); 路径输入2=新路径(文件[1]); 路径输出=新路径(文件[2]); //如果输出已经存在->删除它 FileSystem fs=FileSystem.get(conf); 如果(fs.存在(输出)){ fs.delete(输出,true); } Job Job=Job.getInstance(conf,“字数”); job.setJarByClass(WordCount.class); setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); addInputPath(作业、input1、TextInputFormat.class); addInputPath(作业、input2、TextInputFormat.class); setOutputPath(作业,输出); 系统退出(作业等待完成(真)?0:1; } }

Java orPlatformA+=intValueOfValue; System.out.println(“键='A'”+“值:”+intValueOfValue); System.out.println(“计数器A:+反格式A+”| | |计数器B:+反格式B+”| |--| |”); } else if(keyOfValue.equals(“B”){//如果值来自平台B 反形态B+=intValueOfValue; System.out.println(“键='B'”+“值:”+intValueOfValue); System.out.println(“计数器A:+反格式A+”| | |计数器B:+反格式B+”| |--| |”); } 否则{ //错误 System.out.println(“不等于A或B”); }*/ } 总和+=intValueOfValue; } 写(键,新文本(总和)); } } 公共静态void main(字符串[]args)引发异常{ 如果(参数长度!=3){ System.err.println(“用法:>”; 系统出口(0); } Configuration conf=新配置(); String[]files=new-GenericOptionsParser(conf,args).getRemainingArgs(); 路径输入1=新路径(文件[0]); 路径输入2=新路径(文件[1]); 路径输出=新路径(文件[2]); //如果输出已经存在->删除它 FileSystem fs=FileSystem.get(conf); 如果(fs.存在(输出)){ fs.delete(输出,true); } Job Job=Job.getInstance(conf,“字数”); job.setJarByClass(WordCount.class); setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); addInputPath(作业、input1、TextInputFormat.class); addInputPath(作业、input2、TextInputFormat.class); setOutputPath(作业,输出); 系统退出(作业等待完成(真)?0:1; } },java,hadoop,mapreduce,Java,Hadoop,Mapreduce,它看起来像是您的减速机接收一对文本对象并输出文本。如果是这种情况,您可能会遇到一些问题: 在main中,您有: job.setOutputValueClass(IntWritable.class)可能是job.setOutputValueClass(Text.class) 您还将减速器定义为: 公共静态类IntSumReducer扩展Reducer它应该是公共静态类IntSumReducer扩展Reducer 还原程序正在接收文本值,而不是IntWritables。最终是合并程序。如果您也将re

它看起来像是您的减速机接收一对文本对象并输出文本。如果是这种情况,您可能会遇到一些问题:

main
中,您有:

job.setOutputValueClass(IntWritable.class)
可能是
job.setOutputValueClass(Text.class)

您还将减速器定义为:

公共静态类IntSumReducer扩展Reducer
它应该是
公共静态类IntSumReducer扩展Reducer


还原程序正在接收文本值,而不是IntWritables。

最终是合并程序。如果您也将reducer设置为组合器,那么映射器和reducer之间就不能有不同的类型

import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.Partitioner;

public class WordCount {

  public static class TokenizerMapper
  extends Mapper<Object, Text, Text, Text>{

  //private final static IntWritable result = new IntWritable();
  private Text word = new Text();


public void map(Object key, Text value, Context context
                ) throws IOException, InterruptedException {
  StringTokenizer itr = new StringTokenizer(value.toString(),"\n");
  while (itr.hasMoreTokens()) {

    String nextWord = itr.nextToken().toUpperCase();

    //System.out.println("'"+nextWord+"'");
    if(isFromPlatformB(nextWord)){
    //Procedure for words of Platform B.
        String[] split1 = nextWord.split("(,)|(/)|(\\s)");
        String seriesTitle = split1[0];
        String numOfMonth = split1[2];
        String numOfDay = split1[3];
        String number = split1[4];//VALUE

        int monthInt = Integer.parseInt(numOfMonth);
        String monthString;

        switch (monthInt) {
            case 1:  monthString = "JAN";
                 break;
            case 2:  monthString = "FEB";
                 break;
            case 3:  monthString = "MAR";
                 break;
            case 4:  monthString = "APR";
                 break;
            case 5:  monthString = "MAY";
                 break;
            case 6:  monthString = "JUN";
                 break;
            case 7:  monthString = "JUL";
                 break;
            case 8:  monthString = "AUG";
                 break;
            case 9:  monthString = "SEP";
                 break;
            case 10: monthString = "OCT";
                 break;
            case 11: monthString = "NOV";
                 break;
            case 12: monthString = "DEC";
                 break;
            default: monthString = "ERROR";
                 break;
            }

         //result.set(numberInt);
         word.set(monthString + " " + seriesTitle);
         System.out.println("key: "+monthString + " " + seriesTitle + ",  value: "+number+"/B");
         context.write(word, new Text(number + "/B"));
         //FORMAT : <KEY,VALUE/B>
    }
    else{
         //Procedure for words of Platform A.
         String[] split5 = nextWord.split("(-)|( )|(,)");
         String month = split5[0];
         String seriesTitle = split5[2];
         String value2 = split5[3];//OUTVALUE
         String finalWord = month + " " + seriesTitle;//OUTKEY   KEY: <APR #WESTWORLD>

         word.set(finalWord);
         //result.set(valueInt);
         System.out.println("key: "+finalWord + ",  value: "+value2+"/A");
         context.write(word, new Text(value2 + "/A"));
         //FORMAT : <KEY,VALUE/A>
    } 
  }
}

 /*
 *This method takes the next token and returns true if the token is taken from platform B file,
 *Or it returns false if the token comes from platform A file.
 *
 */
 public boolean isFromPlatformB(String nextToken){
   // B platform has the form of : "#WestWorld ,2017/1/2){
   if(nextToken.charAt(0) == '#'){  
      return true;
   }
   return false;
}
}

  public static class IntSumReducer
   extends Reducer<Text,IntWritable,Text,Text> {
//private IntWritable result = new IntWritable();

public void reduce(Text key, Iterable<Text> values,
                   Context context
                   ) throws IOException, InterruptedException {
  int sum = 0;
  for (Text val : values) {
      String valToString = val.toString();

      String[] split = valToString.split("/");
      //String keyOfValue;
      String valueOfValue;
      int intValueOfValue = 0;

      // FORMAT : <KEY,VALUE/platform>  [<KEY,VALUE>,VALUE = <key,value>]
      //                 [0]      [1]

      if(split.length>1){
             //keyOfValue = split[1];
          valueOfValue = split[0];
          //System.out.println(key);
          //System.out.println(valueOfValue);
          //System.out.println(keyOfValue);
          intValueOfValue = Integer.parseInt(valueOfValue);
          /*if(keyOfValue.equals("A")){//If value is from platform A
              counterForPlatformA += intValueOfValue;
              System.out.println("KEY = 'A' " + "VALUE :"     +intValueOfValue);
              System.out.println("counter A: "+ counterForPlatformA +"||     counter B: "+ counterForPlatformB + "||----||");
          }
          else if(keyOfValue.equals("B")){//If value is from platform B
                 counterForPlatformB += intValueOfValue;
                 System.out.println("KEY = 'B' " + "VALUE :" +intValueOfValue);
                 System.out.println("counter A: "+ counterForPlatformA +"|| counter B: "+ counterForPlatformB + "||----||");
              }
              else{
                    //ERROR
                    System.out.println("Not equal to A or B");
                  }*/

      }
      sum += intValueOfValue;
  }
  context.write(key, new Text(sum));
  }
  }

public static void main(String[] args) throws Exception{

 if (args.length != 3 ){
    System.err.println ("Usage :<inputlocation1> <inputlocation2>   <outputlocation> >");
    System.exit(0);
}
Configuration conf = new Configuration();
String[] files=new GenericOptionsParser(conf,args).getRemainingArgs();
Path input1=new Path(files[0]);
Path input2=new Path(files[1]);
Path output=new Path(files[2]);
//If OUTPUT already exists -> Delete it
FileSystem fs = FileSystem.get(conf);
if(fs.exists(output)){
    fs.delete(output, true);
} 

Job job = Job.getInstance(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
MultipleInputs.addInputPath(job, input1, TextInputFormat.class);
MultipleInputs.addInputPath(job, input2, TextInputFormat.class);
FileOutputFormat.setOutputPath(job, output);
System.exit(job.waitForCompletion(true) ? 0 : 1);

}
}