Java Hadoop MapReduce不写入输出
我创建了一个文件并添加了一些数字,如10、20、220和228。我想在下面的mapper函数中读取这个文件,并检查一个数字是否友好。但是在编译类文件并构建jar之后,输出文件中没有任何内容Java Hadoop MapReduce不写入输出,java,hadoop,Java,Hadoop,我创建了一个文件并添加了一些数字,如10、20、220和228。我想在下面的mapper函数中读取这个文件,并检查一个数字是否友好。但是在编译类文件并构建jar之后,输出文件中没有任何内容 public class FriendlyNumbers { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job
public class FriendlyNumbers {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "befriended numbers");
job.setJarByClass(FriendlyNumbers.class);
job.setMapperClass(FriendlyNumberMapper.class);
// job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(FriendlyNumberKeywordReducer.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(NumberCouple.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
class FriendlyNumberMapper extends Mapper<Object, Text, IntWritable, NumberCouple> {
// process all the input data
// the data come's from the file file0
private IntWritable number = new IntWritable(); // number from file
private IntWritable sum = new IntWritable(); // number from calculateSum()
private NumberCouple numberCouple = new NumberCouple();
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
StringTokenizer numberTokens = new StringTokenizer(value.toString());
// loop trough all given numbers
while (numberTokens.hasMoreTokens()) {
int parsedNumberToken = Integer.parseInt(numberTokens.nextToken());
int calculatedSum = calculateSum(parsedNumberToken);
// set stuff
number.set(parsedNumberToken);
sum.set(calculatedSum);
numberCouple.set(number, sum);
context.write(sum, numberCouple);
if (number.get() != sum.get()) {
context.write(number, numberCouple);
}
}
}
// the actual sum to check if a number is amicable
public int calculateSum(int number) {
int sum = 0;
for (int i = 1; i <= number / 2; i++) {
if (number % i == 0) {
sum += i;
}
}
return sum;
}
}
class FriendlyNumberKeywordReducer extends Reducer<IntWritable, NumberCouple, IntWritable, IntWritable> {
// combine data
// in this case: get only the befriended numbers and remove others
public void reduce(IntWritable key, Iterable<NumberCouple> values, Context context) throws IOException, InterruptedException {
//
}
}
class NumberCouple implements WritableComparable<NumberCouple> {
private IntWritable number;
private IntWritable sum;
public NumberCouple() {
set(new IntWritable(), new IntWritable());
}
public NumberCouple(NumberCouple couple) {
set(new IntWritable(couple.number.get()), new IntWritable(couple.sum.get()));
}
public NumberCouple(int number, int sum) {
set(new IntWritable(number), new IntWritable(sum));
}
public void set(IntWritable number, IntWritable sum) {
this.number = number;
this.sum = sum;
}
public IntWritable getNumber() {
return this.number;
}
public IntWritable getSum() {
return this.sum;
}
@Override
public void write(DataOutput out) throws IOException {
number.write(out);
sum.write(out);
}
@Override
public void readFields(DataInput in) throws IOException {
number.readFields(in);
sum.readFields(in);
}
@Override
public int compareTo(NumberCouple o) {
return number.compareTo(o.number);
}
}
公共类FriendlyNumber{
公共静态void main(字符串[]args)引发异常{
Configuration conf=新配置();
Job Job=Job.getInstance(conf,“befriended number”);
job.setJarByClass(FriendlyNumbers.class);
setMapperClass(FriendlyNumberMapper.class);
//job.setCombinerClass(IntSumReducer.class);
setReducerClass(FriendlyNumber关键字Reducer.class);
setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(numberCoupleClass);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(IntWritable.class);
setInputPaths(作业,新路径(args[0]);
setOutputPath(作业,新路径(args[1]);
系统退出(作业等待完成(真)?0:1;
}
}
类FriendlyNumber映射器扩展映射器{
//处理所有输入数据
//数据来自文件0
private intwriteable number=new intwriteable();//文件中的编号
private intwriteable sum=new intwriteable();//calculateSum()中的数字
私有NumberCouple NumberCouple=新的NumberCouple();
公共void映射(对象键、文本值、上下文上下文)引发IOException、InterruptedException{
StringTokenizer numberTokens=新的StringTokenizer(value.toString());
//循环所有给定的数字
while(numberTokens.hasMoreTokens()){
int parsedNumberToken=Integer.parseInt(numberTokens.nextToken());
int calculatedSum=calculateSum(parsedNumberToken);
//布景
number.set(parsedNumberToken);
sum.set(计算总和);
numberCouple.set(数字、和);
write(和,数对);
if(number.get()!=sum.get()){
context.write(number,numberCouple);
}
}
}
//检查数字是否友好的实际金额
公共整数计算(整数){
整数和=0;
对于(int i=1;i,因为您没有将numReduceTask设置为“0”,所以它将转到Reducer并尝试运行reduce任务
所以,如果您想运行仅映射作业,请将numReduceTask设置为“0”。您不需要设置ReducerClass。在驱动程序类中使用以下命令
Job job = Job.getInstance(conf, "befriended numbers");
// Set this property to Zero to run map-only job
job.setNumReduceTasks(0);
job.setJarByClass(FriendlyNumbers.class);
job.setMapperClass(FriendlyNumberMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(NumberCouple.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(IntWritable.class);
由于您没有将numReduceTask设置为“0”,因此它将转到Reducer并尝试运行reduce任务
所以,如果您想运行仅映射作业,请将numReduceTask设置为“0”。您不需要设置ReducerClass。在驱动程序类中使用以下命令
Job job = Job.getInstance(conf, "befriended numbers");
// Set this property to Zero to run map-only job
job.setNumReduceTasks(0);
job.setJarByClass(FriendlyNumbers.class);
job.setMapperClass(FriendlyNumberMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(NumberCouple.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(IntWritable.class);
您的reduce方法实现在哪里?当我此时仅使用映射器时是否有必要?请检查答案,您需要将numReduceTask设置为0。您的reduce方法实现在哪里?当我此时仅使用映射器时是否有必要?请检查答案,您需要将numReduceTask设置为0。