Java 在hadoop reducer中执行context.write()时出现空指针异常
当我运行MapReduce作业时,出现以下错误 我的职业分类如下:Java 在hadoop reducer中执行context.write()时出现空指针异常,java,hadoop,mapreduce,Java,Hadoop,Mapreduce,当我运行MapReduce作业时,出现以下错误 我的职业分类如下: package mutualfriends; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.inp
package mutualfriends;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class MutualSuggest extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
// TODO Auto-generated method stub
if(args.length !=2)
{
System.err.println("Usage: MutualSuggest <input path> <outputpath>");
System.exit(-1);
}
Job job = new Job();
job.setJarByClass(MutualSuggest.class);
job.setJobName("Mutual Friends");
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));
job.setMapperClass(MutualSuggestMapper.class);
job.setCombinerClass(MutualSuggestReducer.class);
job.setReducerClass(MutualSuggestReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
System.exit(job.waitForCompletion(true) ? 0:1);
boolean success = job.waitForCompletion(true);
return success ? 0 : 1;
}
public static void main(String[] args) throws Exception
{
MutualSuggest driver = new MutualSuggest();
int exitCode = ToolRunner.run(driver, args);
System.exit(exitCode);
}
}
package mutualfriends;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
@SuppressWarnings({ "unchecked","rawtypes"})
public class MutualSuggestMapper extends Mapper<Object, Text, Text, Text>{
public String sorted(String name) {
char[] chars = name.toCharArray();
Arrays.sort(chars);
String sorted = new String(chars);
return sorted;
}
@Override
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] spl = line.split("=");
String user=spl[0];
String[] friends = spl[1].split(",");
Map m = new HashMap();
for (int i=0;i<friends.length;i++)
{
m.put(sorted(user+friends[i]), sorted(spl[1].replace(",","")));
}
Set x=m.keySet();
Iterator ite=x.iterator();
while (ite.hasNext())
{
Object z=ite.next();
context.write(new Text((String) z),new Text((String) m.get(z)));
}
}
}
package mutualfriends;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class MutualSuggestReducer extends Reducer<Text, Text, Text, Text>{
@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void reduce(Text key, Iterable<Text> values,Context context)
throws IOException, InterruptedException {
Map hm=new HashMap();
int z=1;
for (Text val:values)
{
hm.put(z, new Text(val));
z+=1;
}
String s=new String();
String t=new String();
s= hm.get(1).toString();
t= hm.get(2).toString();
//System.out.println(s+" "+t);
String x = s.replaceAll("[^" + t + "]", "");
System.out.println(key+" "+new Text(x));
context.write(new Text(key),new Text(x));
}
}
但在编写输出时,如:
context.write(key,new Text(x));
我得到以下错误:
15/07/03 16:13:10 WARN mapred.LocalJobRunner: job_local1502108935_0001
java.lang.NullPointerException
at mutualfriends.MutualSuggestReducer.reduce(MutualSuggestReducer.java:26)
at mutualfriends.MutualSuggestReducer.reduce(MutualSuggestReducer.java:1)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:177)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:649)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:418)
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:398)
15/07/03 16:13:11 INFO mapred.JobClient: map 100% reduce 0%
如何解决这个问题
提前感谢。您正在索引hm.get1和hm.get2,而不检查这些密钥是否存在于地图中。在从HashMap获取值之前进行检查。非常基本的错误我通过简单的修改解决了这个问题 通过删除job.setCombinerClassMutualSuggestReducer.class;MutualSuggest类中的语句
此错误的原因是,通过调用job.setCombinerClassMutualSuggestReducer.class;它通过调用job.setReducerClassMutualSuggestReducer.class执行一次Reducer函数;程序试图再次执行Reducer函数。因此,我删除了job.setCombinerClassMutualSuggestReducer.class;。程序运行得很好。嗨,库马尔,我修改了代码,检查hm中元素的存在,比如if!hm.isEmpty{s=hm.get1.toString;t=hm.get2.toString;}。但我还是犯了同样的错误,我不认为这是个问题。
15/07/03 16:13:10 WARN mapred.LocalJobRunner: job_local1502108935_0001
java.lang.NullPointerException
at mutualfriends.MutualSuggestReducer.reduce(MutualSuggestReducer.java:26)
at mutualfriends.MutualSuggestReducer.reduce(MutualSuggestReducer.java:1)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:177)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:649)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:418)
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:398)
15/07/03 16:13:11 INFO mapred.JobClient: map 100% reduce 0%