Hadoop Map Reduce的对象序列化 朋友们好,,

Hadoop Map Reduce的对象序列化 朋友们好,,,hadoop,mapreduce,Hadoop,Mapreduce,我正在尝试序列化对象,它可以作为out值从映射器传递到reducer。在这个程序中,我得到以下例外 java.lang.RuntimeException: java.lang.NoSuchMethodException: com.test.objectpass.SerObj.<init>() at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:115) at org.apache.had

我正在尝试序列化对象,它可以作为out值从映射器传递到reducer。在这个程序中,我得到以下例外

java.lang.RuntimeException: java.lang.NoSuchMethodException: com.test.objectpass.SerObj.<init>()
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:115)
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:62)
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:40)
at org.apache.hadoop.mapreduce.ReduceContext.nextKeyValue(ReduceContext.java:116)
at org.apache.hadoop.mapreduce.ReduceContext.nextKey(ReduceContext.java:92)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:175)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:649)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:417)
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:260)

Caused by: java.lang.NoSuchMethodException: com.test.objectpass.SerObj.<init>()
at java.lang.Class.getConstructor0(Class.java:2715)
at java.lang.Class.getDeclaredConstructor(Class.java:1987)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:109)
java.lang.RuntimeException:java.lang.NoSuchMethodException:com.test.objectpass.SerObj.()
位于org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:115)
位于org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:62)
位于org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:40)
位于org.apache.hadoop.mapreduce.ReduceContext.nextKeyValue(ReduceContext.java:116)
位于org.apache.hadoop.mapreduce.ReduceContext.nextKey(ReduceContext.java:92)
位于org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:175)
位于org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:649)
位于org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:417)
位于org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:260)
原因:java.lang.NoSuchMethodException:com.test.objectpass.SerObj。()
位于java.lang.Class.getConstructor0(Class.java:2715)
位于java.lang.Class.getDeclaredConstructor(Class.java:1987)
位于org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:109)
我正在复制需要序列化的对象类的代码-

package com.test.objectpass;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.WritableComparable;

public class SerObj implements WritableComparable<Object> {
    
    private String name = null;
    private String surname = null;
    private Long number = null;
    
    public SerObj(String name, String surname, Long number) {
        super();
        setName(name);
        setNumber(number);
        setSurname(surname);
    }
    @Override
    public final String toString() {
        final StringBuilder string = new StringBuilder();
        string.append("SerObj [name=").append(name).append(", surname=")
                .append(surname).append(", number=").append(number).append("]");
        return string.toString();
    }
    
    public final String getName() { return name; }
    public final void setName(String name) { this.name = name; }
    public final String getSurname() { return surname; }
    public final void setSurname(String surname) { this.surname = surname; }
    public final Long getNumber() { return number; }
    public final void setNumber(Long number) { this.number = number; }
    
    @Override
    public void readFields(DataInput in) throws IOException {
        name =in.readLine();
        surname = in.readLine();
        number = in.readLong();
    }
    
    @Override
    public void write(DataOutput out) throws IOException {
        out.writeBytes(name);
        out.writeBytes(surname);
        out.writeLong(number);
    }
    
    @Override
    public boolean equals(Object o) {
        if (!(o instanceof SerObj))
            return false;
        SerObj other = (SerObj) o;
        return this.number == other.number;
    }
     
    @Override
    public int compareTo(Object o) {
        long thisValue = this.number;
        long thatValue = ((SerObj)o).number;
        return (thisValue<thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
    }
}
package com.test.objectpass;
导入java.io.DataInput;
导入java.io.DataOutput;
导入java.io.IOException;
导入org.apache.hadoop.io.WritableComparable;
公共类SerObj实现了writeablecomparable{
私有字符串名称=null;
私有字符串姓氏=null;
私有长编号=空;
public SerObj(字符串名称、字符串姓氏、长数字){
超级();
集合名(名称);
setNumber(数字);
setSurname(姓氏);
}
@凌驾
公共最终字符串toString(){
最终StringBuilder字符串=新StringBuilder();
string.append(“SerObj[name=).append(name).append(“,姓氏=”)
.append(姓氏)。append(,number=“)。append(number)。append(“]);
返回字符串.toString();
}
公共最终字符串getName(){return name;}
public final void setName(字符串名){this.name=name;}
公共最终字符串getNastname(){返回姓氏;}
public final void setSurname(字符串姓氏){this.names=姓氏;}
public final Long getNumber(){return number;}
公共最终无效集合编号(长编号){this.number=number;}
@凌驾
public void readFields(DataInput in)引发IOException{
name=in.readLine();
姓氏=in.readLine();
number=in.readLong();
}
@凌驾
public void write(DataOutput out)引发IOException{
out.writeBytes(名称);
out.writeBytes(姓氏);
out.writeLong(数字);
}
@凌驾
公共布尔等于(对象o){
if(!(o SerObj实例))
返回false;
SerObj其他=(SerObj)o;
返回this.number==other.number;
}
@凌驾
公共整数比较对象(对象o){
long thisValue=this.number;
长值=((SerObj)o).number;

返回(thisValue错误消息应该足够清楚;您的SerObj类中没有0-arg构造函数。几乎存在的每个序列化框架都要求您的bean没有可用的arg构造函数,以便框架可以在读取其所有数据之前通过反射实例化某个对象,并可写串行化也没什么不同。

谢谢Judge Mental。它起作用了。我需要对写入方法提出另一个建议。每次我都需要使用appaned“\n”来读取.readLine()中的数据输入。是否有其他方法可以提高性能。public void write(DataOutput out)抛出IOException{out.writeBytes(name+“\n”);out.writeBytes(姓氏+“\n”);out.writeLong(数字);}谢谢
package com.test.objectpass;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;

public class ObjectSerialization {
    public static class MyMapper extends Mapper<LongWritable, Text, Text, SerObj> {
        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] values = value.toString().split(" ");
            SerObj obj = new SerObj(values[0], values[1], Long.parseLong(values[2]));
            context.write(new Text(values[0]), obj);
        }
    }

    public static class MyReducer extends Reducer<Text, SerObj, NullWritable, NullWritable> {
        @Override
        public void reduce(Text key, Iterable<SerObj> values, Context context) throws IOException, InterruptedException {
            for (SerObj valueObj : values) {
                System.out.println(valueObj);
            }
        }
    }

    public static void main(String[] args) throws Exception {

        final Configuration conf = new Configuration();
        Job job = new Job(conf, "TEST");
        job.setJarByClass(ObjectSerialization.class);
        job.setMapperClass(MyMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(SerObj.class);
        job.setReducerClass(MyReducer.class);
        job.setOutputFormatClass(NullOutputFormat.class);

        TextInputFormat.addInputPath(job, new Path("/home/pankaj/test"));

        job.waitForCompletion(true);
        System.out.println("Done.");
    }
}