使用泛型类型为Java POJO生成Avro模式

使用泛型类型为Java POJO生成Avro模式,java,generics,reflection,avro,Java,Generics,Reflection,Avro,我正在尝试使用以下方法在运行时获取Avro模式: private Schema getSchema(Class clazz) { Schema s = ReflectData.get().getSchema(clazz); AvroSchema avroSchema = new AvroSchema(s); return avroSchema.getAvroSchema(); } 但由于我的POJO类包含以下泛型: public abstract class Dat

我正在尝试使用以下方法在运行时获取Avro模式:

private Schema getSchema(Class clazz) {
    Schema s = ReflectData.get().getSchema(clazz);
    AvroSchema avroSchema = new AvroSchema(s);
    return avroSchema.getAvroSchema();
  }
但由于我的POJO类包含以下泛型:

public abstract class Data<T> implements Serializable {
    private static final long serialVersionUID = 1L;
    private String dataType;
    private T id;

    public Data() {
    }

    public Data(String dataType) {
        this.dataType = dataType;
    }

    public Data(String dataType, T id) {
        this.dataType = dataType;
        this.id = id;
    }
}
我理解Avro将不支持泛型类型。在运行时生成架构期间,有没有一种方法可以从类中省略某些类字段?

私有字符串writePojoToParquet(List pojo,String fileKey){
private <T> String writePojoToParquet(List<T> pojos, String fileKey){
        String fileName = fileKey + ".parquet";
        Path path = new Path(fileName.replace("/", "_"));
        //No matter what delete file always.
        String strPath = path.toString();
        FileUtils.delete(strPath);
        FileUtils.delete(strPath + ".crc");
        logger.debug("Writing data to parquet file {}", strPath);
        Configuration conf = new Configuration();
        try (ParquetWriter<T> writer =
                     AvroParquetWriter.<T>builder(path)
                             .withSchema(ReflectData.AllowNull.get().getSchema(pojos.get(0).getClass()))
                             .withDataModel(ReflectData.get())
                             .withConf(conf)
                             .withCompressionCodec(CompressionCodecName.SNAPPY)
                             .withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
                             .enableValidation()
                             .enableDictionaryEncoding()
                             .build()) {
            for (T p : pojos) {
                writer.write(p);
            }
            return strPath;
        } catch (IOException e) {
            logger.error("Error while writing data to parquet file {}.", strPath, e);
        }
        return null;
    }
字符串文件名=fileKey+“.parquet”; 路径路径=新路径(fileName.replace(“/”,“”); //无论什么,都要删除文件。 String strPath=path.toString(); FileUtils.delete(strPath); delete(strPath+“.crc”); debug(“将数据写入拼花文件{}”,strPath); Configuration conf=新配置(); try(拼花编剧)= AvroParquetWriter.builder(路径) .withSchema(ReflectData.AllowNull.get().getSchema(pojos.get(0.getClass())) .withDataModel(ReflectData.get()) .withConf(conf) .withCompressionCodec(CompressionCodecName.SNAPPY) .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) .enableValidation() .enableDictionaryEncoding() .build()){ 对于(tp:pojos){ write.write(p); } 返回strPath; }捕获(IOE异常){ error(“将数据写入拼花文件{}时出错”,strPath,e); } 返回null; }
private <T> String writePojoToParquet(List<T> pojos, String fileKey){
        String fileName = fileKey + ".parquet";
        Path path = new Path(fileName.replace("/", "_"));
        //No matter what delete file always.
        String strPath = path.toString();
        FileUtils.delete(strPath);
        FileUtils.delete(strPath + ".crc");
        logger.debug("Writing data to parquet file {}", strPath);
        Configuration conf = new Configuration();
        try (ParquetWriter<T> writer =
                     AvroParquetWriter.<T>builder(path)
                             .withSchema(ReflectData.AllowNull.get().getSchema(pojos.get(0).getClass()))
                             .withDataModel(ReflectData.get())
                             .withConf(conf)
                             .withCompressionCodec(CompressionCodecName.SNAPPY)
                             .withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
                             .enableValidation()
                             .enableDictionaryEncoding()
                             .build()) {
            for (T p : pojos) {
                writer.write(p);
            }
            return strPath;
        } catch (IOException e) {
            logger.error("Error while writing data to parquet file {}.", strPath, e);
        }
        return null;
    }