Sql apache PIG中的超前/滞后函数

ApachePig中是否有类似于SQL中的Lead/Lag函数的函数？或者任何可以回溯到前一行记录的清管器函数？

好，这是我的第一次尝试。请注意，我今天刚开始学习如何编写UDF

Maven的pom.xml文件包含：

<dependency>
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-client</artifactId>
  <version>2.0.0-cdh4.1.0</version>
</dependency>
...

如果我像这样做最后一行：

ToDate（子字符串（替换（滞后（日期字段到滞后），'T'，''，0,19），'yyyy-MM-dd HH:MM:ss'）作为滞后日期

它将返回以下错误

ERROR org.apache.pig.tools.grunt.grunt-错误1066:无法打开别名滞后的迭代器。后端错误：null

检查日志时会发现：

java.lang.NullPointerException
在org.joda.time.format.DateTimeFormatterBuilder$NumberFormatter.parseInto（DateTimeFormatterBuilder.java:1200）

---

因为第一行将包含空值。

这里有一个替代方案：

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;

public class GenericLag2 extends EvalFunc<Tuple>{

    private List<String> lagObjects = null;

    @Override
    public Tuple exec(Tuple input) throws IOException {
        if (lagObjects == null) {
            lagObjects = new ArrayList<String>();
            return null;
        }
        try {
            Tuple output = TupleFactory.getInstance().newTuple(lagObjects.size());
            for (int i = 0; i < lagObjects.size(); i++) {
                output.set(i, lagObjects.get(i));
            }

            lagObjects.clear();

            for (int i = 0; i < input.size(); i++) {
                lagObjects.add(input.get(i).toString());
            }
            return output;
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }

    @Override
    public Schema outputSchema(Schema input) {
        Schema tupleSchema = new Schema();
        try {
            for (int i = 0; i < input.size(); i++) {
                tupleSchema.add(new FieldSchema("lag_" + i, DataType.CHARARRAY));
            }
            return new Schema(new FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), tupleSchema, DataType.TUPLE));
        } catch (FrontendException e) {
            e.printStackTrace();
            return null;
        }
    }
}

---

是的，有预定义的功能。请参阅Piggybank中的说明和方法。Over（）在文档中列出了一些示例。

@DonaldMiner嘿Donald你能提供我可以从哪里开始的链接吗？我最近在MapReduce中做了滞后和超前处理，并使用了二次排序，从而避免了无共享方法的问题，因为给定键的所有值都被发送到正确的减速机并被订购。我来看看我能为UDF做些什么。@DonaldMiner你介意看看我的答案并给我一些反馈吗？忽略关于这不可能的评论。请通过@user3062149查看下面的答案。这样更方便。它包括预构建的功能。好发现！接受的答案应该是@user3062149的答案。

REGISTER /path/to/compiled/file.jar
DEFINE LAG fully.qualified.domain.name.GenericLag();

A = LOAD '/hdfs/path/to/directory' USING PigStorage(',') AS (
    important_order_by_field:int
    ,second_important_order_by_field:string
    ,...
    ,string_field_to_lag:string
    ,int_field_to_lag:int
    ,date_field_to_lag:string
);
B = FOREACH A GENERATE
    important_order_by_field
    ,second_important_order_by_field
    ,...
    ,string_field_to_lag
    ,int_field_to_lag
    ,ToDate(date_field_to_lag, 'yyyy-MM-dd HH:mm:ss')
    ;
C = ORDER A BY important_order_by_field, second_important_order_by_field
D = FOREACH B GENERATE
    important_order_by_field
    ,second_important_order_by_field
    ,...
    ,LAG(string_field_to_lag) AS lag_string
    ,(int) LAG(int_field_to_lag) AS lag_int
    ,(date_field_to_lag IS NULL ? 
        null : 
        ToDate(SUBSTRING(REPLACE(LAG(date_field_to_lag), 'T', ' ') ,0,19), 'yyyy-MM-dd HH:mm:ss')) AS lag_date
    ;
DUMP D;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;

public class GenericLag2 extends EvalFunc<Tuple>{

    private List<String> lagObjects = null;

    @Override
    public Tuple exec(Tuple input) throws IOException {
        if (lagObjects == null) {
            lagObjects = new ArrayList<String>();
            return null;
        }
        try {
            Tuple output = TupleFactory.getInstance().newTuple(lagObjects.size());
            for (int i = 0; i < lagObjects.size(); i++) {
                output.set(i, lagObjects.get(i));
            }

            lagObjects.clear();

            for (int i = 0; i < input.size(); i++) {
                lagObjects.add(input.get(i).toString());
            }
            return output;
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }

    @Override
    public Schema outputSchema(Schema input) {
        Schema tupleSchema = new Schema();
        try {
            for (int i = 0; i < input.size(); i++) {
                tupleSchema.add(new FieldSchema("lag_" + i, DataType.CHARARRAY));
            }
            return new Schema(new FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), tupleSchema, DataType.TUPLE));
        } catch (FrontendException e) {
            e.printStackTrace();
            return null;
        }
    }
}

... 
C = ORDER A BY important_order_by_field, second_important_order_by_field
D = FOREACH B GENERATE
    important_order_by_field
    ,second_important_order_by_field
    ,...
    ,FLATTEN(LAG(
        string_field_to_lag
        ,int_field_to_lag
        ,date_field_to_lag
    ))
    ;
E = FOREACH D GENERATE
    important_order_by_field
    ,second_important_order_by_field
    ,...
    ,string_field_to_lag
    ,(int) int_field_to_lag
    ,(date_field_to_lag IS NULL ? 
        null :
        ToDate(SUBSTRING(REPLACE(date_field_to_lag, 'T', ' '), 0, 19),  'yyyy-MM-dd HH:mm:ss')) 
        as date_field_to_lag
;
DUMP E;