Apache pig 用pig拉丁语选择行

我有如下数据：

(a,b,c)
(a,c,b)
(a,b,d)

是否有类似DISTINCT的东西会产生如下输出

(a,b,c)
(a,b,d)

---

我想忽略顺序，只比较元素。

否。您最好的选择是编写一个UDF，它将接受每一行，对字段进行排序并返回一个有序字符串，然后对其使用distinct

猪

自由民主党

---

考虑到值得考虑，我建议读取数据并将其转换为包，对包进行排序并选择不同的数据

猪字：

REGISTER ORDER_UDF.jar;
A = LOAD 'data.txt' USING PigStorage(',') AS (a1: chararray, a2: chararray, a3: chararray);
B = FOREACH A GENERATE ORDER_UDF.ORDER(CONCAT(CONCAT(a1,a2),a3));
C = DISTINCT B;
D = FOREACH C GENERATE REPLACE($0,'',','); -- Get back the comma separated fields from the concatenated string.
DUMP D;

  import java.io.IOException;
  import java.util.Arrays;
  import org.apache.pig.EvalFunc;
  import org.apache.pig.data.Tuple;

   public class ORDER extends EvalFunc<String>
   {
     public String exec(Tuple input) throws IOException {
        if (input == null || input.size() == 0)
            return null;
        try{
            char tempArray[] = ((String)input).toCharArray();
            Arrays.sort(tempArray);       
            return new String(tempArray);
        }catch(Exception e){
            throw new IOException("Caught exception processing input row ", e);
        }
    }
  }

inp_data = load 'input.csv' USING PigStorage(',') AS (field1:chararray,field2:chararray,field3:chararray);
req_data = FOREACH inp_data GENERATE TOBAG(field1,field2,field3) AS (b:bag{t:(token:chararray)});
sorted_data = FOREACH req_data {
                sorted = ORDER b BY token;
                GENERATE sorted AS (sorted_bag:bag{t:(token:chararray)}); 
}

req_data_fmt = DISTINCT(FOREACH sorted_data GENERATE BagToString(sorted_bag,','));

DUMP req_data_fmt;