Java 将结构数组从数据流写入大查询
我试图将一个Structs字段数组从数据流管道写入大查询,生成的表的模式是正确的,但字段中没有填充数据 我的DoFn函数:Java 将结构数组从数据流写入大查询,java,google-bigquery,google-cloud-dataflow,Java,Google Bigquery,Google Cloud Dataflow,我试图将一个Structs字段数组从数据流管道写入大查询,生成的表的模式是正确的,但字段中没有填充数据 我的DoFn函数: public class ProcessIpBlocks { public static class IpBlocksToIp extends DoFn<TableRow, TableRow> { private static final long serialVersionUID = 1L; @Override
public class ProcessIpBlocks {
public static class IpBlocksToIp extends DoFn<TableRow, TableRow> {
private static final long serialVersionUID = 1L;
@Override
public void processElement(ProcessContext c) throws JSONException {
TableRow row = c.element();
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Calendar cal = Calendar.getInstance();
long startIp = 0L, endIp = 0L;
if(row.get("start_ip") != null)
startIp = Long.parseLong((String)row.get("start_ip"));
if(row.get("end_ip") != null)
endIp = Long.parseLong((String)row.get("end_ip"));
for(long i= startIp; i<=endIp; i++)
{
TableRow outputRow = new TableRow();
outputRow.set("start_ip", startIp);
outputRow.set("ip", i);
if(row.get("postal_code") != null && !((String)row.get("postal_code")).isEmpty()){
System.out.println("This is getting written to logs");
endIp = Long.parseLong((String)row.get("end_ip"));
JSONArray atrArray = new JSONArray();
JSONObject atr = new JSONObject();
atr.put("id", "zippostal_code");
JSONArray atrValueArray = new JSONArray();
atr.put("value", atrValueArray.put((String)row.get("postal_code")));
atr.put("pr", 0.5);
atr.put("dt", cal.getTime());
atrArray.put(atr);
outputRow.set("atr", atrArray);
}
c.output(outputRow);
}
}
}
}
下面的解决方案有效,使用
TableRow
而不是JSONArray
公开课Foo{
public static class Foo extends DoFn<TableRow, TableRow> {
@Override
public void processElement(ProcessContext c) throws JSONException {
TableRow row = c.element();
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Calendar cal = Calendar.getInstance();
long startIp = 0L, endIp = 0L;
if(row.get("start_ip") != null)
startIp = Long.parseLong((String)row.get("start_ip"));
if(row.get("end_ip") != null)
endIp = Long.parseLong((String)row.get("end_ip"));
for(long i= startIp; i<=endIp; i++)
{
TableRow outputRow = new TableRow();
outputRow.set("start_ip", startIp);
outputRow.set("ip", i);
if(row.get("postal_code") != null && !((String)row.get("postal_code")).isEmpty()){
endIp = Long.parseLong((String)row.get("end_ip"));
TableRow atrRow = new TableRow();
atrRow.set("id", "zippostal_code");
atrRow.set("value", new String[] {(String)row.get("postal_code")});
outputRow.set("atr", atrRow);
}
System.out.println(outputRow);
c.output(outputRow);
}
}
}
公共静态类Foo扩展了DoFn{
@凌驾
public void processElement(ProcessContext c)抛出JSONException{
TableRow行=c.元素();
DateFormat DateFormat=新的简化格式(“yyyy-MM-dd HH:MM:ss”);
Calendar cal=Calendar.getInstance();
长星尖=0L,尾端=0L;
if(row.get(“start_ip”)!=null)
startIp=Long.parseLong((字符串)row.get(“start_ip”);
if(row.get(“end_ip”)!=null)
endIp=Long.parseLong((字符串)row.get(“end_ip”);
对于(长i=startIp;i以下解决方案有效,使用TableRow
而不是JSONArray
公开课Foo{
public static class Foo extends DoFn<TableRow, TableRow> {
@Override
public void processElement(ProcessContext c) throws JSONException {
TableRow row = c.element();
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Calendar cal = Calendar.getInstance();
long startIp = 0L, endIp = 0L;
if(row.get("start_ip") != null)
startIp = Long.parseLong((String)row.get("start_ip"));
if(row.get("end_ip") != null)
endIp = Long.parseLong((String)row.get("end_ip"));
for(long i= startIp; i<=endIp; i++)
{
TableRow outputRow = new TableRow();
outputRow.set("start_ip", startIp);
outputRow.set("ip", i);
if(row.get("postal_code") != null && !((String)row.get("postal_code")).isEmpty()){
endIp = Long.parseLong((String)row.get("end_ip"));
TableRow atrRow = new TableRow();
atrRow.set("id", "zippostal_code");
atrRow.set("value", new String[] {(String)row.get("postal_code")});
outputRow.set("atr", atrRow);
}
System.out.println(outputRow);
c.output(outputRow);
}
}
}
公共静态类Foo扩展了DoFn{
@凌驾
public void processElement(ProcessContext c)抛出JSONException{
TableRow行=c.元素();
DateFormat DateFormat=新的简化格式(“yyyy-MM-dd HH:MM:ss”);
Calendar cal=Calendar.getInstance();
长星尖=0L,尾端=0L;
if(row.get(“start_ip”)!=null)
startIp=Long.parseLong((字符串)row.get(“start_ip”);
if(row.get(“end_ip”)!=null)
endIp=Long.parseLong((字符串)row.get(“end_ip”);
对于(长i=星尖;i