Google bigquery BigQuery如何使用合并加载数组列
我有一个测试表,我正试图从GCS存储中加载该表:Google bigquery BigQuery如何使用合并加载数组列,google-bigquery,Google Bigquery,我有一个测试表,我正试图从GCS存储中加载该表: CREATE OR REPLACE TABLE ta_producer_conformed.test ( id NUMERIC, array_string ARRAY<STRING>, array_struct_string_string ARRAY<STRUCT<key STRING, value STRING>>, array_struct_string_numeric ARRAY<
CREATE OR REPLACE TABLE ta_producer_conformed.test
(
id NUMERIC,
array_string ARRAY<STRING>,
array_struct_string_string ARRAY<STRUCT<key STRING, value STRING>>,
array_struct_string_numeric ARRAY<STRUCT<key STRING, value NUMERIC>>,
array_struct_string_int64 ARRAY<STRUCT<key STRING, value INT64>>
)
在其中,我使用JSON保存数组类型:
"id" "array_string" "struct_string_string" "struct_string_numberic" "struct_string_int64"
1 ["one", "two", "three"] [{"key":"one", "value":"1"},{"key":"two", "value":"2"},{"key":"three", "value":"3"}] [{"key":"one", "value":1.1},{"key":"two", "value":2.2}] [{"key":"one", "value":11},{"key":"two", "value":22}]
2 ["four", "five", "six"] [{"key":"four", "value":"4"},{"key":"five", "value":"5"},{"key":"six", "value":"6"}] [{"key":"three", "value":3.3},{"key":"four", "value":4.4}] [{"key":"three", "value":33},{"key":"four", "value":44}]
然后,我想使用合并在目标表中向上插入数据。当我运行此命令时:
CREATE TEMPORARY FUNCTION ARRAY_OF(json STRING)
RETURNS ARRAY<STRING>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_STRING_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING, value STRING>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_NUMERIC_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING, value NUMERIC>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_INT64_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING, value INT64>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
MERGE ta_producer_conformed.test T
USING ta_producer_raw.test_raw S
ON
S.id = T.id
WHEN NOT MATCHED THEN
INSERT (id, array_string, array_struct_string_string, array_struct_string_numeric, array_struct_string_int64)
VALUES (
id,
ARRAY_OF(array_string),
ARRAY_STRUCT_STRING_STRING_OF(struct_string_string),
ARRAY_STRUCT_STRING_NUMERIC_OF(struct_string_numberic),
ARRAY_STRUCT_STRING_INT64_OF(struct_string_int64)
)
WHEN MATCHED THEN UPDATE SET
T.id = S.id,
T.array_string = ARRAY_OF(S.array_string),
T.struct_string_string = ARRAY_STRUCT_STRING_STRING_OF(S.struct_string_string),
T.ARRAY_STRUCT_STRING_NUMERIC_OF(S.struct_string_numberic),
T.ARRAY_STRUCT_STRING_INT64_OF(S.struct_string_int64)
如果我删除了WHEN MATCHED的最后一个部分,这样只插入临时函数就可以了。所以问题似乎是,在final-THEN-updateset部分中,我不能使用临时函数
在理想情况下,如何使用单个MERGE语句从外部bucket加载数据类型(如ARRAY和ARRAY)
更新:我尝试使用公共表表达式来预处理数据,方法是:
WITH cteConvertJason AS (
SELECT
id,
ARRAY_OF(array_string) AS array_string,
ARRAY_STRUCT_STRING_STRING_OF(struct_string_string) AS struct_string_string,
ARRAY_STRUCT_STRING_NUMERIC_OF(struct_string_numberic) AS struct_string_numberic,
ARRAY_STRUCT_STRING_INT64_OF(struct_string_int64) AS struct_string_int64
FROM
ta_producer_raw.test_raw
)
MERGE ta_producer_conformed.test T
USING cteConvertJason S
...
这产生了一个错误,因此看起来您与合并
更新:出于遗留原因,我们尝试了TSV。最好使用换行符分隔的JSON作为格式,这样就不需要显式解析嵌套的或重复的列。事实证明,使用源的合并目标可以使用查询作为源。该查询可以运行临时函数来预处理源数据。然后,MERGE语句的其余部分可以正常工作:
CREATE TEMPORARY FUNCTION ARRAY_OF(json STRING)
RETURNS ARRAY<STRING>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_STRING_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING, value STRING>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_NUMERIC_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING, value NUMERIC>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_INT64_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING, value INT64>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
MERGE ta_producer_conformed.test T
USING
(
SELECT
id,
ARRAY_OF(array_string) AS array_string,
ARRAY_STRUCT_STRING_STRING_OF(struct_string_string) AS struct_string_string,
ARRAY_STRUCT_STRING_NUMERIC_OF(struct_string_numberic) AS struct_string_numberic,
ARRAY_STRUCT_STRING_INT64_OF(struct_string_int64) AS struct_string_int64
FROM
ta_producer_raw.test_raw
)
S
ON
S.id = T.id
WHEN NOT MATCHED THEN
INSERT (id, array_string, array_struct_string_string, array_struct_string_numeric, array_struct_string_int64)
VALUES (
id,
array_string,
struct_string_string,
struct_string_numberic,
struct_string_int64
)
WHEN MATCHED THEN UPDATE SET
T.array_string = S.array_string,
T.array_struct_string_string = S.struct_string_string,
T.array_struct_string_numeric = S.struct_string_numberic,
T.array_struct_string_int64 = S.struct_string_int64
WITH cteConvertJason AS (
SELECT
id,
ARRAY_OF(array_string) AS array_string,
ARRAY_STRUCT_STRING_STRING_OF(struct_string_string) AS struct_string_string,
ARRAY_STRUCT_STRING_NUMERIC_OF(struct_string_numberic) AS struct_string_numberic,
ARRAY_STRUCT_STRING_INT64_OF(struct_string_int64) AS struct_string_int64
FROM
ta_producer_raw.test_raw
)
MERGE ta_producer_conformed.test T
USING cteConvertJason S
...
CREATE TEMPORARY FUNCTION ARRAY_OF(json STRING)
RETURNS ARRAY<STRING>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_STRING_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING, value STRING>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_NUMERIC_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING, value NUMERIC>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_INT64_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING, value INT64>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
MERGE ta_producer_conformed.test T
USING
(
SELECT
id,
ARRAY_OF(array_string) AS array_string,
ARRAY_STRUCT_STRING_STRING_OF(struct_string_string) AS struct_string_string,
ARRAY_STRUCT_STRING_NUMERIC_OF(struct_string_numberic) AS struct_string_numberic,
ARRAY_STRUCT_STRING_INT64_OF(struct_string_int64) AS struct_string_int64
FROM
ta_producer_raw.test_raw
)
S
ON
S.id = T.id
WHEN NOT MATCHED THEN
INSERT (id, array_string, array_struct_string_string, array_struct_string_numeric, array_struct_string_int64)
VALUES (
id,
array_string,
struct_string_string,
struct_string_numberic,
struct_string_int64
)
WHEN MATCHED THEN UPDATE SET
T.array_string = S.array_string,
T.array_struct_string_string = S.struct_string_string,
T.array_struct_string_numeric = S.struct_string_numberic,
T.array_struct_string_int64 = S.struct_string_int64