如何解决BigQuery如何使用MERGE加载数组列
我有一个要从GCS存储中加载的测试表:
CREATE OR REPLACE TABLE ta_producer_conformed.test
(
id NUMERIC,array_string ARRAY<STRING>,array_struct_string_string ARRAY<STRUCT<key STRING,value STRING>>,array_struct_string_numeric ARRAY<STRUCT<key STRING,value NUMERIC>>,array_struct_string_int64 ARRAY<STRUCT<key STRING,value INT64>>
)
我已将外部存储表定义为:
{
"autodetect": true,"csvOptions": {
"encoding": "UTF-8","quote": "\"","fieldDelimiter": "\t"
},"sourceFormat": "CSV","sourceUris": [
"gs://my_bucket/test/input/*.tsv"
]
}
在其中,我使用JSON来保存数组类型:
"id" "array_string" "struct_string_string" "struct_string_numberic" "struct_string_int64"
1 ["one","two","three"] [{"key":"one","value":"1"},{"key":"two","value":"2"},{"key":"three","value":"3"}] [{"key":"one","value":1.1},"value":2.2}] [{"key":"one","value":11},"value":22}]
2 ["four","five","six"] [{"key":"four","value":"4"},{"key":"five","value":"5"},{"key":"six","value":"6"}] [{"key":"three","value":3.3},{"key":"four","value":4.4}] [{"key":"three","value":33},"value":44}]
然后,我想使用MERGE在目标表中插入数据。当我运行此命令时:
CREATE TEMPORARY FUNCTION ARRAY_OF(json STRING)
RETURNS ARRAY<STRING>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_STRING_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING,value STRING>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_NUMERIC_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING,value NUMERIC>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_INT64_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING,value INT64>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
MERGE ta_producer_conformed.test T
USING ta_producer_raw.test_raw S
ON
S.id = T.id
WHEN NOT MATCHED THEN
INSERT (id,array_string,array_struct_string_string,array_struct_string_numeric,array_struct_string_int64)
VALUES (
id,ARRAY_OF(array_string),ARRAY_STRUCT_STRING_STRING_OF(struct_string_string),ARRAY_STRUCT_STRING_NUMERIC_OF(struct_string_numberic),ARRAY_STRUCT_STRING_INT64_OF(struct_string_int64)
)
WHEN MATCHED THEN UPDATE SET
T.id = S.id,T.array_string = ARRAY_OF(S.array_string),T.struct_string_string = ARRAY_STRUCT_STRING_STRING_OF(S.struct_string_string),T.ARRAY_STRUCT_STRING_NUMERIC_OF(S.struct_string_numberic),T.ARRAY_STRUCT_STRING_INT64_OF(S.struct_string_int64)
我收到此错误:
Error in query string: Error processing job 'xxxx-10843454-datamesh-
dev:bqjob_r4c426875_00000173fcfd2294_1': Syntax error: Expected "." or "=" or
"[" but got "(" at [1:1312]
如果我删除了“匹配时”的整个最后一部分,以使其仅插入,则临时功能可以正常工作。因此问题似乎在于,在THEN UPDATE SET
的最后一节中,我无法使用临时功能。
如何理想地使用单个MERGE语句从外部存储桶中加载ARRAY<STRING>
和ARRAY<STRUCT<STRING,STRING>>
之类的数据类型?
更新:我尝试使用通用表表达式来预处理数据,方法是:
WITH cteConvertJason AS (
SELECT
id,ARRAY_OF(array_string) AS array_string,ARRAY_STRUCT_STRING_STRING_OF(struct_string_string) AS struct_string_string,ARRAY_STRUCT_STRING_NUMERIC_OF(struct_string_numberic) AS struct_string_numberic,ARRAY_STRUCT_STRING_INT64_OF(struct_string_int64) AS struct_string_int64
FROM
ta_producer_raw.test_raw
)
MERGE ta_producer_conformed.test T
USING cteConvertJason S
...
这给出了一个错误,因此看起来您将WITH
和MERGE
组合在一起。
更新:出于遗留原因,我们正在尝试TSV。最好使用NEWLINE_DELIMITED_JSON
作为格式,这样您就不必显式解析嵌套或重复的列。
解决方法
事实证明MERGE target USING source
可以使用查询作为源。该查询可以运行临时功能来预处理源数据。然后,MERGE语句的其余部分可以正常工作:
CREATE TEMPORARY FUNCTION ARRAY_OF(json STRING)
RETURNS ARRAY<STRING>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_STRING_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING,value STRING>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_NUMERIC_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING,value NUMERIC>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
CREATE TEMPORARY FUNCTION ARRAY_STRUCT_STRING_INT64_OF(json STRING)
RETURNS ARRAY<STRUCT<key STRING,value INT64>>
LANGUAGE js AS """
let parsed = JSON.parse(json);
return parsed;
""";
MERGE ta_producer_conformed.test T
USING
(
SELECT
id,ARRAY_OF(array_string) AS array_string,ARRAY_STRUCT_STRING_STRING_OF(struct_string_string) AS struct_string_string,ARRAY_STRUCT_STRING_NUMERIC_OF(struct_string_numberic) AS struct_string_numberic,ARRAY_STRUCT_STRING_INT64_OF(struct_string_int64) AS struct_string_int64
FROM
ta_producer_raw.test_raw
)
S
ON
S.id = T.id
WHEN NOT MATCHED THEN
INSERT (id,array_string,array_struct_string_string,array_struct_string_numeric,array_struct_string_int64)
VALUES (
id,struct_string_string,struct_string_numberic,struct_string_int64
)
WHEN MATCHED THEN UPDATE SET
T.array_string = S.array_string,T.array_struct_string_string = S.struct_string_string,T.array_struct_string_numeric = S.struct_string_numberic,T.array_struct_string_int64 = S.struct_string_int64
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。