Firebase BigQuery架构迁移：移动到分区表中？

我收到了一封电子邮件，上面有将我以前在BigQuery中的Firebase表迁移到新模式的说明。他们指出这些指示：

但我更愿意：

• 我宁愿只运行一个执行迁移的查询，而不是运行bash脚本
• 我宁愿将以前的所有结果移动到一个新的日期分区表中，而不是创建许多新表

---

我接受了文档中的脚本并做了一些更改

• 查看所有的

  --Fh

  注释。这些是我的修改
• 选择目标表
• 选择Android和IOS的日期范围
• 请注意，我正在添加一个新列，其中包含一个用于分区的真实时间戳（以及您的便利性）
• 您将只得到一个表，而不是许多新表，但按日期进行分区

修改脚本：

#standardSQL

CREATE OR REPLACE TABLE `fh-bigquery.deleting.delete`
PARTITION BY DATE(ts)
AS 

WITH sources AS ( --Fh
  SELECT * FROM (
    SELECT *, _table_suffix event_date, 'ANDROID' operating_system
    FROM `firebase-public-project.com_firebase_demo_ANDROID.app_events_*` 
    UNION ALL SELECT *, _table_suffix event_date, 'IOS' operating_system 
    FROM `firebase-public-project.com_firebase_demo_IOS.app_events_*`
  )
  WHERE event_date BETWEEN '20180503' AND '20180504'  --Fh: choose your timerange
) 

SELECT
  event_date, --Fh: extracted from original table name
  TIMESTAMP_MICROS(event.timestamp_micros) ts, --Fh: adding a real timestamp column
  event.timestamp_micros AS event_timestamp,
  event.previous_timestamp_micros AS event_previous_timestamp,
  event.name AS event_name,
  event.value_in_usd  AS event_value_in_usd,
  user_dim.bundle_info.bundle_sequence_id AS event_bundle_sequence_id,
  user_dim.bundle_info.server_timestamp_offset_micros as event_server_timestamp_offset,
  (
  SELECT
    ARRAY_AGG(STRUCT(event_param.key AS key,
        STRUCT(event_param.value.string_value AS string_value,
          event_param.value.int_value AS int_value,
          event_param.value.double_value AS double_value, 
          event_param.value.float_value AS float_value) AS value))
  FROM
    UNNEST(event.params) AS event_param) AS event_params,
  user_dim.first_open_timestamp_micros AS user_first_touch_timestamp,
  user_dim.user_id AS user_id,
  user_dim.app_info.app_instance_id AS user_pseudo_id,
  "" AS stream_id,
  user_dim.app_info.app_platform AS platform,
  STRUCT( user_dim.ltv_info.revenue AS revenue,
    user_dim.ltv_info.currency AS currency ) AS user_ltv,
  STRUCT( user_dim.traffic_source.user_acquired_campaign AS name,
      user_dim.traffic_source.user_acquired_medium AS medium,
      user_dim.traffic_source.user_acquired_source AS source ) AS traffic_source,
  STRUCT( user_dim.geo_info.continent AS continent,
    user_dim.geo_info.country AS country,
    user_dim.geo_info.region AS region,
    user_dim.geo_info.city AS city ) AS geo,
  STRUCT( user_dim.device_info.device_category AS category,
    user_dim.device_info.mobile_brand_name,
    user_dim.device_info.mobile_model_name,
    user_dim.device_info.mobile_marketing_name,
    user_dim.device_info.device_model AS mobile_os_hardware_model,
    operating_system, --Fh
    user_dim.device_info.platform_version AS operating_system_version,
    user_dim.device_info.device_id AS vendor_id,
    user_dim.device_info.resettable_device_id AS advertising_id,
    user_dim.device_info.user_default_language AS language,
    user_dim.device_info.device_time_zone_offset_seconds AS time_zone_offset_seconds,
    IF(user_dim.device_info.limited_ad_tracking, "Yes", "No") AS is_limited_ad_tracking ) AS device,
  STRUCT( user_dim.app_info.app_id AS id,
    'app_id'  AS firebase_app_id, --Fh: choose your app id
    user_dim.app_info.app_version AS version,
    user_dim.app_info.app_store AS install_source ) AS app_info,
  ( SELECT ARRAY_AGG(STRUCT(user_property.key AS key,
        STRUCT(user_property.value.value.string_value AS string_value,
          user_property.value.value.int_value AS int_value,
          user_property.value.value.double_value AS double_value,
          user_property.value.value.float_value AS float_value,
          user_property.value.set_timestamp_usec AS set_timestamp_micros ) AS value))
     FROM UNNEST(user_dim.user_properties) AS user_property
  ) AS user_properties
FROM sources -- Fh
  , UNNEST(event_dim) AS event

---

我在文档中使用了脚本并做了一些更改

• 查看所有的

  --Fh

  注释。这些是我的修改
• 选择目标表
• 选择Android和IOS的日期范围
• 请注意，我正在添加一个新列，其中包含一个用于分区的真实时间戳（以及您的便利性）
• 您将只得到一个表，而不是许多新表，但按日期进行分区

修改脚本：

#standardSQL

CREATE OR REPLACE TABLE `fh-bigquery.deleting.delete`
PARTITION BY DATE(ts)
AS 

WITH sources AS ( --Fh
  SELECT * FROM (
    SELECT *, _table_suffix event_date, 'ANDROID' operating_system
    FROM `firebase-public-project.com_firebase_demo_ANDROID.app_events_*` 
    UNION ALL SELECT *, _table_suffix event_date, 'IOS' operating_system 
    FROM `firebase-public-project.com_firebase_demo_IOS.app_events_*`
  )
  WHERE event_date BETWEEN '20180503' AND '20180504'  --Fh: choose your timerange
) 

SELECT
  event_date, --Fh: extracted from original table name
  TIMESTAMP_MICROS(event.timestamp_micros) ts, --Fh: adding a real timestamp column
  event.timestamp_micros AS event_timestamp,
  event.previous_timestamp_micros AS event_previous_timestamp,
  event.name AS event_name,
  event.value_in_usd  AS event_value_in_usd,
  user_dim.bundle_info.bundle_sequence_id AS event_bundle_sequence_id,
  user_dim.bundle_info.server_timestamp_offset_micros as event_server_timestamp_offset,
  (
  SELECT
    ARRAY_AGG(STRUCT(event_param.key AS key,
        STRUCT(event_param.value.string_value AS string_value,
          event_param.value.int_value AS int_value,
          event_param.value.double_value AS double_value, 
          event_param.value.float_value AS float_value) AS value))
  FROM
    UNNEST(event.params) AS event_param) AS event_params,
  user_dim.first_open_timestamp_micros AS user_first_touch_timestamp,
  user_dim.user_id AS user_id,
  user_dim.app_info.app_instance_id AS user_pseudo_id,
  "" AS stream_id,
  user_dim.app_info.app_platform AS platform,
  STRUCT( user_dim.ltv_info.revenue AS revenue,
    user_dim.ltv_info.currency AS currency ) AS user_ltv,
  STRUCT( user_dim.traffic_source.user_acquired_campaign AS name,
      user_dim.traffic_source.user_acquired_medium AS medium,
      user_dim.traffic_source.user_acquired_source AS source ) AS traffic_source,
  STRUCT( user_dim.geo_info.continent AS continent,
    user_dim.geo_info.country AS country,
    user_dim.geo_info.region AS region,
    user_dim.geo_info.city AS city ) AS geo,
  STRUCT( user_dim.device_info.device_category AS category,
    user_dim.device_info.mobile_brand_name,
    user_dim.device_info.mobile_model_name,
    user_dim.device_info.mobile_marketing_name,
    user_dim.device_info.device_model AS mobile_os_hardware_model,
    operating_system, --Fh
    user_dim.device_info.platform_version AS operating_system_version,
    user_dim.device_info.device_id AS vendor_id,
    user_dim.device_info.resettable_device_id AS advertising_id,
    user_dim.device_info.user_default_language AS language,
    user_dim.device_info.device_time_zone_offset_seconds AS time_zone_offset_seconds,
    IF(user_dim.device_info.limited_ad_tracking, "Yes", "No") AS is_limited_ad_tracking ) AS device,
  STRUCT( user_dim.app_info.app_id AS id,
    'app_id'  AS firebase_app_id, --Fh: choose your app id
    user_dim.app_info.app_version AS version,
    user_dim.app_info.app_store AS install_source ) AS app_info,
  ( SELECT ARRAY_AGG(STRUCT(user_property.key AS key,
        STRUCT(user_property.value.value.string_value AS string_value,
          user_property.value.value.int_value AS int_value,
          user_property.value.value.double_value AS double_value,
          user_property.value.value.float_value AS float_value,
          user_property.value.set_timestamp_usec AS set_timestamp_micros ) AS value))
     FROM UNNEST(user_dim.user_properties) AS user_property
  ) AS user_properties
FROM sources -- Fh
  , UNNEST(event_dim) AS event