feat(DENG-9505): migrating amo datasets to glean

sql/moz-fx-data-shared-prod/amo_glean/fenix_addons_by_client/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.amo_glean.fenix_addons_by_client`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.amo_glean_derived.fenix_addons_by_client_v1`

sql/moz-fx-data-shared-prod/amo_glean/firefox_desktop_addons_by_client/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.amo_glean.firefox_desktop_addons_by_client`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.amo_glean_derived.firefox_desktop_addons_by_client_v1`

sql/moz-fx-data-shared-prod/amo_glean/firefox_desktop_stats_installs/view.sql

@@ -0,0 +1,37 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.amo_glean.firefox_desktop_stats_installs`
+AS
+SELECT
+  submission_date,
+  hashed_addon_id,
+  total_downloads,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(downloads_per_campaign)
+  ) AS downloads_per_campaign,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(downloads_per_content)
+  ) AS downloads_per_content,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(downloads_per_source)
+  ) AS downloads_per_source,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(downloads_per_medium)
+  ) AS downloads_per_medium,
+FROM
+  `moz-fx-data-shared-prod.amo_glean_derived.firefox_desktop_stats_installs_v1`

sql/moz-fx-data-shared-prod/amo_glean/stats_dau/view.sql

@@ -0,0 +1,51 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.amo_glean.stats_dau`
+AS
+SELECT
+  submission_date,
+  addon_id,
+  dau,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(dau_by_addon_version)
+  ) AS dau_by_addon_version,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(dau_by_app_os)
+  ) AS dau_by_app_os,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(dau_by_app_version)
+  ) AS dau_by_app_version,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(dau_by_fenix_build)
+  ) AS dau_by_fenix_build,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(dau_by_country)
+  ) AS dau_by_country,
+  ARRAY(
+    SELECT AS STRUCT
+      IFNULL(key, 'Unknown') AS key,
+      value
+    FROM
+      UNNEST(dau_by_locale)
+  ) AS dau_by_locale,
+FROM
+  `moz-fx-data-shared-prod.amo_glean_derived.stats_dau_v1`

sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/metadata.yaml

@@ -0,0 +1,24 @@
+---
+friendly_name: Fenix addons by client
+description: >-
+  Clients_daily-like table on top of the various Firefox for Android channels
+  that records only the dimensions and addon info necessary to power the daily
+  amo_stats_dau_v2 query.
+owners:
+  - [email protected]
+labels:
+  application: amo
+  incremental: true
+  schedule: daily
+  table_table: client_level
+scheduling:
+  dag_name: bqetl_amo_stats
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+    expiration_days: 775
+  clustering:
+    fields:
+      - sample_id

sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/query.sql

@@ -0,0 +1,99 @@
+CREATE TEMP FUNCTION get_fields(m ANY TYPE) AS (
+  STRUCT(
+    m.submission_timestamp,
+    m.client_info.client_id,
+    m.sample_id,
+    m.metrics.string_list.addons_enabled_addons,
+    m.normalized_country_code,
+    m.client_info.locale,
+    m.normalized_os
+  )
+);
+
+WITH unioned AS (
+  SELECT
+    get_fields(release).*,
+    client_info.app_display_version AS app_version,
+  FROM
+    `moz-fx-data-shared-prod.org_mozilla_firefox.metrics` AS release
+  UNION ALL
+  SELECT
+    get_fields(beta).*,
+    -- Bug 1669516 We choose to show beta versions as 80.0.0b1, etc.
+    REPLACE(client_info.app_display_version, '-beta.', 'b') AS app_version,
+  FROM
+    `moz-fx-data-shared-prod.org_mozilla_firefox_beta.metrics` AS beta
+  UNION ALL
+  SELECT
+    get_fields(nightly).*,
+    -- Bug 1669516 Nightly versions have app_display_version like "Nightly <timestamp>",
+    -- so we take the geckoview version instead.
+    nightly.metrics.string.geckoview_version AS app_version,
+  FROM
+    `moz-fx-data-shared-prod.org_mozilla_fenix.metrics` AS nightly
+  UNION ALL
+  SELECT
+    get_fields(preview_nightly).*,
+    preview_nightly.metrics.string.geckoview_version AS app_version,
+  FROM
+    `moz-fx-data-shared-prod.org_mozilla_fenix_nightly.metrics` AS preview_nightly
+  UNION ALL
+  SELECT
+    get_fields(old_fenix_nightly).*,
+    old_fenix_nightly.metrics.string.geckoview_version AS app_version,
+  FROM
+    `moz-fx-data-shared-prod.org_mozilla_fennec_aurora.metrics` AS old_fenix_nightly
+),
+cleaned AS (
+  SELECT
+    * REPLACE (
+      IF(
+        -- Accepts formats: 80.0 80.0.0 80.0.0a1 80.0.0b1
+        REGEXP_CONTAINS(app_version, r'^(\d+\.\d+(\.\d+)?([ab]\d+)?)$'),
+        app_version,
+        NULL
+      ) AS app_version
+    )
+  FROM
+    unioned
+),
+per_client AS (
+  SELECT
+    DATE(submission_timestamp) AS submission_date,
+    client_id,
+    sample_id,
+    ARRAY_CONCAT_AGG(addons_enabled_addons ORDER BY submission_timestamp) AS addons,
+    -- We always want to take the most recent seen version per
+    -- https://bugzilla.mozilla.org/show_bug.cgi?id=1693308
+    ARRAY_AGG(app_version ORDER BY mozfun.norm.truncate_version(app_version, "minor") DESC)[
+      SAFE_OFFSET(0)
+    ] AS app_version,
+    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(normalized_country_code)) AS country,
+    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(locale)) AS locale,
+    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(normalized_os)) AS app_os,
+  FROM
+    cleaned
+  WHERE
+    DATE(submission_timestamp) = @submission_date
+    AND client_id IS NOT NULL
+  GROUP BY
+    submission_date,
+    sample_id,
+    client_id
+)
+SELECT
+  * EXCEPT (addons),
+  ARRAY(
+    SELECT AS STRUCT
+      TRIM(addon) AS addon,
+      -- As of 2020-07-01, the metrics ping from Fenix contains no data about
+      -- the version of installed addons, so we inject null and replace with
+      -- an appropriate placeholder value when we get to the app-facing view.
+      CAST(NULL AS STRING) AS version,
+    FROM
+      UNNEST(addons) AS addon
+    GROUP BY
+      TRIM(addon)
+  ) AS addons
+FROM
+  per_client

sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/schema.yaml

@@ -0,0 +1,32 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+- name: client_id
+  type: STRING
+  mode: NULLABLE
+- name: sample_id
+  type: INTEGER
+  mode: NULLABLE
+- name: app_version
+  type: STRING
+  mode: NULLABLE
+- name: country
+  type: STRING
+  mode: NULLABLE
+- name: locale
+  type: STRING
+  mode: NULLABLE
+- name: app_os
+  type: STRING
+  mode: NULLABLE
+- name: addons
+  type: RECORD
+  mode: REPEATED
+  fields:
+  - name: addon
+    type: STRING
+    mode: NULLABLE
+  - name: version
+    type: STRING
+    mode: NULLABLE

sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/metadata.yaml

@@ -0,0 +1,23 @@
+---
+friendly_name: Desktop addons by client
+description: >-
+  Clients_daily-like table that records only the dimensions and addon info
+  necessary to power daily the amo_stats_dau_v2 query.
+owners:
+  - [email protected]
+labels:
+  application: amo
+  incremental: true
+  schedule: daily
+  table_table: client_level
+scheduling:
+  dag_name: bqetl_amo_stats
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+    expiration_days: 775
+  clustering:
+    fields:
+      - sample_id

sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/query.sql

@@ -0,0 +1,111 @@
+WITH base AS (
+  SELECT
+    submission_timestamp,
+    DATE(submission_timestamp) AS submission_date,
+    client_info.client_id,
+    sample_id,
+    metrics.uuid.legacy_telemetry_client_id,
+    normalized_channel,
+    normalized_country_code,
+    client_info.locale,
+    normalized_os,
+    client_info.app_display_version,
+    -- merging active_addons and addons_theme into a single addons object
+    ARRAY_CONCAT(
+      -- COALESCE to make sure array concat returns results if active_addons is empty and addons_theme is not
+      COALESCE(JSON_QUERY_ARRAY(metrics.object.addons_active_addons, '$'), []),
+      [
+        JSON_QUERY(metrics.object.addons_theme, '$')
+      ] -- wrapping the json object in array to enable array concat
+    ) AS active_addons,
+  FROM
+    `moz-fx-data-shared-prod.firefox_desktop.metrics`
+  WHERE
+    DATE(submission_timestamp) = @submission_date
+    AND client_info.client_id IS NOT NULL
+    AND (
+      (
+        metrics.object.addons_active_addons IS NOT NULL
+        AND ARRAY_LENGTH(JSON_QUERY_ARRAY(metrics.object.addons_active_addons, '$')) > 0
+      )
+      OR metrics.object.addons_theme IS NOT NULL
+    )
+),
+per_clients_without_addons AS (
+  SELECT
+    submission_date,
+    client_id,
+    sample_id,
+    mozfun.stats.mode_last(
+      ARRAY_AGG(legacy_telemetry_client_id ORDER BY submission_timestamp)
+    ) AS legacy_telemetry_client_id,
+    ARRAY_AGG(
+      app_display_version
+      ORDER BY
+        mozfun.norm.truncate_version(app_display_version, "minor") DESC
+    )[SAFE_OFFSET(0)] AS app_version,
+    mozfun.stats.mode_last(
+      ARRAY_AGG(normalized_country_code ORDER BY submission_timestamp)
+    ) AS country,
+    mozfun.stats.mode_last(ARRAY_AGG(locale ORDER BY submission_timestamp)) AS locale,
+    mozfun.stats.mode_last(ARRAY_AGG(normalized_os ORDER BY submission_timestamp)) AS app_os,
+  FROM
+    base
+  GROUP BY
+    ALL
+),
+per_clients_just_addons AS (
+  SELECT
+    submission_date,
+    client_id,
+    sample_id,
+    mozfun.stats.mode_last(
+      ARRAY_AGG(legacy_telemetry_client_id ORDER BY submission_timestamp)
+    ) AS legacy_telemetry_client_id,
+    ARRAY_CONCAT_AGG(
+      ARRAY(
+        SELECT AS STRUCT
+          JSON_VALUE(addon, '$.id') AS id,
+          JSON_VALUE(addon, '$.version') AS `version`
+        FROM
+          UNNEST(active_addons) AS addon
+      )
+    ) AS addons
+  FROM
+    base
+  GROUP BY
+    ALL
+),
+per_client AS (
+  SELECT
+    submission_date,
+    client_id,
+    sample_id,
+    legacy_telemetry_client_id,
+    addons,
+    app_version,
+    country,
+    locale,
+    app_os,
+  FROM
+    per_clients_without_addons
+  INNER JOIN
+    per_clients_just_addons
+    USING (submission_date, client_id, sample_id, legacy_telemetry_client_id)
+)
+SELECT
+  * EXCEPT (addons),
+  ARRAY(
+    SELECT AS STRUCT
+      addon.id,
+      -- Same methodology as for app_version above.
+      ARRAY_AGG(addon.version ORDER BY mozfun.norm.truncate_version(addon.version, "minor") DESC)[
+        SAFE_OFFSET(0)
+      ] AS `version`,
+    FROM
+      UNNEST(addons) AS addon
+    GROUP BY
+      addon.id
+  ) AS addons
+FROM
+  per_client