SVCSE-1595 Setup import of tables from production FxA databases (#4597)

Author: akkomar

bqetl_project.yaml

@@ -52,6 +52,7 @@ dry_run:
   - sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend/accounts/view.sql
+  - sql/moz-fx-data-shared-prod/accounts_db_external/**/*.sql
   - sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/**/*.sql
   - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql
   - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql

sql/moz-fx-data-shared-prod/accounts_db_external/dataset_metadata.yaml

@@ -0,0 +1,14 @@
+---
+friendly_name: Firefox Accounts Databases production External
+description: |-
+  Data extracted from the production FxA backend services databases.
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa for more information.
+
+  Access to this dataset is restricted to accounts-confidential workgroup because some tables here contain sensitive data.
+dataset_base_acl: restricted
+user_facing: false
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:accounts-confidential

sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_customers_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: accountCustomers table from production fxa database
+description: >
+  A mirror of the `accountCustomers` table from the production `fxa` CloudSQL database,
+  updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - [email protected]
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_customers_v1/query.sql

@@ -0,0 +1,17 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  stripeCustomerId,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(updatedAt AS INT)) AS updatedAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
+    """SELECT
+         uid,
+         stripeCustomerId,
+         createdAt,
+         updatedAt
+       FROM
+         fxa.accountCustomers
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_customers_v1/schema.yaml

@@ -0,0 +1,14 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: stripeCustomerId
+  type: STRING
+  mode: NULLABLE
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: updatedAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_reset_tokens_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: accountResetTokens table from production fxa database
+description: >
+  A mirror of the `accountResetTokens` table from the production `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - [email protected]
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_reset_tokens_v1/query.sql

@@ -0,0 +1,13 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
+    """SELECT
+         uid,
+         createdAt
+       FROM
+         fxa.accountResetTokens
+    """
+  )

sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_reset_tokens_v1/schema.yaml

@@ -0,0 +1,8 @@
+fields:
+- name: uid
+  type: STRING
+  mode: NULLABLE
+  description: Account ID in hexadecimal format.
+- name: createdAt
+  type: TIMESTAMP
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/metadata.yaml

@@ -0,0 +1,18 @@
+---
+friendly_name: accounts table from production fxa database
+description: >
+  A mirror of the `accounts` table from the production `fxa` CloudSQL database,
+  excluding columns containing confidential data, updated daily to match the current state of the table.
+
+  See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa
+owners:
+  - [email protected]
+labels:
+  application: accounts_backend
+  schedule: daily
+scheduling:
+  dag_name: bqetl_accounts_backend_external
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  referenced_tables: []

sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/query.sql

@@ -0,0 +1,37 @@
+SELECT
+  TO_HEX(uid) AS uid,
+  normalizedEmail,
+  email,
+  SAFE_CAST(emailVerified AS BOOL) AS emailVerified,
+  verifierVersion,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(verifierSetAt AS INT)) AS verifierSetAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt,
+  locale,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(lockedAt AS INT)) AS lockedAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(profileChangedAt AS INT)) AS profileChangedAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(keysChangedAt AS INT)) AS keysChangedAt,
+  ecosystemAnonId,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(disabledAt AS INT)) AS disabledAt,
+  SAFE.TIMESTAMP_MILLIS(SAFE_CAST(metricsOptOutAt AS INT)) AS metricsOptOutAt,
+FROM
+  EXTERNAL_QUERY(
+    "moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa",
+    """SELECT
+         uid,
+         normalizedEmail,
+         email,
+         emailVerified,
+         verifierVersion,
+         verifierSetAt,
+         createdAt,
+         locale,
+         lockedAt,
+         profileChangedAt,
+         keysChangedAt,
+         ecosystemAnonId,
+         disabledAt,
+         metricsOptOutAt
+       FROM
+         fxa.accounts
+    """
+  )