diff --git a/bigquery_etl/shredder/config.py b/bigquery_etl/shredder/config.py index 9c0049a9214..839987fc0a0 100755 --- a/bigquery_etl/shredder/config.py +++ b/bigquery_etl/shredder/config.py @@ -151,6 +151,9 @@ def fields(self) -> tuple[str, ...]: FXA_FRONTEND_GLEAN_SRC = DeleteSource( table="accounts_frontend_stable.deletion_request_v1", field=GLEAN_CLIENT_ID ) +SUBPLAT_CIRRUS_SRC = DeleteSource( + table="subscription_platform_backend_cirrus.delete_events", field="nimbus_user_id" +) EXPERIMENTER_BACKEND_SRC = DeleteSource( table="experimenter_backend_stable.data_collection_opt_out_v1", field=NIMBUS_USER_ID, @@ -703,6 +706,14 @@ def fields(self) -> tuple[str, ...]: table="experimenter_cirrus_stable.enrollment_status_v1", field=CIRRUS_EVENTS_NIMBUS_USER_ID, ): EXPERIMENTER_BACKEND_SRC, + DeleteTarget( + table="subscription_platform_backend_cirrus_stable.enrollment_v1", + field=CIRRUS_EVENTS_NIMBUS_USER_ID, + ): SUBPLAT_CIRRUS_SRC, + DeleteTarget( + table="subscription_platform_backend_cirrus_stable.enrollment_status_v1", + field=CIRRUS_EVENTS_NIMBUS_USER_ID, + ): SUBPLAT_CIRRUS_SRC, } SEARCH_IGNORE_TABLES = {source.table for source in SOURCES} diff --git a/bqetl_project.yaml b/bqetl_project.yaml index 37e693d38bf..f067728b1da 100644 --- a/bqetl_project.yaml +++ b/bqetl_project.yaml @@ -54,6 +54,7 @@ dry_run: - sql/moz-fx-data-shared-prod/monitoring_derived/table_storage_v1/query.sql - sql/moz-fx-data-shared-prod/activity_stream/impression_stats_by_experiment/view.sql - sql/moz-fx-data-shared-prod/activity_stream/impression_stats_flat/view.sql + - sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql # uses time travel, will error on dates prior to the time travel window - sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql - sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_recovery_phones_counts_v1/query.sql diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus/delete_events/view.sql b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus/delete_events/view.sql new file mode 100644 index 00000000000..8d0f4906c9f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus/delete_events/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.subscription_platform_backend_cirrus.delete_events` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.subscription_platform_backend_cirrus_derived.delete_events_v1` diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/metadata.yaml new file mode 100644 index 00000000000..9ddc354874f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/metadata.yaml @@ -0,0 +1,17 @@ +--- +friendly_name: Subscription Platform Backend Cirrus Delete Events +description: Deletion events for subscription platform backend cirrus, derived + from FxA's delete events and subplat's namespace secrets +owners: + - mwilliams@mozilla.com +labels: + incremental: true + schedule: daily + table_type: client_level +scheduling: + dag_name: bqetl_fxa_events +bigquery: + time_partitioning: + type: day + field: submission_timestamp + require_partition_filter: false diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql new file mode 100644 index 00000000000..21ea9f40d7a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql @@ -0,0 +1,53 @@ +CREATE TEMP FUNCTION format_uuid(value STRING) +RETURNS STRING AS ( + CONCAT( + SUBSTR(value, 1, 8), + "-", + SUBSTR(value, 9, 4), + "-", + SUBSTR(value, 13, 4), + "-", + SUBSTR(value, 17, 4), + "-", + SUBSTR(value, 21, 12) + ) +); + +CREATE TEMP FUNCTION uuid_v5(value STRING, namespace STRING) +RETURNS STRING AS ( + FORMAT_UUID( + TO_HEX( + LEFT( + SHA1(CONCAT(FROM_HEX(REPLACE(namespace, "-", "")), CAST(value AS BYTES FORMAT 'UTF8'))), + 16 + ) -- + & FROM_HEX('ffffffffffff0fff3fffffffffffffff') -- + | FROM_HEX('00000000000050008000000000000000') + ) + ) +); + +WITH subplat_namespaces AS ( + SELECT + CAST( + AEAD.DECRYPT_BYTES( + (SELECT keyset FROM `moz-fx-dataops-secrets.airflow_query_keys.subplat_namespaces_prod`), + ciphertext, + CAST(key_id AS BYTES) + ) AS STRING + ) AS namespace + FROM + `moz-fx-data-shared-prod.subscription_platform_derived.encrypted_keys_v1` + WHERE + key_id = 'subplat_namespaces_prod' +) +SELECT + fxa_delete_events.submission_timestamp, + uuid_v5(fxa_delete_events.user_id_unhashed, subplat_namespaces.namespace) AS nimbus_user_id, +FROM + `moz-fx-data-shared-prod.firefox_accounts.fxa_delete_events` AS fxa_delete_events +CROSS JOIN + subplat_namespaces +WHERE + DATE(fxa_delete_events.submission_timestamp) = @submission_date + AND fxa_delete_events.user_id_unhashed IS NOT NULL diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/schema.yaml new file mode 100644 index 00000000000..a00fd4eb636 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: submission_timestamp + type: TIMESTAMP + mode: NULLABLE +- name: nimbus_user_id + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_derived/encrypted_keys_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/subscription_platform_derived/encrypted_keys_v1/metadata.yaml new file mode 100644 index 00000000000..8de39c757dc --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_derived/encrypted_keys_v1/metadata.yaml @@ -0,0 +1,2 @@ +friendly_name: Encrypted Keys V1 +description: Encrypted keys