Skip to content

Commit ba2bd7c

Browse files
akkomarakkomar
committed
Use time travel to avoid losing data during backfills
1 parent 154d7ed commit ba2bd7c

File tree

4 files changed

+141
-42
lines changed

4 files changed

+141
-42
lines changed

‎bqetl_project.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ dry_run:
4242
- sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_emails_v1/query.sql
4343
- sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
4444
- sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
45+
# uses time travel, will error on current date
46+
- sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql
4547
- sql/moz-fx-data-shared-prod/accounts_db_external/**/*.sql
4648
- sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/**/*.sql
4749
- sql/moz-fx-data-shared-prod/ads/ppa_measurements/*.sql

‎sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/metadata.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,18 @@ description: |-
33
Simple aggregation of counts of records in the FxA DB tables.
44
Enables to identify trends within accounts data. E.g. "How many
55
inactive accounts are there?"
6-
Note: because its source tables are overwritten daily, query used to
7-
populate this table is not idempotent so this table should not be backfilled.
86
owners:
97
- [email protected]
108
labels:
119
incremental: true
1210
owner1: [email protected]
1311
scheduling:
1412
dag_name: bqetl_accounts_derived
13+
parameters:
14+
- "as_of_date:DATE:{{ ds }}"
1515
bigquery:
1616
time_partitioning:
1717
type: day
18-
field: date
18+
field: as_of_date
1919
require_partition_filter: false
2020
expiration_days: null

‎sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql‎

Lines changed: 135 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -3,184 +3,278 @@ WITH table_counts AS (
33
'account_customers' AS table_name,
44
COUNT(*) AS total_rows
55
FROM
6-
`moz-fx-data-shared-prod.accounts_db_external.fxa_account_customers_v1`
6+
`moz-fx-data-shared-prod.accounts_db_external.fxa_account_customers_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
7+
@as_of_date,
8+
'UTC'
9+
)
710
UNION ALL
811
SELECT
912
'account_groups' AS table_name,
1013
COUNT(*) AS total_rows
1114
FROM
12-
`moz-fx-data-shared-prod.accounts_db_external.fxa_account_groups_v1`
15+
`moz-fx-data-shared-prod.accounts_db_external.fxa_account_groups_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
16+
@as_of_date,
17+
'UTC'
18+
)
1319
UNION ALL
1420
SELECT
1521
'account_reset_tokens' AS table_name,
1622
COUNT(*) AS total_rows
1723
FROM
18-
`moz-fx-data-shared-prod.accounts_db_external.fxa_account_reset_tokens_v1`
24+
`moz-fx-data-shared-prod.accounts_db_external.fxa_account_reset_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
25+
@as_of_date,
26+
'UTC'
27+
)
1928
UNION ALL
2029
SELECT
2130
'accounts' AS table_name,
2231
COUNT(*) AS total_rows
2332
FROM
24-
`moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1`
33+
`moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
34+
@as_of_date,
35+
'UTC'
36+
)
2537
UNION ALL
2638
SELECT
2739
'carts' AS table_name,
2840
COUNT(*) AS total_rows
2941
FROM
30-
`moz-fx-data-shared-prod.accounts_db_external.fxa_carts_v1`
42+
`moz-fx-data-shared-prod.accounts_db_external.fxa_carts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
43+
@as_of_date,
44+
'UTC'
45+
)
3146
UNION ALL
3247
SELECT
3348
'device_commands' AS table_name,
3449
COUNT(*) AS total_rows
3550
FROM
36-
`moz-fx-data-shared-prod.accounts_db_external.fxa_device_commands_v1`
51+
`moz-fx-data-shared-prod.accounts_db_external.fxa_device_commands_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
52+
@as_of_date,
53+
'UTC'
54+
)
3755
UNION ALL
3856
SELECT
3957
'devices' AS table_name,
4058
COUNT(*) AS total_rows
4159
FROM
42-
`moz-fx-data-shared-prod.accounts_db_external.fxa_devices_v1`
60+
`moz-fx-data-shared-prod.accounts_db_external.fxa_devices_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
61+
@as_of_date,
62+
'UTC'
63+
)
4364
UNION ALL
4465
SELECT
4566
'email_bounces' AS table_name,
4667
COUNT(*) AS total_rows
4768
FROM
48-
`moz-fx-data-shared-prod.accounts_db_external.fxa_email_bounces_v1`
69+
`moz-fx-data-shared-prod.accounts_db_external.fxa_email_bounces_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
70+
@as_of_date,
71+
'UTC'
72+
)
4973
UNION ALL
5074
SELECT
5175
'emails' AS table_name,
5276
COUNT(*) AS total_rows
5377
FROM
54-
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1`
78+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
79+
@as_of_date,
80+
'UTC'
81+
)
5582
UNION ALL
5683
SELECT
5784
'linked_accounts' AS table_name,
5885
COUNT(*) AS total_rows
5986
FROM
60-
`moz-fx-data-shared-prod.accounts_db_external.fxa_linked_accounts_v1`
87+
`moz-fx-data-shared-prod.accounts_db_external.fxa_linked_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
88+
@as_of_date,
89+
'UTC'
90+
)
6191
UNION ALL
6292
SELECT
6393
'oauth_codes' AS table_name,
6494
COUNT(*) AS total_rows
6595
FROM
66-
`moz-fx-data-shared-prod.accounts_db_external.fxa_oauth_codes_v1`
96+
`moz-fx-data-shared-prod.accounts_db_external.fxa_oauth_codes_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
97+
@as_of_date,
98+
'UTC'
99+
)
67100
UNION ALL
68101
SELECT
69102
'oauth_refresh_tokens' AS table_name,
70103
COUNT(*) AS total_rows
71104
FROM
72-
`moz-fx-data-shared-prod.accounts_db_external.fxa_oauth_refresh_tokens_v1`
105+
`moz-fx-data-shared-prod.accounts_db_external.fxa_oauth_refresh_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
106+
@as_of_date,
107+
'UTC'
108+
)
73109
UNION ALL
74110
SELECT
75111
'oauth_tokens' AS table_name,
76112
COUNT(*) AS total_rows
77113
FROM
78-
`moz-fx-data-shared-prod.accounts_db_external.fxa_oauth_tokens_v1`
114+
`moz-fx-data-shared-prod.accounts_db_external.fxa_oauth_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
115+
@as_of_date,
116+
'UTC'
117+
)
79118
UNION ALL
80119
SELECT
81120
'password_change_tokens' AS table_name,
82121
COUNT(*) AS total_rows
83122
FROM
84-
`moz-fx-data-shared-prod.accounts_db_external.fxa_password_change_tokens_v1`
123+
`moz-fx-data-shared-prod.accounts_db_external.fxa_password_change_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
124+
@as_of_date,
125+
'UTC'
126+
)
85127
UNION ALL
86128
SELECT
87129
'password_forgot_tokens' AS table_name,
88130
COUNT(*) AS total_rows
89131
FROM
90-
`moz-fx-data-shared-prod.accounts_db_external.fxa_password_forgot_tokens_v1`
132+
`moz-fx-data-shared-prod.accounts_db_external.fxa_password_forgot_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
133+
@as_of_date,
134+
'UTC'
135+
)
91136
UNION ALL
92137
SELECT
93138
'paypal_customers' AS table_name,
94139
COUNT(*) AS total_rows
95140
FROM
96-
`moz-fx-data-shared-prod.accounts_db_external.fxa_paypal_customers_v1`
141+
`moz-fx-data-shared-prod.accounts_db_external.fxa_paypal_customers_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
142+
@as_of_date,
143+
'UTC'
144+
)
97145
UNION ALL
98146
SELECT
99147
'recovery_codes' AS table_name,
100148
COUNT(*) AS total_rows
101149
FROM
102-
`moz-fx-data-shared-prod.accounts_db_external.fxa_recovery_codes_v1`
150+
`moz-fx-data-shared-prod.accounts_db_external.fxa_recovery_codes_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
151+
@as_of_date,
152+
'UTC'
153+
)
103154
UNION ALL
104155
SELECT
105156
'security_events' AS table_name,
106157
COUNT(*) AS total_rows
107158
FROM
108-
`moz-fx-data-shared-prod.accounts_db_external.fxa_security_events_v1`
159+
`moz-fx-data-shared-prod.accounts_db_external.fxa_security_events_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
160+
@as_of_date,
161+
'UTC'
162+
)
109163
UNION ALL
110164
SELECT
111165
'sent_emails' AS table_name,
112166
COUNT(*) AS total_rows
113167
FROM
114-
`moz-fx-data-shared-prod.accounts_db_external.fxa_sent_emails_v1`
168+
`moz-fx-data-shared-prod.accounts_db_external.fxa_sent_emails_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
169+
@as_of_date,
170+
'UTC'
171+
)
115172
UNION ALL
116173
SELECT
117174
'session_tokens' AS table_name,
118175
COUNT(*) AS total_rows
119176
FROM
120-
`moz-fx-data-shared-prod.accounts_db_external.fxa_session_tokens_v1`
177+
`moz-fx-data-shared-prod.accounts_db_external.fxa_session_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
178+
@as_of_date,
179+
'UTC'
180+
)
121181
UNION ALL
122182
SELECT
123183
'signin_codes' AS table_name,
124184
COUNT(*) AS total_rows
125185
FROM
126-
`moz-fx-data-shared-prod.accounts_db_external.fxa_signin_codes_v1`
186+
`moz-fx-data-shared-prod.accounts_db_external.fxa_signin_codes_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
187+
@as_of_date,
188+
'UTC'
189+
)
127190
UNION ALL
128191
SELECT
129192
'totp' AS table_name,
130193
COUNT(*) AS total_rows
131194
FROM
132-
`moz-fx-data-shared-prod.accounts_db_external.fxa_totp_v1`
195+
`moz-fx-data-shared-prod.accounts_db_external.fxa_totp_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
196+
@as_of_date,
197+
'UTC'
198+
)
133199
UNION ALL
134200
SELECT
135201
'unblock_codes' AS table_name,
136202
COUNT(*) AS total_rows
137203
FROM
138-
`moz-fx-data-shared-prod.accounts_db_external.fxa_unblock_codes_v1`
204+
`moz-fx-data-shared-prod.accounts_db_external.fxa_unblock_codes_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
205+
@as_of_date,
206+
'UTC'
207+
)
139208
UNION ALL
140209
SELECT
141210
'unverified_tokens' AS table_name,
142211
COUNT(*) AS total_rows
143212
FROM
144-
`moz-fx-data-shared-prod.accounts_db_external.fxa_unverified_tokens_v1`
213+
`moz-fx-data-shared-prod.accounts_db_external.fxa_unverified_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
214+
@as_of_date,
215+
'UTC'
216+
)
145217
UNION ALL
146218
SELECT
147219
'verification_reminders' AS table_name,
148220
COUNT(*) AS total_rows
149221
FROM
150-
`moz-fx-data-shared-prod.accounts_db_external.fxa_verification_reminders_v1`
222+
`moz-fx-data-shared-prod.accounts_db_external.fxa_verification_reminders_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
223+
@as_of_date,
224+
'UTC'
225+
)
151226
UNION ALL
152227
(
153228
SELECT
154229
"accounts_with_secondary_emails" AS table_name,
155-
COUNT(DISTINCT accounts.uid) AS total_rows
230+
COUNT(
231+
DISTINCT `moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1`.uid
232+
) AS total_rows
156233
FROM
157-
`moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1` accounts
234+
`moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
235+
@as_of_date,
236+
'UTC'
237+
)
158238
JOIN
159-
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` emails
160-
ON accounts.uid = emails.uid
239+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
240+
@as_of_date,
241+
'UTC'
242+
)
243+
ON `moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1`.uid = `moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1`.uid
161244
WHERE
162-
emails.isPrimary = FALSE
245+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1`.isPrimary = FALSE
163246
)
164247
UNION ALL
165248
(
166249
SELECT
167250
"accounts_with_unverified_emails" AS table_name,
168-
COUNT(DISTINCT accounts.uid) AS total_rows
251+
COUNT(
252+
DISTINCT `moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1`.uid
253+
) AS total_rows
169254
FROM
170-
`moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1` accounts
255+
`moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
256+
@as_of_date,
257+
'UTC'
258+
)
171259
JOIN
172-
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` emails
173-
ON accounts.uid = emails.uid
260+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
261+
@as_of_date,
262+
'UTC'
263+
)
264+
ON `moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1`.uid = `moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1`.uid
174265
WHERE
175-
emails.isVerified = FALSE
266+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1`.isVerified = FALSE
176267
)
177268
UNION ALL
178269
(
179270
SELECT
180271
"accounts_linked_to_google" AS table_name,
181272
COUNT(uid) AS total_rows
182273
FROM
183-
`moz-fx-data-shared-prod.accounts_db_external.fxa_linked_accounts_v1`
274+
`moz-fx-data-shared-prod.accounts_db_external.fxa_linked_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
275+
@as_of_date,
276+
'UTC'
277+
)
184278
WHERE
185279
providerId = 1 -- see LinkedAccountProviderIds at https://github.com/mozilla/fxa/blob/main/packages/fxa-settings/src/lib/types.ts
186280
)
@@ -190,13 +284,16 @@ WITH table_counts AS (
190284
"accounts_linked_to_apple" AS table_name,
191285
COUNT(uid) AS total_rows
192286
FROM
193-
`moz-fx-data-shared-prod.accounts_db_external.fxa_linked_accounts_v1`
287+
`moz-fx-data-shared-prod.accounts_db_external.fxa_linked_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
288+
@as_of_date,
289+
'UTC'
290+
)
194291
WHERE
195292
providerId = 2 -- see LinkedAccountProviderIds at https://github.com/mozilla/fxa/blob/main/packages/fxa-settings/src/lib/types.ts
196293
)
197294
)
198295
SELECT
199-
@submission_date AS date,
296+
@as_of_date AS as_of_date,
200297
table_name,
201298
total_rows
202299
FROM

‎sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/schema.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
fields:
2-
- name: date
2+
- name: as_of_date
33
type: DATE
44
mode: NULLABLE
55
- name: table_name

0 commit comments

Comments
 (0)