Skip to content

Commit aa5073d

Browse files
akkomarsean-rose
akkomar
and
sean-rose
authored
DSRE-1779 Add FxA monitoring query (#6511)
* Add FxA monitoring query * Add schema * Fix schema * Use time travel to avoid losing data during backfills * update time travel comment * Update sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/metadata.yaml Co-authored-by: Sean Rose <[email protected]> * Update sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql Co-authored-by: Sean Rose <[email protected]> * s/@as_of_date,/@as_of_date+1,/g --------- Co-authored-by: Sean Rose <[email protected]>
1 parent aa0e1de commit aa5073d

File tree

4 files changed

+331
-0
lines changed

4 files changed

+331
-0
lines changed

‎bqetl_project.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ dry_run:
4242
- sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_emails_v1/query.sql
4343
- sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
4444
- sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
45+
# uses time travel, will error on dates prior to the time travel window
46+
- sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql
4547
- sql/moz-fx-data-shared-prod/accounts_db_external/**/*.sql
4648
- sql/moz-fx-data-shared-prod/accounts_db_nonprod_external/**/*.sql
4749
- sql/moz-fx-data-shared-prod/ads/ppa_measurements/*.sql

‎sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/metadata.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
friendly_name: FxA DB Counts Monitoring
2+
description: |-
3+
Simple aggregation of counts of records in the FxA DB tables.
4+
Enables to identify trends within accounts data. E.g. "How many
5+
inactive accounts are there?"
6+
owners:
7+
- [email protected]
8+
labels:
9+
incremental: true
10+
owner1: [email protected]
11+
scheduling:
12+
dag_name: bqetl_accounts_derived
13+
date_partition_parameter: as_of_date
14+
bigquery:
15+
time_partitioning:
16+
type: day
17+
field: as_of_date
18+
require_partition_filter: false
19+
expiration_days: null

‎sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql‎

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
WITH table_counts AS (
2+
SELECT
3+
'account_customers' AS table_name,
4+
COUNT(*) AS total_rows
5+
FROM
6+
`moz-fx-data-shared-prod.accounts_db_external.fxa_account_customers_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
7+
@as_of_date + 1,
8+
'UTC'
9+
)
10+
UNION ALL
11+
SELECT
12+
'account_groups' AS table_name,
13+
COUNT(*) AS total_rows
14+
FROM
15+
`moz-fx-data-shared-prod.accounts_db_external.fxa_account_groups_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
16+
@as_of_date + 1,
17+
'UTC'
18+
)
19+
UNION ALL
20+
SELECT
21+
'account_reset_tokens' AS table_name,
22+
COUNT(*) AS total_rows
23+
FROM
24+
`moz-fx-data-shared-prod.accounts_db_external.fxa_account_reset_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
25+
@as_of_date + 1,
26+
'UTC'
27+
)
28+
UNION ALL
29+
SELECT
30+
'accounts' AS table_name,
31+
COUNT(*) AS total_rows
32+
FROM
33+
`moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
34+
@as_of_date + 1,
35+
'UTC'
36+
)
37+
UNION ALL
38+
SELECT
39+
'carts' AS table_name,
40+
COUNT(*) AS total_rows
41+
FROM
42+
`moz-fx-data-shared-prod.accounts_db_external.fxa_carts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
43+
@as_of_date + 1,
44+
'UTC'
45+
)
46+
UNION ALL
47+
SELECT
48+
'device_commands' AS table_name,
49+
COUNT(*) AS total_rows
50+
FROM
51+
`moz-fx-data-shared-prod.accounts_db_external.fxa_device_commands_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
52+
@as_of_date + 1,
53+
'UTC'
54+
)
55+
UNION ALL
56+
SELECT
57+
'devices' AS table_name,
58+
COUNT(*) AS total_rows
59+
FROM
60+
`moz-fx-data-shared-prod.accounts_db_external.fxa_devices_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
61+
@as_of_date + 1,
62+
'UTC'
63+
)
64+
UNION ALL
65+
SELECT
66+
'email_bounces' AS table_name,
67+
COUNT(*) AS total_rows
68+
FROM
69+
`moz-fx-data-shared-prod.accounts_db_external.fxa_email_bounces_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
70+
@as_of_date + 1,
71+
'UTC'
72+
)
73+
UNION ALL
74+
SELECT
75+
'emails' AS table_name,
76+
COUNT(*) AS total_rows
77+
FROM
78+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
79+
@as_of_date + 1,
80+
'UTC'
81+
)
82+
UNION ALL
83+
SELECT
84+
'linked_accounts' AS table_name,
85+
COUNT(*) AS total_rows
86+
FROM
87+
`moz-fx-data-shared-prod.accounts_db_external.fxa_linked_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
88+
@as_of_date + 1,
89+
'UTC'
90+
)
91+
UNION ALL
92+
SELECT
93+
'oauth_codes' AS table_name,
94+
COUNT(*) AS total_rows
95+
FROM
96+
`moz-fx-data-shared-prod.accounts_db_external.fxa_oauth_codes_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
97+
@as_of_date + 1,
98+
'UTC'
99+
)
100+
UNION ALL
101+
SELECT
102+
'oauth_refresh_tokens' AS table_name,
103+
COUNT(*) AS total_rows
104+
FROM
105+
`moz-fx-data-shared-prod.accounts_db_external.fxa_oauth_refresh_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
106+
@as_of_date + 1,
107+
'UTC'
108+
)
109+
UNION ALL
110+
SELECT
111+
'oauth_tokens' AS table_name,
112+
COUNT(*) AS total_rows
113+
FROM
114+
`moz-fx-data-shared-prod.accounts_db_external.fxa_oauth_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
115+
@as_of_date + 1,
116+
'UTC'
117+
)
118+
UNION ALL
119+
SELECT
120+
'password_change_tokens' AS table_name,
121+
COUNT(*) AS total_rows
122+
FROM
123+
`moz-fx-data-shared-prod.accounts_db_external.fxa_password_change_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
124+
@as_of_date + 1,
125+
'UTC'
126+
)
127+
UNION ALL
128+
SELECT
129+
'password_forgot_tokens' AS table_name,
130+
COUNT(*) AS total_rows
131+
FROM
132+
`moz-fx-data-shared-prod.accounts_db_external.fxa_password_forgot_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
133+
@as_of_date + 1,
134+
'UTC'
135+
)
136+
UNION ALL
137+
SELECT
138+
'paypal_customers' AS table_name,
139+
COUNT(*) AS total_rows
140+
FROM
141+
`moz-fx-data-shared-prod.accounts_db_external.fxa_paypal_customers_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
142+
@as_of_date + 1,
143+
'UTC'
144+
)
145+
UNION ALL
146+
SELECT
147+
'recovery_codes' AS table_name,
148+
COUNT(*) AS total_rows
149+
FROM
150+
`moz-fx-data-shared-prod.accounts_db_external.fxa_recovery_codes_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
151+
@as_of_date + 1,
152+
'UTC'
153+
)
154+
UNION ALL
155+
SELECT
156+
'security_events' AS table_name,
157+
COUNT(*) AS total_rows
158+
FROM
159+
`moz-fx-data-shared-prod.accounts_db_external.fxa_security_events_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
160+
@as_of_date + 1,
161+
'UTC'
162+
)
163+
UNION ALL
164+
SELECT
165+
'sent_emails' AS table_name,
166+
COUNT(*) AS total_rows
167+
FROM
168+
`moz-fx-data-shared-prod.accounts_db_external.fxa_sent_emails_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
169+
@as_of_date + 1,
170+
'UTC'
171+
)
172+
UNION ALL
173+
SELECT
174+
'session_tokens' AS table_name,
175+
COUNT(*) AS total_rows
176+
FROM
177+
`moz-fx-data-shared-prod.accounts_db_external.fxa_session_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
178+
@as_of_date + 1,
179+
'UTC'
180+
)
181+
UNION ALL
182+
SELECT
183+
'signin_codes' AS table_name,
184+
COUNT(*) AS total_rows
185+
FROM
186+
`moz-fx-data-shared-prod.accounts_db_external.fxa_signin_codes_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
187+
@as_of_date + 1,
188+
'UTC'
189+
)
190+
UNION ALL
191+
SELECT
192+
'totp' AS table_name,
193+
COUNT(*) AS total_rows
194+
FROM
195+
`moz-fx-data-shared-prod.accounts_db_external.fxa_totp_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
196+
@as_of_date + 1,
197+
'UTC'
198+
)
199+
UNION ALL
200+
SELECT
201+
'unblock_codes' AS table_name,
202+
COUNT(*) AS total_rows
203+
FROM
204+
`moz-fx-data-shared-prod.accounts_db_external.fxa_unblock_codes_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
205+
@as_of_date + 1,
206+
'UTC'
207+
)
208+
UNION ALL
209+
SELECT
210+
'unverified_tokens' AS table_name,
211+
COUNT(*) AS total_rows
212+
FROM
213+
`moz-fx-data-shared-prod.accounts_db_external.fxa_unverified_tokens_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
214+
@as_of_date + 1,
215+
'UTC'
216+
)
217+
UNION ALL
218+
SELECT
219+
'verification_reminders' AS table_name,
220+
COUNT(*) AS total_rows
221+
FROM
222+
`moz-fx-data-shared-prod.accounts_db_external.fxa_verification_reminders_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
223+
@as_of_date + 1,
224+
'UTC'
225+
)
226+
UNION ALL
227+
(
228+
SELECT
229+
"accounts_with_secondary_emails" AS table_name,
230+
COUNT(
231+
DISTINCT `moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1`.uid
232+
) AS total_rows
233+
FROM
234+
`moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
235+
@as_of_date + 1,
236+
'UTC'
237+
)
238+
JOIN
239+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
240+
@as_of_date + 1,
241+
'UTC'
242+
)
243+
ON `moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1`.uid = `moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1`.uid
244+
WHERE
245+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1`.isPrimary = FALSE
246+
)
247+
UNION ALL
248+
(
249+
SELECT
250+
"accounts_with_unverified_emails" AS table_name,
251+
COUNT(
252+
DISTINCT `moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1`.uid
253+
) AS total_rows
254+
FROM
255+
`moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
256+
@as_of_date + 1,
257+
'UTC'
258+
)
259+
JOIN
260+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
261+
@as_of_date + 1,
262+
'UTC'
263+
)
264+
ON `moz-fx-data-shared-prod.accounts_db_external.fxa_accounts_v1`.uid = `moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1`.uid
265+
WHERE
266+
`moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1`.isVerified = FALSE
267+
)
268+
UNION ALL
269+
(
270+
SELECT
271+
"accounts_linked_to_google" AS table_name,
272+
COUNT(uid) AS total_rows
273+
FROM
274+
`moz-fx-data-shared-prod.accounts_db_external.fxa_linked_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
275+
@as_of_date + 1,
276+
'UTC'
277+
)
278+
WHERE
279+
providerId = 1 -- see LinkedAccountProviderIds at https://github.com/mozilla/fxa/blob/main/packages/fxa-settings/src/lib/types.ts
280+
)
281+
UNION ALL
282+
(
283+
SELECT
284+
"accounts_linked_to_apple" AS table_name,
285+
COUNT(uid) AS total_rows
286+
FROM
287+
`moz-fx-data-shared-prod.accounts_db_external.fxa_linked_accounts_v1` FOR SYSTEM_TIME AS OF TIMESTAMP(
288+
@as_of_date + 1,
289+
'UTC'
290+
)
291+
WHERE
292+
providerId = 2 -- see LinkedAccountProviderIds at https://github.com/mozilla/fxa/blob/main/packages/fxa-settings/src/lib/types.ts
293+
)
294+
)
295+
SELECT
296+
@as_of_date AS as_of_date,
297+
table_name,
298+
total_rows
299+
FROM
300+
table_counts

‎sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/schema.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
fields:
2+
- name: as_of_date
3+
type: DATE
4+
mode: NULLABLE
5+
- name: table_name
6+
type: STRING
7+
mode: NULLABLE
8+
- name: total_rows
9+
type: INTEGER
10+
mode: NULLABLE

0 commit comments

Comments
 (0)