Merge branch 'main' into query-schema-generation-dryrun-caching

Author: scholtzan

bigquery_etl/format_sql/format.py

@@ -4,7 +4,7 @@
 import os
 import os.path
 import sys
-from functools import partial
+from functools import cache, partial
 from multiprocessing.pool import Pool
 from pathlib import Path
 from typing import Tuple
@@ -16,6 +16,7 @@
 from bigquery_etl.util.common import qualify_table_references_in_file
 
 
+@cache
 def skip_format():
     """Return a list of configured queries for which formatting should be skipped."""
     return [
@@ -25,6 +26,7 @@ def skip_format():
     ]
 
 
+@cache
 def skip_qualifying_references():
     """Return a list of configured queries where fully qualifying references should be skipped."""
     return [

sql/moz-fx-data-shared-prod/monitoring_derived/shredder_targets_new_mismatched_v1/query.sql

@@ -4,7 +4,7 @@ WITH unmatching_sources AS (
   FROM
     `moz-fx-data-shared-prod.monitoring_derived.shredder_targets_joined_v1`
   WHERE
-    matching_sources IS FALSE
+    (matching_sources IS FALSE AND ARRAY_LENGTH(detected_sources) > 0)
     OR ARRAY_LENGTH(current_sources) = 0
 ),
 previous_config AS (

sql/moz-fx-data-shared-prod/monitoring_derived/shredder_targets_v1/metadata.yaml

@@ -10,6 +10,8 @@ labels:
 scheduling:
   dag_name: bqetl_shredder_monitoring
   arguments: ["--output-table", "moz-fx-data-shared-prod.monitoring_derived.shredder_targets_v1", "--run-date", "{{ ds }}"]
+  # downstream alert job uses previous date to determine new table
+  depends_on_past: true
 bigquery:
   time_partitioning:
     type: day

sql/moz-fx-data-shared-prod/monitoring_derived/shredder_targets_v1/query.py

@@ -69,6 +69,8 @@
   AND table_schema != "backfills_staging_derived"
   AND table_name != 'deletion_request_v1'
   AND table_name != 'deletion_request_v4'
+  -- ignore recently created tables that may not have lineage populated yet
+  AND DATE(creation_time) < DATE_SUB('2025-11-24', INTERVAL 1 DAY)
 """
 
 

sql/moz-fx-data-shared-prod/mozilla_org/desktop_conversion_events/metadata.yaml

@@ -8,7 +8,7 @@ labels:
   schedule: daily
   dag: bqetl_google_analytics_derived
   owner1: ascholtz
-#scheduling:
+# scheduling:
 #  dag_name: bqetl_google_analytics_derived
 bigquery:
   time_partitioning:

sql/moz-fx-data-shared-prod/mozilla_org/desktop_conversion_events/schema.yaml

@@ -32,7 +32,7 @@ labels:
   incremental: true
   schedule: daily
   owner1: gleonard
-#scheduling:
+# scheduling:
 #  dag_name: bqetl_google_analytics_derived
 bigquery:
   time_partitioning:

sql/moz-fx-data-shared-prod/mozilla_org/desktop_conversion_events/view.sql

@@ -5,7 +5,7 @@ description: |-
 owners:
 - [email protected]
 scheduling:
-  #dag_name: bqetl_mozilla_org_derived - GA3 feed turned off 2024-06-30 so no need to run anymore
+  # dag_name: bqetl_mozilla_org_derived - GA3 feed turned off 2024-06-30 so no need to run anymore
   depends_on_past: true
   date_partition_parameter: null
   parameters: ["session_date:DATE:{{ds}}"]