[run-tests] Fix get_table_schemas for views

Author: scholtzan

bigquery_etl/dryrun.py

@@ -106,10 +106,12 @@ def __init__(
         dataset=None,
         table=None,
         billing_project=None,
+        ignore_content=False,
     ):
         """Instantiate DryRun class."""
         self.sqlfile = sqlfile
         self.content = content
+        self.ignore_content = ignore_content
         self.query_parameters = query_parameters
         self.strip_dml = strip_dml
         self.use_cloud_function = use_cloud_function
@@ -228,10 +230,13 @@ def get_sql(self):
     @cached_property
     def dry_run_result(self):
         """Dry run the provided SQL file."""
-        if self.content:
-            sql = self.content
+        if self.ignore_content:
+            sql = None
         else:
-            sql = self.get_sql()
+            if self.content:
+                sql = self.content
+            elif self.content != "":
+                sql = self.get_sql()
 
         query_parameters = []
         if self.query_parameters:
@@ -595,6 +600,7 @@ def validate_schema(self):
             client=self.client,
             id_token=self.id_token,
             partitioned_by=partitioned_by,
+            filename=basename(self.sqlfile),
         )
 
         # This check relies on the new schema being deployed to prod

bigquery_etl/schema/__init__.py

@@ -13,6 +13,7 @@
 from google.cloud.bigquery import SchemaField
 
 from .. import dryrun
+from ..config import ConfigLoader
 
 SCHEMA_FILE = "schema.yaml"
 
@@ -58,24 +59,30 @@ def from_json(cls, json_schema):
         return cls(json_schema)
 
     @classmethod
-    def for_table(cls, project, dataset, table, partitioned_by=None, *args, **kwargs):
+    def for_table(
+        cls,
+        project,
+        dataset,
+        table,
+        partitioned_by=None,
+        filename="query.sql",
+        *args,
+        **kwargs,
+    ):
         """Get the schema for a BigQuery table."""
-        query = f"SELECT * FROM `{project}.{dataset}.{table}`"
-
-        if partitioned_by:
-            query += f" WHERE DATE(`{partitioned_by}`) = DATE('2020-01-01')"
-
         try:
+            sql_dir = ConfigLoader.get("default", "sql_dir")
             return cls(
                 dryrun.DryRun(
-                    os.path.join(project, dataset, table, "query.sql"),
-                    query,
+                    os.path.join(sql_dir, project, dataset, table, filename),
                     project=project,
                     dataset=dataset,
                     table=table,
+                    respect_skip=False,
+                    ignore_content=True,
                     *args,
                     **kwargs,
-                ).get_schema()
+                ).get_table_schema()
             )
         except Exception as e:
             print(f"Cannot get schema for {project}.{dataset}.{table}: {e}")

sql_generators/README.md

@@ -9,3 +9,4 @@ The directories in `sql_generators/` represent the generated queries and will co
 Each `__init__.py` file needs to implement a `generate()` method that is configured as a [click command](https://click.palletsprojects.com/en/8.0.x/). The `bqetl` CLI will automatically add these commands to the `./bqetl query generate` command group.
 
 After changes to a schema or adding new tables, the schema is automatically derived from the query and deployed the next day in DAG [bqetl_artifact_deployment](https://workflow.telemetry.mozilla.org/dags/bqetl_artifact_deployment/grid). Alternatively, it can be manually generated and deployed using `./bqetl generate all` and `./bqetl query schema deploy`.
+

sql_generators/glean_usage/glean_app_ping_views.py

@@ -149,6 +149,7 @@ def _process_ping(ping_name):
                                 partitioned_by="submission_timestamp",
                                 use_cloud_function=use_cloud_function,
                                 id_token=id_token,
+                                filename="view.sql",
                             )
                             if schema.schema["fields"] != []:
                                 break