Add related files to set Freshness and Volume on all glean tables

bigquery_etl/cli/monitoring.py

@@ -173,7 +173,7 @@ def update(name: str, sql_dir: Optional[str], project_id: Optional[str]) -> None
                                     ),
                                     metric_schedule=SimpleMetricSchedule(
                                         named_schedule=SimpleNamedSchedule(
-                                            name="Default Schedule - 17:00 UTC"
+                                            name="Default Schedule - 13:00 UTC"
                                         )
                                     ),
                                 )

sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/releases_v1/bigconfig.yml

@@ -10,10 +10,10 @@ table_deployments:
         predefined_metric: FRESHNESS
       metric_schedule:
         named_schedule:
-          name: Default Schedule - 17:00 UTC
+          name: Default Schedule - 13:00 UTC
     - metric_type:
         type: PREDEFINED
         predefined_metric: VOLUME
       metric_schedule:
         named_schedule:
-          name: Default Schedule - 17:00 UTC
+          name: Default Schedule - 13:00 UTC

sql_generators/glean_usage/common.py

@@ -227,6 +227,7 @@ def generate_per_app_id(
         view_metadata_filename = f"{self.target_table_id[:-3]}.metadata.yaml"
         table_metadata_filename = f"{self.target_table_id}.metadata.yaml"
         schema_filename = f"{self.target_table_id}.schema.yaml"
+        bigconfig_filename = f"{self.target_table_id}.bigconfig.yml"
 
         table = tables[f"{self.prefix}_table"]
         view = tables[f"{self.prefix}_view"]
@@ -248,6 +249,7 @@ def generate_per_app_id(
             app_name=app_name,
             has_distribution_id=app_name in APPS_WITH_DISTRIBUTION_ID,
             has_profile_group_id= app_name in APPS_WITH_PROFILE_GROUP_ID,
+            target_table=f"{self.target_table_id}",
         )
 
         render_kwargs.update(self.custom_render_kwargs)
@@ -291,6 +293,17 @@ def generate_per_app_id(
             )
         except TemplateNotFound:
             schema = None
+
+         # Bigconfig files are optional
+        try:
+            bigconfig = render(
+                bigconfig_filename,
+                format=False,
+                template_folder=PATH / "templates",
+                **render_kwargs,
+            )
+        except TemplateNotFound:
+            bigconfig = None
 
         # generated files to update
         Artifact = namedtuple("Artifact", "table_id basename sql")
@@ -319,6 +332,9 @@ def generate_per_app_id(
 
             if schema:
                 artifacts.append(Artifact(table, "schema.yaml", schema))
+
+            if bigconfig:
+                artifacts.append(Artifact(table, "bigconfig.yml", bigconfig))
 
             for artifact in artifacts:
                 destination = (

sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.bigconfig.yml

@@ -0,0 +1,20 @@
+type: BIGCONFIG_FILE
+table_deployments:
+- collection:
+    name: Test
+  deployments:
+  - fq_table_name: {{ project_id }}.{{ project_id }}.{{ derived_dataset }}.{{ target_table }}
+    table_metrics:
+    - metric_type:
+        type: PREDEFINED
+        predefined_metric: FRESHNESS
+      metric_schedule:
+        named_schedule:
+          name: Default Schedule - 13:00 UTC
+    - metric_type:
+        type: PREDEFINED
+        predefined_metric: VOLUME
+      metric_schedule:
+        named_schedule:
+          name: Default Schedule - 13:00 UTC
+

sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml

@@ -28,3 +28,6 @@ bigquery:
 schema:
   derived_from:
   - table: ['moz-fx-data-shared-prod', '{{ derived_dataset }}', 'baseline_clients_daily_v1']
+monitoring:
+  enabled: true
+