From f697220b95c74164177a85187a6d14df6f3c027a Mon Sep 17 00:00:00 2001 From: Jasper Ginn Date: Fri, 1 Mar 2024 14:22:41 +0100 Subject: [PATCH] chore: add cron schedule --- .../src/luchtmeetnet_ingestion/assets/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/assets/__init__.py b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/assets/__init__.py index 3a3d4c0..96e9bc2 100644 --- a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/assets/__init__.py +++ b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/assets/__init__.py @@ -93,7 +93,11 @@ def daily_air_quality_data( description="Luchtmeetnet API stations", partitions_def=stations_partition, # Setting max_materializations_per_minute disables the rate limiter - auto_materialize_policy=AutoMaterializePolicy.eager(max_materializations_per_minute=None), + auto_materialize_policy=AutoMaterializePolicy.eager( + max_materializations_per_minute=None + ).with_rules( + AutoMaterializeRule.materialize_on_cron("0 0 1 * *", all_partitions=True), + ), group_name="stations", ) def station_names( @@ -116,7 +120,7 @@ def station_names( AutoMaterializeRule.materialize_on_required_for_freshness(), ), freshness_policy=FreshnessPolicy( - maximum_lag_minutes=60 * 24, + maximum_lag_minutes=10, cron_schedule="0 0 1 * *", ), group_name="stations",