chore: remove job config and request all stations

JasperHG90 · Jan 24, 2024 · 1364e2e · 1364e2e
1 parent 3992847
commit 1364e2e
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 8 deletions.
diff --git a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/jobs.py b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/jobs.py
@@ -14,11 +14,4 @@
     description="Ingestion job for air quality data",
     partitions_def=daily_station_partition,
     executor_def=multiprocess_executor if environment == "dev" else k8s_job_executor,
-    config={
-        "execution": {
-            "config": {
-                "max_concurrent": 3,
-            }
-        }
-    },
 )
diff --git a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/sensors.py b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/sensors.py
@@ -52,7 +52,7 @@ def slack_message_on_failure(context: RunFailureSensorContext, slack: SlackResou
 @sensor(job=ingestion_job)
 def stations_sensor(context, luchtmeetnet_api: LuchtMeetNetResource):
     # Only take first three stations for demo purposes
-    stations_request = luchtmeetnet_api.request("stations")[:2]
+    stations_request = luchtmeetnet_api.request("stations")
     context.log.debug(stations_request)
     stations = [
         f["number"]