Specify config per job type

openclimatefix · Sep 15, 2023 · c46e57d · c46e57d
1 parent 43f0663
commit c46e57d
Showing 1 changed file with 16 additions and 4 deletions.
diff --git a/nwp/jobs.py b/nwp/jobs.py
@@ -65,22 +65,34 @@ def build_config_on_runtime(model: str, run: str, delay: int = 0) -> dict:
 
 
 @daily_partitioned_config(start_date=dt.datetime(2021, 1, 1))
-def ecmwf_daily_partitioned_config(start: dt.datetime, _end: dt.datetime):
+def ecmwf_daily_partitioned_config_docker(start: dt.datetime, _end: dt.datetime):
     config: NWPConsumerConfig = NWPConsumerConfig(
         date_from=start.strftime("%Y-%m-%d"),
         date_to=start.strftime("%Y-%m-%d"),
         source="ecmwf-mars",
         env_vars=["ECMWF_API_URL", "ECMWF_API_KEY", "ECMWF_API_EMAIL"],
         docker_volumes=['/mnt/storage_b/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/NWP/ECMWF:/tmp'],
+        zarr_dir='/tmp/zarr',
+        raw_dir='/tmp/raw',
+    )
+    return {"ops": {
+        "nwp_consumer_docker_op": {"config": json.loads(config.json())},
+    }}
+
+
+@daily_partitioned_config(start_date=dt.datetime(2021, 1, 1))
+def ecmwf_daily_partitioned_config(start: dt.datetime, _end: dt.datetime):
+    config: NWPConsumerConfig = NWPConsumerConfig(
+        date_from=start.strftime("%Y-%m-%d"),
+        date_to=start.strftime("%Y-%m-%d"),
+        source="ecmwf-mars",
+        env_vars=["ECMWF_API_URL", "ECMWF_API_KEY", "ECMWF_API_EMAIL"],
         zarr_dir='/mnt/storage_b/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/NWP/ECMWF/zarr',
         raw_dir='/mnt/storage_b/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/NWP/ECMWF/raw',
     )
     return {"ops": {
         "nwp_consumer_download_op": {"config": json.loads(config.json())},
-        "nwp_consumer_convert_op": {"config": json.loads(config.json())},
-        "nwp_consumer_docker_op": {"config": json.loads(config.json())},
     }}
-
 @job(config=ecmwf_daily_partitioned_config)
 def ecmwf_daily_local_archive():
     nwp_consumer_convert_op(nwp_consumer_download_op())