Skip to content

Commit

Permalink
fix(sat-etl): Use subprocess instead
Browse files Browse the repository at this point in the history
  • Loading branch information
devsjc committed Sep 19, 2024
1 parent e747049 commit d8a3015
Showing 1 changed file with 8 additions and 16 deletions.
24 changes: 8 additions & 16 deletions local_archives/sat/eumetsat/eumetsat_iodc.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from typing import Any

import dagster as dg
from dagster_docker import PipesDockerClient

from constants import LOCATIONS_BY_ENVIRONMENT

Expand All @@ -27,7 +26,7 @@
"source": dg.MetadataValue.text("eumetsat"),
"expected_runtime": dg.MetadataValue.text("TBD"),
},
compute_kind="docker",
compute_kind="subprocess",
automation_condition=dg.AutomationCondition.eager(),
tags={
# "dagster/max_runtime": str(60 * 60 * 10), # Should take 6 ish hours
Expand All @@ -41,27 +40,20 @@
)
def iodc_monthly(
context: dg.AssetExecutionContext,
pipes_docker_client: PipesDockerClient,
pipes_subprocess_client: dg.PipesSubprocessClient,
) -> Any:
image: str = "ghcr.io/openclimatefix/sat-etl:main"
it: dt.datetime = context.partition_time_window.start
return pipes_docker_client.run(
image=image,
return pipes_subprocess_client.run(
command=[
"iodc",
"-m",
"/home/dagster/mambaforge/envs/sat-etl/bin/python",
"/home/dagster/dags/containers/sat/download_process_sat.py",
"--month",
it.strftime("%Y-%m"),
"--path",
f"/store_a_0/sat/eumetsat/india",
f"/mnt/storage_a/sat/eumetsat/india",
"--rm",
],
env={
"EUMETSAT_CONSUMER_KEY": os.environ["EUMETSAT_CONSUMER_KEY"],
"EUMETSAT_CONSUMER_SECRET": os.environ["EUMETSAT_CONSUMER_SECRET"],
},
container_kwargs={
"volumes": [f"/store_a_0/sat/eumetsat/india:/store_a_0/sat/eumetsat/india"],
},
context=context,
).get_results()
).get_materialize_result()

0 comments on commit d8a3015

Please sign in to comment.