diff --git a/nwp/assets/ecmwf/mars.py b/nwp/assets/ecmwf/mars.py index e641132..ae57baf 100644 --- a/nwp/assets/ecmwf/mars.py +++ b/nwp/assets/ecmwf/mars.py @@ -1,5 +1,8 @@ +import os import subprocess -from dagster import Config, OpExecutionContext, op, graph +import sys + +from dagster import Config, OpExecutionContext, op from dagster_docker import execute_docker_container @@ -33,27 +36,25 @@ def nwp_consumer_docker_op(context: OpExecutionContext, config: NWPConsumerConfi @op def nwp_consumer_download_op(context: OpExecutionContext, config: NWPConsumerConfig): process = subprocess.run( - ["nwp-consumer", "download", + [f"{os.path.dirname(sys.executable)}/nwp-consumer", "download", f'--source={config.source}', f'--from={config.date_from}', f'--to={config.date_to}', f'--rdir={config.raw_dir}', f'--zdir={config.zarr_dir}'], - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, + capture_output=True, text=True ) - code = process.returncode print(process.stdout) print(process.stderr) + process.check_returncode() return config @op def nwp_consumer_convert_op(context: OpExecutionContext, downloadedConfig: NWPConsumerConfig): process = subprocess.run( - ["nwp-consumer", "convert", + [f"{os.path.dirname(sys.executable)}/nwp-consumer", "convert", f'--source={downloadedConfig.source}', f'--from={downloadedConfig.date_from}', f'--to={downloadedConfig.date_to}', f'--rdir={downloadedConfig.raw_dir}', f'--zdir={downloadedConfig.zarr_dir}'], - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, + capture_output=True, text=True ) print(process.stdout)