From 932ecf4e61b9825f843fcdb611c90b1cb6af0ba0 Mon Sep 17 00:00:00 2001 From: Nathan Freeman Date: Wed, 27 Mar 2024 15:28:55 -0500 Subject: [PATCH] Remove validation for ETL jobs --- src/api/src/backend/utils/__init__.py | 4 ++++ src/api/src/backend/views/http/tapis_etl.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/api/src/backend/utils/__init__.py b/src/api/src/backend/utils/__init__.py index 529d3f1a..0ec63520 100644 --- a/src/api/src/backend/utils/__init__.py +++ b/src/api/src/backend/utils/__init__.py @@ -64,6 +64,10 @@ def build_etl_pipeline_env(body): "type": "string", "value": json.dumps(body.remote_inbox.dict()) }, + "DEFAULT_ETL_JOBS": { + "type": "string", + "value": json.dumps(body.jobs) + } } \ No newline at end of file diff --git a/src/api/src/backend/views/http/tapis_etl.py b/src/api/src/backend/views/http/tapis_etl.py index bffe7262..f00c4d43 100644 --- a/src/api/src/backend/views/http/tapis_etl.py +++ b/src/api/src/backend/views/http/tapis_etl.py @@ -20,7 +20,7 @@ class RODataProfile(TapisIOSystemProfile, DataProfile): path: str = "/ETL/REMOTE-OUTBOX/DATA" class ROManifestsProfile(TapisIOSystemProfile, ManifestsProfile): - path: str = "/ETL/REMOTE-OUTBOX/MANFIFESTS" + path: str = "/ETL/REMOTE-OUTBOX/MANIFESTS" generation_policy: EnumManifestGenerationPolicy = EnumManifestGenerationPolicy.AutoOnePerFile priority: EnumManifestPriority = EnumManifestPriority.Oldest @@ -32,7 +32,7 @@ class LIDataProfile(TapisIOSystemProfile, DataProfile): path: str = "/ETL/LOCAL-INBOX/DATA" class LIManifestsProfile(TapisIOSystemProfile, ManifestsProfile): - path: str = "/ETL/LOCAL-INBOX/MANFIFESTS" + path: str = "/ETL/LOCAL-INBOX/MANIFESTS" generation_policy: EnumManifestGenerationPolicy = None priority: EnumManifestPriority = EnumManifestPriority.Oldest @@ -94,15 +94,15 @@ class TapisETLPipeline(Pipeline): before: ActionFilter = None remote_outbox: RemoteOutbox local_inbox: LocalInbox - jobs: List[ExetendedTapisJob] + jobs: List[ExetendedTapisJob] = [] local_outbox: LocalOutbox remote_inbox: RemoteInbox after: ActionFilter = None - @validator("jobs") - def one_or_more_jobs(cls, value): - # Check that the pipeline contains at least 1 tapis job definition - if len(value) < 1: - raise ValueError("A Tapis ETL pipeline must contain at least 1 Tapis Job definition") + # @validator("jobs") + # def one_or_more_jobs(cls, value): + # # Check that the pipeline contains at least 1 tapis job definition + # if len(value) < 1: + # raise ValueError("A Tapis ETL pipeline must contain at least 1 Tapis Job definition") - return value \ No newline at end of file + # return value \ No newline at end of file