From 4e50c362b1d9acdcd0bc805b12d3eedb39ab5e17 Mon Sep 17 00:00:00 2001 From: Nathan Freeman Date: Fri, 15 Mar 2024 12:19:30 -0500 Subject: [PATCH] Change *_pattern to *_patterns --- src/api/src/backend/views/http/etl.py | 12 ++--- src/api/src/backend/views/http/tapis_etl.py | 14 +++--- src/api/src/tests/TestTapisETLPipeline.py | 4 +- .../tests/fixtures/tapis-etl-pipeline.json | 50 +++++++++---------- 4 files changed, 38 insertions(+), 42 deletions(-) diff --git a/src/api/src/backend/views/http/etl.py b/src/api/src/backend/views/http/etl.py index d618a4a9..17a03f32 100644 --- a/src/api/src/backend/views/http/etl.py +++ b/src/api/src/backend/views/http/etl.py @@ -28,8 +28,8 @@ class ByteCheckDataIntegrityProfile(BaseDataIntegrityProfile): class DoneFileDataIntegrityProfile(BaseDataIntegrityProfile): type: Literal["done_file"] done_files_path: str - include_pattern: str = None - exclude_pattern: str = None + include_patterns: str = [] + exclude_patterns: str = [] DataIntegrityProfile = Annotated[ Union[ @@ -42,8 +42,8 @@ class DoneFileDataIntegrityProfile(BaseDataIntegrityProfile): class IOSystemProfile(BaseModel): path: str - include_pattern: str = None - exclude_pattern: str = None + include_patterns: str = [] + exclude_patterns: str = [] class DataProfile(IOSystemProfile): integrity_profile: DataIntegrityProfile = None @@ -52,11 +52,7 @@ class ManifestsProfile(IOSystemProfile): generation_policy: EnumManifestGenerationPolicy = None priority: EnumManifestPriority = None -class IngressProfile(IOSystemProfile): - pass - class IOSystem(BaseModel): - ingress: IngressProfile = None data: DataProfile manifests: ManifestsProfile diff --git a/src/api/src/backend/views/http/tapis_etl.py b/src/api/src/backend/views/http/tapis_etl.py index 34a0d5c4..bffe7262 100644 --- a/src/api/src/backend/views/http/tapis_etl.py +++ b/src/api/src/backend/views/http/tapis_etl.py @@ -1,6 +1,7 @@ from typing import List +from os.path import join -from pydantic import BaseModel, validator, Extra, root_validator, conlist +from pydantic import BaseModel, validator, Extra, conlist from .requests import Pipeline @@ -9,7 +10,6 @@ EnumManifestPriority, DataProfile, ManifestsProfile, - IngressProfile, IOSystem ) @@ -21,7 +21,7 @@ class RODataProfile(TapisIOSystemProfile, DataProfile): class ROManifestsProfile(TapisIOSystemProfile, ManifestsProfile): path: str = "/ETL/REMOTE-OUTBOX/MANFIFESTS" - generation_policy: EnumManifestGenerationPolicy = None + generation_policy: EnumManifestGenerationPolicy = EnumManifestGenerationPolicy.AutoOnePerFile priority: EnumManifestPriority = EnumManifestPriority.Oldest class RemoteOutbox(IOSystem): @@ -33,16 +33,16 @@ class LIDataProfile(TapisIOSystemProfile, DataProfile): class LIManifestsProfile(TapisIOSystemProfile, ManifestsProfile): path: str = "/ETL/LOCAL-INBOX/MANFIFESTS" - generation_policy: EnumManifestGenerationPolicy = EnumManifestGenerationPolicy.AutoOnePerFile + generation_policy: EnumManifestGenerationPolicy = None priority: EnumManifestPriority = EnumManifestPriority.Oldest -class LIIngressProfile(TapisIOSystemProfile, IngressProfile): - path: str +class LIControlProfile(TapisIOSystemProfile): + path: str = "/ETL/LOCAL-INBOX/CONTROL" class LocalInbox(IOSystem): data: LIDataProfile manifests: LIManifestsProfile - ingress: LIIngressProfile + control: LIControlProfile class LODataProfile(TapisIOSystemProfile, DataProfile): path: str = "/ETL/LOCAL-OUTBOX/DATA" diff --git a/src/api/src/tests/TestTapisETLPipeline.py b/src/api/src/tests/TestTapisETLPipeline.py index 9d2660ba..7a44e4e4 100644 --- a/src/api/src/tests/TestTapisETLPipeline.py +++ b/src/api/src/tests/TestTapisETLPipeline.py @@ -22,8 +22,8 @@ def testETLEnvBuilderUtil(self): env = build_etl_pipeline_env(self.pipeline) assert type(env) == dict - assert json.loads(env.get("REMOTE_OUTBOX").get("value")).get("manifests").get("generation_policy") == "manual" - assert json.loads(env.get("LOCAL_INBOX").get("value")).get("manifests").get("generation_policy") == "auto_one_per_file" + assert json.loads(env.get("REMOTE_OUTBOX").get("value")).get("manifests").get("generation_policy") == "auto_one_per_file" + assert json.loads(env.get("LOCAL_INBOX").get("value")).get("manifests").get("generation_policy") == None assert json.loads(env.get("LOCAL_OUTBOX").get("value")).get("manifests").get("generation_policy") == "auto_one_for_all" assert json.loads(env.get("REMOTE_INBOX").get("value")).get("manifests").get("generation_policy") == None diff --git a/src/api/src/tests/fixtures/tapis-etl-pipeline.json b/src/api/src/tests/fixtures/tapis-etl-pipeline.json index c1fcab79..ac8978c4 100644 --- a/src/api/src/tests/fixtures/tapis-etl-pipeline.json +++ b/src/api/src/tests/fixtures/tapis-etl-pipeline.json @@ -16,7 +16,7 @@ }, "manifests": { "system_id": "test.etl.ls6.writable", - "generation_policy": "manual", + "generation_policy": "auto_one_per_file", "priority": "oldest", "path": "/ETL/REMOTE-OUTBOX/MANIFESTS", "include_pattern": null, @@ -24,9 +24,9 @@ } }, "local_inbox": { - "ingress": { + "control": { "system_id": "test.etl.ls6.writable", - "path": "/ETL/LOCAL-INBOX/INGRESS" + "path": "/ETL/LOCAL-INBOX/CONTROL" }, "data": { "system_id": "test.etl.ls6.xfer", @@ -41,27 +41,6 @@ "exclude_pattern": null } }, - "local_outbox": { - "data": { - "system_id": "test.etl.ls6.xfer", - "path": "/ETL/LOCAL-OUTBOX/DATA" - }, - "manifests": { - "system_id": "test.etl.ls6.writable", - "path": "/ETL/LOCAL-OUTBOX/MANIFESTS" - } - }, - "remote_inbox": { - "data": { - "system_id": "test.etl.ls6.xfer", - "path": "/ETL/REMOTE-INBOX/DATA" - }, - "manifests": { - "system_id": "test.etl.ls6.writable", - "path": "/ETL/REMOTE-INBOX/MANIFESTS" - } - }, - "after": null, "jobs": [ { "name": "string-transform", @@ -126,5 +105,26 @@ } } } - ] + ], + "local_outbox": { + "data": { + "system_id": "test.etl.ls6.xfer", + "path": "/ETL/LOCAL-OUTBOX/DATA" + }, + "manifests": { + "system_id": "test.etl.ls6.writable", + "path": "/ETL/LOCAL-OUTBOX/MANIFESTS" + } + }, + "remote_inbox": { + "data": { + "system_id": "test.etl.ls6.xfer", + "path": "/ETL/REMOTE-INBOX/DATA" + }, + "manifests": { + "system_id": "test.etl.ls6.writable", + "path": "/ETL/REMOTE-INBOX/MANIFESTS" + } + }, + "after": null } \ No newline at end of file