Change *_pattern to *_patterns

tapis-project · Mar 15, 2024 · 4e50c36 · 4e50c36
1 parent b224e77
commit 4e50c36
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 42 deletions.
diff --git a/src/api/src/backend/views/http/etl.py b/src/api/src/backend/views/http/etl.py
@@ -28,8 +28,8 @@ class ByteCheckDataIntegrityProfile(BaseDataIntegrityProfile):
 class DoneFileDataIntegrityProfile(BaseDataIntegrityProfile):
     type: Literal["done_file"]
     done_files_path: str
-    include_pattern: str = None
-    exclude_pattern: str = None
+    include_patterns: str = []
+    exclude_patterns: str = []
 
 DataIntegrityProfile = Annotated[
     Union[
@@ -42,8 +42,8 @@ class DoneFileDataIntegrityProfile(BaseDataIntegrityProfile):
 
 class IOSystemProfile(BaseModel):
     path: str
-    include_pattern: str = None
-    exclude_pattern: str = None
+    include_patterns: str = []
+    exclude_patterns: str = []
 
 class DataProfile(IOSystemProfile):
     integrity_profile: DataIntegrityProfile = None
@@ -52,11 +52,7 @@ class ManifestsProfile(IOSystemProfile):
     generation_policy: EnumManifestGenerationPolicy = None
     priority: EnumManifestPriority = None
 
-class IngressProfile(IOSystemProfile):
-    pass
-
 class IOSystem(BaseModel):
-    ingress: IngressProfile = None
     data: DataProfile
     manifests: ManifestsProfile
 
diff --git a/src/api/src/backend/views/http/tapis_etl.py b/src/api/src/backend/views/http/tapis_etl.py
@@ -1,6 +1,7 @@
 from typing import List
+from os.path import join
 
-from pydantic import BaseModel, validator, Extra, root_validator, conlist
+from pydantic import BaseModel, validator, Extra, conlist
 
 from .requests import Pipeline
 
@@ -9,7 +10,6 @@
     EnumManifestPriority,
     DataProfile,
     ManifestsProfile,
-    IngressProfile,
     IOSystem
 )
 
@@ -21,7 +21,7 @@ class RODataProfile(TapisIOSystemProfile, DataProfile):
 
 class ROManifestsProfile(TapisIOSystemProfile, ManifestsProfile):
     path: str = "/ETL/REMOTE-OUTBOX/MANFIFESTS"
-    generation_policy: EnumManifestGenerationPolicy = None
+    generation_policy: EnumManifestGenerationPolicy = EnumManifestGenerationPolicy.AutoOnePerFile
     priority: EnumManifestPriority = EnumManifestPriority.Oldest
 
 class RemoteOutbox(IOSystem):
@@ -33,16 +33,16 @@ class LIDataProfile(TapisIOSystemProfile, DataProfile):
 
 class LIManifestsProfile(TapisIOSystemProfile, ManifestsProfile):
     path: str = "/ETL/LOCAL-INBOX/MANFIFESTS"
-    generation_policy: EnumManifestGenerationPolicy = EnumManifestGenerationPolicy.AutoOnePerFile
+    generation_policy: EnumManifestGenerationPolicy = None
     priority: EnumManifestPriority = EnumManifestPriority.Oldest
 
-class LIIngressProfile(TapisIOSystemProfile, IngressProfile):
-    path: str
+class LIControlProfile(TapisIOSystemProfile):
+    path: str = "/ETL/LOCAL-INBOX/CONTROL"
 
 class LocalInbox(IOSystem):
     data: LIDataProfile
     manifests: LIManifestsProfile
-    ingress: LIIngressProfile
+    control: LIControlProfile
 
 class LODataProfile(TapisIOSystemProfile, DataProfile):
     path: str = "/ETL/LOCAL-OUTBOX/DATA"

diff --git a/src/api/src/tests/TestTapisETLPipeline.py b/src/api/src/tests/TestTapisETLPipeline.py
@@ -22,8 +22,8 @@ def testETLEnvBuilderUtil(self):
         env = build_etl_pipeline_env(self.pipeline)
 
         assert type(env) == dict
-        assert json.loads(env.get("REMOTE_OUTBOX").get("value")).get("manifests").get("generation_policy") == "manual"
-        assert json.loads(env.get("LOCAL_INBOX").get("value")).get("manifests").get("generation_policy") == "auto_one_per_file"
+        assert json.loads(env.get("REMOTE_OUTBOX").get("value")).get("manifests").get("generation_policy") == "auto_one_per_file"
+        assert json.loads(env.get("LOCAL_INBOX").get("value")).get("manifests").get("generation_policy") == None
         assert json.loads(env.get("LOCAL_OUTBOX").get("value")).get("manifests").get("generation_policy") == "auto_one_for_all"
         assert json.loads(env.get("REMOTE_INBOX").get("value")).get("manifests").get("generation_policy") == None
 

diff --git a/src/api/src/tests/fixtures/tapis-etl-pipeline.json b/src/api/src/tests/fixtures/tapis-etl-pipeline.json
@@ -16,17 +16,17 @@
         },
         "manifests": {
             "system_id": "test.etl.ls6.writable",
-            "generation_policy": "manual",
+            "generation_policy": "auto_one_per_file",
             "priority": "oldest",
             "path": "/ETL/REMOTE-OUTBOX/MANIFESTS",
             "include_pattern": null,
             "exclude_pattern": null
         }
     },
     "local_inbox": {
-        "ingress": {
+        "control": {
             "system_id": "test.etl.ls6.writable",
-            "path": "/ETL/LOCAL-INBOX/INGRESS"
+            "path": "/ETL/LOCAL-INBOX/CONTROL"
         },
         "data": {
             "system_id": "test.etl.ls6.xfer",
@@ -41,27 +41,6 @@
             "exclude_pattern": null
         }
     },
-    "local_outbox": {
-        "data": {
-            "system_id": "test.etl.ls6.xfer",
-            "path": "/ETL/LOCAL-OUTBOX/DATA"
-        },
-        "manifests": {
-            "system_id": "test.etl.ls6.writable",
-            "path": "/ETL/LOCAL-OUTBOX/MANIFESTS"
-        }
-    },
-    "remote_inbox": {
-        "data": {
-            "system_id": "test.etl.ls6.xfer",
-            "path": "/ETL/REMOTE-INBOX/DATA"
-        },
-        "manifests": {
-            "system_id": "test.etl.ls6.writable",
-            "path": "/ETL/REMOTE-INBOX/MANIFESTS"
-        }
-    },
-    "after": null,
     "jobs": [
         {
             "name": "string-transform",
@@ -126,5 +105,26 @@
                 }
             }
         }
-    ]
+    ],
+    "local_outbox": {
+        "data": {
+            "system_id": "test.etl.ls6.xfer",
+            "path": "/ETL/LOCAL-OUTBOX/DATA"
+        },
+        "manifests": {
+            "system_id": "test.etl.ls6.writable",
+            "path": "/ETL/LOCAL-OUTBOX/MANIFESTS"
+        }
+    },
+    "remote_inbox": {
+        "data": {
+            "system_id": "test.etl.ls6.xfer",
+            "path": "/ETL/REMOTE-INBOX/DATA"
+        },
+        "manifests": {
+            "system_id": "test.etl.ls6.writable",
+            "path": "/ETL/REMOTE-INBOX/MANIFESTS"
+        }
+    },
+    "after": null
 }