From e372eaebaa50d57fee8acbdb4ea00b6e28714f31 Mon Sep 17 00:00:00 2001
From: Nathan Freeman <nathan.dean.freeman@gmail.com>
Date: Tue, 2 Jul 2024 19:15:36 -0500
Subject: [PATCH] Update spec and changelog

---
 CHANGELOG.md                    |   5 +
 src/api/specs/WorkflowsAPI.yaml | 198 +++++++++++++++++++-------------
 2 files changed, 124 insertions(+), 79 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 383fb00d..8ed2eeef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [v1.6.x] - 2024-xx-xx
+
+### Features
+- Added cooperative pipeline locking and queue functionality
+
 ## [v1.6.0] - 2024-01-xx
 
 ### Breaking Changes
diff --git a/src/api/specs/WorkflowsAPI.yaml b/src/api/specs/WorkflowsAPI.yaml
index 2b2a9c84..5aa52528 100644
--- a/src/api/specs/WorkflowsAPI.yaml
+++ b/src/api/specs/WorkflowsAPI.yaml
@@ -2318,6 +2318,8 @@ components:
               $ref: "#/components/schemas/EnumInstaller"
             command:
               type: string
+            entrypoint:
+              type: string
             code:
               type: string
               format: byte
@@ -2913,7 +2915,6 @@ components:
           required:
             - runtime
             - installer
-            - code
           properties:
             git_repositories:
               type: array
@@ -2925,6 +2926,8 @@ components:
               $ref: "#/components/schemas/EnumInstaller"
             command:
               type: string
+            entrypoint:
+              type: string
             code:
               type: string
               format: byte
@@ -2959,51 +2962,76 @@ components:
     ########################### ETL
 
     ReqCreateETLPipeline:
+      allOf:
+        - $ref: "#/components/schemas/ReqPipeline"
+        - type: object
+          required:
+            - id
+            - remote_outbox
+            - local_inbox
+            - jobs
+            - local_outbox
+            - remote_inbox
+          properties:
+            id:
+              $ref: '#/components/schemas/ID'
+            description:
+              type: string
+            before:
+              $ref: "#/components/schemas/ETLActionFilter"
+            remote_outbox:
+              $ref: '#/components/schemas/ETLRemoteOutbox'
+            local_inbox:
+              $ref: '#/components/schemas/ETLLocalInbox'
+            jobs:
+              type: array
+              items:
+                $ref: '#/components/schemas/TapisETLExtendedTapisJobDef'
+            local_outbox:
+              $ref: '#/components/schemas/ETLLocalOutbox'
+            remote_inbox:
+              $ref: '#/components/schemas/ETLRemoteInbox'
+            after:
+              $ref: "#/components/schemas/ETLActionFilter"
+
+    ETLActionFilter:
       type: object
-      required:
-        - id
-        - local_inbox
-        - jobs
-        - local_outbox
-        - remote_inbox
       properties:
-        id:
-          $ref: '#/components/schemas/ID'
-        description:
-          type: string
-        execution_profile:
-          $ref: '#/components/schemas/PipelineExecutionProfile'
-        remote_outbox:
-          $ref: '#/components/schemas/ETLRemoteOutbox'
-        local_inbox:
-          $ref: '#/components/schemas/ETLLocalInbox'
-        jobs:
+        pipeline_ids:
           type: array
           items:
-            $ref: '#/components/schemas/WorkflowsExtendedTapisJobDef'
-        local_outbox:
-          $ref: '#/components/schemas/ETLLocalOutbox'
-        remote_inbox:
-          $ref: '#/components/schemas/ETLRemoteInbox'
+            type: string
+          default: []
+        run_async:
+          type: boolean
+          default: true
 
-    WorkflowsTapisJobDefExtension:
+    TapisETLExtension:
       type: object
       properties:
-        workflows:
+        env_mappings:
           type: object
-          properties:
-            etl:
-              type: object
-              properties:
-                input:
-                  type: string
-                output:
-                  type: string
+          additionalProperties: true
+        last_status:
+          type: string
+          default: PENDING
+    
+    TapisJobExtensions:
+      type: object
+      properties:
+        tapis_etl:
+          $ref: "#/components/schemas/TapisETLExtension"
 
-    WorkflowsExtendedTapisJobDef:
+    ExtendedTapisJob:
+      type: object
+      properties:
+        extensions:
+          $ref: "#/components/schemas/TapisJobExtensions"
+
+    TapisETLExtendedTapisJobDef:
       allOf:
         - $ref: "#/components/schemas/TapisJobDef"
-        - $ref: "#/components/schemas/WorkflowsTapisJobDefExtension"
+        - $ref: "#/components/schemas/ExtendedTapisJob"
 
     BaseDataIntegrityProfile:
       type: object
@@ -3032,10 +3060,14 @@ components:
               type: string
             done_files_path:
               type: string
-            include_pattern:
-              type: string
-            exclude_pattern:
-              type: string
+            include_patterns:
+              type: array
+              items:
+                type: string
+            exclude_patterns:
+              type: array
+              items:
+                type: string
 
     ByteCheckDataIntegrityProfile:
       allOf:
@@ -3057,66 +3089,74 @@ components:
           done_file: "#/components/schemas/DoneFileDataIntegrityProfile"
           byte_check: "#/components/schemas/ByteCheckDataIntegrityProfile"
 
-    ETLLocalIOBox:
+    ETLSystem:
       type: object
-      required:
-        - system_id
-        - data_path
       properties:
         system_id:
           type: string
-        data_path:
-          type: string
-        manifests_path:
-          type: string
-        manifest_generation_policy:
-          $ref: "#/components/schemas/ETLEnumManifestGenerationPolicy"
-        manifest_priority:
-          $ref: "#/components/schemas/ETLEnumManifestPriority"
-          default: 'oldest'
-        data_integrity_profile:
-          $ref: "#/components/schemas/DataIntegrityProfile"
-        include_pattern:
-          type: string
-        exclude_pattern:
+        path:
           type: string
+        include_patterns:
+          type: array
+          items: 
+            type: string
+        exclude_patterns:
+          type: array
+          items: 
+            type: string
 
-    ETLRemoteOutbox:
-      type: object
-      additionalProperties: true
-
-    ETLLocalInbox:
+    ETLManifestsSystem:
       allOf:
-        - $ref: "#/components/schemas/ETLLocalIOBox"
+        - $ref: "#/components/schemas/ETLSystem"
         - type: object
           properties:
-            manifest_generation_policy:
+            generation_policy:
               $ref: "#/components/schemas/ETLEnumManifestGenerationPolicy"
-              default: 'auto_one_per_file'
+            priority:
+              $ref: "#/components/schemas/ETLEnumManifestPriority"
 
-    ETLLocalOutbox:
+    ETLDataSystem:
       allOf:
-        - $ref: "#/components/schemas/ETLLocalIOBox"
+        - $ref: "#/components/schemas/ETLSystem"
         - type: object
           properties:
-            manifest_generation_policy:
-              $ref: "#/components/schemas/ETLEnumManifestGenerationPolicy"
-              default: 'auto_one_for_all'
+            integrity_profile:
+              $ref: "#/components/schemas/DataIntegrityProfile"
 
-    ETLTapisSystemRemoteInbox:
+    ETLControlSystem:
+      allOf:
+        - $ref: "#/components/schemas/ETLSystem"
+
+    ETLSystemsConfiguration:
       type: object
       required:
-        - system_id
-        - path
+        - manifests
+        - data
       properties:
-        system_id:
-          type: string
-        path:
-          type: string
+        manifests:
+          $ref: "#/components/schemas/ETLManifestsSystem"
+        data:
+          $ref: "#/components/schemas/ETLDataSystem"
+    
+    ETLRemoteOutbox:
+      allOf:
+        - $ref: "#/components/schemas/ETLSystemsConfiguration"
 
-    ETLRemoteInbox:
+    ETLLocalInbox:
       allOf:
-        - $ref: "#/components/schemas/ETLTapisSystemRemoteInbox"
+        - $ref: "#/components/schemas/ETLSystemsConfiguration"
+        - type: object
+          properties:
+            control:
+              $ref: "#/components/schemas/ETLControlSystem"
+
+    ETLLocalOutbox:
+      allOf:
+        - $ref: "#/components/schemas/ETLSystemsConfiguration"
+
+    ETLRemoteInbox:
+      allOf: 
+        - $ref: "#/components/schemas/ETLSystemsConfiguration"
 
     ETLEnumManifestGenerationPolicy:
       type: string