Merge branch 'dev' into staging

CHANGELOG.md

@@ -4,14 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [v1.7.0] - 2024-xx-xx
+## [v1.7.0] - 2024-09-17
 
 ### Features
 - Added cooperative pipeline locking and queue functionality
 - Added user secrets
 - Added group secrets
 - Added task patching for all task types
 - Added delete group endpoint which will delete the group and all objects owned by that group
+- Added a Secrets Engine middleware to the Tapis Plugin for the Workflow Engine
 
 ## [v1.6.0] - 2024-01-xx
 

src/api/specs/WorkflowsAPI.yaml

@@ -729,7 +729,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/RespResourceURL'
+                $ref: '#/components/schemas/RespGroupSecret'
         '400':
           description: Bad request. Invalid JSON.
           content:

src/api/src/backend/services/GroupService.py

@@ -1,10 +1,14 @@
 from typing import Union
 
 from backend.models import Group, GroupUser
+from backend.conf.constants import PERMITTED_SERVICES
 
 
 class GroupService:
     def user_in_group(self, username, group_id, tenant_id, is_admin: Union[bool, None]=None):
+        # Grant access to all Group-level objects to all services
+        if username in PERMITTED_SERVICES: return True
+
         kwargs = {}
         if is_admin is not None:
             kwargs["is_admin"] = is_admin

src/api/src/backend/services/SecretService.py

@@ -22,7 +22,7 @@ def create(self, tenant_id, owner, req_secret: ReqCreateSecret):
                 secretName=sk_secret_name,
                 user=TAPIS_SERVICE_ACCOUNT,
                 tenant=SECRETS_TENANT,
-                data=req_secret.data,
+                data={"secret": req_secret.data},
                 _tapis_set_x_headers_from_service=True
             )
 

src/api/src/backend/views/http/secrets.py

@@ -1,9 +1,15 @@
-from typing import Any, Dict
+from typing import Any, Dict, Union
 
 from pydantic import BaseModel
 
 
 class ReqCreateSecret(BaseModel):
     id: str
     description: str = None
-    data: Dict[str, Any]
+    data: Union[
+        Dict[str, Any],
+        str,
+        int,
+        float,
+        bool
+    ]

src/engine/src/core/Server.py → src/engine/src/Server.py

@@ -34,8 +34,8 @@
 )
 from owe_python_sdk.schema import WorkflowSubmissionRequest, EmptyObject
 
-from core.workers import WorkerPool
-from core.workflows import WorkflowExecutor
+from workers import WorkerPool
+from workflows import WorkflowExecutor
 from utils import serialize_request, load_plugins, lbuffer_str as lbuf
 from errors import NoAvailableWorkers, WorkflowTerminated
 

src/engine/src/contrib/tapis/TapisPlugin.py

@@ -16,7 +16,7 @@
 )
 from contrib.tapis.middleware.event_handlers.archivers import TapisSystemArchiver
 from contrib.tapis.middleware.event_handlers.notifications import TapisWorkflowsAPIBackend
-from contrib.tapis.middleware.engines import TapisSKSecretsEngine
+from contrib.tapis.middleware.engines import TapisWorkflowsGroupSecretsEngine
 from contrib.tapis.executors import TapisActor, TapisJob
 from contrib.tapis.constants import TASK_TYPE_TAPIS_ACTOR, TASK_TYPE_TAPIS_JOB, ARCHIVER_TYPE_TAPIS_SYSTEM
 
@@ -49,7 +49,7 @@ def __init__(self, name):
         )
         self.register("task_executor", {TASK_TYPE_TAPIS_ACTOR: TapisActor})
         self.register("task_executor", {TASK_TYPE_TAPIS_JOB: TapisJob})
-        self.register("engine", {"tapis-security-kernal": TapisSKSecretsEngine})
+        self.register("engine", {"tapis-workflows-group-secrets": TapisWorkflowsGroupSecretsEngine})
         self.register("schema_extension", SchemaExtension(
             _type="task_executor",
             sub_type="function",

src/engine/src/contrib/tapis/middleware/engines/TapisSKSecretsEngine.py → src/engine/src/contrib/tapis/middleware/engines/TapisWorkflowsGroupSecretsEngine.py

@@ -1,7 +1,7 @@
 from contrib.tapis.helpers import TapisServiceAPIGateway
 
 
-class TapisSKSecretsEngine:
+class TapisWorkflowsGroupSecretsEngine:
     def __init__(self):
 
         service_api_gateway = TapisServiceAPIGateway()
@@ -16,17 +16,25 @@ def __init__(self):
         #     }
         # }
 
-    def __call__(self, tapis_tenant_id, sk_id):
+    def __call__(self, pk, args):
         try:
+            group_secret = self.service_client.workflows.getGroupSecret(
+                _tapis_set_x_headers_from_service=True,
+                _x_tapis_tenant=args["tapis_tenant_id"].value,
+                _x_tapis_user=args["tapis_pipeline_owner"].value,
+                group_id=args["tapis_workflows_group_id"].value,
+                group_secret_id=pk
+            )
+
             resp = self.service_client.sk.readSecret(
                 secretType="user",
-                secretName=sk_id,
+                secretName=group_secret.secret.sk_secret_name,
                 user="workflows",
-                tenant=tapis_tenant_id,
+                tenant="admin",
                 version=0,
                 _tapis_set_x_headers_from_service=True
             )
-            # ctx.args["tapis_tenant_id"].value
-            return resp.secretMap.__dict__
+
+            return resp.secretMap.__dict__["data"]
         except Exception as e:
             return None # TODO catch network error

src/engine/src/core/daos/WorkflowExecutorStateDAO.py → src/engine/src/daos/WorkflowExecutorStateDAO.py

@@ -1,4 +1,4 @@
-from core.state import ReactiveState
+from state import ReactiveState
 
 
 class WorkflowExecutorStateDAO:

src/engine/src/daos/__init__.py

@@ -0,0 +1,2 @@
+from daos.FileSystemDAO import FileSystemDAO
+from daos.WorkflowExecutorStateDAO import WorkflowExecutorStateDAO

src/engine/src/core/expressions/ConditionalExpressionEvaluator.py → src/engine/src/expressions/ConditionalExpressionEvaluator.py

@@ -9,7 +9,7 @@
     ConditionalExpression,
     ConditionalExpressions
 )
-from core.expressions import OperandResolver
+from expressions import OperandResolver
 from errors.tasks import ConditionalExpressionEvalError, OperandResolutionError
 
 

src/engine/src/core/expressions/OperandResolver.py → src/engine/src/expressions/OperandResolver.py

@@ -1,6 +1,6 @@
 from owe_python_sdk.schema import Operand
 
-from core.workflows import ValueFromService
+from workflows import ValueFromService
 from errors.tasks import OperandResolutionError
 
 

src/engine/src/expressions/__init__.py

@@ -0,0 +1,2 @@
+from expressions.ConditionalExpressionEvaluator import ConditionalExpressionEvaluator
+from expressions.OperandResolver import OperandResolver

src/engine/src/core/ioc/IOCContainerFactory.py → src/engine/src/ioc/IOCContainerFactory.py

@@ -1,21 +1,21 @@
 from typing import List
 
 from owe_python_sdk.Plugin import Plugin
-from core.ioc import IOCContainer
-from core.state import ReactiveState
-from core.daos import (
+from ioc import IOCContainer
+from state import ReactiveState
+from daos import (
     WorkflowExecutorStateDAO,
     FileSystemDAO
 )
-from core.mappers import (
+from mappers import (
     ArgMapper,
     ArgValueFileMapper,
     EnvMapper,
     EnvVarValueFileMapper,
     TaskMapper,
     TaskOutputMapper
 )
-from core.repositories import (
+from repositories import (
     ArgRepository,
     ArgValueFileRepository,
     EnvRepository,
@@ -25,12 +25,12 @@
     TaskRepository,
     TemplateRepository
 )
-from core.tasks.TaskInputFileStagingService import TaskInputFileStagingService
-from core.workflows import (
+from tasks.TaskInputFileStagingService import TaskInputFileStagingService
+from workflows import (
     GraphValidator,
     ValueFromService
 )
-from core.expressions import (
+from expressions import (
     ConditionalExpressionEvaluator,
     OperandResolver
 )

src/engine/src/ioc/__init__.py

@@ -0,0 +1,2 @@
+from ioc.IOCContainer import IOCContainer
+from ioc.IOCContainerFactory import IOCContainerFactory

src/engine/src/main.py

@@ -1,6 +1,6 @@
 import sys, logging
 
-from core.Server import Server
+from Server import Server
 
 
 # Set all third-party library loggers to critical

src/engine/src/core/mappers/ArgMapper.py → src/engine/src/mappers/ArgMapper.py

@@ -1,4 +1,4 @@
-from core.daos import WorkflowExecutorStateDAO
+from daos import WorkflowExecutorStateDAO
 
 
 class ArgMapper:
@@ -8,4 +8,7 @@ def __init__(self, dao: WorkflowExecutorStateDAO):
     def get_value_by_key(self, key):
         arg = self._dao.get_state().ctx.args.get(key, None)
         if arg == None: return None
-        return arg.value
+        return arg.value
+
+    def get_all(self):
+        return self._dao.get_state().ctx.args

src/engine/src/core/mappers/ArgValueFileMapper.py → src/engine/src/mappers/ArgValueFileMapper.py

@@ -1,4 +1,4 @@
-from core.daos import FileSystemDAO
+from daos import FileSystemDAO
 
 
 class ArgValueFileMapper:

src/engine/src/core/mappers/EnvMapper.py → src/engine/src/mappers/EnvMapper.py

@@ -1,4 +1,4 @@
-from core.daos import WorkflowExecutorStateDAO
+from daos import WorkflowExecutorStateDAO
 
 
 class EnvMapper:

src/engine/src/core/mappers/EnvVarValueFileMapper.py → src/engine/src/mappers/EnvVarValueFileMapper.py

@@ -1,4 +1,4 @@
-from core.daos import FileSystemDAO
+from daos import FileSystemDAO
 
 
 class EnvVarValueFileMapper:

src/engine/src/core/mappers/TaskMapper.py → src/engine/src/mappers/TaskMapper.py

@@ -1,4 +1,4 @@
-from core.daos import WorkflowExecutorStateDAO
+from daos import WorkflowExecutorStateDAO
 
 
 class TaskMapper:

src/engine/src/core/mappers/TaskOutputMapper.py → src/engine/src/mappers/TaskOutputMapper.py

@@ -1,6 +1,6 @@
 import os
 
-from core.daos import FileSystemDAO
+from daos import FileSystemDAO
 
 
 class TaskOutputMapper:

src/engine/src/mappers/__init__.py

@@ -0,0 +1,6 @@
+from mappers.ArgMapper import ArgMapper
+from mappers.ArgValueFileMapper import ArgValueFileMapper
+from mappers.EnvVarValueFileMapper import EnvVarValueFileMapper
+from mappers.EnvMapper import EnvMapper
+from mappers.TaskMapper import TaskMapper
+from mappers.TaskOutputMapper import TaskOutputMapper

src/engine/src/middleware/archivers/__init__.py

@@ -0,0 +1,4 @@
+from middleware.archivers.S3Archiver import S3Archiver
+from middleware.archivers.IRODSArchiver import IRODSArchiver
+
+

src/engine/src/owe_python_sdk/TaskExecutor.py

@@ -9,7 +9,7 @@
 from owe_python_sdk.TaskOutputFile import TaskOutputFile
 from owe_python_sdk.constants import STDERR, STDOUT
 from utils import lbuffer_str as lbuf
-from core.resources import Resource, ResourceType
+from resources import Resource, ResourceType
 from conf.constants import (
     DEFAULT_POLLING_INTERVAL,
     KUBERNETES_NAMESPACE,

src/engine/src/core/repositories/ArgRepository.py → src/engine/src/repositories/ArgRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import ArgMapper
+from mappers import ArgMapper
 
 
 class ArgRepository:
@@ -7,4 +7,7 @@ def __init__(self, mapper: ArgMapper):
 
     def get_value_by_key(self, key):
         value = self._mapper.get_value_by_key(key)
-        return value
+        return value
+
+    def get_all(self):
+        return self._mapper.get_all()

src/engine/src/core/repositories/ArgValueFileRepository.py → src/engine/src/repositories/ArgValueFileRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import ArgValueFileMapper
+from mappers import ArgValueFileMapper
 
 
 class ArgValueFileRepository:

src/engine/src/core/repositories/EnvRepository.py → src/engine/src/repositories/EnvRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import EnvMapper
+from mappers import EnvMapper
 
 
 class EnvRepository:

src/engine/src/core/repositories/EnvVarValueFileRepository.py → src/engine/src/repositories/EnvVarValueFileRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import EnvVarValueFileMapper
+from mappers import EnvVarValueFileMapper
 
 
 class EnvVarValueFileRepository:

src/engine/src/core/repositories/TaskOutputRepository.py → src/engine/src/repositories/TaskOutputRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import TaskOutputMapper
+from mappers import TaskOutputMapper
 
 from owe_python_sdk.schema import Task
 

src/engine/src/core/repositories/TaskRepository.py → src/engine/src/repositories/TaskRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import TaskMapper
+from mappers import TaskMapper
 
 
 class TaskRepository: