refactor

src/engine/src/core/Server.py → src/engine/src/Server.py

@@ -34,8 +34,8 @@
 )
 from owe_python_sdk.schema import WorkflowSubmissionRequest, EmptyObject
 
-from core.workers import WorkerPool
-from core.workflows import WorkflowExecutor
+from workers import WorkerPool
+from workflows import WorkflowExecutor
 from utils import serialize_request, load_plugins, lbuffer_str as lbuf
 from errors import NoAvailableWorkers, WorkflowTerminated
 

src/engine/src/core/daos/WorkflowExecutorStateDAO.py → src/engine/src/daos/WorkflowExecutorStateDAO.py

@@ -1,4 +1,4 @@
-from core.state import ReactiveState
+from state import ReactiveState
 
 
 class WorkflowExecutorStateDAO:

src/engine/src/daos/__init__.py

@@ -0,0 +1,2 @@
+from daos.FileSystemDAO import FileSystemDAO
+from daos.WorkflowExecutorStateDAO import WorkflowExecutorStateDAO

src/engine/src/core/expressions/ConditionalExpressionEvaluator.py → src/engine/src/expressions/ConditionalExpressionEvaluator.py

@@ -9,7 +9,7 @@
     ConditionalExpression,
     ConditionalExpressions
 )
-from core.expressions import OperandResolver
+from expressions import OperandResolver
 from errors.tasks import ConditionalExpressionEvalError, OperandResolutionError
 
 

src/engine/src/core/expressions/OperandResolver.py → src/engine/src/expressions/OperandResolver.py

@@ -1,6 +1,6 @@
 from owe_python_sdk.schema import Operand
 
-from core.workflows import ValueFromService
+from workflows import ValueFromService
 from errors.tasks import OperandResolutionError
 
 

src/engine/src/expressions/__init__.py

@@ -0,0 +1,2 @@
+from expressions.ConditionalExpressionEvaluator import ConditionalExpressionEvaluator
+from expressions.OperandResolver import OperandResolver

src/engine/src/core/ioc/IOCContainerFactory.py → src/engine/src/ioc/IOCContainerFactory.py

@@ -1,21 +1,21 @@
 from typing import List
 
 from owe_python_sdk.Plugin import Plugin
-from core.ioc import IOCContainer
-from core.state import ReactiveState
-from core.daos import (
+from ioc import IOCContainer
+from state import ReactiveState
+from daos import (
     WorkflowExecutorStateDAO,
     FileSystemDAO
 )
-from core.mappers import (
+from mappers import (
     ArgMapper,
     ArgValueFileMapper,
     EnvMapper,
     EnvVarValueFileMapper,
     TaskMapper,
     TaskOutputMapper
 )
-from core.repositories import (
+from repositories import (
     ArgRepository,
     ArgValueFileRepository,
     EnvRepository,
@@ -25,12 +25,12 @@
     TaskRepository,
     TemplateRepository
 )
-from core.tasks.TaskInputFileStagingService import TaskInputFileStagingService
-from core.workflows import (
+from tasks.TaskInputFileStagingService import TaskInputFileStagingService
+from workflows import (
     GraphValidator,
     ValueFromService
 )
-from core.expressions import (
+from expressions import (
     ConditionalExpressionEvaluator,
     OperandResolver
 )

src/engine/src/ioc/__init__.py

@@ -0,0 +1,2 @@
+from ioc.IOCContainer import IOCContainer
+from ioc.IOCContainerFactory import IOCContainerFactory

src/engine/src/main.py

@@ -1,6 +1,6 @@
 import sys, logging
 
-from core.Server import Server
+from Server import Server
 
 
 # Set all third-party library loggers to critical

src/engine/src/core/mappers/ArgMapper.py → src/engine/src/mappers/ArgMapper.py

@@ -1,4 +1,4 @@
-from core.daos import WorkflowExecutorStateDAO
+from daos import WorkflowExecutorStateDAO
 
 
 class ArgMapper:

src/engine/src/core/mappers/ArgValueFileMapper.py → src/engine/src/mappers/ArgValueFileMapper.py

@@ -1,4 +1,4 @@
-from core.daos import FileSystemDAO
+from daos import FileSystemDAO
 
 
 class ArgValueFileMapper:

src/engine/src/core/mappers/EnvMapper.py → src/engine/src/mappers/EnvMapper.py

@@ -1,4 +1,4 @@
-from core.daos import WorkflowExecutorStateDAO
+from daos import WorkflowExecutorStateDAO
 
 
 class EnvMapper:

src/engine/src/core/mappers/EnvVarValueFileMapper.py → src/engine/src/mappers/EnvVarValueFileMapper.py

@@ -1,4 +1,4 @@
-from core.daos import FileSystemDAO
+from daos import FileSystemDAO
 
 
 class EnvVarValueFileMapper:

src/engine/src/core/mappers/TaskMapper.py → src/engine/src/mappers/TaskMapper.py

@@ -1,4 +1,4 @@
-from core.daos import WorkflowExecutorStateDAO
+from daos import WorkflowExecutorStateDAO
 
 
 class TaskMapper:

src/engine/src/core/mappers/TaskOutputMapper.py → src/engine/src/mappers/TaskOutputMapper.py

@@ -1,6 +1,6 @@
 import os
 
-from core.daos import FileSystemDAO
+from daos import FileSystemDAO
 
 
 class TaskOutputMapper:

src/engine/src/mappers/__init__.py

@@ -0,0 +1,6 @@
+from mappers.ArgMapper import ArgMapper
+from mappers.ArgValueFileMapper import ArgValueFileMapper
+from mappers.EnvVarValueFileMapper import EnvVarValueFileMapper
+from mappers.EnvMapper import EnvMapper
+from mappers.TaskMapper import TaskMapper
+from mappers.TaskOutputMapper import TaskOutputMapper

src/engine/src/middleware/archivers/__init__.py

@@ -0,0 +1,4 @@
+from middleware.archivers.S3Archiver import S3Archiver
+from middleware.archivers.IRODSArchiver import IRODSArchiver
+
+

src/engine/src/owe_python_sdk/TaskExecutor.py

@@ -9,7 +9,7 @@
 from owe_python_sdk.TaskOutputFile import TaskOutputFile
 from owe_python_sdk.constants import STDERR, STDOUT
 from utils import lbuffer_str as lbuf
-from core.resources import Resource, ResourceType
+from resources import Resource, ResourceType
 from conf.constants import (
     DEFAULT_POLLING_INTERVAL,
     KUBERNETES_NAMESPACE,

src/engine/src/core/repositories/ArgRepository.py → src/engine/src/repositories/ArgRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import ArgMapper
+from mappers import ArgMapper
 
 
 class ArgRepository:

src/engine/src/core/repositories/ArgValueFileRepository.py → src/engine/src/repositories/ArgValueFileRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import ArgValueFileMapper
+from mappers import ArgValueFileMapper
 
 
 class ArgValueFileRepository:

src/engine/src/core/repositories/EnvRepository.py → src/engine/src/repositories/EnvRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import EnvMapper
+from mappers import EnvMapper
 
 
 class EnvRepository:

src/engine/src/core/repositories/EnvVarValueFileRepository.py → src/engine/src/repositories/EnvVarValueFileRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import EnvVarValueFileMapper
+from mappers import EnvVarValueFileMapper
 
 
 class EnvVarValueFileRepository:

src/engine/src/core/repositories/TaskOutputRepository.py → src/engine/src/repositories/TaskOutputRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import TaskOutputMapper
+from mappers import TaskOutputMapper
 
 from owe_python_sdk.schema import Task
 

src/engine/src/core/repositories/TaskRepository.py → src/engine/src/repositories/TaskRepository.py

@@ -1,4 +1,4 @@
-from core.mappers import TaskMapper
+from mappers import TaskMapper
 
 
 class TaskRepository:

src/engine/src/core/repositories/TemplateRepository.py → src/engine/src/repositories/TemplateRepository.py

@@ -2,7 +2,7 @@
 
 from urllib.parse import urlparse
 
-from core.repositories import GitCacheRepository
+from repositories import GitCacheRepository
 from owe_python_sdk.schema import Uses
 
 

src/engine/src/repositories/__init__.py

@@ -0,0 +1,8 @@
+from repositories.EnvRepository import EnvRepository
+from repositories.GitCacheRepository import GitCacheRepository
+from repositories.ArgRepository import ArgRepository
+from repositories.ArgValueFileRepository import ArgValueFileRepository
+from repositories.EnvVarValueFileRepository import EnvVarValueFileRepository
+from repositories.TaskOutputRepository import TaskOutputRepository
+from repositories.TaskRepository import TaskRepository
+from repositories.TemplateRepository import TemplateRepository

src/engine/src/core/state/ReactiveState.py → src/engine/src/state/ReactiveState.py

@@ -3,7 +3,7 @@
 from typing import Any
 from threading import Lock
 
-from core.state import Hook
+from state import Hook
 
 
 class ReactiveState(object):

src/engine/src/state/__init__.py

@@ -0,0 +1,2 @@
+from state.Hook import Hook, method_hook
+from state.ReactiveState import ReactiveState

src/engine/src/core/tasks/BaseBuildExecutor.py → src/engine/src/tasks/BaseBuildExecutor.py

@@ -1,6 +1,6 @@
 import json, base64, os
 
-from core.tasks.executors.builders.helpers.ContextResolver import context_resolver
+from tasks.executors.builders.helpers.ContextResolver import context_resolver
 from errors.credentials import CredentialsError
 from owe_python_sdk.TaskExecutor import TaskExecutor
 

src/engine/src/core/tasks/TaskExecutorFactory.py → src/engine/src/tasks/TaskExecutorFactory.py

@@ -1,10 +1,10 @@
 import logging
 
 from owe_python_sdk.events import EventExchange
-from core.tasks.BuildTaskExecutorResolver import build_task_executor_resolver
-from core.tasks.executors.requesters.HTTP import HTTP
-from core.tasks.executors.Function import Function
-from core.tasks.executors.Application import Application
+from tasks.BuildTaskExecutorResolver import build_task_executor_resolver
+from tasks.executors.requesters.HTTP import HTTP
+from tasks.executors.Function import Function
+from tasks.executors.Application import Application
 from errors.tasks import InvalidTaskTypeError
 
 

src/engine/src/core/tasks/TaskInputFileStagingService.py → src/engine/src/tasks/TaskInputFileStagingService.py

@@ -1,6 +1,6 @@
 import os
 
-from core.workflows import ValueFromService
+from workflows import ValueFromService
 from owe_python_sdk.schema import Task
 from errors.tasks import TaskInputStagingError
 from owe_python_sdk.utils import select_field

src/engine/src/core/tasks/executors/Application.py → src/engine/src/tasks/executors/Application.py

@@ -6,7 +6,7 @@
 
 from owe_python_sdk.TaskExecutor import TaskExecutor
 from conf.constants import KUBERNETES_NAMESPACE
-from core.resources import PodResource
+from resources import PodResource
 from utils import get_flavor
 from utils.k8s import flavor_to_k8s_resource_reqs, gen_resource_name
 

src/engine/src/core/tasks/executors/Function.py → src/engine/src/tasks/executors/Function.py

@@ -10,11 +10,11 @@
     KUBERNETES_NAMESPACE,
     OWE_PYTHON_SDK_DIR,
 )
-from core.resources import JobResource
+from resources import JobResource
 from utils import get_flavor
 from utils.k8s import flavor_to_k8s_resource_reqs, gen_resource_name
-from core.tasks import function_bootstrap
-from core.repositories import GitCacheRepository
+from tasks import function_bootstrap
+from repositories import GitCacheRepository
 
 
 class ContainerDetails:

src/engine/src/core/tasks/executors/builders/kaniko/Kaniko.py → src/engine/src/tasks/executors/builders/kaniko/Kaniko.py

@@ -8,8 +8,8 @@
     KANIKO_IMAGE_URL,
     KANIKO_IMAGE_TAG
 )
-from core.tasks.BaseBuildExecutor import BaseBuildExecutor
-from core.resources import ConfigMapResource, JobResource
+from tasks.BaseBuildExecutor import BaseBuildExecutor
+from resources import ConfigMapResource, JobResource
 from utils import get_flavor, lbuffer_str as lbuf
 from utils.k8s import flavor_to_k8s_resource_reqs, gen_resource_name
 from errors import WorkflowTerminated

src/engine/src/tasks/executors/builders/kaniko/__init__.py

@@ -0,0 +1 @@
+from tasks.executors.builders.kaniko.Kaniko import Kaniko

src/engine/src/core/tasks/executors/builders/singularity/Singularity.py → src/engine/src/tasks/executors/builders/singularity/Singularity.py

@@ -13,9 +13,9 @@
 )
 
 from conf.constants import KUBERNETES_NAMESPACE, WORKFLOW_NFS_SERVER
-from core.resources import JobResource
-from core.tasks.BaseBuildExecutor import BaseBuildExecutor
-from core.tasks.executors.builders.singularity.helpers.ContainerBuilder import container_builder
+from resources import JobResource
+from tasks.BaseBuildExecutor import BaseBuildExecutor
+from tasks.executors.builders.singularity.helpers.ContainerBuilder import container_builder
 from errors import WorkflowTerminated
 from utils.k8s import gen_resource_name
 

src/engine/src/tasks/executors/builders/singularity/__init__.py

@@ -0,0 +1 @@
+from tasks.executors.builders.singularity.Singularity import Singularity

src/engine/src/core/templating/TemplateMapper.py → src/engine/src/templating/TemplateMapper.py

@@ -1,6 +1,6 @@
 from typing import Union
 
-from core.repositories import TemplateRepository
+from repositories import TemplateRepository
 from owe_python_sdk.schema import (
     Uses,
     Pipeline,

src/engine/src/templating/__init__.py

@@ -0,0 +1 @@
+from templating.TemplateMapper import TemplateMapper

src/engine/src/tests/TestConditionalExpressionEvaluator.py

@@ -2,7 +2,7 @@
 
 from unittest.mock import Mock
 
-from core.expressions import (
+from expressions import (
     ConditionalExpressionEvaluator,
 
 )

src/engine/src/tests/TestIOCContainerFactory.py

@@ -1,7 +1,7 @@
 import unittest
 
-from core.ioc import IOCContainerFactory
-from core.daos import WorkflowExecutorStateDAO
+from ioc import IOCContainerFactory
+from daos import WorkflowExecutorStateDAO
 
 
 class TestIOCContainerFactory(unittest.TestCase):