From 6fd0f5a7fcb5307c67a9de9e9ac7d3cabe15996f Mon Sep 17 00:00:00 2001 From: Nathan Freeman Date: Wed, 18 Oct 2023 11:36:36 -0500 Subject: [PATCH] Convert TemplateRepository to class --- .../workflows/executors/WorkflowExecutor.py | 13 ++--- src/engine/src/helpers/TemplateMapper.py | 4 +- src/engine/src/helpers/TemplateRepository.py | 55 ++++++++++--------- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/src/engine/src/core/workflows/executors/WorkflowExecutor.py b/src/engine/src/core/workflows/executors/WorkflowExecutor.py index 945249dc..fcbca2dc 100644 --- a/src/engine/src/core/workflows/executors/WorkflowExecutor.py +++ b/src/engine/src/core/workflows/executors/WorkflowExecutor.py @@ -20,7 +20,6 @@ ) from helpers.GraphValidator import GraphValidator # From shared from helpers.TemplateMapper import TemplateMapper -from helpers.TemplateRepository import TemplateRepository from errors.tasks import ( InvalidTaskTypeError, MissingInitialTasksError, @@ -441,12 +440,12 @@ def _prepare_pipeline(self): # Create all of the directories needed for the pipeline to run and persist results and cache self._prepare_pipeline_fs() - template = TemplateRepository( - self.state.ctx.pipeline.uses, - cache_dir=self.state.ctx.pipeline.git_cache_dir - ) - - # TODO map the template props to the pipeline + # template_mapper = TemplateMapper(cache_dir=self.state.ctx.pipeline.git_cache_dir) + # if self.state.ctx.pipeline.uses != None: + # self.state.ctx.pipeline = template_mapper.map( + # self.state.ctx.pipeline, + # self.state.ctx.pipeline.uses + # ) @interceptable() def _prepare_pipeline_fs(self): diff --git a/src/engine/src/helpers/TemplateMapper.py b/src/engine/src/helpers/TemplateMapper.py index 905dfcc3..c2dbee97 100644 --- a/src/engine/src/helpers/TemplateMapper.py +++ b/src/engine/src/helpers/TemplateMapper.py @@ -16,7 +16,6 @@ class TemplateMapper: def __init__(self, cache_dir: str): - self.cache_dir = cache_dir self.task_map_by_type = { "function": FunctionTask, "application": ApplicationTask, @@ -25,6 +24,7 @@ def __init__(self, cache_dir: str): "tapis_job": TapisJobTask, "tapis_actor": TapisActorTask } + self.template_repo = TemplateRepository(cache_dir=cache_dir) def map(self, obj: Union[Pipeline, Task], uses: Uses) -> Union[Pipeline, Task]: """This method takes an object(Pipeline or Task object), and updates its @@ -35,7 +35,7 @@ def map(self, obj: Union[Pipeline, Task], uses: Uses) -> Union[Pipeline, Task]: """ # Clone git repository specified on the pipeline.uses if exists - template = TemplateRepository(uses, cache_dir=self.cache_dir) + template = self.template_repo.get_by_uses(uses) # Resolve which class the final object should have obj_class = Pipeline diff --git a/src/engine/src/helpers/TemplateRepository.py b/src/engine/src/helpers/TemplateRepository.py index c1379808..2964e8f9 100644 --- a/src/engine/src/helpers/TemplateRepository.py +++ b/src/engine/src/helpers/TemplateRepository.py @@ -3,32 +3,35 @@ from .GitCacheService import GitCacheService from ..owe_python_sdk.schema import Uses -def TemplateRepository(uses: Uses, cache_dir: str): - # Clone git repository specified on the pipeline.uses if exists - git_cache_service = GitCacheService(cache_dir=cache_dir) +class TemplateRepository: + def __init__(self, cache_dir: str): + # Clone git repository specified on the pipeline.uses if exists + self.cache_dir = cache_dir + self.git_cache_service = GitCacheService(cache_dir=cache_dir) - git_cache_service.add_or_update( - uses.source.url, - # NOTE Using the url as the directory to clone into is intentional - uses.source.url - ) + def get_by_uses(self, uses: Uses): + self.git_cache_service.add_or_update( + uses.source.url, + # NOTE Using the url as the directory to clone into is intentional + uses.source.url + ) - template_root_dir = os.path.join(cache_dir, uses.source.url) - - try: - # Open the owe-config.json file - with open(os.path.join(template_root_dir, "owe-config.json")) as file: - owe_config = json.loads(file.read()) - - # Open the etl pipeline schema.json - with open( - os.path.join( - template_root_dir, - owe_config.get(uses.name).get("path") - ) - ) as file: - template = json.loads(file.read()) - except Exception as e: - raise Exception(f"Templating configuration Error (owe-config.json): {str(e)}") + template_root_dir = os.path.join(self.cache_dir, uses.source.url) - return template \ No newline at end of file + try: + # Open the owe-config.json file + with open(os.path.join(template_root_dir, "owe-config.json")) as file: + owe_config = json.loads(file.read()) + + # Open the etl pipeline schema.json + with open( + os.path.join( + template_root_dir, + owe_config.get(uses.name).get("path") + ) + ) as file: + template = json.loads(file.read()) + except Exception as e: + raise Exception(f"Templating configuration Error (owe-config.json): {str(e)}") + + return template \ No newline at end of file