From d063df5b7d27e0c7f5081957ebb256a9cd4262fa Mon Sep 17 00:00:00 2001 From: Nathan Freeman Date: Mon, 9 Oct 2023 13:36:45 -0500 Subject: [PATCH] add uses prop to pipeline --- src/api/src/backend/urls.py | 2 +- src/api/src/backend/views/ETLPipelines.py | 61 +++++----------------- src/api/src/backend/views/Pipelines.py | 6 +-- src/api/src/backend/views/http/etl.py | 6 +-- src/api/src/backend/views/http/requests.py | 4 +- 5 files changed, 22 insertions(+), 57 deletions(-) diff --git a/src/api/src/backend/urls.py b/src/api/src/backend/urls.py index 43bd0eef..b9ef477d 100644 --- a/src/api/src/backend/urls.py +++ b/src/api/src/backend/urls.py @@ -48,7 +48,7 @@ # Pipelines path("groups//ci", Pipelines.as_view(), name="ci"), - path("groups//etl", ETLPipelines.as_view(), name="etl"), + path("groups//etl", ETLPipelines.as_view(), name="etl"), path("groups//pipelines", Pipelines.as_view(), name="pipelines"), path("groups//pipelines/", Pipelines.as_view(), name="pipeline"), path("groups//pipelines//owner/", ChangePipelineOwner.as_view(), name="changePipelineOwner"), diff --git a/src/api/src/backend/views/ETLPipelines.py b/src/api/src/backend/views/ETLPipelines.py index b2b2b54a..663dd8ef 100644 --- a/src/api/src/backend/views/ETLPipelines.py +++ b/src/api/src/backend/views/ETLPipelines.py @@ -11,12 +11,11 @@ Forbidden, ServerError as ServerErrorResp ) -from backend.views.http.responses.models import ModelListResponse from backend.views.http.responses import BaseResponse, ResourceURLResponse -from backend.views.http.requests import WorkflowPipeline, CIPipeline, ImageBuildTask +from backend.views.http.requests import ImageBuildTask from backend.views.http.etl import TapisETLPipeline from backend.models import ( - Pipeline, + Pipeline as PipelineModel, Archive, PipelineArchive, TASK_TYPE_IMAGE_BUILD @@ -40,7 +39,16 @@ def post(self, request, group_id, *_, **__): return Forbidden(message="You do not have access to this group") # Validate the request body based on the type of pipeline specified - prepared_request = self.prepare(TapisETLPipeline) + prepared_request = self.prepare( + TapisETLPipeline, + uses={ + "name": "tapis/etl-pipeline@v1beta", + "source": { + "url": "https://github.com/tapis-project/tapis-owe-templates.git", + "branch": "master" + } + } + ) # Return the failure view instance if validation failed if not prepared_request.is_valid: @@ -50,12 +58,12 @@ def post(self, request, group_id, *_, **__): body = prepared_request.body # Check that the id of the pipeline is unique - if Pipeline.objects.filter(id=body.id, group=group).exists(): + if PipelineModel.objects.filter(id=body.id, group=group).exists(): return Conflict(f"A Pipeline already exists with the id '{body.id}'") # Create the pipeline try: - pipeline = Pipeline.objects.create( + pipeline = PipelineModel.objects.create( id=body.id, group=group, owner=request.username, @@ -100,50 +108,9 @@ def post(self, request, group_id, *_, **__): [pipeline_archive.delete() for pipeline_archive in pipeline_archives] return BadRequest(message=e.__cause__) - # Fetch the function for building the pipeline according to its type. - fn = getattr(self, PIPELINE_TYPE_MAPPING[body.type]) return fn(request, body, pipeline) - def delete(self, request, group_id, pipeline_id, *_, **__): - # Get the group - group = group_service.get(group_id, request.tenant_id) - if group == None: - return NotFound(f"No group found with id '{group_id}'") - - # Check that the user belongs to the group - if not group_service.user_in_group(request.username, group_id, request.tenant_id): - return Forbidden(message="You do not have access to this group") - - # Get the pipeline by the id provided in the path params - pipeline = Pipeline.objects.filter( - id=pipeline_id, - group=group - ).prefetch_related("tasks").first() - - - if pipeline == None: - return NotFound(f"Pipeline not found with id '{pipeline_id}'") - - # Delete operation only allowed by owner - if request.username != pipeline.owner: - return Forbidden(message="Only the owner of this pipeline can delete it") - - # Delete the pipeline - try: - # Delete the tasks - tasks = pipeline.tasks.all() - task_service.delete(tasks) - except (DatabaseError, OperationalError) as e: - return ServerErrorResp(message=e.__cause__) - except ServerError as e: - return ServerErrorResp(message=e) - - pipeline.delete() - - msg = f"Pipeline '{pipeline_id}' deleted. {len(tasks)} task(s) deleted." - return BaseResponse(message=msg, result=msg) - def build_ci_pipeline(self, request, body, pipeline): try: # Build an task_request from the pipeline request body diff --git a/src/api/src/backend/views/Pipelines.py b/src/api/src/backend/views/Pipelines.py index 970726c5..391d0661 100644 --- a/src/api/src/backend/views/Pipelines.py +++ b/src/api/src/backend/views/Pipelines.py @@ -13,7 +13,7 @@ ) from backend.views.http.responses.models import ModelListResponse from backend.views.http.responses import BaseResponse, ResourceURLResponse -from backend.views.http.requests import WorkflowPipeline, CIPipeline, ImageBuildTask +from backend.views.http.requests import Pipeline, CIPipeline, ImageBuildTask from backend.models import ( Pipeline, Archive, @@ -96,8 +96,8 @@ def post(self, request, group_id, *_, **__): ): return BadRequest(f"Request body must inlcude a 'type' property with one of the following values: {PIPELINE_TYPES}") - # Determine the proper request type. Default will be a WorkflowPipeline request - PipelineCreateRequest = WorkflowPipeline + # Determine the proper request type. Default will be a normal pipeline request + PipelineCreateRequest = Pipeline if self.request_body["type"] == PIPELINE_TYPE_CI: PipelineCreateRequest = CIPipeline diff --git a/src/api/src/backend/views/http/etl.py b/src/api/src/backend/views/http/etl.py index e56d4f6b..044f2e9b 100644 --- a/src/api/src/backend/views/http/etl.py +++ b/src/api/src/backend/views/http/etl.py @@ -3,7 +3,7 @@ from pydantic import BaseModel -from . import _EnumMeta +from .requests import _EnumMeta, Pipeline class EnumManifestGenerationPolicy(str, Enum, metaclass=_EnumMeta): @@ -51,7 +51,7 @@ class S3RemoteInbox(BaseModel): url: str bucket: str -class TapisETLPipeline(BaseModel): +class TapisETLPipeline(Pipeline): remote_outbox: Dict = None local_inbox: LocalInbox jobs: List[Dict] @@ -60,4 +60,4 @@ class TapisETLPipeline(BaseModel): GlobusRemoteInbox, S3RemoteInbox ] - follow_tasks: List[Dict] = [] \ No newline at end of file + followup_tasks: List[Dict] = [] \ No newline at end of file diff --git a/src/api/src/backend/views/http/requests.py b/src/api/src/backend/views/http/requests.py index 4bad8fa1..fcb3d8d2 100644 --- a/src/api/src/backend/views/http/requests.py +++ b/src/api/src/backend/views/http/requests.py @@ -674,6 +674,7 @@ class FunctionTask(BaseTask): class Pipeline(BaseModel): id: ID type: EnumPipelineType = EnumPipelineType.Workflow + uses: Union[str, Uses] tasks: List[ Annotated[ Union[ @@ -716,9 +717,6 @@ def backwards_compatibility_transforms(cls, values): class Config: extra = Extra.allow -class WorkflowPipeline(Pipeline): - pass - class CIPipeline(Pipeline): cache: bool = False builder: str