add uses prop to pipeline

src/api/src/backend/urls.py

@@ -48,7 +48,7 @@
 
     # Pipelines
     path("groups/<str:group_id>/ci", Pipelines.as_view(), name="ci"),
-     path("groups/<str:group_id>/etl", ETLPipelines.as_view(), name="etl"),
+    path("groups/<str:group_id>/etl", ETLPipelines.as_view(), name="etl"),
     path("groups/<str:group_id>/pipelines", Pipelines.as_view(), name="pipelines"),
     path("groups/<str:group_id>/pipelines/<str:pipeline_id>", Pipelines.as_view(), name="pipeline"),
     path("groups/<str:group_id>/pipelines/<str:pipeline_id>/owner/<str:username>", ChangePipelineOwner.as_view(), name="changePipelineOwner"),

src/api/src/backend/views/ETLPipelines.py

@@ -11,12 +11,11 @@
     Forbidden,
     ServerError as ServerErrorResp
 )
-from backend.views.http.responses.models import ModelListResponse
 from backend.views.http.responses import BaseResponse, ResourceURLResponse
-from backend.views.http.requests import WorkflowPipeline, CIPipeline, ImageBuildTask
+from backend.views.http.requests import ImageBuildTask
 from backend.views.http.etl import TapisETLPipeline
 from backend.models import (
-    Pipeline,
+    Pipeline as PipelineModel,
     Archive,
     PipelineArchive,
     TASK_TYPE_IMAGE_BUILD
@@ -40,7 +39,16 @@ def post(self, request, group_id, *_, **__):
             return Forbidden(message="You do not have access to this group")
 
         # Validate the request body based on the type of pipeline specified
-        prepared_request = self.prepare(TapisETLPipeline)
+        prepared_request = self.prepare(
+            TapisETLPipeline,
+            uses={
+                "name": "tapis/etl-pipeline@v1beta",
+                "source": {
+                    "url": "https://github.com/tapis-project/tapis-owe-templates.git",
+                    "branch": "master"
+                }
+            }
+        )
 
         # Return the failure view instance if validation failed
         if not prepared_request.is_valid:
@@ -50,12 +58,12 @@ def post(self, request, group_id, *_, **__):
         body = prepared_request.body
 
         # Check that the id of the pipeline is unique
-        if Pipeline.objects.filter(id=body.id, group=group).exists():
+        if PipelineModel.objects.filter(id=body.id, group=group).exists():
             return Conflict(f"A Pipeline already exists with the id '{body.id}'")
 
         # Create the pipeline
         try:
-            pipeline = Pipeline.objects.create(
+            pipeline = PipelineModel.objects.create(
                 id=body.id,
                 group=group,
                 owner=request.username,
@@ -100,50 +108,9 @@ def post(self, request, group_id, *_, **__):
             [pipeline_archive.delete() for pipeline_archive in pipeline_archives]
             return BadRequest(message=e.__cause__)
 
-        # Fetch the function for building the pipeline according to its type.
-        fn = getattr(self, PIPELINE_TYPE_MAPPING[body.type])
 
         return fn(request, body, pipeline)
 
-    def delete(self, request, group_id, pipeline_id, *_, **__):
-        # Get the group
-        group = group_service.get(group_id, request.tenant_id)
-        if group == None:
-            return NotFound(f"No group found with id '{group_id}'")
-
-        # Check that the user belongs to the group
-        if not group_service.user_in_group(request.username, group_id, request.tenant_id):
-            return Forbidden(message="You do not have access to this group")
-
-        # Get the pipeline by the id provided in the path params
-        pipeline = Pipeline.objects.filter(
-            id=pipeline_id,
-            group=group
-        ).prefetch_related("tasks").first()
-
-
-        if pipeline == None:
-            return NotFound(f"Pipeline not found with id '{pipeline_id}'")
-
-        # Delete operation only allowed by owner
-        if request.username != pipeline.owner:
-            return Forbidden(message="Only the owner of this pipeline can delete it")
-
-        # Delete the pipeline
-        try:
-            # Delete the tasks
-            tasks = pipeline.tasks.all()
-            task_service.delete(tasks)
-        except (DatabaseError, OperationalError) as e:
-            return ServerErrorResp(message=e.__cause__)
-        except ServerError as e:
-            return ServerErrorResp(message=e)
-
-        pipeline.delete()
-
-        msg = f"Pipeline '{pipeline_id}' deleted. {len(tasks)} task(s) deleted."
-        return BaseResponse(message=msg, result=msg)
-
     def build_ci_pipeline(self, request, body, pipeline):
         try:
             # Build an task_request from the pipeline request body

src/api/src/backend/views/Pipelines.py

@@ -13,7 +13,7 @@
 )
 from backend.views.http.responses.models import ModelListResponse
 from backend.views.http.responses import BaseResponse, ResourceURLResponse
-from backend.views.http.requests import WorkflowPipeline, CIPipeline, ImageBuildTask
+from backend.views.http.requests import Pipeline, CIPipeline, ImageBuildTask
 from backend.models import (
     Pipeline,
     Archive,
@@ -96,8 +96,8 @@ def post(self, request, group_id, *_, **__):
         ):
             return BadRequest(f"Request body must inlcude a 'type' property with one of the following values: {PIPELINE_TYPES}")
 
-        # Determine the proper request type. Default will be a WorkflowPipeline request
-        PipelineCreateRequest = WorkflowPipeline
+        # Determine the proper request type. Default will be a normal pipeline request
+        PipelineCreateRequest = Pipeline
         if self.request_body["type"] == PIPELINE_TYPE_CI:
             PipelineCreateRequest = CIPipeline
 

src/api/src/backend/views/http/etl.py

@@ -3,7 +3,7 @@
 
 from pydantic import BaseModel
 
-from . import _EnumMeta
+from .requests import _EnumMeta, Pipeline
 
 
 class EnumManifestGenerationPolicy(str, Enum, metaclass=_EnumMeta):
@@ -51,7 +51,7 @@ class S3RemoteInbox(BaseModel):
     url: str
     bucket: str
 
-class TapisETLPipeline(BaseModel):
+class TapisETLPipeline(Pipeline):
     remote_outbox: Dict = None
     local_inbox: LocalInbox
     jobs: List[Dict]
@@ -60,4 +60,4 @@ class TapisETLPipeline(BaseModel):
         GlobusRemoteInbox,
         S3RemoteInbox
     ]
-    follow_tasks: List[Dict] = []
+    followup_tasks: List[Dict] = []

src/api/src/backend/views/http/requests.py

@@ -674,6 +674,7 @@ class FunctionTask(BaseTask):
 class Pipeline(BaseModel):
     id: ID
     type: EnumPipelineType = EnumPipelineType.Workflow
+    uses: Union[str, Uses]
     tasks: List[
         Annotated[
             Union[
@@ -716,9 +717,6 @@ def backwards_compatibility_transforms(cls, values):
     class Config:
         extra = Extra.allow
 
-class WorkflowPipeline(Pipeline):
-    pass
-
 class CIPipeline(Pipeline):
     cache: bool = False
     builder: str