Add Secrets and GroupSecrets endpoints and models

src/api/src/backend/helpers/PipelineDispatchRequestBuilder.py

@@ -7,8 +7,8 @@
 
 
 class PipelineDispatchRequestBuilder:
-    def __init__(self, secret_service):
-        self.secret_service = secret_service
+    def __init__(self, credentials_service):
+        self.credentials_service = credentials_service
 
     def build(
         self,
@@ -119,7 +119,7 @@ def _image_build(self, task_request, task):
 
         if context_creds != None:
             # Get the context credentials data
-            context_cred_data = self.secret_service.get_secret(context_creds.sk_id)
+            context_cred_data = self.credentials_service.get_secret(context_creds.sk_id)
             task_request["context"]["credentials"] = context_cred_data
 
         # NOTE Workflow engine expect build_file_path and not recipe_file_path
@@ -134,7 +134,7 @@ def _image_build(self, task_request, task):
 
         if destination_creds != None:
             # Get the context credentials data
-            destination_cred_data = self.secret_service.get_secret(destination_creds.sk_id)
+            destination_cred_data = self.credentials_service.get_secret(destination_creds.sk_id)
             task_request["destination"]["credentials"] = destination_cred_data
 
         return task_request

src/api/src/backend/models.py

@@ -325,6 +325,7 @@ class Destination(models.Model):
 
 class Group(models.Model):
     id = models.CharField(validators=[validate_id], max_length=128, unique=True)
+    description = models.TextField(null=True)
     created_at = models.DateTimeField(auto_now_add=True)
     owner = models.CharField(max_length=64)
     tenant_id = models.CharField(max_length=128)
@@ -341,6 +342,20 @@ class Meta:
             )
         ]
 
+class GroupSecret(models.Model):
+    id = models.CharField(max_length=128)
+    group = models.ForeignKey("backend.Group", related_name="groupsecrets", on_delete=models.CASCADE)
+    secret = models.ForeignKey("backend.Secret", related_name="groupsecrets", on_delete=models.CASCADE)
+    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["group", "id"],
+                name="groupsecret_secret_and_id"
+            )
+        ]
+
 class GroupUser(models.Model):
     group = models.ForeignKey("backend.Group", related_name="users", on_delete=models.CASCADE)
     username = models.CharField(max_length=64)
@@ -435,6 +450,26 @@ class PipelineRun(models.Model):
     started_at = models.DateTimeField(null=True)
     uuid = models.UUIDField(primary_key=True)
 
+class Secret(models.Model):
+    id = models.CharField(max_length=128)
+    tenant_id = models.CharField(max_length=128)
+    description = models.TextField(null=True)
+    sk_secret_name = models.CharField(max_length=128, unique=True)
+    owner = models.CharField(max_length=64)
+    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4)
+
+    class Meta:
+        indexes = [
+            models.Index(fields=["sk_secret_name"]),
+            models.Index(fields=["owner", "tenant_id"])
+        ]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["id", "tenant_id", "owner"],
+                name="secret_id_tenant_id_owner"
+            )
+        ]
+
 class Task(models.Model):
     class Meta:
         constraints = [

src/api/src/backend/serializers/GroupSecretSerializer.py

@@ -0,0 +1,14 @@
+from backend.serializers.UUIDSerializer import UUIDSerializer
+from backend.serializers.SecretSerializer import SecretSerializer
+
+
+class GroupSecretSerializer:
+    @staticmethod
+    def serialize(model):
+        group_secret = {}
+        group_secret["id"] = model.id
+        group_secret["group_id"] = model.group.id
+        group_secret["secret"] = SecretSerializer.serialize(model.secret)
+        group_secret["uuid"] = UUIDSerializer.serialize(model.uuid)
+
+        return group_secret

src/api/src/backend/serializers/SecretSerializer.py

@@ -0,0 +1,15 @@
+from backend.serializers.UUIDSerializer import UUIDSerializer
+
+
+class SecretSerializer:
+    @staticmethod
+    def serialize(model):
+        secret = {}
+        secret["id"] = model.id
+        secret["description"] = model.description
+        secret["tenant_id"] = model.tenant_id
+        secret["owner"] = model.owner
+        secret["sk_secret_name"] = model.sk_secret_name
+        secret["uuid"] = UUIDSerializer.serialize(model.uuid)
+
+        return secret

src/api/src/backend/serializers/__init__.py

@@ -14,4 +14,6 @@
 from backend.serializers.ContextSerializer import ContextSerializer
 from backend.serializers.DestinationSerializer import DestinationSerializer
 from backend.serializers.CredentialsSerializer import CredentialsSerializer
-from backend.serializers.PipelineSerializer import PipelineSerializer
+from backend.serializers.PipelineSerializer import PipelineSerializer
+from backend.serializers.SecretSerializer import SecretSerializer
+from backend.serializers.GroupSecretSerializer import GroupSecretSerializer

src/api/src/backend/services/CredentialsService.py

@@ -0,0 +1,79 @@
+import uuid
+
+from typing import Dict
+
+from backend.models import Credentials, Secret
+from backend.conf.constants import SECRETS_TENANT, TAPIS_SERVICE_ACCOUNT, SECRETS_TENANT
+from backend.services.TapisServiceAPIGateway import TapisServiceAPIGateway
+from backend.services.Service import Service
+from backend.views.http.secrets import ReqSecret
+
+
+class CredentialsService(Service):
+    def __init__(self):
+        self.tapis_service_api_gateway = TapisServiceAPIGateway()
+        Service.__init__(self)
+
+    def save(self, owner: str, data: Dict[str, str]):
+        service_client = self.tapis_service_api_gateway.get_client()
+
+        sk_secret_name = f"tapis+workflows+{owner}+{uuid.uuid4()}"
+        try:
+            service_client.sk.writeSecret(
+                secretType="user",
+                secretName=sk_secret_name,
+                user=TAPIS_SERVICE_ACCOUNT,
+                tenant=SECRETS_TENANT,
+                data=data,
+                _tapis_set_x_headers_from_service=True
+            )
+        except Exception as e:
+            raise e
+
+        credentials = Credentials.objects.create(sk_id=sk_secret_name, owner=owner)
+
+        return credentials
+
+    def delete(self, sk_secret_name: str):
+        service_client = self.tapis_service_api_gateway.get_client()
+
+        service_client.sk.deleteSecret(
+            secretType="user",
+            secretName=sk_secret_name,
+            user=TAPIS_SERVICE_ACCOUNT,
+            tenant=SECRETS_TENANT,
+            versions=[],
+            _tapis_set_x_headers_from_service=True
+        )
+
+        credentials = Credentials.objects.filter(sk_id=sk_secret_name).first()
+        if credentials is not None:
+            credentials.delete()
+
+    def get(self, sk_secret_name: str):
+        if Credentials.objects.filter(sk_id=sk_secret_name).exists():
+            return Credentials.objects.filter(sk_id=sk_secret_name)[0]
+
+        return None
+
+    def get_secret(self, sk_secret_name: str):
+        service_client = self.tapis_service_api_gateway.get_client()
+
+        try:
+            res = service_client.sk.readSecret(
+                secretType="user",
+                secretName=sk_secret_name,
+                user=TAPIS_SERVICE_ACCOUNT,
+                tenant=SECRETS_TENANT,
+                version=0,
+                _tapis_set_x_headers_from_service=True
+            )
+
+            return res.secretMap.__dict__
+        except Exception as e:
+            return None # TODO catch network error
+
+    def _format_secret_name(self, secret_name: str):
+        return secret_name.replace(" ", "-")
+
+service = CredentialsService()

src/api/src/backend/services/SecretService.py

@@ -1,78 +1,55 @@
-import uuid
-
 from typing import Dict
 
-from backend.models import Credentials
+from backend.models import Secret
 from backend.conf.constants import SECRETS_TENANT, TAPIS_SERVICE_ACCOUNT, SECRETS_TENANT
 from backend.services.TapisServiceAPIGateway import TapisServiceAPIGateway
 from backend.services.Service import Service
+from backend.views.http.secrets import ReqCreateSecret
 
 
 class SecretService(Service):
     def __init__(self):
         self.tapis_service_api_gateway = TapisServiceAPIGateway()
         Service.__init__(self)
 
-    def save(self, owner: str, data: Dict[str, str]):
+    def create(self, tenant_id, owner, req_secret: ReqCreateSecret):
         service_client = self.tapis_service_api_gateway.get_client()
 
-        sk_id = f"workflows+{owner}+{uuid.uuid4()}"
+        sk_secret_name = f"tapis+{tenant_id}+workflows+{owner}+{req_secret.id}"
         try:
             service_client.sk.writeSecret(
                 secretType="user",
-                secretName=sk_id,
+                secretName=sk_secret_name,
                 user=TAPIS_SERVICE_ACCOUNT,
                 tenant=SECRETS_TENANT,
-                data=data,
+                data=req_secret.data,
                 _tapis_set_x_headers_from_service=True
             )
+
+            return Secret.objects.create(
+                id=req_secret.id,
+                description=req_secret.description,
+                sk_secret_name=sk_secret_name,
+                owner=owner,
+                tenant_id=tenant_id
+            )
         except Exception as e:
             raise e
 
-        credentials = Credentials.objects.create(sk_id=sk_id, owner=owner)
-
-        return credentials
-
-    def delete(self, sk_id: str):
+    def delete(self, secret_id, tenant_id, owner):
         service_client = self.tapis_service_api_gateway.get_client()
 
-        service_client.sk.deleteSecret(
+        secret = Secret.objects.filter(secret_id=secret_id, tenant_id=tenant_id, owner=owner).first()
+        if secret is not None:
+            secret.delete()
+
+        service_client.sk.destroySecret(
             secretType="user",
-            secretName=sk_id,
+            secretName=secret.sk_secret_name,
             user=TAPIS_SERVICE_ACCOUNT,
             tenant=SECRETS_TENANT,
             versions=[],
             _tapis_set_x_headers_from_service=True
         )
 
-        credentials = Credentials.objects.filter(sk_id=sk_id).first()
-        if credentials is not None:
-            credentials.delete()
-
-    def get(self, sk_id: str):
-        if Credentials.objects.filter(sk_id=sk_id).exists():
-            return Credentials.objects.filter(sk_id=sk_id)[0]
-
-        return None
-
-    def get_secret(self, sk_id: str):
-        service_client = self.tapis_service_api_gateway.get_client()
-
-        try:
-            res = service_client.sk.readSecret(
-                secretType="user",
-                secretName=sk_id,
-                user=TAPIS_SERVICE_ACCOUNT,
-                tenant=SECRETS_TENANT,
-                version=0,
-                _tapis_set_x_headers_from_service=True
-            )
-
-            return res.secretMap.__dict__
-        except Exception as e:
-            return None # TODO catch network error
-
-    def _format_secret_name(self, secret_name: str):
-        return secret_name.replace(" ", "-")
-
 service = SecretService()

src/api/src/backend/services/TaskService.py

@@ -30,7 +30,7 @@
     DockerhubDestination,
     LocalDestination
 )
-from backend.services.SecretService import service as secret_service
+from backend.services.CredentialsService import service as credentials_service
 from backend.services.Service import Service
 from backend.errors.api import BadRequestError, ServerError
 
@@ -190,11 +190,11 @@ def _resolve_authn_source(self, request, pipeline, accessor):
                     cred_data[key] = getattr(credentials, key)
 
             try:
-                cred = secret_service.save(f"pipeline:{pipeline.id}", cred_data)
+                cred = credentials_service.save(f"pipeline:{pipeline.id}", cred_data)
 
                 # Register a rollback funtion(partial) that will be used to delete the credentials
                 # should any subsequent model creations fail
-                self._add_rollback(secret_service.delete, cred.sk_id)
+                self._add_rollback(credentials_service.delete, cred.sk_id)
             except Exception as e:
                 raise ServerError(str(e))
 

src/api/src/backend/urls.py

@@ -23,6 +23,8 @@
 from backend.views.ETLPipelines import ETLPipelines
 from backend.views.PipelineLocksGetList import PipelineLocksGetList
 from backend.views.PipelineLocksPostDelete import PipelineLocksPostDelete
+from backend.views.Secrets import Secrets
+from backend.views.GroupSecrets import GroupSecrets
 
 
 urlpatterns = [
@@ -34,11 +36,19 @@
     # Identities
     path("identities", Identities.as_view(), name="identities"),
     path("identities/<str:identity_uuid>", Identities.as_view(), name="identity"),
+
+    # Secrets
+    path("secrets", Secrets.as_view(), name="secrets"),
+    path("secrets/<str:secret_id>", Secrets.as_view(), name="secret"),
 
     # Groups
     path("groups", Groups.as_view(), name="groups"),
     path("groups/<str:group_id>", Groups.as_view(), name="group"),
 
+    # Group Secrets
+    path("groups/<str:group_id>/secrets", GroupSecrets.as_view(), name="groupSecrets"),
+    path("groups/<str:group_id>/secrets/<str:group_secret_id>", GroupSecrets.as_view(), name="groupSecret"),
+
     # Group Users
     path("groups/<str:group_id>/users", Users.as_view(), name="users"),
     path("groups/<str:group_id>/users/<str:username>", Users.as_view(), name="user"),

src/api/src/backend/views/APIView.py

@@ -6,7 +6,7 @@
 from django.views.decorators.csrf import csrf_exempt
 
 from backend.views.http.responses.errors import MethodNotAllowed, UnsupportedMediaType, BadRequest
-from backend.views.http.requests import PreparedRequest
+from backend.views.http.PreparedRequest import PreparedRequest
 from backend.conf.constants import (
     PERMITTED_HTTP_METHODS,
     PERMITTED_CONTENT_TYPES,

src/api/src/backend/views/Credentials.py

@@ -1,5 +1,5 @@
 from backend.models import Credentials
-from backend.services.SecretService import service as secret_service
+from backend.services.CredentialsService import service as credentials_service
 
 from backend.views.RestrictedAPIView import RestrictedAPIView
 from backend.views.http.responses.BaseResponse import BaseResponse
@@ -12,7 +12,7 @@ def get(self, request):
 
         creds = []
         for cred in credentials:
-            creds.append(secret_service.get_secret(cred.sk_id))
+            creds.append(credentials_service.get_secret(cred.sk_id))
 
         return BaseResponse(result=creds)
 
@@ -21,6 +21,6 @@ def delete(self, request):
         credentials_list = Credentials.objects.all()
 
         for credentials in credentials_list:
-            secret_service.delete(credentials.sk_id)
+            credentials_service.delete(credentials.sk_id)
 
         return BaseResponse(message="Credentials deleted")