diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ed2eeef..ea17c5c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [v1.6.x] - 2024-xx-xx +## [v1.7.0] - 2024-xx-xx ### Features - Added cooperative pipeline locking and queue functionality +- Added user secrets +- Added group secrets +- Added task patching for all task types +- Added delete group endpoint which will delete the group and all objects owned by that group ## [v1.6.0] - 2024-01-xx diff --git a/src/api/specs/WorkflowsAPI.yaml b/src/api/specs/WorkflowsAPI.yaml index 670fa9d7..50323175 100644 --- a/src/api/specs/WorkflowsAPI.yaml +++ b/src/api/specs/WorkflowsAPI.yaml @@ -237,6 +237,156 @@ paths: schema: $ref: '#/components/schemas/RespError' + '/v3/workflows/secrets': + get: + tags: + - Secrets + summary: Retrieve secrets + description: | + Retrieve all secrets for a user + operationId: listSecrets + responses: + '200': + description: List secrets successful. + content: + application/json: + schema: + $ref: '#/components/schemas/RespSecretList' + '401': + description: Not authorized. Invalid or Expired Token. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '500': + description: Server error. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + post: + tags: + - Secrets + summary: Create a secret + description: | + Create a secret. + operationId: createSecret + requestBody: + required: true + description: A JSON object specifying information for the pipeline to be created. + content: + application/json: + schema: + $ref: '#/components/schemas/ReqCreateSecret' + responses: + '201': + description: Secert created. + content: + application/json: + schema: + $ref: '#/components/schemas/RespSecret' + '400': + description: Bad request. Invalid JSON. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '401': + description: Not authorized. Invalid or Expired Token. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '409': + description: Conflict. Secret with provided id already exists. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '415': + description: Unsupported media type. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '500': + description: Server error. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + + '/v3/workflows/secrets/{secret_id}': + get: + tags: + - Secrets + summary: Retrieve a secret + description: | + Retrieve a secret + parameters: + - name: secret_id + in: path + required: true + schema: + $ref: '#/components/schemas/ID' + operationId: getSecret + responses: + '200': + description: Get secret successful. + content: + application/json: + schema: + $ref: '#/components/schemas/RespSecret' + '401': + description: Not authorized. Invalid or Expired Token. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '500': + description: Server error. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + delete: + tags: + - Secrets + summary: Delete a secret + description: Delete a secret and all of the objects that belong to them + parameters: + - name: secret_id + in: path + required: true + schema: + $ref: '#/components/schemas/ID' + operationId: deleteSecret + responses: + '200': + description: Delete secret. + content: + application/json: + schema: + $ref: '#/components/schemas/RespString' + '401': + description: Not authorized. Invalid or Expired Token. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '403': + description: Cannot remove a user to this secret. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '404': + description: Secret does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '/v3/workflows/groups/{group_id}/users': get: tags: @@ -502,6 +652,216 @@ paths: schema: $ref: '#/components/schemas/RespError' + '/v3/workflows/groups/{group_id}/secrets': + get: + tags: + - GroupSecrets + summary: List group secrets + description: | + List group_secrets. + parameters: + - name: group_id + in: path + required: true + schema: + $ref: '#/components/schemas/ID' + operationId: listGroupSecrets + responses: + '200': + description: List Group Secrets successful. + content: + application/json: + schema: + $ref: '#/components/schemas/RespGroupSecretList' + '400': + description: Bad request. Invalid JSON. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '401': + description: Not authorized. Invalid or Expired Token. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '403': + description: Cannot access resources for this Group. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '404': + description: Group does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '500': + description: Server error. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + post: + tags: + - GroupSecrets + summary: Add a user's secret to a group + description: | + Add a user's secret to a group. + parameters: + - name: group_id + in: path + required: true + schema: + $ref: '#/components/schemas/ID' + operationId: addGroupSecret + requestBody: + required: true + description: A JSON object specifying the GroupSecret to add. + content: + application/json: + schema: + $ref: '#/components/schemas/ReqGroupSecret' + responses: + '201': + description: GroupSecret successfully added. + content: + application/json: + schema: + $ref: '#/components/schemas/RespResourceURL' + '400': + description: Bad request. Invalid JSON. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '401': + description: Not authorized. Invalid or Expired Token. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '403': + description: Cannot add a secret to this group. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '404': + description: Group does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '500': + description: Server error. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + + '/v3/workflows/groups/{group_id}/secrets/{group_secret_id}': + get: + tags: + - GroupSecrets + summary: Get group secret + description: | + Get a group secret + parameters: + - name: group_id + in: path + required: true + schema: + $ref: '#/components/schemas/ID' + - name: group_secret_id + in: path + required: true + schema: + $ref: '#/components/schemas/ID' + operationId: getGroupSecret + responses: + '200': + description: Get GroupSecret successful. + content: + application/json: + schema: + $ref: '#/components/schemas/RespGroupSecret' + '400': + description: Bad request. Invalid JSON. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '401': + description: Not authorized. Invalid or Expired Token. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '403': + description: Cannot access resources for this Group. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '404': + description: Group does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '500': + description: Server error. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + + delete: + tags: + - GroupSecrets + summary: Remove user from group + description: | + Remove a user from a group. + parameters: + - name: group_id + in: path + required: true + schema: + $ref: '#/components/schemas/ID' + - name: group_secret_id + in: path + required: true + schema: + $ref: '#/components/schemas/ID' + operationId: removeGroupSecret + responses: + '200': + description: GroupSecret removed. + content: + application/json: + schema: + $ref: '#/components/schemas/RespObject' + '401': + description: Not authorized. Invalid or Expired Token. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '403': + description: Cannot remove a secret from this group. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + '404': + description: Group or GroupSecret does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/RespError' + # --- Paths for identities --- '/v3/workflows/identities': post: @@ -2728,6 +3088,36 @@ components: type: string format: uuid + Secret: + type: object + properties: + id: + $ref: "#/components/schemas/ID" + description: + type: string + tenant_id: + type: string + owner: + type: string + sk_secret_name: + type: string + uuid: + type: string + format: uuid + + GroupSecret: + type: object + properties: + id: + $ref: "#/components/schemas/ID" + group_id: + $ref: "#/components/schemas/ID" + uuid: + type: string + format: uuid + secret: + $ref: "#/components/schemas/Secret" + Identity: type: object properties: @@ -2970,6 +3360,30 @@ components: type: boolean default: false + ReqCreateSecret: + type: object + required: + - id + - data + properties: + id: + $ref: '#/components/schemas/ID' + description: + type: string + data: + type: object + + ReqGroupSecret: + type: object + required: + - secret_id + properties: + secret_id: + $ref: '#/components/schemas/ID' + id: + type: string + description: The unique identifier of the secret in the group. Defaults to the secret_id of no id is provided + ReqUpdateGroupUser: type: object required: @@ -3872,6 +4286,42 @@ components: items: $ref: '#/components/schemas/GroupUser' + RespSecret: + allOf: + - $ref: '#/components/schemas/RespObject' + - type: object + properties: + result: + $ref: '#/components/schemas/Secret' + + RespSecretList: + allOf: + - $ref: '#/components/schemas/RespList' + - type: object + properties: + result: + type: array + items: + $ref: '#/components/schemas/Secret' + + RespGroupSecret: + allOf: + - $ref: '#/components/schemas/RespObject' + - type: object + properties: + result: + $ref: '#/components/schemas/GroupSecret' + + RespGroupSecretList: + allOf: + - $ref: '#/components/schemas/RespList' + - type: object + properties: + result: + type: array + items: + $ref: '#/components/schemas/GroupSecret' + RespGroupList: allOf: - $ref: '#/components/schemas/RespList' diff --git a/src/api/src/backend/helpers/PipelineDispatchRequestBuilder.py b/src/api/src/backend/helpers/PipelineDispatchRequestBuilder.py index 50e1079c..c634cd20 100644 --- a/src/api/src/backend/helpers/PipelineDispatchRequestBuilder.py +++ b/src/api/src/backend/helpers/PipelineDispatchRequestBuilder.py @@ -7,8 +7,8 @@ class PipelineDispatchRequestBuilder: - def __init__(self, secret_service): - self.secret_service = secret_service + def __init__(self, credentials_service): + self.credentials_service = credentials_service def build( self, @@ -119,7 +119,7 @@ def _image_build(self, task_request, task): if context_creds != None: # Get the context credentials data - context_cred_data = self.secret_service.get_secret(context_creds.sk_id) + context_cred_data = self.credentials_service.get_secret(context_creds.sk_id) task_request["context"]["credentials"] = context_cred_data # NOTE Workflow engine expect build_file_path and not recipe_file_path @@ -134,7 +134,7 @@ def _image_build(self, task_request, task): if destination_creds != None: # Get the context credentials data - destination_cred_data = self.secret_service.get_secret(destination_creds.sk_id) + destination_cred_data = self.credentials_service.get_secret(destination_creds.sk_id) task_request["destination"]["credentials"] = destination_cred_data return task_request diff --git a/src/api/src/backend/migrations/0031_groupsecret_secret_group_description_and_more.py b/src/api/src/backend/migrations/0031_groupsecret_secret_group_description_and_more.py new file mode 100644 index 00000000..c8c04c0f --- /dev/null +++ b/src/api/src/backend/migrations/0031_groupsecret_secret_group_description_and_more.py @@ -0,0 +1,64 @@ +# Generated by Django 4.1.2 on 2024-09-12 00:18 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0030_pipeline_enabled_pipeline_lock_expiration_policy_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='GroupSecret', + fields=[ + ('id', models.CharField(max_length=128)), + ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ], + ), + migrations.CreateModel( + name='Secret', + fields=[ + ('id', models.CharField(max_length=128)), + ('tenant_id', models.CharField(max_length=128)), + ('description', models.TextField(null=True)), + ('sk_secret_name', models.CharField(max_length=128, unique=True)), + ('owner', models.CharField(max_length=64)), + ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ], + ), + migrations.AddField( + model_name='group', + name='description', + field=models.TextField(null=True), + ), + migrations.AddIndex( + model_name='secret', + index=models.Index(fields=['sk_secret_name'], name='backend_sec_sk_secr_f50be2_idx'), + ), + migrations.AddIndex( + model_name='secret', + index=models.Index(fields=['owner', 'tenant_id'], name='backend_sec_owner_6f6779_idx'), + ), + migrations.AddConstraint( + model_name='secret', + constraint=models.UniqueConstraint(fields=('id', 'tenant_id', 'owner'), name='secret_id_tenant_id_owner'), + ), + migrations.AddField( + model_name='groupsecret', + name='group', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groupsecrets', to='backend.group'), + ), + migrations.AddField( + model_name='groupsecret', + name='secret', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groupsecrets', to='backend.secret'), + ), + migrations.AddConstraint( + model_name='groupsecret', + constraint=models.UniqueConstraint(fields=('group', 'id'), name='groupsecret_secret_and_id'), + ), + ] \ No newline at end of file diff --git a/src/api/src/backend/models.py b/src/api/src/backend/models.py index 9cf89c68..561c37b5 100644 --- a/src/api/src/backend/models.py +++ b/src/api/src/backend/models.py @@ -325,6 +325,7 @@ class Destination(models.Model): class Group(models.Model): id = models.CharField(validators=[validate_id], max_length=128, unique=True) + description = models.TextField(null=True) created_at = models.DateTimeField(auto_now_add=True) owner = models.CharField(max_length=64) tenant_id = models.CharField(max_length=128) @@ -341,6 +342,20 @@ class Meta: ) ] +class GroupSecret(models.Model): + id = models.CharField(max_length=128) + group = models.ForeignKey("backend.Group", related_name="groupsecrets", on_delete=models.CASCADE) + secret = models.ForeignKey("backend.Secret", related_name="groupsecrets", on_delete=models.CASCADE) + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["group", "id"], + name="groupsecret_secret_and_id" + ) + ] + class GroupUser(models.Model): group = models.ForeignKey("backend.Group", related_name="users", on_delete=models.CASCADE) username = models.CharField(max_length=64) @@ -435,6 +450,26 @@ class PipelineRun(models.Model): started_at = models.DateTimeField(null=True) uuid = models.UUIDField(primary_key=True) +class Secret(models.Model): + id = models.CharField(max_length=128) + tenant_id = models.CharField(max_length=128) + description = models.TextField(null=True) + sk_secret_name = models.CharField(max_length=128, unique=True) + owner = models.CharField(max_length=64) + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4) + + class Meta: + indexes = [ + models.Index(fields=["sk_secret_name"]), + models.Index(fields=["owner", "tenant_id"]) + ] + constraints = [ + models.UniqueConstraint( + fields=["id", "tenant_id", "owner"], + name="secret_id_tenant_id_owner" + ) + ] + class Task(models.Model): class Meta: constraints = [ diff --git a/src/api/src/backend/serializers/GroupSecretSerializer.py b/src/api/src/backend/serializers/GroupSecretSerializer.py new file mode 100644 index 00000000..2ddf18c5 --- /dev/null +++ b/src/api/src/backend/serializers/GroupSecretSerializer.py @@ -0,0 +1,14 @@ +from backend.serializers.UUIDSerializer import UUIDSerializer +from backend.serializers.SecretSerializer import SecretSerializer + + +class GroupSecretSerializer: + @staticmethod + def serialize(model): + group_secret = {} + group_secret["id"] = model.id + group_secret["group_id"] = model.group.id + group_secret["secret"] = SecretSerializer.serialize(model.secret) + group_secret["uuid"] = UUIDSerializer.serialize(model.uuid) + + return group_secret \ No newline at end of file diff --git a/src/api/src/backend/serializers/SecretSerializer.py b/src/api/src/backend/serializers/SecretSerializer.py new file mode 100644 index 00000000..84080da9 --- /dev/null +++ b/src/api/src/backend/serializers/SecretSerializer.py @@ -0,0 +1,15 @@ +from backend.serializers.UUIDSerializer import UUIDSerializer + + +class SecretSerializer: + @staticmethod + def serialize(model): + secret = {} + secret["id"] = model.id + secret["description"] = model.description + secret["tenant_id"] = model.tenant_id + secret["owner"] = model.owner + secret["sk_secret_name"] = model.sk_secret_name + secret["uuid"] = UUIDSerializer.serialize(model.uuid) + + return secret \ No newline at end of file diff --git a/src/api/src/backend/serializers/__init__.py b/src/api/src/backend/serializers/__init__.py index 6ed429ca..54185ef8 100644 --- a/src/api/src/backend/serializers/__init__.py +++ b/src/api/src/backend/serializers/__init__.py @@ -14,4 +14,6 @@ from backend.serializers.ContextSerializer import ContextSerializer from backend.serializers.DestinationSerializer import DestinationSerializer from backend.serializers.CredentialsSerializer import CredentialsSerializer -from backend.serializers.PipelineSerializer import PipelineSerializer \ No newline at end of file +from backend.serializers.PipelineSerializer import PipelineSerializer +from backend.serializers.SecretSerializer import SecretSerializer +from backend.serializers.GroupSecretSerializer import GroupSecretSerializer \ No newline at end of file diff --git a/src/api/src/backend/services/CredentialsService.py b/src/api/src/backend/services/CredentialsService.py new file mode 100644 index 00000000..b2f45ada --- /dev/null +++ b/src/api/src/backend/services/CredentialsService.py @@ -0,0 +1,78 @@ +import uuid + +from typing import Dict + +from backend.models import Credentials, Secret +from backend.conf.constants import SECRETS_TENANT, TAPIS_SERVICE_ACCOUNT, SECRETS_TENANT +from backend.services.TapisServiceAPIGateway import TapisServiceAPIGateway +from backend.services.Service import Service + + +class CredentialsService(Service): + def __init__(self): + self.tapis_service_api_gateway = TapisServiceAPIGateway() + Service.__init__(self) + + def save(self, owner: str, data: Dict[str, str]): + service_client = self.tapis_service_api_gateway.get_client() + + sk_secret_name = f"tapis+workflows+{owner}+{uuid.uuid4()}" + try: + service_client.sk.writeSecret( + secretType="user", + secretName=sk_secret_name, + user=TAPIS_SERVICE_ACCOUNT, + tenant=SECRETS_TENANT, + data=data, + _tapis_set_x_headers_from_service=True + ) + except Exception as e: + raise e + + credentials = Credentials.objects.create(sk_id=sk_secret_name, owner=owner) + + return credentials + + def delete(self, sk_secret_name: str): + service_client = self.tapis_service_api_gateway.get_client() + + service_client.sk.deleteSecret( + secretType="user", + secretName=sk_secret_name, + user=TAPIS_SERVICE_ACCOUNT, + tenant=SECRETS_TENANT, + versions=[], + _tapis_set_x_headers_from_service=True + ) + + credentials = Credentials.objects.filter(sk_id=sk_secret_name).first() + if credentials is not None: + credentials.delete() + + def get(self, sk_secret_name: str): + if Credentials.objects.filter(sk_id=sk_secret_name).exists(): + return Credentials.objects.filter(sk_id=sk_secret_name)[0] + + return None + + def get_secret(self, sk_secret_name: str): + service_client = self.tapis_service_api_gateway.get_client() + + try: + res = service_client.sk.readSecret( + secretType="user", + secretName=sk_secret_name, + user=TAPIS_SERVICE_ACCOUNT, + tenant=SECRETS_TENANT, + version=0, + _tapis_set_x_headers_from_service=True + ) + + return res.secretMap.__dict__ + except Exception as e: + return None # TODO catch network error + + def _format_secret_name(self, secret_name: str): + return secret_name.replace(" ", "-") + +service = CredentialsService() diff --git a/src/api/src/backend/services/SecretService.py b/src/api/src/backend/services/SecretService.py index a63a1e5e..99552251 100644 --- a/src/api/src/backend/services/SecretService.py +++ b/src/api/src/backend/services/SecretService.py @@ -1,11 +1,10 @@ -import uuid - from typing import Dict -from backend.models import Credentials +from backend.models import Secret from backend.conf.constants import SECRETS_TENANT, TAPIS_SERVICE_ACCOUNT, SECRETS_TENANT from backend.services.TapisServiceAPIGateway import TapisServiceAPIGateway from backend.services.Service import Service +from backend.views.http.secrets import ReqCreateSecret class SecretService(Service): @@ -13,66 +12,44 @@ def __init__(self): self.tapis_service_api_gateway = TapisServiceAPIGateway() Service.__init__(self) - def save(self, owner: str, data: Dict[str, str]): + def create(self, tenant_id, owner, req_secret: ReqCreateSecret): service_client = self.tapis_service_api_gateway.get_client() - sk_id = f"workflows+{owner}+{uuid.uuid4()}" + sk_secret_name = f"tapis+{tenant_id}+workflows+{owner}+{req_secret.id}" try: service_client.sk.writeSecret( secretType="user", - secretName=sk_id, + secretName=sk_secret_name, user=TAPIS_SERVICE_ACCOUNT, tenant=SECRETS_TENANT, - data=data, + data=req_secret.data, _tapis_set_x_headers_from_service=True ) + + return Secret.objects.create( + id=req_secret.id, + description=req_secret.description, + sk_secret_name=sk_secret_name, + owner=owner, + tenant_id=tenant_id + ) except Exception as e: raise e - credentials = Credentials.objects.create(sk_id=sk_id, owner=owner) - - return credentials - - def delete(self, sk_id: str): + def delete(self, secret_id, tenant_id, owner): service_client = self.tapis_service_api_gateway.get_client() - service_client.sk.deleteSecret( + secret = Secret.objects.filter(id=secret_id, tenant_id=tenant_id, owner=owner).first() + if secret is not None: + secret.delete() + + service_client.sk.destroySecret( secretType="user", - secretName=sk_id, + secretName=secret.sk_secret_name, user=TAPIS_SERVICE_ACCOUNT, tenant=SECRETS_TENANT, versions=[], _tapis_set_x_headers_from_service=True ) - credentials = Credentials.objects.filter(sk_id=sk_id).first() - if credentials is not None: - credentials.delete() - - def get(self, sk_id: str): - if Credentials.objects.filter(sk_id=sk_id).exists(): - return Credentials.objects.filter(sk_id=sk_id)[0] - - return None - - def get_secret(self, sk_id: str): - service_client = self.tapis_service_api_gateway.get_client() - - try: - res = service_client.sk.readSecret( - secretType="user", - secretName=sk_id, - user=TAPIS_SERVICE_ACCOUNT, - tenant=SECRETS_TENANT, - version=0, - _tapis_set_x_headers_from_service=True - ) - - return res.secretMap.__dict__ - except Exception as e: - return None # TODO catch network error - - def _format_secret_name(self, secret_name: str): - return secret_name.replace(" ", "-") - service = SecretService() diff --git a/src/api/src/backend/services/TaskService.py b/src/api/src/backend/services/TaskService.py index 00fc15d6..2758c99d 100644 --- a/src/api/src/backend/services/TaskService.py +++ b/src/api/src/backend/services/TaskService.py @@ -30,7 +30,7 @@ DockerhubDestination, LocalDestination ) -from backend.services.SecretService import service as secret_service +from backend.services.CredentialsService import service as credentials_service from backend.services.Service import Service from backend.errors.api import BadRequestError, ServerError @@ -190,11 +190,11 @@ def _resolve_authn_source(self, request, pipeline, accessor): cred_data[key] = getattr(credentials, key) try: - cred = secret_service.save(f"pipeline:{pipeline.id}", cred_data) + cred = credentials_service.save(f"pipeline:{pipeline.id}", cred_data) # Register a rollback funtion(partial) that will be used to delete the credentials # should any subsequent model creations fail - self._add_rollback(secret_service.delete, cred.sk_id) + self._add_rollback(credentials_service.delete, cred.sk_id) except Exception as e: raise ServerError(str(e)) diff --git a/src/api/src/backend/urls.py b/src/api/src/backend/urls.py index 39de82bb..fb8a66d5 100644 --- a/src/api/src/backend/urls.py +++ b/src/api/src/backend/urls.py @@ -23,6 +23,8 @@ from backend.views.ETLPipelines import ETLPipelines from backend.views.PipelineLocksGetList import PipelineLocksGetList from backend.views.PipelineLocksPostDelete import PipelineLocksPostDelete +from backend.views.Secrets import Secrets +from backend.views.GroupSecrets import GroupSecrets urlpatterns = [ @@ -34,11 +36,19 @@ # Identities path("identities", Identities.as_view(), name="identities"), path("identities/", Identities.as_view(), name="identity"), + + # Secrets + path("secrets", Secrets.as_view(), name="secrets"), + path("secrets/", Secrets.as_view(), name="secret"), # Groups path("groups", Groups.as_view(), name="groups"), path("groups/", Groups.as_view(), name="group"), + # Group Secrets + path("groups//secrets", GroupSecrets.as_view(), name="groupSecrets"), + path("groups//secrets/", GroupSecrets.as_view(), name="groupSecret"), + # Group Users path("groups//users", Users.as_view(), name="users"), path("groups//users/", Users.as_view(), name="user"), diff --git a/src/api/src/backend/views/APIView.py b/src/api/src/backend/views/APIView.py index 069b708d..02c3401a 100644 --- a/src/api/src/backend/views/APIView.py +++ b/src/api/src/backend/views/APIView.py @@ -6,7 +6,7 @@ from django.views.decorators.csrf import csrf_exempt from backend.views.http.responses.errors import MethodNotAllowed, UnsupportedMediaType, BadRequest -from backend.views.http.requests import PreparedRequest +from backend.views.http.PreparedRequest import PreparedRequest from backend.conf.constants import ( PERMITTED_HTTP_METHODS, PERMITTED_CONTENT_TYPES, diff --git a/src/api/src/backend/views/Credentials.py b/src/api/src/backend/views/Credentials.py index 81924a7c..6e800596 100644 --- a/src/api/src/backend/views/Credentials.py +++ b/src/api/src/backend/views/Credentials.py @@ -1,5 +1,5 @@ from backend.models import Credentials -from backend.services.SecretService import service as secret_service +from backend.services.CredentialsService import service as credentials_service from backend.views.RestrictedAPIView import RestrictedAPIView from backend.views.http.responses.BaseResponse import BaseResponse @@ -12,7 +12,7 @@ def get(self, request): creds = [] for cred in credentials: - creds.append(secret_service.get_secret(cred.sk_id)) + creds.append(credentials_service.get_secret(cred.sk_id)) return BaseResponse(result=creds) @@ -21,6 +21,6 @@ def delete(self, request): credentials_list = Credentials.objects.all() for credentials in credentials_list: - secret_service.delete(credentials.sk_id) + credentials_service.delete(credentials.sk_id) return BaseResponse(message="Credentials deleted") \ No newline at end of file diff --git a/src/api/src/backend/views/GroupSecrets.py b/src/api/src/backend/views/GroupSecrets.py new file mode 100644 index 00000000..7294a07d --- /dev/null +++ b/src/api/src/backend/views/GroupSecrets.py @@ -0,0 +1,146 @@ +from django.db import IntegrityError, OperationalError, DatabaseError + +from backend.models import GroupSecret, Secret +from backend.views.RestrictedAPIView import RestrictedAPIView +from backend.views.http.responses import BaseResponse, NoContentResponse +from backend.views.http.responses.errors import BadRequest, Forbidden, NotFound, MethodNotAllowed, ServerError, Conflict +from backend.views.http.responses import BaseResponse +from backend.services.GroupService import service as group_service +from backend.serializers import GroupSecretSerializer +from backend.utils import logger + + +class GroupSecrets(RestrictedAPIView): + def get(self, request, group_id, group_secret_id=None): + try: + # Get the group + group = group_service.get(group_id, request.tenant_id) + if group == None: + return NotFound(f"No group found with id '{group_id}'") + + # Check that the user belongs to the group + if not group_service.user_in_group(request.username, group_id, request.tenant_id): + return Forbidden(message="You do not have access to this group") + + # Return a GroupSecretList response + if group_secret_id == None: + return self.list(group) + + # Get the group secret by id + group_secret = GroupSecret.objects.filter( + id=group_secret_id, + group=group + ).prefetch_related("secret", "group").first() + + # Return if NotFound if no group secret found with the provided id + if group_secret == None: + return NotFound(f"GroupSecret with id '{group_secret_id}' does not exist does in group '{group.id}'") + + return BaseResponse(result=GroupSecretSerializer.serialize(group_secret)) + except Exception as e: + logger.exception(e.__cuase__) + return ServerError(str(e)) + + def list(self, group, *_, **__): + group_secrets = [] + try: + group_secret_models = GroupSecret.objects.filter( + group=group + ).prefetch_related("secret", "group").all() + + for group_secret_model in group_secret_models: + group_secrets.append(GroupSecretSerializer.serialize(group_secret_model)) + + return BaseResponse(result=group_secrets) + except Exception as e: + logger.exception(e.__cause__) + return ServerError(f"{e}") + + def post(self, request, group_id, *_, **__): + try: + secret_id = self.request_body.get("secret_id") + if secret_id == None: + return BadRequest("Property 'secret_id' missing from request") + + # Get the group + group = group_service.get(group_id, request.tenant_id) + if group == None: + return NotFound(f"No group found with id '{group_id}'") + + # Check that the user belongs to the group + if not group_service.user_in_group(request.username, group_id, request.tenant_id): + return Forbidden(message="You do not have access to this group") + + # Fetch the secret + secret = Secret.objects.filter( + id=secret_id, + tenant_id=request.tenant_id, + owner=request.username + ).first() + + if secret == None: + return NotFound(message=f"No secret found with id '{secret_id}'") + + group_secret_id = self.request_body.get("id") + if group_secret_id == None: + group_secret_id = secret.id + + if GroupSecret.objects.filter(group=group, id=group_secret_id).exists(): + return Conflict(message=f"A GroupSecret already exists with id '{group_secret_id}'") + + # Create group secret + group_secret = GroupSecret.objects.create( + id=group_secret_id, + group=group, + secret=secret + ) + + return BaseResponse(result=GroupSecretSerializer.serialize(group_secret)) + + except (IntegrityError, OperationalError, DatabaseError) as e: + return BadRequest(message=e.__cause__) + except Exception as e: + logger.exception(e.__cause__) + return ServerError(f"{e}") + + def put(self, *_, **__): + return MethodNotAllowed("Method 'PUT' not allowed for 'GroupSecret' objects") + + def patch(self, *_, **__): + return MethodNotAllowed("Method 'PATCH' not allowed for 'GroupSecret' objects") + + def delete(self, request, group_id, group_secret_id): + try: + # Get the group + group = group_service.get(group_id, request.tenant_id) + if group == None: + return NotFound(f"No group found with id '{group_id}'") + + # Check that the user belongs to the group + if not group_service.user_in_group(request.username, group_id, request.tenant_id): + return Forbidden(message="You do not have access to this group") + + # Get the group secret + group_secret = GroupSecret.objects.filter( + group=group, + id=group_secret_id + ).prefetch_related("secret").first() + + if group_secret == None: + return NotFound(f"Secret with id '{group_secret_id}' not found in group '{group.id}'") + + # Only group_secret owners can delete the group_secret + if ( + request.username != group_secret.secret.owner + and not group_service.user_in_group(request.username, group_id, request.tenant_id, is_admin=True) + ): + return Forbidden(message="Only GroupSecret owners and group admins can delete a GroupSecret") + + group_secret.delete() + except Exception as e: + logger.exception(e.__cause__) + return ServerError(str(e)) + + return NoContentResponse(message=f"Deleted GroupSecret '{group_secret_id}' from group '{group.id}") + + \ No newline at end of file diff --git a/src/api/src/backend/views/Identities.py b/src/api/src/backend/views/Identities.py index 73db3684..a845ded4 100644 --- a/src/api/src/backend/views/Identities.py +++ b/src/api/src/backend/views/Identities.py @@ -6,7 +6,7 @@ from backend.views.http.responses.errors import BadRequest, NotFound, Forbidden, ServerError from backend.views.http.responses import ResourceURLResponse from backend.views.http.requests import IdentityCreateRequest -from backend.services.SecretService import service as secret_service +from backend.services.CredentialsService import service as credentials_service from utils.cred_validators import validate_by_type from backend.helpers import resource_url_builder @@ -55,7 +55,7 @@ def post(self, request, *_, **__): # Persist the credentials in sk try: - credentials = secret_service.save(request.username, body.credentials) + credentials = credentials_service.save(request.username, body.credentials) except Exception as e: return BadRequest(message=e) @@ -69,7 +69,7 @@ def post(self, request, *_, **__): tenant_id=request.tenant_id ) except IntegrityError as e: - secret_service.delete(credentials.sk_id) + credentials_service.delete(credentials.sk_id) return BadRequest(message=e.__cause__) return ResourceURLResponse( @@ -89,7 +89,7 @@ def delete(self, request, identity_uuid): if identity.owner != request.username: return Forbidden("You do not have access to this identity") - secret_service.delete(identity.credentials.sk_id) + credentials_service.delete(identity.credentials.sk_id) identity.delete() diff --git a/src/api/src/backend/views/Nuke.py b/src/api/src/backend/views/Nuke.py index 25dcae01..30a1fe6b 100644 --- a/src/api/src/backend/views/Nuke.py +++ b/src/api/src/backend/views/Nuke.py @@ -1,6 +1,6 @@ from backend.views.RestrictedAPIView import RestrictedAPIView from backend.views.http.responses.BaseResponse import BaseResponse -from backend.services.SecretService import service as secret_service +from backend.services.CredentialsService import service as credentials_service from backend.models import Task, Identity, Credentials, Context, Destination, Group, GroupUser, Pipeline models = [ @@ -24,6 +24,6 @@ def delete(self, request): credentials_list = Credentials.objects.all() for credentials in credentials_list: - secret_service.delete(credentials.sk_id) + credentials_service.delete(credentials.sk_id) return BaseResponse(message="Boom") \ No newline at end of file diff --git a/src/api/src/backend/views/RestrictedAPIView.py b/src/api/src/backend/views/RestrictedAPIView.py index fb2d4881..5e4fd759 100644 --- a/src/api/src/backend/views/RestrictedAPIView.py +++ b/src/api/src/backend/views/RestrictedAPIView.py @@ -8,7 +8,7 @@ from backend.views.http.responses.errors import MethodNotAllowed, UnsupportedMediaType, BadRequest, Unauthorized, ServerError from backend.views.http.responses import NoContentResponse -from backend.views.http.requests import PreparedRequest +from backend.views.http.PreparedRequest import PreparedRequest from backend.services.TapisAPIGateway import TapisAPIGateway from backend.services.TapisServiceAPIGateway import TapisServiceAPIGateway from backend.utils import one_in diff --git a/src/api/src/backend/views/RunPipeline.py b/src/api/src/backend/views/RunPipeline.py index b79b6c3b..23a09c83 100644 --- a/src/api/src/backend/views/RunPipeline.py +++ b/src/api/src/backend/views/RunPipeline.py @@ -10,11 +10,11 @@ from backend.helpers.PipelineDispatchRequestBuilder import PipelineDispatchRequestBuilder from backend.services.PipelineDispatcher import service as pipeline_dispatcher from backend.services.GroupService import service as group_service -from backend.services.SecretService import service as secret_service +from backend.services.CredentialsService import service as credentials_service from backend.models import Pipeline -request_builder = PipelineDispatchRequestBuilder(secret_service) +request_builder = PipelineDispatchRequestBuilder(credentials_service) class RunPipeline(RestrictedAPIView): def post(self, request, group_id, pipeline_id, *_, **__): diff --git a/src/api/src/backend/views/Secrets.py b/src/api/src/backend/views/Secrets.py new file mode 100644 index 00000000..b13893af --- /dev/null +++ b/src/api/src/backend/views/Secrets.py @@ -0,0 +1,107 @@ +import json, pprint + +from django.db import IntegrityError, OperationalError, DatabaseError + +from backend.models import Secret +from backend.views.RestrictedAPIView import RestrictedAPIView +from backend.views.http.secrets import ReqCreateSecret +from backend.views.http.responses import BaseResponse, NoContentResponse +from backend.views.http.responses.errors import BadRequest, Forbidden, NotFound, MethodNotAllowed, ServerError, Conflict +from backend.views.http.responses import BaseResponse +from backend.services.SecretService import service as secret_service +from backend.serializers import SecretSerializer +from backend.helpers import resource_url_builder +from backend.utils import logger + + +class Secrets(RestrictedAPIView): + def get(self, request, secret_id=None): + try: + if secret_id == None: + return self.list(request.username, request.tenant_id) + + # Get the secret + secret = Secret.objects.filter( + id=secret_id, + tenant_id=request.tenant_id, + owner=request.username, + ).first() + + # Return if BadRequest if no secret found + if secret == None: + return BadRequest(f"Secret with id '{secret_id}' does not exist for user '{request.username}'") + + return BaseResponse(result=SecretSerializer.serialize(secret)) + except Exception as e: + logger.exception(e.__cuase__) + return ServerError(str(e)) + + def list(self, username, tenant_id, *_, **__): + secrets = [] + try: + secret_models = Secret.objects.filter(owner=username, tenant_id=tenant_id) + for secret_model in secret_models: + secrets.append(SecretSerializer.serialize(secret_model)) + + return BaseResponse(result=secrets) + except Exception as e: + logger.exception(e.__cause__) + return ServerError(f"{e}") + + def post(self, request, *_, **__): + # Validate and prepare the create reqeust + prepared_request = self.prepare(ReqCreateSecret) + + # Return the failure view instance if validation failed + if not prepared_request.is_valid: + return prepared_request.failure_view + + # Get the JSON encoded body from the validation result + req_secret = prepared_request.body + + # Check if secret exists + if Secret.objects.filter(id=req_secret.id, owner=request.username, tenant_id=request.tenant_id).exists(): + return Conflict(f"A secret already exists for owner '{request.username}' with the id '{req_secret.id}'") + + # Create secret + try: + secret = secret_service.create(request.tenant_id, request.username, req_secret) + except (IntegrityError, OperationalError, DatabaseError) as e: + logger.exception(e.__cause__) + return ServerError(message=e.__cause__) + except Exception as e: + logger.exception(e.__cause__) + return ServerError(f"{e}") + + return BaseResponse(result=SecretSerializer.serialize(secret)) + + def put(self, *_, **__): + return MethodNotAllowed("Method 'PUT' not allowed for 'Secret' objects") + + def patch(self, *_, **__): + return MethodNotAllowed("Method 'PATCH' not allowed for 'Secret' objects") + + def delete(self, request, secret_id): + # Get the secret + secret = Secret.objects.filter( + owner=request.username, + tenant_id=request.tenant_id, + id=secret_id + ).first() + + if secret == None: + return NotFound(f"Secret with id '{secret_id}' not found for user {request.username}'") + + # Only secret owners can delete the secret + if request.username != secret.owner: + return Forbidden(message="Only secret owners can delete a secret") + + try: + secret_service.delete(secret_id=secret.id, tenant_id=request.tenant_id, owner=secret.owner) + except Exception as e: + logger.exception(e.__cause__) + return ServerError(str(e)) + + return NoContentResponse(message=f"Deleted secret '{secret_id}'") + + \ No newline at end of file diff --git a/src/api/src/backend/views/http/PreparedRequest.py b/src/api/src/backend/views/http/PreparedRequest.py new file mode 100644 index 00000000..6b69d3e2 --- /dev/null +++ b/src/api/src/backend/views/http/PreparedRequest.py @@ -0,0 +1,12 @@ +class PreparedRequest: + def __init__( + self, + is_valid=True, + body=None, + message=None, + failure_view=None + ): + self.is_valid = is_valid + self.body = body + self.message = message + self.failure_view = failure_view \ No newline at end of file diff --git a/src/api/src/backend/views/http/requests.py b/src/api/src/backend/views/http/requests.py index 9e0672af..8d108cd4 100644 --- a/src/api/src/backend/views/http/requests.py +++ b/src/api/src/backend/views/http/requests.py @@ -863,16 +863,3 @@ def backwards_compatibility_transforms(cls, values): # Generic object. NOTE Only used in idempotency key resolution class EmptyObject(BaseModel): pass - -class PreparedRequest: - def __init__( - self, - is_valid=True, - body=None, - message=None, - failure_view=None - ): - self.is_valid = is_valid - self.body = body - self.message = message - self.failure_view = failure_view \ No newline at end of file diff --git a/src/api/src/backend/views/http/secrets.py b/src/api/src/backend/views/http/secrets.py new file mode 100644 index 00000000..995eb7a6 --- /dev/null +++ b/src/api/src/backend/views/http/secrets.py @@ -0,0 +1,9 @@ +from typing import Any, Dict + +from pydantic import BaseModel + + +class ReqCreateSecret(BaseModel): + id: str + description: str = None + data: Dict[str, Any] \ No newline at end of file diff --git a/src/engine/src/owe_python_sdk/schema.py b/src/engine/src/owe_python_sdk/schema.py index 9e0672af..a7cac090 100644 --- a/src/engine/src/owe_python_sdk/schema.py +++ b/src/engine/src/owe_python_sdk/schema.py @@ -862,17 +862,4 @@ def backwards_compatibility_transforms(cls, values): # Generic object. NOTE Only used in idempotency key resolution class EmptyObject(BaseModel): - pass - -class PreparedRequest: - def __init__( - self, - is_valid=True, - body=None, - message=None, - failure_view=None - ): - self.is_valid = is_valid - self.body = body - self.message = message - self.failure_view = failure_view \ No newline at end of file + pass \ No newline at end of file