diff --git a/datameta/alembic/versions/20210625_f6a70402119f.py b/datameta/alembic/versions/20210625_f6a70402119f.py new file mode 100644 index 00000000..6d6ec4ff --- /dev/null +++ b/datameta/alembic/versions/20210625_f6a70402119f.py @@ -0,0 +1,55 @@ +"""added msetreplacements table, moved update-related fields from mset, added user.can_update flag + +Revision ID: f6a70402119f +Revises: 7fdc829db18d +Create Date: 2021-06-25 15:27:37.638601 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'f6a70402119f' +down_revision = '7fdc829db18d' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'msetreplacementevents', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('datetime', sa.DateTime(), nullable=False), + sa.Column('new_metadataset_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacementevents_new_metadataset_id_metadatasets')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacementevents_user_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacementevents')), + sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacementevents_uuid')) + ) + op.add_column('metadatasets', sa.Column('replaced_via_event_id', sa.Integer(), nullable=True)) + op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey') + op.create_foreign_key(op.f('fk_metadatasets_replaced_via_event_id_msetreplacementevents'), 'metadatasets', 'msetreplacementevents', ['replaced_via_event_id'], ['id'], use_alter=True) + op.drop_column('metadatasets', 'deprecated_label') + op.drop_column('metadatasets', 'replaced_by_id') + op.drop_column('metadatasets', 'is_deprecated') + op.add_column('users', sa.Column('can_update', sa.Boolean(create_constraint=False), nullable=True)) + op.execute('UPDATE users SET can_update = site_admin;') + op.alter_column('users', 'can_update', nullable=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('users', 'can_update') + op.add_column('metadatasets', sa.Column('is_deprecated', sa.BOOLEAN(), autoincrement=False, nullable=True)) + op.add_column('metadatasets', sa.Column('replaced_by_id', sa.INTEGER(), autoincrement=False, nullable=True)) + op.add_column('metadatasets', sa.Column('deprecated_label', sa.VARCHAR(), autoincrement=False, nullable=True)) + op.drop_constraint(op.f('fk_metadatasets_replaced_via_event_id_msetreplacementevents'), 'metadatasets', type_='foreignkey') + op.create_foreign_key('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', 'metadatasets', ['replaced_by_id'], ['id']) + op.drop_column('metadatasets', 'replaced_via_event_id') + op.drop_table('msetreplacementevents') + # ### end Alembic commands ### diff --git a/datameta/api/__init__.py b/datameta/api/__init__.py index 0ffef4ad..4120783e 100644 --- a/datameta/api/__init__.py +++ b/datameta/api/__init__.py @@ -62,6 +62,7 @@ def includeme(config: Configurator) -> None: config.add_route("groups_id", base_url + "/groups/{id}") config.add_route("rpc_delete_files", base_url + "/rpc/delete-files") config.add_route("rpc_delete_metadatasets", base_url + "/rpc/delete-metadatasets") + config.add_route("rpc_replace_metadatasets", base_url + "/rpc/replace-metadatasets") config.add_route("rpc_get_file_url", base_url + "/rpc/get-file-url/{id}") config.add_route('register_submit', base_url + "/registrations") config.add_route("register_settings", base_url + "/registrationsettings") diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index e131a8a1..a0f2f6e1 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -13,23 +13,24 @@ # limitations under the License. from dataclasses import dataclass +from datameta.models.db import MsetReplacementEvent from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound, HTTPNoContent from pyramid.view import view_config from pyramid.request import Request from sqlalchemy.orm import joinedload from sqlalchemy import and_ -from typing import Optional, Dict, List +from typing import Optional, Dict, List, Any from ..linting import validate_metadataset_record from .. import security, siteid, resource, validation from ..models import MetaDatum, MetaDataSet, ServiceExecution, Service, MetaDatumRecord, Submission, File from ..security import authz -import datetime -from datetime import timezone +from datetime import timezone, datetime from ..resource import resource_by_id, resource_query_by_id, get_identifier from ..utils import get_record_from_metadataset from . import DataHolderBase from .. import errors from .metadata import get_all_metadata, get_service_metadata, get_metadata_with_access +from datameta.api.submissions import SubmissionResponse @dataclass @@ -40,6 +41,31 @@ class MetaDataSetServiceExecution(DataHolderBase): user_id : dict +@dataclass +class ReplacementMsetResponse(DataHolderBase): + id : dict + record : Dict[str, Optional[str]] + file_ids : Dict[str, Optional[Dict[str, str]]] + user_id : str + replaces : List[Any] + submission_id : Optional[str] = None + service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None + + @classmethod + def from_metadataset(cls, metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): + + replaces = [get_identifier(mset) for event in metadataset.replaces_via_event for mset in event.replaced_metadatasets] # if metadataset.replaces_via_event else None + return cls( + id = get_identifier(metadataset), + record = get_record_from_metadataset(metadataset, metadata_with_access), + file_ids = get_mset_associated_files(metadataset, metadata_with_access), + user_id = get_identifier(metadataset.user), + replaces = replaces, + submission_id = get_identifier(metadataset.submission) if metadataset.submission else None, + service_executions = collect_service_executions(metadata_with_access, metadataset), + ) + + @dataclass class MetaDataSetResponse(DataHolderBase): """MetaDataSetResponse container for OpenApi communication""" @@ -49,6 +75,8 @@ class MetaDataSetResponse(DataHolderBase): user_id : str submission_id : Optional[str] = None service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None + replaces : Optional[List[str]] = None + replaced_by : Optional[str] = None @staticmethod def from_metadataset(metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): @@ -56,21 +84,28 @@ def from_metadataset(metadataset: MetaDataSet, metadata_with_access: Dict[str, M a dictionary of metadata [MetaDatum.name, MetaDatum] that the receiving user has read access to.""" - # Identify file ids associated with this metadataset for metadata with access - file_ids = { mdrec.metadatum.name : resource.get_identifier_or_none(mdrec.file) - for mdrec in metadataset.metadatumrecords - if mdrec.metadatum.isfile and mdrec.metadatum.name in metadata_with_access} # Build the metadataset response return MetaDataSetResponse( id = get_identifier(metadataset), record = get_record_from_metadataset(metadataset, metadata_with_access), - file_ids = file_ids, + file_ids = get_mset_associated_files(metadataset, metadata_with_access), user_id = get_identifier(metadataset.user), submission_id = get_identifier(metadataset.submission) if metadataset.submission else None, - service_executions = collect_service_executions(metadata_with_access, metadataset) + service_executions = collect_service_executions(metadata_with_access, metadataset), + replaces = [get_identifier(mset) for event in metadataset.replaces_via_event for mset in event.replaced_metadatasets] if metadataset.replaces_via_event else None, + replaced_by = get_identifier(metadataset.replaced_via_event.new_metadataset) if metadataset.replaced_via_event else None ) +def get_mset_associated_files(metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): + """ Identify file ids associated with this metadataset for metadata with access """ + return { + mdrec.metadatum.name : resource.get_identifier_or_none(mdrec.file) + for mdrec in metadataset.metadatumrecords + if mdrec.metadatum.isfile and mdrec.metadatum.name in metadata_with_access + } + + def record_to_strings(record: Dict[str, str]): return { k: str(v) if v is not None else None @@ -88,7 +123,7 @@ def render_record_values(metadata: Dict[str, MetaDatum], record: dict) -> dict: continue elif record_rendered[field] and metadata[field].datetimefmt: # if MetaDatum is a datetime field, render the value in isoformat - record_rendered[field] = datetime.datetime.strptime( + record_rendered[field] = datetime.strptime( record_rendered[field], metadata[field].datetimefmt ).isoformat() @@ -136,17 +171,117 @@ def delete_metadatasets(request: Request) -> HTTPNoContent: return HTTPNoContent() -@view_config( - route_name="metadatasets", - renderer='json', - request_method="POST", - openapi=True -) -def post(request: Request) -> MetaDataSetResponse: - """Create new metadataset""" - auth_user = security.revalidate_user(request) - db = request.dbsession +def initialize_service_metadata(db, mset_id): + """ + For the specified metadataset, set initial metadatumrecords + for all service-provided metadata fields. + """ + service_metadata = get_service_metadata(db) + for s_mdatum in service_metadata.values(): + mdatum_rec = MetaDatumRecord( + metadatum_id = s_mdatum.id, + metadataset_id = mset_id, + file_id = None, + value = None + ) + db.add(mdatum_rec) + + +def add_metadata_from_request(db, record, metadata, mset_id): + """ + For the specified metadataset, set non-service metadatum records + as supplied in the request body. + """ + for name, value in record.items(): + mdatum_rec = MetaDatumRecord( + metadatum_id = metadata[name].id, + metadataset_id = mset_id, + file_id = None, + value = value + ) + db.add(mdatum_rec) + + +def validate_associated_files(db, file_ids, auth_user): + """ + Check that a list of files is accessible by an authenticated user. + """ + + # Collect files, drop duplicates + db_files = { file_id : resource.resource_query_by_id(db, File, file_id).options(joinedload(File.metadatumrecord)).one_or_none() for file_id in set(file_ids) } + + # Validate submission access to the specified files + validation.validate_submission_access(db, db_files, {}, auth_user) + + return db_files + + +def link_files(db, mdata_set, db_files, ignore_submitted=False): + """ + Perform association checks between a specified metadataset and list of files + and, upon success, link the files to the metadataset. + """ + + # Validate the associations between files and records + fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { mdata_set.site_id : mdata_set }, ignore_submitted_metadatasets=ignore_submitted) + + # If there were any validation errors, return 400 + if val_errors: + entities, fields, messages = zip(*val_errors) + raise errors.get_validation_error(messages=list(messages), fields=fields, entities=list(entities)) + + # Given that validation hasn't failed, we know that file names are unique. Flatten the dict. + fnames = { k : v[0] for k, v in fnames.items() } + + # Associate the files with the metadata + for fname, mdatrec in ref_fnames.items(): + mdatrec.file = fnames[fname] + db.add(mdatrec) + + +def execute_mset_replacement(db, new_mset_id, replaced_msets, user_id): + """ + Evaluate if a replacement of a list of metadatasets is possible + - validate target metadataset ids + - ensure target metadatasets have not already been replaced + Upon success, generate a replacement event and mark the target datasets as replaced. + """ + + msets = [ + (mset_id, resource_by_id(db, MetaDataSet, mset_id)) + for mset_id in replaced_msets + ] + + missing_msets = [ f"Invalid metadataset id: {mset_id}" for mset_id, target_mset in msets if target_mset is None] + + if missing_msets: + raise errors.get_validation_error(messages=missing_msets) + + already_replaced = [ + (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset).get('site')}", target_mset) + for mset_id, target_mset in msets + if target_mset.replaced_via_event_id is not None + ] + + if already_replaced: + messages, entities = zip(*already_replaced) + raise errors.get_validation_error(messages=list(messages), entities=list(entities)) + + mset_repl_evt = MsetReplacementEvent( + user_id = user_id, + datetime = datetime.utcnow(), + new_metadataset_id = new_mset_id + ) + db.add(mset_repl_evt) + db.flush() + + for _, target_mset in msets: + target_mset.replaced_via_event_id = mset_repl_evt.id + + return msets + +def prepare_record(db, request): # Obtain string converted version of the record record = record_to_strings(request.openapi_validated.body["record"]) @@ -160,35 +295,91 @@ def post(request: Request) -> MetaDataSetResponse: # Render records according to MetaDatum constraints. record = render_record_values(metadata, record) + return record, metadata + + +@view_config( + route_name="rpc_replace_metadatasets", + renderer="json", + request_method="POST", + openapi=True +) +def replace_metadatasets(request: Request) -> SubmissionResponse: + auth_user = security.revalidate_user(request) + db = request.dbsession + + record, metadata = prepare_record(db, request) + # construct new MetaDataSet: mdata_set = MetaDataSet( site_id = siteid.generate(request, MetaDataSet), user_id = auth_user.id, submission_id = None ) + db.add(mdata_set) db.flush() - # Add NULL values for service metadata - service_metadata = get_service_metadata(db) - for s_mdatum in service_metadata.values(): - mdatum_rec = MetaDatumRecord( - metadatum_id = s_mdatum.id, - metadataset_id = mdata_set.id, - file_id = None, - value = None - ) - db.add(mdatum_rec) + replaces = request.openapi_validated.body["replaces"] + replaces_label = request.openapi_validated.body["replacesLabel"] - # Add the non-service metadata as specified in the request body - for name, value in record.items(): - mdatum_rec = MetaDatumRecord( - metadatum_id = metadata[name].id, - metadataset_id = mdata_set.id, - file_id = None, - value = value - ) - db.add(mdatum_rec) + execute_mset_replacement(db, mdata_set.id, replaces, auth_user.id) + + initialize_service_metadata(db, mdata_set.id) + + add_metadata_from_request(db, record, metadata, mdata_set.id) + + db_files = validate_associated_files(db, set(request.openapi_validated.body['fileIds']), auth_user) + + # Validate the provided records + validate_metadataset_record(metadata, record, return_err_message=False, rendered=True) + + link_files(db, mdata_set, db_files, ignore_submitted=False) + + mdata_set = resource.resource_query_by_id(db, MetaDataSet, mdata_set.site_id).options(joinedload(MetaDataSet.metadatumrecords).joinedload(MetaDatumRecord.metadatum)).one_or_none() + + # Add a submission + submission = Submission( + site_id = siteid.generate(request, Submission), + label = replaces_label, + date = datetime.utcnow(), + metadatasets = [mdata_set], + group_id = auth_user.group.id + ) + db.add(submission) + + # Check which metadata of this metadataset the user is allowed to view + metadata_with_access = get_metadata_with_access(db, auth_user) + + return ReplacementMsetResponse.from_metadataset(mdata_set, metadata_with_access) + + +@view_config( + route_name="metadatasets", + renderer='json', + request_method="POST", + openapi=True +) +def post(request: Request) -> MetaDataSetResponse: + """Create new metadataset""" + auth_user = security.revalidate_user(request) + db = request.dbsession + + record, metadata = prepare_record(db, request) + + # construct new MetaDataSet: + mdata_set = MetaDataSet( + site_id = siteid.generate(request, MetaDataSet), + user_id = auth_user.id, + submission_id = None + ) + + db.add(mdata_set) + db.flush() + + initialize_service_metadata(db, mdata_set.id) + + add_metadata_from_request(db, record, metadata, mdata_set.id) return MetaDataSetResponse( id = get_identifier(mdata_set), @@ -279,7 +470,7 @@ def get_metadatasets(request: Request) -> List[MetaDataSetResponse]: query = query.filter(Submission.date < submitted_before.astimezone(timezone.utc)) if awaiting_service is not None: if awaiting_service not in readable_services_by_id: - raise errors.get_validation_error(messages=['Invalid service ID specified'], fields=['awaitingServices']) + raise errors.get_validation_error(messages=['Invalid service ID specified'], fields=['awaitingService']) query = query.outerjoin(ServiceExecution, and_( MetaDataSet.id == ServiceExecution.metadataset_id, ServiceExecution.service_id == readable_services_by_id[awaiting_service].id @@ -293,9 +484,9 @@ def get_metadatasets(request: Request) -> List[MetaDataSetResponse]: raise HTTPNotFound() return [ - MetaDataSetResponse.from_metadataset(mdata_set, metadata_with_access) - for mdata_set in mdata_sets - ] + MetaDataSetResponse.from_metadataset(mdata_set, metadata_with_access) + for mdata_set in mdata_sets + ] @view_config( @@ -406,12 +597,7 @@ def set_metadata_via_service(request: Request) -> MetaDataSetResponse: messages, fields = zip(*val_errors) raise errors.get_validation_error(messages, fields) - # Collect files, drop duplicates - file_ids = set(request.openapi_validated.body['fileIds']) - db_files = { file_id : resource.resource_query_by_id(db, File, file_id).options(joinedload(File.metadatumrecord)).one_or_none() for file_id in file_ids } - - # Validate submission access to the specified files - validation.validate_submission_access(db, db_files, {}, auth_user) + db_files = validate_associated_files(db, set(request.openapi_validated.body['fileIds']), auth_user) # Validate the provided records validate_metadataset_record(service_metadata, records, return_err_message=False, rendered=False) @@ -423,29 +609,15 @@ def set_metadata_via_service(request: Request) -> MetaDataSetResponse: db_rec.value = record_value db.add(db_rec) - # Validate the associations between files and records - fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { metadataset.site_id : metadataset }, ignore_submitted_metadatasets=True) - - # If there were any validation errors, return 400 - if val_errors: - entities, fields, messages = zip(*val_errors) - raise errors.get_validation_error(messages=messages, fields=fields, entities=entities) - - # Given that validation hasn't failed, we know that file names are unique. Flatten the dict. - fnames = { k : v[0] for k, v in fnames.items() } - - # Associate the files with the metadata - for fname, mdatrec in ref_fnames.items(): - mdatrec.file = fnames[fname] - db.add(mdatrec) + link_files(db, metadataset, db_files, ignore_submitted=True) # Create a service execution sexec = ServiceExecution( - service = service, - metadataset = metadataset, - user = auth_user, - datetime = datetime.datetime.utcnow() - ) + service = service, + metadataset = metadataset, + user = auth_user, + datetime = datetime.utcnow() + ) db.add(sexec) diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index 043d4182..870bdc29 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -15,7 +15,7 @@ openapi: 3.0.0 info: description: DataMeta - version: 1.4.0 + version: 1.5.0 title: DataMeta servers: @@ -104,7 +104,7 @@ paths: [Attention this endpoint is not RESTful, the result should not be cached.] tags: - Remote Procedure Calls - operationId: GetUserInformation + operationId: GetOwnUserInfo responses: "200": description: OK @@ -189,6 +189,43 @@ paths: '500': description: Internal Server Error + /rpc/replace-metadatasets: + post: + summary: Replace a range of metadatasets. + description: >- + Replace metadatasets. + tags: + - Remote Procedure Calls + operationId: ReplaceMetaDataSets + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ReplacementMetaDataSet" + description: >- + Provide all properties for one MetaDataSet and a list of MetaDataSets to be replaced. + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ReplacementMsetResponse" + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not found + '400': + description: Validation Error + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorModel" + '500': + description: Internal Server Error + /rpc/get-file-url/{id}: get: @@ -347,7 +384,7 @@ paths: Get information about a user. tags: - Authentication and Users - operationId: UserInformationRequest + operationId: GetUserInfo parameters: - name: id in: path @@ -563,7 +600,7 @@ paths: format: date-time - name: awaitingService in: query - description: Identifier for a service. Restricts the result to metadatsets for which the specified service has not been executed yet. + description: Identifier for a service. Restricts the result to metadatasets for which the specified service has not been executed yet. schema: type: string responses: @@ -998,7 +1035,7 @@ paths: Get information about a group. tags: - Groups - operationId: GroupInformationRequest + operationId: GetGroupInfo parameters: - name: id in: path @@ -1448,6 +1485,31 @@ components: - record additionalProperties: false + ReplacementMetaDataSet: + type: object + properties: + record: + type: object + additionalProperties: true + # a free-form object, + # any property is allowed + replaces: + type: array + items: + type: string + replacesLabel: + type: string + fileIds: + type: array + items: + type: string + required: + - record + - replaces + - replacesLabel + - fileIds + additionalProperties: false + ServiceExecution: type: object properties: @@ -1467,6 +1529,42 @@ components: - fileIds additionalProperties: false + ReplacementMsetResponse: + type: object + properties: + record: + type: object + additionalProperties: true + # a free-form object, any property is allowed + replaces: + type: array + items: + type: object + fileIds: + type: object + additionalProperties: true + # a free-form object mapping the field names to file IDs + serviceExecutions: + type: object + # nullable as a whole for responses to users who cannot see service executions + nullable: true + additionalProperties: + # Maps record field names to service executions. The individual + # fields are also nullable for services that haven't been executed + # yet. + $ref: "#/components/schemas/MetaDataSetServiceExecution" + id: + $ref: "#/components/schemas/Identifier" + submissionId: + $ref: "#/components/schemas/NullableIdentifier" + userId: + $ref: "#/components/schemas/Identifier" + required: + - record + - replaces + - fileIds + additionalProperties: false + MetaDataSetResponse: type: object properties: @@ -1493,6 +1591,14 @@ components: $ref: "#/components/schemas/NullableIdentifier" userId: $ref: "#/components/schemas/Identifier" + replaces: + type: array + items: + type: object + nullable: true + replacedBy: + type: object + nullable: true required: - record additionalProperties: false @@ -1733,6 +1839,8 @@ components: type: boolean siteRead: type: boolean + canUpdate: + type: boolean email: type: string group: @@ -1748,6 +1856,7 @@ components: - groupAdmin - siteAdmin - siteRead + - canUpdate - email - group additionalProperties: false @@ -1773,6 +1882,9 @@ components: nullable: true groupId: $ref: "#/components/schemas/Identifier" + canUpdate: + type: boolean + nullable: true required: - id - name @@ -1794,6 +1906,8 @@ components: type: boolean enabled: type: boolean + canUpdate: + type: boolean MetaDataResponse: type: object diff --git a/datameta/api/ui/admin.py b/datameta/api/ui/admin.py index 18e83edd..9cb005ea 100644 --- a/datameta/api/ui/admin.py +++ b/datameta/api/ui/admin.py @@ -86,6 +86,7 @@ def v_admin_put_request(request): enabled = True, site_admin = False, site_read = False, + can_update = False, group_admin = newuser_make_admin, pwhash = '!') try: diff --git a/datameta/api/ui/view.py b/datameta/api/ui/view.py index 6812edb4..18ba3fd8 100644 --- a/datameta/api/ui/view.py +++ b/datameta/api/ui/view.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datameta.models.db import MsetReplacementEvent from sqlalchemy.orm import joinedload, aliased from sqlalchemy import func, and_, or_, desc, asc @@ -182,6 +183,7 @@ def post(request: Request): .options(joinedload(MetaDataSet.user))\ .options(joinedload(MetaDataSet.service_executions).joinedload(ServiceExecution.user))\ .options(joinedload(MetaDataSet.service_executions).joinedload(ServiceExecution.service).joinedload(Service.target_metadata))\ + .options(joinedload(MetaDataSet.replaced_via_event).joinedload(MsetReplacementEvent.new_metadataset))\ .offset(start)\ .limit(length)\ @@ -224,6 +226,7 @@ def post(request: Request): submission_datetime = mdata_set.submission.date.isoformat(), submission_label = mdata_set.submission.label, service_executions = service_executions, + replaced_by = get_identifier(mdata_set.replaced_via_event.new_metadataset) if mdata_set.replaced_via_event else None ) for (mdata_set, _), service_executions in zip(mdata_sets, service_executions_all) ] diff --git a/datameta/api/users.py b/datameta/api/users.py index 103bf752..e3c26e1b 100644 --- a/datameta/api/users.py +++ b/datameta/api/users.py @@ -36,6 +36,7 @@ class UserUpdateRequest(DataHolderBase): siteAdmin: bool enabled: bool siteRead: bool + canUpdate: bool @dataclass @@ -48,6 +49,11 @@ class UserResponseElement(DataHolderBase): site_admin: Optional[bool] = None site_read: Optional[bool] = None email: Optional[str] = None + can_update: Optional[bool] = None + + @staticmethod + def get_restricted_fields(): + return ["group_admin", "site_admin", "site_read", "email", "can_update"] @classmethod def from_user(cls, target_user, requesting_user): @@ -55,10 +61,8 @@ def from_user(cls, target_user, requesting_user): if authz.view_restricted_user_info(requesting_user, target_user): restricted_fields.update({ - "group_admin": target_user.group_admin, - "site_admin": target_user.site_admin, - "site_read": target_user.site_read, - "email": target_user.email + field: getattr(target_user, field) + for field in cls.get_restricted_fields() }) return cls(id=get_identifier(target_user), name=target_user.fullname, group_id=get_identifier(target_user.group), **restricted_fields) @@ -72,6 +76,7 @@ class WhoamiResponseElement(DataHolderBase): group_admin: bool site_admin: bool site_read: bool + can_update: bool email: str group: dict @@ -92,6 +97,7 @@ def get_whoami(request: Request) -> WhoamiResponseElement: group_admin = auth_user.group_admin, site_admin = auth_user.site_admin, site_read = auth_user.site_read, + can_update = auth_user.can_update, email = auth_user.email, group = {"id": get_identifier(auth_user.group), "name": auth_user.group.name} ) @@ -138,6 +144,7 @@ def put(request: Request): site_admin = request.openapi_validated.body.get("siteAdmin") enabled = request.openapi_validated.body.get("enabled") site_read = request.openapi_validated.body.get("siteRead") + can_update = request.openapi_validated.body.get("canUpdate") db = request.dbsession @@ -176,6 +183,10 @@ def put(request: Request): if name is not None and not authz.update_user_name(auth_user, target_user): raise HTTPForbidden() + # The user has to be site admin to change update privileges for a user + if can_update is not None and not authz.update_user_can_update(auth_user): + raise HTTPForbidden() + # Now, make the corresponding changes if group_id is not None: new_group = resource_by_id(db, Group, group_id) @@ -192,5 +203,7 @@ def put(request: Request): target_user.enabled = enabled if name is not None: target_user.fullname = name + if can_update is not None: + target_user.can_update = can_update return HTTPNoContent() diff --git a/datameta/models/db.py b/datameta/models/db.py index 935f9d18..a97ab656 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -28,7 +28,7 @@ Table ) from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import relationship from .meta import Base @@ -82,6 +82,8 @@ class User(Base): site_admin = Column(Boolean(create_constraint=False), nullable=False) group_admin = Column(Boolean(create_constraint=False), nullable=False) site_read = Column(Boolean(create_constraint=False), nullable=False) + can_update = Column(Boolean(create_constraint=False), nullable=False) + # Relationships group = relationship('Group', back_populates='user') metadatasets = relationship('MetaDataSet', back_populates='user') @@ -90,6 +92,7 @@ class User(Base): apikeys = relationship('ApiKey', back_populates='user') services = relationship('Service', secondary=user_service_table, back_populates='users') service_executions = relationship('ServiceExecution', back_populates='user') + mset_replacements = relationship('MsetReplacementEvent', back_populates='user') class ApiKey(Base): @@ -209,20 +212,36 @@ class MetaDatumRecord(Base): class MetaDataSet(Base): """A MetaDataSet represents all metadata associated with *one* record""" __tablename__ = 'metadatasets' - id = Column(Integer, primary_key=True) - site_id = Column(String(50), unique=True, nullable=False, index=True) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) - user_id = Column(Integer, ForeignKey('users.id'), nullable=False) - submission_id = Column(Integer, ForeignKey('submissions.id'), nullable=True) - is_deprecated = Column(Boolean, default=False) - deprecated_label = Column(String, nullable=True) - replaced_by_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=True) + id = Column(Integer, primary_key=True) + site_id = Column(String(50), unique=True, nullable=False, index=True) + uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) + user_id = Column(Integer, ForeignKey('users.id'), nullable=False) + submission_id = Column(Integer, ForeignKey('submissions.id'), nullable=True) + replaced_via_event_id = Column(Integer, ForeignKey('msetreplacementevents.id', use_alter=True), nullable=True) + # Relationships user = relationship('User', back_populates='metadatasets') submission = relationship('Submission', back_populates='metadatasets') metadatumrecords = relationship('MetaDatumRecord', back_populates='metadataset') - replaces = relationship('MetaDataSet', backref=backref('replaced_by', remote_side=[id])) - service_executions = relationship('ServiceExecution', back_populates = 'metadataset') + service_executions = relationship('ServiceExecution', back_populates ='metadataset') + + replaced_via_event = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.replaced_via_event_id==MsetReplacementEvent.id', back_populates='new_metadataset') + replaces_via_event = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.id==MsetReplacementEvent.new_metadataset_id', back_populates='replaced_metadatasets') + + +class MsetReplacementEvent(Base): + """ Stores information about an mset replacement event """ + __tablename__ = 'msetreplacementevents' + id = Column(Integer, primary_key=True) + uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) + user_id = Column(Integer, ForeignKey('users.id'), nullable=False) + datetime = Column(DateTime, nullable=False) + new_metadataset_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=False) + + # Relationships + user = relationship('User', back_populates='mset_replacements') + new_metadataset = relationship("MetaDataSet", primaryjoin='MetaDataSet.id==MsetReplacementEvent.new_metadataset_id', back_populates='replaces_via_event') + replaced_metadatasets = relationship("MetaDataSet", primaryjoin='MetaDataSet.replaced_via_event_id==MsetReplacementEvent.id', back_populates='replaced_via_event') class ApplicationSetting(Base): @@ -244,7 +263,7 @@ class Service(Base): site_id = Column(String(50), unique=True, nullable=False, index=True) name = Column(Text, nullable=True, unique=True) # Relationships - users = relationship('User', secondary=user_service_table, back_populates='services') + users = relationship('User', secondary=user_service_table, back_populates='services') # unfortunately, 'metadata' is a reserved keyword for sqlalchemy classes service_executions = relationship('ServiceExecution', back_populates = 'service') target_metadata = relationship('MetaDatum', back_populates = 'service') diff --git a/datameta/scripts/initialize_db.py b/datameta/scripts/initialize_db.py index a1a16ca4..c2b3df3a 100644 --- a/datameta/scripts/initialize_db.py +++ b/datameta/scripts/initialize_db.py @@ -68,7 +68,8 @@ def create_initial_user(request, email, fullname, password, groupname): group=init_group, group_admin=True, site_admin=True, - site_read=True + site_read=True, + can_update=True ) db.add(root) diff --git a/datameta/security/authz.py b/datameta/security/authz.py index b357dfab..d64fecaa 100644 --- a/datameta/security/authz.py +++ b/datameta/security/authz.py @@ -176,6 +176,10 @@ def update_user_name(user, target_user): )) +def update_user_can_update(user): + return user.site_admin + + def view_restricted_user_info(user, target_user): return has_group_rights(user, target_user.group) diff --git a/datameta/static/js/view.js b/datameta/static/js/view.js index 6f69fe24..277b387d 100644 --- a/datameta/static/js/view.js +++ b/datameta/static/js/view.js @@ -29,19 +29,26 @@ DataMeta.view.buildColumns = function(mdata) { // Special case NULL and service metadatum with access but not run yet if (mdataset.serviceExecutions !== null && mdatum.name in mdataset.serviceExecutions - && mdataset.serviceExecutions[mdatum.name]===null) + && mdataset.serviceExecutions[mdatum.name] === null) return ' pending'; // Special case NULL if (mdataset.record[mdatum.name] === null) return 'empty'; - // Speical case file - if (mdataset.fileIds[mdatum.name]) return ' '+mdataset.record[mdatum.name]+''; + // Special case file + if (mdataset.fileIds[mdatum.name]) { + var record_str = mdataset.replacedBy === null ? mdataset.record[mdatum.name] : '' + mdataset.record[mdatum.name] + ''; + return ' ' + record_str + ''; + } // All other cases - return mdataset.record[mdatum.name]; + return mdataset.replacedBy === null ? mdataset.record[mdatum.name] : '' + mdataset.record[mdatum.name] + ''; } }; }); } +function format_cell(data, is_replaced) { + return is_replaced ? '
' + data + '
' : '
' + '
' + data + '
' +} + DataMeta.view.initTable = function() { // Fetch metadata information from the API @@ -57,23 +64,44 @@ DataMeta.view.initTable = function() { // Extract field names from the metadata information var mdata = json; + + //var cell_style_start = data.replacedBy === null ? '
' : '
'; + //var cell_style_end = data.replacedBy === null ? '
' : '
'; + var cell_style_start = ''; + var cell_style_end = ''; + + + var columns = [ { title: ' Submission', data: null, className: "id_col", render: function(data) { - var label = data.submissionLabel ? data.submissionLabel : 'empty'; - return '
' + label + '
' + data.submissionId.site + '
' + var label = data.submissionLabel ? data.submissionLabel : 'empty'; //'empty'; + return '
' + format_cell(label, data.replacedBy !== null) + '
' + data.submissionId.site + '
'; }}, { title: ' Submission Time', data: null, className: "id_col", render: function(data) { var d = moment(new Date(data.submissionDatetime)); - return '
' + d.format('YYYY-MM-DD HH:mm:ss') + - '
' + 'GMT' + d.format('ZZ') + '
' + return '
' + format_cell(d.format('YYYY-MM-DD HH:mm:ss'), data.replacedBy !== null) + '
' + 'GMT' + d.format('ZZ') + '
'; }}, { title: ' User', data: null, className: "id_col", render: data => - '
'+data.userName+'
'+data.userId.site+'
' + '
' + format_cell(data.userName, data.replacedBy !== null) + '
' + data.userId.site + '
' }, { title: ' Group', data: null, className: "id_col", render: data => - '
'+data.groupName+'
'+data.groupId.site+'
' + '
' + format_cell(data.groupName, data.replacedBy !== null) + '
' + data.groupId.site + '
' }, - { title: ' ID', data: "id.site", className: "id_col", render: data => '' + data + ''} + { title: ' ID', data: null, className: "id_col", render: function(data) { + + var return_str = format_cell(data.id.site, data.replacedBy !== null); + return '
' + return_str + (data.replacedBy === null ? '' : '
replaced by ' + data.replacedBy.site + '
') + '
'; + if (data.replacedBy !== null) { + return_str += '
' + data.id.site + '
'; + return_str += '
replaced by ' + data.replacedBy.site + '
'; + } else { + return_str += '
' + data.id.site + '
'; + } + + return return_str + '' + + } + }, ].concat(DataMeta.view.buildColumns(mdata)) // Build table based on field names diff --git a/tests/integration/fixtures/files/group_x_file_1.txt b/tests/integration/fixtures/files/group_x_file_1.txt new file mode 100644 index 00000000..f226f2ae --- /dev/null +++ b/tests/integration/fixtures/files/group_x_file_1.txt @@ -0,0 +1 @@ +user_a_file_1 diff --git a/tests/integration/fixtures/files/group_x_file_2.txt b/tests/integration/fixtures/files/group_x_file_2.txt new file mode 100644 index 00000000..b2a72147 --- /dev/null +++ b/tests/integration/fixtures/files/group_x_file_2.txt @@ -0,0 +1 @@ +user_a_file_2 diff --git a/tests/integration/fixtures/files/group_x_file_3.txt b/tests/integration/fixtures/files/group_x_file_3.txt new file mode 100644 index 00000000..f226f2ae --- /dev/null +++ b/tests/integration/fixtures/files/group_x_file_3.txt @@ -0,0 +1 @@ +user_a_file_1 diff --git a/tests/integration/fixtures/files/group_x_file_4.txt b/tests/integration/fixtures/files/group_x_file_4.txt new file mode 100644 index 00000000..b2a72147 --- /dev/null +++ b/tests/integration/fixtures/files/group_x_file_4.txt @@ -0,0 +1 @@ +user_a_file_2 diff --git a/tests/integration/fixtures/files_independent.yaml b/tests/integration/fixtures/files_independent.yaml index 3f669ae9..c9c64331 100644 --- a/tests/integration/fixtures/files_independent.yaml +++ b/tests/integration/fixtures/files_independent.yaml @@ -35,3 +35,51 @@ user_a_file_2: user: fixtureset: users name: user_a + +group_x_file_1: + class: File + attributes: + site_id: group_x_file_1 + name: group_x_file_1.txt + content_uploaded: True + checksum: bbe230ca2f18d04af27f03f7f13631af + references: + user: + fixtureset: users + name: group_x_admin + +group_x_file_2: + class: File + attributes: + site_id: group_x_file_2 + name: group_x_file_2.txt + content_uploaded: True + checksum: f7a3deca77966ae714b0e9d73402659b + references: + user: + fixtureset: users + name: group_x_admin + +group_x_file_3: + class: File + attributes: + site_id: group_x_file_3 + name: group_x_file_3.txt + content_uploaded: True + checksum: bbe230ca2f18d04af27f03f7f13631af + references: + user: + fixtureset: users + name: group_x_admin + +group_x_file_4: + class: File + attributes: + site_id: group_x_file_4 + name: group_x_file_4.txt + content_uploaded: True + checksum: f7a3deca77966ae714b0e9d73402659b + references: + user: + fixtureset: users + name: group_x_admin diff --git a/tests/integration/fixtures/holders.py b/tests/integration/fixtures/holders.py index e1464711..9e8a3a30 100644 --- a/tests/integration/fixtures/holders.py +++ b/tests/integration/fixtures/holders.py @@ -42,6 +42,7 @@ class UserFixture(Entity): site_admin : bool site_read : bool enabled : bool + can_update : bool @dataclass diff --git a/tests/integration/fixtures/users.yaml b/tests/integration/fixtures/users.yaml index 51e89508..a41f63a3 100644 --- a/tests/integration/fixtures/users.yaml +++ b/tests/integration/fixtures/users.yaml @@ -25,6 +25,7 @@ admin: group_admin: true site_admin: true site_read: true + can_update: true enabled: true references: group: @@ -44,6 +45,7 @@ group_x_admin: group_admin: true site_admin: false site_read: false + can_update: true enabled: true references: group: @@ -63,6 +65,7 @@ user_a: group_admin: false site_admin: false site_read: false + can_update: false enabled: true references: group: @@ -82,6 +85,7 @@ user_b: group_admin: false site_admin: false site_read: false + can_update: false enabled: true references: group: @@ -101,6 +105,7 @@ group_y_admin: group_admin: true site_admin: false site_read: false + can_update: true enabled: true references: group: @@ -120,6 +125,7 @@ user_c: group_admin: false site_admin: false site_read: false + can_update: false enabled: true references: group: @@ -139,6 +145,7 @@ user_site_read: group_admin: false site_admin: false site_read: true + can_update: false enabled: true references: group: @@ -158,6 +165,7 @@ service_user_0: group_admin: false site_admin: false site_read: true + can_update: false enabled: true references: group: diff --git a/tests/integration/test_bulkdelete.py b/tests/integration/test_bulkdelete.py index 67837c84..dfb49cdc 100644 --- a/tests/integration/test_bulkdelete.py +++ b/tests/integration/test_bulkdelete.py @@ -18,9 +18,8 @@ def test_file_deletion(self): user = self.fixture_manager.get_fixture('users', 'user_a') auth_headers = self.apikey_auth(user) - files = self.fixture_manager.get_fixtureset('files_independent') - file_ids = [ file.site_id for file in files.values() ] + file_ids = ["user_a_file_1", "user_a_file_2"] self.testapp.post_json( f"{base_url}/rpc/delete-files", diff --git a/tests/integration/test_information_requests.py b/tests/integration/test_information_requests.py index ebcc9f74..238a918d 100644 --- a/tests/integration/test_information_requests.py +++ b/tests/integration/test_information_requests.py @@ -18,6 +18,7 @@ from . import BaseIntegrationTest from datameta.api import base_url +from datameta.api.users import UserResponseElement class TestUserInformationRequest(BaseIntegrationTest): @@ -46,6 +47,9 @@ def do_request(self, user_uuid, **params): ("site_admin_query_wrong_id" , "admin" , "flopsy" , 404), ]) def test_user_query(self, testname: str, executing_user: str, target_user: str, expected_response: int): + def snake_to_camel(s): + return ''.join(x.capitalize() if i else x for i, x in enumerate(s.split('_'))) + user = self.fixture_manager.get_fixture('users', executing_user) if testname == "site_admin_query_wrong_id": @@ -63,7 +67,7 @@ def test_user_query(self, testname: str, executing_user: str, target_user: str, successful_request = expected_response == 200 privileged_request = "admin" in executing_user - restricted_fields = ["groupAdmin", "siteAdmin", "siteRead", "email"] + restricted_fields = map(snake_to_camel, UserResponseElement.get_restricted_fields()) assert any(( not successful_request, diff --git a/tests/integration/test_mset_replacement.py b/tests/integration/test_mset_replacement.py new file mode 100644 index 00000000..c2e92154 --- /dev/null +++ b/tests/integration/test_mset_replacement.py @@ -0,0 +1,97 @@ +# Copyright 2021 Universität Tübingen, DKFZ and EMBL for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from parameterized import parameterized + +from datameta.api import base_url + +from . import BaseIntegrationTest + + +class MsetReplacementTest(BaseIntegrationTest): + def setUp(self): + super().setUp() + self.fixture_manager.load_fixtureset('groups') + self.fixture_manager.load_fixtureset('users') + self.fixture_manager.load_fixtureset('apikeys') + self.fixture_manager.load_fixtureset('services') + self.fixture_manager.load_fixtureset('metadata') + self.fixture_manager.load_fixtureset('files_msets') + self.fixture_manager.load_fixtureset('files_independent') + self.fixture_manager.load_fixtureset('submissions') + self.fixture_manager.load_fixtureset('metadatasets') + self.fixture_manager.load_fixtureset('serviceexecutions') + self.fixture_manager.copy_files_to_storage() + self.fixture_manager.populate_metadatasets() + + @parameterized.expand([ + ("success", "group_x_admin", "mset_a", 200), + ("insufficient_access_rights", "user_a", "mset_a", 400), + ("insufficient_access_rights_other_group", "user_b", "mset_a", 400), + ("target_mset_does_not_exist", "group_x_admin", "blob", 400), + ("target_mset_already_replaced", "group_x_admin", "mset_a", 400) + ]) + def test_mset_replacement(self, testname: str, executing_user: str, replaced_mset: str, expected_status: int): + user = self.fixture_manager.get_fixture("users", executing_user) if executing_user else None + auth_headers = self.apikey_auth(user) if user else {} + + request_body = { + "record": { + "Date": "2021-07-01", + "ZIP Code": "108", + "ID": "XC13", + "FileR2": "group_x_file_1.txt", + "FileR1": "group_x_file_2.txt" + }, + "replaces": [ + replaced_mset + ], + "replacesLabel": "because i can", + "fileIds": [ + "group_x_file_1", "group_x_file_2" + ] + } + + self.testapp.post_json( + url = f"{base_url}/rpc/replace-metadatasets", + headers = auth_headers, + status = expected_status if testname != "target_mset_already_replaced" else 200, + params = request_body + ) + + if testname == "target_mset_already_replaced": + + request_body = { + "record": { + "Date": "2021-07-01", + "ZIP Code": "108", + "ID": "blargh", + "FileR2": "group_x_file_3.txt", + "FileR1": "group_x_file_4.txt" + }, + "replaces": [ + replaced_mset + ], + "replacesLabel": "because i can", + "fileIds": [ + "group_x_file_3", "group_x_file_4" + ] + } + + self.testapp.post_json( + url = f"{base_url}/rpc/replace-metadatasets", + headers = auth_headers, + status = expected_status, + params = request_body + )