From 827db60181ecb6c54bd43947330f01292c2d317b Mon Sep 17 00:00:00 2001 From: Luigi Pellecchia Date: Fri, 9 Aug 2024 22:13:06 +0200 Subject: [PATCH 01/22] Support for mapping a Document Work Item to the reference specification Signed-off-by: Luigi Pellecchia --- Dockerfile-api | 7 +- api/api.py | 564 ++++++++++++++++-- app/src/app/Constants/constants.tsx | 128 +++- app/src/app/Mapping/Form/CommentForm.tsx | 5 + app/src/app/Mapping/Form/DocumentForm.tsx | 520 ++++++++++++++++ app/src/app/Mapping/MappingListingTable.tsx | 203 ++++++- app/src/app/Mapping/MappingPageSection.tsx | 82 ++- .../app/Mapping/Menu/DocumentMenuKebab.tsx | 105 ++++ .../Mapping/Menu/MappingSectionMenuKebab.tsx | 11 + .../app/Mapping/Modal/MappingCommentModal.tsx | 5 + .../Mapping/Modal/MappingDocumentModal.tsx | 167 ++++++ app/src/app/Mapping/Search/DocumentSearch.tsx | 277 +++++++++ app/src/app/Notification/Notification.tsx | 3 + db/models/api_document.py | 210 +++++++ db/models/document.py | 186 ++++++ db/models/init_db.py | 2 + pyproject.toml | 2 +- 17 files changed, 2409 insertions(+), 68 deletions(-) create mode 100644 app/src/app/Mapping/Form/DocumentForm.tsx create mode 100644 app/src/app/Mapping/Menu/DocumentMenuKebab.tsx create mode 100644 app/src/app/Mapping/Modal/MappingDocumentModal.tsx create mode 100644 app/src/app/Mapping/Search/DocumentSearch.tsx create mode 100644 db/models/api_document.py create mode 100644 db/models/document.py diff --git a/Dockerfile-api b/Dockerfile-api index 250e934..ab8b022 100644 --- a/Dockerfile-api +++ b/Dockerfile-api @@ -4,7 +4,10 @@ FROM registry.fedoraproject.org/fedora:39 USER root RUN mkdir BASIL-API WORKDIR /BASIL-API -COPY . /BASIL-API +COPY api/ /BASIL-API/api +COPY db/ /BASIL-API/db +COPY misc/ /BASIL-API/misc +COPY requirements.txt pyproject.toml /BASIL-API/ RUN dnf install -y curl git patch python3 python3-pip podman rsync && \ pip3 install --no-cache-dir -r requirements.txt @@ -15,7 +18,7 @@ ENV BASIL_ADMIN_PASSWORD=${ADMIN_PASSWORD} BASIL_API_PORT=${API_PORT} # Init the database and # Write permission to db -RUN cd db/models && \ +RUN mkdir -p /var/tmp && cd /BASIL-API/db/models && \ python3 init_db.py && \ chmod a+rw /BASIL-API/db diff --git a/api/api.py b/api/api.py index d4eed64..2308c5a 100644 --- a/api/api.py +++ b/api/api.py @@ -64,6 +64,7 @@ 'write_permissions'] from db import db_orm +from db.models.api_document import ApiDocumentModel, ApiDocumentHistoryModel from db.models.api_justification import ApiJustificationModel, ApiJustificationHistoryModel from db.models.api_sw_requirement import ApiSwRequirementModel, ApiSwRequirementHistoryModel from db.models.api_test_case import ApiTestCaseModel, ApiTestCaseHistoryModel @@ -71,6 +72,7 @@ from db.models.api_test_specification import ApiTestSpecificationHistoryModel from db.models.api import ApiModel, ApiHistoryModel from db.models.comment import CommentModel +from db.models.document import DocumentModel, DocumentHistoryModel from db.models.justification import JustificationModel, JustificationHistoryModel from db.models.notification import NotificationModel from db.models.ssh_key import SshKeyModel @@ -108,6 +110,8 @@ _TCs = f'{_TC}s' _J = 'justification' _Js = f'{_J}s' +_D = 'document' +_Ds = f'{_D}s' def get_api_from_request(_request, _db_session): @@ -368,6 +372,16 @@ def get_model_editable_fields(_model, _is_history): return [x for x in all_fields if x not in not_editable_model_history_fields] +def get_dict_with_db_format_keys(_dict): + """ + return a dict with keys formatted with _ instead of - + """ + tmp = {} + for iKey in list(_dict.keys()): + tmp[iKey.replace("-", "_")] = _dict[iKey] + return tmp + + def filter_query(_query, _args, _model, _is_history): fields = get_model_editable_fields(_model, _is_history) for arg_key in _args.keys(): @@ -465,7 +479,8 @@ def split_section(_to_splits, _that_split, _work_item_type): _Js: [], _TCs: [], _TSs: [], - _SRs: []} + _SRs: [], + _Ds: []} for j in range(len(_to_split[_SRs])): tmp_section[_SRs].append(_to_split[_SRs][j].copy()) @@ -475,6 +490,8 @@ def split_section(_to_splits, _that_split, _work_item_type): tmp_section[_TSs].append(_to_split[_TSs][j].copy()) for j in range(len(_to_split[_Js])): tmp_section[_Js].append(_to_split[_Js][j].copy()) + for j in range(len(_to_split[_Ds])): + tmp_section[_Ds].append(_to_split[_Ds][j].copy()) sections.append(tmp_section) @@ -495,7 +512,8 @@ def split_section(_to_splits, _that_split, _work_item_type): _Js: [], _TCs: [], _TSs: [], - _SRs: []} + _SRs: [], + _Ds: []} for j in range(len(_to_split[_SRs])): tmp_section[_SRs].append(_to_split[_SRs][j].copy()) @@ -505,6 +523,8 @@ def split_section(_to_splits, _that_split, _work_item_type): tmp_section[_TSs].append(_to_split[_TSs][j].copy()) for j in range(len(_to_split[_Js])): tmp_section[_Js].append(_to_split[_Js][j].copy()) + for j in range(len(_to_split[_Ds])): + tmp_section[_Ds].append(_to_split[_Ds][j].copy()) sections.append(tmp_section) @@ -536,7 +556,8 @@ def get_split_sections(_specification, _mapping, _work_item_types): _TCs: [], _TSs: [], _SRs: [], - _Js: []}] + _Js: [], + _Ds: []}] for iWIT in range(len(_work_item_types)): _items_key = f"{_work_item_types[iWIT]}s" @@ -570,7 +591,8 @@ def get_split_sections(_specification, _mapping, _work_item_types): if sum([len(mapped_sections[iS][_SRs]), len(mapped_sections[iS][_TSs]), len(mapped_sections[iS][_TCs]), - len(mapped_sections[iS][_Js])]) == 0: + len(mapped_sections[iS][_Js]), + len(mapped_sections[iS][_Ds])]) == 0: mapped_sections[iS]['delete'] = True coverage_total = 0 @@ -582,6 +604,8 @@ def get_split_sections(_specification, _mapping, _work_item_types): coverage_total += mapped_sections[iS][_TSs][j]['covered'] for j in range(len(mapped_sections[iS][_Js])): coverage_total += mapped_sections[iS][_Js][j]['covered'] + for j in range(len(mapped_sections[iS][_Ds])): + coverage_total += mapped_sections[iS][_Ds][j]['covered'] mapped_sections[iS]['covered'] = min(max(coverage_total, 0), 100) # Remove Section with section: \n and no work items @@ -702,6 +726,11 @@ def get_api_sw_requirements_mapping_sections(dbi, api): ApiSwRequirementModel.offset.asc()).all() sr_mapping = [x.as_dict(db_session=dbi.session) for x in sr] + documents = dbi.session.query(ApiDocumentModel).filter( + ApiDocumentModel.api_id == api.id).order_by( + ApiDocumentModel.offset.asc()).all() + documents_mapping = [x.as_dict(db_session=dbi.session) for x in documents] + justifications = dbi.session.query(ApiJustificationModel).filter( ApiJustificationModel.api_id == api.id).order_by( ApiJustificationModel.offset.asc()).all() @@ -709,20 +738,17 @@ def get_api_sw_requirements_mapping_sections(dbi, api): mapping = {_A: api.as_dict(), _SRs: sr_mapping, - _Js: justifications_mapping} - - for iMapping in range(len(mapping[_SRs])): - current_offset = mapping[_SRs][iMapping]['offset'] - current_section = mapping[_SRs][iMapping]['section'] - mapping[_SRs][iMapping]['match'] = ( - api_specification[current_offset:current_offset + len(current_section)] == current_section) - for iMapping in range(len(mapping[_Js])): - current_offset = mapping[_Js][iMapping]['offset'] - current_section = mapping[_Js][iMapping]['section'] - mapping[_Js][iMapping]['match'] = ( - api_specification[current_offset:current_offset + len(current_section)] == current_section) - - mapped_sections = get_split_sections(api_specification, mapping, [_SR, _J]) + _Js: justifications_mapping, + _Ds: documents_mapping} + + for iType in [_SRs, _Js, _Ds]: + for iMapping in range(len(mapping[iType])): + current_offset = mapping[iType][iMapping]['offset'] + current_section = mapping[iType][iMapping]['section'] + mapping[iType][iMapping]['match'] = ( + api_specification[current_offset:current_offset + len(current_section)] == current_section) + + mapped_sections = get_split_sections(api_specification, mapping, [_SR, _J, _D]) unmapped_sections = [x for x in mapping[_SRs] if not x['match']] unmapped_sections += [x for x in mapping[_Js] if not x['match']] @@ -1016,6 +1042,89 @@ def get(self): return ret +class Document(Resource): + def get(self): + """ + /documents + """ + args = get_query_string_args(request.args) + dbi = db_orm.DbInterface(get_db()) + query = dbi.session.query(DocumentModel) + query = filter_query(query, args, DocumentModel, False) + docs = [doc.as_dict() for doc in query.all()] + dbi.engine.dispose() + return docs + + +class RemoteDocument(Resource): + def get(self): + """ + /remote-documents + """ + + args = get_query_string_args(request.args) + ret = {} + if "api-id" not in args.keys(): + return {} + + if "id" not in args.keys() and "url" not in args.keys(): + return {} + + dbi = db_orm.DbInterface(get_db()) + + # User + user_id = get_user_id_from_request(args, dbi.session) + + api = get_api_from_request(args, dbi.session) + if not api: + dbi.engine.dispose() + return NOT_FOUND_MESSAGE, NOT_FOUND_STATUS + + # Permissions + permissions = get_api_user_permissions(api, user_id, dbi.session) + if 'r' not in permissions: + dbi.engine.dispose() + return UNAUTHORIZED_MESSAGE, UNAUTHORIZED_STATUS + + if "url" in args.keys(): + ret = {"url": args["url"], + "valid": None, # not evaluated + "content": get_api_specification(args["url"])} + elif "id" in args.keys(): + try: + document = dbi.session.query(DocumentModel).filter( + DocumentModel.id == args["id"]).one() + content = get_api_specification(document.url) + valid = False + if content: + valid = (content[document.offset:document.offset + len(document.section)] == document.section) + if int(valid) == 0 and document.valid == 1: + document.valid = int(valid) + dbi.session.add(document) + + # Add Notifications + notification = f'Document {document.title} ' \ + f'mapped to ' \ + f'{api.api} as part of the library {api.library} it is not valid anymore' + notifications = NotificationModel(api, + NOTIFICATION_CATEGORY_EDIT, + f'Document {document.title} it is not valid anymore', + notification, + "", + f'/mapping/{api.id}') + dbi.session.add(notifications) + dbi.session.commit() + + except NoResultFound: + return f"Unable to find the Document id {id}", 400 + ret = {"id": args["id"], + "content": content, + "valid": valid} + + dbi.engine.dispose() + return ret + + class FixNewSpecificationWarnings(Resource): fields = ['id'] @@ -1227,6 +1336,19 @@ def post(self): user) dbi.session.add(tmp) + # Clone ApiDocument + api_documents = dbi.session.query(ApiDocumentModel).filter( + ApiDocumentModel.api_id == source_api[0].id + ).all() + for api_document in api_documents: + tmp = ApiDocumentModel(new_api, + api_document.document, + api_document.section, + api_document.offset, + api_document.coverage, + user) + dbi.session.add(tmp) + # Clone ApiSwRequirement api_sw_requirements = dbi.session.query(ApiSwRequirementModel).filter( ApiSwRequirementModel.api_id == source_api[0].id @@ -1577,6 +1699,11 @@ def get(self): ApiTestSpecificationModel.offset.asc()).all() ts_mapping = [x.as_dict(db_session=dbi.session) for x in ts] + documents = dbi.session.query(ApiDocumentModel).filter( + ApiDocumentModel.api_id == api.id).order_by( + ApiDocumentModel.offset.asc()).all() + documents_mapping = [x.as_dict(db_session=dbi.session) for x in documents] + justifications = dbi.session.query(ApiJustificationModel).filter( ApiJustificationModel.api_id == api.id).order_by( ApiJustificationModel.offset.asc()).all() @@ -1584,7 +1711,8 @@ def get(self): mapping = {_A: api.as_dict(), _TSs: ts_mapping, - _Js: justifications_mapping} + _Js: justifications_mapping, + _Ds: documents_mapping} for iTS in range(len(mapping[_TSs])): # Indirect Test Cases @@ -1596,18 +1724,14 @@ def get(self): mapping[_TSs][iTS][_TS][_TCs] = [get_dict_without_keys(x.as_dict(db_session=dbi.session), undesired_keys + ['api']) for x in ind_tc] - for iMapping in range(len(mapping[_TSs])): - current_offset = mapping[_TSs][iMapping]['offset'] - current_section = mapping[_TSs][iMapping]['section'] - mapping[_TSs][iMapping]['match'] = ( - api_specification[current_offset:current_offset + len(current_section)] == current_section) - for iMapping in range(len(mapping[_Js])): - current_offset = mapping[_Js][iMapping]['offset'] - current_section = mapping[_Js][iMapping]['section'] - mapping[_Js][iMapping]['match'] = ( - api_specification[current_offset:current_offset + len(current_section)] == current_section) + for iType in [_TSs, _Js, _Ds]: + for iMapping in range(len(mapping[iType])): + current_offset = mapping[iType][iMapping]['offset'] + current_section = mapping[iType][iMapping]['section'] + mapping[iType][iMapping]['match'] = ( + api_specification[current_offset:current_offset + len(current_section)] == current_section) - mapped_sections = get_split_sections(api_specification, mapping, [_TS, _J]) + mapped_sections = get_split_sections(api_specification, mapping, [_TS, _J, _D]) unmapped_sections = [x for x in mapping[_TSs] if not x['match']] unmapped_sections += [x for x in mapping[_Js] if not x['match']] ret = {'mapped': mapped_sections, @@ -1908,6 +2032,11 @@ def get(self): ApiTestCaseModel.offset.asc()).all() tc_mapping = [x.as_dict(db_session=dbi.session) for x in tc] + documents = dbi.session.query(ApiDocumentModel).filter( + ApiDocumentModel.api_id == api.id).order_by( + ApiDocumentModel.offset.asc()).all() + documents_mapping = [x.as_dict(db_session=dbi.session) for x in documents] + justifications = dbi.session.query(ApiJustificationModel).filter( ApiJustificationModel.api_id == api.id).order_by( ApiJustificationModel.offset.asc()).all() @@ -1915,20 +2044,17 @@ def get(self): mapping = {_A: api.as_dict(), _TCs: tc_mapping, - _Js: justifications_mapping} + _Js: justifications_mapping, + _Ds: documents_mapping} - for iMapping in range(len(mapping[_TCs])): - current_offset = mapping[_TCs][iMapping]['offset'] - current_section = mapping[_TCs][iMapping]['section'] - mapping[_TCs][iMapping]['match'] = ( - api_specification[current_offset:current_offset + len(current_section)] == current_section) - for iMapping in range(len(mapping[_Js])): - current_offset = mapping[_Js][iMapping]['offset'] - current_section = mapping[_Js][iMapping]['section'] - mapping[_Js][iMapping]['match'] = ( - api_specification[current_offset:current_offset + len(current_section)] == current_section) + for iType in [_TCs, _Js, _Ds]: + for iMapping in range(len(mapping[iType])): + current_offset = mapping[iType][iMapping]['offset'] + current_section = mapping[iType][iMapping]['section'] + mapping[iType][iMapping]['match'] = ( + api_specification[current_offset:current_offset + len(current_section)] == current_section) - mapped_sections = get_split_sections(api_specification, mapping, [_TC, _J]) + mapped_sections = get_split_sections(api_specification, mapping, [_TC, _J, _D]) unmapped_sections = [x for x in mapping[_TCs] if not x['match']] unmapped_sections += [x for x in mapping[_Js] if not x['match']] ret = {'mapped': mapped_sections, @@ -2201,6 +2327,11 @@ def get(self): _model_map = ApiJustificationModel _model_map_history = ApiJustificationHistoryModel _model_history = JustificationHistoryModel + elif args['work_item_type'] == 'document': + _model = DocumentModel + _model_map = ApiDocumentModel + _model_map_history = ApiDocumentHistoryModel + _model_history = DocumentHistoryModel elif args['work_item_type'] == 'sw-requirement': _model = SwRequirementModel _model_map = ApiSwRequirementModel @@ -2367,6 +2498,12 @@ def get(self): model.justification_id == _id ).all() api_ids = [x.api_id for x in api_data] + elif args['work_item_type'] == "document": + model = ApiDocumentModel + api_data = dbi.session.query(model).filter( + model.document_id == _id + ).all() + api_ids = [x.api_id for x in api_data] elif args['work_item_type'] == "sw-requirement": model = ApiSwRequirementModel api_data = dbi.session.query(model).filter( @@ -2530,11 +2667,12 @@ def get(self): mapping = {_A: api.as_dict(), _Js: justifications_mapping} - for iMapping in range(len(mapping[_Js])): - current_offset = mapping[_Js][iMapping]['offset'] - current_section = mapping[_Js][iMapping]['section'] - mapping[_Js][iMapping]['match'] = ( - api_specification[current_offset:current_offset + len(current_section)] == current_section) + for iType in [_Js]: + for iMapping in range(len(mapping[iType])): + current_offset = mapping[iType][iMapping]['offset'] + current_section = mapping[iType][iMapping]['section'] + mapping[iType][iMapping]['match'] = ( + api_specification[current_offset:current_offset + len(current_section)] == current_section) mapped_sections = get_split_sections(api_specification, mapping, [_J]) unmapped_sections = [x for x in mapping[_Js] if not x['match']] @@ -2775,6 +2913,335 @@ def delete(self): return True +class ApiDocumentsMapping(Resource): + fields = ['api-id', + 'coverage', + 'document', + 'offset', + 'section'] + + document_fields = ['description', + 'document_type', + 'offset', + 'section', + 'spdx_relation', + 'title', + 'url'] + + def get(self): + """ + curl /api/documents?api-id= + """ + + args = get_query_string_args(request.args) + if not check_fields_in_request(['api-id'], args): + return 'bad request!', 400 + + # undesired_keys = ['section', 'offset'] + + dbi = db_orm.DbInterface(get_db()) + + # User + user_id = get_user_id_from_request(args, dbi.session) + + # Find api + api = get_api_from_request(args, dbi.session) + if not api: + dbi.engine.dispose() + return NOT_FOUND_MESSAGE, NOT_FOUND_STATUS + + # Permissions + permissions = get_api_user_permissions(api, user_id, dbi.session) + if 'r' not in permissions: + dbi.engine.dispose() + return UNAUTHORIZED_MESSAGE, UNAUTHORIZED_STATUS + + api_specification = get_api_specification(api.raw_specification_url) + if api_specification is None: + return [] + + documents = dbi.session.query(ApiDocumentModel).filter( + ApiDocumentModel.api_id == api.id).order_by( + ApiDocumentModel.offset.asc()).all() + documents_mapping = [x.as_dict(db_session=dbi.session) for x in documents] + + mapping = {_A: api.as_dict(), + _Ds: documents_mapping} + + for iType in [_Ds]: + for iMapping in range(len(mapping[iType])): + current_offset = mapping[iType][iMapping]['offset'] + current_section = mapping[iType][iMapping]['section'] + mapping[iType][iMapping]['match'] = ( + api_specification[current_offset:current_offset + len(current_section)] == current_section) + + mapped_sections = get_split_sections(api_specification, mapping, [_D]) + unmapped_sections = [x for x in mapping[_Ds] if not x['match']] + ret = {'mapped': mapped_sections, + 'unmapped': unmapped_sections} + + dbi.engine.dispose() + return ret + + def post(self): + request_data = request.get_json(force=True) + + if not check_fields_in_request(self.fields, request_data): + return 'bad request!', 400 + + dbi = db_orm.DbInterface(get_db()) + + # User + user = get_active_user_from_request(request_data, dbi.session) + if not isinstance(user, UserModel): + return UNAUTHORIZED_MESSAGE, UNAUTHORIZED_STATUS + + # Find api + api = get_api_from_request(request_data, dbi.session) + if not api: + dbi.engine.dispose() + return NOT_FOUND_MESSAGE, NOT_FOUND_STATUS + + # Permissions + permissions = get_api_user_permissions(api, user.id, dbi.session) + if 'w' not in permissions or user.role not in USER_ROLES_WRITE_PERMISSIONS: + dbi.engine.dispose() + return UNAUTHORIZED_MESSAGE, UNAUTHORIZED_STATUS + + mapping_section = request_data['section'] + mapping_offset = request_data['offset'] + mapping_coverage = request_data['coverage'] + + if 'id' not in request_data['document'].keys(): + + if not check_fields_in_request(self.document_fields, request_data['document']): + dbi.engine.dispose() + return 'bad request!!', 400 + + doc_title = request_data['document']['title'] + doc_description = request_data['document']['description'] + doc_type = request_data['document']['document_type'] + doc_spdx_relation = request_data['document']['spdx_relation'] + doc_url = request_data['document']['url'] + doc_section = request_data['document']['section'] + doc_offset = request_data['document']['offset'] + doc_valid = 0 # default 0, it will be evaluated once rendered + new_document = DocumentModel(doc_title, + doc_description, + doc_type, + doc_spdx_relation, + doc_url, + doc_section, + doc_offset, + doc_valid, + user) + new_document_mapping_api = ApiDocumentModel(api, + new_document, + mapping_section, + mapping_offset, + mapping_coverage, + user) + dbi.session.add(new_document) + dbi.session.add(new_document_mapping_api) + + else: + id = request_data['document']['id'] + if len(dbi.session.query(ApiDocumentModel).filter(ApiDocumentModel.api_id == api.id).filter( + ApiDocumentModel.document_id == id).filter( + ApiDocumentModel.section == mapping_section).all()) > 0: + dbi.engine.dispose() + return "Document already associated to the selected api Specification section.", 409 + + try: + existing_document = dbi.session.query(DocumentModel).filter( + DocumentModel.id == id).one() + except NoResultFound: + return f"Unable to find the Document id {id}", 400 + + new_document_mapping_api = ApiDocumentModel(api, + existing_document, + mapping_section, + mapping_offset, + mapping_coverage, + user) + + dbi.session.add(new_document_mapping_api) + dbi.session.commit() # TO have the id of the new document mapping + + # Add Notifications + notification = f'{user.email} added document {new_document_mapping_api.document.id} ' \ + f'mapped to ' \ + f'{api.api} as part of the library {api.library}' + notifications = NotificationModel(api, + NOTIFICATION_CATEGORY_NEW, + f'Document mapping {new_document_mapping_api.id} has been added', + notification, + str(user.id), + f'/mapping/{api.id}') + dbi.session.add(notifications) + + dbi.session.commit() + dbi.engine.dispose() + return new_document_mapping_api.as_dict() + + def put(self): + request_data = request.get_json(force=True) + document_fields = self.document_fields + ['status'] + + if not check_fields_in_request(self.fields + ["relation-id"], request_data): + return 'bad request!', 400 + + if not check_fields_in_request(document_fields, request_data['document']): + return 'bad request!!', 400 + + dbi = db_orm.DbInterface(get_db()) + + # User + user = get_active_user_from_request(request_data, dbi.session) + if not isinstance(user, UserModel): + return UNAUTHORIZED_MESSAGE, UNAUTHORIZED_STATUS + + # Find api + api = get_api_from_request(request_data, dbi.session) + if not api: + dbi.engine.dispose() + return NOT_FOUND_MESSAGE, NOT_FOUND_STATUS + + # Permissions + permissions = get_api_user_permissions(api, user.id, dbi.session) + if 'w' not in permissions or user.role not in USER_ROLES_WRITE_PERMISSIONS: + dbi.engine.dispose() + return UNAUTHORIZED_MESSAGE, UNAUTHORIZED_STATUS + + # check if api ... + try: + document_mapping_api = dbi.session.query(ApiDocumentModel).filter( + ApiDocumentModel.id == request_data["relation-id"]).one() + except NoResultFound: + return f"Unable to find the Document mapping to Api id {request_data['relation-id']}", 400 + + document = document_mapping_api.document + + # Update only modified fields + modified_d = False + request_document_data = get_dict_with_db_format_keys(request_data["document"]) + print(f"request_document_data: {request_document_data}") + print(f"document: {document}") + for field in document_fields: + if field in request_document_data.keys(): + if getattr(document, field) != request_document_data[field]: + modified_d = True + setattr(document, field, request_document_data[field]) + if field == "url": # move valid to false + setattr(document, "valid", 0) + + if modified_d: + setattr(document, 'edited_by_id', user.id) + + modified_da = False # da: document mapping api + if request_data['section'] != document_mapping_api.section: + modified_da = True + document_mapping_api.section = request_data["section"] + + if request_data['offset'] != document_mapping_api.offset: + modified_da = True + document_mapping_api.offset = request_data["offset"] + + if request_data['coverage'] != document_mapping_api.coverage: + modified_da = True + document_mapping_api.coverage = request_data["coverage"] + + if modified_da: + document_mapping_api.edited_by_id = user.id + + if modified_da or modified_d: + # Add Notifications + notification = f'{user.email} modified document {document_mapping_api.document.id} ' \ + f'mapped to ' \ + f'{api.api} as part of the library {api.library}' + notifications = NotificationModel(api, + NOTIFICATION_CATEGORY_EDIT, + f'Document mapping {document_mapping_api.id} ' + f'has been modified', + notification, + str(user.id), + f'/mapping/{api.id}') + dbi.session.add(notifications) + + dbi.session.commit() + dbi.engine.dispose() + return document_mapping_api.as_dict() + + def delete(self): + request_data = request.get_json(force=True) + + if not check_fields_in_request(['relation-id', 'api-id'], request_data): + return 'bad request!', 400 + + dbi = db_orm.DbInterface(get_db()) + + # User + user = get_active_user_from_request(request_data, dbi.session) + if not isinstance(user, UserModel): + return UNAUTHORIZED_MESSAGE, UNAUTHORIZED_STATUS + + # Find api + api = get_api_from_request(request_data, dbi.session) + if not api: + dbi.engine.dispose() + return NOT_FOUND_MESSAGE, NOT_FOUND_STATUS + + # Permissions + permissions = get_api_user_permissions(api, user.id, dbi.session) + if 'w' not in permissions or user.role not in USER_ROLES_WRITE_PERMISSIONS: + dbi.engine.dispose() + return UNAUTHORIZED_MESSAGE, UNAUTHORIZED_STATUS + + # check if api ... + document_mapping_api = dbi.session.query(ApiDocumentModel).filter( + ApiDocumentModel.id == request_data["relation-id"]).all() + + if len(document_mapping_api) != 1: + dbi.engine.dispose() + return 'bad request!', 401 + + document_mapping_api = document_mapping_api[0] + + if document_mapping_api.api.id != api.id: + dbi.engine.dispose() + return 'bad request!', 401 + + notification_d_id = document_mapping_api.document.id + dbi.session.delete(document_mapping_api) + dbi.session.commit() + + # Add Notifications + notification = f'{user.email} deleted document {notification_d_id} ' \ + f'mapped to ' \ + f'{api.api} as part of the library {api.library}' + notifications = NotificationModel(api, + NOTIFICATION_CATEGORY_DELETE, + 'Document mapping has been deleted', + notification, + str(user.id), + f'/mapping/{api.id}') + dbi.session.add(notifications) + + # TODO: Remove work item only user request to do + """ + document = document_mapping_api.document + + if len(dbi.session.query(ApiDocumentModel).filter( \ + ApiDocumentModel.api_id == api.id).filter( \ + ApiDocumentModel.document_id == document.id).all()) == 0: + dbi.session.delete(document) + """ + + dbi.session.commit() + dbi.engine.dispose() + return True + + class ApiSwRequirementsMapping(Resource): fields = ['api-id', 'sw-requirement', 'section', 'coverage'] @@ -5690,6 +6157,8 @@ def get(self): api.add_resource(ApiSpecification, '/api-specifications') api.add_resource(Library, '/libraries') api.add_resource(SPDXLibrary, '/spdx/libraries') +api.add_resource(Document, '/documents') +api.add_resource(RemoteDocument, '/remote-documents') api.add_resource(Justification, '/justifications') api.add_resource(SwRequirement, '/sw-requirements') api.add_resource(TestSpecification, '/test-specifications') @@ -5697,6 +6166,7 @@ def get(self): # Mapping # - Direct api.add_resource(ApiSpecificationsMapping, '/mapping/api/specifications') +api.add_resource(ApiDocumentsMapping, '/mapping/api/documents') api.add_resource(ApiJustificationsMapping, '/mapping/api/justifications') api.add_resource(ApiSwRequirementsMapping, '/mapping/api/sw-requirements') api.add_resource(ApiTestSpecificationsMapping, '/mapping/api/test-specifications') diff --git a/app/src/app/Constants/constants.tsx b/app/src/app/Constants/constants.tsx index 92dd552..5f7f124 100644 --- a/app/src/app/Constants/constants.tsx +++ b/app/src/app/Constants/constants.tsx @@ -3,6 +3,8 @@ export const API_BASE_URL = 'http://localhost:5000' export const force_reload = true export const _A = 'api' +export const _D = 'document' +export const _Ds = 'documents' export const _J = 'justification' export const _Js = 'justifications' export const _M_ = '_mapping_' @@ -23,6 +25,83 @@ export const _TSs_ = 'test_specifications' export const DEFAULT_VIEW = _SRs export type validate = 'success' | 'warning' | 'error' | 'error2' | 'default' | 'indeterminate' | 'undefined' +export const document_type = [ + { value: 'file', label: 'file', disabled: false }, + { value: 'text', label: 'text', disabled: false } +] + +export const provision_type = [ + { value: '', label: '', disabled: false }, + { value: 'container', label: 'Fedora Container', disabled: false }, + { value: 'connect', label: 'SSH', disabled: false } +] + +export const spdx_relations = [ + { value: '', label: 'Select a value', disabled: false }, + { value: 'AFFECTS', label: 'AFFECTS', disabled: false }, + { value: 'AMENDS', label: 'AMENDS', disabled: false }, + { value: 'ANCESTOR', label: 'ANCESTOR', disabled: false }, + { value: 'AVAILABLE_FROM', label: 'AVAILABLE_FROM', disabled: false }, + { value: 'BUILD_DEPENDENCY', label: 'BUILD_DEPENDENCY', disabled: false }, + { value: 'BUILD_TOOL', label: 'BUILD_TOOL', disabled: false }, + { value: 'COORDINATED_BY', label: 'COORDINATED_BY', disabled: false }, + { value: 'CONTAINS', label: 'CONTAINS', disabled: false }, + { value: 'CONFIG_OF', label: 'CONFIG_OF', disabled: false }, + { value: 'COPY', label: 'COPY', disabled: false }, + { value: 'DATA_FILE', label: 'DATA_FILE', disabled: false }, + { value: 'DEPENDENCY_MANIFEST', label: 'DEPENDENCY_MANIFEST', disabled: false }, + { value: 'DEPENDS_ON', label: 'DEPENDS_ON', disabled: false }, + { value: 'DESCENDANT', label: 'DESCENDANT', disabled: false }, + { value: 'DESCRIBES', label: 'DESCRIBES', disabled: false }, + { value: 'DEV_DEPENDENCY', label: 'DEV_DEPENDENCY', disabled: false }, + { value: 'DEV_TOOL', label: 'DEV_TOOL', disabled: false }, + { value: 'DISTRIBUTION_ARTIFACT', label: 'DISTRIBUTION_ARTIFACT', disabled: false }, + { value: 'DOCUMENTATION', label: 'DOCUMENTATION', disabled: false }, + { value: 'DOES_NOT_AFFECT', label: 'DOES_NOT_AFFECT', disabled: false }, + { value: 'DYNAMIC_LINK', label: 'DYNAMIC_LINK', disabled: false }, + { value: 'EXAMPLE', label: 'EXAMPLE', disabled: false }, + { value: 'EVIDENCE_FOR', label: 'EVIDENCE_FOR', disabled: false }, + { value: 'EXPANDED_FROM_ARCHIVE', label: 'EXPANDED_FROM_ARCHIVE', disabled: false }, + { value: 'EXPLOIT_CREATED_BY', label: 'EXPLOIT_CREATED_BY', disabled: false }, + { value: 'FILE_ADDED', label: 'FILE_ADDED', disabled: false }, + { value: 'FILE_DELETED', label: 'FILE_DELETED', disabled: false }, + { value: 'FILE_MODIFIED', label: 'FILE_MODIFIED', disabled: false }, + { value: 'FIXED_BY', label: 'FIXED_BY', disabled: false }, + { value: 'FIXED_IN', label: 'FIXED_IN', disabled: false }, + { value: 'FOUND_BY', label: 'FOUND_BY', disabled: false }, + { value: 'GENERATES', label: 'GENERATES', disabled: false }, + { value: 'HAS_ASSESSMENT_FOR', label: 'HAS_ASSESSMENT_FOR', disabled: false }, + { value: 'HAS_ASSOCIATED_VULNERABILITY', label: 'HAS_ASSOCIATED_VULNERABILITY', disabled: false }, + { value: 'HOST_OF', label: 'HOST_OF', disabled: false }, + { value: 'INPUT_OF', label: 'INPUT_OF', disabled: false }, + { value: 'INVOKED_BY', label: 'INVOKED_BY', disabled: false }, + { value: 'METAFILE', label: 'METAFILE', disabled: false }, + { value: 'ON_BEHALF_OF', label: 'ON_BEHALF_OF', disabled: false }, + { value: 'OPTIONAL_COMPONENT', label: 'OPTIONAL_COMPONENT', disabled: false }, + { value: 'OPTIONAL_DEPENDENCY', label: 'OPTIONAL_DEPENDENCY', disabled: false }, + { value: 'OTHER', label: 'OTHER', disabled: false }, + { value: 'OUTPUT_OF', label: 'OUTPUT_OF', disabled: false }, + { value: 'PACKAGES', label: 'PACKAGES', disabled: false }, + { value: 'PATCH', label: 'PATCH', disabled: false }, + { value: 'PREREQUISITE', label: 'PREREQUISITE', disabled: false }, + { value: 'PROVIDED_DEPENDENCY', label: 'PROVIDED_DEPENDENCY', disabled: false }, + { value: 'PUBLISHED_BY', label: 'PUBLISHED_BY', disabled: false }, + { value: 'REPORTED_BY', label: 'REPORTED_BY', disabled: false }, + { value: 'REPUBLISHED_BY', label: 'REPUBLISHED_BY', disabled: false }, + { value: 'REQUIREMENT_FOR', label: 'REQUIREMENT_FOR', disabled: false }, + { value: 'RUNTIME_DEPENDENCY', label: 'RUNTIME_DEPENDENCY', disabled: false }, + { value: 'SPECIFICATION_FOR', label: 'SPECIFICATION_FOR', disabled: false }, + { value: 'STATIC_LINK', label: 'STATIC_LINK', disabled: false }, + { value: 'TEST', label: 'TEST', disabled: false }, + { value: 'TEST_CASE', label: 'TEST_CASE', disabled: false }, + { value: 'TEST_DEPENDENCY', label: 'TEST_DEPENDENCY', disabled: false }, + { value: 'TEST_TOOL', label: 'TEST_TOOL', disabled: false }, + { value: 'TESTED_ON', label: 'TESTED_ON', disabled: false }, + { value: 'TRAINED_ON', label: 'TRAINED_ON', disabled: false }, + { value: 'UNDER_INVESTIGATION_FOR', label: 'UNDER_INVESTIGATION_FOR', disabled: false }, + { value: 'VARIANT', label: 'VARIANT', disabled: false } +] + export const status_options = [ { value: 'NEW', label: 'New', disabled: false }, { value: 'IN PROGRESS', label: 'in Progress', disabled: false }, @@ -32,18 +111,55 @@ export const status_options = [ { value: 'APPROVED', label: 'Approved', disabled: false } ] -export const provision_type = [ - { value: '', label: '', disabled: false }, - { value: 'container', label: 'Fedora Container', disabled: false }, - { value: 'connect', label: 'SSH', disabled: false } -] - export const capitalizeFirstWithoutHashes = (_string: string) => { let tmp = _string.split('-').join(' ') tmp = tmp.split('_').join(' ') return tmp.charAt(0).toUpperCase() + tmp.slice(1) } +export const tcFormEmpty = { + coverage: 0, + description: '', + id: 0, + relative_path: '', + repository: '', + title: '' +} + +export const tsFormEmpty = { + coverage: 0, + expected_behavior: '', + id: 0, + preconditions: '', + test_description: '', + title: '' +} + +export const srFormEmpty = { + coverage: 0, + description: '', + id: 0, + title: '' +} + +export const jFormEmpty = { + coverage: 100, + description: '', + id: 0 +} + +export const docFormEmpty = { + coverage: 100, + document_type: 'file', + description: '', + id: 0, + section: '', + offset: 0, + spdx_relation: '', + title: '', + url: '' +} + export const logObject = (obj) => { let i let k diff --git a/app/src/app/Mapping/Form/CommentForm.tsx b/app/src/app/Mapping/Form/CommentForm.tsx index 4e1482e..f7c952f 100644 --- a/app/src/app/Mapping/Form/CommentForm.tsx +++ b/app/src/app/Mapping/Form/CommentForm.tsx @@ -74,6 +74,11 @@ export const CommentForm: React.FunctionComponent = ({ parent_table = Constants._J + Constants._M_ + Constants._A parent_id = relationData.relation_id } + } else if (workItemType == Constants._D) { + if (parentType == Constants._A) { + parent_table = Constants._D + Constants._M_ + Constants._A + parent_id = relationData.relation_id + } } else if (workItemType == Constants._SR) { if (parentType == Constants._A) { parent_table = Constants._SR_ + Constants._M_ + Constants._A diff --git a/app/src/app/Mapping/Form/DocumentForm.tsx b/app/src/app/Mapping/Form/DocumentForm.tsx new file mode 100644 index 0000000..695358f --- /dev/null +++ b/app/src/app/Mapping/Form/DocumentForm.tsx @@ -0,0 +1,520 @@ +import React from 'react' +import * as Constants from '../../Constants/constants' +import { + ActionGroup, + Button, + CodeBlock, + CodeBlockCode, + Form, + FormGroup, + FormHelperText, + FormSelect, + FormSelectOption, + HelperText, + HelperTextItem, + Hint, + HintBody, + TextArea, + TextInput +} from '@patternfly/react-core' +import { useAuth } from '../../User/AuthProvider' + +export interface DocumentFormProps { + api + formAction: string + formData + formDefaultButtons: number + formMessage: string + formVerb: string + handleModalToggle + loadMappingData + modalFormSubmitState: string + modalOffset + modalSection + parentData +} + +export const DocumentForm: React.FunctionComponent = ({ + api, + formAction = 'add', + formData = Constants.docFormEmpty, + formDefaultButtons = 1, + formMessage = '', + formVerb, + handleModalToggle, + loadMappingData, + modalFormSubmitState = 'waiting', + modalOffset, + modalSection, + parentData +}: DocumentFormProps) => { + const auth = useAuth() + const [descriptionValue, setDescriptionValue] = React.useState(formData.description) + const [SPDXRelationValue, setSPDXRelationValue] = React.useState(formData.spdx_relation) + const [documentContentValue, setDocumentContentValue] = React.useState('') + const [titleValue, setTitleValue] = React.useState(formData.title) + const [urlValue, setUrlValue] = React.useState(formData.url) + const [offsetValue, setOffsetValue] = React.useState(formData.offset) + const [sectionValue, setSectionValue] = React.useState(formData.section) + const [validatedDescriptionValue, setValidatedDescriptionValue] = React.useState('error') + const [validatedTitleValue, setValidatedTitleValue] = React.useState('error') + const [validatedUrlValue, setValidatedUrlValue] = React.useState('error') + const [coverageValue, setCoverageValue] = React.useState(formData.coverage) + const [validatedCoverageValue, setValidatedCoverageValue] = React.useState('error') + const [documentStatusValue, setDocumentStatusValue] = React.useState(formData.status) + const [documentTypeValue, setDocumentTypeValue] = React.useState(formData.document_type) + const [validatedOffsetValue, setValidatedOffsetValue] = React.useState('error') + const [validatedSectionValue, setValidatedSectionValue] = React.useState('error') + const [messageValue, setMessageValue] = React.useState(formMessage) + const [statusValue, setStatusValue] = React.useState('waiting') + + const resetForm = () => { + setDescriptionValue('') + setTitleValue('') + setUrlValue('') + setSectionValue('') + setOffsetValue(0) + setSPDXRelationValue('') + setCoverageValue('100') + setDocumentContentValue('') + } + + React.useEffect(() => { + if (descriptionValue == undefined) { + setDescriptionValue('') + } else { + if (descriptionValue.trim() === '') { + setValidatedDescriptionValue('error') + } else { + setValidatedDescriptionValue('success') + } + } + }, [descriptionValue]) + + React.useEffect(() => { + if (titleValue == undefined) { + setTitleValue('') + } else { + if (titleValue.trim() === '') { + setValidatedTitleValue('error') + } else { + setValidatedTitleValue('success') + } + } + }, [titleValue]) + + React.useEffect(() => { + if (urlValue == undefined) { + setUrlValue('') + } else { + if (urlValue.trim() === '') { + setValidatedUrlValue('error') + } else { + setValidatedUrlValue('success') + } + } + }, [urlValue]) + + React.useEffect(() => { + if (coverageValue === '') { + setValidatedCoverageValue('error') + } else if (/^\d+$/.test(coverageValue)) { + if (coverageValue >= 0 && coverageValue <= 100) { + setValidatedCoverageValue('success') + } else { + setValidatedCoverageValue('error') + } + } else { + setValidatedCoverageValue('error') + } + }, [coverageValue]) + + React.useEffect(() => { + if (statusValue == 'submitted') { + handleSubmit() + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [statusValue]) + + React.useEffect(() => { + if (modalFormSubmitState == 'submitted') { + handleSubmit() + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [modalFormSubmitState]) + + React.useEffect(() => { + if (sectionValue.trim() === '') { + setValidatedSectionValue('error') + } else { + setValidatedSectionValue('success') + } + }, [sectionValue]) + + React.useEffect(() => { + if (offsetValue === '') { + setValidatedOffsetValue('default') + } else if (/^\d+$/.test(offsetValue)) { + if (offsetValue >= 0 && offsetValue <= documentContentValue.length) { + setValidatedOffsetValue('success') + } else { + setValidatedOffsetValue('error') + } + } else { + setValidatedOffsetValue('error') + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [offsetValue]) + + const handleDescriptionValueChange = (_event, value: string) => { + setDescriptionValue(value) + } + + const handleTitleValueChange = (_event, value: string) => { + setTitleValue(value) + } + + const handleUrlValueChange = (_event, value: string) => { + setUrlValue(value) + } + + const handleSPDXRelationValueChange = (_event, value: string) => { + setSPDXRelationValue(value) + } + + const handleSectionValueChange = () => { + const currentSelection = getSelection()?.toString() as string | '' + if (currentSelection != '') { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if (((getSelection()?.anchorNode?.parentNode as any)?.id as string | '') == 'div-document-${formAction}-content-${formData.id}') { + setSectionValue(currentSelection) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + setOffsetValue(Math.min((getSelection() as any)?.baseOffset, (getSelection() as any)?.extentOffset)) + } + } + } + + const handleCoverageValueChange = (_event, value: string) => { + setCoverageValue(value) + } + + const handleDocumentStatusChange = (_event, value: string) => { + setDocumentStatusValue(value) + } + + const handleDocumentTypeChange = (_event, value: string) => { + setDocumentTypeValue(value) + } + + const handleSubmit = () => { + if (validatedDescriptionValue != 'success') { + setMessageValue('Document Description is mandatory.') + setStatusValue('waiting') + return + } else if (validatedTitleValue != 'success') { + setMessageValue('Document Title is mandatory.') + setStatusValue('waiting') + return + } else if (validatedUrlValue != 'success') { + setMessageValue('Document Url is mandatory.') + setStatusValue('waiting') + return + } else if (validatedCoverageValue != 'success') { + setMessageValue('Document Coverage is mandatory and must be a integer value in the range 0-100.') + setStatusValue('waiting') + return + } else if (modalSection.trim().length == 0) { + setMessageValue('Section of the software component specification is mandatory.') + setStatusValue('waiting') + return + } + + setMessageValue('') + + const data = { + 'api-id': api.id, + document: { + title: titleValue, + description: descriptionValue, + document_type: documentTypeValue, + url: urlValue, + spdx_relation: SPDXRelationValue, + offset: offsetValue, + section: sectionValue + }, + section: modalSection, + offset: modalOffset, + coverage: coverageValue, + 'user-id': auth.userId, + token: auth.token + } + + if (formVerb == 'PUT' || formVerb == 'DELETE') { + data['relation-id'] = parentData.relation_id + data['document']['id'] = formData.id + } + + if (formVerb == 'PUT') { + data['document']['status'] = documentStatusValue + } + + fetch(Constants.API_BASE_URL + '/mapping/api/documents', { + method: formVerb, + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify(data) + }) + .then((response) => { + if (response.status !== 200) { + setMessageValue(response.statusText) + setStatusValue('waiting') + } else { + handleModalToggle() + setMessageValue('') + setStatusValue('waiting') + loadMappingData(Constants.force_reload) + } + }) + .catch((err) => { + setMessageValue(err.toString()) + setStatusValue('waiting') + }) + } + + const readRemoteTextFile = () => { + if (documentTypeValue != 'text') { + return + } + + let url = Constants.API_BASE_URL + '/remote-documents?url=' + urlValue + url += '&api-id=' + api.id + + if (auth.isLogged()) { + url += '&user-id=' + auth.userId + '&token=' + auth.token + } else { + return + } + + fetch(url) + .then((res) => res.json()) + .then((data) => { + setDocumentContentValue(data['content']) + }) + .catch((err) => { + console.log(err.message) + }) + } + + const setSectionAsUnmatching = () => { + const unmatching_section = '?????????' + const unmatching_offset = 0 + setSectionValue(unmatching_section) + setOffsetValue(unmatching_offset) + } + + const setSectionAsFullDocument = () => { + setSectionValue(documentContentValue) + setOffsetValue(0) + } + + return ( +
+ {formAction == 'edit' ? ( + + + {Constants.status_options.map((option, index) => ( + + ))} + + + ) : ( + '' + )} + + handleTitleValueChange(_ev, value)} + /> + {validatedTitleValue !== 'success' && ( + + + {validatedTitleValue === 'error' ? 'This field is mandatory' : ''} + + + )} + + +