diff --git a/apps/envited.ascs.digital/common/archive/archive.test.ts b/apps/envited.ascs.digital/common/archive/archive.test.ts index 342724e0..82fc853a 100644 --- a/apps/envited.ascs.digital/common/archive/archive.test.ts +++ b/apps/envited.ascs.digital/common/archive/archive.test.ts @@ -50,4 +50,30 @@ describe('common/archive', () => { expect(result).toEqual(undefined) }) }) + + describe('countAmountOfFilesInZip', () => { + it('Should return the amount of files', async () => { + // when ... we want to read a file from a zip archive + // then ... it should return the entries from the zip archive + + const getEntriesStub = jest + .fn() + .mockResolvedValue([ + { filename: 'FILENAME.EXT', directory: true }, + { filename: 'FILENAME_1.EXT' }, + { filename: 'FILENAME_2.EXT' }, + ]) + const closeStub = jest.fn() + const zipReaderStub = jest.fn().mockImplementation(() => ({ + getEntries: getEntriesStub, + close: closeStub, + })) + + const result = await SUT._countAmountOfFilesInZip({ ZipReader: zipReaderStub })('FILE' as any) + expect(result).toEqual(2) + expect(closeStub).toHaveBeenCalledWith() + expect(getEntriesStub).toHaveBeenCalledWith() + expect(zipReaderStub).toHaveBeenCalledWith({ blob: 'FILE', size: undefined }) + }) + }) }) diff --git a/apps/envited.ascs.digital/common/archive/archive.ts b/apps/envited.ascs.digital/common/archive/archive.ts index 17a2937c..441b8a14 100644 --- a/apps/envited.ascs.digital/common/archive/archive.ts +++ b/apps/envited.ascs.digital/common/archive/archive.ts @@ -48,3 +48,17 @@ export const _readContentFromJsonFile = read(file).then(JSON.parse) export const readContentFromJsonFile = _readContentFromJsonFile({ read }) + +export const _countAmountOfFilesInZip = + ({ ZipReader }: { ZipReader: any }) => + async (file: File) => { + const reader = new ZipReader(new BlobReader(file)) + + return reader + .getEntries() + .then((entries: Entry[]) => entries.filter(entry => !entry.directory).length) + .catch(() => undefined) + .finally(() => reader.close()) + } + +export const countAmountOfFilesInZip = _countAmountOfFilesInZip({ ZipReader }) diff --git a/apps/envited.ascs.digital/common/archive/index.ts b/apps/envited.ascs.digital/common/archive/index.ts index 7fef532f..dd249ab4 100644 --- a/apps/envited.ascs.digital/common/archive/index.ts +++ b/apps/envited.ascs.digital/common/archive/index.ts @@ -1 +1,8 @@ -export { extract, extractFromFile, extractFromByteArray, read, readContentFromJsonFile } from './archive' +export { + countAmountOfFilesInZip, + extract, + extractFromFile, + extractFromByteArray, + read, + readContentFromJsonFile, +} from './archive' diff --git a/apps/envited.ascs.digital/common/asset/constants.ts b/apps/envited.ascs.digital/common/asset/constants.ts index 2ae6071e..b5e4a64f 100644 --- a/apps/envited.ascs.digital/common/asset/constants.ts +++ b/apps/envited.ascs.digital/common/asset/constants.ts @@ -2,4 +2,6 @@ export const MANIFEST_FILE = 'manifest.json' export const LICENSE_FILE = 'LICENSE' +export const README_FILE = 'README.md' + export const DOMAIN_METADATA_FILE = 'metadata/domainMetadata.json' diff --git a/apps/envited.ascs.digital/common/constants/errors.ts b/apps/envited.ascs.digital/common/constants/errors.ts index 75ad86d9..71209937 100644 --- a/apps/envited.ascs.digital/common/constants/errors.ts +++ b/apps/envited.ascs.digital/common/constants/errors.ts @@ -8,6 +8,10 @@ export const ERRORS = { ASSETS_NOT_FOUND: 'Assets not found', ASSET_INVALID: 'Asset validation failed', ASSET_FILE_NOT_FOUND: 'No valid metadata.json found', + DOMAIN_METADATA_INVALID: 'domainMetadata.json validation failed', + MANIFEST_INVALID: 'manifest.json validation failed', + FILES_NOT_FOUND: 'File(s) not found', + README_FILE_NOT_FOUND: 'README.md file not found', NOT_ALLOWED_TO_DELETE_ASSET: 'Not allowed to delete asset', MINTED_ASSET_CANNOT_BE_DELETED: 'Minted asset cannot be deleted', } diff --git a/apps/envited.ascs.digital/common/validator/shacl/shacl.constants.ts b/apps/envited.ascs.digital/common/validator/shacl/shacl.constants.ts index eec020ec..0ecdde78 100644 --- a/apps/envited.ascs.digital/common/validator/shacl/shacl.constants.ts +++ b/apps/envited.ascs.digital/common/validator/shacl/shacl.constants.ts @@ -17,3 +17,5 @@ export const SCHEMA_MAP = { } export const CONTEXT_DROP_SCHEMAS = [Schema.sh, Schema.skos, Schema.xsd] + +export const AMOUNT_OF_UNDEFINED_FILES_IN_MANIFEST = 2 diff --git a/apps/envited.ascs.digital/common/validator/shacl/shacl.test.ts b/apps/envited.ascs.digital/common/validator/shacl/shacl.test.ts index bb1bce94..07ef9a24 100644 --- a/apps/envited.ascs.digital/common/validator/shacl/shacl.test.ts +++ b/apps/envited.ascs.digital/common/validator/shacl/shacl.test.ts @@ -5,26 +5,34 @@ describe('common/validator/shacl', () => { it('Should return a valid result', async () => { // when ... we want to validate a asset file const file = 'ZIP' - const validateManifestStub = jest.fn().mockReturnValue({ + const validateReadmeStub = jest.fn().mockResolvedValue('FILE_NAME') + const validateManifestStub = jest.fn().mockResolvedValue({ conforms: true, data: { file: 'FILE_NAME', }, }) - const validateDomainMetadataStub = jest.fn().mockReturnValue({ + const validateDomainMetadataStub = jest.fn().mockResolvedValue({ conforms: true, data: { name: 'NAME', }, }) - const checkIfAllFilesInManifestExistsStub = jest.fn().mockReturnValue({}) + const checkIfAllFilesInManifestExistStub = jest.fn().mockResolvedValue({ + errors: [], + amount: 12, + }) + + const countAmountOfFilesInZipStub = jest.fn().mockResolvedValue(14) // then ... we should get a valid response const result = await SUT._validateShaclFile({ validateManifest: validateManifestStub, validateDomainMetadata: validateDomainMetadataStub, - checkIfAllFilesInManifestExists: checkIfAllFilesInManifestExistsStub, + checkIfAllFilesInManifestExist: checkIfAllFilesInManifestExistStub, + countAmountOfFilesInZip: countAmountOfFilesInZipStub, + validateReadme: validateReadmeStub, })(file as any) expect(validateManifestStub).toHaveBeenCalledWith('ZIP') diff --git a/apps/envited.ascs.digital/common/validator/shacl/shacl.ts b/apps/envited.ascs.digital/common/validator/shacl/shacl.ts index c4a0da8a..2e85283b 100644 --- a/apps/envited.ascs.digital/common/validator/shacl/shacl.ts +++ b/apps/envited.ascs.digital/common/validator/shacl/shacl.ts @@ -1,41 +1,83 @@ import { DatasetCore, Quad } from '@rdfjs/types' import { Dataset } from '@zazuko/env/lib/Dataset' import { Entry } from '@zip.js/zip.js' -import { all, equals, keys, omit, pipe } from 'ramda' +import { all, equals, has, isEmpty, keys, omit, pipe } from 'ramda' import ValidationReport from 'rdf-validate-shacl/src/validation-report' -import { extractFromFile, read } from '../../archive' -import { MANIFEST_FILE } from '../../asset/constants' +import { countAmountOfFilesInZip, extractFromFile, read } from '../../archive' +import { MANIFEST_FILE, README_FILE } from '../../asset/constants' import { Manifest } from '../../asset/types' import { getAllManifestLinksAndFormatPaths, getDomainMetadataPath } from '../../asset/validateAndCreateMetadata.utils' import { ERRORS } from '../../constants' import { CONTEXT_DROP_SCHEMAS } from './shacl.constants' import { ContentType, Schema, ValidationSchema } from './shacl.types' -import { fetchShaclSchema, loadDataset, parseStreamToDataset, validateShacl } from './shacl.utils' +import { + fetchShaclSchema, + formatFilesErrorMessage, + loadDataset, + parseStreamToDataset, + subtractManifestAndReadMeFiles, + validateShacl, +} from './shacl.utils' export const _validateShaclFile = ({ validateManifest, validateDomainMetadata, - checkIfAllFilesInManifestExists, + checkIfAllFilesInManifestExist, + countAmountOfFilesInZip, + validateReadme, }: { validateManifest: (file: File) => Promise<{ conforms: boolean; data: any }> validateDomainMetadata: (file: File, manifest: Manifest) => Promise<{ conforms: boolean; data: any }> - checkIfAllFilesInManifestExists: (file: File, manifest: Manifest) => any + checkIfAllFilesInManifestExist: ( + file: File, + manifest: Manifest, + ) => Promise<{ errors: { error: string }[]; amount: number }> + countAmountOfFilesInZip: (file: File) => Promise + validateReadme: (file: File) => Promise }) => async (file: File) => { try { + const readmeExists = await validateReadme(file) + if (!readmeExists) { + return { + isValid: false, + data: {}, + error: ERRORS.README_FILE_NOT_FOUND, + } + } + const { conforms: manifestConforms, data: manifest } = await validateManifest(file) - await checkIfAllFilesInManifestExists(file, manifest) + const manifestFiles = await checkIfAllFilesInManifestExist(file, manifest) + + if (!isEmpty(manifestFiles.errors)) { + return { + isValid: false, + data: {}, + error: formatFilesErrorMessage(manifestFiles.errors), + } + } + + const amountOfFilesInZip = await countAmountOfFilesInZip(file) + const amountWithoutManifestAndReadme = subtractManifestAndReadMeFiles(amountOfFilesInZip) + + if (!equals(amountWithoutManifestAndReadme)(manifestFiles.amount)) { + return { + isValid: false, + data: {}, + error: `${amountWithoutManifestAndReadme} files found, should be ${manifestFiles.amount} files`, + } + } const { conforms: domainMetadataConforms, data: domainMetadata } = await validateDomainMetadata(file, manifest) if (!manifestConforms) { - return { isValid: false, data: {}, error: ERRORS.ASSET_INVALID } + return { isValid: false, data: {}, error: ERRORS.MANIFEST_INVALID } } if (!domainMetadataConforms) { - return { isValid: false, data: {}, error: ERRORS.ASSET_INVALID } + return { isValid: false, data: {}, error: ERRORS.DOMAIN_METADATA_INVALID } } return { isValid: true, data: { manifest, domainMetadata } } @@ -113,6 +155,22 @@ export const validateManifest = _validateManifest({ validateShaclSchema, }) +export const _validateReadme = + ({ getShaclDataFromZip }: { getShaclDataFromZip: (file: File, fileName: string) => Promise }) => + async (file: File) => { + try { + await getShaclDataFromZip(file, README_FILE) + + return true + } catch { + return false + } + } + +export const validateReadme = _validateReadme({ + getShaclDataFromZip, +}) + export const _validateDomainMetadata = ({ getShaclDataFromZip, @@ -163,23 +221,42 @@ export const validateDomainMetadata = _validateDomainMetadata({ getDomainMetadataPath, }) -export const _checkIfAllFilesInManifestExists = +export const _checkIfAllFilesInManifestExist = ({ getShaclDataFromZip }: { getShaclDataFromZip: (file: File, fileName: string) => Promise }) => async (file: File, manifest: Manifest) => { const files = getAllManifestLinksAndFormatPaths(manifest) - const validationPromises = files.map((fileName: string) => getShaclDataFromZip(file, fileName)) + const validationPromises = files.map((fileName: string) => ({ + fileName, + promise: getShaclDataFromZip(file, fileName), + })) - return Promise.all(validationPromises) + const wrappedPromises = validationPromises.map(({ fileName, promise }) => + promise.catch(() => ({ error: fileName })), + ) + + const errors = await Promise.all(wrappedPromises).then( + (results: (string | { error: string })[]) => + results.filter((result: string | { error: string }) => has('error')(result) && result.error) as { + error: string + }[], + ) + + return { + errors, + amount: files.length, + } } -export const checkIfAllFilesInManifestExists = _checkIfAllFilesInManifestExists({ +export const checkIfAllFilesInManifestExist = _checkIfAllFilesInManifestExist({ getShaclDataFromZip, }) export const validateShaclFile = _validateShaclFile({ validateDomainMetadata, validateManifest, - checkIfAllFilesInManifestExists, + checkIfAllFilesInManifestExist, + countAmountOfFilesInZip, + validateReadme, }) export const _validateShaclDataWithSchema = diff --git a/apps/envited.ascs.digital/common/validator/shacl/shacl.utils.test.ts b/apps/envited.ascs.digital/common/validator/shacl/shacl.utils.test.ts index c4657c8a..c4fa9a80 100644 --- a/apps/envited.ascs.digital/common/validator/shacl/shacl.utils.test.ts +++ b/apps/envited.ascs.digital/common/validator/shacl/shacl.utils.test.ts @@ -58,4 +58,26 @@ describe('common/validator/shacl/shacl.utils', () => { expect(result._readableState.highWaterMark).toEqual(expected) }) }) + + describe('formatFilesErrorMessage', () => { + const errors = [ + { + error: 'Error 1', + }, + { + error: 'Error 2', + }, + ] + + const expected = 'File(s) not found - Error 1, Error 2' + const result = SUT.formatFilesErrorMessage(errors) + + expect(result).toEqual(expected) + }) + + describe('subtractManifestAndReadMeFiles', () => { + const result = SUT.subtractManifestAndReadMeFiles(16) + + expect(result).toEqual(14) + }) }) diff --git a/apps/envited.ascs.digital/common/validator/shacl/shacl.utils.ts b/apps/envited.ascs.digital/common/validator/shacl/shacl.utils.ts index 82e690cb..fb71c011 100644 --- a/apps/envited.ascs.digital/common/validator/shacl/shacl.utils.ts +++ b/apps/envited.ascs.digital/common/validator/shacl/shacl.utils.ts @@ -1,11 +1,13 @@ import { DatasetCore, Quad } from '@rdfjs/types' import rdf, { DefaultEnv } from '@zazuko/env' import { Dataset } from '@zazuko/env/lib/Dataset' +import { flip, join, map, pipe, subtract } from 'ramda' import rdfParser, { RdfParser } from 'rdf-parse' import SHACLValidator from 'rdf-validate-shacl' import { Readable } from 'stream' -import { SCHEMA_MAP } from './shacl.constants' +import { ERRORS } from '../../constants' +import { AMOUNT_OF_UNDEFINED_FILES_IN_MANIFEST, SCHEMA_MAP } from './shacl.constants' import { ContentType, ValidationSchema } from './shacl.types' export const validateShacl = (shapes: DatasetCore) => async (data: DatasetCore) => { @@ -56,3 +58,12 @@ export const loadDataset = _loadDataset({ createReadableStream, parseStreamToDataset, }) + +export const formatFilesErrorMessage = (errors: { error: string }[]) => + pipe( + map(({ error }: { error: string }) => error), + join(', '), + (x: string) => `${ERRORS.FILES_NOT_FOUND} - ${x}`, + )(errors) + +export const subtractManifestAndReadMeFiles = flip(subtract)(AMOUNT_OF_UNDEFINED_FILES_IN_MANIFEST)