Skip to content

Commit

Permalink
feat(envited.ascs.digital): Error handling asset verification (#398)
Browse files Browse the repository at this point in the history
* feat(envited.ascs.digital): Error handling asset verification

Signed-off-by: Jeroen Branje <[email protected]>

* feat: replace any type

Signed-off-by: Jeroen Branje <[email protected]>

* feat: add branch to workflow

Signed-off-by: Jeroen Branje <[email protected]>

* feat: add unexpected files amount check

Signed-off-by: Jeroen Branje <[email protected]>

* feat: remove branch from workflow

Signed-off-by: Jeroen Branje <[email protected]>

* feat: add check if readme exists

Signed-off-by: Jeroen Branje <[email protected]>

* feat: add error message for manifest and domainMetadata

Signed-off-by: Jeroen Branje <[email protected]>

* feat: rename exists

Signed-off-by: Jeroen Branje <[email protected]>

---------

Signed-off-by: Jeroen Branje <[email protected]>
Co-authored-by: Carlo van Driesten <[email protected]>
  • Loading branch information
jeroenbranje and jdsika authored Dec 17, 2024
1 parent 6d526bb commit 93f7c15
Show file tree
Hide file tree
Showing 10 changed files with 193 additions and 20 deletions.
26 changes: 26 additions & 0 deletions apps/envited.ascs.digital/common/archive/archive.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,4 +50,30 @@ describe('common/archive', () => {
expect(result).toEqual(undefined)
})
})

describe('countAmountOfFilesInZip', () => {
it('Should return the amount of files', async () => {
// when ... we want to read a file from a zip archive
// then ... it should return the entries from the zip archive

const getEntriesStub = jest
.fn()
.mockResolvedValue([
{ filename: 'FILENAME.EXT', directory: true },
{ filename: 'FILENAME_1.EXT' },
{ filename: 'FILENAME_2.EXT' },
])
const closeStub = jest.fn()
const zipReaderStub = jest.fn().mockImplementation(() => ({
getEntries: getEntriesStub,
close: closeStub,
}))

const result = await SUT._countAmountOfFilesInZip({ ZipReader: zipReaderStub })('FILE' as any)
expect(result).toEqual(2)
expect(closeStub).toHaveBeenCalledWith()
expect(getEntriesStub).toHaveBeenCalledWith()
expect(zipReaderStub).toHaveBeenCalledWith({ blob: 'FILE', size: undefined })
})
})
})
14 changes: 14 additions & 0 deletions apps/envited.ascs.digital/common/archive/archive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,17 @@ export const _readContentFromJsonFile =
read(file).then(JSON.parse)

export const readContentFromJsonFile = _readContentFromJsonFile({ read })

export const _countAmountOfFilesInZip =
({ ZipReader }: { ZipReader: any }) =>
async (file: File) => {
const reader = new ZipReader(new BlobReader(file))

return reader
.getEntries()
.then((entries: Entry[]) => entries.filter(entry => !entry.directory).length)
.catch(() => undefined)
.finally(() => reader.close())
}

export const countAmountOfFilesInZip = _countAmountOfFilesInZip({ ZipReader })
9 changes: 8 additions & 1 deletion apps/envited.ascs.digital/common/archive/index.ts
Original file line number Diff line number Diff line change
@@ -1 +1,8 @@
export { extract, extractFromFile, extractFromByteArray, read, readContentFromJsonFile } from './archive'
export {
countAmountOfFilesInZip,
extract,
extractFromFile,
extractFromByteArray,
read,
readContentFromJsonFile,
} from './archive'
2 changes: 2 additions & 0 deletions apps/envited.ascs.digital/common/asset/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,6 @@ export const MANIFEST_FILE = 'manifest.json'

export const LICENSE_FILE = 'LICENSE'

export const README_FILE = 'README.md'

export const DOMAIN_METADATA_FILE = 'metadata/domainMetadata.json'
4 changes: 4 additions & 0 deletions apps/envited.ascs.digital/common/constants/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ export const ERRORS = {
ASSETS_NOT_FOUND: 'Assets not found',
ASSET_INVALID: 'Asset validation failed',
ASSET_FILE_NOT_FOUND: 'No valid metadata.json found',
DOMAIN_METADATA_INVALID: 'domainMetadata.json validation failed',
MANIFEST_INVALID: 'manifest.json validation failed',
FILES_NOT_FOUND: 'File(s) not found',
README_FILE_NOT_FOUND: 'README.md file not found',
NOT_ALLOWED_TO_DELETE_ASSET: 'Not allowed to delete asset',
MINTED_ASSET_CANNOT_BE_DELETED: 'Minted asset cannot be deleted',
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,5 @@ export const SCHEMA_MAP = {
}

export const CONTEXT_DROP_SCHEMAS = [Schema.sh, Schema.skos, Schema.xsd]

export const AMOUNT_OF_UNDEFINED_FILES_IN_MANIFEST = 2
16 changes: 12 additions & 4 deletions apps/envited.ascs.digital/common/validator/shacl/shacl.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,34 @@ describe('common/validator/shacl', () => {
it('Should return a valid result', async () => {
// when ... we want to validate a asset file
const file = 'ZIP'
const validateManifestStub = jest.fn().mockReturnValue({
const validateReadmeStub = jest.fn().mockResolvedValue('FILE_NAME')
const validateManifestStub = jest.fn().mockResolvedValue({
conforms: true,
data: {
file: 'FILE_NAME',
},
})
const validateDomainMetadataStub = jest.fn().mockReturnValue({
const validateDomainMetadataStub = jest.fn().mockResolvedValue({
conforms: true,
data: {
name: 'NAME',
},
})

const checkIfAllFilesInManifestExistsStub = jest.fn().mockReturnValue({})
const checkIfAllFilesInManifestExistStub = jest.fn().mockResolvedValue({
errors: [],
amount: 12,
})

const countAmountOfFilesInZipStub = jest.fn().mockResolvedValue(14)

// then ... we should get a valid response
const result = await SUT._validateShaclFile({
validateManifest: validateManifestStub,
validateDomainMetadata: validateDomainMetadataStub,
checkIfAllFilesInManifestExists: checkIfAllFilesInManifestExistsStub,
checkIfAllFilesInManifestExist: checkIfAllFilesInManifestExistStub,
countAmountOfFilesInZip: countAmountOfFilesInZipStub,
validateReadme: validateReadmeStub,
})(file as any)

expect(validateManifestStub).toHaveBeenCalledWith('ZIP')
Expand Down
105 changes: 91 additions & 14 deletions apps/envited.ascs.digital/common/validator/shacl/shacl.ts
Original file line number Diff line number Diff line change
@@ -1,41 +1,83 @@
import { DatasetCore, Quad } from '@rdfjs/types'
import { Dataset } from '@zazuko/env/lib/Dataset'
import { Entry } from '@zip.js/zip.js'
import { all, equals, keys, omit, pipe } from 'ramda'
import { all, equals, has, isEmpty, keys, omit, pipe } from 'ramda'
import ValidationReport from 'rdf-validate-shacl/src/validation-report'

import { extractFromFile, read } from '../../archive'
import { MANIFEST_FILE } from '../../asset/constants'
import { countAmountOfFilesInZip, extractFromFile, read } from '../../archive'
import { MANIFEST_FILE, README_FILE } from '../../asset/constants'
import { Manifest } from '../../asset/types'
import { getAllManifestLinksAndFormatPaths, getDomainMetadataPath } from '../../asset/validateAndCreateMetadata.utils'
import { ERRORS } from '../../constants'
import { CONTEXT_DROP_SCHEMAS } from './shacl.constants'
import { ContentType, Schema, ValidationSchema } from './shacl.types'
import { fetchShaclSchema, loadDataset, parseStreamToDataset, validateShacl } from './shacl.utils'
import {
fetchShaclSchema,
formatFilesErrorMessage,
loadDataset,
parseStreamToDataset,
subtractManifestAndReadMeFiles,
validateShacl,
} from './shacl.utils'

export const _validateShaclFile =
({
validateManifest,
validateDomainMetadata,
checkIfAllFilesInManifestExists,
checkIfAllFilesInManifestExist,
countAmountOfFilesInZip,
validateReadme,
}: {
validateManifest: (file: File) => Promise<{ conforms: boolean; data: any }>
validateDomainMetadata: (file: File, manifest: Manifest) => Promise<{ conforms: boolean; data: any }>
checkIfAllFilesInManifestExists: (file: File, manifest: Manifest) => any
checkIfAllFilesInManifestExist: (
file: File,
manifest: Manifest,
) => Promise<{ errors: { error: string }[]; amount: number }>
countAmountOfFilesInZip: (file: File) => Promise<number>
validateReadme: (file: File) => Promise<boolean>
}) =>
async (file: File) => {
try {
const readmeExists = await validateReadme(file)
if (!readmeExists) {
return {
isValid: false,
data: {},
error: ERRORS.README_FILE_NOT_FOUND,
}
}

const { conforms: manifestConforms, data: manifest } = await validateManifest(file)
await checkIfAllFilesInManifestExists(file, manifest)
const manifestFiles = await checkIfAllFilesInManifestExist(file, manifest)

if (!isEmpty(manifestFiles.errors)) {
return {
isValid: false,
data: {},
error: formatFilesErrorMessage(manifestFiles.errors),
}
}

const amountOfFilesInZip = await countAmountOfFilesInZip(file)
const amountWithoutManifestAndReadme = subtractManifestAndReadMeFiles(amountOfFilesInZip)

if (!equals(amountWithoutManifestAndReadme)(manifestFiles.amount)) {
return {
isValid: false,
data: {},
error: `${amountWithoutManifestAndReadme} files found, should be ${manifestFiles.amount} files`,
}
}

const { conforms: domainMetadataConforms, data: domainMetadata } = await validateDomainMetadata(file, manifest)

if (!manifestConforms) {
return { isValid: false, data: {}, error: ERRORS.ASSET_INVALID }
return { isValid: false, data: {}, error: ERRORS.MANIFEST_INVALID }
}

if (!domainMetadataConforms) {
return { isValid: false, data: {}, error: ERRORS.ASSET_INVALID }
return { isValid: false, data: {}, error: ERRORS.DOMAIN_METADATA_INVALID }
}

return { isValid: true, data: { manifest, domainMetadata } }
Expand Down Expand Up @@ -113,6 +155,22 @@ export const validateManifest = _validateManifest({
validateShaclSchema,
})

export const _validateReadme =
({ getShaclDataFromZip }: { getShaclDataFromZip: (file: File, fileName: string) => Promise<string> }) =>
async (file: File) => {
try {
await getShaclDataFromZip(file, README_FILE)

return true
} catch {
return false
}
}

export const validateReadme = _validateReadme({
getShaclDataFromZip,
})

export const _validateDomainMetadata =
({
getShaclDataFromZip,
Expand Down Expand Up @@ -163,23 +221,42 @@ export const validateDomainMetadata = _validateDomainMetadata({
getDomainMetadataPath,
})

export const _checkIfAllFilesInManifestExists =
export const _checkIfAllFilesInManifestExist =
({ getShaclDataFromZip }: { getShaclDataFromZip: (file: File, fileName: string) => Promise<string> }) =>
async (file: File, manifest: Manifest) => {
const files = getAllManifestLinksAndFormatPaths(manifest)
const validationPromises = files.map((fileName: string) => getShaclDataFromZip(file, fileName))
const validationPromises = files.map((fileName: string) => ({
fileName,
promise: getShaclDataFromZip(file, fileName),
}))

return Promise.all(validationPromises)
const wrappedPromises = validationPromises.map(({ fileName, promise }) =>
promise.catch(() => ({ error: fileName })),
)

const errors = await Promise.all(wrappedPromises).then(
(results: (string | { error: string })[]) =>
results.filter((result: string | { error: string }) => has('error')(result) && result.error) as {
error: string
}[],
)

return {
errors,
amount: files.length,
}
}

export const checkIfAllFilesInManifestExists = _checkIfAllFilesInManifestExists({
export const checkIfAllFilesInManifestExist = _checkIfAllFilesInManifestExist({
getShaclDataFromZip,
})

export const validateShaclFile = _validateShaclFile({
validateDomainMetadata,
validateManifest,
checkIfAllFilesInManifestExists,
checkIfAllFilesInManifestExist,
countAmountOfFilesInZip,
validateReadme,
})

export const _validateShaclDataWithSchema =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,26 @@ describe('common/validator/shacl/shacl.utils', () => {
expect(result._readableState.highWaterMark).toEqual(expected)
})
})

describe('formatFilesErrorMessage', () => {
const errors = [
{
error: 'Error 1',
},
{
error: 'Error 2',
},
]

const expected = 'File(s) not found - Error 1, Error 2'
const result = SUT.formatFilesErrorMessage(errors)

expect(result).toEqual(expected)
})

describe('subtractManifestAndReadMeFiles', () => {
const result = SUT.subtractManifestAndReadMeFiles(16)

expect(result).toEqual(14)
})
})
13 changes: 12 additions & 1 deletion apps/envited.ascs.digital/common/validator/shacl/shacl.utils.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import { DatasetCore, Quad } from '@rdfjs/types'
import rdf, { DefaultEnv } from '@zazuko/env'
import { Dataset } from '@zazuko/env/lib/Dataset'
import { flip, join, map, pipe, subtract } from 'ramda'
import rdfParser, { RdfParser } from 'rdf-parse'
import SHACLValidator from 'rdf-validate-shacl'
import { Readable } from 'stream'

import { SCHEMA_MAP } from './shacl.constants'
import { ERRORS } from '../../constants'
import { AMOUNT_OF_UNDEFINED_FILES_IN_MANIFEST, SCHEMA_MAP } from './shacl.constants'
import { ContentType, ValidationSchema } from './shacl.types'

export const validateShacl = (shapes: DatasetCore<Quad, Quad>) => async (data: DatasetCore<Quad, Quad>) => {
Expand Down Expand Up @@ -56,3 +58,12 @@ export const loadDataset = _loadDataset({
createReadableStream,
parseStreamToDataset,
})

export const formatFilesErrorMessage = (errors: { error: string }[]) =>
pipe(
map(({ error }: { error: string }) => error),
join(', '),
(x: string) => `${ERRORS.FILES_NOT_FOUND} - ${x}`,
)(errors)

export const subtractManifestAndReadMeFiles = flip(subtract)(AMOUNT_OF_UNDEFINED_FILES_IN_MANIFEST)

0 comments on commit 93f7c15

Please sign in to comment.