diff --git a/package.json b/package.json index a40fbd154..45257eb88 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,9 @@ "compose:postgres": "docker-compose -f docker-compose.test.yml up -d postgres", "compose:build": "docker-compose -f docker-compose.test.yml build", "compose:down": "docker-compose -f docker-compose.test.yml down", - "prettify": "balena-lint -e js -e ts --fix src build typings Gruntfile.ts" + "prettify": "balena-lint -e js -e ts --fix src build typings Gruntfile.ts", + "serve-openapi": "redoc-cli serve --ssr ./example-openapi.json", + "build-openapi": "redoc-cli build ./example-openapi.json" }, "dependencies": { "@balena/abstract-sql-compiler": "^7.20.0", @@ -61,6 +63,7 @@ "express-session": "^1.17.3", "lodash": "^4.17.21", "memoizee": "^0.4.15", + "odata-openapi": "^0.18.1", "pinejs-client-core": "^6.10.2", "randomstring": "^1.2.2", "typed-error": "^3.2.1" @@ -91,6 +94,7 @@ "load-grunt-tasks": "^5.1.0", "mocha": "^9.2.2", "raw-loader": "^4.0.2", + "redoc-cli": "^0.13.18", "require-npm4-to-publish": "^1.0.0", "supertest": "^6.2.4", "terser-webpack-plugin": "^5.3.3", diff --git a/src/odata-metadata/odata-metadata-generator.ts b/src/odata-metadata/odata-metadata-generator.ts index f36adfa2d..3da3f6c10 100644 --- a/src/odata-metadata/odata-metadata-generator.ts +++ b/src/odata-metadata/odata-metadata-generator.ts @@ -4,10 +4,105 @@ import type { } from '@balena/abstract-sql-compiler'; import * as sbvrTypes from '@balena/sbvr-types'; +import { PermissionLookup } from '../sbvr-api/permissions'; +import * as odataMetadata from 'odata-openapi'; // tslint:disable-next-line:no-var-requires const { version }: { version: string } = require('../../package.json'); +type dict = { [key: string]: any }; +interface OdataCsdl { + $Version: string; + $EntityContainer: string; + [key: string]: any; +} + +interface ODataNameSpaceType { + $Alias: string; + '@Core.DefaultNamespace': boolean; + [key: string]: any; +} +interface ODataEntityContainerType { + $Kind: 'EntityContainer'; + [key: string]: any; +} + +interface ODataEntityContainerEntryType { + $Kind: 'EntityType' | 'ComplexType' | 'NavigationProperty'; + [key: string]: any; +} + +interface AbstractModel { + abstractSqlModel: AbstractSqlModel; + permissionLookup: PermissionLookup; +} + +/** OData JSON v4 CSDL Vocabulary constants + * + * http://docs.oasis-open.org/odata/odata-vocabularies/v4.0/odata-vocabularies-v4.0.html + * + */ + +const odataVocabularyReferences = { + 'https://oasis-tcs.github.io/odata-vocabularies/vocabularies/Org.OData.Core.V1.json': + { + $Include: [ + { + $Namespace: 'Org.OData.Core.V1', + $Alias: 'Core', + '@Core.DefaultNamespace': true, + }, + ], + }, + 'https://oasis-tcs.github.io/odata-vocabularies/vocabularies/Org.OData.Measures.V1.json': + { + $Include: [ + { + $Namespace: 'Org.OData.Measures.V1', + $Alias: 'Measures', + }, + ], + }, + 'https://oasis-tcs.github.io/odata-vocabularies/vocabularies/Org.OData.Aggregation.V1.json': + { + $Include: [ + { + $Namespace: 'Org.OData.Aggregation.V1', + $Alias: 'Aggregation', + }, + ], + }, + 'https://oasis-tcs.github.io/odata-vocabularies/vocabularies/Org.OData.Capabilities.V1.json': + { + $Include: [ + { + $Namespace: 'Org.OData.Capabilities.V1', + $Alias: 'Capabilities', + }, + ], + }, +}; + +// https://github.com/oasis-tcs/odata-vocabularies/blob/main/vocabularies/Org.OData.Capabilities.V1.md +const restrictionsLookup = { + update: { + capability: 'UpdateRestrictions', + ValueIdentifier: 'Updatable', + }, + delete: { + capability: 'DeleteRestrictions', + ValueIdentifier: 'Deletable', + }, + create: { + capability: 'InsertRestrictions', + ValueIdentifier: 'Insertable', + }, + read: { + capability: 'ReadRestrictions', + ValueIdentifier: 'Readable', + }, +}; + const getResourceName = (resourceName: string): string => resourceName .split('-') @@ -15,17 +110,25 @@ const getResourceName = (resourceName: string): string => .join('__'); const forEachUniqueTable = ( - model: AbstractSqlModel['tables'], - callback: (tableName: string, table: AbstractSqlTable) => T, + model: AbstractModel, + callback: ( + tableName: string, + table: AbstractSqlTable & { referenceScheme: string }, + ) => T, ): T[] => { const usedTableNames: { [tableName: string]: true } = {}; const result = []; - for (const [key, table] of Object.entries(model)) { + + for (const key of Object.keys(model.abstractSqlModel.tables)) { + const table = model.abstractSqlModel.tables[key] as AbstractSqlTable & { + referenceScheme: string; + }; if ( typeof table !== 'string' && !table.primitive && - !usedTableNames[table.name] + !usedTableNames[table.name] && + model.permissionLookup ) { usedTableNames[table.name] = true; result.push(callback(key, table)); @@ -34,9 +137,48 @@ const forEachUniqueTable = ( return result; }; +/** + * parsing dictionary of vocabulary.resource.operation permissions string + * into dictionary of resource to operation for later lookup + */ +const preparePermissionsLookup = (permissionLookup: PermissionLookup): dict => { + const pathsAndOps: dict = {}; + + for (const pathOpsAuths of Object.keys(permissionLookup)) { + const [vocabulary, path, rule] = pathOpsAuths.split('.'); + + pathsAndOps[vocabulary] = Object.assign( + { [path]: {} }, + pathsAndOps[vocabulary], + ); + if (rule === 'all') { + pathsAndOps[vocabulary][path] = Object.assign( + { + ['read']: true, + ['create']: true, + ['update']: true, + ['delete']: true, + }, + pathsAndOps[vocabulary][path], + ); + } else if (rule === undefined) { + // just true no operation to be named + pathsAndOps[vocabulary][path] = true; + } else { + pathsAndOps[vocabulary][path] = Object.assign( + { [rule]: true }, + pathsAndOps[vocabulary][path], + ); + } + } + + return pathsAndOps; +}; + export const generateODataMetadata = ( vocabulary: string, abstractSqlModel: AbstractSqlModel, + permissionsLookup?: PermissionLookup, ) => { const complexTypes: { [fieldType: string]: string } = {}; const resolveDataType = (fieldType: string): string => { @@ -51,132 +193,189 @@ export const generateODataMetadata = ( return sbvrTypes[fieldType].types.odata.name; }; - const model = abstractSqlModel.tables; - const associations: Array<{ - name: string; - ends: Array<{ - resourceName: string; - cardinality: '1' | '0..1' | '*'; - }>; - }> = []; - forEachUniqueTable(model, (_key, { name: resourceName, fields }) => { - resourceName = getResourceName(resourceName); - for (const { dataType, required, references } of fields) { - if (dataType === 'ForeignKey' && references != null) { - const { resourceName: referencedResource } = references; - associations.push({ - name: resourceName + referencedResource, - ends: [ - { resourceName, cardinality: required ? '1' : '0..1' }, - { resourceName: referencedResource, cardinality: '*' }, - ], - }); - } - } - }); + const prepPermissionsLookup = permissionsLookup + ? preparePermissionsLookup(permissionsLookup) + : {}; - return ( - ` - - - - - - ` + - forEachUniqueTable( - model, - (_key, { idField, name: resourceName, fields }) => { - resourceName = getResourceName(resourceName); - return ( - ` - - - - - - ` + - fields - .filter(({ dataType }) => dataType !== 'ForeignKey') - .map(({ dataType, fieldName, required }) => { - dataType = resolveDataType(dataType); - fieldName = getResourceName(fieldName); - return ``; - }) - .join('\n') + - '\n' + - fields - .filter( - ({ dataType, references }) => - dataType === 'ForeignKey' && references != null, - ) - .map(({ fieldName, references }) => { - const { resourceName: referencedResource } = references!; - fieldName = getResourceName(fieldName); - return ``; - }) - .join('\n') + - '\n' + - ` - ` - ); - }, - ).join('\n\n') + - associations - .map(({ name, ends }) => { - name = getResourceName(name); - return ( - `` + - '\n\t' + - ends - .map( - ({ resourceName, cardinality }) => - ``, - ) - .join('\n\t') + - '\n' + - `` - ); - }) - .join('\n') + - ` - + const model: AbstractModel = { + abstractSqlModel, + permissionLookup: + prepPermissionsLookup[vocabulary] ?? prepPermissionsLookup['resource'], + }; - ` + - forEachUniqueTable(model, (_key, { name: resourceName }) => { + let metaBalena: ODataNameSpaceType = { + $Alias: vocabulary, + '@Core.DefaultNamespace': true, + }; + + let metaBalenaEntries: dict = {}; + let entityContainerEntries: dict = {}; + forEachUniqueTable( + model, + (_key, { idField, name: resourceName, fields, referenceScheme }) => { resourceName = getResourceName(resourceName); - return ``; - }).join('\n') + - '\n' + - associations - .map(({ name, ends }) => { - name = getResourceName(name); - return ( - `` + - '\n\t' + - ends - .map( - ({ resourceName }) => - ``, - ) - .join('\n\t') + - ` - ` + // no path nor entity when permissions not contain resource + if ( + !model?.permissionLookup?.[resourceName] && + !(model?.permissionLookup?.['all'] === true) + ) { + return; + } + + const uniqueTable: ODataEntityContainerEntryType = { + $Kind: 'EntityType', + $Key: [idField], + '@Core.LongDescription': + '{"x-ref-scheme": ["' + referenceScheme + '"]}', + }; + + fields + .filter(({ dataType }) => dataType !== 'ForeignKey') + .map(({ dataType, fieldName, required }) => { + dataType = resolveDataType(dataType); + fieldName = getResourceName(fieldName); + + uniqueTable[fieldName] = { + $Type: dataType, + $Nullable: !required, + '@Core.Computed': + fieldName === 'created_at' || fieldName === 'modified_at' + ? true + : false, + }; + }); + + fields + .filter( + ({ dataType, references }) => + dataType === 'ForeignKey' && references != null, + ) + .map(({ fieldName, references, required }) => { + const { resourceName: referencedResource } = references!; + const referencedResourceName = + model.abstractSqlModel.tables[referencedResource]?.name; + const typeReference = referencedResourceName || referencedResource; + + fieldName = getResourceName(fieldName); + uniqueTable[fieldName] = { + $Kind: 'NavigationProperty', + $Partner: resourceName, + $Nullable: !required, + $Type: vocabulary + '.' + getResourceName(typeReference), + }; + }); + + metaBalenaEntries[resourceName] = uniqueTable; + + entityContainerEntries[resourceName] = { + $Collection: true, + $Type: vocabulary + '.' + resourceName, + }; + + for (const [key, value] of Object.entries(restrictionsLookup)) { + let capabilitiesEnabled = false; + if ( + model?.permissionLookup?.[resourceName]?.hasOwnProperty(key) || + model?.permissionLookup?.['all'] === true + ) { + capabilitiesEnabled = true; + } + const restriction = { + ['@Capabilities.' + value.capability]: { + [value.ValueIdentifier]: capabilitiesEnabled, + }, + }; + + entityContainerEntries[resourceName] = Object.assign( + entityContainerEntries[resourceName], + restriction, ); - }) - .join('\n') + - ` - ` + - Object.values(complexTypes).join('\n') + - ` - - - ` + } + }, + ); + + metaBalenaEntries = Object.keys(metaBalenaEntries) + .sort() + .reduce((r, k) => ((r[k] = metaBalenaEntries[k]), r), {} as dict); + + metaBalena = { ...metaBalena, ...metaBalenaEntries }; + + let oDataApi: ODataEntityContainerType = { + $Kind: 'EntityContainer', + '@Capabilities.BatchSupported': false, + }; + + const odataCsdl: OdataCsdl = { + $Version: '4.01', // because of odata2openapi transformer has a hacky switch on === 4.0 that we don't want. Other checks are checking for >=4.0. + $EntityContainer: vocabulary + '.ODataApi', + $Reference: odataVocabularyReferences, + }; + + entityContainerEntries = Object.keys(entityContainerEntries) + .sort() + .reduce((r, k) => ((r[k] = entityContainerEntries[k]), r), {} as dict); + + oDataApi = { ...oDataApi, ...entityContainerEntries }; + + metaBalena['ODataApi'] = oDataApi; + + odataCsdl[vocabulary] = metaBalena; + + return odataCsdl; +}; + +export const generateODataOpenAPI = ( + vocabulary: string, + abstractSqlModel: AbstractSqlModel, + permissionsLookup?: PermissionLookup, + versionBasePathUrl: string = '', + hostname: string = '', +) => { + const odataCsdl = generateODataMetadata( + vocabulary, + abstractSqlModel, + permissionsLookup, ); + const openAPIJson: any = odataMetadata.csdl2openapi(odataCsdl, { + scheme: 'https', + host: hostname, + basePath: versionBasePathUrl, + diagram: false, + maxLevels: 5, + }); + + /** + * HACK + * Rewrite odata body response schema properties from `value:` to `d:` + * Currently pinejs is returning `d:` + * https://www.odata.org/documentation/odata-version-2-0/json-format/ (6. Representing Collections of Entries) + * https://www.odata.org/documentation/odata-version-3-0/json-verbose-format/ (6.1 Response body) + * + * New v4 odata specifies the body response with `value:` + * http://docs.oasis-open.org/odata/odata-json-format/v4.01/odata-json-format-v4.01.html#sec_IndividualPropertyorOperationRespons + * + * Used oasis translator generates openapi according to v4 spec (`value:`) + */ + + Object.keys(openAPIJson.paths).forEach((i) => { + if ( + openAPIJson?.paths[i]?.get?.responses?.['200']?.content?.[ + 'application/json' + ]?.schema?.properties?.value + ) { + openAPIJson.paths[i].get.responses['200'].content[ + 'application/json' + ].schema.properties['d'] = + openAPIJson.paths[i].get.responses['200'].content[ + 'application/json' + ].schema.properties.value; + delete openAPIJson.paths[i].get.responses['200'].content[ + 'application/json' + ].schema.properties.value; + } + }); + + return openAPIJson; }; generateODataMetadata.version = version; diff --git a/src/sbvr-api/permissions.ts b/src/sbvr-api/permissions.ts index 1227be777..d40e066e0 100644 --- a/src/sbvr-api/permissions.ts +++ b/src/sbvr-api/permissions.ts @@ -22,7 +22,7 @@ import type { } from '@balena/odata-parser'; import type { Tx } from '../database-layer/db'; import type { ApiKey, User } from '../sbvr-api/sbvr-utils'; -import type { AnyObject, Dictionary } from './common-types'; +import type { AnyObject } from './common-types'; import { isBindReference, @@ -312,7 +312,7 @@ const namespaceRelationships = ( }); }; -type PermissionLookup = Dictionary; +export type PermissionLookup = _.Dictionary; const getPermissionsLookup = env.createCache( 'permissionsLookup', @@ -1610,7 +1610,7 @@ const getGuestPermissions = memoize( { promise: true }, ); -const getReqPermissions = async ( +export const getReqPermissions = async ( req: PermissionReq, odataBinds: ODataBinds = [], ) => { diff --git a/src/sbvr-api/sbvr-utils.ts b/src/sbvr-api/sbvr-utils.ts index dbc7a84d4..8835dca08 100644 --- a/src/sbvr-api/sbvr-utils.ts +++ b/src/sbvr-api/sbvr-utils.ts @@ -34,7 +34,10 @@ import { PinejsClientCore, PromiseResultTypes } from 'pinejs-client-core'; import { ExtendedSBVRParser } from '../extended-sbvr-parser/extended-sbvr-parser'; import * as syncMigrator from '../migrator/sync'; -import { generateODataMetadata } from '../odata-metadata/odata-metadata-generator'; +import { + generateODataMetadata, + generateODataOpenAPI, +} from '../odata-metadata/odata-metadata-generator'; // tslint:disable-next-line:no-var-requires const devModel = require('./dev.sbvr'); @@ -1533,11 +1536,34 @@ const respondGet = async ( return response; } else { if (request.resourceName === '$metadata') { - return { - statusCode: 200, - body: models[vocab].odataMetadata, - headers: { 'content-type': 'xml' }, - }; + const { openapi } = req.body; + const permLookup = await permissions.getReqPermissions(req); + let specJson = {}; + if (openapi) { + specJson = generateODataOpenAPI( + vocab, + models[vocab].abstractSql, + permLookup, + req.originalUrl.replace('/$metadata', ''), + req.hostname, + ); + return { + statusCode: 200, + body: specJson, + headers: { 'content-type': 'application/json' }, + }; + } else { + specJson = generateODataMetadata( + vocab, + models[vocab].abstractSql, + permLookup, + ); + return { + statusCode: 200, + body: models[vocab].odataMetadata, + headers: { 'content-type': 'application/json' }, + }; + } } else { // TODO: request.resourceName can be '$serviceroot' or a resource and we should return an odata xml document based on that return { @@ -1547,6 +1573,8 @@ const respondGet = async ( } }; +// paths./any/.get.responses.200.content.application/json.schema.d + const runPost = async ( _req: Express.Request, request: uriParser.ODataRequest, diff --git a/test/03-metadata.test.ts b/test/03-metadata.test.ts new file mode 100644 index 000000000..ff7a1c540 --- /dev/null +++ b/test/03-metadata.test.ts @@ -0,0 +1,38 @@ +import * as fs from 'fs'; +import { expect } from 'chai'; +import * as supertest from 'supertest'; +const fixturePath = __dirname + '/fixtures/03-metadata/config'; +import { testInit, testDeInit, testLocalServer } from './lib/test-init'; + +describe('00 basic tests', function () { + let pineServer: Awaited>; + before(async () => { + pineServer = await testInit(fixturePath); + }); + + after(async () => { + await testDeInit(pineServer); + }); + + describe('Basic', () => { + it('check /ping route is OK', async () => { + await supertest(testLocalServer).get('/ping').expect(200, 'OK'); + }); + }); + + // TODO Check deprecation for endpoints + + describe('get metadata', () => { + it('check /example/$metadata is served by pinejs', async () => { + const res = await supertest(testLocalServer) + .get('/example/$metadata') + .send({ openapi: true }) + .expect(200); + expect(res.body.paths).to.be.an('object'); + console.log(res.body.paths); + expect(res.body.paths).to.have.property('/device'); + expect(res.body.paths).to.have.property('/application'); + await fs.writeFileSync('example-openapi.json', JSON.stringify(res.body)); + }); + }); +}); diff --git a/test/fixtures/03-metadata/config.ts b/test/fixtures/03-metadata/config.ts new file mode 100644 index 000000000..a68c04631 --- /dev/null +++ b/test/fixtures/03-metadata/config.ts @@ -0,0 +1,18 @@ +import type { ConfigLoader } from '../../../src/server-glue/module'; + +export default { + models: [ + { + apiRoot: 'example', + modelFile: __dirname + '/example.sbvr', + modelName: 'example', + }, + ], + users: [ + { + username: 'guest', + password: ' ', + permissions: ['resource.all'], + }, + ], +} as ConfigLoader.Config; diff --git a/test/fixtures/03-metadata/example.sbvr b/test/fixtures/03-metadata/example.sbvr new file mode 100644 index 000000000..dfce088b5 --- /dev/null +++ b/test/fixtures/03-metadata/example.sbvr @@ -0,0 +1,36 @@ +Vocabulary: example + +Term: name + Concept Type: Short Text (Type) + +Term: note + Concept Type: Text (Type) + +Term: type + Concept Type: Short Text (Type) + + +Term: application + +Fact Type: application has name + Necessity: each application has at most one name. + +Fact Type: application has note + Necessity: each application has at most one note. + +Fact Type: application has type + Necessity: each application has exactly one type. + +Term: device + +Fact Type: device has name + Necessity: each device has at most one name. + +Fact Type: device has note + Necessity: each device has at most one note. + +Fact Type: device has type + Necessity: each device has exactly one type. + +Fact Type: device belongs to application + Necessity: each device belongs to exactly one application diff --git a/test/fixtures/03-metadata/openapi.json b/test/fixtures/03-metadata/openapi.json new file mode 100644 index 000000000..47f402d64 --- /dev/null +++ b/test/fixtures/03-metadata/openapi.json @@ -0,0 +1,23 @@ +{ + "req": { + "method": "GET", + "url": "http://localhost:1337/example/$metadata", + "data": { + "openapi": true + }, + "headers": { + "content-type": "application/json" + } + }, + "header": { + "x-powered-by": "Express", + "cache-control": "no-cache", + "content-type": "application/json; charset=utf-8", + "content-length": "2139", + "etag": "W/\"85b-bxp1fjUPz+3tZ4a5Ox6dkCS1b3c\"", + "date": "Mon, 25 Jul 2022 10:13:47 GMT", + "connection": "close" + }, + "status": 200, + "text": "{\"openapi\":\"3.0.2\",\"info\":{\"title\":\"Service for namespace example\",\"description\":\"This service is located at [https://localhost/example/](https://localhost/example/)\",\"version\":\"\"},\"servers\":[{\"url\":\"https://localhost/example\"}],\"tags\":[],\"paths\":{},\"components\":{\"schemas\":{\"count\":{\"anyOf\":[{\"type\":\"integer\",\"minimum\":0},{\"type\":\"string\"}],\"description\":\"The number of entities in the collection. Available when using the [$count](http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_SystemQueryOptioncount) query option.\"},\"error\":{\"type\":\"object\",\"required\":[\"error\"],\"properties\":{\"error\":{\"type\":\"object\",\"required\":[\"code\",\"message\"],\"properties\":{\"code\":{\"type\":\"string\"},\"message\":{\"type\":\"string\"},\"target\":{\"type\":\"string\"},\"details\":{\"type\":\"array\",\"items\":{\"type\":\"object\",\"required\":[\"code\",\"message\"],\"properties\":{\"code\":{\"type\":\"string\"},\"message\":{\"type\":\"string\"},\"target\":{\"type\":\"string\"}}}},\"innererror\":{\"type\":\"object\",\"description\":\"The structure of this object is service-specific\"}}}}}},\"parameters\":{\"top\":{\"name\":\"top\",\"in\":\"query\",\"description\":\"Show only the first n items, see [Paging - Top](http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_SystemQueryOptiontop)\",\"schema\":{\"type\":\"integer\",\"minimum\":0},\"example\":50},\"skip\":{\"name\":\"skip\",\"in\":\"query\",\"description\":\"Skip the first n items, see [Paging - Skip](http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_SystemQueryOptionskip)\",\"schema\":{\"type\":\"integer\",\"minimum\":0}},\"count\":{\"name\":\"count\",\"in\":\"query\",\"description\":\"Include count of items, see [Count](http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_SystemQueryOptioncount)\",\"schema\":{\"type\":\"boolean\"}},\"search\":{\"name\":\"search\",\"in\":\"query\",\"description\":\"Search items by search phrases, see [Searching](http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_SystemQueryOptionsearch)\",\"schema\":{\"type\":\"string\"}}},\"responses\":{\"error\":{\"description\":\"Error\",\"content\":{\"application/json\":{\"schema\":{\"$ref\":\"#/components/schemas/error\"}}}}}}}" +} \ No newline at end of file diff --git a/typings/odata-openapi.d.ts b/typings/odata-openapi.d.ts new file mode 100644 index 000000000..b91ee894d --- /dev/null +++ b/typings/odata-openapi.d.ts @@ -0,0 +1,6 @@ +declare module 'odata-openapi' { + export const csdl2openapi: ( + csdl, + { scheme, host, basePath, diagram, maxLevels } = {}, + ) => object; +}