diff --git a/packages/openactive-integration-tests/README.md b/packages/openactive-integration-tests/README.md index d49876a847..29c2c33243 100644 --- a/packages/openactive-integration-tests/README.md +++ b/packages/openactive-integration-tests/README.md @@ -22,11 +22,11 @@ To run `openactive-integration-tests` in separate terminal window to `openactive ### Running specific tests -`npm start-tests -- test/features/core/availability-check/implemented/availability-confirmed-test.js` +`npm run start-tests -- test/features/core/availability-check/implemented/availability-confirmed-test.js` ### Running core tests in a single process -`npm start-tests -- --runInBand test/features/core/` +`npm run start-tests -- --runInBand test/features/core/` ## Configuration for `integrationTests` within `./config/{NODE_ENV}.json` diff --git a/packages/openactive-integration-tests/package-lock.json b/packages/openactive-integration-tests/package-lock.json index 7b349473b7..8fca283cb0 100644 --- a/packages/openactive-integration-tests/package-lock.json +++ b/packages/openactive-integration-tests/package-lock.json @@ -43,6 +43,7 @@ "ramda": "^0.27.1", "rmfr": "^2.0.0", "shortid": "^2.2.16", + "showdown": "^2.1.0", "strip-ansi": "^6.0.0", "uuid": "^8.3.0", "yargs": "^16.2.0" @@ -114,6 +115,7 @@ "eslint-plugin-jest": "^27.9.0", "fast-check": "^3.15.1", "jest": "^29.7.0", + "rimraf": "^5.0.5", "typescript": "^5.1.6" }, @@ -5248,6 +5250,14 @@ "node": ">= 0.8" } }, + "node_modules/commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "engines": { + "node": "^12.20.0 || >=14" + } + }, "node_modules/component-emitter": { "version": "1.3.0", "license": "MIT" @@ -11829,6 +11839,21 @@ "nanoid": "^2.1.0" } }, + "node_modules/showdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz", + "integrity": "sha512-/6NVYu4U819R2pUIk79n67SYgJHWCce0a5xTP979WbNp0FL9MN1I1QK662IDU1b6JzKTvmhgI7T7JYIxBi3kMQ==", + "dependencies": { + "commander": "^9.0.0" + }, + "bin": { + "showdown": "bin/showdown.js" + }, + "funding": { + "type": "individual", + "url": "https://www.paypal.me/tiviesantos" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", diff --git a/packages/openactive-integration-tests/package.json b/packages/openactive-integration-tests/package.json index 01c9c1bdb9..757d8acf61 100644 --- a/packages/openactive-integration-tests/package.json +++ b/packages/openactive-integration-tests/package.json @@ -51,6 +51,7 @@ "ramda": "^0.27.1", "rmfr": "^2.0.0", "shortid": "^2.2.16", + "showdown": "^2.1.0", "strip-ansi": "^6.0.0", "uuid": "^8.3.0", "yargs": "^16.2.0" diff --git a/packages/openactive-integration-tests/test/certification/certification-writer.js b/packages/openactive-integration-tests/test/certification/certification-writer.js index c3fb642ee3..2a19cf2fb8 100644 --- a/packages/openactive-integration-tests/test/certification/certification-writer.js +++ b/packages/openactive-integration-tests/test/certification/certification-writer.js @@ -153,9 +153,9 @@ class CertificationWriter { async generateZip(loggers, generator) { const evidenceFilePaths = [].concat( loggers.map(logger => ({ path: logger.metaPath, zipPath: `json/${logger.metaLocalPath}` })), - loggers.map(logger => ({ path: logger.markdownPath, zipPath: `markdown/${logger.markdownLocalPath}` })), + loggers.map(logger => ({ path: logger.htmlPath, zipPath: `html/${logger.htmlLocalPath}` })), { path: generator.summaryMetaPath, zipPath: 'json/index.json' }, - { path: generator.reportMarkdownPath, zipPath: 'markdown/summary.md' }, + { path: generator.reportHtmlPath, zipPath: 'html/summary.html' }, ); const zip = new JSZip(); diff --git a/packages/openactive-integration-tests/test/helpers/logger.js b/packages/openactive-integration-tests/test/helpers/logger.js index ebedfbdede..3c48f412b5 100644 --- a/packages/openactive-integration-tests/test/helpers/logger.js +++ b/packages/openactive-integration-tests/test/helpers/logger.js @@ -244,12 +244,12 @@ class BaseLogger { return `${OUTPUT_PATH}json/${this.metaLocalPath}`; } - get markdownLocalPath () { - return `${this.uniqueSuiteName.replace(/\s+/g, '_')}.md`; + get htmlLocalPath () { + return `${this.uniqueSuiteName.replace(/\s+/g, '_')}.html`; } - get markdownPath () { - return `${OUTPUT_PATH}${this.markdownLocalPath}`; + get htmlPath () { + return `${OUTPUT_PATH}${this.htmlLocalPath}`; } get validationStatusCounts () { @@ -285,6 +285,18 @@ class BaseLogger { }; } + get specStatusCountsForEachSuiteName() { + if (this._specStatusCountsBySuiteName) return this._specStatusCountsBySuiteName; + + let statusBySuiteName = _.chain(this.logs) + .filter(log => log.type === "spec") + .groupBy(log => log.ancestorTitles.join(" > ")) + .mapValues(logs => _.countBy(logs, log => log.spec.status)) + .value(); + + return this._specStatusCountsBySuiteName = statusBySuiteName; + } + get overallStatus() { let spec = this.specStatusCounts; let validation = this.validationStatusCounts; @@ -454,6 +466,17 @@ class ReporterLogger extends BaseLogger { return result; } + + statusFor (suiteName) { + let specStatusCountsBySuiteName = this.specStatusCountsForEachSuiteName; + let joinedSuiteName = suiteName.join(" > "); + let spec = specStatusCountsBySuiteName[joinedSuiteName]; + if (spec) { + if (spec.failed > 0) return "failed"; + return "passed" + } + return ""; + } } /** diff --git a/packages/openactive-integration-tests/test/report-generator.js b/packages/openactive-integration-tests/test/report-generator.js index 5e3a47bb9e..913dda24b0 100644 --- a/packages/openactive-integration-tests/test/report-generator.js +++ b/packages/openactive-integration-tests/test/report-generator.js @@ -5,6 +5,7 @@ const stripAnsi = require("strip-ansi"); const {ReporterLogger} = require("./helpers/logger"); const _ = require("lodash"); const { getConfigVarOrThrow } = require('./helpers/config-utils'); +const showdown = require('showdown'); const USE_RANDOM_OPPORTUNITIES = getConfigVarOrThrow('integrationTests', 'useRandomOpportunities'); const OUTPUT_PATH = getConfigVarOrThrow('integrationTests', 'outputPath'); @@ -126,6 +127,10 @@ class BaseReportGenerator { return ret; }, + "statusFor": (suite) => { + let status = this.logger.statusFor(suite); + return status; + }, "eachSorted": (context, options) => { var ret = ""; Object.keys(context).sort().forEach(function(key) { @@ -140,7 +145,7 @@ class BaseReportGenerator { return {}; } - get reportMarkdownPath () { + get reportHtmlPath () { throw "Not Implemented"; } @@ -160,7 +165,7 @@ class BaseReportGenerator { } } - async writeMarkdown () { + async writeHtml () { let template = await this.getTemplate(`${this.templateName}.md`); let data = template(this.templateData, { @@ -169,12 +174,19 @@ class BaseReportGenerator { helpers: this.helpers, }); - await fs.writeFile(this.reportMarkdownPath, data); + const converter = new showdown.Converter(); + converter.setOption('completeHTMLDocument', true); + converter.setOption('moreStyling', true) + converter.setOption('openLinksInNewWindow', true) + const html = converter.makeHtml(data); + + + await fs.writeFile(this.reportHtmlPath, html); } async report(silentOnConsole) { if (!silentOnConsole) await this.outputConsole(); - await this.writeMarkdown(); + await this.writeHtml(); } async getTemplate (name) { @@ -199,8 +211,8 @@ class ReportGenerator extends BaseReportGenerator { return this.logger; } - get reportMarkdownPath () { - return this.logger.markdownPath; + get reportHtmlPath () { + return this.logger.htmlPath; } } @@ -274,8 +286,8 @@ class SummaryReportGenerator extends BaseReportGenerator { return `${OUTPUT_PATH}json/summary.json`; } - get reportMarkdownPath () { - return `${OUTPUT_PATH}summary.md`; + get reportHtmlPath () { + return `${OUTPUT_PATH}summary.html`; } get opportunityTypeGroups () { diff --git a/packages/openactive-integration-tests/test/report-templates/report-cli.handlebars b/packages/openactive-integration-tests/test/report-templates/report-cli.handlebars index 580a4fc4e6..46bb563f38 100644 --- a/packages/openactive-integration-tests/test/report-templates/report-cli.handlebars +++ b/packages/openactive-integration-tests/test/report-templates/report-cli.handlebars @@ -6,7 +6,7 @@ Feature Implemented: {{{ implemented }}} {{ consoleValidationIcon overallStatus }} {{ numPassed }} passed with {{ numFailed }} {{ pluralise "failure" numFailed }}, {{ numWarnings }} {{ pluralise "warning" numWarnings }} and {{{ numSuggestions }}} {{ pluralise "suggestion" numSuggestions }} -See `{{{ markdownPath }}}` for more detailed info. +See `{{{ htmlPath }}}` for more detailed info. {{#each activeSuites }} {{#each . }}{{# chalk "bold" "yellow" }}>>{{/chalk}} {{# chalk "bold" "green" }}{{{ . }}}{{/chalk}} {{/each}} diff --git a/packages/openactive-integration-tests/test/report-templates/report.md.handlebars b/packages/openactive-integration-tests/test/report-templates/report.md.handlebars index c516956b46..a21944a579 100644 --- a/packages/openactive-integration-tests/test/report-templates/report.md.handlebars +++ b/packages/openactive-integration-tests/test/report-templates/report.md.handlebars @@ -1,4 +1,25 @@ -[< Return to Summary](summary.md) | File Generated: {{{ timestamp }}} +
+ + + + +[< Return to Summary](summary.html) | File Generated: {{{ timestamp }}} + + + + + # {{{ title }}} @@ -30,7 +51,7 @@ The [OpenActive Reference Implementation test result for this test]({{{ referenc {{#each activeSuites }} -## {{{ renderSuiteName . }}} +## {{ validationIcon (statusFor . ) }} {{{ renderSuiteName . }}} {{#logsFor . "information"}} ### {{title}} @@ -141,3 +162,104 @@ Test could not be completed as it timed out while a request was still pending: {{/if~}} {{/logsFor}} + + + + + + diff --git a/packages/openactive-integration-tests/test/report-templates/summary-cli.handlebars b/packages/openactive-integration-tests/test/report-templates/summary-cli.handlebars index 2b7ca22ef2..1b48a87b90 100644 --- a/packages/openactive-integration-tests/test/report-templates/summary-cli.handlebars +++ b/packages/openactive-integration-tests/test/report-templates/summary-cli.handlebars @@ -1,6 +1,6 @@ Summary -See `{{{ reportMarkdownPath }}}` for a Markdown version. +See `{{{ reportHtmlPath }}}` for a Markdown version. {{#eachSorted opportunityTypeGroups }} {{# chalk "bold"}}{{{ opportunityTypeName }}}{{/chalk}} @@ -9,7 +9,7 @@ See `{{{ reportMarkdownPath }}}` for a Markdown version. {{#each featureGroups }} - {{{ consoleValidationIcon overallStatus }}} {{{ featureName }}} ({{implementedDisplayLabel}}) {{#each loggers}} - - {{{ consoleValidationIcon overallStatus }}} {{{ suiteName }}} ({{{ numFailed }}} failures, {{{ numWarnings }}} warnings, {{{ numSuggestions }}} suggestions, {{{ numPassed }}} passes): {{{ markdownPath }}} + - {{{ consoleValidationIcon overallStatus }}} {{{ suiteName }}} ({{{ numFailed }}} failures, {{{ numWarnings }}} warnings, {{{ numSuggestions }}} suggestions, {{{ numPassed }}} passes): {{{ htmlPath }}} {{/each}} {{/each}} diff --git a/packages/openactive-integration-tests/test/report-templates/summary.md.handlebars b/packages/openactive-integration-tests/test/report-templates/summary.md.handlebars index 9c88561516..10241f315b 100644 --- a/packages/openactive-integration-tests/test/report-templates/summary.md.handlebars +++ b/packages/openactive-integration-tests/test/report-templates/summary.md.handlebars @@ -14,7 +14,7 @@ Mode: **{{ useRandomOpportunitiesMode }}** {{#each featureGroups }} * {{{ validationIcon overallStatus }}} {{{ featureName }}} ({{implementedDisplayLabel}}) {{#each loggers}} - - {{{ validationIcon overallStatus }}} [{{{ suiteName }}}]({{{ markdownLocalPath }}}): ({{{ numFailed }}} failures, {{{ numWarnings }}} warnings, {{{ numSuggestions }}} suggestions, {{{ numPassed }}} passes) + - {{{ validationIcon overallStatus }}} [{{{ suiteName }}}]({{{ htmlLocalPath }}}): ({{{ numFailed }}} failures, {{{ numWarnings }}} warnings, {{{ numSuggestions }}} suggestions, {{{ numPassed }}} passes) {{/each}} {{/each}}