feat: change test outputs to be html and add toggle buttons (#630)

* feat: change test outputs to be html and add toggle buttons

* uninstall marked

* fix typo

* fix html bugs and add report status icons

* reenable tests

* add h2 css borders

packages/openactive-integration-tests/README.md

@@ -22,11 +22,11 @@ To run `openactive-integration-tests` in separate terminal window to `openactive
 
 ### Running specific tests
 
-`npm start-tests -- test/features/core/availability-check/implemented/availability-confirmed-test.js`
+`npm run start-tests -- test/features/core/availability-check/implemented/availability-confirmed-test.js`
 
 ### Running core tests in a single process
 
-`npm start-tests -- --runInBand test/features/core/`
+`npm run start-tests -- --runInBand test/features/core/`
 
 ## Configuration for `integrationTests` within `./config/{NODE_ENV}.json`
 

packages/openactive-integration-tests/package.json

@@ -51,6 +51,7 @@
     "ramda": "^0.27.1",
     "rmfr": "^2.0.0",
     "shortid": "^2.2.16",
+    "showdown": "^2.1.0",
     "strip-ansi": "^6.0.0",
     "uuid": "^8.3.0",
     "yargs": "^16.2.0"

packages/openactive-integration-tests/test/certification/certification-writer.js

@@ -153,9 +153,9 @@ class CertificationWriter {
   async generateZip(loggers, generator) {
     const evidenceFilePaths = [].concat(
       loggers.map(logger => ({ path: logger.metaPath, zipPath: `json/${logger.metaLocalPath}` })),
-      loggers.map(logger => ({ path: logger.markdownPath, zipPath: `markdown/${logger.markdownLocalPath}` })),
+      loggers.map(logger => ({ path: logger.htmlPath, zipPath: `html/${logger.htmlLocalPath}` })),
       { path: generator.summaryMetaPath, zipPath: 'json/index.json' },
-      { path: generator.reportMarkdownPath, zipPath: 'markdown/summary.md' },
+      { path: generator.reportHtmlPath, zipPath: 'html/summary.html' },
     );
 
     const zip = new JSZip();

packages/openactive-integration-tests/test/helpers/logger.js

@@ -244,12 +244,12 @@ class BaseLogger {
     return `${OUTPUT_PATH}json/${this.metaLocalPath}`;
   }
 
-  get markdownLocalPath () {
-    return `${this.uniqueSuiteName.replace(/\s+/g, '_')}.md`;
+  get htmlLocalPath () {
+    return `${this.uniqueSuiteName.replace(/\s+/g, '_')}.html`;
   }
 
-  get markdownPath () {
-    return `${OUTPUT_PATH}${this.markdownLocalPath}`;
+  get htmlPath () {
+    return `${OUTPUT_PATH}${this.htmlLocalPath}`;
   }
 
   get validationStatusCounts () {
@@ -285,6 +285,18 @@ class BaseLogger {
     };
   }
 
+  get specStatusCountsForEachSuiteName()  {
+    if (this._specStatusCountsBySuiteName) return this._specStatusCountsBySuiteName;
+
+    let statusBySuiteName =  _.chain(this.logs)
+    .filter(log => log.type === "spec")
+    .groupBy(log => log.ancestorTitles.join(" > "))
+    .mapValues(logs => _.countBy(logs, log => log.spec.status))
+    .value();
+
+    return this._specStatusCountsBySuiteName = statusBySuiteName;
+  }
+
   get overallStatus() {
     let spec = this.specStatusCounts;
     let validation = this.validationStatusCounts;
@@ -454,6 +466,17 @@ class ReporterLogger extends BaseLogger {
 
     return result;
   }
+
+  statusFor (suiteName) {
+    let specStatusCountsBySuiteName = this.specStatusCountsForEachSuiteName;
+    let joinedSuiteName = suiteName.join(" > ");
+    let spec = specStatusCountsBySuiteName[joinedSuiteName];
+    if (spec) {
+      if (spec.failed > 0) return "failed";
+      return "passed"
+    }
+    return "";
+  }
 }
 
 /**

packages/openactive-integration-tests/test/report-generator.js

@@ -5,6 +5,7 @@ const stripAnsi = require("strip-ansi");
 const {ReporterLogger} = require("./helpers/logger");
 const _ = require("lodash");
 const { getConfigVarOrThrow } = require('./helpers/config-utils');
+const showdown = require('showdown');
 
 const USE_RANDOM_OPPORTUNITIES = getConfigVarOrThrow('integrationTests', 'useRandomOpportunities');
 const OUTPUT_PATH = getConfigVarOrThrow('integrationTests', 'outputPath');
@@ -126,6 +127,10 @@ class BaseReportGenerator {
 
         return ret;
       },
+      "statusFor": (suite) => {
+        let status = this.logger.statusFor(suite);
+        return status;
+      },
       "eachSorted": (context, options) => {
         var ret = "";
         Object.keys(context).sort().forEach(function(key) {
@@ -140,7 +145,7 @@ class BaseReportGenerator {
     return {};
   }
 
-  get reportMarkdownPath () {
+  get reportHtmlPath () {
     throw "Not Implemented";
   }
 
@@ -160,7 +165,7 @@ class BaseReportGenerator {
     }
   }
 
-  async writeMarkdown () {
+  async writeHtml () {
     let template = await this.getTemplate(`${this.templateName}.md`);
 
     let data = template(this.templateData, {
@@ -169,12 +174,19 @@ class BaseReportGenerator {
       helpers: this.helpers,
     });
 
-    await fs.writeFile(this.reportMarkdownPath, data);
+    const converter = new showdown.Converter();
+    converter.setOption('completeHTMLDocument', true);
+    converter.setOption('moreStyling', true)
+    converter.setOption('openLinksInNewWindow', true)
+    const html = converter.makeHtml(data);
+
+
+    await fs.writeFile(this.reportHtmlPath, html);
   }
 
   async report(silentOnConsole) {
     if (!silentOnConsole) await this.outputConsole();
-    await this.writeMarkdown();
+    await this.writeHtml();
   }
 
   async getTemplate (name) {
@@ -199,8 +211,8 @@ class ReportGenerator extends BaseReportGenerator {
     return this.logger;
   }
 
-  get reportMarkdownPath () {
-    return this.logger.markdownPath;
+  get reportHtmlPath () {
+    return this.logger.htmlPath;
   }
 }
 
@@ -274,8 +286,8 @@ class SummaryReportGenerator extends BaseReportGenerator {
     return `${OUTPUT_PATH}json/summary.json`;
   }
 
-  get reportMarkdownPath () {
-    return `${OUTPUT_PATH}summary.md`;
+  get reportHtmlPath () {
+    return `${OUTPUT_PATH}summary.html`;
   }
 
   get opportunityTypeGroups () {

packages/openactive-integration-tests/test/report-templates/report-cli.handlebars

@@ -6,7 +6,7 @@ Feature Implemented: {{{ implemented }}}
 
 {{ consoleValidationIcon overallStatus }} {{ numPassed }} passed with {{ numFailed }} {{ pluralise "failure" numFailed }}, {{ numWarnings }} {{ pluralise "warning" numWarnings }} and {{{ numSuggestions }}} {{ pluralise "suggestion" numSuggestions }} 
 
-See `{{{ markdownPath }}}` for more detailed info.
+See `{{{ htmlPath }}}` for more detailed info.
 
 {{#each activeSuites }}
 {{#each . }}{{# chalk "bold" "yellow" }}>>{{/chalk}} {{# chalk "bold" "green" }}{{{ . }}}{{/chalk}} {{/each}}

packages/openactive-integration-tests/test/report-templates/report.md.handlebars

@@ -1,4 +1,25 @@
-[< Return to Summary](summary.md) | File Generated: {{{ timestamp }}}
+<head>
+  <style>
+h2 {
+    cursor: pointer;
+    border-top-color: #CCC;
+    border-top-width: 1px;
+    border-top-style: solid;
+    padding-top: 5px;
+}
+h2:hover {
+    color: #0056b3; /* Example color change on hover */
+}
+</style>
+
+</head>
+
+[< Return to Summary](summary.html) | File Generated: {{{ timestamp }}}
+
+<button id="collapseH2Button">Collapse All Sections</button>
+<button id="showH2Button">Show All Sections</button>
+<button id="collapseAllButFirstError">Show Only First Error</button>
+
 
 # {{{ title }}}
 
@@ -30,7 +51,7 @@ The [OpenActive Reference Implementation test result for this test]({{{ referenc
 
 {{#each activeSuites }}
 
-## {{{ renderSuiteName . }}}
+## {{ validationIcon (statusFor . ) }} {{{ renderSuiteName . }}}
     {{#logsFor . "information"}}
 
 ### {{title}}
@@ -141,3 +162,104 @@ Test could not be completed as it timed out while a request was still pending:
 
     {{/if~}}
 {{/logsFor}}
+
+<script>
+  function collapseH2Section(nextElement) {
+    while(nextElement && nextElement.tagName !== 'H2') {
+        if (nextElement.style.display !== 'none') {
+            nextElement.style.display = 'none';
+        }
+        nextElement = nextElement.nextElementSibling;
+    }
+  }
+
+  function showH2Section(nextElement) {
+    while(nextElement && nextElement.tagName !== 'H2') {
+        if (nextElement.style.display !== '') {
+            nextElement.style.display = '';
+        }
+        nextElement = nextElement.nextElementSibling;
+    }
+  }
+
+  function toggleH2Section(nextElement) {
+    while(nextElement && nextElement.tagName !== 'H2') {
+              // Toggle visibility
+              nextElement.style.display = nextElement.style.display === 'none' ? '' : 'none';
+              nextElement = nextElement.nextElementSibling;
+          }
+  }
+
+  function collapseAllH2Sections() {
+    var h2Elements = document.querySelectorAll('h2');
+    h2Elements.forEach(function(h2) {
+        var nextElement = h2.nextElementSibling;
+        collapseH2Section(nextElement);
+    });
+  }
+  function showAllH2Sections() {
+    var h2Elements = document.querySelectorAll('h2');
+    h2Elements.forEach(function(h2) {
+        var nextElement = h2.nextElementSibling;
+        showH2Section(nextElement);
+    });
+  }
+
+  function collapseExceptFirstSpecialH2Section() {
+    const h2Elements = document.querySelectorAll('h2');
+    let specialSectionFound = false;
+
+    h2Elements.forEach((h2) => {
+        let nextNode = h2.nextElementSibling;
+        let sectionContainsSpecialChar = false;
+
+        // Iterate through sibling elements until the next H2 or no more siblings until the error section is found
+        while (nextNode && nextNode.tagName !== 'H2') {
+            if (!specialSectionFound && nextNode.textContent.includes('❌')) {
+                sectionContainsSpecialChar = true;
+                specialSectionFound = true; 
+                break;
+              }
+              showH2Section(nextNode);
+            nextNode = nextNode.nextElementSibling;
+        }
+
+        // If this section is not the one with the error, collapse it
+        if (!sectionContainsSpecialChar) {
+            nextNode = h2.nextElementSibling; 
+            collapseH2Section(nextNode);
+        }
+    });
+}
+
+
+  // Show/Hide each header section
+  document.querySelectorAll('h2').forEach(function(h2) {
+      h2.addEventListener('click', function() {
+          let nextElement = this.nextElementSibling;
+          toggleH2Section(nextElement)
+      });
+  });
+
+  // Show all header sections
+  document.getElementById('showH2Button').addEventListener('click', function() {
+      showAllH2Sections();
+  });
+
+  // Collapse all header sections
+  document.getElementById('collapseH2Button').addEventListener('click', function() {
+      collapseAllH2Sections();
+  });
+
+  // Collapse all but first error section button
+    document.getElementById('collapseAllButFirstError').addEventListener('click', function() {
+      collapseExceptFirstSpecialH2Section();
+  });
+
+  // Hide all header sections by default
+  collapseExceptFirstSpecialH2Section();
+  </script>
+
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css">
+<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
+<script>hljs.highlightAll();</script>

packages/openactive-integration-tests/test/report-templates/summary-cli.handlebars

@@ -1,6 +1,6 @@
 Summary
 
-See `{{{ reportMarkdownPath }}}` for a Markdown version.
+See `{{{ reportHtmlPath }}}` for a Markdown version.
 
 {{#eachSorted opportunityTypeGroups }}
 {{# chalk "bold"}}{{{ opportunityTypeName }}}{{/chalk}}
@@ -9,7 +9,7 @@ See `{{{ reportMarkdownPath }}}` for a Markdown version.
   {{#each featureGroups }}
   - {{{ consoleValidationIcon overallStatus }}} {{{ featureName }}} ({{implementedDisplayLabel}})
     {{#each loggers}}
-    - {{{ consoleValidationIcon overallStatus }}} {{{ suiteName }}} ({{{ numFailed }}} failures, {{{ numWarnings }}} warnings, {{{ numSuggestions }}} suggestions, {{{ numPassed }}} passes): {{{ markdownPath }}}
+    - {{{ consoleValidationIcon overallStatus }}} {{{ suiteName }}} ({{{ numFailed }}} failures, {{{ numWarnings }}} warnings, {{{ numSuggestions }}} suggestions, {{{ numPassed }}} passes): {{{ htmlPath }}}
     {{/each}}
   {{/each}}
 

packages/openactive-integration-tests/test/report-templates/summary.md.handlebars

@@ -14,7 +14,7 @@ Mode: **{{ useRandomOpportunitiesMode }}**
   {{#each featureGroups }}
   * {{{ validationIcon overallStatus }}} {{{ featureName }}} ({{implementedDisplayLabel}})
     {{#each loggers}}
-    - {{{ validationIcon overallStatus }}} [{{{ suiteName }}}]({{{ markdownLocalPath }}}): ({{{ numFailed }}} failures, {{{ numWarnings }}} warnings, {{{ numSuggestions }}} suggestions, {{{ numPassed }}} passes)
+    - {{{ validationIcon overallStatus }}} [{{{ suiteName }}}]({{{ htmlLocalPath }}}): ({{{ numFailed }}} failures, {{{ numWarnings }}} warnings, {{{ numSuggestions }}} suggestions, {{{ numPassed }}} passes)
     {{/each}}
   {{/each}}