feat: Handle page caching (#26) (#28)

.github/workflows/node.js.yml

@@ -22,5 +22,4 @@ jobs:
         node-version: 14.x
 
     - run: npm install
-    - run: npm run build
     - run: npm test

.github/workflows/publish-to-npm.yaml

@@ -0,0 +1,51 @@
+name: Publish to npm
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.author.email, '[email protected]')"
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        token: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN }}
+    - name: Identify
+      run: |
+        git config user.name OpenActive Bot
+        git config user.email [email protected]
+    - name: Use Node.js 14.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 14
+        registry-url: https://registry.npmjs.org/
+    - name: Install
+      run: npm install
+    - name: Test
+      run: npm test
+    - name: Increment Version
+      run: npm version patch
+    - name: Publish to npm
+      run: npm publish
+      env:
+        NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
+    - name: Push version update
+      run: git push
+
+  dispatch:
+    needs: publish
+    strategy:
+      matrix:
+        repo: ['data-model-validator-site', 'openactive-test-suite']
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger tooling update
+        uses: peter-evans/repository-dispatch@v1
+        with:
+          token: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN }}
+          repository: openactive/${{ matrix.repo }}
+          event-type: rpde-validator-update

.gitignore

@@ -78,5 +78,5 @@ typings/
 # OSX
 .DS_Store
 
-# Dist
-dist
+# Local bash history
+bash_history.txt

README.md

@@ -22,7 +22,7 @@ $ npm install @openactive/rpde-validator
 ### Usage
 
 ```js
-import { RpdeValidator } from '@openactive/rpde-validator';
+const { RpdeValidator } = require('@openactive/rpde-validator');
 
 RpdeValidator(
   url, // The URL of the feed to test

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openactive/rpde-validator",
-  "version": "2.0.10",
+  "version": "2.0.15",
   "description": "A library to walk and validate an RPDE feed",
   "homepage": "https://openactive.io",
   "author": "OpenActive Community <[email protected]>",
@@ -10,7 +10,7 @@
     "validator",
     "json"
   ],
-  "main": "dist/index.js",
+  "main": "src/index.js",
   "engines": {
     "node": "14.16.0"
   },
@@ -23,32 +23,26 @@
   },
   "license": "MIT",
   "scripts": {
-    "build": "npm run build:genversion && npm run build:babel",
-    "build:genversion": "genversion --es6 --semi src/version.js",
-    "build:babel": "babel src --out-dir dist",
-    "prepare": "npm run build",
     "lint": "eslint \"src/**/*.js\"",
-    "e2e": "babel-node src/e2e.js",
+    "lint-fix": "eslint \"src/**/*.js\" --fix",
+    "e2e": "node src/e2e.js",
     "pretest": "npm run lint",
     "test": "npm run test-no-lint",
-    "test-no-lint": "BABEL_ENV=test babel-node spec/run.js",
+    "test-no-lint": "jasmine",
+    "test-debug": "node --inspect-brk -i ./node_modules/jasmine/bin/jasmine.js",
     "postpublish": "git push",
     "publish-patch": "npm test && git pull && git push && npm version patch && npm publish"
   },
   "devDependencies": {
-    "babel-cli": "^6.26.0",
-    "eslint": "^5.6.1",
-    "eslint-config-airbnb": "^17.1.0",
-    "eslint-plugin-import": "^2.14.0",
-    "jasmine": "^3.2.0"
+    "eslint": "^8.36.0",
+    "eslint-config-airbnb": "^19.0.4",
+    "eslint-plugin-import": "^2.27.5",
+    "jasmine": "^4.6.0"
   },
   "dependencies": {
     "@openactive/data-model-validator": "^2.0.0",
-    "babel-core": "^6.26.3",
-    "babel-plugin-transform-builtin-extend": "^1.1.2",
-    "babel-preset-env": "^1.7.0",
-    "genversion": "^2.1.0",
-    "jsonpath": "^1.0.0",
+    "cache-parser": "^1.2.4",
+    "jsonpath": "^1.1.1",
     "luxon": "^1.4.2",
     "node-fetch": "^2.2.0"
   }

spec/run.js

@@ -1,4 +1,4 @@
-import Jasmine from 'jasmine';
+const Jasmine = require('jasmine');
 
 const jasmine = new Jasmine();
 jasmine.loadConfigFile('spec/support/jasmine.json');

spec/support/jasmine.json

@@ -4,7 +4,6 @@
     "**/*[sS]pec.js"
   ],
   "helpers": [
-    "../spec/helpers/babel.js",
     "../spec/helpers/**/*.js"
   ],
   "stopSpecOnExpectationFailure": false,

src/e2e.js

@@ -1,5 +1,5 @@
-import util from 'util';
-import { RpdeValidator } from './index';
+const util = require('util');
+const { RpdeValidator } = require('./index');
 
 const urls = [
   // 'https://activenewham-openactive.herokuapp.com/',

src/errors/extendable-error.js

@@ -1,4 +1,4 @@
-export default class ExtendableError extends Error {
+module.exports = class ExtendableError extends Error {
   constructor(message) {
     super(message);
     this.name = this.constructor.name;
@@ -8,4 +8,4 @@ export default class ExtendableError extends Error {
       this.stack = (new Error(message)).stack;
     }
   }
-}
+};

src/errors/rpde-error-type.js

@@ -24,6 +24,8 @@ const RpdeErrorType = {
   NO_MODELLING_DATA: 'no_modelling_data',
   HAS_MODELLING_DATA: 'has_modelling_data',
   HTTP_ERROR: 'http_error',
+  MISSING_CACHE_CONTROL: 'missing_cache_control',
+  EXCESSIVE_CACHE_CONTROL: 'excessive_cache_control',
 };
 
-export default Object.freeze(RpdeErrorType);
+module.exports = Object.freeze(RpdeErrorType);

src/errors/timeout-error.js

@@ -1,3 +1,3 @@
-import ExtendableError from './extendable-error';
+const ExtendableError = require('./extendable-error');
 
-export default class TimeoutError extends ExtendableError {}
+module.exports = class TimeoutError extends ExtendableError {};

src/feed-checker.js

@@ -1,16 +1,16 @@
-import {
+const {
   ValidationErrorCategory,
   ValidationErrorSeverity,
   ValidationError,
-} from '@openactive/data-model-validator';
-import { URL } from 'url';
-import FeedLog from './feed-log';
-import RpdeNode from './rpde-node';
-import Rules from './rules';
-import LastPageHelper from './helpers/last-page-helper';
-import UrlHelper from './helpers/url-helper';
-import { version } from './version';
-import RpdeErrorType from './errors/rpde-error-type';
+} = require('@openactive/data-model-validator');
+const { URL } = require('url');
+const FeedLog = require('./feed-log');
+const RpdeNode = require('./rpde-node');
+const Rules = require('./rules');
+const LastPageHelper = require('./helpers/last-page-helper');
+const UrlHelper = require('./helpers/url-helper');
+const RpdeErrorType = require('./errors/rpde-error-type');
+const { version } = require('../package.json');
 
 class FeedChecker {
   constructor(url, options = {}) {
@@ -64,16 +64,12 @@ class FeedChecker {
   logMessage(msg, extra = {}) {
     if (typeof this.logCallback === 'function') {
       this.logCallback(
-        Object.assign(
-          {
-            percentage: this.percentageComplete(),
-            verbosity: 1,
-          },
-          extra,
-          {
-            message: msg,
-          },
-        ),
+        {
+          percentage: this.percentageComplete(),
+          verbosity: 1,
+          ...extra,
+          message: msg,
+        },
       );
     }
   }
@@ -88,7 +84,7 @@ class FeedChecker {
 
   walk(urlOverride) {
     const url = urlOverride || this.url;
-    return this.load(url)
+    return this.load(url, false)
       .then((json) => {
         if (typeof json !== 'object' || json === null) {
           return null;
@@ -225,7 +221,7 @@ class FeedChecker {
     }
 
     if (hasUrl) {
-      return this.load(lastPageUrl)
+      return this.load(lastPageUrl, true)
         .then((json) => {
           const lastPageNode = new RpdeNode(
             lastPageUrl,
@@ -250,7 +246,7 @@ class FeedChecker {
           if (typeof json === 'object' && typeof json.next !== 'undefined') {
             const nextUrlRaw = UrlHelper.deriveUrl(json.next, lastPageUrl);
 
-            return this.load(nextUrlRaw)
+            return this.load(nextUrlRaw, true)
               .then((nextJson) => {
                 const nextNode = new RpdeNode(
                   nextUrlRaw,
@@ -280,7 +276,7 @@ class FeedChecker {
     return null;
   }
 
-  load(url) {
+  load(url, isLastPage) {
     this.logMessage(`Loading ${url}...`);
     this.log.addPage(url);
     const options = {
@@ -301,10 +297,13 @@ class FeedChecker {
           url,
           {
             contentType: res.headers.get('content-type'),
+            cacheControl: res.headers.get('Cache-Control'),
             status: res.status,
             body,
           },
           this.log,
+          undefined,
+          isLastPage,
         );
 
         this.logMessage(`Applying HTTP rules to ${url}`);
@@ -348,4 +347,4 @@ class FeedChecker {
   }
 }
 
-export default FeedChecker;
+module.exports = FeedChecker;

src/feed-log-page.js

@@ -9,4 +9,4 @@ class FeedLogPage {
   }
 }
 
-export default FeedLogPage;
+module.exports = FeedLogPage;

src/feed-log.js

@@ -1,4 +1,4 @@
-import FeedLogPage from './feed-log-page';
+const FeedLogPage = require('./feed-log-page');
 
 class FeedLog {
   constructor(url) {
@@ -30,4 +30,4 @@ class FeedLog {
   }
 }
 
-export default FeedLog;
+module.exports = FeedLog;

src/feed-page-checker.js

@@ -1,6 +1,7 @@
-import FeedLog from './feed-log';
-import RpdeNode from './rpde-node';
-import Rules from './rules';
+const { parse: parseCacheControlHeader } = require('cache-parser');
+const FeedLog = require('./feed-log');
+const RpdeNode = require('./rpde-node');
+const Rules = require('./rules');
 
 class FeedPageChecker {
   constructor() {
@@ -21,7 +22,7 @@ class FeedPageChecker {
   }
 
   validateRpdePage({
-    url, json, pageIndex, contentType, status, isInitialHarvestComplete, isOrdersFeed,
+    url, json, pageIndex, contentType, cacheControl, status, isInitialHarvestComplete, isOrdersFeed,
   }) {
     const log = new FeedLog(url);
 
@@ -31,13 +32,17 @@ class FeedPageChecker {
       && json.items.length === 0
     );
 
+    // Item duplication is only permissible if Test Suite harvesting is complete (and therefore state change is expected within the underlying booking system), or if RPDE page caching is in place
+    const cacheControlComponents = parseCacheControlHeader(cacheControl) || {};
+    const isItemDuplicationPermissible = isInitialHarvestComplete || cacheControlComponents.public;
+
     const node = new RpdeNode(
       url,
       json,
       log,
       pageIndex,
       isLastPage,
-      isInitialHarvestComplete,
+      isItemDuplicationPermissible,
       isOrdersFeed,
     );
 
@@ -55,22 +60,23 @@ class FeedPageChecker {
       url,
       {
         contentType,
+        cacheControl,
         status,
         body: json,
       },
       log,
       undefined,
-      undefined,
-      undefined,
+      isLastPage,
+      isItemDuplicationPermissible,
       isOrdersFeed,
     );
 
     for (const rule of this.httpRules) {
       rule.validate(rawNode);
     }
 
-    return log.pages.flatMap(page => page.errors).filter(error => error.severity === 'failure');
+    return log.pages.flatMap((page) => page.errors).filter((error) => error.severity === 'failure');
   }
 }
 
-export default FeedPageChecker;
+module.exports = FeedPageChecker;

src/feed-page-checker.spec.js

@@ -1,5 +1,5 @@
-import FeedPageChecker from './feed-page-checker';
-import RpdeErrorType from './errors/rpde-error-type';
+const FeedPageChecker = require('./feed-page-checker');
+const RpdeErrorType = require('./errors/rpde-error-type');
 
 describe('FeedPageChecker', () => {
   let checker;