Add JWT, test endpoints, and service account.

.devcontainer/devcontainer.json

@@ -0,0 +1,5 @@
+{
+    "dockerComposeFile": "../docker-compose.yml",
+    "service": "spider",
+    "workspaceFolder": "/usr/local/bin/node"
+}

.env.ci

@@ -0,0 +1,10 @@
+###
+# Environment variables for CI
+###
+
+AWS_ACCESS_KEY_ID=test-key-id
+AWS_SECRET_ACCESS_KEY=test-access-key
+S3_BUCKET_NAME=test-bucket
+S3_ENDPOINT=http://minio:9000
+ALLOWED_AGENCIES="hq,htcts"
+JWT=test-jwt

.github/workflows/deploy.yml

@@ -32,7 +32,7 @@ jobs:
           # AWS_CLOUDFRONT_PUBLIC_KEY_EXPIRING: "${{ secrets.AWS_CLOUDFRONT_PUBLIC_KEY_B }}"
           BASIC_AUTH_USER: ${{ secrets.BASIC_AUTH_USER }}
           BASIC_AUTH_PASS: ${{ secrets.BASIC_AUTH_PASS }}
-          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+          JWT: ${{ secrets.JWT }}
         run: |
           ## - - - - - - - - - -
           ## CloudFront  - - - -

.github/workflows/test.yml

@@ -0,0 +1,18 @@
+name: Test
+
+on: [pull_request]
+
+jobs:
+  test:
+    if: github.event.ref != 'refs/heads/main'
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Run tests
+      run: |
+        cp .env.ci .env
+        docker compose run --rm -e JWT=${JWT} spider sh -c "npm ci && npm run test"
+      env:
+        JWT: ${{ secrets.JWT }}

Dockerfile

@@ -93,12 +93,12 @@ WORKDIR /usr/local/bin/node
 COPY conf/entrypoint/start-node-prod.sh /docker-entrypoint.d/
 RUN chmod -R +x /docker-entrypoint.d/start-node-prod.sh
 
-# Install the node modules.
-RUN npm ci
-
-# Change the environment to production for runtime.
+# Set the environment to production for install & runtime.
 ENV NODE_ENV=production
 
+# Install the node modules.
+RUN npm ci --only=prod 
+
 # Remove the npm package manager.
 RUN apt remove -y curl npm unzip
 

conf/entrypoint/start-node-prod.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-nohup node /usr/local/bin/node/process.js &
+nohup node /usr/local/bin/node/server.js &
 

conf/nginx.conf

@@ -9,6 +9,27 @@ server {
       proxy_pass http://localhost:2000/spider;
   }
 
+  location /fetch-test {
+      proxy_set_header X-Forwarded-Host $host;
+      proxy_set_header X-Forwarded-Server $host;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_pass http://localhost:2000/fetch-test;
+  }
+
+  location /bucket-test {
+      proxy_set_header X-Forwarded-Host $host;
+      proxy_set_header X-Forwarded-Server $host;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_pass http://localhost:2000/bucket-test;
+  }
+
+  location /set-cf-cookie {
+      proxy_set_header X-Forwarded-Host $host;
+      proxy_set_header X-Forwarded-Server $host;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_pass http://localhost:2000/set-cf-cookie;
+  }
+
   location / {
     root   /usr/share/nginx/html;
     index  index.html;

conf/node/constants.js

@@ -0,0 +1,46 @@
+export const port = 2000;
+export const jwt = process.env.JWT;
+
+/**
+ * S3
+ */
+
+export const s3Region = 'eu-west-2';
+export const s3BucketName = process.env.S3_BUCKET_NAME;
+export const s3Credentials = process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY && {
+  accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+}
+
+/**
+ * Options
+ */
+
+export const corsOptions ={
+  methods: ["POST"],
+  origin: [
+    "http://spider.intranet.docker/",
+    "https://dev-intranet-archive.apps.live.cloud-platform.service.justice.gov.uk/",
+  ],
+};
+
+/**
+ * Defaults
+ */
+
+export const defaultAgency = "hq";
+export const defaultUrl = "https://intranet.justice.gov.uk/";
+
+/**
+ * Validation
+ */
+
+export const allowedTargetHosts = [
+  "intranet.docker",
+  "intranet.justice.gov.uk",
+  "dev.intranet.justice.gov.uk",
+  "staging.intranet.justice.gov.uk",
+  "demo.intranet.justice.gov.uk",
+];
+
+export const allowedTargetAgencies = process.env.ALLOWED_AGENCIES?.split(",") ?? [];

conf/node/controllers/__snapshots__/httrack.test.js.snap

@@ -0,0 +1,33 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`getHttrackArgs should return an array of arguments 1`] = `
+[
+  "-%W",
+  "/archiver/strip_x_amz_query_param.so",
+  "https://intranet.justice.gov.uk/",
+  "+*.png",
+  "+*.gif",
+  "+*.jpg",
+  "+*.jpeg",
+  "+*.css",
+  "+*.js",
+  "-ad.doubleclick.net/*",
+  "-*intranet.justice.gov.uk/agency-switcher/",
+  "-*intranet.justice.gov.uk/?*agency=*",
+  "-*intranet.justice.gov.uk/?p=*",
+  "-*intranet.justice.gov.uk/?page_id=*",
+  "-*intranet.justice.gov.uk/wp-json/*/embed*",
+  "-*intranet.justice.gov.uk/wp/*",
+  "+*intranet.justice.gov.uk/?*agency=hq",
+  "-s0",
+  "-V",
+  ""sed -i 's/srcset="[^"]*"//g' $0"",
+  "-%k",
+  "-F",
+  "intranet-archive",
+  "-%X",
+  "Cookie: dw_agency=hq; jwt=***",
+  "-O",
+  "/tmp/test-snapshot",
+]
+`;

conf/node/controllers/cloudfront.js

@@ -0,0 +1,33 @@
+const { getSignedCookies } = require("@aws-sdk/cloudfront-signer");
+
+// TODO: Implement the getCookies function
+
+const getCookies = () => {
+  const cloudfrontDistributionDomain = "https://d111111abcdef8.cloudfront.net";
+  const s3ObjectKey = "private-content/private.jpeg";
+  const url = `${cloudfrontDistributionDomain}/${s3ObjectKey}`;
+  const privateKey = "CONTENTS-OF-PRIVATE-KEY";
+  const keyPairId = "PUBLIC-KEY-ID-OF-CLOUDFRONT-KEY-PAIR";
+  const dateLessThan = "2022-01-01";
+
+  const policy = {
+    Statement: [
+      {
+        Resource: url,
+        Condition: {
+          DateLessThan: {
+            "AWS:EpochTime": new Date(dateLessThan).getTime() / 1000, // time in seconds
+          },
+        },
+      },
+    ],
+  };
+
+  const policyString = JSON.stringify(policy);
+
+  return getSignedCookies({
+    keyPairId,
+    privateKey,
+    policy: policyString,
+  });
+};