CDPT-2295 Update documentation

.env.ci

@@ -6,7 +6,6 @@ CI="true"
 AWS_ACCESS_KEY_ID=test-key-id
 AWS_SECRET_ACCESS_KEY=test-access-key
 S3_BUCKET_NAME=test-bucket
-S3_ENDPOINT=http://minio:9000
 ALLOWED_AGENCIES="hq,hmcts"
 INTRANET_JWT_DEV=test-jwt
 INTRANET_ARCHIVE_SHARED_SECRET=test-shared-secret

.env.example

@@ -2,8 +2,20 @@
 # Needed environment variables
 ###
 
-S3_BUCKET_NAME=my-bucket-name
-S3_ACCESS_KEY_ID=my-iam-access-key
-S3_SECRET_ACCESS_KEY=my-iam-secret-key
+ALLOWED_AGENCIES="hq,hmcts"
 
-SNAPSHOT_SCHEDULE="hq:Mon:17:30,hmcts:Tue:17:30"
+SNAPSHOT_SCHEDULE="dev::hmcts::Wed::16:08::3"
+
+INTRANET_JWT_DEV=""
+INTRANET_JWT_STAGING=""
+INTRANET_JWT_PRODUCTION=""
+
+# This should match the value generated for the intranet.
+INTRANET_ARCHIVE_SHARED_SECRET=""
+
+# Minio/AWS credentials - for local only.
+# On Cloud Platform, a service account is used.
+AWS_ACCESS_KEY_ID=local-key-id
+AWS_SECRET_ACCESS_KEY=local-access-key
+
+S3_BUCKET_NAME=local-bucket

Dockerfile

@@ -33,6 +33,7 @@ USER node
 CMD []
 
 
+
 # Create a production image, from the base image.
 # The image is used for deployment, and can be run locally.
 FROM base AS build-prod

Makefile

@@ -1,7 +1,5 @@
 default: launch
 
-IMAGE := ministryofjustice/intranet-archive
-
 # Start the application
 run: env dory
 	docker compose up
@@ -20,15 +18,11 @@ down:
 dory:
 	@chmod +x ./bin/dory-start.sh && ./bin/dory-start.sh
 
-
 launch:
 	@bin/launch.sh
-	@echo "\n Intranet spider available here: http://spider.intranet.docker/\n"
+	@echo "\n Intranet archive available here: http://app.archive.intranet.docker/status\n"
 	@docker compose logs -f spider
 
-image: Dockerfile Makefile build
-	docker build -t $(IMAGE) .
-
 # Get inside the spider container
 bash:
 	docker compose exec spider /bin/ash
@@ -40,26 +34,51 @@ build-prod:
 up-prod:
 	docker compose -f docker-compose.prod.yml up
 
+# Generate the shared secret used by the intranet for signing `/access` requests.
+key-gen-shared-secret:
+	@openssl rand -base64 64 | tr -d '\n' | pbcopy
+	@echo "Shared secret copied to clipboard - either:"
+	@echo "A - Paste it into .env"
+	@echo "    Once for the intranet project and once for the intranet-archive project"
+	@echo "B - Paste it into GitHub secrets"
+	@echo "    Once for the intranet project and once for the intranet-archive project"
+	@echo "    and repeat this command for each environment"
+	@echo "Use the name INTRANET_ARCHIVE_SHARED_SECRET"
+
 # The following key-gen-* commands are for CloudFront RSA key generation/management.
 key-gen-private:
 	@openssl genrsa -out /tmp/private_key.pem 2048 && pbcopy < /tmp/private_key.pem
-	@echo "Private key copied to clipboard - paste it into GitHub secrets"
-	@echo "Use the name AWS_CLOUDFRONT_PRIVATE_KEY_A or AWS_CLOUDFRONT_PRIVATE_KEY_B"
+	@echo "Private key copied to clipboard - either:"
+	@echo "A - Paste it into .env"
+	@echo "    Use the name AWS_CLOUDFRONT_PRIVATE_KEY"
+	@echo "B - Paste it into GitHub secrets"
+	@echo "    Use the name AWS_CLOUDFRONT_PRIVATE_KEY_A or AWS_CLOUDFRONT_PRIVATE_KEY_B"
+	@echo "C - Paste it into GitHub secrets"
+	@echo "    Use the name TEST_AWS_CLOUDFRONT_PRIVATE_KEY"
 	@echo "Then run 'make key-gen-public'"
 
 key-gen-public:
 	@openssl rsa -in /tmp/private_key.pem -pubout -out /tmp/public_key.pem && pbcopy < /tmp/public_key.pem
-	@echo "Public key copied to clipboard - paste it into GitHub secrets"
-	@echo "Use the name AWS_CLOUDFRONT_PUBLIC_KEY_A or AWS_CLOUDFRONT_PUBLIC_KEY_B"
-	@echo "Optionally run 'make key-gen-object' if you are making a test object for GitHub actions"
+	@echo "Public key copied to clipboard - either:"
+	@echo "A - Paste it into .env"
+	@echo "    Use the name AWS_CLOUDFRONT_PUBLIC_KEY"
+	@echo "    Next run 'make key-gen-object'"
+	@echo "B - Paste it into GitHub secrets"
+	@echo "    Use the name AWS_CLOUDFRONT_PUBLIC_KEY_A or AWS_CLOUDFRONT_PUBLIC_KEY_B"
+	@echo "C - Paste it into GitHub secrets"
+	@echo "    Use the name TEST_AWS_CLOUDFRONT_PUBLIC_KEY"
+	@echo "Optionally run 'make key-gen-object' if you are populating .env or TEST_ secrets in GitHub actions"
 	@echo "Then run 'make key-gen-clean'"
 
 key-gen-object:
 	@echo "[{\"id\":\"GENERATED_BY_AWS\",\"comment\":\"$(shell cat /tmp/public_key.pem | openssl dgst -binary -sha256 | xxd -p -c 32 | cut -c 1-8)\"}]" | pbcopy
-	@echo "Public keys object copied to clipboard - paste it into GitHub secrets"
-	@echo "This is only used for testing. Use the name TEST_AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT"
+	@echo "Public keys object copied to clipboard - either: paste it into GitHub secrets"
+	@echo "A - Paste it into .env"
+	@echo "    Use the name AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT"
+	@echo "C - Paste it into GitHub secrets"
+	@echo "    Use the name TEST_AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT - This is only used for testing"
 	@echo "Finally run 'make key-gen-clean'"
 
 key-gen-clean:
 	@rm /tmp/private_key.pem /tmp/public_key.pem && echo "" | pbcopy
-	@echo "Keys removed from /tmp"
+	@echo "Keys removed from /tmp"

bin/launch.sh

@@ -5,16 +5,7 @@ DOTS="\n \033[0;32m***\033[0m"
 echo -e "${DOTS} ${DOTS} Checking Dory... ${DOTS}\n"
 chmod +x ./bin/dory-start.sh && ./bin/dory-start.sh
 
-echo -e "${DOTS} ${DOTS} Firing the website up... ${DOTS}\n"
+echo -e "${DOTS} ${DOTS} Firing the application up... ${DOTS}\n"
 
 # bring docker online (background)
 docker compose up -d
-
-# launch in browser
-echo -e "${DOTS} ${DOTS} Launching your default browser... ${DOTS}\n"
-sleep 2
-
-if command -v python &> /dev/null
-then
-  python -m webbrowser http://spider.intranet.docker
-fi

conf/node/constants.js

@@ -1,5 +1,6 @@
 import { parseScheduleString } from "./controllers/schedule.js";
 
+export const isLocal = process.env.NODE_ENV === "development";
 export const ordinalNumber = parseInt(process.env.ORDINAL_NUMBER);
 export const port = 2000;
 

conf/node/controllers/main.test.js

@@ -1,20 +1,39 @@
 import fs from "fs/promises";
 import { afterAll, beforeEach, expect, it, jest } from "@jest/globals";
 
-import { S3Client, ListObjectsV2Command, GetObjectCommand } from "@aws-sdk/client-s3";
+import {
+  S3Client,
+  ListObjectsV2Command,
+  GetObjectCommand,
+} from "@aws-sdk/client-s3";
 
 import { main } from "./main.js";
 import { getSnapshotPaths } from "./paths.js";
 import { s3Options, s3EmptyDir } from "./s3.js";
-import { intranetUrls, s3BucketName } from "../constants.js";
+import { intranetUrls, intranetJwts, s3BucketName } from "../constants.js";
 
 // Skip tests when running on CI, because this environment doesn't have access to the intranet.
 const skipAllTests = process.env.CI === "true";
 
 // Skip long tests when running in watch mode.
 const skipLongTests = process.env.npm_lifecycle_event === "test:watch";
 
-const envs = ['dev', 'production'];
+const envs = ["dev", "production"];
+
+/**
+ * Can we access the intranet?
+ *
+ * @param {string} env
+ * @returns {Promise<boolean>}
+ */
+
+const canFetchEnv = async (env) => {
+  const { status } = await fetch(intranetUrls[env], {
+    redirect: "manual",
+    headers: { Cookie: `jwt=${intranetJwts[env]}` },
+  });
+  return status === 200;
+};
 
 describe.each(envs)("main - %s", (env) => {
   if (skipAllTests) {
@@ -31,7 +50,17 @@ describe.each(envs)("main - %s", (env) => {
   const paths = getSnapshotPaths({ env, agency });
   const s3Client = new S3Client(s3Options);
 
+  // Can we access the intranet? i.e. is our JWT valid?
+  let access = false;
+
   beforeAll(async () => {
+    access = await canFetchEnv(env);
+    if (!access) {
+      console.info(
+        `Could not access ${env}.\nAdd JWT to your .env file to access the intranet.`,
+      );
+    }
+
     // Mock console.log so the tests are quiet.
     jest.spyOn(console, "log").mockImplementation(() => {});
   });
@@ -53,6 +82,10 @@ describe.each(envs)("main - %s", (env) => {
   });
 
   it("should get index files on a shallow scrape", async () => {
+    if (!access) {
+      return expect(access).toBe(true);
+    }
+
     await main({ env, agency, depth: 1 });
 
     // The snapshot should be on s3
@@ -78,6 +111,10 @@ describe.each(envs)("main - %s", (env) => {
   }, 10_000);
 
   it("should delete sensitive files and cleanup local fs", async () => {
+    if (!access) {
+      return expect(access).toBe(true);
+    }
+
     await main({ env, agency, depth: 1 });
 
     // The snapshot should be on s3
@@ -108,24 +145,32 @@ describe.each(envs)("main - %s", (env) => {
   }, 10_000);
 
   it("should create an auth/heartbeat file", async () => {
+    if (!access) {
+      return expect(access).toBe(true);
+    }
+
     await main({ env, agency, depth: 1 });
 
     // The snapshot should be on s3
     const objects = await s3Client.send(
       new ListObjectsV2Command({
         Bucket: s3BucketName,
-        Prefix: 'auth/heartbeat',
+        Prefix: "auth/heartbeat",
       }),
     );
 
     const heartbeat = objects.Contents.find(
-      (object) => object.Key === 'auth/heartbeat',
+      (object) => object.Key === "auth/heartbeat",
     );
 
     expect(heartbeat).toBeDefined();
   }, 10_000);
 
   it("should create root and agency index files", async () => {
+    if (!access) {
+      return expect(access).toBe(true);
+    }
+
     await main({ env, agency, depth: 1 });
 
     const rootIndexHtml = await s3Client.send(
@@ -134,7 +179,7 @@ describe.each(envs)("main - %s", (env) => {
         Key: "production" === env ? `index.html` : `${env}.html`,
       }),
     );
-    
+
     expect(rootIndexHtml).toBeDefined();
 
     const agencyIndexHtml = await s3Client.send(
@@ -157,6 +202,10 @@ describe.each(envs)("main - %s", (env) => {
   }
 
   it("should get styles.css from the cdn", async () => {
+    if (!access) {
+      return expect(access).toBe(true);
+    }
+
     await main({ env, agency, depth: 2 });
 
     // The snapshot should be on s3

conf/node/middleware.js

@@ -162,6 +162,8 @@ export const checkSignature = (req, _res, next) => {
  */
 
 export const errorHandler = (err, _req, res, _next) => {
+  console.log(err);
+
   if (err.status === 400) {
     res
       .status(400)
@@ -176,6 +178,6 @@ export const errorHandler = (err, _req, res, _next) => {
     return;
   }
 
-  // For everthing else, return a 500 error
+  // For everything else, return a 500 error
   res.status(500).sendFile("static/error-pages/500.html", { root: __dirname });
 };

conf/node/server.js

@@ -8,6 +8,7 @@ import express from "express";
 
 // Relative
 import {
+  isLocal,
   ordinalNumber,
   intranetUrls,
   intranetJwts,
@@ -72,6 +73,10 @@ app.get("/status", async function (_req, res, next) {
       .filter(([, jwt]) => jwt)
       .map(([env]) => env);
 
+    if(isLocal) {
+      envs.push("local");
+    }
+
     const fetchStatuses = await Promise.all(
       envs.map(async (env) => {
         const url = intranetUrls[env];
@@ -93,8 +98,7 @@ app.get("/status", async function (_req, res, next) {
     res.status(200).send(data);
   } catch (err) {
     // Handling errors like this will send the error to the default Express error handler.
-    // It will log the error to the console, return a 500 error page,
-    // and show the error message on dev environments, but hide it on production.
+    // It will log the error to the console, return a 500 error page.
     next(err);
   }
 });
@@ -160,7 +164,7 @@ app.post("/access", async function (req, res, next) {
 
 app.use(function (_req, res) {
   // Return a 404 page if no route is matched
-  res.status(404).sendFile("static/404.html", { root: __dirname });
+  res.status(404).sendFile("static/error-pages/404.html", { root: __dirname });
 });
 
 /**

docker-compose.yml

@@ -14,6 +14,7 @@ services:
       ORDINAL_NUMBER: 0
       VIRTUAL_HOST: app.archive.intranet.docker
       VIRTUAL_PORT: "2000"
+      S3_ENDPOINT: "http://minio:9000"
     volumes:
       - node_modules:/home/node/app/node_modules
       - ./conf/node:/home/node/app
@@ -23,6 +24,9 @@ services:
       minio-init:
         # Wait for minio-init to complete before starting.
         condition: service_completed_successfully
+    # Requests to intranet.docker should go to host machine
+    extra_hosts:
+      - "intranet.docker:host-gateway"
 
   minio:
     image: minio/minio