diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 23a8bb2..1a732bd 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,5 +1,12 @@
{
"dockerComposeFile": "../docker-compose.yml",
"service": "spider",
- "workspaceFolder": "/usr/local/bin/node"
+ "workspaceFolder": "/usr/local/bin/node",
+ "customizations": {
+ "vscode": {
+ "extensions": [
+ "esbenp.prettier-vscode"
+ ]
+ }
+ }
}
diff --git a/conf/node/controllers/__snapshots__/generate-indexes.test.js.snap b/conf/node/controllers/__snapshots__/generate-indexes.test.js.snap
index 41592e8..1a208ca 100644
--- a/conf/node/controllers/__snapshots__/generate-indexes.test.js.snap
+++ b/conf/node/controllers/__snapshots__/generate-indexes.test.js.snap
@@ -28,7 +28,8 @@ exports[`generateAgencyIndex should return the snapshots as a string 1`] = `
Ministry of Justice Intranet Archive
test.generate.indexes - hmcts
@@ -66,9 +67,12 @@ exports[`generateRootIndex should return the agencies as a string 1`] = `
Ministry of Justice Intranet Archive
test.generate.indexes
diff --git a/conf/node/controllers/generate-indexes.js b/conf/node/controllers/generate-indexes.js
index 590d4f0..da2c05f 100644
--- a/conf/node/controllers/generate-indexes.js
+++ b/conf/node/controllers/generate-indexes.js
@@ -26,11 +26,14 @@ export const generateRootIndex = async (bucket = s3BucketName, host) => {
Ministry of Justice Intranet Archive
${host}
- ${agencies.map(
- (agency) => `-
+ ${agencies
+ .map(
+ (agency) => `
+
-
${agency}
`,
- )}
+ )
+ .join("\n")}
@@ -74,11 +77,14 @@ export const generateAgencyIndex = async (
Ministry of Justice Intranet Archive
${host} - ${agency}
- ${snapshots.map(
- (snapshot) => `-
+ ${snapshots
+ .map(
+ (snapshot) => `
+
-
${snapshot}
`,
- )}
+ )
+ .join("\n")}
diff --git a/conf/node/controllers/main.js b/conf/node/controllers/main.js
index ff9adff..d6fb6ce 100644
--- a/conf/node/controllers/main.js
+++ b/conf/node/controllers/main.js
@@ -59,12 +59,23 @@ export const main = async ({ url, agency, depth }) => {
await fs.rm(paths.fs, { recursive: true, force: true });
// Generate and write content for the agency index file.
- const agencyIndexHtml = await generateAgencyIndex(s3BucketName, url.host, agency);
- await writeToS3(s3BucketName, `${url.host}/${agency}/index.html`, agencyIndexHtml);
+ const agencyIndexHtml = await generateAgencyIndex(
+ s3BucketName,
+ url.host,
+ agency,
+ );
+ await writeToS3(
+ s3BucketName,
+ `${url.host}/${agency}/index.html`,
+ agencyIndexHtml,
+ { cacheMaxAge: 600 },
+ );
// Generate and write content for the root index file.
const rootIndexHtml = await generateRootIndex(s3BucketName, url.host);
- await writeToS3(s3BucketName, `index.html`, rootIndexHtml);
+ await writeToS3(s3BucketName, `index.html`, rootIndexHtml, {
+ cacheMaxAge: 600,
+ });
console.log("Snapshot complete", { url: url.href, agency, depth });
};
diff --git a/conf/node/controllers/s3.js b/conf/node/controllers/s3.js
index 475b39b..16e56d1 100644
--- a/conf/node/controllers/s3.js
+++ b/conf/node/controllers/s3.js
@@ -1,5 +1,9 @@
import fs from "fs/promises";
-import { S3Client, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
+import {
+ S3Client,
+ ListObjectsV2Command,
+ PutObjectCommand,
+} from "@aws-sdk/client-s3";
import { S3SyncClient } from "s3-sync-client";
import mime from "mime-types";
@@ -7,7 +11,7 @@ import {
s3BucketName,
s3Credentials as credentials,
s3Region,
- heartbeatEndpoint
+ heartbeatEndpoint,
} from "../constants.js";
/**
@@ -37,20 +41,23 @@ export const client = new S3Client(s3Options);
/**
* Create dummy /auth/heartbeat at bucket root, if it doesn't exist.
- *
+ *
* @param {string} bucket - The bucket name, defaults to the s3BucketName constant
* @returns {Promise}
- *
+ *
* @throws {Error}
*/
-export const createHeartbeat = async (bucket = s3BucketName, file = heartbeatEndpoint) => {
+export const createHeartbeat = async (
+ bucket = s3BucketName,
+ file = heartbeatEndpoint,
+) => {
const objects = await client.send(
new ListObjectsV2Command({
Bucket: bucket,
Prefix: file,
}),
- )
+ );
if (!objects.Contents?.length) {
const response = await client.send(
@@ -58,8 +65,8 @@ export const createHeartbeat = async (bucket = s3BucketName, file = heartbeatEnd
Bucket: bucket,
Key: file,
Body: "OK",
- })
- )
+ }),
+ );
}
return;
@@ -140,7 +147,7 @@ export const checkAccess = async (bucket = s3BucketName) => {
* @param {string} bucket - The bucket name, defaults to the s3BucketName constant
* @param {string} host - The host to the intranet e.g. intranet.justice.gov.uk or dev.intranet.justice.gov.uk
* @returns {Promise}
- *
+ *
* @throws {Error}
*/
@@ -164,12 +171,12 @@ export const getAgenciesFromS3 = async (bucket = s3BucketName, host) => {
/**
* Get an agencies snapshots from S3
- *
+ *
* @param {string} bucket - The bucket name, defaults to the s3BucketName constant
* @param {string} host - The host to the intranet e.g. intranet.justice.gov.uk or dev.intranet.justice.gov.uk
* @param {string} agency - The agency to get snapshots for e.g. hq, hmcts etc.
* @returns {Promise}
- *
+ *
* @throws {Error}
*/
@@ -194,28 +201,40 @@ export const getSnapshotsFromS3 = async (
const { CommonPrefixes } = await client.send(command);
- return CommonPrefixes.map((folder) =>
- folder.Prefix.replace(`${host}/${agency}/`, "").replace("/", ""),
+ return CommonPrefixes.map(({ Prefix }) =>
+ // Remove the host and agency from the Prefix
+ Prefix.replace(`${host}/${agency}/`, "").replace("/", ""),
+ ).filter((folder) =>
+ // Filter out any folders that are not in the format YYYY-MM-DD
+ /^\d{4}-\d{2}-\d{2}$/.test(folder),
);
};
/**
* Write string to an S3 file.
- *
+ *
* @param {string} bucket - The bucket name, defaults to the s3BucketName constant
* @param {string} path - The path to write to
* @param {string} content - The content to write
- *
+ * @param {Object} [options] - The options object
+ * @param {number} [options.cacheMaxAge] - The cache max age in seconds - a simpler alternative to invalidating CloudFront cache
+ *
* @returns {Promise}
*/
-export const writeToS3 = async (bucket = s3BucketName, path, content) => {
+export const writeToS3 = async (
+ bucket = s3BucketName,
+ path,
+ content,
+ { cacheMaxAge } = {},
+) => {
const command = new PutObjectCommand({
Bucket: bucket,
Key: path,
Body: content,
ContentType: mime.lookup(path) || "text/html",
+ ...(cacheMaxAge && { CacheControl: `max-age=${cacheMaxAge}` }),
});
await client.send(command);
-}
+};
diff --git a/conf/node/controllers/s3.test.js b/conf/node/controllers/s3.test.js
index 8224626..9f2a78f 100644
--- a/conf/node/controllers/s3.test.js
+++ b/conf/node/controllers/s3.test.js
@@ -230,7 +230,9 @@ describe("writeToS3", () => {
await writeToS3(undefined, keyPlain, fileContent);
- const resPlain = await client.send(new GetObjectCommand({...commandArgs, Key: keyPlain}));
+ const resPlain = await client.send(
+ new GetObjectCommand({ ...commandArgs, Key: keyPlain }),
+ );
expect(resPlain.ContentType).toBe("text/plain");
@@ -239,10 +241,21 @@ describe("writeToS3", () => {
await writeToS3(undefined, keyHtml, fileContent);
- const resHtml = await client.send(new GetObjectCommand({...commandArgs, Key: keyHtml}));
+ const resHtml = await client.send(
+ new GetObjectCommand({ ...commandArgs, Key: keyHtml }),
+ );
expect(resHtml.ContentType).toBe("text/html");
+ });
+
+ it("should write the correct cache control", async () => {
+ await writeToS3(undefined, commandArgs.Key, fileContent, {
+ cacheMaxAge: 60,
+ });
+ const res = await client.send(new GetObjectCommand(commandArgs));
+
+ expect(res.CacheControl).toBe("max-age=60");
});
});
@@ -312,6 +325,14 @@ describe("getSnapshotsFromS3", () => {
Body: "test",
}),
);
+
+ await client.send(
+ new PutObjectCommand({
+ Bucket: s3BucketName,
+ Key: "test.get.snapshots/hq/non-date-folder/index.html",
+ Body: "test",
+ }),
+ );
});
afterAll(() => {