diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 23a8bb2..1a732bd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,12 @@ { "dockerComposeFile": "../docker-compose.yml", "service": "spider", - "workspaceFolder": "/usr/local/bin/node" + "workspaceFolder": "/usr/local/bin/node", + "customizations": { + "vscode": { + "extensions": [ + "esbenp.prettier-vscode" + ] + } + } } diff --git a/conf/node/controllers/__snapshots__/generate-indexes.test.js.snap b/conf/node/controllers/__snapshots__/generate-indexes.test.js.snap index 41592e8..1a208ca 100644 --- a/conf/node/controllers/__snapshots__/generate-indexes.test.js.snap +++ b/conf/node/controllers/__snapshots__/generate-indexes.test.js.snap @@ -28,7 +28,8 @@ exports[`generateAgencyIndex should return the snapshots as a string 1`] = `

Ministry of Justice Intranet Archive

test.generate.indexes - hmcts

@@ -66,9 +67,12 @@ exports[`generateRootIndex should return the agencies as a string 1`] = `

Ministry of Justice Intranet Archive

test.generate.indexes

diff --git a/conf/node/controllers/generate-indexes.js b/conf/node/controllers/generate-indexes.js index 590d4f0..da2c05f 100644 --- a/conf/node/controllers/generate-indexes.js +++ b/conf/node/controllers/generate-indexes.js @@ -26,11 +26,14 @@ export const generateRootIndex = async (bucket = s3BucketName, host) => {

Ministry of Justice Intranet Archive

${host}

@@ -74,11 +77,14 @@ export const generateAgencyIndex = async (

Ministry of Justice Intranet Archive

${host} - ${agency}

diff --git a/conf/node/controllers/main.js b/conf/node/controllers/main.js index ff9adff..d6fb6ce 100644 --- a/conf/node/controllers/main.js +++ b/conf/node/controllers/main.js @@ -59,12 +59,23 @@ export const main = async ({ url, agency, depth }) => { await fs.rm(paths.fs, { recursive: true, force: true }); // Generate and write content for the agency index file. - const agencyIndexHtml = await generateAgencyIndex(s3BucketName, url.host, agency); - await writeToS3(s3BucketName, `${url.host}/${agency}/index.html`, agencyIndexHtml); + const agencyIndexHtml = await generateAgencyIndex( + s3BucketName, + url.host, + agency, + ); + await writeToS3( + s3BucketName, + `${url.host}/${agency}/index.html`, + agencyIndexHtml, + { cacheMaxAge: 600 }, + ); // Generate and write content for the root index file. const rootIndexHtml = await generateRootIndex(s3BucketName, url.host); - await writeToS3(s3BucketName, `index.html`, rootIndexHtml); + await writeToS3(s3BucketName, `index.html`, rootIndexHtml, { + cacheMaxAge: 600, + }); console.log("Snapshot complete", { url: url.href, agency, depth }); }; diff --git a/conf/node/controllers/s3.js b/conf/node/controllers/s3.js index 475b39b..16e56d1 100644 --- a/conf/node/controllers/s3.js +++ b/conf/node/controllers/s3.js @@ -1,5 +1,9 @@ import fs from "fs/promises"; -import { S3Client, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3"; +import { + S3Client, + ListObjectsV2Command, + PutObjectCommand, +} from "@aws-sdk/client-s3"; import { S3SyncClient } from "s3-sync-client"; import mime from "mime-types"; @@ -7,7 +11,7 @@ import { s3BucketName, s3Credentials as credentials, s3Region, - heartbeatEndpoint + heartbeatEndpoint, } from "../constants.js"; /** @@ -37,20 +41,23 @@ export const client = new S3Client(s3Options); /** * Create dummy /auth/heartbeat at bucket root, if it doesn't exist. - * + * * @param {string} bucket - The bucket name, defaults to the s3BucketName constant * @returns {Promise} - * + * * @throws {Error} */ -export const createHeartbeat = async (bucket = s3BucketName, file = heartbeatEndpoint) => { +export const createHeartbeat = async ( + bucket = s3BucketName, + file = heartbeatEndpoint, +) => { const objects = await client.send( new ListObjectsV2Command({ Bucket: bucket, Prefix: file, }), - ) + ); if (!objects.Contents?.length) { const response = await client.send( @@ -58,8 +65,8 @@ export const createHeartbeat = async (bucket = s3BucketName, file = heartbeatEnd Bucket: bucket, Key: file, Body: "OK", - }) - ) + }), + ); } return; @@ -140,7 +147,7 @@ export const checkAccess = async (bucket = s3BucketName) => { * @param {string} bucket - The bucket name, defaults to the s3BucketName constant * @param {string} host - The host to the intranet e.g. intranet.justice.gov.uk or dev.intranet.justice.gov.uk * @returns {Promise} - * + * * @throws {Error} */ @@ -164,12 +171,12 @@ export const getAgenciesFromS3 = async (bucket = s3BucketName, host) => { /** * Get an agencies snapshots from S3 - * + * * @param {string} bucket - The bucket name, defaults to the s3BucketName constant * @param {string} host - The host to the intranet e.g. intranet.justice.gov.uk or dev.intranet.justice.gov.uk * @param {string} agency - The agency to get snapshots for e.g. hq, hmcts etc. * @returns {Promise} - * + * * @throws {Error} */ @@ -194,28 +201,40 @@ export const getSnapshotsFromS3 = async ( const { CommonPrefixes } = await client.send(command); - return CommonPrefixes.map((folder) => - folder.Prefix.replace(`${host}/${agency}/`, "").replace("/", ""), + return CommonPrefixes.map(({ Prefix }) => + // Remove the host and agency from the Prefix + Prefix.replace(`${host}/${agency}/`, "").replace("/", ""), + ).filter((folder) => + // Filter out any folders that are not in the format YYYY-MM-DD + /^\d{4}-\d{2}-\d{2}$/.test(folder), ); }; /** * Write string to an S3 file. - * + * * @param {string} bucket - The bucket name, defaults to the s3BucketName constant * @param {string} path - The path to write to * @param {string} content - The content to write - * + * @param {Object} [options] - The options object + * @param {number} [options.cacheMaxAge] - The cache max age in seconds - a simpler alternative to invalidating CloudFront cache + * * @returns {Promise} */ -export const writeToS3 = async (bucket = s3BucketName, path, content) => { +export const writeToS3 = async ( + bucket = s3BucketName, + path, + content, + { cacheMaxAge } = {}, +) => { const command = new PutObjectCommand({ Bucket: bucket, Key: path, Body: content, ContentType: mime.lookup(path) || "text/html", + ...(cacheMaxAge && { CacheControl: `max-age=${cacheMaxAge}` }), }); await client.send(command); -} +}; diff --git a/conf/node/controllers/s3.test.js b/conf/node/controllers/s3.test.js index 8224626..9f2a78f 100644 --- a/conf/node/controllers/s3.test.js +++ b/conf/node/controllers/s3.test.js @@ -230,7 +230,9 @@ describe("writeToS3", () => { await writeToS3(undefined, keyPlain, fileContent); - const resPlain = await client.send(new GetObjectCommand({...commandArgs, Key: keyPlain})); + const resPlain = await client.send( + new GetObjectCommand({ ...commandArgs, Key: keyPlain }), + ); expect(resPlain.ContentType).toBe("text/plain"); @@ -239,10 +241,21 @@ describe("writeToS3", () => { await writeToS3(undefined, keyHtml, fileContent); - const resHtml = await client.send(new GetObjectCommand({...commandArgs, Key: keyHtml})); + const resHtml = await client.send( + new GetObjectCommand({ ...commandArgs, Key: keyHtml }), + ); expect(resHtml.ContentType).toBe("text/html"); + }); + + it("should write the correct cache control", async () => { + await writeToS3(undefined, commandArgs.Key, fileContent, { + cacheMaxAge: 60, + }); + const res = await client.send(new GetObjectCommand(commandArgs)); + + expect(res.CacheControl).toBe("max-age=60"); }); }); @@ -312,6 +325,14 @@ describe("getSnapshotsFromS3", () => { Body: "test", }), ); + + await client.send( + new PutObjectCommand({ + Bucket: s3BucketName, + Key: "test.get.snapshots/hq/non-date-folder/index.html", + Body: "test", + }), + ); }); afterAll(() => {