diff --git a/conf/local.Caddyfile b/conf/local.Caddyfile new file mode 100644 index 0000000..3c617f4 --- /dev/null +++ b/conf/local.Caddyfile @@ -0,0 +1,8 @@ +# A local mock CDN to proxy requests to the Minio (S3) server. +# Mimics AWS CloudFront, and removes the bucket path from the URL. +# e.g. Request: http://archive.intranet.docker/intranet.justice.gov.uk/hmcts/2024-12-13/index.html +# proxies to : http://minio:9000/bucket-name/intranet.justice.gov.uk/hmcts/2024-12-13/index.html + +:2019 +rewrite * /{$S3_BUCKET_NAME}{uri} +reverse_proxy minio:9000 diff --git a/conf/node/constants.js b/conf/node/constants.js index cb4c441..89d22b5 100644 --- a/conf/node/constants.js +++ b/conf/node/constants.js @@ -67,6 +67,12 @@ export const sensitiveFiles = [ `hts-cache/new.zip`, ]; +/** + * Intranet application + */ + +export const heartbeatEndpoint = "auth/heartbeat"; + /** * Index pages */ diff --git a/conf/node/controllers/main.js b/conf/node/controllers/main.js index 3b5f985..41492b9 100644 --- a/conf/node/controllers/main.js +++ b/conf/node/controllers/main.js @@ -8,8 +8,8 @@ import { getHttrackProgress, waitForHttrackComplete, } from "./httrack.js"; +import { createHeartbeat, sync, writeToS3 } from "./s3.js"; import { generateRootIndex, generateAgencyIndex } from "./generate-indexes.js"; -import { sync, writeToS3 } from "./s3.js"; /** * @@ -49,6 +49,9 @@ export const main = async ({ url, agency, depth }) => { sensitiveFiles.map((file) => fs.rm(`${paths.fs}/${file}`, { force: true })), ); + // Add a file at /auth/heartbeat for the intranet's heartbeat script. + await createHeartbeat(); + // Sync the snapshot to S3 await sync(paths.fs, `s3://${s3BucketName}/${paths.s3}`); diff --git a/conf/node/controllers/main.test.js b/conf/node/controllers/main.test.js index 5e5ed54..0d0fe4a 100644 --- a/conf/node/controllers/main.test.js +++ b/conf/node/controllers/main.test.js @@ -92,6 +92,24 @@ describe("main", () => { expect(pathExists).toBe(false); }, 10_000); + it("should create an auth/heartbeat file", async () => { + await main({ url, agency, depth: 1 }); + + // The snapshot should be on s3 + const objects = await s3Client.send( + new ListObjectsV2Command({ + Bucket: s3BucketName, + Prefix: 'auth/heartbeat', + }), + ); + + const heartbeat = objects.Contents.find( + (object) => object.Key === 'auth/heartbeat', + ); + + expect(heartbeat).toBeDefined(); + }, 10_000); + it("should create root and agency index files", async () => { await main({ url, agency, depth: 1 }); diff --git a/conf/node/controllers/s3.js b/conf/node/controllers/s3.js index 73623e6..78bac32 100644 --- a/conf/node/controllers/s3.js +++ b/conf/node/controllers/s3.js @@ -7,6 +7,7 @@ import { s3BucketName, s3Credentials as credentials, s3Region, + heartbeatEndpoint } from "../constants.js"; /** @@ -34,6 +35,36 @@ export const s3Options = { export const client = new S3Client(s3Options); +/** + * Create dummy /auth/heartbeat at bucket root, if it doesn't exist. + * + * @param {string} bucket - The bucket name, defaults to the s3BucketName constant + * @returns {Promise} + * + * @throws {Error} + */ + +export const createHeartbeat = async (bucket = s3BucketName, file = heartbeatEndpoint) => { + const objects = await client.send( + new ListObjectsV2Command({ + Bucket: bucket, + Prefix: file, + }), + ) + + if (!objects.Contents?.length) { + const response = await client.send( + new PutObjectCommand({ + Bucket: bucket, + Key: file, + Body: "OK", + }) + ) + } + + return; +}; + /** * S3 sync client * diff --git a/conf/node/controllers/s3.test.js b/conf/node/controllers/s3.test.js index eac6f37..e4aa63b 100644 --- a/conf/node/controllers/s3.test.js +++ b/conf/node/controllers/s3.test.js @@ -11,6 +11,7 @@ import { s3BucketName } from "../constants.js"; import { s3Options, checkAccess, + createHeartbeat, sync, s3EmptyDir, writeToS3, @@ -35,6 +36,29 @@ describe("checkAccess", () => { }); }); +describe("createHeartbeat", () => { + let client; + + beforeAll(() => { + client = new S3Client(s3Options); + }); + + it("should create /auth/heartbeat if it doesn't exist", async () => { + await s3EmptyDir("test/auth"); + + await createHeartbeat(undefined, "test/auth/heartbeat"); + + const objects = await client.send( + new ListObjectsV2Command({ + Bucket: s3BucketName, + Prefix: "test/auth/heartbeat", + }), + ); + + expect(objects.Contents.length).toBe(1); + }); +}); + describe("sync", () => { let client; diff --git a/docker-compose.yml b/docker-compose.yml index 99cacf1..08eccfe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,9 +35,6 @@ services: environment: MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID} MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY} - # Accessible at this domain, so we can manually check that CloudFront cookies have been set correctly. - VIRTUAL_HOST: archive.intranet.docker - VIRTUAL_PORT: "9001" command: server --console-address ":9001" /data healthcheck: test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1 @@ -58,3 +55,14 @@ services: mc anonymous set download intranet-archive/${S3_BUCKET_NAME}; exit 0 " + + cdn: + image: caddy:2-alpine + volumes: + - ./conf/local.Caddyfile:/etc/caddy/Caddyfile + environment: + S3_BUCKET_NAME: ${S3_BUCKET_NAME} + VIRTUAL_HOST: archive.intranet.docker + VIRTUAL_PORT: 2019 + depends_on: + - minio