Add content types during S3 sync. (#38)

ministryofjustice · Dec 6, 2024 · 6e4d379 · 6e4d379
1 parent 314edd0
commit 6e4d379
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 9 deletions.
diff --git a/conf/node/controllers/main.js b/conf/node/controllers/main.js
@@ -21,7 +21,7 @@ import { sync } from "./s3.js";
 export const main = async ({ url, agency, depth }) => {
   const paths = getSnapshotPaths({ host: url.host, agency });
 
-  const { complete } = getHttrackProgress(paths.fs);
+  const { complete } = await getHttrackProgress(paths.fs);
 
   // If the snapshot is already complete, skip httrack
   if (!complete) {

diff --git a/conf/node/controllers/s3.js b/conf/node/controllers/s3.js
@@ -1,6 +1,7 @@
 import fs from "fs/promises";
 import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3";
 import { S3SyncClient } from "s3-sync-client";
+import mime from "mime-types";
 
 import {
   s3BucketName,
@@ -36,18 +37,33 @@ export const client = new S3Client(s3Options);
 /**
  * S3 sync client
  *
- * @type {S3SyncClient}
- * 
  * @see https://github.com/jeanbmar/s3-sync-client
  */
 
-const { sync } = new S3SyncClient({ client });
+const { sync: originalSync } = new S3SyncClient({ client });
 
-export { sync };
+/**
+ * Sync a local directory to an S3 bucket
+ *
+ * @param {string} source - The source directory
+ * @param {string} destination - The destination directory
+ * @param {?import('s3-sync-client/dist/commands/SyncCommand').SyncOptions} options - The sync options
+ *
+ * @returns {Promise<import('s3-sync-client/dist/commands/SyncCommand').SyncCommandOutput>}
+ */
+
+export const sync = async (source, destination, options = {}) => {
+  // Set the content type for each file
+  options.commandInput = (input) => ({
+    ContentType: mime.lookup(input.Key) || "text/html",
+  });
+
+  return originalSync(source, destination, options);
+};
 
 /**
  * Empty an S3 folder by using sync and deleting all files
- * 
+ *
  * @param {string} path - The path to empty
  */
 
@@ -63,7 +79,7 @@ export const s3EmptyDir = async (path) => {
 
   // Remove the empty directory
   await fs.rm(emptyDir, { recursive: true });
-}
+};
 
 /**
  * Check if the bucket is accessible by listing the root of the bucket

diff --git a/conf/node/controllers/s3.test.js b/conf/node/controllers/s3.test.js
@@ -7,8 +7,8 @@ import {
   ListObjectsV2Command,
 } from "@aws-sdk/client-s3";
 
-import { s3Options, checkAccess, sync, s3EmptyDir } from "./s3";
 import { s3BucketName } from "../constants.js";
+import { s3Options, checkAccess, sync, s3EmptyDir } from "./s3";
 
 describe("checkAccess", () => {
   let client;
@@ -55,11 +55,17 @@ describe("sync", () => {
 
     // Create a test file in /tmp/s3-test
     await fs.promises.writeFile("/tmp/s3-test/test.txt", fileContent);
+
+    await fs.promises.writeFile(
+      "/tmp/s3-test/test.html",
+      "<html><body><h1>Hello, World!</h1></body></html>",
+    );
   });
 
   afterAll(async () => {
-    // Remove the test file
+    // Remove the test files
     await fs.promises.unlink("/tmp/s3-test/test.txt");
+    await fs.promises.unlink("/tmp/s3-test/test.html");
 
     await client.destroy();
   });
@@ -75,6 +81,28 @@ describe("sync", () => {
 
     expect(bodyString).toBe(fileContent);
   });
+
+  it("should add content type to the destination files", async () => {
+    await sync("/tmp/s3-test", `s3://${s3BucketName}/test-types`);
+
+    const object = await client.send(
+      new GetObjectCommand({
+        Bucket: s3BucketName,
+        Key: "test-types/test.txt",
+      }),
+    );
+
+    expect(object.ContentType).toBe("text/plain");
+
+    const htmlObject = await client.send(
+      new GetObjectCommand({
+        Bucket: s3BucketName,
+        Key: "test-types/test.html",
+      }),
+    );
+
+    expect(htmlObject.ContentType).toBe("text/html");
+  });
 });
 
 describe("S3EmptyDir", () => {

diff --git a/conf/node/package-lock.json b/conf/node/package-lock.json
diff --git a/conf/node/package.json b/conf/node/package.json
@@ -24,6 +24,7 @@
     "@aws-sdk/cloudfront-signer": "^3.696.0",
     "cors": "^2.8.5",
     "express": "^4.21.1",
+    "mime-types": "^2.1.35",
     "s3-sync-client": "^4.3.1"
   },
   "devDependencies": {