From ab1363937323552b13af8aa1320e88a552ee7d37 Mon Sep 17 00:00:00 2001 From: EarthlingDavey <15802017+EarthlingDavey@users.noreply.github.com> Date: Fri, 6 Dec 2024 11:38:26 +0000 Subject: [PATCH] Create signed cookie for CloudFront (#29) --- .github/workflows/test.yml | 14 ++- conf/nginx.conf | 4 +- conf/node/constants.js | 8 ++ conf/node/controllers/cloudfront.js | 137 +++++++++++++++++++++-- conf/node/controllers/cloudfront.test.js | 49 ++++++++ conf/node/controllers/httrack.js | 2 +- conf/node/controllers/main.js | 2 +- conf/node/server.js | 69 +++++++----- 8 files changed, 237 insertions(+), 48 deletions(-) create mode 100644 conf/node/controllers/cloudfront.test.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ccbd825..086339c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,10 +13,18 @@ jobs: - name: Run tests run: | cp .env.ci .env - docker compose run --rm -e JWT=${JWT} spider sh -c "npm ci && npm run test" + + # Add secrets to .env + echo "JWT=$JWT" >> .env + + # These secrets are used for testing, and are not used in production. + echo "AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT=$AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT" >> .env + echo "AWS_CLOUDFRONT_PRIVATE_KEY=\"$AWS_CLOUDFRONT_PRIVATE_KEY\"" >> .env + echo "AWS_CLOUDFRONT_PUBLIC_KEY=\"$AWS_CLOUDFRONT_PUBLIC_KEY\"" >> .env + + docker compose run --rm spider sh -c "npm ci && npm run test" env: JWT: ${{ secrets.JWT }} - # Use mock AWS CloudFront keys, these do not grant permission to anything. + AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT: ${{ secrets.TEST_AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT }} AWS_CLOUDFRONT_PRIVATE_KEY: ${{ secrets.TEST_AWS_CLOUDFRONT_PRIVATE_KEY }} AWS_CLOUDFRONT_PUBLIC_KEY: ${{ secrets.TEST_AWS_CLOUDFRONT_PUBLIC_KEY }} - AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT: ${{ secrets.TEST_AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT }} diff --git a/conf/nginx.conf b/conf/nginx.conf index 53e8098..77b6481 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -23,11 +23,11 @@ server { proxy_pass http://localhost:2000/bucket-test; } - location /set-cf-cookie { + location /access-archive { proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Server $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_pass http://localhost:2000/set-cf-cookie; + proxy_pass http://localhost:2000/access-archive; } location / { diff --git a/conf/node/constants.js b/conf/node/constants.js index be86487..924c827 100644 --- a/conf/node/constants.js +++ b/conf/node/constants.js @@ -13,6 +13,14 @@ export const s3Credentials = process.env.AWS_ACCESS_KEY_ID && secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, }; +/** + * CloudFront + */ + +export const cloudFrontKeysObject = process.env.AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT; +export const cloudFrontPublicKey = process.env.AWS_CLOUDFRONT_PUBLIC_KEY; +export const cloudFrontPrivateKey = process.env.AWS_CLOUDFRONT_PRIVATE_KEY; + /** * Options */ diff --git a/conf/node/controllers/cloudfront.js b/conf/node/controllers/cloudfront.js index 7573ee9..883c7e1 100644 --- a/conf/node/controllers/cloudfront.js +++ b/conf/node/controllers/cloudfront.js @@ -1,19 +1,124 @@ -const { getSignedCookies } = require("@aws-sdk/cloudfront-signer"); +import crypto from "node:crypto"; +import { getSignedCookies } from "@aws-sdk/cloudfront-signer"; -// TODO: Implement the getCookies function +import { + cloudFrontKeysObject as keysObject, + cloudFrontPublicKey as publicKey, + cloudFrontPrivateKey as privateKey, +} from "../constants.js"; -const getCookies = () => { - const cloudfrontDistributionDomain = "https://d111111abcdef8.cloudfront.net"; - const s3ObjectKey = "private-content/private.jpeg"; - const url = `${cloudfrontDistributionDomain}/${s3ObjectKey}`; - const privateKey = "CONTENTS-OF-PRIVATE-KEY"; - const keyPairId = "PUBLIC-KEY-ID-OF-CLOUDFRONT-KEY-PAIR"; - const dateLessThan = "2022-01-01"; +/** + * @typedef {Object} CookieSet + * @property {import('@aws-sdk/cloudfront-signer').CloudfrontSignedCookiesOutput} value + * @property {number} dateLessThan - epoch in milliseconds + */ + +/** + * @typedef {Object} Cache + * @property {string|null} keyPairId - will only change on server restart + * @property {[key: string]: CookieSet} cookieSets + */ + +/** @type {Cache} */ +const cache = { + keyPairId: null, + cookieSets: {}, +}; + +/** + * Infer the CloudFront CDN URL from the app host + * + * @param {string} appHost + * @returns {URL} cdnURL - The CloudFront CDN URL + * @throws {Error} If the host is invalid + */ + +export const getCdnUrl = (appHost) => { + // Check appHost starts with `app.` + if (!appHost.startsWith("app.")) { + throw new Error("Invalid host"); + } + + const cdnHost = appHost.replace(/^app\./, ""); + + // Use regex to replace the initial app. with an empty string. + return new URL(`https://${cdnHost}`); +}; + +/** + * Get the CloudFront key pair ID from the public key and keys object + * + * @returns {string} keyPairId - The CloudFront key pair ID + */ + +export const getKeyPairId = () => { + // Return the cached value if it exists + if (cache.keyPairId) { + return cache.keyPairId; + } + + // Get sha256 hash of the public key, and get the first 8 characters. + const publicKeyShort = crypto + .createHash("sha256") + .update(`${publicKey.trim()}\n`, "utf8") + .digest("hex") + .substring(0, 8); + + const keyPairId = JSON.parse(keysObject).find( + (key) => key.comment === publicKeyShort && key.id?.length, + )?.id; + + if (!keyPairId) { + throw new Error("Key pair ID not found"); + } + + cache.keyPairId = keyPairId; + + return keyPairId; +}; + +/** + * Return the epoch value for midnight today, or midnight tomorrow if past midday + * + * @returns {number} dateLessThan - The date in seconds + */ + +export const getDateLessThan = () => { + const date = new Date(); + const hours = date.getHours(); + + if (hours >= 12) { + return parseInt(new Date(date.setHours(24, 0, 0, 0)).getTime() / 1000); + } + + // This should be midnight tomorrow. + date.setDate(date.getDate() + 1); + return parseInt(new Date(date.setHours(0, 0, 0, 0)).getTime() / 1000); +}; + +/** + * Get signed CloudFront cookies + * + * @param {Object} props + * @param {string} props.resource + * @param {number} props.dateLessThan + * @returns {import('@aws-sdk/cloudfront-signer').CloudfrontSignedCookiesOutput} cookies - The signed CloudFront cookies + */ + +export const getCookies = ({ resource, dateLessThan }) => { + // Check if the cache has a value for the resource + const cachedValue = + cache.cookieSets?.[resource]?.dateLessThan === dateLessThan; + + // Return the cached value if it exists + if (cachedValue) { + return cachedValue; + } const policy = { Statement: [ { - Resource: url, + Resource: resource, Condition: { DateLessThan: { "AWS:EpochTime": new Date(dateLessThan).getTime() / 1000, // time in seconds @@ -25,9 +130,17 @@ const getCookies = () => { const policyString = JSON.stringify(policy); - return getSignedCookies({ - keyPairId, + const signedCookies = getSignedCookies({ + keyPairId: getKeyPairId(), privateKey, policy: policyString, }); + + // Set the cache + cache.cookieSets[resource] = { + dateLessThan, + value: signedCookies, + }; + + return signedCookies; }; diff --git a/conf/node/controllers/cloudfront.test.js b/conf/node/controllers/cloudfront.test.js new file mode 100644 index 0000000..7b6f3fa --- /dev/null +++ b/conf/node/controllers/cloudfront.test.js @@ -0,0 +1,49 @@ +import { + getCdnUrl, + getKeyPairId, + getCookies, + getDateLessThan, +} from "./cloudfront.js"; + +describe("getCdnUrl", () => { + it("should return a cdn URL object", () => { + const result = getCdnUrl("app.archive.example.com"); + expect(result.host).toBe("archive.example.com"); + expect(result.origin).toBe("https://archive.example.com"); + }); + + it("should throw an error for invalid host", () => { + expect(() => getCdnUrl("archive.example.com")).toThrow("Invalid host"); + }); +}); + +describe("getKeyPairId", () => { + it("should return an id", () => { + const id = getKeyPairId(); + expect(id).toBe("GENERATED_BY_AWS"); + }); +}); + +describe("getDateLessThan", () => { + it("should return a date in the future", () => { + const date = getDateLessThan(); + expect(date).toBeGreaterThan(Date.now() / 1000); + }); +}); + +describe("getCookies", () => { + it("should return cookies for CloudFront", () => { + const dateLessThan = getDateLessThan(); + const resource = "https://archive.example.com/*"; + + const result = getCookies({ + resource, + dateLessThan, + }); + + expect(result).toBeDefined(); + expect(result["CloudFront-Key-Pair-Id"]).toBeDefined(); + expect(result["CloudFront-Policy"]).toBeDefined(); + expect(result["CloudFront-Signature"]).toBeDefined(); + }); +}); diff --git a/conf/node/controllers/httrack.js b/conf/node/controllers/httrack.js index ef524bb..98872b5 100644 --- a/conf/node/controllers/httrack.js +++ b/conf/node/controllers/httrack.js @@ -6,7 +6,7 @@ import { jwt } from "../constants.js"; /** * A helper function to get the directory for the snapshot. * - * @param {props} props + * @param {Object} props * @param {string} props.host * @param {string} props.agency * @returns {Object} object diff --git a/conf/node/controllers/main.js b/conf/node/controllers/main.js index dded6da..36d910d 100644 --- a/conf/node/controllers/main.js +++ b/conf/node/controllers/main.js @@ -12,7 +12,7 @@ import { sync } from "./s3.js"; /** * - * @param {props} props + * @param {Object} props * @param {URL} props.url * @param {string} props.agency * @param {?number} props.depth diff --git a/conf/node/server.js b/conf/node/server.js index a2d8cfb..731ed0e 100644 --- a/conf/node/server.js +++ b/conf/node/server.js @@ -15,6 +15,11 @@ import express from "express"; import { corsOptions, jwt, port } from "./constants.js"; import { parseBody } from "./middleware.js"; import { checkAccess as checkS3Access } from "./controllers/s3.js"; +import { + getCdnUrl, + getCookies, + getDateLessThan, +} from "./controllers/cloudfront.js"; import { main } from "./controllers/main.js"; const app = express(); @@ -62,40 +67,46 @@ app.post("/bucket-test", async function (_req, res, next) { } }); -app.post("/set-cf-cookie", async function (request, response) { - // Get the current domain from the request - const appHost = request.get("X-Forwarded-Host") || request.get("host"); - if (!appHost.startsWith("app.")) { - console.error("Invalid host"); - response.status(400).send({ status: 400 }); - return; - } - - // Use regex to replace the initial app. with an empty string. - // e.g. app.archive.intranet.docker -> archive.intranet.docker - const cdnHost = appHost.replace(/^app\./, ""); +app.post("/spider", function (req, res) { + // Start the main function - without awiting for the result. + main(req.mirror); + // Handle the response + res.status(200).sendFile(path.join("/usr/share/nginx/html/working.html")); +}); - // TODO Generate CloudFront cookies. +app.get("/access-archive", async function (req, res, next) { + try { + // Get the current domain from the request + const appHost = req.headers["x-forwarded-host"] || req.headers["host"]; + + // Get the CloudFront CDN URL + const cdnUrl = getCdnUrl(appHost); + + // Get the CloudFront cookies + const cookies = getCookies({ + resource: `${cdnUrl.origin}/*`, + dateLessThan: getDateLessThan(), + }); - // Set the cookie on the response - // response.cookie('jwt', jwt, { - // domain: cdnHost, - // secure: true, - // sameSite: 'None', - // httpOnly: true, - // }); + // Set the cookies on the response + Object.entries(cookies).forEach(([name, value]) => { + res.cookie(name, value, { + path: "/", + domain: cdnUrl.host, + secure: true, + sameSite: "Lax", + httpOnly: true, + }); + }); - response.status(200).send({ appHost, cdnHost }); -}); + // Send a metadata html tag to redirect to the cdnUrl + const html = ``; -app.post("/spider", function (request, response) { - // Start the main function - without awiting for the result. - main(request.mirror); - // Handle the response - response - .status(200) - .sendFile(path.join("/usr/share/nginx/html/working.html")); + res.status(200).send(html); + } catch (err) { + next(err); + } }); app.listen(port);