diff --git a/config.ts b/config.ts index 17cb4db8ba..211a295e51 100644 --- a/config.ts +++ b/config.ts @@ -1,5 +1,6 @@ import type { ConfigLoader } from '@balena/pinejs'; import * as balenaModel from './src/balena'; +import { getFileUploadHandler } from './src/fileupload-handler'; export = { models: [balenaModel], @@ -25,4 +26,5 @@ export = { ], }, ], + webResourceHandler: getFileUploadHandler(), } as ConfigLoader.Config; diff --git a/config/confd/conf.d/cloudront-pk.pem.toml b/config/confd/conf.d/cloudront-pk.pem.toml new file mode 100644 index 0000000000..8127f27590 --- /dev/null +++ b/config/confd/conf.d/cloudront-pk.pem.toml @@ -0,0 +1,7 @@ +[template] +src = "cloudfront-pk.pem.tmpl" +dest = "/etc/ssl/private/cloudfront-pk.pem" +keys = [ + "WEBRESOURCES_CLOUDFRONT_PRIVATEKEY", +] +mode = "0400" diff --git a/config/confd/conf.d/env.toml b/config/confd/conf.d/env.toml index e7d28b0fc6..44d08be616 100644 --- a/config/confd/conf.d/env.toml +++ b/config/confd/conf.d/env.toml @@ -53,5 +53,14 @@ keys = [ "VPN_PORT", "VPN_SERVICE_API_KEY", "VPN_GUEST_API_KEY", - "VPN_CONNECT_PROXY_PORT" + "VPN_CONNECT_PROXY_PORT", + "WEBRESOURCES_S3_ACCESS_KEY", + "WEBRESOURCES_S3_SECRET_KEY", + "WEBRESOURCES_S3_REGION", + "WEBRESOURCES_S3_HOST", + "WEBRESOURCES_S3_BUCKET", + "WEBRESOURCES_S3_MAX_FILESIZE", + "WEBRESOURCES_CLOUDFRONT_PUBLICKEY", + "WEBRESOURCES_CLOUDFRONT_HOST", + "WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH" ] diff --git a/config/confd/templates/cloudfront-pk.pem.tmpl b/config/confd/templates/cloudfront-pk.pem.tmpl new file mode 100644 index 0000000000..068ad8bd9f --- /dev/null +++ b/config/confd/templates/cloudfront-pk.pem.tmpl @@ -0,0 +1 @@ +{{base64Decode (getenv "WEBRESOURCES_CLOUDFRONT_PRIVATEKEY" "")}} diff --git a/config/confd/templates/env.tmpl b/config/confd/templates/env.tmpl index f6284dba7f..123bbdbdb7 100644 --- a/config/confd/templates/env.tmpl +++ b/config/confd/templates/env.tmpl @@ -79,3 +79,12 @@ VPN_SERVICE_API_KEY={{getenv "VPN_SERVICE_API_KEY"}} {{if getenv "VPN_GUEST_API_KEY"}}VPN_GUEST_API_KEY={{getenv "VPN_GUEST_API_KEY"}}{{end}} {{if getenv "AUTH_RESINOS_REGISTRY_CODE"}}AUTH_RESINOS_REGISTRY_CODE={{getenv "AUTH_RESINOS_REGISTRY_CODE"}}{{end}} {{if getenv "BROTLI_COMPRESSION_QUALITY"}}BROTLI_COMPRESSION_QUALITY={{getenv "BROTLI_COMPRESSION_QUALITY"}}{{end}} +{{if getenv "WEBRESOURCES_S3_ACCESS_KEY"}}WEBRESOURCES_S3_ACCESS_KEY={{getenv "WEBRESOURCES_S3_ACCESS_KEY"}}{{end}} +{{if getenv "WEBRESOURCES_S3_SECRET_KEY"}}WEBRESOURCES_S3_SECRET_KEY={{getenv "WEBRESOURCES_S3_SECRET_KEY"}}{{end}} +{{if getenv "WEBRESOURCES_S3_REGION"}}WEBRESOURCES_S3_REGION={{getenv "WEBRESOURCES_S3_REGION"}}{{end}} +{{if getenv "WEBRESOURCES_S3_HOST"}}WEBRESOURCES_S3_HOST={{getenv "WEBRESOURCES_S3_HOST"}}{{end}} +{{if getenv "WEBRESOURCES_S3_BUCKET"}}WEBRESOURCES_S3_BUCKET={{getenv "WEBRESOURCES_S3_BUCKET"}}{{end}} +{{if getenv "WEBRESOURCES_S3_MAX_FILESIZE"}}WEBRESOURCES_S3_MAX_FILESIZE={{getenv "WEBRESOURCES_S3_MAX_FILESIZE"}}{{end}} +{{if getenv "WEBRESOURCES_CLOUDFRONT_PUBLICKEY"}}WEBRESOURCES_CLOUDFRONT_PUBLICKEY={{getenv "WEBRESOURCES_CLOUDFRONT_PUBLICKEY"}}{{end}} +{{if getenv "WEBRESOURCES_CLOUDFRONT_HOST"}}WEBRESOURCES_CLOUDFRONT_HOST={{getenv "WEBRESOURCES_CLOUDFRONT_HOST"}}{{end}} +WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH=/etc/ssl/private/cloudfront-pk.pem diff --git a/package-lock.json b/package-lock.json index 943488ab32..c8ba55cb06 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "@balena/env-parsing": "^1.1.5", "@balena/es-version": "^1.0.2", "@balena/node-metrics-gatherer": "^6.0.3", - "@balena/pinejs": "^15.3.4", + "@balena/pinejs": "15.3.8-build-fixes-expand-on-parent-resource-0cca44384bf053447b7778ed3c2d78ef2fe8559e-1 build-fixes-expand-on-parent-resource", + "@balena/pinejs-webresource-cloudfront": "^0.0.4", "@sentry/node": "^7.49.0", "@types/basic-auth": "^1.1.3", "@types/bluebird": "^3.5.38", @@ -484,6 +485,17 @@ "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", "optional": true }, + "node_modules/@aws-sdk/cloudfront-signer": { + "version": "3.398.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/cloudfront-signer/-/cloudfront-signer-3.398.0.tgz", + "integrity": "sha512-fyUty9SNI3oiOSvgVcK0S2OmihawzqWCR5TdcZ2EWbpiLk0V94U5BaKoIKu6jYY+57OgGjr/vUDrDideT/0cMw==", + "dependencies": { + "@smithy/url-parser": "^2.0.5" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@aws-sdk/config-resolver": { "version": "3.200.0", "resolved": "https://registry.npmjs.org/@aws-sdk/config-resolver/-/config-resolver-3.200.0.tgz", @@ -2907,12 +2919,92 @@ } }, "node_modules/@balena/pinejs": { - "version": "15.3.4", - "resolved": "https://registry.npmjs.org/@balena/pinejs/-/pinejs-15.3.4.tgz", - "integrity": "sha512-kFEUeg3DduSR6vIPO/kkuUs/T4LkklVreXLKKDjHuQ6IOqIxHmJcej4S5PptmAjNVVeHD6HfMr4Ke07uY6TbLg==", + "version": "15.3.8-build-fixes-expand-on-parent-resource-0cca44384bf053447b7778ed3c2d78ef2fe8559e-1", + "resolved": "https://registry.npmjs.org/@balena/pinejs/-/pinejs-15.3.8-build-fixes-expand-on-parent-resource-0cca44384bf053447b7778ed3c2d78ef2fe8559e-1.tgz", + "integrity": "sha512-cwXHTHcfG9c0RaNFCtl/uzsL2KXxolJPK640TIF3ojV1sXNA23skyLrzwPPZ5netEH+qnnWfdoT83+HsN1wu8Q==", "dependencies": { "@balena/abstract-sql-compiler": "^9.0.3", - "@balena/abstract-sql-to-typescript": "^2.1.0", + "@balena/abstract-sql-to-typescript": "^2.1.1", + "@balena/env-parsing": "^1.1.5", + "@balena/lf-to-abstract-sql": "^5.0.0", + "@balena/odata-parser": "^3.0.0", + "@balena/odata-to-abstract-sql": "^6.0.1", + "@balena/sbvr-parser": "^1.4.3", + "@balena/sbvr-types": "^6.0.0", + "@types/body-parser": "^1.19.2", + "@types/compression": "^1.7.2", + "@types/cookie-parser": "^1.4.3", + "@types/deep-freeze": "^0.1.2", + "@types/express": "^4.17.17", + "@types/express-session": "^1.17.7", + "@types/lodash": "^4.14.194", + "@types/memoizee": "^0.4.8", + "@types/method-override": "^0.0.32", + "@types/multer": "^1.4.7", + "@types/mysql": "^2.15.21", + "@types/node": "^18.16.1", + "@types/passport": "^1.0.12", + "@types/passport-local": "^1.0.35", + "@types/passport-strategy": "^0.2.35", + "@types/pg": "^8.6.6", + "@types/randomstring": "^1.1.8", + "@types/websql": "^0.0.27", + "busboy": "^1.6.0", + "commander": "^10.0.1", + "deep-freeze": "^0.0.1", + "eventemitter3": "^5.0.0", + "express-session": "^1.17.3", + "lodash": "^4.17.21", + "memoizee": "^0.4.15", + "pinejs-client-core": "^6.13.0", + "randomstring": "^1.2.3", + "type-is": "^1.6.18", + "typed-error": "^3.2.2" + }, + "bin": { + "abstract-sql-compiler": "bin/abstract-sql-compiler.js", + "odata-compiler": "bin/odata-compiler.js", + "sbvr-compiler": "bin/sbvr-compiler.js" + }, + "engines": { + "node": ">=16.13.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@aws-sdk/client-s3": "^3.200.0", + "@aws-sdk/lib-storage": "^3.200.0", + "@aws-sdk/s3-request-presigner": "^3.200.0", + "bcrypt": "^5.1.0", + "body-parser": "^1.20.2", + "compression": "^1.7.4", + "cookie-parser": "^1.4.6", + "express": "^4.18.2", + "method-override": "^3.0.0", + "mysql": "^2.18.1", + "passport": "^0.6.0", + "passport-local": "^1.0.0", + "pg": "^8.10.0", + "pg-connection-string": "^2.5.0", + "serve-static": "^1.15.0" + } + }, + "node_modules/@balena/pinejs-webresource-cloudfront": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@balena/pinejs-webresource-cloudfront/-/pinejs-webresource-cloudfront-0.0.4.tgz", + "integrity": "sha512-pACee791NLsxih/4nwZe6dIyWy4UP+F41ZoyO9udIIUlHJ0pZin7mviWAuRoAqyfT3iC0Nhm0r/0CcT4nA08kg==", + "dependencies": { + "@aws-sdk/cloudfront-signer": "^3.398.0", + "@balena/pinejs": "^15.3.3", + "memoizee": "^0.4.15" + } + }, + "node_modules/@balena/pinejs-webresource-cloudfront/node_modules/@balena/pinejs": { + "version": "15.3.7", + "resolved": "https://registry.npmjs.org/@balena/pinejs/-/pinejs-15.3.7.tgz", + "integrity": "sha512-rarHogcBXL/nCp+3HwSkGeCgntpQAbH037eISW88ZS20UcsXerJmT+ZfrmPmfAfRnY5hsuiJ930idTBGV1qLfQ==", + "dependencies": { + "@balena/abstract-sql-compiler": "^9.0.3", + "@balena/abstract-sql-to-typescript": "^2.1.1", "@balena/env-parsing": "^1.1.5", "@balena/lf-to-abstract-sql": "^5.0.0", "@balena/odata-parser": "^3.0.0", @@ -3981,12 +4073,11 @@ "optional": true }, "node_modules/@smithy/querystring-parser": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-2.0.1.tgz", - "integrity": "sha512-h+e7k1z+IvI2sSbUBG9Aq46JsgLl4UqIUl6aigAlRBj+P6ocNXpM6Yn1vMBw5ijtXeZbYpd1YvCxwDgdw3jhmg==", - "optional": true, + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-2.0.5.tgz", + "integrity": "sha512-C2stCULH0r54KBksv3AWcN8CLS3u9+WsEW8nBrvctrJ5rQTNa1waHkffpVaiKvcW2nP0aIMBPCobD/kYf/q9mA==", "dependencies": { - "@smithy/types": "^2.0.2", + "@smithy/types": "^2.2.2", "tslib": "^2.5.0" }, "engines": { @@ -4046,10 +4137,9 @@ "optional": true }, "node_modules/@smithy/types": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.0.2.tgz", - "integrity": "sha512-wcymEjIXQ9+NEfE5Yt5TInAqe1o4n+Nh+rh00AwoazppmUt8tdo6URhc5gkDcOYrcvlDVAZE7uG69nDpEGUKxw==", - "optional": true, + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.2.2.tgz", + "integrity": "sha512-4PS0y1VxDnELGHGgBWlDksB2LJK8TG8lcvlWxIsgR+8vROI7Ms8h1P4FQUx+ftAX2QZv5g1CJCdhdRmQKyonyw==", "dependencies": { "tslib": "^2.5.0" }, @@ -4064,13 +4154,12 @@ "optional": true }, "node_modules/@smithy/url-parser": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.1.tgz", - "integrity": "sha512-NpHVOAwddo+OyyIoujDL9zGL96piHWrTNXqltWmBvlUoWgt1HPyBuKs6oHjioyFnNZXUqveTOkEEq0U5w6Uv8A==", - "optional": true, + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.5.tgz", + "integrity": "sha512-OdMBvZhpckQSkugCXNJQCvqJ71wE7Ftxce92UOQLQ9pwF6hoS5PLL7wEfpnuEXtStzBqJYkzu1C1ZfjuFGOXAA==", "dependencies": { - "@smithy/querystring-parser": "^2.0.1", - "@smithy/types": "^2.0.2", + "@smithy/querystring-parser": "^2.0.5", + "@smithy/types": "^2.2.2", "tslib": "^2.5.0" } }, diff --git a/package.json b/package.json index 06a1815e1b..e0368f217f 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,8 @@ "@balena/env-parsing": "^1.1.5", "@balena/es-version": "^1.0.2", "@balena/node-metrics-gatherer": "^6.0.3", - "@balena/pinejs": "^15.3.4", + "@balena/pinejs": "15.3.8-build-fixes-expand-on-parent-resource-0cca44384bf053447b7778ed3c2d78ef2fe8559e-1 build-fixes-expand-on-parent-resource", + "@balena/pinejs-webresource-cloudfront": "^0.0.4", "@sentry/node": "^7.49.0", "@types/basic-auth": "^1.1.3", "@types/bluebird": "^3.5.38", diff --git a/src/fileupload-handler.ts b/src/fileupload-handler.ts new file mode 100644 index 0000000000..dda78416e2 --- /dev/null +++ b/src/fileupload-handler.ts @@ -0,0 +1,102 @@ +import { webResourceHandler } from '@balena/pinejs'; +import { + CloudFrontHandler, + CloudFrontHandlerProps, +} from '@balena/pinejs-webresource-cloudfront'; +import * as fs from 'fs'; + +import { + WEBRESOURCES_S3_ACCESS_KEY, + WEBRESOURCES_S3_SECRET_KEY, + WEBRESOURCES_S3_REGION, + WEBRESOURCES_S3_HOST, + WEBRESOURCES_S3_BUCKET, + WEBRESOURCES_S3_MAX_FILESIZE, + WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH, + WEBRESOURCES_CLOUDFRONT_PUBLICKEY, + WEBRESOURCES_CLOUDFRONT_HOST, +} from './lib/config'; + +const getEndpointFromHost = (host: string): string => { + return host.startsWith('http') ? host : `https://${host}`; +}; + +const getS3Config = (): webResourceHandler.S3HandlerProps | undefined => { + if ( + WEBRESOURCES_S3_ACCESS_KEY != null && + WEBRESOURCES_S3_SECRET_KEY != null && + WEBRESOURCES_S3_REGION != null && + WEBRESOURCES_S3_HOST != null && + WEBRESOURCES_S3_BUCKET != null + ) { + return { + endpoint: getEndpointFromHost(WEBRESOURCES_S3_HOST), + accessKey: WEBRESOURCES_S3_ACCESS_KEY, + secretKey: WEBRESOURCES_S3_SECRET_KEY, + region: WEBRESOURCES_S3_REGION, + bucket: WEBRESOURCES_S3_BUCKET, + maxSize: WEBRESOURCES_S3_MAX_FILESIZE, + }; + } +}; + +const getCloudfrontConfig = (): CloudFrontHandlerProps | undefined => { + const s3Config = getS3Config(); + if ( + s3Config != null && + WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH != null && + WEBRESOURCES_CLOUDFRONT_PUBLICKEY != null && + WEBRESOURCES_CLOUDFRONT_HOST != null + ) { + let cfSecretKey: string; + try { + cfSecretKey = fs.readFileSync( + WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH, + 'utf-8', + ); + } catch (e) { + console.error('Failed to start cloudfront with error', e); + return; + } + + return { + cfDistributionDomain: getEndpointFromHost(WEBRESOURCES_CLOUDFRONT_HOST), + cfPublicKeyId: WEBRESOURCES_CLOUDFRONT_PUBLICKEY, + cfSecretKey, + ...s3Config, + }; + } +}; + +let handler: webResourceHandler.WebResourceHandler | undefined; +export const getFileUploadHandler = () => { + if (handler == null) { + const cfConfig = getCloudfrontConfig(); + if (cfConfig != null) { + handler = new CloudFrontHandler(cfConfig); + console.log('Successfully initialised webresource CloudFront handler.'); + console.log({ + region: cfConfig.region, + endpoint: cfConfig.endpoint, + bucket: cfConfig.bucket, + cfHost: cfConfig.cfDistributionDomain, + }); + return handler; + } + + const s3Config = getS3Config(); + if (s3Config != null) { + handler = new webResourceHandler.S3Handler(s3Config); + console.log('Successfully initialised webresource S3 handler.'); + console.log({ + region: s3Config.region, + endpoint: s3Config.endpoint, + bucket: s3Config.bucket, + }); + return handler; + } + + console.log('No webresource handler loaded.'); + } + return handler; +}; diff --git a/src/index.ts b/src/index.ts index 1f62777e5d..3617e1e527 100644 --- a/src/index.ts +++ b/src/index.ts @@ -164,6 +164,7 @@ export * as scheduler from './infra/scheduler'; export * as cache from './infra/cache'; export * as config from './lib/config'; export * as abstractSql from './abstract-sql-utils'; +export { getFileUploadHandler } from './fileupload-handler'; export * as deviceState from './features/device-state'; export const errors = { diff --git a/src/lib/config.ts b/src/lib/config.ts index def2a5009a..25ce84ea27 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -416,6 +416,29 @@ export const IGNORE_FROZEN_DEVICE_PERMISSIONS = boolVar( false, ); +export const WEBRESOURCES_S3_HOST = optionalVar('WEBRESOURCES_S3_HOST'); +export const WEBRESOURCES_S3_REGION = optionalVar('WEBRESOURCES_S3_REGION'); +export const WEBRESOURCES_S3_ACCESS_KEY = optionalVar( + 'WEBRESOURCES_S3_ACCESS_KEY', +); +export const WEBRESOURCES_S3_SECRET_KEY = optionalVar( + 'WEBRESOURCES_S3_SECRET_KEY', +); +export const WEBRESOURCES_S3_BUCKET = optionalVar('WEBRESOURCES_S3_BUCKET'); +export const WEBRESOURCES_S3_MAX_FILESIZE = intVar( + 'WEBRESOURCES_S3_MAX_FILESIZE', + 10000000, +); +export const WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH = optionalVar( + 'WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH', +); +export const WEBRESOURCES_CLOUDFRONT_PUBLICKEY = optionalVar( + 'WEBRESOURCES_CLOUDFRONT_PUBLICKEY', +); +export const WEBRESOURCES_CLOUDFRONT_HOST = optionalVar( + 'WEBRESOURCES_CLOUDFRONT_HOST', +); + /** * Splits an env var in the format of `${username}:${password}` * into a RedisAuth object. Auth is optional, so this can return