From 1525f9217c83b662909619adb124b5d3dd60ba72 Mon Sep 17 00:00:00 2001 From: Nicolas Froidure Date: Mon, 30 Mar 2020 10:06:12 +0200 Subject: [PATCH] feat(@whook/gcp-functions): add Google Cloud Functions build --- packages/whook-aws-lambda/API.md | 54 -- packages/whook-aws-lambda/README.md | 54 -- packages/whook-gcp-functions/API.md | 1 + packages/whook-gcp-functions/LICENSE | 20 + packages/whook-gcp-functions/README.md | 151 +++++ .../whook-gcp-functions/package-lock.json | 37 ++ packages/whook-gcp-functions/package.json | 198 +++++++ .../src/commands/testHTTPFunction.ts | 181 ++++++ packages/whook-gcp-functions/src/index.ts | 413 +++++++++++++ .../whook-gcp-functions/src/libs/utils.ts | 42 ++ .../src/services/_autoload.ts | 117 ++++ .../src/services/compiler.ts | 220 +++++++ .../src/services/log.test.ts | 7 + .../whook-gcp-functions/src/services/log.ts | 4 + .../src/wrappers/googleHTTPFunction.ts | 549 ++++++++++++++++++ packages/whook-gcp-functions/tsconfig.json | 19 + packages/whook-graphql/API.md | 29 - packages/whook-graphql/README.md | 29 - packages/whook/API.md | 2 +- packages/whook/README.md | 2 +- 20 files changed, 1961 insertions(+), 168 deletions(-) create mode 100644 packages/whook-gcp-functions/API.md create mode 100644 packages/whook-gcp-functions/LICENSE create mode 100644 packages/whook-gcp-functions/README.md create mode 100644 packages/whook-gcp-functions/package-lock.json create mode 100644 packages/whook-gcp-functions/package.json create mode 100644 packages/whook-gcp-functions/src/commands/testHTTPFunction.ts create mode 100644 packages/whook-gcp-functions/src/index.ts create mode 100644 packages/whook-gcp-functions/src/libs/utils.ts create mode 100644 packages/whook-gcp-functions/src/services/_autoload.ts create mode 100644 packages/whook-gcp-functions/src/services/compiler.ts create mode 100644 packages/whook-gcp-functions/src/services/log.test.ts create mode 100644 packages/whook-gcp-functions/src/services/log.ts create mode 100644 packages/whook-gcp-functions/src/wrappers/googleHTTPFunction.ts create mode 100644 packages/whook-gcp-functions/tsconfig.json diff --git a/packages/whook-aws-lambda/API.md b/packages/whook-aws-lambda/API.md index 5956cfa9..59327929 100644 --- a/packages/whook-aws-lambda/API.md +++ b/packages/whook-aws-lambda/API.md @@ -1,55 +1 @@ # API -## Members - -
-
defaultPromise.<Object>
-

Wrap the ENV service in order to filter ENV vars for the build

-
-
- -## Functions - -
-
initBuildConstants(constants)Promise.<Object>
-

Allow to proxy constants directly by serializing it in the - build, saving some computing and increasing boot time of - lambdas.

-
-
- - - -## default ⇒ Promise.<Object> -Wrap the ENV service in order to filter ENV vars for the build - -**Kind**: global variable -**Returns**: Promise.<Object> - A promise of an object containing the reshaped env vars. - -| Param | Type | Default | Description | -| --- | --- | --- | --- | -| services | Object | | The services ENV depends on | -| services.NODE_ENV | Object | | The injected NODE_ENV value to add it to the build env | -| [services.PROXYED_ENV_VARS] | Object | {} | A list of environment variable names to proxy | -| [log] | Object | noop | An optional logging service | - - - -## initBuildConstants(constants) ⇒ Promise.<Object> -Allow to proxy constants directly by serializing it in the - build, saving some computing and increasing boot time of - lambdas. - -**Kind**: global function -**Returns**: Promise.<Object> - A promise of an object containing the gathered constants. - -| Param | Type | Description | -| --- | --- | --- | -| constants | Object | The serializable constants to gather | - -**Example** -```js -import { initBuildConstants } from '@whook/aws-lambda'; -import { alsoInject } from 'knifecycle'; - -export default alsoInject(['MY_OWN_CONSTANT'], initBuildConstants); -``` diff --git a/packages/whook-aws-lambda/README.md b/packages/whook-aws-lambda/README.md index bc2e0d3a..946b09c1 100644 --- a/packages/whook-aws-lambda/README.md +++ b/packages/whook-aws-lambda/README.md @@ -110,60 +110,6 @@ There is a complete example on how to deploy your lambdas [//]: # (::contents:end) # API -## Members - -
-
defaultPromise.<Object>
-

Wrap the ENV service in order to filter ENV vars for the build

-
-
- -## Functions - -
-
initBuildConstants(constants)Promise.<Object>
-

Allow to proxy constants directly by serializing it in the - build, saving some computing and increasing boot time of - lambdas.

-
-
- - - -## default ⇒ Promise.<Object> -Wrap the ENV service in order to filter ENV vars for the build - -**Kind**: global variable -**Returns**: Promise.<Object> - A promise of an object containing the reshaped env vars. - -| Param | Type | Default | Description | -| --- | --- | --- | --- | -| services | Object | | The services ENV depends on | -| services.NODE_ENV | Object | | The injected NODE_ENV value to add it to the build env | -| [services.PROXYED_ENV_VARS] | Object | {} | A list of environment variable names to proxy | -| [log] | Object | noop | An optional logging service | - - - -## initBuildConstants(constants) ⇒ Promise.<Object> -Allow to proxy constants directly by serializing it in the - build, saving some computing and increasing boot time of - lambdas. - -**Kind**: global function -**Returns**: Promise.<Object> - A promise of an object containing the gathered constants. - -| Param | Type | Description | -| --- | --- | --- | -| constants | Object | The serializable constants to gather | - -**Example** -```js -import { initBuildConstants } from '@whook/aws-lambda'; -import { alsoInject } from 'knifecycle'; - -export default alsoInject(['MY_OWN_CONSTANT'], initBuildConstants); -``` # Authors - [Nicolas Froidure](http://insertafter.com/en/index.html) diff --git a/packages/whook-gcp-functions/API.md b/packages/whook-gcp-functions/API.md new file mode 100644 index 00000000..59327929 --- /dev/null +++ b/packages/whook-gcp-functions/API.md @@ -0,0 +1 @@ +# API diff --git a/packages/whook-gcp-functions/LICENSE b/packages/whook-gcp-functions/LICENSE new file mode 100644 index 00000000..df9966e5 --- /dev/null +++ b/packages/whook-gcp-functions/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) +Copyright © 2017 Nicolas Froidure + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/whook-gcp-functions/README.md b/packages/whook-gcp-functions/README.md new file mode 100644 index 00000000..c91ed6b8 --- /dev/null +++ b/packages/whook-gcp-functions/README.md @@ -0,0 +1,151 @@ +[//]: # ( ) +[//]: # (This file is automatically generated by a `metapak`) +[//]: # (module. Do not change it except between the) +[//]: # (`content:start/end` flags, your changes would) +[//]: # (be overridden.) +[//]: # ( ) +# @whook/gcp-functions +> Build and deploy to GCP Cloud Functions with Whook. + +[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/nfroidure/whook/blob/master/packages/whook-gcp-functions/LICENSE) +[![NPM version](https://badge.fury.io/js/%40whook%2Fgcp-functions.svg)](https://npmjs.org/package/@whook/gcp-functions) + + +[//]: # (::contents:start) + +This module is aimed to help you to build and deploy your Whook server + to [Google Cloud Functions](https://cloud.google.com/functions). + +You can find a complete setup with a Terraform deployment example in + [this pull request](https://github.com/nfroidure/whook/pull/66). + +## Quick setup + +Install this module and its peer dependencies : +```sh +npm i @whook/gcp-functions; +npm i --save-dev @whook/http-transaction babel-loader babel-plugin-knifecycle webpack +``` + +Add this module to your Whook plugins and tweak the 2 build functions + in your `index.ts` main file: +```diff +import { + runBuild as runBaseBuild, + prepareBuildEnvironment as prepareBaseBuildEnvironment, +} from '@whook/gcp-functions'; + +// (...) + +export async function prepareEnvironment( + $: Knifecycle = new Knifecycle(), +): Promise { + + // (...) + + // Setup your own whook plugins or avoid whook defaults by leaving it empty +- $.register(constant('WHOOK_PLUGINS', ['@whook/cli', '@whook/whook'])); ++ $.register(constant('WHOOK_PLUGINS', [ ++ '@whook/gcp-functions', ++ '@whook/cli', ++ '@whook/whook', ++ ])); + + // (...) + +} + +// (...) + +// The `runBuild` function is intended to build the +// project +export async function runBuild( + innerPrepareEnvironment = prepareBuildEnvironment, +): Promise { + throw new YError('E_NO_BUILD_IMPLEMENTED'); + + // Usually, here you call the installed build +- // return runBaseBuild(innerPrepareEnvironment); ++ return runBaseBuild(innerPrepareEnvironment); +} + +// (...) + +// The `prepareBuildEnvironment` create the build +// environment +export async function prepareBuildEnvironment( + $: Knifecycle = new Knifecycle(), +): Promise { + $ = await prepareEnvironment($); + + // (...) + +- // Usually, here you call the installed build env +- // $ = await prepareBaseBuildEnvironment($); ++ // Calling the GCP specific build ++ $ = await prepareBaseBuildEnvironment($); + + + // The build often need to know were initializer + // can be found to create a static build and + // remove the need to create an injector + $.register( + constant('INITIALIZER_PATH_MAP', { + ENV: require.resolve('@whook/whook/dist/services/ProxyedENV'), + apm: require.resolve('@whook/http-transaction/dist/services/apm'), + obfuscator: require.resolve( + '@whook/http-transaction/dist/services/obfuscator', + ), +- log: require.resolve('common-services/dist/log'), ++ log: require.resolve('@whook/gcp-functions/dist/services/log'), + time: require.resolve('common-services/dist/time'), + delay: require.resolve('common-services/dist/delay'), + }), + ); + + // (...) + +} +``` + +# Build + +To build your functions : +```sh +# Build all functions +npm run compile && npm run build +# Build only one function +npm run compile && npm run build -- getPing +``` + +# Debug + +You can easily test your function builds by adding `@whook/gcp-functions` + to your `WHOOK_PLUGINS` list. It provides you some commands like + the `testHTTPFunction` one: +```sh +npx whook testHTTPFunction --name getPing +``` + +To get more insights when errors happens: +```sh +npm run whook-dev -- testHTTPFunction --name getPing +``` + +## Deployment + +We recommend using [Terraform](https://terraform.io) to deploy your + functions. + +There is a complete example on how to deploy your functions + [in this pull request](https://github.com/nfroidure/whook/pull/54). + +[//]: # (::contents:end) + +# API + +# Authors +- [Nicolas Froidure](http://insertafter.com/en/index.html) + +# License +[MIT](https://github.com/nfroidure/whook/blob/master/packages/whook-gcp-functions/LICENSE) diff --git a/packages/whook-gcp-functions/package-lock.json b/packages/whook-gcp-functions/package-lock.json new file mode 100644 index 00000000..49093391 --- /dev/null +++ b/packages/whook-gcp-functions/package-lock.json @@ -0,0 +1,37 @@ +{ + "name": "@whook/gcp-functions", + "version": "3.1.3", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "strict-qs": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/strict-qs/-/strict-qs-6.0.2.tgz", + "integrity": "sha512-zmcWbXwobbMwFQnpI3denPcHhIW17vNd6wDo//vOBTU07/baOTWFPViaDfQQmtvfKcnvdGOBZri0bIHwHdUs3Q==", + "requires": { + "debug": "^4.1.1", + "yerror": "^5.0.0" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "yerror": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yerror/-/yerror-5.0.0.tgz", + "integrity": "sha512-b4I0lhWrHDcFGVSP5SkSqGL8P+cuIvmoGvWfnm7YLMWMvUwngCZy9qNLjRbnABh02K/q/yFzBBz8dQuUVEoMAw==" + } + } + } + } +} diff --git a/packages/whook-gcp-functions/package.json b/packages/whook-gcp-functions/package.json new file mode 100644 index 00000000..1f952419 --- /dev/null +++ b/packages/whook-gcp-functions/package.json @@ -0,0 +1,198 @@ +{ + "name": "@whook/gcp-functions", + "version": "3.1.3", + "description": "Build and deploy to GCP Cloud Functions with Whook.", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "metapak": { + "configs": [ + "main", + "readme", + "eslint", + "babel", + "jest", + "jsdocs", + "typescript" + ], + "data": { + "childPackage": true, + "files": "'src/**/*.ts'", + "testsFiles": "'src/**/*.test.ts'", + "distFiles": "'dist/**/*.js'", + "ignore": [ + "dist", + "builds", + ".bin", + ".terraform", + "*.plan", + "*.tfstate.d", + ".credentials.json" + ], + "bundleFiles": [ + "dist", + "src" + ] + } + }, + "author": { + "name": "Nicolas Froidure", + "email": "nicolas.froidure@insertafter.com", + "url": "http://insertafter.com/en/index.html" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/nfroidure/whook.git" + }, + "bugs": { + "url": "https://github.com/nfroidure/whook/issues" + }, + "homepage": "https://github.com/nfroidure/whook", + "peerDependencies": { + "babel-loader": "^8.0.6", + "babel-plugin-knifecycle": "^1.2.0", + "webpack": "4.41.5" + }, + "dependencies": { + "@whook/cli": "^3.1.3", + "@whook/http-router": "^3.1.3", + "@whook/whook": "^3.1.3", + "ajv": "^6.11.0", + "camel-case": "^4.1.1", + "common-services": "^7.0.0", + "cpr": "3.0.1", + "knifecycle": "^9.0.0", + "memfs": "3.0.4", + "memory-fs": "0.5.0", + "openapi-types": "^1.3.5", + "qs": "^6.9.1", + "strict-qs": "^6.0.1", + "yerror": "^5.0.0" + }, + "devDependencies": { + "@babel/cli": "^7.8.4", + "@babel/core": "^7.8.7", + "@babel/plugin-proposal-class-properties": "^7.8.3", + "@babel/plugin-proposal-object-rest-spread": "^7.8.3", + "@babel/preset-env": "^7.8.7", + "@babel/preset-typescript": "^7.8.3", + "@babel/register": "^7.8.6", + "@types/jest": "^24.9.0", + "@typescript-eslint/eslint-plugin": "^2.24.0", + "@typescript-eslint/parser": "^2.24.0", + "babel-eslint": "^10.1.0", + "babel-loader": "^8.0.6", + "babel-plugin-knifecycle": "^1.2.0", + "eslint": "^6.8.0", + "eslint-plugin-prettier": "^3.1.2", + "jest": "^25.1.0", + "jsdoc-to-markdown": "^5.0.3", + "metapak": "^3.1.8", + "metapak-nfroidure": "10.0.1", + "prettier": "^1.19.1", + "rimraf": "^3.0.2", + "typescript": "^3.8.3", + "webpack": "4.41.5" + }, + "contributors": [], + "engines": { + "node": ">=10.19.0" + }, + "files": [ + "dist", + "src", + "LICENSE", + "README.md", + "CHANGELOG.md" + ], + "eslintConfig": { + "extends": "eslint:recommended", + "parserOptions": { + "ecmaVersion": 2018, + "sourceType": "module", + "modules": true + }, + "env": { + "es6": true, + "node": true, + "jest": true, + "mocha": true + }, + "plugins": [ + "prettier" + ], + "rules": { + "prettier/prettier": "error" + }, + "parser": "@typescript-eslint/parser", + "ignorePatterns": [ + "*.d.ts" + ], + "overrides": [ + { + "files": [ + "*.ts" + ], + "rules": { + "no-unused-vars": [ + 1, + { + "args": "none" + } + ] + } + } + ] + }, + "prettier": { + "semi": true, + "printWidth": 80, + "singleQuote": true, + "trailingComma": "all", + "proseWrap": "always" + }, + "babel": { + "presets": [ + "@babel/typescript", + [ + "@babel/env", + { + "targets": { + "node": "10.19.0" + } + } + ] + ], + "plugins": [ + "@babel/proposal-class-properties", + "@babel/plugin-proposal-object-rest-spread", + "babel-plugin-knifecycle" + ] + }, + "jest": { + "coverageReporters": [ + "lcov", + "html" + ], + "testPathIgnorePatterns": [ + "/node_modules/" + ], + "roots": [ + "/src" + ] + }, + "scripts": { + "cli": "env NODE_ENV=${NODE_ENV:-cli}", + "compile": "babel --extensions '.ts,.js' src --out-dir=dist --source-maps=true", + "cover": "npm run jest -- --coverage", + "doc": "echo \"# API\" > API.md; jsdoc2md 'dist/**/*.js' >> API.md && git add API.md", + "jest": "NODE_ENV=test jest", + "lint": "eslint 'src/**/*.ts'", + "metapak": "metapak", + "precz": "npm run compile", + "prettier": "prettier --write 'src/**/*.ts'", + "preversion": "npm run compile", + "test": "npm run jest", + "types": "rimraf -f 'dist/**/*.d.ts' && tsc --project . --declaration --emitDeclarationOnly --outDir dist" + } +} \ No newline at end of file diff --git a/packages/whook-gcp-functions/src/commands/testHTTPFunction.ts b/packages/whook-gcp-functions/src/commands/testHTTPFunction.ts new file mode 100644 index 00000000..15fbf763 --- /dev/null +++ b/packages/whook-gcp-functions/src/commands/testHTTPFunction.ts @@ -0,0 +1,181 @@ +import { loadLambda } from '../libs/utils'; +import { extra, autoService } from 'knifecycle'; +import { LogService, TimeService } from 'common-services'; +import { + readArgs, + WhookCommandArgs, + WhookCommandDefinition, + WhookCommandNamedArgs, +} from '@whook/cli'; +import YError from 'yerror'; +import Knifecycle from 'knifecycle'; +import { flattenOpenAPI, getOpenAPIOperations } from '@whook/http-router'; +import uuid from 'uuid'; +import { camelCase } from 'camel-case'; +import { WhookConfig } from '@whook/whook'; +import { OpenAPIV3 } from 'openapi-types'; +import { PassThrough } from 'stream'; + +const SEARCH_SEPARATOR = '?'; +const PATH_SEPARATOR = '/'; + +export const definition: WhookCommandDefinition = { + description: 'A command for testing AWS HTTP lambda', + example: `whook testHTTPLambda --name getPing`, + arguments: { + type: 'object', + additionalProperties: false, + required: ['name'], + properties: { + name: { + description: 'Name of the lamda to run', + type: 'string', + }, + type: { + description: 'Type of lambda to test', + type: 'string', + enum: ['main', 'index'], + default: 'index', + }, + contentType: { + description: 'Content type of the payload', + type: 'string', + default: 'application/json', + }, + parameters: { + description: 'The HTTP call parameters', + type: 'string', + default: '{}', + }, + }, + }, +}; + +export default extra(definition, autoService(initTestHTTPLambdaCommand)); + +async function initTestHTTPLambdaCommand({ + NODE_ENV, + PROJECT_DIR, + API, + time, + log, + args, +}: { + NODE_ENV: string; + PROJECT_DIR: string; + API: OpenAPIV3.Document; + time: TimeService; + log: LogService; + args: WhookCommandArgs; +}) { + return async () => { + const { + name, + type, + contentType, + parameters: rawParameters, + }: WhookCommandNamedArgs = readArgs(definition.arguments, args) as { + name: string; + type: string; + contentType: string; + parameters: string; + }; + const handler = await loadLambda( + { PROJECT_DIR, log }, + NODE_ENV, + name, + type, + ); + const OPERATION = ( + await getOpenAPIOperations(await flattenOpenAPI(API)) + ).find(({ operationId }) => operationId === name); + + if (!OPERATION) { + throw new YError('E_OPERATION_NOT_FOUND'); + } + + const search = ((OPERATION.parameters || []) as OpenAPIV3.ParameterObject[]) + .filter(p => p.in === 'query') + .reduce((accSearch, p) => { + if (null != parameters[p.name]) { + return ( + accSearch + + (accSearch ? '&' : '') + + p.name + + '=' + + parameters[p.name] + ); + } + return accSearch; + }, ''); + + const path = OPERATION.path + .split(PATH_SEPARATOR) + + .map((part, index) => { + const matches = /^\{([\d\w]+)\}$/i.exec(part); + + if (matches) { + return parameters[matches[1]]; + } + return part; + }) + .join(PATH_SEPARATOR); + const hasBody = !!OPERATION.requestBody; + const parameters = JSON.parse(rawParameters); + const gcpfRequest = { + method: OPERATION.method, + originalUrl: path + (search ? SEARCH_SEPARATOR + search : ''), + headers: ((OPERATION.parameters || []) as OpenAPIV3.ParameterObject[]) + .filter(p => p.in === 'header') + .reduce((headerParameters, p) => { + headerParameters[p.name] = parameters[camelCase(p.name)]; + return headerParameters; + }, {}), + rawBody: Buffer.from( + hasBody + ? contentType === 'application/json' + ? JSON.stringify(parameters.body) + : parameters.body + : '', + ), + }; + if (hasBody) { + gcpfRequest.headers['content-type'] = `${contentType};charset=UTF-8`; + } + log('info', 'GCPF_REQUEST:', gcpfRequest); + + const response = { + status: 0, + headers: {}, + data: '', + }; + await new Promise((resolve, reject) => { + const gcpfResponse: any = new PassThrough(); + + gcpfResponse.set = (name: string, value: string) => { + response.headers[name] = value; + }; + gcpfResponse.status = (code: number) => { + response.status = code; + }; + + handler(gcpfRequest, gcpfResponse).catch(reject); + + const chunks = []; + + gcpfResponse.once('end', () => { + response.data = Buffer.concat(chunks).toString(); + resolve(); + }); + gcpfResponse.once('error', reject); + gcpfResponse.on('readable', () => { + let data; + while ((data = gcpfResponse.read())) { + chunks.push(data); + } + }); + }); + log('info', 'SUCCESS:', response); + }; +} diff --git a/packages/whook-gcp-functions/src/index.ts b/packages/whook-gcp-functions/src/index.ts new file mode 100644 index 00000000..09aea4fc --- /dev/null +++ b/packages/whook-gcp-functions/src/index.ts @@ -0,0 +1,413 @@ +/* eslint global-require:0 */ +import joinPath from 'memory-fs/lib/join'; +import fs from 'fs'; +import util from 'util'; +import path from 'path'; +import mkdirp from 'mkdirp'; +import cpr from 'cpr'; +import YError from 'yerror'; +import initInitializerBuilder from 'knifecycle/dist/build'; +import initCompiler, { + WhookCompilerOptions, + WhookCompilerService, + WhookCompilerConfig, + DEFAULT_COMPILER_OPTIONS, +} from './services/compiler'; +import initBuildAutoloader from './services/_autoload'; +import Knifecycle, { SPECIAL_PROPS, constant, Autoloader } from 'knifecycle'; +import { WhookAPIOperationAddition } from '@whook/whook'; +import { + flattenOpenAPI, + getOpenAPIOperations, +} from '@whook/http-router/dist/utils'; +import { OpenAPIV3 } from 'openapi-types'; +import { LogService } from 'common-services'; + +export { + WhookCompilerConfig, + WhookCompilerOptions, + WhookCompilerService, + DEFAULT_COMPILER_OPTIONS, +}; +export type WhookAPIOperationAWSLambdaConfig = { + type?: 'http' | 'cron' | 'consumer' | 'transformer'; + enabled?: boolean; + sourceOperationId?: string; + staticFiles?: string[]; + compilerOptios?: WhookCompilerOptions; +}; +type WhookAPIAWSLambdaOperation = OpenAPIV3.OperationObject & + WhookAPIOperationAddition; + +const readFileAsync = util.promisify(fs.readFile) as ( + path: string, + encoding: string, +) => Promise; +const writeFileAsync = util.promisify(fs.writeFile) as ( + path: string, + content: string, + encoding: string, +) => Promise; +const cprAsync = util.promisify(cpr) as ( + source: string, + destination: string, + options: any, +) => Promise; + +const BUILD_DEFINITIONS: { + [type: string]: { + type: string; + wrapper: { name: string; path: string }; + suffix?: string; + }; +} = { + http: { + type: 'HTTP', + wrapper: { + name: 'wrapHandlerForGoogleHTTPFunction', + path: path.join(__dirname, 'wrappers', 'googleHTTPFunction'), + }, + suffix: 'Wrapped', + }, +}; + +export async function prepareBuildEnvironment( + $: Knifecycle = new Knifecycle(), +): Promise { + $.register( + constant('INITIALIZER_PATH_MAP', { + ENV: '@whook/gcp-functions/services/ENV', + log: '@whook/gcp-functions/services/log', + time: 'common-services/dist/time', + }), + ); + $.register(initInitializerBuilder); + $.register(initBuildAutoloader); + $.register(initCompiler); + $.register(constant('BUILD_PARALLELISM', 10)); + $.register(constant('PORT', 1337)); + $.register(constant('HOST', 'localhost')); + + return $; +} + +export async function runBuild( + aPrepareBuildEnvironment: typeof prepareBuildEnvironment, +): Promise { + try { + const handlerName = process.argv[2]; + const $ = await aPrepareBuildEnvironment(); + const { + NODE_ENV, + BUILD_PARALLELISM, + PROJECT_DIR, + compiler, + log, + $autoload, + API, + buildInitializer, + }: { + NODE_ENV: string; + BUILD_PARALLELISM: number; + PROJECT_DIR: string; + compiler: WhookCompilerService; + log: LogService; + $autoload: Autoloader; + API: OpenAPIV3.Document; + buildInitializer: Function; + } = await $.run([ + 'NODE_ENV', + 'BUILD_PARALLELISM', + 'PROJECT_DIR', + 'process', + 'compiler', + 'log', + '$autoload', + 'API', + 'buildInitializer', + ]); + + log('info', 'Environment initialized 🚀🌕'); + + const operations: WhookAPIAWSLambdaOperation[] = ( + await flattenOpenAPI(API).then(getOpenAPIOperations) + ).filter((operation: WhookAPIAWSLambdaOperation) => { + if (handlerName) { + const sourceOperationId = + operation['x-whook'] && operation['x-whook'].sourceOperationId; + + return ( + handlerName === operation.operationId || + handlerName === sourceOperationId + ); + } + return true; + }); + + log('info', `${operations.length} operations to process.`); + await processOperations( + { + NODE_ENV, + BUILD_PARALLELISM, + PROJECT_DIR, + compiler, + log, + $autoload, + buildInitializer, + }, + operations, + ); + await $.destroy(); + process.exit(); + } catch (err) { + // eslint-disable-next-line + console.error('💀 - Cannot launch the build:', err.stack); + process.exit(1); + } +} + +async function processOperations( + { + NODE_ENV, + BUILD_PARALLELISM, + PROJECT_DIR, + compiler, + log, + $autoload, + buildInitializer, + }: { + NODE_ENV: string; + BUILD_PARALLELISM: number; + PROJECT_DIR: string; + compiler: WhookCompilerService; + log: LogService; + $autoload: Autoloader; + buildInitializer: Function; + }, + operations: WhookAPIAWSLambdaOperation[], +) { + const operationsLeft = operations.slice(BUILD_PARALLELISM); + + await Promise.all( + operations + .slice(0, BUILD_PARALLELISM) + .map(operation => + buildAnyLambda( + { NODE_ENV, PROJECT_DIR, compiler, log, $autoload, buildInitializer }, + operation, + ), + ), + ); + + if (operationsLeft.length) { + log('info', operationsLeft.length, ' operations left.'); + return processOperations( + { + NODE_ENV, + BUILD_PARALLELISM, + PROJECT_DIR, + compiler, + log, + $autoload, + buildInitializer, + }, + operationsLeft, + ); + } + log('info', 'No more operations.'); +} + +async function buildAnyLambda( + { NODE_ENV, PROJECT_DIR, compiler, log, $autoload, buildInitializer }, + operation, +) { + const { operationId } = operation; + + try { + const whookConfig: WhookAPIOperationAWSLambdaConfig = + operation['x-whook'] || {}; + const operationType = whookConfig.type || 'http'; + const sourceOperationId = whookConfig.sourceOperationId; + const entryPoint = operationId; + const finalEntryPoint = + (sourceOperationId ? sourceOperationId : operationId) + + ((operation['x-whook'] || {}).suffix || ''); + log('info', `Building ${operationType} '${finalEntryPoint}'...`); + const buildDefinition = BUILD_DEFINITIONS[operationType]; + const applyWrapper = require(buildDefinition.wrapper.path).default; + const rootNode = await $autoload( + entryPoint + (buildDefinition.suffix || ''), + ); + const lambdaPath = path.join( + PROJECT_DIR, + 'builds', + NODE_ENV, + finalEntryPoint, + ); + const finalHandlerInitializer = applyWrapper(rootNode.initializer); + + const initializerContent = await buildInitializer( + finalHandlerInitializer[SPECIAL_PROPS.INJECT].map(name => + name === 'OPERATION' ? `OPERATION>OPERATION_${finalEntryPoint}` : name, + ), + ); + const indexContent = await buildLambdaIndex(rootNode, { + name: buildDefinition.wrapper.name, + path: buildDefinition.wrapper.path, + }); + + await mkdirpAsync(lambdaPath); + await Promise.all([ + copyStaticFiles( + { PROJECT_DIR, log }, + lambdaPath, + whookConfig.staticFiles || [], + ), + ensureFileAsync( + { log }, + path.join(lambdaPath, 'initialize.js'), + initializerContent, + ), + ensureFileAsync({ log }, path.join(lambdaPath, 'main.js'), indexContent), + ]); + await buildFinalLambda( + { NODE_ENV, compiler, log }, + lambdaPath, + whookConfig, + ); + } catch (err) { + log('error', `Error building ${operationId}'...`); + log('stack', err.stack); + throw YError.wrap(err, 'E_LAMBDA_BUILD', operationId); + } +} + +async function buildLambdaIndex(rootNode, buildWrapper) { + return `import initHandler from '${rootNode.path}'; +import ${buildWrapper.name} from '${buildWrapper.path}'; +import { initialize } from './initialize'; + +const handlerInitializer = ${buildWrapper.name}( + initHandler +); + +const handlerPromise = initialize() + .then(handlerInitializer); + +export default function handler (req, res) { + return handlerPromise + .then(handler => handler(req, res)); +}; +`; +} + +async function buildFinalLambda( + { NODE_ENV, compiler, log }, + lambdaPath, + whookConfig, +) { + const entryPoint = `${lambdaPath}/main.js`; + const { contents, mappings } = await compiler(entryPoint); + + await Promise.all([ + ensureFileAsync({ log }, `${lambdaPath}/index.js`, contents, 'utf-8'), + mappings + ? ensureFileAsync( + { log }, + `${lambdaPath}/index.js.map`, + mappings, + 'utf-8', + ) + : Promise.resolve(), + ]); +} + +async function copyStaticFiles( + { PROJECT_DIR, log }: { PROJECT_DIR: string; log: LogService }, + lambdaPath: string, + staticFiles: string[] = [], +) { + await Promise.all( + staticFiles.map( + async staticFile => + await copyFiles( + { log }, + path.join(PROJECT_DIR, 'node_modules', staticFile), + path.join(lambdaPath, 'node_modules', staticFile), + ), + ), + ); +} + +async function copyFiles( + { log }: { log: LogService }, + source: string, + destination: string, +) { + let theError; + try { + await mkdirpAsync(destination); + const data = await readFileAsync(source, 'utf-8'); + await ensureFileAsync({ log }, destination, data, 'utf-8'); + } catch (err) { + theError = err; + } + if (theError) { + if ('EISDIR' !== theError.code) { + throw theError; + } + await cprAsync(source, destination, { + overwrite: true, + }); + } +} + +async function ensureFileAsync( + { log }: { log: LogService }, + path: string, + content: string, + encoding: string = 'utf-8', +) { + try { + const oldContent = await readFileAsync(path, encoding); + + if (oldContent === content) { + log('debug', 'Ignore unchanged file:', path); + return; + } + } catch (err) { + log('debug', 'Write new file:', path); + return await writeFileAsync(path, content, encoding); + } + log('debug', 'Write changed file:', path); + return await writeFileAsync(path, content, encoding); +} + +// Cannot promisify mkdirp easily so doing it by hand +// https://github.com/substack/node-mkdirp/issues/136 +async function mkdirpAsync(path: string) { + return new Promise((resolve, reject) => { + mkdirp(path, err => { + if (err) { + reject(err); + return; + } + resolve(); + }); + }); +} + +// Taken from https://github.com/streamich/memfs/issues/404#issuecomment-522450466 +// Awaiting for Webpack to avoid using .join on fs +function ensureWebpackMemoryFs(fs) { + // Return it back, when it has Webpack 'join' method + if (fs.join) { + return fs; + } + + // Create FS proxy, adding `join` method to memfs, but not modifying original object + const nextFs = Object.create(fs); + nextFs.join = joinPath; + + return nextFs; +} diff --git a/packages/whook-gcp-functions/src/libs/utils.ts b/packages/whook-gcp-functions/src/libs/utils.ts new file mode 100644 index 00000000..45c98223 --- /dev/null +++ b/packages/whook-gcp-functions/src/libs/utils.ts @@ -0,0 +1,42 @@ +import YError from 'yerror'; +import path from 'path'; +import { LogService } from 'common-services'; + +export async function loadLambda( + { + PROJECT_DIR, + log, + }: { + PROJECT_DIR: string; + log: LogService; + }, + target: string, + operationId: string, + type: string, +): Promise { + const modulePath = path.join( + PROJECT_DIR, + 'builds', + target, + operationId, + type, + ); + + log('debug', '⛏️ - Loading lambda module at path:', modulePath); + + try { + const module = require(modulePath); + + if (!module) { + throw new YError('E_MODULE_NOT_FOUND', module); + } + + if (!module.default) { + throw new YError('E_LAMBDA_NOT_FOUND', module, Object.keys(module)); + } + + return module.default; + } catch (err) { + throw YError.wrap(err, 'E_LAMBDA_LOAD'); + } +} diff --git a/packages/whook-gcp-functions/src/services/_autoload.ts b/packages/whook-gcp-functions/src/services/_autoload.ts new file mode 100644 index 00000000..24a4d441 --- /dev/null +++ b/packages/whook-gcp-functions/src/services/_autoload.ts @@ -0,0 +1,117 @@ +import { WhookBuildConstantsService, initAutoload, noop } from '@whook/whook'; +import Knifecycle, { + SPECIAL_PROPS, + wrapInitializer, + constant, + alsoInject, + Injector, +} from 'knifecycle'; +import YError from 'yerror'; +import { flattenOpenAPI, getOpenAPIOperations } from '@whook/http-router'; +import { LogService } from 'common-services'; + +/** + * Wrap the _autoload service in order to build AWS + * Lambda compatible code. + * @param {Object} services + * The services ENV depends on + * @param {Object} services.NODE_ENV + * The injected NODE_ENV value to add it to the build env + * @param {Object} [services.PROXYED_ENV_VARS={}] + * A list of environment variable names to proxy + * @param {Object} [log=noop] + * An optional logging service + * @return {Promise} + * A promise of an object containing the reshaped env vars. + */ +export default alsoInject( + ['?BUILD_CONSTANTS', '$instance', '$injector', '?log'], + wrapInitializer( + async ( + { + BUILD_CONSTANTS = {}, + $injector, + $instance, + log = noop, + }: { + BUILD_CONSTANTS?: WhookBuildConstantsService; + $injector: Injector; + $instance: Knifecycle; + log: LogService; + }, + $autoload, + ) => { + let API_OPERATIONS; + const getAPIOperations = (() => { + return async () => { + // eslint-disable-next-line + API_OPERATIONS = + API_OPERATIONS || + (await getOpenAPIOperations( + await flattenOpenAPI((await $injector(['API'])).API), + )); + return API_OPERATIONS; + }; + })(); + + log('debug', '🤖 - Initializing the `$autoload` build wrapper.'); + + return async serviceName => { + try { + // TODO: add initializer map to knifecycle public API + const initializer = ($instance as any)._initializers.get(serviceName); + + if (initializer && initializer[SPECIAL_PROPS.TYPE] === 'constant') { + log( + 'debug', + '🤖 - Reusing a constant initializer directly from the Knifecycle instance:', + serviceName, + ); + return { + initializer, + path: `instance://${serviceName}`, + }; + } + + if (serviceName.startsWith('OPERATION_')) { + const OPERATION = (await getAPIOperations()).find( + operation => + serviceName === + (((operation['x-whook'] || {}).sourceOperationId && + 'OPERATION_' + + (operation['x-whook'] || {}).sourceOperationId) || + 'OPERATION_' + operation.operationId) + + ((operation['x-whook'] || {}).suffix || ''), + ); + + if (!OPERATION) { + log( + 'error', + '💥 - Unable to find a lambda operation definition!', + ); + throw new YError('E_OPERATION_NOT_FOUND', serviceName); + } + + return { + initializer: constant(serviceName, OPERATION), + path: `api://${serviceName}`, + }; + } + + if (BUILD_CONSTANTS[serviceName]) { + return { + initializer: constant(serviceName, BUILD_CONSTANTS[serviceName]), + path: `constant://${serviceName}`, + }; + } + + return $autoload(serviceName); + } catch (err) { + log('error', `Build error while loading ${serviceName} `); + log('stack', err.stack); + } + }; + }, + initAutoload, + ), +); diff --git a/packages/whook-gcp-functions/src/services/compiler.ts b/packages/whook-gcp-functions/src/services/compiler.ts new file mode 100644 index 00000000..ab012cde --- /dev/null +++ b/packages/whook-gcp-functions/src/services/compiler.ts @@ -0,0 +1,220 @@ +import path from 'path'; +import YError from 'yerror'; +import { noop } from '@whook/whook'; +import joinPath from 'memory-fs/lib/join'; +import { Volume, createFsFromVolume } from 'memfs'; +import webpack from 'webpack'; +import { autoService } from 'knifecycle'; +import { LogService } from 'common-services'; + +export default autoService(initCompiler); + +export type WhookCompilerOptions = { + externalModules?: string[]; + ignoredModules?: string[]; + extensions?: string[]; +}; +export type WhookCompilerConfig = { + NODE_ENV?: string; + DEBUG_NODE_ENVS: string[]; + COMPILER_OPTIONS?: WhookCompilerOptions; +}; +export type WhookCompilerDependencies = WhookCompilerConfig & { + NODE_ENV: string; + log?: LogService; +}; +type WhookCompilationResult = { contents: string; mappings: string }; +export type WhookCompilerService = ( + entryPoint: string, +) => Promise; + +export const DEFAULT_COMPILER_OPTIONS: WhookCompilerOptions = { + externalModules: ['ecstatic'], + ignoredModules: [], + extensions: ['.ts', '.js', '.json'], +}; + +async function initCompiler({ + NODE_ENV, + DEBUG_NODE_ENVS, + COMPILER_OPTIONS = DEFAULT_COMPILER_OPTIONS, + log = noop, +}: WhookCompilerDependencies): Promise { + return async function compiler( + entryPoint: string, + options?: {}, + ): Promise { + const debugging = DEBUG_NODE_ENVS.includes(NODE_ENV); + const basePath = path.dirname(entryPoint); + const compilerOptions: WhookCompilerOptions = { + ...COMPILER_OPTIONS, + ...options, + }; + const memoryFS = createFsFromVolume(new Volume()); + // Configurations inspired from modes + // See https://webpack.js.org/configuration/mode/ + const compiler = webpack({ + entry: entryPoint, + target: 'node', + mode: 'none', + devtool: debugging ? 'eval' : 'source-map', + cache: debugging, + performance: { + hints: debugging ? false : 'warning', + }, + output: { + pathinfo: debugging, + libraryTarget: 'commonjs2', + path: basePath, + filename: 'index.js', + }, + optimization: { + nodeEnv: NODE_ENV, + minimize: false, + ...(debugging + ? { + namedModules: true, + namedChunks: true, + flagIncludedChunks: false, + occurrenceOrder: false, + sideEffects: false, + usedExports: false, + concatenateModules: false, + splitChunks: { + hidePathInfo: false, + minSize: 10000, + maxAsyncRequests: Infinity, + maxInitialRequests: Infinity, + }, + noEmitOnErrors: false, + checkWasmTypes: false, + removeAvailableModules: false, + } + : { + namedModules: false, + namedChunks: false, + flagIncludedChunks: true, + occurrenceOrder: true, + sideEffects: true, + usedExports: true, + concatenateModules: true, + splitChunks: { + hidePathInfo: true, + minSize: 30000, + maxAsyncRequests: 5, + maxInitialRequests: 3, + }, + noEmitOnErrors: true, + checkWasmTypes: true, + }), + }, + plugins: [ + new webpack.DefinePlugin({ + 'process.env.NODE_ENV': JSON.stringify(NODE_ENV), + }), + ...(compilerOptions.ignoredModules || []).map( + ignoredModule => new webpack.IgnorePlugin(new RegExp(ignoredModule)), + ), + ...(debugging + ? [new webpack.NamedModulesPlugin(), new webpack.NamedChunksPlugin()] + : [ + new webpack.optimize.ModuleConcatenationPlugin(), + new webpack.NoEmitOnErrorsPlugin(), + ]), + ], + node: { + __dirname: true, + }, + resolve: { + extensions: compilerOptions.extensions || ['.js'], + }, + externals: compilerOptions.externalModules || [], + module: { + rules: [ + // This rule must be added to handle deep dependencies usage + // of the .esm extension. It should be safe someday to remove + // it but who knows when ¯\_(ツ)_/¯ + { + test: /\.mjs$/, + type: 'javascript/auto', + }, + { + test: /\.(js|ts)$/, + exclude: /node_modules/, + use: { + loader: 'babel-loader', + options: { + presets: [ + '@babel/typescript', + [ + '@babel/env', + { + modules: false, + targets: { + node: '12.13', + }, + }, + ], + ], + plugins: [ + '@babel/proposal-class-properties', + '@babel/proposal-object-rest-spread', + 'babel-plugin-knifecycle', + ], + babelrc: false, + }, + }, + }, + ], + }, + }); + + compiler.outputFileSystem = ensureWebpackMemoryFs(memoryFS); + + await new Promise((resolve, reject) => { + compiler.run((err, stats) => { + if (err) { + reject(YError.wrap(err, 'E_WEBPACK', err.details)); + return; + } + if (stats.hasErrors()) { + reject(new YError('E_WEBPACK', stats.toJson().errors)); + return; + } + if (stats.hasWarnings()) { + log('warn', stats.toJson().warnings); + } + + resolve(); + }); + }); + + const contents: string = (memoryFS.readFileSync( + `${basePath}/index.js`, + 'utf-8', + ) as unknown) as string; + const mappings: string = debugging + ? '' + : ((memoryFS.readFileSync( + `${basePath}/index.js.map`, + 'utf-8', + ) as unknown) as string); + + return { contents, mappings }; + }; +} + +// Taken from https://github.com/streamich/memfs/issues/404#issuecomment-522450466 +// Awaiting for Webpack to avoid using .join on fs +function ensureWebpackMemoryFs(fs) { + // Return it back, when it has Webpack 'join' method + if (fs.join) { + return fs; + } + + // Create FS proxy, adding `join` method to memfs, but not modifying original object + const nextFs = Object.create(fs); + nextFs.join = joinPath; + + return nextFs; +} diff --git a/packages/whook-gcp-functions/src/services/log.test.ts b/packages/whook-gcp-functions/src/services/log.test.ts new file mode 100644 index 00000000..7914089e --- /dev/null +++ b/packages/whook-gcp-functions/src/services/log.test.ts @@ -0,0 +1,7 @@ +import initLogService from './log'; + +describe('initLogService', () => { + it('should work', async () => { + await initLogService(); + }); +}); diff --git a/packages/whook-gcp-functions/src/services/log.ts b/packages/whook-gcp-functions/src/services/log.ts new file mode 100644 index 00000000..0be3848f --- /dev/null +++ b/packages/whook-gcp-functions/src/services/log.ts @@ -0,0 +1,4 @@ +import { service } from 'knifecycle'; + +// eslint-disable-next-line +export default service(async () => console.log.bind(console), 'log'); diff --git a/packages/whook-gcp-functions/src/wrappers/googleHTTPFunction.ts b/packages/whook-gcp-functions/src/wrappers/googleHTTPFunction.ts new file mode 100644 index 00000000..9d0658b5 --- /dev/null +++ b/packages/whook-gcp-functions/src/wrappers/googleHTTPFunction.ts @@ -0,0 +1,549 @@ +import { + DEFAULT_DEBUG_NODE_ENVS, + DEFAULT_BUFFER_LIMIT, + DEFAULT_PARSERS, + DEFAULT_STRINGIFYERS, + DEFAULT_DECODERS, + DEFAULT_ENCODERS, + WhookQueryStringParser, +} from '@whook/http-router'; +import { reuseSpecialProps, alsoInject, ServiceInitializer } from 'knifecycle'; +import Ajv from 'ajv'; +import HTTPError from 'yhttperror'; +import { + prepareParametersValidators, + prepareBodyValidator, + applyValidators, + filterHeaders, +} from '@whook/http-router/dist/validation'; +import { + extractBodySpec, + extractResponseSpec, + checkResponseCharset, + checkResponseMediaType, + executeHandler, + extractProduceableMediaTypes, + extractConsumableMediaTypes, +} from '@whook/http-router/dist/lib'; +import { getBody, sendBody } from '@whook/http-router/dist/body'; +import { + noop, + compose, + WhookRequest, + WhookResponse, + WhookHandler, + ObfuscatorService, + WhookOperation, + APMService, + WhookWrapper, + identity, +} from '@whook/whook'; +import { PassThrough } from 'stream'; +import qs from 'qs'; +import { parseReentrantNumber, parseBoolean } from 'strict-qs'; +import { camelCase } from 'camel-case'; +import { TimeService, LogService } from 'common-services'; +import { OpenAPIV3 } from 'openapi-types'; + +type HTTPWrapperDependencies = { + NODE_ENV: string; + DEBUG_NODE_ENVS?: string[]; + OPERATION: WhookOperation; + DECODERS?: typeof DEFAULT_DECODERS; + ENCODERS?: typeof DEFAULT_ENCODERS; + PARSED_HEADERS?: string[]; + PARSERS?: typeof DEFAULT_PARSERS; + STRINGIFYERS?: typeof DEFAULT_STRINGIFYERS; + QUERY_PARSER: WhookQueryStringParser; + BUFFER_LIMIT?: string; + apm: APMService; + obfuscator: ObfuscatorService; + time?: TimeService; + log?: LogService; + WRAPPERS: WhookWrapper[]; +}; + +const SEARCH_SEPARATOR = '?'; +const PATH_SEPARATOR = '/'; +const uuidPattern = + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; + +export default function wrapHandlerForAWSHTTPLambda( + initHandler: ServiceInitializer, +): ServiceInitializer { + return alsoInject( + [ + 'OPERATION', + 'WRAPPERS', + '?DEBUG_NODE_ENVS', + 'NODE_ENV', + '?DECODERS', + '?ENCODERS', + '?PARSED_HEADERS', + '?PARSERS', + '?STRINGIFYERS', + '?BUFFER_LIMIT', + 'QUERY_PARSER', + 'apm', + 'obfuscator', + '?log', + '?time', + ], + reuseSpecialProps( + initHandler, + initHandlerForAWSHTTPLambda.bind(null, initHandler), + ), + ); +} + +async function initHandlerForAWSHTTPLambda( + initHandler, + { + OPERATION, + WRAPPERS, + NODE_ENV, + DEBUG_NODE_ENVS = DEFAULT_DEBUG_NODE_ENVS, + DECODERS = DEFAULT_DECODERS, + ENCODERS = DEFAULT_ENCODERS, + PARSED_HEADERS = [], + log = noop, + time = Date.now.bind(Date), + ...services + }, +) { + const consumableCharsets = Object.keys(DECODERS); + const produceableCharsets = Object.keys(ENCODERS); + const consumableMediaTypes = extractConsumableMediaTypes(OPERATION); + const produceableMediaTypes = extractProduceableMediaTypes(OPERATION); + const ajv = new Ajv({ + verbose: DEBUG_NODE_ENVS.includes(NODE_ENV), + useDefaults: true, + coerceTypes: true, + strictKeywords: true, + }); + const validators = prepareParametersValidators( + ajv, + OPERATION.operationId, + OPERATION.parameters || [], + ); + const bodyValidator = prepareBodyValidator(ajv, OPERATION); + const applyWrappers = compose(...WRAPPERS); + + const handler = await applyWrappers(initHandler)({ + OPERATION, + DEBUG_NODE_ENVS, + NODE_ENV, + ...services, + time, + log, + }); + + return handleForAWSHTTPLambda.bind( + null, + { + OPERATION, + NODE_ENV, + DEBUG_NODE_ENVS, + DECODERS, + ENCODERS, + PARSED_HEADERS, + log, + time, + ...services, + }, + { + consumableMediaTypes, + produceableMediaTypes, + consumableCharsets, + produceableCharsets, + validators, + bodyValidator, + }, + handler, + ); +} + +async function handleForAWSHTTPLambda( + { + OPERATION, + DEBUG_NODE_ENVS, + NODE_ENV, + ENCODERS, + DECODERS, + PARSERS = DEFAULT_PARSERS, + STRINGIFYERS = DEFAULT_STRINGIFYERS, + BUFFER_LIMIT = DEFAULT_BUFFER_LIMIT, + PARSED_HEADERS, + QUERY_PARSER, + // TODO: Better handling of CORS in errors should + // be found + CORS, + log, + time, + apm, + obfuscator, + }: HTTPWrapperDependencies & { CORS: any }, + { + consumableMediaTypes, + produceableMediaTypes, + consumableCharsets, + produceableCharsets, + validators, + bodyValidator, + }, + handler: WhookHandler, + req: any, + res: any, +) { + const debugging = DEBUG_NODE_ENVS.includes(NODE_ENV); + const startTime = time(); + + log( + 'info', + 'GCP_FUNCTIONS_REQUEST', + JSON.stringify({ + url: req.originalUrl, + method: req.method, + body: req.body, + // body: obfuscateEventBody(obfuscator, req.body), + headers: obfuscator.obfuscateSensibleHeaders(req.headers), + }), + ); + + const request = await gcpfReqToRequest(req); + let parameters; + let response; + let responseLog; + let responseSpec; + + log( + 'debug', + 'REQUEST', + JSON.stringify({ + ...request, + body: request.body ? 'Stream' : undefined, + headers: obfuscator.obfuscateSensibleHeaders(request.headers), + }), + ); + + try { + const operation = OPERATION; + const bodySpec = extractBodySpec( + request, + consumableMediaTypes, + consumableCharsets, + ); + + responseSpec = extractResponseSpec( + operation, + request, + produceableMediaTypes, + produceableCharsets, + ); + + try { + const body = await getBody( + { + DECODERS, + PARSERS, + bufferLimit: BUFFER_LIMIT, + }, + operation, + request.body, + bodySpec, + ); + const path = request.url.split(SEARCH_SEPARATOR)[0]; + const parts = path.split(PATH_SEPARATOR).filter(identity); + const search = request.url.substr( + request.url.split(SEARCH_SEPARATOR)[0].length, + ); + + const pathParameters = OPERATION.path + .split(PATH_SEPARATOR) + .filter(identity) + .map((part, index) => { + const matches = /^\{([\d\w]+)\}$/i.exec(part); + + if (matches) { + return { + name: matches[1], + value: parts[index], + }; + } + }) + .filter(identity) + .reduce( + (accParameters, { name, value }) => ({ + ...accParameters, + [name]: value, + }), + {}, + ); + + // TODO: Update strictQS to handle OpenAPI 3 + const retroCompatibleQueryParameters = (OPERATION.parameters || []) + .filter(p => (p as any).in === 'query') + .map(p => ({ ...p, ...(p as any).schema })); + + parameters = { + ...pathParameters, + ...QUERY_PARSER(retroCompatibleQueryParameters, search), + ...filterHeaders( + operation.parameters as OpenAPIV3.ParameterObject[], + request.headers, + ), + }; + + parameters = ((OPERATION.parameters || + []) as OpenAPIV3.ParameterObject[]).reduce( + (cleanParameters, parameter) => { + const parameterName = + parameter.in === 'header' + ? camelCase(parameter.name) + : parameter.name; + + cleanParameters[parameterName] = castParameterValue( + parameter.schema, + parameters[parameterName], + ); + + return cleanParameters; + }, + { + // TODO: Use the security of the operation to infer + // authorization parameters, see: + // https://github.com/nfroidure/whook/blob/06ccae93d1d52d97ff70fd5e19fa826bdabf3968/packages/whook-http-router/src/validation.js#L110 + authorization: parameters.authorization, + }, + ); + + applyValidators(operation, validators, parameters); + + bodyValidator(operation, bodySpec.contentType, body); + + parameters = { + ...parameters, + ...('undefined' !== typeof body ? { body } : {}), + }; + } catch (err) { + throw HTTPError.cast(err, 400); + } + + response = await executeHandler(operation, handler, parameters); + + if (response.body) { + response.headers['content-type'] = + response.headers['content-type'] || responseSpec.contentTypes[0]; + } + + // Check the stringifyer only when a schema is + // specified and it is not a binary one + const responseObject = + operation.responses && + (operation.responses[response.status] as OpenAPIV3.ResponseObject); + const responseSchema = + responseObject && + responseObject.content && + responseObject.content[response.headers['content-type']] && + (responseObject.content[response.headers['content-type']] + .schema as OpenAPIV3.SchemaObject); + const responseHasSchema = + responseSchema && + (responseSchema.type !== 'string' || responseSchema.format !== 'binary'); + + if (responseHasSchema && !STRINGIFYERS[response.headers['content-type']]) { + throw new HTTPError( + 500, + 'E_STRINGIFYER_LACK', + response.headers['content-type'], + ); + } + if (response.body) { + checkResponseMediaType(request, responseSpec, produceableMediaTypes); + checkResponseCharset(request, responseSpec, produceableCharsets); + } + responseLog = { + type: 'success', + status: response.status, + }; + log('debug', JSON.stringify(responseLog)); + } catch (err) { + const castedError = HTTPError.cast(err); + + responseLog = { + type: 'error', + code: castedError.code, + statusCode: castedError.httpCode, + params: castedError.params || [], + stack: castedError.stack, + }; + + log('error', JSON.stringify(responseLog)); + response = { + status: castedError.httpCode, + headers: { + ...CORS, + 'content-type': 'application/json', + }, + body: { + error: { + code: castedError.code, + stack: debugging ? responseLog.stack : undefined, + params: debugging ? responseLog.params : undefined, + }, + }, + }; + } + + log( + 'debug', + 'RESPONSE', + JSON.stringify({ + ...response, + body: obfuscateEventBody(obfuscator, response.body), + headers: obfuscator.obfuscateSensibleHeaders(response.headers), + }), + ); + + await pipeResponseInGCPFResponse( + await sendBody( + { + ENCODERS, + STRINGIFYERS, + }, + response, + ), + res, + ); + + // log( + // 'debug', + // 'AWS_RESPONSE_EVENT', + // JSON.stringify({ + // ...awsResponse, + // body: obfuscateEventBody(obfuscator, awsResponse.body), + // headers: obfuscator.obfuscateSensibleHeaders(awsResponse.headers), + // }), + // ); + + // apm('CALL', { + // id: event.requestContext.requestId, + // transactionId: + // request.headers['x-transaction-id'] && + // new RegExp(uuidPattern).test(request.headers['x-transaction-id']) + // ? event.headers['x-transaction-id'] + // : event.requestContext.requestId, + // environment: NODE_ENV, + // method: event.requestContext.httpMethod, + // resourcePath: event.requestContext.resourcePath, + // path: event.requestContext.path, + // userAgent: + // event.requestContext.identity && event.requestContext.identity.userAgent, + // triggerTime: event.requestContext.requestTimeEpoch, + // lambdaName: OPERATION.operationId, + // parameters: obfuscator.obfuscateSensibleProps(parameters), + // status: response.status, + // headers: obfuscator.obfuscateSensibleHeaders( + // Object.keys(response.headers).reduce( + // (finalHeaders, headerName) => ({ + // ...finalHeaders, + // ...(PARSED_HEADERS.includes(headerName) + // ? {} + // : { + // [headerName]: response.headers[headerName], + // }), + // }), + // {}, + // ), + // ), + // bodyLength: awsResponse.body ? awsResponse.body.length : 0, + // type: responseLog.type, + // stack: responseLog.stack, + // code: responseLog.code, + // params: responseLog.params, + // startTime, + // endTime: time(), + // ...PARSED_HEADERS.reduce( + // (result, parsedHeader) => ({ + // ...result, + // ...(response.headers[parsedHeader] + // ? JSON.parse(response.headers[parsedHeader]) + // : {}), + // }), + // {}, + // ), + // }); +} + +async function gcpfReqToRequest(req: any): Promise { + const request: WhookRequest = { + method: req.method.toLowerCase(), + headers: lowerCaseHeaders(req.headers || {}), + url: req.originalUrl, + }; + + if (req.rawBody) { + request.headers['content-length'] = req.rawBody.length.toString(); + request.body = new PassThrough(); + request.body.write(req.rawBody); + request.body.end(); + } + + return request; +} + +async function pipeResponseInGCPFResponse( + response: WhookResponse, + res: any, +): Promise { + Object.keys(response.headers).forEach(headerName => { + res.set(headerName, response.headers[headerName]); + }); + res.status(response.status); + + if (response.body) { + response.body.pipe(res); + return; + } + + res.end(); +} + +function lowerCaseHeaders(headers) { + return Object.keys(headers).reduce((newHeaders, name) => { + const newName = name.toLowerCase(); + + newHeaders[newName] = headers[name]; + return newHeaders; + }, {}); +} + +export function castParameterValue(parameter, value) { + if ('undefined' !== typeof value) { + if ('boolean' === parameter.type) { + value = parseBoolean(value); + } else if ('number' === parameter.type) { + value = parseReentrantNumber(value); + } else if ('array' === parameter.type) { + value = ('' + value) + .split(',') + .map(castParameterValue.bind(null, parameter.items)); + } + if (parameter.enum && !parameter.enum.includes(value)) { + throw new HTTPError(400, 'E_NOT_IN_ENUM', value, parameter.enum); + } + } + return value; +} + +function obfuscateEventBody(obfuscator, rawBody) { + if (typeof rawBody === 'string') { + try { + const jsonBody = JSON.parse(rawBody); + + return JSON.stringify(obfuscator.obfuscateSensibleProps(jsonBody)); + // eslint-disable-next-line + } catch (err) {} + } + return rawBody; +} diff --git a/packages/whook-gcp-functions/tsconfig.json b/packages/whook-gcp-functions/tsconfig.json new file mode 100644 index 00000000..91192cdb --- /dev/null +++ b/packages/whook-gcp-functions/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "module": "esnext", + "moduleResolution": "node", + "target": "esnext", + "noImplicitAny": false, + "removeComments": false, + "preserveConstEnums": true, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "declaration": true + }, + "include": [ + "**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} \ No newline at end of file diff --git a/packages/whook-graphql/API.md b/packages/whook-graphql/API.md index 5e7dd8d5..9f6b727a 100644 --- a/packages/whook-graphql/API.md +++ b/packages/whook-graphql/API.md @@ -1,33 +1,4 @@ # API -## Members - -
-
defaultPromise
-

Initialize the GraphQL service

-
-
defaultPromise
-

Initialize the GraphQL service

-
-
- - - -## default ⇒ Promise -Initialize the GraphQL service - -**Kind**: global variable -**Returns**: Promise - A promise of a GraphQL service - -| Param | Type | Default | Description | -| --- | --- | --- | --- | -| services | Object | | The services the server depends on | -| services.NODE_ENV | Object | | The injected NODE_ENV value | -| [services.GRAPHQL_OPTIONS] | Array | | The GraphQL options to pass to the schema | -| ENV | String | | The process environment | -| [graphQLFragments] | String | | Fragments of GraphQL schemas/resolvers declaration | -| [services.log] | function | noop | A logging function | -| [services.time] | function | | A function returning the current timestamp | - ## default ⇒ Promise diff --git a/packages/whook-graphql/README.md b/packages/whook-graphql/README.md index 64930847..f30d6318 100644 --- a/packages/whook-graphql/README.md +++ b/packages/whook-graphql/README.md @@ -92,35 +92,6 @@ See [this repository tests](./src/intex.test.ts) for more examples. [//]: # (::contents:end) # API -## Members - -
-
defaultPromise
-

Initialize the GraphQL service

-
-
defaultPromise
-

Initialize the GraphQL service

-
-
- - - -## default ⇒ Promise -Initialize the GraphQL service - -**Kind**: global variable -**Returns**: Promise - A promise of a GraphQL service - -| Param | Type | Default | Description | -| --- | --- | --- | --- | -| services | Object | | The services the server depends on | -| services.NODE_ENV | Object | | The injected NODE_ENV value | -| [services.GRAPHQL_OPTIONS] | Array | | The GraphQL options to pass to the schema | -| ENV | String | | The process environment | -| [graphQLFragments] | String | | Fragments of GraphQL schemas/resolvers declaration | -| [services.log] | function | noop | A logging function | -| [services.time] | function | | A function returning the current timestamp | - ## default ⇒ Promise diff --git a/packages/whook/API.md b/packages/whook/API.md index 3ccde6f8..53908d85 100644 --- a/packages/whook/API.md +++ b/packages/whook/API.md @@ -203,7 +203,7 @@ Allow to proxy constants directly by serializing it in the **Example** ```js -import { initBuildConstants } from '@whook/aws-lambda'; +import { initBuildConstants } from '@whook/whook'; import { alsoInject } from 'knifecycle'; export default alsoInject(['MY_OWN_CONSTANT'], initBuildConstants); diff --git a/packages/whook/README.md b/packages/whook/README.md index 9bfa8e14..9935db16 100644 --- a/packages/whook/README.md +++ b/packages/whook/README.md @@ -240,7 +240,7 @@ Allow to proxy constants directly by serializing it in the **Example** ```js -import { initBuildConstants } from '@whook/aws-lambda'; +import { initBuildConstants } from '@whook/whook'; import { alsoInject } from 'knifecycle'; export default alsoInject(['MY_OWN_CONSTANT'], initBuildConstants);