From 8a1724903450b13cc77d358da347f41b7c3b6b2d Mon Sep 17 00:00:00 2001 From: Nicolas Froidure Date: Mon, 30 Mar 2020 10:06:55 +0200 Subject: [PATCH] feat(@whook/example): add Google Cloud Functions build --- packages/whook-example/README.md | 51 ++++ packages/whook-example/package.json | 2 + .../src/commands/terraformValues.ts | 241 ++++++++++++++++++ packages/whook-example/src/index.ts | 21 +- packages/whook-example/terraform/functions.tf | 53 ++++ packages/whook-example/terraform/main.tf | 67 +++++ 6 files changed, 428 insertions(+), 7 deletions(-) create mode 100644 packages/whook-example/src/commands/terraformValues.ts create mode 100644 packages/whook-example/terraform/functions.tf create mode 100644 packages/whook-example/terraform/main.tf diff --git a/packages/whook-example/README.md b/packages/whook-example/README.md index dca4e184..de11b0af 100644 --- a/packages/whook-example/README.md +++ b/packages/whook-example/README.md @@ -57,6 +57,57 @@ Debug `knifecycle` internals (dependency injection issues): DEBUG=knifecycle npm run dev ``` +# Deploying with Google Cloud Functions + +Create a project and save its credentials to `.credentials.json`. + +Then install Terraform: +```sh +wget https://releases.hashicorp.com/terraform/0.12.24/terraform_0.12.24_linux_amd64.zip +mkdir .bin +unzip -d .bin terraform_0.12.24_linux_amd64.zip +rm terraform_0.12.24_linux_amd64.zip +``` + +Then initialize the Terraform configuration: +```sh +.bin/terraform init ./terraform; +``` + +Create a new workspace: +```sh +.bin/terraform workspace new staging +``` + +Build the functions: +```sh +NODE_ENV=staging npm run build +``` + +Build the Whook commands Terraform depends on: +```sh +npm run compile +``` + +Plan the deployment: +```sh +.bin/terraform plan -var="project_id=my-project-1664" \ + -out=terraform.plan terraform +``` + +Apply changes: +```sh +# parallelism may be necessary to avoid hitting +# timeouts with a slow connection +.bin/terraform apply -parallelism=1 terraform.plan +``` + +Finally retrieve the API URL and enjoy! +```sh +.bin/terraform -var="project_id=my-project-1664" \ + output api_url +``` + [//]: # (::contents:end) # Authors diff --git a/packages/whook-example/package.json b/packages/whook-example/package.json index 120345e3..9e29db7d 100644 --- a/packages/whook-example/package.json +++ b/packages/whook-example/package.json @@ -77,6 +77,7 @@ "@whook/authorization": "^3.1.3", "@whook/cli": "^3.1.3", "@whook/cors": "^3.1.3", + "@whook/gcp-functions": "^3.1.3", "@whook/http-router": "^3.1.3", "@whook/http-transaction": "^3.1.3", "@whook/swagger-ui": "^3.1.3", @@ -84,6 +85,7 @@ "common-services": "^7.0.0", "ecstatic": "^4.1.2", "http-auth-utils": "^2.3.0", + "js-yaml": "^3.13.1", "knifecycle": "^9.0.0", "strict-qs": "^6.0.2", "yerror": "^5.0.0", diff --git a/packages/whook-example/src/commands/terraformValues.ts b/packages/whook-example/src/commands/terraformValues.ts new file mode 100644 index 00000000..a9b77fcc --- /dev/null +++ b/packages/whook-example/src/commands/terraformValues.ts @@ -0,0 +1,241 @@ +import { extra, autoService } from 'knifecycle'; +import { LogService } from 'common-services'; +import { ENVService, identity } from '@whook/whook'; +import { readArgs, WhookCommandArgs, WhookCommandDefinition } from '@whook/cli'; +import { OpenAPIV3 } from 'openapi-types'; +import { getOpenAPIOperations } from '@whook/http-router'; +import YError from 'yerror'; +import { exec } from 'child_process'; +import crypto from 'crypto'; +import yaml from 'js-yaml'; + +export type WhookGoogleFunctionsBaseBuildConfiguration = { + private?: boolean; + memory?: number; + timeout?: number; + suffix?: string; + sourceOperationId?: string; +}; +export type WhookGoogleFunctionsBuildConfiguration = { + type: 'http'; +} & WhookGoogleFunctionsBaseBuildConfiguration; + +export const definition: WhookCommandDefinition = { + description: 'A command printing functions informations for Terraform', + example: `whook terraformValues --type paths`, + arguments: { + type: 'object', + additionalProperties: false, + required: ['type'], + properties: { + type: { + description: 'Type of values to return', + type: 'string', + enum: ['globals', 'paths', 'functions', 'function'], + }, + pretty: { + description: 'Pretty print JSON values', + type: 'boolean', + }, + functionName: { + description: 'Name of the function', + type: 'string', + }, + pathsIndex: { + description: 'Index of the paths to retrieve', + type: 'number', + }, + functionType: { + description: 'Types of the functions to return', + type: 'string', + }, + }, + }, +}; + +export default extra(definition, autoService(initTerraformValuesCommand)); + +async function initTerraformValuesCommand({ + API, + BASE_PATH, + log, + args, + execAsync = _execAsync, +}: { + API: OpenAPIV3.Document; + BASE_PATH: string; + log: LogService; + args: WhookCommandArgs; + execAsync: typeof _execAsync; +}) { + return async () => { + const { type, pretty, functionName, pathsIndex, functionType } = readArgs( + definition.arguments, + args, + ) as { + type: string; + pretty: boolean; + functionName: string; + pathsIndex: number; + functionType: string; + }; + const operations = getOpenAPIOperations(API); + const configurations = operations.map(operation => { + const whookConfiguration = (operation['x-whook'] || { + type: 'http', + }) as WhookGoogleFunctionsBuildConfiguration; + const configuration = { + type: 'http', + timeout: '10', + memory: '128', + description: operation.summary || '', + enabled: 'true', + sourceOperationId: operation.operationId, + suffix: '', + ...Object.keys(whookConfiguration).reduce( + (accConfigurations, key) => ({ + ...accConfigurations, + [key]: whookConfiguration[key].toString(), + }), + {}, + ), + }; + const qualifiedOperationId = + (configuration.sourceOperationId || operation.operationId) + + (configuration.suffix || ''); + + return { + qualifiedOperationId, + method: operation.method.toUpperCase(), + path: operation.path, + ...configuration, + }; + }); + + if (type === 'globals') { + const commitHash = await execAsync(`git rev-parse HEAD`); + const commitMessage = ( + await execAsync(`git rev-list --format=%B --max-count=1 HEAD`) + ).split('\n')[1]; + const openapi2 = yaml.safeDump({ + swagger: '2.0', + info: { + title: API.info.title, + description: API.info.description, + version: API.info.version, + }, + host: '${infos_host}', + basePath: BASE_PATH, + schemes: ['https'], + produces: ['application/json'], + paths: configurations.reduce((accPaths, configuration) => { + return { + ...accPaths, + [configuration.path]: { + ...(accPaths[configuration.path] || {}), + [configuration.method.toLowerCase()]: { + summary: configuration.description || '', + operationId: configuration.qualifiedOperationId, + 'x-google-backend': { + address: `\${function_${configuration.qualifiedOperationId}}`, + }, + responses: { + '200': { description: 'x', schema: { type: 'string' } }, + }, + }, + }, + }; + }, {}), + }); + const openapiHash = crypto + .createHash('md5') + .update(JSON.stringify(API)) + .digest('hex'); + const infos = { + commitHash, + commitMessage, + openapi2, + openapiHash, + }; + log('info', JSON.stringify(infos)); + return; + } + + if (type === 'functions') { + const functions = configurations + .filter(configuration => + functionType ? configuration.type === functionType : true, + ) + .reduce( + (accLambdas, configuration) => ({ + ...accLambdas, + [configuration.qualifiedOperationId]: + configuration.qualifiedOperationId, + }), + {}, + ); + + log('info', `${JSON.stringify(functions, null, pretty ? 2 : 0)}`); + return; + } + + if (!functionName) { + throw new YError('E_FUNCTION_NAME_REQUIRED'); + } + + const functionConfiguration = configurations.find( + ({ qualifiedOperationId }) => qualifiedOperationId === functionName, + ); + + log( + 'info', + `${JSON.stringify(functionConfiguration, null, pretty ? 2 : 0)}`, + ); + }; +} + +function buildPartName(parts: string[]): string { + return parts + .map((aPart, anIndex) => { + const realPartName = aPart.replace(/[{}]/g, ''); + + return anIndex + ? realPartName[0].toUpperCase() + realPartName.slice(1) + : realPartName; + }) + .join(''); +} + +function fixAWSSchedule(schedule: string): string { + if (typeof schedule === 'undefined') { + return ''; + } + + // The last wildcard is for years. + // This is a non-standard AWS addition... + // Also, we have to put a `?` in either + // day(month) or day(week) to fit AWS + // way of building cron tabs... + const fields = schedule.split(' '); + + if ('*' === fields[4]) { + fields[4] = '?'; + } else if ('*' === fields[2]) { + fields[2] = '?'; + } else { + throw new YError('E_BAD_AWS_SCHEDULE', schedule); + } + return `cron(${fields.concat('*').join(' ')})`; +} + +async function _execAsync(command: string): Promise { + return await new Promise((resolve, reject) => { + exec(command, (err: Error, stdout: string, stderr: string) => { + if (err) { + reject(YError.wrap(err, 'E_EXEC_FAILURE', stderr)); + return; + } + resolve(stdout.trim()); + }); + }); +} diff --git a/packages/whook-example/src/index.ts b/packages/whook-example/src/index.ts index ca57aa1c..6f5d3d2a 100644 --- a/packages/whook-example/src/index.ts +++ b/packages/whook-example/src/index.ts @@ -9,7 +9,10 @@ import { } from '@whook/whook'; import initHTTPRouter from '@whook/http-router'; import wrapHTTPRouterWithSwaggerUI from '@whook/swagger-ui'; -import YError from 'yerror'; +import { + runBuild as runBaseBuild, + prepareBuildEnvironment as prepareBaseBuildEnvironment, +} from '@whook/gcp-functions'; // Per convention a Whook server main file must export // the following 3 functions to be composable: @@ -72,7 +75,13 @@ export async function prepareEnvironment( $.register(constant('TRANSACTIONS', {})); // Setup your own whook plugins or avoid whook defaults by leaving it empty - $.register(constant('WHOOK_PLUGINS', ['@whook/cli', '@whook/whook'])); + $.register( + constant('WHOOK_PLUGINS', [ + '@whook/cli', + '@whook/whook', + '@whook/gcp-functions', + ]), + ); return $; } @@ -85,10 +94,8 @@ export async function prepareEnvironment( export async function runBuild( innerPrepareEnvironment = prepareBuildEnvironment, ): Promise { - throw new YError('E_NO_BUILD_IMPLEMENTED'); - // Usually, here you call the installed build - // return runBaseBuild(innerPrepareEnvironment); + return runBaseBuild(innerPrepareEnvironment); } // The `prepareBuildEnvironment` create the build @@ -99,7 +106,7 @@ export async function prepareBuildEnvironment( $ = await prepareEnvironment($); // Usually, here you call the installed build env - // $ = await prepareBaseBuildEnvironment($); + $ = await prepareBaseBuildEnvironment($); // The build often need to know were initializer // can be found to create a static build and @@ -111,7 +118,7 @@ export async function prepareBuildEnvironment( obfuscator: require.resolve( '@whook/http-transaction/dist/services/obfuscator', ), - log: require.resolve('common-services/dist/log'), + log: require.resolve('@whook/gcp-functions/dist/services/log'), time: require.resolve('common-services/dist/time'), delay: require.resolve('common-services/dist/delay'), }), diff --git a/packages/whook-example/terraform/functions.tf b/packages/whook-example/terraform/functions.tf new file mode 100644 index 00000000..c19b9f70 --- /dev/null +++ b/packages/whook-example/terraform/functions.tf @@ -0,0 +1,53 @@ +data "external" "functionConfiguration" { + for_each = data.external.functions.result + + program = ["env", "NODE_ENV=${terraform.workspace}", "npx", "whook", "terraformValues", "--type='function'", "--functionName='${each.key}'"] +} + +resource "google_storage_bucket" "storage_bucket" { + name = "whook_functions" +} + +data "archive_file" "functions" { + for_each = data.external.functions.result + + type = "zip" + source_dir = "./builds/${terraform.workspace}/${each.key}" + output_path = "./builds/${terraform.workspace}/${each.key}.zip" +} + +resource "google_storage_bucket_object" "storage_bucket_object" { + for_each = data.external.functions.result + + name = "${terraform.workspace}_${each.key}" + source = "./builds/${terraform.workspace}/${each.key}.zip" + bucket = google_storage_bucket.storage_bucket.name + depends_on = [data.archive_file.functions] +} + +resource "google_cloudfunctions_function" "cloudfunctions_function" { + for_each = data.external.functions.result + + name = "${terraform.workspace}_${each.key}" + description = data.external.functionConfiguration[each.key].result["description"] + runtime = "nodejs10" + + available_memory_mb = data.external.functionConfiguration[each.key].result["memory"] + timeout = data.external.functionConfiguration[each.key].result["timeout"] + source_archive_bucket = google_storage_bucket.storage_bucket.name + source_archive_object = google_storage_bucket_object.storage_bucket_object[each.key].name + trigger_http = true + entry_point = "default" +} + +# Seems to not work (no idea why) +# resource "google_cloudfunctions_function_iam_member" "invoker" { +# for_each = data.external.functions.result + +# project = google_cloudfunctions_function.cloudfunctions_function[each.key].project +# region = google_cloudfunctions_function.cloudfunctions_function[each.key].region +# cloud_function = google_cloudfunctions_function.cloudfunctions_function[each.key].name + +# role = "roles/cloudfunctions.invoker" +# member = "allUsers" +# } diff --git a/packages/whook-example/terraform/main.tf b/packages/whook-example/terraform/main.tf new file mode 100644 index 00000000..43b451b7 --- /dev/null +++ b/packages/whook-example/terraform/main.tf @@ -0,0 +1,67 @@ +variable "project_id" { + type = string +} + +variable "region" { + type = string + default = "europe-west1" +} + +variable "zone" { + type = string + default = "europe-west1-b" +} + +variable "api_name" { + type = string + default = "api" +} + +provider "google" { + version = "~> 3.14" + project = var.project_id + region = var.region + zone = var.zone + credentials = file(".credentials.json") +} + +provider "archive" { + version = "~> 1.3" +} + +provider "template" { + version = "~> 2.1.2" +} + +output "api_url" { + value = google_endpoints_service.endpoints_service.dns_address +} + +data "google_project" "project" { + project_id = var.project_id +} + +# imports the functions list +data "external" "functions" { + program = ["env", "NODE_ENV=${terraform.workspace}", "npx", "whook", "terraformValues", "--type='functions'", "--functionType='http'"] +} +data "external" "globals" { + program = ["env", "NODE_ENV=${terraform.workspace}", "npx", "whook", "terraformValues", "--type='globals'"] +} + +data "template_file" "template_file" { + template = data.external.globals.result["openapi2"] + + vars = merge({ + "infos_host" : "${var.api_name}.endpoints.${data.google_project.project.project_id}.cloud.goog" + }, zipmap( + [for key in keys(data.external.functions.result) : "function_${key}"], + [for key in keys(data.external.functions.result) : google_cloudfunctions_function.cloudfunctions_function[key].https_trigger_url] + )) +} + +resource "google_endpoints_service" "endpoints_service" { + service_name = "${var.api_name}.endpoints.${data.google_project.project.project_id}.cloud.goog" + project = data.google_project.project.project_id + openapi_config = data.template_file.template_file.rendered +}