From 9b66ff1981ca23467b97a089af4a70d782ce3d4b Mon Sep 17 00:00:00 2001 From: Nicolas Froidure Date: Mon, 21 Aug 2023 07:35:28 +0200 Subject: [PATCH] feat(deploy): add the terraform deployment --- packages/whook-example/README.md | 51 ++++ packages/whook-example/src/cli.test.ts | 3 +- .../src/commands/terraformValues.ts | 228 ++++++++++++++++++ packages/whook-example/terraform/functions.tf | 53 ++++ packages/whook-example/terraform/main.tf | 48 ++++ packages/whook-example/terraform/variables.tf | 23 ++ 6 files changed, 405 insertions(+), 1 deletion(-) create mode 100644 packages/whook-example/src/commands/terraformValues.ts create mode 100644 packages/whook-example/terraform/functions.tf create mode 100644 packages/whook-example/terraform/main.tf create mode 100644 packages/whook-example/terraform/variables.tf diff --git a/packages/whook-example/README.md b/packages/whook-example/README.md index 44a52881..ad048a3d 100644 --- a/packages/whook-example/README.md +++ b/packages/whook-example/README.md @@ -102,6 +102,57 @@ Debug `knifecycle` internals (dependency injection issues): DEBUG=knifecycle npm run dev ``` +## Deploying with Google Cloud Functions + +Create a project and save its credentials to `.credentials.json`. + +Then install Terraform: +```sh +wget https://releases.hashicorp.com/terraform/0.12.24/terraform_0.12.24_linux_amd64.zip +mkdir .bin +unzip -d .bin terraform_0.12.24_linux_amd64.zip +rm terraform_0.12.24_linux_amd64.zip +``` + +Then initialize the Terraform configuration: +```sh +.bin/terraform init ./terraform; +``` + +Create a new workspace: +```sh +.bin/terraform workspace new staging +``` + +Build the functions: +```sh +NODE_ENV=staging npm run build +``` + +Build the Whook commands Terraform depends on: +```sh +npm run compile +``` + +Plan the deployment: +```sh +.bin/terraform plan -var="project_id=my-project-1664" \ + -out=terraform.plan terraform +``` + +Apply changes: +```sh +# parallelism may be necessary to avoid hitting +# timeouts with a slow connection +.bin/terraform apply -parallelism=1 terraform.plan +``` + +Finally retrieve the API URL and enjoy! +```sh +.bin/terraform -var="project_id=my-project-1664" \ + output api_url +``` + ## Testing the GCP Functions ```sh diff --git a/packages/whook-example/src/cli.test.ts b/packages/whook-example/src/cli.test.ts index 843c0c0e..0ed9ed39 100644 --- a/packages/whook-example/src/cli.test.ts +++ b/packages/whook-example/src/cli.test.ts @@ -19,8 +19,9 @@ On air 🚀🌕 ", "stdout": " -# Provided by "@whook/example": 1 commands +# Provided by "@whook/example": 2 commands - printEnv: A command printing every env values +- terraformValues: A command printing functions informations for Terraform # Provided by "@whook/whook": 8 commands diff --git a/packages/whook-example/src/commands/terraformValues.ts b/packages/whook-example/src/commands/terraformValues.ts new file mode 100644 index 00000000..bc49f782 --- /dev/null +++ b/packages/whook-example/src/commands/terraformValues.ts @@ -0,0 +1,228 @@ +import { extra, autoService } from 'knifecycle'; +import { readArgs } from '@whook/whook'; +import { getOpenAPIOperations } from '@whook/http-router'; +import { YError } from 'yerror'; +import { exec } from 'child_process'; +import crypto from 'crypto'; +import yaml from 'js-yaml'; +import type { ExecException } from 'child_process'; +import type { LogService } from 'common-services'; +import type { + WhookCommandArgs, + WhookCommandDefinition, + WhookAPIHandlerDefinition, +} from '@whook/whook'; +import type { OpenAPIV3_1 } from 'openapi-types'; + +export const definition: WhookCommandDefinition = { + description: 'A command printing functions informations for Terraform', + example: `whook terraformValues --type paths`, + arguments: { + type: 'object', + additionalProperties: false, + required: ['type'], + properties: { + type: { + description: 'Type of values to return', + type: 'string', + enum: ['globals', 'paths', 'functions', 'function'], + }, + pretty: { + description: 'Pretty print JSON values', + type: 'boolean', + }, + functionName: { + description: 'Name of the function', + type: 'string', + }, + pathsIndex: { + description: 'Index of the paths to retrieve', + type: 'number', + }, + functionType: { + description: 'Types of the functions to return', + type: 'string', + }, + }, + }, +}; + +export default extra(definition, autoService(initTerraformValuesCommand)); + +async function initTerraformValuesCommand({ + API, + BASE_PATH, + log, + args, + execAsync = _execAsync, +}: { + API: OpenAPIV3_1.Document; + BASE_PATH: string; + log: LogService; + args: WhookCommandArgs; + execAsync: typeof _execAsync; +}) { + return async () => { + const { + namedArguments: { type, pretty, functionName, functionType }, + } = readArgs<{ + type: string; + pretty: boolean; + functionName: string; + functionType: string; + }>(definition.arguments, args); + const operations = + getOpenAPIOperations( + API, + ); + const configurations = operations.map((operation) => { + const whookConfiguration = (operation['x-whook'] || { + type: 'http', + }) as WhookAPIHandlerDefinition['operation']['x-whook']; + const configuration = { + type: 'http', + timeout: '10', + memory: '128', + description: operation.summary || '', + enabled: 'true', + sourceOperationId: operation.operationId, + suffix: '', + ...Object.keys(whookConfiguration || {}).reduce( + (accConfigurations, key) => ({ + ...accConfigurations, + [key]: ( + ( + whookConfiguration as NonNullable< + WhookAPIHandlerDefinition['operation']['x-whook'] + > + )[key] as string + ).toString(), + }), + {}, + ), + }; + const qualifiedOperationId = + (configuration.sourceOperationId || operation.operationId) + + (configuration.suffix || ''); + + return { + qualifiedOperationId, + method: operation.method.toUpperCase(), + path: operation.path, + ...configuration, + }; + }); + + if (type === 'globals') { + const commitHash = await execAsync(`git rev-parse HEAD`); + const commitMessage = ( + await execAsync(`git rev-list --format=%B --max-count=1 HEAD`) + ).split('\n')[1]; + const openapi2 = yaml.safeDump({ + swagger: '2.0', + info: { + title: API.info.title, + description: API.info.description, + version: API.info.version, + }, + host: '${infos_host}', + basePath: BASE_PATH, + schemes: ['https'], + produces: ['application/json'], + paths: configurations.reduce((accPaths, configuration) => { + const operation = operations.find( + ({ operationId }) => + operationId === configuration.sourceOperationId, + ); + + return { + ...accPaths, + [configuration.path]: { + ...(accPaths[configuration.path] || {}), + [configuration.method.toLowerCase()]: { + summary: configuration.description || '', + operationId: configuration.qualifiedOperationId, + ...((operation?.parameters || []).length + ? { + parameters: ( + operation?.parameters as OpenAPIV3_1.ParameterObject[] + ).map(({ in: theIn, name, required }) => ({ + in: theIn, + name, + type: 'string', + required: required || false, + })), + } + : undefined), + 'x-google-backend': { + address: `\${function_${configuration.qualifiedOperationId}}`, + }, + responses: { + '200': { description: 'x', schema: { type: 'string' } }, + }, + }, + }, + }; + }, {}), + }); + const openapiHash = crypto + .createHash('md5') + .update(JSON.stringify(API)) + .digest('hex'); + const infos = { + commitHash, + commitMessage, + openapi2, + openapiHash, + }; + log('info', JSON.stringify(infos)); + return; + } + + if (type === 'functions') { + const functions = configurations + .filter((configuration) => + functionType ? configuration.type === functionType : true, + ) + .reduce( + (accLambdas, configuration) => ({ + ...accLambdas, + [configuration.qualifiedOperationId]: + configuration.qualifiedOperationId, + }), + {}, + ); + + log('info', `${JSON.stringify(functions, null, pretty ? 2 : 0)}`); + return; + } + + if (!functionName) { + throw new YError('E_FUNCTION_NAME_REQUIRED'); + } + + const functionConfiguration = configurations.find( + ({ qualifiedOperationId }) => qualifiedOperationId === functionName, + ); + + log( + 'info', + `${JSON.stringify(functionConfiguration, null, pretty ? 2 : 0)}`, + ); + }; +} + +async function _execAsync(command: string): Promise { + return await new Promise((resolve, reject) => { + exec( + command, + (err: ExecException | null, stdout: string, stderr: string) => { + if (err) { + reject(YError.wrap(err, 'E_EXEC_FAILURE', stderr)); + return; + } + resolve(stdout.trim()); + }, + ); + }); +} diff --git a/packages/whook-example/terraform/functions.tf b/packages/whook-example/terraform/functions.tf new file mode 100644 index 00000000..9b63dcb5 --- /dev/null +++ b/packages/whook-example/terraform/functions.tf @@ -0,0 +1,53 @@ +data "external" "functionConfiguration" { + for_each = data.external.functions.result + + program = ["env", "APP_ENV=${terraform.workspace}", "NODE_ENV=${var.node_env}", "npx", "whook", "terraformValues", "--type='function'", "--functionName='${each.key}'"] +} + +resource "google_storage_bucket" "storage_bucket" { + name = "whook_functions" +} + +data "archive_file" "functions" { + for_each = data.external.functions.result + + type = "zip" + source_dir = "./builds/${terraform.workspace}/${each.key}" + output_path = "./builds/${terraform.workspace}/${each.key}.zip" +} + +resource "google_storage_bucket_object" "storage_bucket_object" { + for_each = data.external.functions.result + + name = "${terraform.workspace}_${each.key}" + source = "./builds/${terraform.workspace}/${each.key}.zip" + bucket = google_storage_bucket.storage_bucket.name + depends_on = [data.archive_file.functions] +} + +resource "google_cloudfunctions_function" "cloudfunctions_function" { + for_each = data.external.functions.result + + name = "${terraform.workspace}_${each.key}" + description = data.external.functionConfiguration[each.key].result["description"] + runtime = "nodejs10" + + available_memory_mb = data.external.functionConfiguration[each.key].result["memory"] + timeout = data.external.functionConfiguration[each.key].result["timeout"] + source_archive_bucket = google_storage_bucket.storage_bucket.name + source_archive_object = google_storage_bucket_object.storage_bucket_object[each.key].name + trigger_http = true + entry_point = "default" +} + +# Seems to not work (no idea why) +# resource "google_cloudfunctions_function_iam_member" "invoker" { +# for_each = data.external.functions.result + +# project = google_cloudfunctions_function.cloudfunctions_function[each.key].project +# region = google_cloudfunctions_function.cloudfunctions_function[each.key].region +# cloud_function = google_cloudfunctions_function.cloudfunctions_function[each.key].name + +# role = "roles/cloudfunctions.invoker" +# member = "allUsers" +# } diff --git a/packages/whook-example/terraform/main.tf b/packages/whook-example/terraform/main.tf new file mode 100644 index 00000000..4b8b1afb --- /dev/null +++ b/packages/whook-example/terraform/main.tf @@ -0,0 +1,48 @@ +provider "google" { + version = "~> 3.14" + project = var.project_id + region = var.region + zone = var.zone + credentials = file(".credentials.json") +} + +provider "archive" { + version = "~> 1.3" +} + +provider "template" { + version = "~> 2.1.2" +} + +output "api_url" { + value = google_endpoints_service.endpoints_service.dns_address +} + +data "google_project" "project" { + project_id = var.project_id +} + +# imports the functions list +data "external" "functions" { + program = ["env", "NODE_ENV=${terraform.workspace}", "npx", "whook", "terraformValues", "--type='functions'", "--functionType='http'"] +} +data "external" "globals" { + program = ["env", "NODE_ENV=${terraform.workspace}", "npx", "whook", "terraformValues", "--type='globals'"] +} + +data "template_file" "template_file" { + template = data.external.globals.result["openapi2"] + + vars = merge({ + "infos_host" : "${var.api_name}.endpoints.${data.google_project.project.project_id}.cloud.goog" + }, zipmap( + [for key in keys(data.external.functions.result) : "function_${key}"], + [for key in keys(data.external.functions.result) : google_cloudfunctions_function.cloudfunctions_function[key].https_trigger_url] + )) +} + +resource "google_endpoints_service" "endpoints_service" { + service_name = "${var.api_name}.endpoints.${data.google_project.project.project_id}.cloud.goog" + project = data.google_project.project.project_id + openapi_config = data.template_file.template_file.rendered +} diff --git a/packages/whook-example/terraform/variables.tf b/packages/whook-example/terraform/variables.tf new file mode 100644 index 00000000..6b5d8cfb --- /dev/null +++ b/packages/whook-example/terraform/variables.tf @@ -0,0 +1,23 @@ +variable "node_env" { + type = string + default = "production" +} + +variable "project_id" { + type = string +} + +variable "region" { + type = string + default = "europe-west1" +} + +variable "zone" { + type = string + default = "europe-west1-b" +} + +variable "api_name" { + type = string + default = "api2" +} \ No newline at end of file