From 13e38cba4e23461ef64de063f040a17426a649e5 Mon Sep 17 00:00:00 2001 From: Jeremy Daly Date: Sun, 28 Apr 2019 01:00:38 -0400 Subject: [PATCH] initial commit --- .gitignore | 2 + README.md | 102 ++++++++++++++++ index.js | 301 ++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 13 ++ package.json | 30 +++++ 5 files changed, 448 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 index.js create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e8427e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# package directories +node_modules diff --git a/README.md b/README.md new file mode 100644 index 0000000..fdfa9b3 --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +# serverless-cloudside-plugin + +[Serverless](http://www.serverless.com) plugin for using _cloudside_ resources when developing functions locally. + +[![Serverless](http://public.serverless.com/badges/v3.svg)](http://www.serverless.com) +[![npm](https://img.shields.io/npm/v/serverless-cloudside-plugin.svg)](https://www.npmjs.com/package/serverless-cloudside-plugin) +[![npm](https://img.shields.io/npm/l/serverless-cloudside-plugin.svg)](https://www.npmjs.com/package/serverless-cloudside-plugin) + +This plugin allows you to use AWS CloudFormation intrinsic functions (such as `!Ref` and `!GetAt`) to reference cloud resources during local development. When added to your `environment` variables, these values are replaced with the same identifiers used when deployed to the cloud. You can invoke your functions locally, use the `serverless-offline` plugin, or use a compatible test runner that uses the `serverless invoke test` command. You can now keep your `serverless.yml` files free from pseudo variables and other concatenated strings and simply use the the built-in CloudFormation features. + +## Installation + +#### Install using npm + +Install the module using npm: +```bash +npm install serverless-cloudside-plugin --save-dev +``` + +Add `serverless-cloudside-plugin` to the plugin list of your `serverless.yml` file: + +```yaml +plugins: + - serverless-cloudside-plugin +``` + +## Usage + +When executing your function locally, the `serverless-cloudside-plugin` will replace any environment variable that contains either a `!Ref` or a `!GetAt` that references a CloudFormation resource within your `serverless.yml` file. + +In the example below, we are creating an SQS Queue named `myQueue` and referencing it (using a CloudFormation intrinsic function) in an environment variable named `QUEUE`. + +```yaml +functions: + myFunction: + handler: myFunction.handler + environment: + QUEUE: !Ref myQueue + +resources: + Resources: + myQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: ${self:service}-${self:provider.stage}-myQueue +``` + +If we deploy this to the cloud, our `!Ref myQueue` will be replaced with a `QueueUrl` (e.g. _https://sqs.us-east-1.amazonaws.com/1234567890/sample-service-dev-myQueue_). We can then use that when invoking the AWS SDK and working with our queue. +However, if we were to invoke this function locally using `sls invoke local -f myFunction`, our `QUEUE` environment variable would return `[object Object]` instead of our `QueueUrl`. This is because the Serverless Framework is actually replacing our `!Ref` with: `{ "Ref": "myQueue" }`. + +There are workarounds to this, typically involving using pseudo variables to construct our own URL. But this method is error prone and requires us to hardcode formats for the different service types. Using the `serverless-cloudside-plugin`, you can now use the simple reference format above, and always retrieve the correct `PhysicalResourceId` for the resource. + +#### Invoking a function locally +Once the plugin is installed, you will have a new `invoke` option named `invoke cloudside`. Simply run this command with a function and it will resolve all of your cloud variables and then execute the standard `invoke local` command. + +```bash +sls invoke cloudside -f myFunction +``` + +**PLEASE NOTE** that in order for resources to be referenced, you must deploy your service to the cloud at least initially. References to non-deployed resources will be populated with **"RESOURCE NOT DEPLOYED"**. + +All `invoke local` parameters are supported such as `--stage` and `--path`, as well as the new `--docker` flag that lets you run your function locally in a Docker container. This mimics the Lambda environment much more closely than your local machine. + +By default, the plugin will reference resources from your current CloudFormation stack (including your "stage" if it is part of your stack name). You can change the cloudside stage by using the `--cloudStage` option and supplying the stage name that you'd like to use. For example, if you are developing in your `dev` stage locally, but want to use a DynamoDB table that is deployed to the `test` stage, you can do the following: + +```bash +sls invoke cloudside -f myFunction -s dev --cloudStage test +``` + +This will populate any `${opt:stage}` references with `dev`, but your `!Ref` values will use the ones from your `test` stage. + +You might also want to pull values from an entirely different CloudFormation stack. You can do this by using the `--stackName` option and supplying the complete stack name. For example: + +```bash +sls invoke cloudside -f myFunction --stackName someOtherStack-dev +``` + +#### Using with the serverless-offline plugin +The `serverless-offline` plugin is a great tool for testing your serverless APIs locally, but it has the same problem referencing CloudFormation resources. The `serverless-cloudside-plugin` lets you run `serverless-offline` with all of your cloud variables correctly replaced. + +```bash +sls offline cloudside +``` + +The above command will start the API Gateway emulator and allow you to test your functions locally. The `--cloudStage` and `--stackName` options are supported as well as all of the `serverless-offline` options. + +#### Using with a test runner +You can use this plugin with other test runner plugins such as `serverless-mocha-plugin`. This will make it easier to run integration tests (including in your CI/CD systems) before deploying. Simply run the following when invoking your tests: + +```bash +sls invoke test cloudside -f myFunction +``` + +This plugin extends the `invoke test` command, so any test runner plugin that uses that format should work correctly. All plugin options should remain available. + +## Available Functions +This plugin currently supports the `!Ref` function that returns the `PhysicalResourceId` from CloudFormation. For most resources, this is the value you will need to interact with the corresponding service in the AWS SDK (e.g. `QueueUrl` for SQS, `TopicArn` for SNS, etc.). + +There is also initial (and limited) support for using `!GetAt` to retrieve an **ARN**. For example, you may use `!GetAt myQueue.Arn` to retrieve the ARN for `myQueue`. The plugin generates the ARN based on the service type. For supported types, it will return a properly formatted ARN. For others, it will replace the value with **"FUNCTION NOT SUPPORTED"**. In most cases, it should be possible to support generating an ARN for a resource, but the format will need to be added to the plugin. + +## Contributions +Contributions, ideas and bug reports are welcome and greatly appreciated. Please add [issues](https://github.com/jeremydaly/serverless-cloudside-plugin/issues) for suggestions and bug reports or create a pull request. diff --git a/index.js b/index.js new file mode 100644 index 0000000..b16b0ce --- /dev/null +++ b/index.js @@ -0,0 +1,301 @@ +'use strict'; + +const BbPromise = require('bluebird') + +class InvokeCloudside { + constructor(serverless, options) { + this.serverless = serverless + this.options = options + + this.commands = { + invoke: { + commands: { + cloudside: { + usage: 'Invoke function locally using cloudside resources', + lifecycleEvents: [ + 'loadCloudsideEnvVars', + 'invoke', + ], + options: { + function: { + usage: 'Name of the function', + shortcut: 'f', + required: true, + }, + cloudStage: { + usage: 'Stage to use for cloudside resources', + shortcut: 'S', + }, + stackName: { + usage: 'CloudFormation stack to use for cloudside resources', + shortcut: 'y', + }, + path: { + usage: 'Path to JSON or YAML file holding input data', + shortcut: 'p', + }, + data: { + usage: 'input data', + shortcut: 'd', + }, + raw: { + usage: 'Flag to pass input data as a raw string', + }, + context: { + usage: 'Context of the service', + shortcut: 'c', + }, + contextPath: { + usage: 'Path to JSON or YAML file holding context data', + shortcut: 'x', + }, + env: { + usage: 'Override environment variables. e.g. --env VAR1=val1 --env VAR2=val2', + shortcut: 'e', + }, + docker: { usage: 'Flag to turn on docker use for node/python/ruby/java' }, + 'docker-arg': { + usage: 'Arguments to docker run command. e.g. --docker-arg "-p 9229:9229"', + }, + }, + + }, + test: { + usage: 'NOT INSTALLED - A testing plugin must be installed to use "invoke test cloudside"', + commands: { + cloudside: { + usage: 'Invoke test(s) using cloudside resources', + lifecycleEvents: [ + 'loadCloudsideEnvVars', + 'start' + ], + options: { + cloudStage: { + usage: 'Stage to use for cloudside resources', + shortcut: 'S', + }, + stackName: { + usage: 'CloudFormation stack to use for cloudside resources', + shortcut: 'y', + } + } + } + } + } + }, + }, + offline: { + lifecycleEvents: [ + 'checkInstall' + ], + usage: 'NOT INSTALLED - This plugin must be installed to use "offline cloudside"', + commands: { + cloudside: { + usage: 'Simulates API Gateway to call your lambda functions offline using cloudside resources', + lifecycleEvents: [ + 'checkInstall', + 'loadCloudsideEnvVars', + 'start' + ], + options: { + cloudStage: { + usage: 'Stage to use for cloudside resources', + shortcut: 'S', + }, + stackName: { + usage: 'CloudFormation stack to use for cloudside resources', + shortcut: 'y', + } + } + } + } + } + }; + + this.hooks = { + 'invoke:cloudside:loadCloudsideEnvVars': () => BbPromise.bind(this).then(this.loadCloudsideEnvVars), + 'invoke:test:cloudside:loadCloudsideEnvVars': () => BbPromise.bind(this).then(this.loadCloudsideEnvVars), + 'offline:cloudside:loadCloudsideEnvVars': () => BbPromise.bind(this).then(this.loadCloudsideEnvVars), + 'after:invoke:cloudside:invoke': () => this.serverless.pluginManager.run(['invoke', 'local']), + 'after:offline:cloudside:start': () => this.serverless.pluginManager.run(['offline', 'start']), + 'offline:checkInstall': () => BbPromise.bind(this).then(this.checkInstall('serverless-offline')), + 'offline:cloudside:checkInstall': () => BbPromise.bind(this).then(this.checkInstall('serverless-offline')), + 'after:invoke:test:cloudside:start': () => this.serverless.pluginManager.run(['invoke', 'test']), + }; + } + + checkInstall(plugin) { + console.log(this.serverless.service); + + if (!this.serverless.service.plugins.includes(plugin)) { + throw Error(`You must install "${plugin}" to use the "offline cloudside" feature.\n\n You can install it by running "serverless plugin install --name ${plugin}"`) + } + return BbPromise.resolve() + } + + // Set environment variables for "invoke cloudside" + loadCloudsideEnvVars() { + + // Get the stage (use the cloudstage option first, then stage option, then provider) + const stage = this.options.cloudStage ? this.options.cloudStage : + this.options.stage ? this.options.stage : + this.serverless.service.provider.stage + + // Get the stack name (use provided stackName or fallback to service name) + const stackName = this.options.stackName ? this.options.stackName : + this.serverless.service.provider.stackName ? + this.serverless.service.provider.stackName : + `${this.serverless.service.service}-${stage}` + + this.serverless.cli.log(`Loading cloudside resources for '${stackName}' stack.`) + + this.serverless.service.provider.environment.IS_CLOUDSIDE = true + this.serverless.service.provider.environment.CLOUDSIDE_STACK = stackName + + // Find all envs with CloudFormation Refs + let cloudsideVars = parseEnvs(this.serverless.service.provider.environment) + + let functions = this.options.function ? + { [this.options.function] : this.serverless.service.functions[this.options.function] } + : this.serverless.service.functions + + Object.keys(functions).map(fn => { + if (this.serverless.service.functions[fn].environment) { + let vars = parseEnvs(this.serverless.service.functions[fn].environment,fn) + for (let key in vars) { + cloudsideVars[key] = cloudsideVars[key] ? cloudsideVars[key].concat(vars[key]) : vars[key] + } + } + }) + + // If references need resolving, call CF + if (Object.keys(cloudsideVars).length > 0) { + + const options = { useCache: true } + + return this.serverless.getProvider('aws') + .request('CloudFormation', + 'describeStackResources', + { StackName: stackName }, + options) + .then(res => { + if (res.StackResources) { + // Loop through the returned StackResources + for (let i = 0; i < res.StackResources.length; i++) { + + let resource = cloudsideVars[res.StackResources[i].LogicalResourceId] + + // If the logicial id exists, add the PhysicalResourceId to the ENV + if (resource) { + for (let j = 0; j < resource.length; j++) { + + let value = resource[j].type == 'Ref' ? res.StackResources[i].PhysicalResourceId + : buildCloudValue(res.StackResources[i],resource[j].type) + + if (resource[j].fn) { + this.serverless.service.functions[resource[j].fn].environment[ + resource[j].env + ] = value + } else { + this.serverless.service.provider.environment[ + resource[j].env + ] = value + } + + } // end for + // Remove the cloudside variable + delete(cloudsideVars[res.StackResources[i].LogicalResourceId]) + } // end if + } // end for + } // end if StackResources + + // Replace remaining variables with warning + Object.keys(cloudsideVars).map(x => { + for (let j = 0; j < cloudsideVars[x].length; j++) { + if (cloudsideVars[x][j].fn) { + this.serverless.service.functions[cloudsideVars[x][j].fn].environment[ + cloudsideVars[x][j].env + ] = '' + } else { + this.serverless.service.provider.environment[ + cloudsideVars[x][j].env + ] = '' + } + } + }) + + return true + }).error(e => { + console.log(e) + }) + + } else { + return BbPromise.resolve() + } + } + +} + + + +// Parse the environment variables and return formatted mappings +const parseEnvs = (envs = {},fn) => Object.keys(envs).reduce((vars,key) => { + let logicalId,ref + + if (envs[key].Ref) { + logicalId = envs[key].Ref + ref = { type: 'Ref', env: key, fn } + } else if (envs[key]['Fn::GetAtt']) { + logicalId = envs[key]['Fn::GetAtt'][0] + ref = { type: envs[key]['Fn::GetAtt'][1], env: key, fn } + } else { + return vars + } + + vars[logicalId] = vars[logicalId] ? + vars[logicalId].concat([ref]) : [ref] + + return vars +},{}) + + + +// Build the cloud value based on type +const buildCloudValue = (resource,type) => { + switch(type) { + case 'Arn': + return generateArn(resource) + default: + return '' + } +} + +// Generate the ARN based on service type +// TODO: add more service types or figure out a better way to do this +const generateArn = resource => { + + let stack = resource.StackId.split(':').slice(0,5) + let resourceType = resource.ResourceType.split('::') + let serviceType = resourceType[1].toLowerCase() + stack[2] = serviceType + + switch(serviceType) { + case 'sqs': + stack.push(resource.PhysicalResourceId.split('/').slice(-1)) + break + case 'dynamodb': + case 'kinesis': + stack.push(resourceType[2].toLowerCase()+'/'+resource.PhysicalResourceId) + break + case 's3': + stack.splice(3,5,'','') + stack.push(resource.PhysicalResourceId) + break + default: + return '' + } + + return stack.join(':') +} + +module.exports = InvokeCloudside diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..39c81a1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,13 @@ +{ + "name": "serverless-cloudside-plugin", + "version": "0.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "bluebird": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.4.tgz", + "integrity": "sha512-FG+nFEZChJrbQ9tIccIfZJBz3J7mLrAhxakAbnrJWn8d7aKOC+LWifa0G+p4ZqKp4y13T7juYvdhq9NzKdsrjw==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..9cd9bf7 --- /dev/null +++ b/package.json @@ -0,0 +1,30 @@ +{ + "name": "serverless-cloudside-plugin", + "version": "1.0.0", + "description": "Serverless plugin for using cloudside resources during local development", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/jeremydaly/serverless-cloudside-plugin.git" + }, + "keywords": [ + "serverless", + "serverless-plugin", + "aws", + "nodejs", + "local development", + "offline" + ], + "author": "Jeremy Daly ", + "license": "MIT", + "bugs": { + "url": "https://github.com/jeremydaly/serverless-cloudside-plugin/issues" + }, + "homepage": "https://github.com/jeremydaly/serverless-cloudside-plugin#readme", + "dependencies": { + "bluebird": "^3.5.4" + } +}