From fe288d086f3d03e8ad22da62f572a71674d148a1 Mon Sep 17 00:00:00 2001 From: Frank Schmid Date: Sat, 4 Mar 2017 17:45:40 +0100 Subject: [PATCH 1/2] Fixes #5 Allow resource sets to be different in aliases. --- lib/aliasRestructureStack.js | 159 ++++++++++++++++++++++++++++++++++- lib/stackInformation.js | 3 +- 2 files changed, 158 insertions(+), 4 deletions(-) diff --git a/lib/aliasRestructureStack.js b/lib/aliasRestructureStack.js index 13dfef4..90e0157 100644 --- a/lib/aliasRestructureStack.js +++ b/lib/aliasRestructureStack.js @@ -74,7 +74,7 @@ module.exports = { aliasHandleFunctions(currentTemplate, aliasStackTemplates) { - this._serverless.cli.log('Resolving function versions ...'); + this.options.verbose && this._serverless.cli.log('Processing functions'); const stackName = this._provider.naming.getStackName(); const stageStack = this._serverless.service.provider.compiledCloudFormationTemplate; @@ -173,6 +173,8 @@ module.exports = { if (exposeApi) { + this.options.verbose && this._serverless.cli.log('Processing API'); + // Export the API for the alias stacks stageStack.Outputs.ApiGatewayRestApi = { Description: 'API Gateway API', @@ -272,8 +274,6 @@ module.exports = { aliasResources.push(apiResources); aliasResources.push(apiMethods); aliasResources.push(apiLambdaPermissions); - //console.log(JSON.stringify(apiLambdaPermissions, null, 2)); - //throw new Error('iwzgeiug'); } @@ -282,12 +282,165 @@ module.exports = { return BbPromise.resolve([ currentTemplate, aliasStackTemplates ]); }, + aliasHandleUserResources(currentTemplate, aliasStackTemplates) { + + const stageStack = this._serverless.service.provider.compiledCloudFormationTemplate; + const aliasStack = this._serverless.service.provider.compiledCloudFormationAliasTemplate; + const userResources = _.get(this._serverless.service, 'resources', { Resources: {}, Outputs: {} }); + + this.options.verbose && this._serverless.cli.log('Processing custom resources'); + + // Retrieve all resources referenced from other aliases + const aliasDependencies = _.reduce(aliasStackTemplates, (result, template) => { + try { + const resourceRefs = JSON.parse(_.get(template, 'Outputs.AliasResources.Value', "[]")); + const outputRefs = JSON.parse(_.get(template, 'Outputs.AliasOutputs.Value', "[]")); + const resources = _.assign({}, _.pick(_.get(currentTemplate, 'Resources'), resourceRefs, {})); + const outputs = _.assign({}, _.pick(_.get(currentTemplate, 'Outputs'), outputRefs, {})); + + // Check if there are IAM policy references for the alias resources and integrate them into + // the lambda policy. + + _.assign(result.Resources, resources); + _.assign(result.Outputs, outputs); + return result; + } catch (e) { + return result; + } + }, { Resources: {}, Outputs: {} }); + + // Logical resource ids are unique per stage + // Alias stacks reference the used resources through an Output reference + // On deploy, the plugin checks if a resource is already deployed from a stack + // and does a validation of the resource properties + // All used resources are copied from the current template + + // Extract the user resources that are not overrides of existing Serverless resources + const currentResources = + _.assign({}, + _.omitBy(_.get(userResources, 'Resources', {}), (value, name) => _.includes(_.keys(stageStack.Resources), name))); + + const currentOutputs = _.get(userResources, 'Outputs', {}); + + // Add the alias resources as output to the alias stack + aliasStack.Outputs.AliasResources = { + Description: 'Custom resource references', + Value: JSON.stringify(_.keys(currentResources)) + }; + + // Add the outputs as output to the alias stack + aliasStack.Outputs.AliasOutputs = { + Description: 'Custom output references', + Value: JSON.stringify(_.keys(currentOutputs)) + }; + + // FIXME: Deployments to the master (stage) alias should be allowed to reconfigure + // resources and outputs. Otherwise a "merge" of feature branches into a + // release branch would not be possible as resources would be rendered + // immutable otherwise. + + // Check if the resource is already used anywhere else with a different definition + _.forOwn(currentResources, (resource, name) => { + if (_.has(aliasDependencies.Resources, name) && !_.isMatch(aliasDependencies.Resources[name], resource)) { + + // If we deploy the master alias, allow reconfiguration of resources + if (this._alias === this._stage && resource.Type === aliasDependencies.Resources[name].Type) { + this._serverless.cli.log(`Reconfigure resource ${name}. Remember to update it in other aliases too.`); + } else { + return BbPromise.reject(new Error(`Resource ${name} is already deployed in another alias with a different configuration. Either you change your resource to match the other definition, or you change the logical resource id to deploy your resource separately.`)); + } + } + }); + + // Check if the output is already used anywhere else with a different definition + _.forOwn(currentOutputs, (output, name) => { + if (_.has(aliasDependencies.Outputs, name) && !_.isMatch(aliasDependencies.Outputs[name], output)) { + if (this._alias === this._stage) { + this._serverless.cli.log(`Reconfigure output ${name}. Remember to update it in other aliases too.`); + } else { + return BbPromise.reject(new Error(`Output ${name} is already deployed in another alias with a different configuration. Either you change your output to match the other definition, or you change the logical resource id to deploy your output separately.`)); + } + } + }); + + // Merge used alias resources and outputs into the stage + _.defaults(stageStack.Resources, aliasDependencies.Resources); + _.defaults(stageStack.Outputs, aliasDependencies.Outputs); + + //console.log(JSON.stringify(aliasDependencies, null, 2)); + //throw new Error('iwzgeiug'); + + return BbPromise.resolve([ currentTemplate, aliasStackTemplates ]); + }, + + /** + * Merge alias and current stack policies, so that all alias policy statements + * are present and active + */ + aliasHandleLambdaRole(currentTemplate, aliasStackTemplates) { + + const stageStack = this._serverless.service.provider.compiledCloudFormationTemplate; + const stageRolePolicies = _.get(stageStack, 'Resources.IamRoleLambdaExecution.Properties.Policies', []); + const currentRolePolicies = _.get(currentTemplate, 'Resources.IamRoleLambdaExecution.Properties.Policies', []); + + // For now we only merge the first policy document and exit if SLS changes this behavior. + if (stageRolePolicies.length !== 1 || currentRolePolicies.length !== 1) { + return BbPromise.reject(new Error('Policy count should be 1! Please report this error to the alias plugin owner.')); + } + + const stageRolePolicyStatements = _.get(stageRolePolicies[0], 'PolicyDocument.Statement', []); + const currentRolePolicyStatements = _.get(currentRolePolicies[0], 'PolicyDocument.Statement', []); + + _.forEach(currentRolePolicyStatements, statement => { + // Check if there is already a statement with the same actions and effect. + const sameStageStatement = _.find(stageRolePolicyStatements, value => value.Effect === statement.Effect && + value.Action.length === statement.Action.length && + _.every(value.Action, action => _.includes(statement.Action, action))); + + if (sameStageStatement) { + // Merge the resources + sameStageStatement.Resource = _.unionWith(sameStageStatement.Resource, statement.Resource, (a,b) => _.isMatch(a,b)); + } else { + // Add the different statement + stageRolePolicyStatements.push(statement); + } + }); + + // Insert statement dependencies + const dependencies = _.reject((() => { + const result = []; + const stack = [ _.first(stageRolePolicyStatements) ]; + while (!_.isEmpty(stack)) { + const statement = stack.pop(); + + _.forOwn(statement, (value, key) => { + if (key === 'Ref') { + result.push(value); + } else if (key === 'Fn::GetAtt') { + result.push(value[0]); + } else if (_.isObject(value)) { + stack.push(value); + } + }); + } + return result; + })(), dependency => _.has(stageStack.Resources, dependency)); + + _.forEach(dependencies, dependency => { + stageStack.Resources[dependency] = currentTemplate.Resources[dependency]; + }); + + return BbPromise.resolve([ currentTemplate, aliasStackTemplates ]); + }, + aliasRestructureStack() { this._serverless.cli.log('Preparing aliases ...'); return BbPromise.bind(this) .then(this.aliasLoadCurrentCFStackAndDependencies) + .spread(this.aliasHandleUserResources) + .spread(this.aliasHandleLambdaRole) .spread(this.aliasHandleFunctions) .spread(this.aliasHandleApiGateway) .then(() => BbPromise.resolve()); diff --git a/lib/stackInformation.js b/lib/stackInformation.js index 4e8726e..0f3398c 100644 --- a/lib/stackInformation.js +++ b/lib/stackInformation.js @@ -37,7 +37,7 @@ module.exports = { }, /** - * Load all deployed alias stack templates. + * Load all deployed alias stack templates excluding the current alias. */ aliasStackLoadAliasTemplates() { @@ -51,6 +51,7 @@ module.exports = { this._options.stage, this._options.region) .then(cfData => BbPromise.resolve(cfData.Imports)) + .filter(stack => stack !== `${this._provider.naming.getStackName()}-${this._alias}`) .mapSeries(stack => { const importParams = { From 0452d05857da21c9066f05fbac0177f627fd74e6 Mon Sep 17 00:00:00 2001 From: Frank Schmid Date: Mon, 6 Mar 2017 14:11:28 +0100 Subject: [PATCH 2/2] Modified README resource section --- README.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e90a1a7..aaf9562 100644 --- a/README.md +++ b/README.md @@ -67,15 +67,27 @@ cleaner and the removal of an alias will also remove all logs associated to the The log group is named `/serverless/`. So you can clearly see what is deployed through Serverless and what by other means. -## Resources (not yet finished) +## Resources Resources are deployed per alias. So you can create new resources without destroying the main alias for the stage. If you remove an alias the referenced resources will be removed too. -*BEWARE: Currently the custom resources per alias must not be different. As soon -as the resource handling is implemented, the custom resources will behave exactly -like the SLS resources and can be different per alias!* +However, logical resource ids are unique per stage. If you deploy a resource into +one alias, it cannot be deployed with the same logical resource id and a different +configuration into a different alias. Nevertheless, you can have the same resource +defined within multiple aliases with the same configuration. + +This behavior exactly resembles the workflow of multiple developers working on +different VCS branches. + +The master alias for the stage has a slightly different behavior. If you deploy +here, you are allowed to change the configuration of the resource (e.g. the +capacities of a DynamoDB table). This deployment will reconfigure the resource +and on the next alias deployment of other developers, they will get an error +that they have to update their configuration too - most likely, they updated it +already, because normally you rebase or merge your upstream and get the changes +automatically. ## Serverless info integration