diff --git a/.gitignore b/.gitignore index 8bb22129f381..26fc540918a1 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,5 @@ doc/source/tutorial/services.rst # Pyenv .python-version +.env diff --git a/awscli/customizations/cloudformation/artifact_exporter.py b/awscli/customizations/cloudformation/artifact_exporter.py index 7fba8dddbba3..7fabb4dfed87 100644 --- a/awscli/customizations/cloudformation/artifact_exporter.py +++ b/awscli/customizations/cloudformation/artifact_exporter.py @@ -25,11 +25,15 @@ from awscli.customizations.cloudformation import exceptions from awscli.customizations.cloudformation.yamlhelper import yaml_dump, \ yaml_parse +from awscli.customizations.cloudformation import modules +from awscli.customizations.cloudformation import module_constants import jmespath LOG = logging.getLogger(__name__) +MODULES = "Modules" +RESOURCES = "Resources" def is_path_value_valid(path): return isinstance(path, str) @@ -139,6 +143,9 @@ def upload_local_artifacts(resource_id, resource_dict, property_name, local_path = make_abs_path(parent_dir, local_path) + if uploader is None: + raise exceptions.PackageBucketRequiredError() + # Or, pointing to a folder. Zip the folder and upload if is_local_folder(local_path): return zip_and_upload(local_path, uploader) @@ -154,6 +161,8 @@ def upload_local_artifacts(resource_id, resource_dict, property_name, def zip_and_upload(local_path, uploader): + if uploader is None: + raise exceptions.PackageBucketRequiredError() with zip_folder(local_path) as zipfile: return uploader.upload_with_dedup(zipfile) @@ -472,6 +481,9 @@ def do_export(self, resource_id, resource_dict, parent_dir): exported_template_str = yaml_dump(exported_template_dict) + if self.uploader is None: + raise exceptions.PackageBucketRequiredError() + with mktempfile() as temporary_file: temporary_file.write(exported_template_str) temporary_file.flush() @@ -558,6 +570,9 @@ def include_transform_export_handler(template_dict, uploader, parent_dir): return template_dict # We are confident at this point that `include_location` is a string containing the local path + if uploader is None: + raise exceptions.PackageBucketRequiredError() + abs_include_location = os.path.join(parent_dir, include_location) if is_local_file(abs_include_location): template_dict["Parameters"]["Location"] = uploader.upload_with_dedup(abs_include_location) @@ -591,8 +606,8 @@ def __init__(self, template_path, parent_dir, uploader, raise ValueError("parent_dir parameter must be " "an absolute path to a folder {0}" .format(parent_dir)) - abs_template_path = make_abs_path(parent_dir, template_path) + self.module_parent_path = abs_template_path template_dir = os.path.dirname(abs_template_path) with open(abs_template_path, "r") as handle: @@ -651,14 +666,44 @@ def export(self): :return: The template with references to artifacts that have been exported to s3. """ + + # Process constants + constants = module_constants.process_constants(self.template_dict) + if constants is not None: + module_constants.replace_constants(constants, self.template_dict) + + # Process modules + try: + self.template_dict = modules.process_module_section( + self.template_dict, + self.template_dir, + self.module_parent_path) + except Exception as e: + msg=f"Failed to process Modules section: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + self.template_dict = self.export_metadata(self.template_dict) - if "Resources" not in self.template_dict: + if RESOURCES not in self.template_dict: return self.template_dict + # Process modules that are specified as Resources, not in Modules + try: + self.template_dict = modules.process_resources_section( + self.template_dict, + self.template_dir, + self.module_parent_path, + None) + except Exception as e: + msg=f"Failed to process modules in Resources: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + + self.template_dict = self.export_global_artifacts(self.template_dict) - self.export_resources(self.template_dict["Resources"]) + self.export_resources(self.template_dict[RESOURCES]) return self.template_dict diff --git a/awscli/customizations/cloudformation/exceptions.py b/awscli/customizations/cloudformation/exceptions.py index b2625cdd27f9..68f478fb9bf5 100644 --- a/awscli/customizations/cloudformation/exceptions.py +++ b/awscli/customizations/cloudformation/exceptions.py @@ -53,7 +53,15 @@ class DeployBucketRequiredError(CloudFormationCommandError): "via an S3 Bucket. Please add the --s3-bucket parameter to your " "command. The local template will be copied to that S3 bucket and " "then deployed.") - + +class PackageBucketRequiredError(CloudFormationCommandError): + fmt = "Add the --s3-bucket parameter to your command to upload artifacts to S3" class InvalidForEachIntrinsicFunctionError(CloudFormationCommandError): fmt = 'The value of {resource_id} has an invalid "Fn::ForEach::" format: Must be a list of three entries' + +class InvalidModulePathError(CloudFormationCommandError): + fmt = 'The value of {source} is not a valid path to a local file' + +class InvalidModuleError(CloudFormationCommandError): + fmt = 'Invalid module: {msg}' diff --git a/awscli/customizations/cloudformation/module_conditions.py b/awscli/customizations/cloudformation/module_conditions.py new file mode 100644 index 000000000000..992424f10fce --- /dev/null +++ b/awscli/customizations/cloudformation/module_conditions.py @@ -0,0 +1,116 @@ +# Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +# pylint: disable=fixme + +""" +Parse the Conditions section in a module + +This section is not emitted into the output. +We have to be able to fully resolve it locally. +""" + +from awscli.customizations.cloudformation import exceptions + +AND = "Fn::And" +EQUALS = "Fn::Equals" +IF = "Fn::If" +NOT = "Fn::Not" +OR = "Fn::Or" +REF = "Ref" +CONDITION = "Condition" + + +def parse_conditions(d, find_ref): + """Parse conditions and return a map of name:boolean""" + retval = {} + + for k, v in d.items(): + retval[k] = istrue(v, find_ref, retval) + + return retval + + +def resolve_if(v, find_ref, prior): + "Resolve Fn::If" + msg = f"If expression should be a list with 3 elements: {v}" + if not isinstance(v, list): + raise exceptions.InvalidModuleError(msg=msg) + if len(v) != 3: + raise exceptions.InvalidModuleError(msg=msg) + if istrue(v[0], find_ref, prior): + return v[1] + return v[2] + + +# pylint: disable=too-many-branches,too-many-statements +def istrue(v, find_ref, prior): + "Recursive function to evaluate a Condition" + retval = False + if EQUALS in v: + eq = v[EQUALS] + if len(eq) == 2: + val0 = eq[0] + val1 = eq[1] + if IF in val0: + val0 = resolve_if(val0[IF], find_ref, prior) + if IF in val1: + val1 = resolve_if(val1[IF], find_ref, prior) + if REF in val0: + val0 = find_ref(val0[REF]) + if REF in val1: + val1 = find_ref(val1[REF]) + retval = val0 == val1 + else: + msg = f"Equals expression should be a list with 2 elements: {eq}" + raise exceptions.InvalidModuleError(msg=msg) + if NOT in v: + if not isinstance(v[NOT], list): + msg = f"Not expression should be a list with 1 element: {v[NOT]}" + raise exceptions.InvalidModuleError(msg=msg) + retval = not istrue(v[NOT][0], find_ref, prior) + if AND in v: + vand = v[AND] + msg = f"And expression should be a list with 2 elements: {vand}" + if not isinstance(vand, list): + raise exceptions.InvalidModuleError(msg=msg) + if len(vand) != 2: + raise exceptions.InvalidModuleError(msg=msg) + retval = istrue(vand[0], find_ref, prior) and istrue( + vand[1], find_ref, prior + ) + if OR in v: + vor = v[OR] + msg = f"Or expression should be a list with 2 elements: {vor}" + if not isinstance(vor, list): + raise exceptions.InvalidModuleError(msg=msg) + if len(vor) != 2: + raise exceptions.InvalidModuleError(msg=msg) + retval = istrue(vor[0], find_ref, prior) or istrue( + vor[1], find_ref, prior + ) + if IF in v: + # Shouldn't ever see an IF here + msg = f"Unexpected If: {v[IF]}" + raise exceptions.InvalidModuleError(msg=msg) + if CONDITION in v: + condition_name = v[CONDITION] + if condition_name in prior: + retval = prior[condition_name] + else: + msg = f"Condition {condition_name} was not evaluated yet" + raise exceptions.InvalidModuleError(msg=msg) + # TODO: Should we re-order the conditions? + # We are depending on the author putting them in order + + return retval diff --git a/awscli/customizations/cloudformation/module_constants.py b/awscli/customizations/cloudformation/module_constants.py new file mode 100644 index 000000000000..db28525118af --- /dev/null +++ b/awscli/customizations/cloudformation/module_constants.py @@ -0,0 +1,133 @@ +# Copyright 2012-2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +""" +Module constants. + +Add a Constants section to the module or the parent template for +string constants, to help reduce copy-paste within the template. + +Refer to constants in Sub strings later in the template using ${Constant::name} + +Constants can refer to other constants that were defined previously. + +Adding constants to a template has side effects, since we have to +re-write all Subs in the template! Ideally nothing will change but +it's possible. + +Example: + + Constants: + foo: bar + baz: abc-${AWS::AccountId}-${Constant::foo} + + Resources: + Bucket: + Type: AWS::S3::Bucket + Metadata: + Test: !Sub ${Constant:baz} + Properties: + BucketName: !Sub ${Constant::foo} + +""" + +from collections import OrderedDict +from awscli.customizations.cloudformation.parse_sub import WordType +from awscli.customizations.cloudformation.parse_sub import parse_sub +from awscli.customizations.cloudformation.module_visitor import Visitor +from awscli.customizations.cloudformation import exceptions + +CONSTANTS = "Constants" +SUB = "Fn::Sub" + + +def process_constants(d): + """ + Look for a Constants item in d and if it's found, return it + as a dict. Looks for references to previously defined constants + and substitutes them. + Deletes the Constants item from d. + Returns a dict of the constants. + """ + if CONSTANTS not in d: + return None + + constants = {} + for k, v in d[CONSTANTS].items(): + s = replace_constants(constants, v) + constants[k] = s + + del d[CONSTANTS] + + return constants + + +def replace_constants(constants, s): + """ + Replace all constants in a string or in an entire dictionary. + If s is a string, returns the modified string. + If s is a dictionary, modifies the dictionary in place. + """ + if isinstance(s, str): + retval = "" + words = parse_sub(s) + for w in words: + if w.t == WordType.STR: + retval += w.w + if w.t == WordType.REF: + retval += f"${{{w.w}}}" + if w.t == WordType.AWS: + retval += f"${{AWS::{w.w}}}" + if w.t == WordType.GETATT: + retval += f"${{{w.w}}}" + if w.t == WordType.CONSTANT: + if w.w in constants: + retval += constants[w.w] + else: + msg = f"Unknown constant: {w.w}" + raise exceptions.InvalidModuleError(msg=msg) + return retval + + if isdict(s): + + # Recursively dive into d and replace all string constants + # that are found in Subs. Error if the constant does not exist. + + def vf(v): + if isdict(v.d) and SUB in v.d and v.p is not None: + s = v.d[SUB] + if isinstance(s, str): + newval = replace_constants(constants, s) + if is_sub_needed(newval): + v.p[v.k] = {SUB: newval} + else: + v.p[v.k] = newval + + v = Visitor(s) + v.visit(vf) + + return None + + +def isdict(d): + "Returns true if d is a dict" + return isinstance(d, (dict, OrderedDict)) + + +def is_sub_needed(s): + "Returns true if the string has any Sub variables" + words = parse_sub(s) + for w in words: + if w.t != WordType.STR: + return True + return False diff --git a/awscli/customizations/cloudformation/module_visitor.py b/awscli/customizations/cloudformation/module_visitor.py new file mode 100644 index 000000000000..f12fdebc3c71 --- /dev/null +++ b/awscli/customizations/cloudformation/module_visitor.py @@ -0,0 +1,67 @@ +# Copyright 2012-2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +""" +Visitor pattern for recursively modifying all elements in a dictionary. +""" + +from collections import OrderedDict + + +class Visitor: + """ + This class implements a visitor pattern, to run a function on + all elements of a template or subset. + + Example of a visitor that replaces all strings: + + v = Visitor(template_dict) + def vf(v): + if isinstance(v.d, string) and v.p is not None: + v.p[v.k] = "replacement" + v.vist(vf) + """ + + def __init__(self, d, p=None, k=""): + """ + Initialize the visitor with a dictionary. This can be the entire + template or a subset. + + :param d A dict or OrderedDict + :param p The parent dictionary (default is None for the template) + :param k the key for d (p[k] = d) (default is "" for the template) + """ + self.d = d + self.p = p + self.k = k + + def __str__(self): + return f"d: {self.d}, p: {self.p}, k: {self.k}" + + def visit(self, visit_func): + """ + Run the specified function on all nodes. + + :param visit_func A function that accepts a Visitor + """ + + def walk(visitor): + visit_func(visitor) + if isinstance(visitor.d, (dict, OrderedDict)): + for k, v in visitor.d.items(): + walk(Visitor(v, visitor.d, k)) + if isinstance(visitor.d, list): + for i, v in enumerate(visitor.d): + walk(Visitor(v, visitor.d, i)) + + walk(self) diff --git a/awscli/customizations/cloudformation/modules.py b/awscli/customizations/cloudformation/modules.py new file mode 100644 index 000000000000..0b03097988d6 --- /dev/null +++ b/awscli/customizations/cloudformation/modules.py @@ -0,0 +1,855 @@ +# Copyright 2012-2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +""" +This file implements local module support for the package command + +See tests/unit/customizations/cloudformation/modules for examples of what the +Modules section of a template looks like. + +Modules can be referenced in a new Modules section in the template, or they can +be referenced as Resources with the Type LocalModule. Modules have a Source +attribute pointing to a local file, a Properties attribute that corresponds to +Parameters in the modules, and an Overrides attribute that can override module +output. + +The `Modules` section. + +```yaml +Modules: + Content: + Source: ./module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +``` + +A module configured as a `Resource`. + +```yaml +Resources: + Content: + Type: LocalModule + Source: ./module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +``` + +A module is itself basically a CloudFormation template, with a Parameters +section and Resources that are injected into the parent template. The +Properties defined in the Modules section correspond to the Parameters in the +module. These modules operate in a similar way to registry modules. + +The name of the module in the Modules section is used as a prefix to logical +ids that are defined in the module. Or if the module is referenced in the Type +attribute of a Resource, the logical id of the resource is used as the prefix. + +In addition to the parent setting Properties, all attributes of the module can +be overridden with Overrides, which require the consumer to know how the module +is structured. This "escape hatch" is considered a first class citizen in the +design, to avoid excessive Parameter definitions to cover every possible use +case. One caveat is that using Overrides is less stable, since the module +author might change logical ids. Using module Outputs can mitigate this. + +Module Parameters (set by Properties in the parent) are handled with Refs, +Subs, and GetAtts in the module. These are handled in a way that fixes +references to match module prefixes, fully resolving values that are actually +strings and leaving others to be resolved at deploy time. + +Modules can contain other modules, with no enforced limit to the levels of +nesting. + +Modules can define Outputs, which are key-value pairs that can be referenced by +the parent. + +When using modules, you can use a comma-delimited list to create a number of +similar resources. This is simpler than using `Fn::ForEach` but has the +limitation of requiring the list to be resolved at build time. See +tests/unit/customizations/cloudformation/modules/vpc-module.yaml. + +An example of a Map is defining subnets in a VPC. + +```yaml +Parameters: + CidrBlock: + Type: String + PrivateCidrBlocks: + Type: CommaDelimitedList + PublicCidrBlocks: + Type: CommaDelimitedList +Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: !Ref CidrBlock + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default + PublicSubnet: + Type: LocalModule + Map: !Ref PublicCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + PrivateSubnet: + Type: LocalModule + Map: !Ref PrivateCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex +``` + +""" + +# pylint: disable=fixme,too-many-instance-attributes + +import copy +import logging +import os +import urllib +from collections import OrderedDict + +from awscli.customizations.cloudformation import exceptions +from awscli.customizations.cloudformation import yamlhelper +from awscli.customizations.cloudformation.module_constants import ( + process_constants, + replace_constants, +) + +from awscli.customizations.cloudformation.parse_sub import WordType +from awscli.customizations.cloudformation.parse_sub import parse_sub +from awscli.customizations.cloudformation.module_conditions import ( + parse_conditions, +) + +LOG = logging.getLogger(__name__) + +RESOURCES = "Resources" +METADATA = "Metadata" +OVERRIDES = "Overrides" +DEPENDSON = "DependsOn" +PROPERTIES = "Properties" +CREATIONPOLICY = "CreationPolicy" +UPDATEPOLICY = "UpdatePolicy" +DELETIONPOLICY = "DeletionPolicy" +UPDATEREPLACEPOLICY = "UpdateReplacePolicy" +CONDITION = "Condition" +DEFAULT = "Default" +NAME = "Name" +SOURCE = "Source" +REF = "Ref" +SUB = "Fn::Sub" +GETATT = "Fn::GetAtt" +PARAMETERS = "Parameters" +MODULES = "Modules" +TYPE = "Type" +LOCAL_MODULE = "LocalModule" +OUTPUTS = "Outputs" +MAP = "Map" +MAP_PLACEHOLDER = "$MapValue" +INDEX_PLACEHOLDER = "$MapIndex" +CONDITIONS = "Conditions" +CONDITION = "Condition" + + +def process_module_section(template, base_path, parent_path): + "Recursively process the Modules section of a template" + if MODULES in template: + if not isdict(template[MODULES]): + msg = "Modules section is invalid" + raise exceptions.InvalidModuleError(msg=msg) + + # Process each Module node separately + for k, v in template[MODULES].items(): + module = make_module(template, k, v, base_path, parent_path) + template = module.process() + + # Remove the Modules section from the template + del template[MODULES] + + return template + + +def make_module(template, name, config, base_path, parent_path): + "Create an instance of a module based on a template and the module config" + module_config = {} + module_config[NAME] = name + if SOURCE not in config: + msg = f"{name} missing {SOURCE}" + raise exceptions.InvalidModulePathError(msg=msg) + + source_path = config[SOURCE] + + if not is_url(source_path): + relative_path = source_path + module_config[SOURCE] = os.path.join(base_path, relative_path) + module_config[SOURCE] = os.path.normpath(module_config[SOURCE]) + else: + module_config[SOURCE] = source_path + + if module_config[SOURCE] == parent_path: + msg = f"Module refers to itself: {parent_path}" + raise exceptions.InvalidModuleError(msg=msg) + if PROPERTIES in config: + module_config[PROPERTIES] = config[PROPERTIES] + if OVERRIDES in config: + module_config[OVERRIDES] = config[OVERRIDES] + return Module(template, module_config) + + +def map_placeholders(i, token, val): + "Replace $MapValue and $MapIndex" + if SUB in val: + sub = val[SUB] + r = sub.replace(MAP_PLACEHOLDER, token) + r = r.replace(INDEX_PLACEHOLDER, f"{i}") + words = parse_sub(r) + need_sub = False + for word in words: + if word.t != WordType.STR: + need_sub = True + break + if need_sub: + return {SUB: r} + return r + r = val.replace(MAP_PLACEHOLDER, token) + r = r.replace(INDEX_PLACEHOLDER, f"{i}") + return r + + +# pylint: disable=too-many-locals,too-many-nested-blocks +def process_resources_section(template, base_path, parent_path, parent_module): + "Recursively process the Resources section of the template" + if parent_module is None: + # Make a fake Module instance to handle find_ref for Maps + # The only valid way to do this at the template level + # is to specify a default for a Parameter, since we need to + # resolve the actual value client-side + parent_module = Module(template, {NAME: "", SOURCE: ""}) + if PARAMETERS in template: + parent_module.params = template[PARAMETERS] + + for k, v in template[RESOURCES].copy().items(): + if TYPE in v and v[TYPE] == LOCAL_MODULE: + # First, pre-process local modules that are looping over a list + if MAP in v: + # Expect Map to be a CSV or ref to a CSV + m = v[MAP] + if isdict(m) and REF in m: + if parent_module is None: + msg = "Map is only valid in a module" + raise exceptions.InvalidModuleError(msg=msg) + m = parent_module.find_ref(m[REF]) + if m is None: + msg = f"{k} has an invalid Map Ref" + raise exceptions.InvalidModuleError(msg=msg) + tokens = m.split(",") # TODO - use an actual csv parser? + for i, token in enumerate(tokens): + # Make a new resource + logical_id = f"{k}{i}" + resource = copy.deepcopy(v) + del resource[MAP] + # Replace $Map and $Index placeholders + for prop, val in resource[PROPERTIES].copy().items(): + resource[PROPERTIES][prop] = map_placeholders( + i, token, val + ) + template[RESOURCES][logical_id] = resource + + del template[RESOURCES][k] + + # Start over after pre-processing maps + for k, v in template[RESOURCES].copy().items(): + if TYPE in v and v[TYPE] == LOCAL_MODULE: + module = make_module(template, k, v, base_path, parent_path) + template = module.process() + del template[RESOURCES][k] + return template + + +def isdict(v): + "Returns True if the type is a dict or OrderedDict" + return isinstance(v, (dict, OrderedDict)) + + +def is_url(p): + "Returns true if the path looks like a URL instead of a local file" + return p.startswith("https") + + +def read_source(source): + "Read the source file and return the content as a string" + + if not isinstance(source, str): + raise exceptions.InvalidModulePathError(source=source) + + if is_url(source): + try: + with urllib.request.urlopen(source) as response: + return response.read() + except Exception as e: + print(e) + raise exceptions.InvalidModulePathError(source=source) + + if not os.path.isfile(source): + raise exceptions.InvalidModulePathError(source=source) + + with open(source, "r", encoding="utf-8") as s: + return s.read() + + +def merge_props(original, overrides): + """ + This function merges dicts, replacing values in the original with + overrides. This function is recursive and can act on lists and scalars. + See the unit tests for example merges. + See tests/unit/customizations/cloudformation/modules/policy-*.yaml + + :return A new value with the overridden properties + """ + original_type = type(original) + override_type = type(overrides) + if not isdict(overrides) and override_type is not list: + return overrides + + if original_type is not override_type: + return overrides + + if isdict(original): + retval = original.copy() + for k in original: + if k in overrides: + retval[k] = merge_props(retval[k], overrides[k]) + for k in overrides: + if k not in original: + retval[k] = overrides[k] + return retval + + # original and overrides are lists + new_list = [] + for item in original: + new_list.append(item) + for item in overrides: + new_list.append(item) + return new_list + + +class Module: + """ + Process client-side modules. + + """ + + def __init__(self, template, module_config): + """ + Initialize the module with values from the parent template + + :param template The parent template dictionary + :param module_config The configuration from the parent Modules section + """ + + # The parent template dictionary + self.template = template + if RESOURCES not in self.template: + # The parent might only have Modules + self.template[RESOURCES] = {} + + # The name of the module, which is used as a logical id prefix + self.name = module_config[NAME] + + # The location of the source for the module, a URI string + self.source = module_config[SOURCE] + + # The Properties from the parent template + self.props = {} + if PROPERTIES in module_config: + self.props = module_config[PROPERTIES] + + # The Overrides from the parent template + self.overrides = {} + if OVERRIDES in module_config: + self.overrides = module_config[OVERRIDES] + + # Resources defined in the module + self.resources = {} + + # Parameters defined in the module + self.params = {} + + # Outputs defined in the module + self.outputs = {} + + # Conditions defined in the module + self.conditions = {} + + def __str__(self): + "Print out a string with module details for logs" + return ( + f"module name: {self.name}, " + + f"source: {self.source}, props: {self.props}" + ) + + def process(self): + """ + Read the module source process it. + + :return: The modified parent template dictionary + """ + + content = read_source(self.source) + + module_dict = yamlhelper.yaml_parse(content) + + # Process constants + constants = process_constants(module_dict) + if constants is not None: + replace_constants(constants, module_dict) + + if RESOURCES not in module_dict: + # The module may only have sub modules in the Modules section + self.resources = {} + else: + self.resources = module_dict[RESOURCES] + + if PARAMETERS in module_dict: + self.params = module_dict[PARAMETERS] + + if OUTPUTS in module_dict: + self.outputs = module_dict[OUTPUTS] + + # Recurse on nested modules + bp = os.path.dirname(self.source) + section = "" + try: + section = MODULES + process_module_section(module_dict, bp, self.source) + section = RESOURCES + process_resources_section(module_dict, bp, self.source, self) + except Exception as e: + msg = f"Failed to process {self.source} {section} section: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + + self.validate_overrides() + + if CONDITIONS in module_dict: + cs = module_dict[CONDITIONS] + + def find_ref(v): + return self.find_ref(v) + + try: + self.conditions = parse_conditions(cs, find_ref) + except Exception as e: + msg = f"Failed to process conditions in {self.source}: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + + for logical_id, resource in self.resources.items(): + self.process_resource(logical_id, resource) + + self.process_outputs() + + return self.template + + def process_outputs(self): + """ + Fix parent template output references. + + In the parent you can !GetAtt ModuleName.OutputName + This will be converted so that it's correct in the packaged template. + + Recurse over all sections in the parent template looking for + GetAtts and Subs that reference a module output value. + """ + sections = [RESOURCES, OUTPUTS] # TODO: Any others? + for section in sections: + if section not in self.template: + continue + for k, v in self.template[section].items(): + self.resolve_outputs(k, v, self.template, section) + + def resolve_outputs(self, k, v, d, n): + """ + Recursively resolve GetAtts and Subs that reference module outputs. + + :param name The name of the output + :param output The output dict + :param k The name of the node + :param v The value of the node + :param d The dict that holds the parent of k + :param n The name of the node that holds k + + If a reference is found, this function sets the value of d[n] + """ + if k == SUB: + self.resolve_output_sub(v, d, n) + elif k == GETATT: + self.resolve_output_getatt(v, d, n) + else: + if isdict(v): + for k2, v2 in v.copy().items(): + self.resolve_outputs(k2, v2, d[n], k) + elif isinstance(v, list): + idx = -1 + for v2 in v: + idx = idx + 1 + if isdict(v2): + for k3, v3 in v2.copy().items(): + self.resolve_outputs(k3, v3, v, idx) + + def resolve_output_sub(self, v, d, n): + "Resolve a Sub that refers to a module output" + words = parse_sub(v, True) + sub = "" + for word in words: + if word.t == WordType.STR: + sub += word.w + elif word.t == WordType.AWS: + sub += "${AWS::" + word.w + "}" + elif word.t == WordType.REF: + # A reference to an output has to be a getatt + resolved = "${" + word.w + "}" + sub += resolved + elif word.t == WordType.GETATT: + resolved = "${" + word.w + "}" + tokens = word.w.split(".", 1) + if len(tokens) < 2: + msg = f"GetAtt {word.w} has unexpected number of tokens" + raise exceptions.InvalidModuleError(msg=msg) + # !Sub ${Content.BucketArn} -> !Sub ${ContentBucket.Arn} + if tokens[0] == self.name and tokens[1] in self.outputs: + output = self.outputs[tokens[1]] + if GETATT in output: + getatt = output[GETATT] + resolved = "${" + self.name + ".".join(getatt) + "}" + elif SUB in output: + resolved = "${" + self.name + output[SUB] + "}" + sub += resolved + + d[n] = {SUB: sub} + + # pylint:disable=too-many-branches + def resolve_output_getatt(self, v, d, n): + "Resolve a GetAtt that refers to a module output" + if not isinstance(v, list) or len(v) < 2: + msg = f"GetAtt {v} invalid" + raise exceptions.InvalidModuleError(msg=msg) + if v[0] == self.name and v[1] in self.outputs: + output = self.outputs[v[1]] + if GETATT in output: + getatt = output[GETATT] + if len(getatt) < 2: + msg = f"GetAtt {getatt} in Output {v[1]} is invalid" + raise exceptions.InvalidModuleError(msg=msg) + d[n] = {GETATT: [self.name + getatt[0], getatt[1]]} + elif SUB in output: + # Parse the Sub in the module output + words = parse_sub(output[SUB], True) + sub = "" + for word in words: + if word.t == WordType.STR: + sub += word.w + elif word.t == WordType.AWS: + sub += "${AWS::" + word.w + "}" + elif word.t == WordType.REF: + # This is a ref to a param or resource + # TODO: If it's a ref to a param...? is this allowed? + # If it's a resource, concatenante the name + resolved = "${" + word.w + "}" + if word.w in self.resources: + resolved = "${" + self.name + word.w + "}" + sub += resolved + elif word.t == WordType.GETATT: + resolved = "${" + word.w + "}" + tokens = word.w.split(".", 1) + if len(tokens) < 2: + msg = f"GetAtt {word.w} unexpected length" + raise exceptions.InvalidModuleError(msg=msg) + if tokens[0] in self.resources: + resolved = "${" + self.name + word.w + "}" + sub += resolved + + d[n] = {SUB: sub} + + def validate_overrides(self): + "Make sure resources referenced by overrides actually exist" + for logical_id in self.overrides: + if logical_id not in self.resources: + msg = f"Override {logical_id} not found in {self.source}" + raise exceptions.InvalidModuleError(msg=msg) + + def process_resource(self, logical_id, resource): + "Process a single resource" + + # First, check to see if a Conditions omits this resource + if CONDITION in resource: + if resource[CONDITION] in self.conditions: + if self.conditions[resource[CONDITION]] is False: + return + del resource[CONDITION] + # else leave it and assume it's in the parent? + + # For each property (and property-like attribute), + # replace the value if it appears in parent overrides. + attrs = [ + PROPERTIES, + CREATIONPOLICY, + METADATA, + UPDATEPOLICY, + DELETIONPOLICY, + CONDITION, + UPDATEREPLACEPOLICY, + DEPENDSON, + ] + for a in attrs: + self.process_overrides(logical_id, resource, a) + + # Resolve refs, subs, and getatts + # (Process module Parameters and parent Properties) + container = {} + # We need the container for the first iteration of the recursion + container[RESOURCES] = self.resources + self.resolve(logical_id, resource, container, RESOURCES) + + self.template[RESOURCES][self.name + logical_id] = resource + + def process_overrides(self, logical_id, resource, attr_name): + """ + Replace overridden values in a property-like attribute of a resource. + + (Properties, Metadata, CreationPolicy, and UpdatePolicy) + + Overrides are a way to customize modules without needing a Parameter. + + Example template.yaml: + + Modules: + Foo: + Source: ./module.yaml + Overrides: + Bar: + Properties: + Name: bbb + + Example module.yaml: + + Resources: + Bar: + Type: A::B::C + Properties: + Name: aaa + + Output yaml: + + Resources: + Bar: + Type: A::B::C + Properties: + Name: bbb + """ + + if logical_id not in self.overrides: + return + + resource_overrides = self.overrides[logical_id] + if resource_overrides is None: + return + if attr_name not in resource_overrides: + return + + # Might be overriding something that's not in the module at all, + # like a Bucket with no Properties + if attr_name not in resource: + if attr_name in resource_overrides: + resource[attr_name] = resource_overrides[attr_name] + else: + return + + original = resource[attr_name] + overrides = resource_overrides[attr_name] + resource[attr_name] = merge_props(original, overrides) + + def resolve(self, k, v, d, n): + """ + Resolve Refs, Subs, and GetAtts recursively. + + :param k The name of the node + :param v The value of the node + :param d The dict that is the parent of the dict that holds k, v + :param n The name of the dict that holds k, v + + Example + + Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name + + In the above example, + k = !Ref, v = Name, d = Properties{}, n = BucketName + + So we can set d[n] = resolved_value (which replaces {k,v}) + + In the prior iteration, + k = BucketName, v = {!Ref, Name}, d = Bucket{}, n = Properties + """ + + # print(f"k: {k}, v: {v}, d: {d}, n: {n}") + + if k == REF: + self.resolve_ref(v, d, n) + elif k == SUB: + self.resolve_sub(v, d, n) + elif k == GETATT: + self.resolve_getatt(v, d, n) + else: + if isdict(v): + vc = v.copy() + for k2, v2 in vc.items(): + self.resolve(k2, v2, d[n], k) + elif isinstance(v, list): + idx = -1 + for v2 in v: + idx = idx + 1 + if isdict(v2): + v2c = v2.copy() + for k3, v3 in v2c.items(): + self.resolve(k3, v3, v, idx) + + def resolve_ref(self, v, d, n): + """ + Look for the Ref in the parent template Properties if it matches + a module Parameter name. If it's not there, use the default if + there is one. If not, raise an error. + + If there is no matching Parameter, look for a resource with that + name in this module and fix the logical id so it has the prefix. + + Otherwise just leave it be and assume the module author is + expecting the parent template to have that Reference. + """ + if not isinstance(v, str): + msg = f"Ref should be a string: {v}" + raise exceptions.InvalidModuleError(msg=msg) + + found = self.find_ref(v) + if found is not None: + d[n] = found + + # pylint: disable=too-many-branches,unused-argument + def resolve_sub(self, v, d, n): + """ + Parse the Sub string and break it into tokens. + + If we can fully resolve it, we can replace it with a string. + + Use the same logic as with resolve_ref. + """ + # print(f"resolve_sub v: {v}, d: {d}, n: {n}") + words = parse_sub(v, True) + sub = "" + need_sub = False + for word in words: + if word.t == WordType.STR: + sub += word.w + elif word.t == WordType.AWS: + sub += "${AWS::" + word.w + "}" + need_sub = True + elif word.t == WordType.REF: + resolved = "${" + word.w + "}" + need_sub = True + found = self.find_ref(word.w) + if found is not None: + if isinstance(found, str): + # need_sub = False + resolved = found + else: + if REF in found: + resolved = "${" + found[REF] + "}" + elif SUB in found: + resolved = found[SUB] + sub += resolved + elif word.t == WordType.GETATT: + need_sub = True + resolved = "${" + word.w + "}" + tokens = word.w.split(".", 1) + if len(tokens) < 2: + msg = f"GetAtt {word.w} has unexpected number of tokens" + raise exceptions.InvalidModuleError(msg=msg) + if tokens[0] in self.resources: + tokens[0] = self.name + tokens[0] + resolved = "${" + tokens[0] + "." + tokens[1] + "}" + sub += resolved + + if need_sub: + d[n] = {SUB: sub} + else: + d[n] = sub + + def resolve_getatt(self, v, d, n): + """ + Resolve a GetAtt. All we do here is add the prefix. + + !GetAtt Foo.Bar becomes !GetAtt ModuleNameFoo.Bar + """ + if not isinstance(v, list): + msg = f"GetAtt {v} is not a list" + raise exceptions.InvalidModuleError(msg=msg) + logical_id = self.name + v[0] + d[n] = {GETATT: [logical_id, v[1]]} + + def find_ref(self, name): + """ + Find a Ref. + + A Ref might be to a module Parameter with a matching parent + template Property, or a Parameter Default. It could also + be a reference to another resource in this module. + + :param name The name to search for + :return The referenced element or None + """ + # print(f"find_ref {name}, props: {self.props}, params: {self.params}") + if name in self.props: + if name not in self.params: + # The parent tried to set a property that doesn't exist + # in the Parameters section of this module + msg = f"{name} not found in module Parameters: {self.source}" + raise exceptions.InvalidModuleError(msg=msg) + return self.props[name] + + if name in self.params: + param = self.params[name] + if DEFAULT in param: + # Use the default value of the Parameter + return param[DEFAULT] + msg = f"{name} does not have a Default and is not a Property" + raise exceptions.InvalidModuleError(msg=msg) + + for logical_id in self.resources: + if name == logical_id: + # Simply rename local references to include the module name + return {REF: self.name + logical_id} + + return None diff --git a/awscli/customizations/cloudformation/package.py b/awscli/customizations/cloudformation/package.py index 9bc7464d442e..2a4f861f074e 100644 --- a/awscli/customizations/cloudformation/package.py +++ b/awscli/customizations/cloudformation/package.py @@ -57,7 +57,6 @@ class PackageCommand(BasicCommand): { 'name': 's3-bucket', - 'required': True, 'help_text': ( 'The name of the S3 bucket where this command uploads' ' the artifacts that are referenced in your template.' @@ -124,11 +123,7 @@ class PackageCommand(BasicCommand): ] def _run_main(self, parsed_args, parsed_globals): - s3_client = self._session.create_client( - "s3", - config=Config(signature_version='s3v4'), - region_name=parsed_globals.region, - verify=parsed_globals.verify_ssl) + template_path = parsed_args.template_file if not os.path.isfile(template_path): @@ -137,13 +132,25 @@ def _run_main(self, parsed_args, parsed_globals): bucket = parsed_args.s3_bucket - self.s3_uploader = S3Uploader(s3_client, - bucket, - parsed_args.s3_prefix, - parsed_args.kms_key_id, - parsed_args.force_upload) - # attach the given metadata to the artifacts to be uploaded - self.s3_uploader.artifact_metadata = parsed_args.metadata + # Only create the s3 uploaded if we need it, + # since this command now also supports local modules. + # Local modules should be able to run without credentials. + if bucket: + s3_client = self._session.create_client( + "s3", + config=Config(signature_version='s3v4'), + region_name=parsed_globals.region, + verify=parsed_globals.verify_ssl) + + self.s3_uploader = S3Uploader(s3_client, + bucket, + parsed_args.s3_prefix, + parsed_args.kms_key_id, + parsed_args.force_upload) + # attach the given metadata to the artifacts to be uploaded + self.s3_uploader.artifact_metadata = parsed_args.metadata + else: + self.s3_uploader = None output_file = parsed_args.output_template_file use_json = parsed_args.use_json diff --git a/awscli/customizations/cloudformation/parse_sub.py b/awscli/customizations/cloudformation/parse_sub.py new file mode 100644 index 000000000000..18153c9f63f5 --- /dev/null +++ b/awscli/customizations/cloudformation/parse_sub.py @@ -0,0 +1,136 @@ +""" +This module parses CloudFormation Sub strings. + +For example: + + !Sub abc-${AWS::Region}-def-${Foo} + +The string is broken down into "words" that are one of four types: + + String: A literal string component + Ref: A reference to another resource or paramter like ${Foo} + AWS: An AWS pseudo-parameter like ${AWS::Region} + GetAtt: A reference to an attribute like ${Foo.Bar} +""" +#pylint: disable=too-few-public-methods + +from enum import Enum + +DATA = ' ' # Any other character +DOLLAR = '$' +OPEN = '{' +CLOSE = '}' +BANG = '!' +SPACE = ' ' + +class WordType(Enum): + "Word type enumeration" + STR = 0 # A literal string fragment + REF = 1 # ${ParamOrResourceName} + AWS = 2 # ${AWS::X} + GETATT = 3 # ${X.Y} + CONSTANT = 4 # ${Constant::name} + +class State(Enum): + "State machine enumeration" + READSTR = 0 + READVAR = 1 + MAYBE = 2 + READLIT = 3 + +class SubWord: + "A single word with a type and the word itself" + def __init__(self, word_type, word): + self.t = word_type + self.w = word # Does not include the ${} if it's not a STR + + def __str__(self): + return f"{self.t} {self.w}" + +#pylint: disable=too-many-branches,too-many-statements +def parse_sub(sub_str, leave_bang=False): + """ + Parse a Sub string + + :param leave_bang If this is True, leave the ! in literals + :return list of words + """ + words = [] + state = State.READSTR + buf = '' + last = '' + for i, char in enumerate(sub_str): + if char == DOLLAR: + if state != State.READVAR: + state = State.MAYBE + else: + # This is a literal $ inside a variable: "${AB$C}" + buf += char + elif char == OPEN: + if state == State.MAYBE: + # Peek to see if we're about to start a LITERAL ! + if len(sub_str) > i+1 and sub_str[i+1] == BANG: + # Treat this as part of the string, not a var + buf += "${" + state = State.READLIT + else: + state = State.READVAR + # We're about to start reading a variable. + # Append the last word in the buffer if it's not empty + if buf: + words.append(SubWord(WordType.STR, buf)) + buf = '' + else: + buf += char + elif char == CLOSE: + if state == State.READVAR: + # Figure out what type it is + if buf.startswith("AWS::"): + word_type = WordType.AWS + elif buf.startswith("Constant::") or buf.startswith("Constants::"): + word_type = WordType.CONSTANT + elif '.' in buf: + word_type = WordType.GETATT + else: + word_type = WordType.REF + buf = buf.replace("AWS::", "", 1) + buf = buf.replace("Constant::", "", 1) + # Very common typo to put Constants instead of Constant + buf = buf.replace("Constants::", "", 1) + words.append(SubWord(word_type, buf)) + buf = '' + state = State.READSTR + else: + buf += char + elif char == BANG: + # ${!LITERAL} becomes ${LITERAL} + if state == State.READLIT: + # Don't write the ! to the string + state = State.READSTR + if leave_bang: + # Unless we actually want it + buf += char + else: + # This is a ! somewhere not related to a LITERAL + buf += char + elif char == SPACE: + # Ignore spaces around Refs. ${ ABC } == ${ABC} + if state != State.READVAR: + buf += char + else: + if state == State.MAYBE: + buf += last # Put the $ back on the buffer + state = State.READSTR + buf += char + + last = char + + if buf: + words.append(SubWord(WordType.STR, buf)) + + # Handle malformed strings, like "ABC${XYZ" + if state != State.READSTR: + # Ended the string in the middle of a variable? + raise ValueError("invalid string, unclosed variable") + + return words diff --git a/awscli/examples/cloudformation/_package_description.rst b/awscli/examples/cloudformation/_package_description.rst index f47ec2212916..f75a788f2a53 100644 --- a/awscli/examples/cloudformation/_package_description.rst +++ b/awscli/examples/cloudformation/_package_description.rst @@ -1,10 +1,13 @@ -Packages the local artifacts (local paths) that your AWS CloudFormation template -references. The command uploads local artifacts, such as source code for an AWS -Lambda function or a Swagger file for an AWS API Gateway REST API, to an S3 -bucket. The command returns a copy of your template, replacing references to -local artifacts with the S3 location where the command uploaded the artifacts. +Packages the local artifacts (local paths) that your AWS CloudFormation +template references. The command uploads local artifacts, such as source code +for an AWS Lambda function or a Swagger file for an AWS API Gateway REST API, +to an S3 bucket. The command can also process local modules, which are +parameterized CloudFormation snippets that are merged into the parent template. +The command returns a copy of your template, replacing references to local +artifacts with the S3 location where the command uploaded the artifacts, or +replacing local and remote module references with the resources in the module. -Use this command to quickly upload local artifacts that might be required by +Use this command to quickly process local artifacts that might be required by your template. After you package your template's artifacts, run the ``deploy`` command to deploy the returned template. @@ -32,25 +35,42 @@ This command can upload local artifacts referenced in the following places: - ``S3`` property for the ``AWS::CodeCommit::Repository`` resource -To specify a local artifact in your template, specify a path to a local file or folder, -as either an absolute or relative path. The relative path is a location -that is relative to your template's location. +To specify a local artifact in your template, specify a path to a local file or +folder, as either an absolute or relative path. The relative path is a location +that is relative to your template's location. If the artifact is a module, the +path can be a local file or a remote URL starting with 'https'. For example, if your AWS Lambda function source code is in the -``/home/user/code/lambdafunction/`` folder, specify -``CodeUri: /home/user/code/lambdafunction`` for the -``AWS::Serverless::Function`` resource. The command returns a template and replaces -the local path with the S3 location: ``CodeUri: s3://mybucket/lambdafunction.zip``. +``/home/user/code/lambdafunction/`` folder, specify ``CodeUri: +/home/user/code/lambdafunction`` for the ``AWS::Serverless::Function`` +resource. The command returns a template and replaces the local path with the +S3 location: ``CodeUri: s3://mybucket/lambdafunction.zip``. If you specify a file, the command directly uploads it to the S3 bucket. If you specify a folder, the command zips the folder and then uploads the .zip file. -For most resources, if you don't specify a path, the command zips and uploads the -current working directory. The exception is ``AWS::ApiGateway::RestApi``; -if you don't specify a ``BodyS3Location``, this command will not upload an artifact to S3. +For most resources, if you don't specify a path, the command zips and uploads +the current working directory. The exception is ``AWS::ApiGateway::RestApi``; +if you don't specify a ``BodyS3Location``, this command will not upload an +artifact to S3. Before the command uploads artifacts, it checks if the artifacts are already present in the S3 bucket to prevent unnecessary uploads. The command uses MD5 checksums to compare files. If the values match, the command doesn't upload the -artifacts. Use the ``--force-upload flag`` to skip this check and always upload the -artifacts. +artifacts. Use the ``--force-upload flag`` to skip this check and always upload +the artifacts. + +Modules can be referenced either in the top level ``Modules`` section of the +template, or from a Resource with ``Type: LocalModule``. Module references have +a ``Source`` attribute pointing to the module, either a local file or an +``https`` URL, a ``Properties`` attribute that corresponds to the module's +parameters, and an ``Overrides`` attribute that can override module output. + +This command also allows you to add a ``Constants`` section to the template +or to a local module. This section is a simple set of key-value pairs that +can be used to reduce copy-paste within the template. Constants values are +strings that can be references within ``Fn::Sub`` functions using the format +``${Constant::NAME}``. + + + diff --git a/awscli/examples/cloudformation/package.rst b/awscli/examples/cloudformation/package.rst index 159b9b4f844e..2de7d63b1576 100644 --- a/awscli/examples/cloudformation/package.rst +++ b/awscli/examples/cloudformation/package.rst @@ -1,6 +1,69 @@ -Following command exports a template named ``template.json`` by uploading local +Following command exports a template named ``template.yaml`` by uploading local artifacts to S3 bucket ``bucket-name`` and writes the exported template to -``packaged-template.json``:: +``packaged-template.yaml``:: - aws cloudformation package --template-file /path_to_template/template.json --s3-bucket bucket-name --output-template-file packaged-template.json --use-json + aws cloudformation package --template-file /path_to_template/template.yaml --s3-bucket bucket-name --output-template-file packaged-template.yaml + + +The following is an example of a template with a ``Modules`` section:: + + Modules: + Content: + Source: ./module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Metadata: + OverrideMe: def + +A module configured as a Resource:: + + Resources: + Content: + Type: LocalModule + Source: ./module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Metadata: + OverrideMe: def + +An example module:: + + Parameters: + Name: + Type: String + Resources: + Bucket: + Type: AWS::S3::Bucket + Metadata: + OverrideMe: abc + Properties: + BucketName: !Ref Name + Outputs: + BucketArn: !GetAtt Bucket.Arn + +Packaging the template with this module would result in the following output:: + + Resources: + ContentBucket: + Type: AWS::S3::Bucket + Metadata: + OverrideMe: def + Properties: + BucketName: foo + +The following is an example of adding constants to a template:: + + Constants: + foo: bar + baz: ${Constant:foo}-xyz-${AWS::AccountId} + + Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub ${Constant::baz} diff --git a/tests/unit/customizations/cloudformation/modules/basic-expect.yaml b/tests/unit/customizations/cloudformation/modules/basic-expect.yaml new file mode 100644 index 000000000000..1010e68cf134 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/basic-expect.yaml @@ -0,0 +1,16 @@ +Resources: + OtherResource: + Type: AWS::S3::Bucket + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + OverrideMe: def + SubName: foo + ContentBucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: + Fn::GetAtt: + - ContentBucket + - Something diff --git a/tests/unit/customizations/cloudformation/modules/basic-module.yaml b/tests/unit/customizations/cloudformation/modules/basic-module.yaml new file mode 100644 index 000000000000..cbb5d4bbd90b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/basic-module.yaml @@ -0,0 +1,14 @@ +Parameters: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name + OverrideMe: abc + SubName: !Sub ${Name} + Bucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: !GetAtt Bucket.Something diff --git a/tests/unit/customizations/cloudformation/modules/basic-template.yaml b/tests/unit/customizations/cloudformation/modules/basic-template.yaml new file mode 100644 index 000000000000..712eb2a8a87b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/basic-template.yaml @@ -0,0 +1,12 @@ +Modules: + Content: + Source: ./basic-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +Resources: + OtherResource: + Type: AWS::S3::Bucket diff --git a/tests/unit/customizations/cloudformation/modules/cond-intrinsics-expect.yaml b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-expect.yaml new file mode 100644 index 000000000000..649ec70a9de9 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-expect.yaml @@ -0,0 +1,7 @@ +Resources: + ABucket0: + Type: AWS::S3::Bucket + ABucket1: + Type: AWS::S3::Bucket + ABucket2: + Type: AWS::S3::Bucket diff --git a/tests/unit/customizations/cloudformation/modules/cond-intrinsics-module.yaml b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-module.yaml new file mode 100644 index 000000000000..b4e1353addab --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-module.yaml @@ -0,0 +1,40 @@ +Parameters: + Foo: + Type: String + +Conditions: + Show0: + Fn::Equals: + - !Ref Foo + - bar + Show1: + Fn::And: + - Fn::Equals: + - !Ref Foo + - bar + - Condition: Show0 + Show2: + Fn::Not: + - Fn::Or: + - Fn::Equals: + - !Ref Foo + - baz + - Fn::Equals: + - Fn::If: + - Fn::Equals: + - a + - a + - x + - y + - y + +Resources: + Bucket0: + Type: AWS::S3::Bucket + Condition: Show0 + Bucket1: + Type: AWS::S3::Bucket + Condition: Show1 + Bucket2: + Type: AWS::S3::Bucket + Condition: Show2 diff --git a/tests/unit/customizations/cloudformation/modules/cond-intrinsics-template.yaml b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-template.yaml new file mode 100644 index 000000000000..c3daddf7c8f2 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-template.yaml @@ -0,0 +1,9 @@ +Modules: + A: + Source: ./cond-intrinsics-module.yaml + Properties: + Foo: bar + B: + Source: ./cond-intrinsics-module.yaml + Properties: + Foo: baz diff --git a/tests/unit/customizations/cloudformation/modules/conditional-expect.yaml b/tests/unit/customizations/cloudformation/modules/conditional-expect.yaml new file mode 100644 index 000000000000..65e290258657 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/conditional-expect.yaml @@ -0,0 +1,46 @@ +Resources: + ATable: + Type: AWS::DynamoDB::Table + Properties: + BillingMode: PAY_PER_REQUEST + TableName: table-a + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + ALambdaPolicy: + Type: AWS::IAM::RolePolicy + Condition: IfLambdaRoleIsSet + Metadata: + Comment: This resource is created only if the LambdaRoleArn is set + Properties: + PolicyDocument: + Statement: + - Action: + - dynamodb:BatchGetItem + - dynamodb:GetItem + - dynamodb:Query + - dynamodb:Scan + - dynamodb:BatchWriteItem + - dynamodb:PutItem + - dynamodb:UpdateItem + Effect: Allow + Resource: + - Fn::GetAtt: + - ATable + - Arn + PolicyName: table-a-policy + RoleName: my-role + BTable: + Type: AWS::DynamoDB::Table + Properties: + BillingMode: PAY_PER_REQUEST + TableName: table-b + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH diff --git a/tests/unit/customizations/cloudformation/modules/conditional-module.yaml b/tests/unit/customizations/cloudformation/modules/conditional-module.yaml new file mode 100644 index 000000000000..f2639ba3b53e --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/conditional-module.yaml @@ -0,0 +1,52 @@ +Parameters: + + TableName: + Type: String + + LambdaRoleName: + Type: String + Description: If set, allow the lambda function to access this table + Default: "" + +Conditions: + IfLambdaRoleIsSet: + Fn::Not: + - Fn::Equals: + - !Ref LambdaRoleName + - "" + +Resources: + Table: + Type: AWS::DynamoDB::Table + Properties: + BillingMode: PAY_PER_REQUEST + TableName: !Sub ${TableName} + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + + LambdaPolicy: + Type: AWS::IAM::RolePolicy + Condition: IfLambdaRoleIsSet + Metadata: + Comment: This resource is created only if the LambdaRoleArn is set + Properties: + PolicyDocument: + Statement: + - Action: + - dynamodb:BatchGetItem + - dynamodb:GetItem + - dynamodb:Query + - dynamodb:Scan + - dynamodb:BatchWriteItem + - dynamodb:PutItem + - dynamodb:UpdateItem + Effect: Allow + Resource: + - !GetAtt Table.Arn + PolicyName: !Sub ${TableName}-policy + RoleName: !Ref LambdaRoleName + diff --git a/tests/unit/customizations/cloudformation/modules/conditional-template.yaml b/tests/unit/customizations/cloudformation/modules/conditional-template.yaml new file mode 100644 index 000000000000..1e7b3ca6345b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/conditional-template.yaml @@ -0,0 +1,13 @@ +Resources: + A: + Type: LocalModule + Source: ./conditional-module.yaml + Properties: + TableName: table-a + LambdaRoleName: my-role + B: + Type: LocalModule + Source: ./conditional-module.yaml + Properties: + TableName: table-b + diff --git a/tests/unit/customizations/cloudformation/modules/constant-expect.yaml b/tests/unit/customizations/cloudformation/modules/constant-expect.yaml new file mode 100644 index 000000000000..d9074de4d355 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/constant-expect.yaml @@ -0,0 +1,10 @@ +Metadata: + test: zzz +Resources: + ContentBucket: + Type: AWS::S3::Bucket + Metadata: + Test: + Fn::Sub: bar-abc-${AWS::AccountId}-xyz + Properties: + BucketName: bar diff --git a/tests/unit/customizations/cloudformation/modules/constant-module.yaml b/tests/unit/customizations/cloudformation/modules/constant-module.yaml new file mode 100644 index 000000000000..768e11418294 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/constant-module.yaml @@ -0,0 +1,12 @@ +Constants: + foo: bar + sub: "${Constant::foo}-abc-${AWS::AccountId}" + +Resources: + Bucket: + Type: AWS::S3::Bucket + Metadata: + Test: !Sub ${Constant::sub}-xyz + Properties: + BucketName: !Sub ${Constant::foo} + diff --git a/tests/unit/customizations/cloudformation/modules/constant-template.yaml b/tests/unit/customizations/cloudformation/modules/constant-template.yaml new file mode 100644 index 000000000000..b76f325aa509 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/constant-template.yaml @@ -0,0 +1,9 @@ +Constants: + yyy: zzz +Metadata: + test: !Sub ${Constant::yyy} +Resources: + Content: + Type: LocalModule + Source: ./constant-module.yaml + diff --git a/tests/unit/customizations/cloudformation/modules/example-expect.yaml b/tests/unit/customizations/cloudformation/modules/example-expect.yaml new file mode 100644 index 000000000000..0314736240bd --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/example-expect.yaml @@ -0,0 +1,7 @@ +Resources: + ContentBucket: + Type: AWS::S3::Bucket + Metadata: + OverrideMe: def + Properties: + BucketName: foo diff --git a/tests/unit/customizations/cloudformation/modules/example-module.yaml b/tests/unit/customizations/cloudformation/modules/example-module.yaml new file mode 100644 index 000000000000..66bd95fecb2a --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/example-module.yaml @@ -0,0 +1,11 @@ +Parameters: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Metadata: + OverrideMe: abc + Properties: + BucketName: !Ref Name + diff --git a/tests/unit/customizations/cloudformation/modules/example-template.yaml b/tests/unit/customizations/cloudformation/modules/example-template.yaml new file mode 100644 index 000000000000..780cac34a992 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/example-template.yaml @@ -0,0 +1,10 @@ +Resources: + Content: + Type: LocalModule + Source: ./example-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Metadata: + OverrideMe: def diff --git a/tests/unit/customizations/cloudformation/modules/getatt-expect.yaml b/tests/unit/customizations/cloudformation/modules/getatt-expect.yaml new file mode 100644 index 000000000000..00c96a04d2dc --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/getatt-expect.yaml @@ -0,0 +1,11 @@ +Resources: + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo +Outputs: + BucketName: + Value: + Fn::GetAtt: + - ContentBucket + - BucketName diff --git a/tests/unit/customizations/cloudformation/modules/getatt-module.yaml b/tests/unit/customizations/cloudformation/modules/getatt-module.yaml new file mode 100644 index 000000000000..349b4a403cef --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/getatt-module.yaml @@ -0,0 +1,10 @@ +Parameters: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name +Outputs: + BucketName: !GetAtt "Bucket.BucketName" diff --git a/tests/unit/customizations/cloudformation/modules/getatt-template.yaml b/tests/unit/customizations/cloudformation/modules/getatt-template.yaml new file mode 100644 index 000000000000..70685556220e --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/getatt-template.yaml @@ -0,0 +1,9 @@ +Resources: + Content: + Type: LocalModule + Source: ./getatt-module.yaml + Properties: + Name: foo +Outputs: + BucketName: + Value: !GetAtt Content.BucketName diff --git a/tests/unit/customizations/cloudformation/modules/map-expect.yaml b/tests/unit/customizations/cloudformation/modules/map-expect.yaml new file mode 100644 index 000000000000..69b789b9a716 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/map-expect.yaml @@ -0,0 +1,17 @@ +Parameters: + List: + Type: CommaDelimitedList + Default: A,B,C +Resources: + Content0Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: my-bucket-A + Content1Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: my-bucket-B + Content2Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: my-bucket-C diff --git a/tests/unit/customizations/cloudformation/modules/map-module.yaml b/tests/unit/customizations/cloudformation/modules/map-module.yaml new file mode 100644 index 000000000000..e96d93d1733e --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/map-module.yaml @@ -0,0 +1,9 @@ +Parameters: + Name: + Type: String + +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name diff --git a/tests/unit/customizations/cloudformation/modules/map-template.yaml b/tests/unit/customizations/cloudformation/modules/map-template.yaml new file mode 100644 index 000000000000..ba6a67dcd30d --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/map-template.yaml @@ -0,0 +1,12 @@ +Parameters: + List: + Type: CommaDelimitedList + Default: A,B,C + +Resources: + Content: + Type: LocalModule + Source: ./map-module.yaml + Map: !Ref List + Properties: + Name: !Sub my-bucket-$MapValue diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml new file mode 100644 index 000000000000..561947b750a2 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml @@ -0,0 +1,14 @@ +Parameters: + ParentVal: + Type: String + AppName: + Type: String +Resources: + MySubBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: mod-in-mod-bucket + XName: + Fn::Sub: ${ParentVal}-abc + YName: + Fn::Sub: ${AppName}-xyz diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-module.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-module.yaml new file mode 100644 index 000000000000..208307bd37f5 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-module.yaml @@ -0,0 +1,11 @@ +Parameters: + AppName: + Type: String +Resources: + Sub: + Type: LocalModule + Source: "./modinmod-submodule.yaml" + Properties: + X: !Sub ${ParentVal}-abc + Y: !Sub ${AppName}-xyz + diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml new file mode 100644 index 000000000000..745292ee0711 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml @@ -0,0 +1,12 @@ +Parameters: + X: + Type: String + Y: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: mod-in-mod-bucket + XName: !Ref X + YName: !Ref Y diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-template.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-template.yaml new file mode 100644 index 000000000000..99cd480b512d --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-template.yaml @@ -0,0 +1,11 @@ +Parameters: + ParentVal: + Type: String + AppName: + Type: String +Resources: + My: + Type: LocalModule + Source: ./modinmod-module.yaml + Properties: + AppName: !Ref AppName diff --git a/tests/unit/customizations/cloudformation/modules/output-expect.yaml b/tests/unit/customizations/cloudformation/modules/output-expect.yaml new file mode 100644 index 000000000000..a4ebb8fe8ea5 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-expect.yaml @@ -0,0 +1,17 @@ +Outputs: + ExampleOutput: + Value: + Fn::GetAtt: + - ContentBucket + - Arn + ExampleSub: + Value: + Fn::Sub: ${ContentBucket.Arn} + ExampleGetSub: + Value: + Fn::Sub: ${ContentBucket.Arn} +Resources: + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo diff --git a/tests/unit/customizations/cloudformation/modules/output-module.yaml b/tests/unit/customizations/cloudformation/modules/output-module.yaml new file mode 100644 index 000000000000..f685ca154d84 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-module.yaml @@ -0,0 +1,12 @@ +Parameters: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name +Outputs: + BucketArn: !GetAtt Bucket.Arn + BucketArnSub: !Sub ${Bucket.Arn} + diff --git a/tests/unit/customizations/cloudformation/modules/output-template.yaml b/tests/unit/customizations/cloudformation/modules/output-template.yaml new file mode 100644 index 000000000000..872d3423597b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-template.yaml @@ -0,0 +1,12 @@ +Modules: + Content: + Source: ./output-module.yaml + Properties: + Name: foo +Outputs: + ExampleOutput: + Value: !GetAtt Content.BucketArn + ExampleSub: + Value: !Sub ${Content.BucketArn} + ExampleGetSub: + Value: !GetAtt Content.BucketArnSub diff --git a/tests/unit/customizations/cloudformation/modules/policy-expect.yaml b/tests/unit/customizations/cloudformation/modules/policy-expect.yaml new file mode 100644 index 000000000000..8d524c1fb4ce --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/policy-expect.yaml @@ -0,0 +1,16 @@ +Resources: + AccessPolicy: + Type: AWS::IAM::RolePolicy + Properties: + PolicyDocument: + Statement: + - Effect: ALLOW + Action: s3:List* + Resource: + - arn:aws:s3:::foo + - arn:aws:s3:::foo/* + - Effect: DENY + Action: s3:Put* + Resource: arn:aws:s3:::bar + PolicyName: my-policy + RoleName: foo diff --git a/tests/unit/customizations/cloudformation/modules/policy-module.yaml b/tests/unit/customizations/cloudformation/modules/policy-module.yaml new file mode 100644 index 000000000000..7de41d792584 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/policy-module.yaml @@ -0,0 +1,19 @@ +Parameters: + Role: + Type: String + BucketName: + Default: foo +Resources: + Policy: + Type: AWS::IAM::RolePolicy + Properties: + PolicyDocument: + Statement: + - Effect: ALLOW + Action: s3:List* + Resource: + - !Sub arn:aws:s3:::${BucketName} + - !Sub arn:aws:s3:::${BucketName}/* + PolicyName: my-policy + RoleName: !Ref Role + diff --git a/tests/unit/customizations/cloudformation/modules/policy-template.yaml b/tests/unit/customizations/cloudformation/modules/policy-template.yaml new file mode 100644 index 000000000000..9f587f7bda9d --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/policy-template.yaml @@ -0,0 +1,13 @@ +Modules: + Access: + Source: ./policy-module.yaml + Properties: + Role: foo + Overrides: + Policy: + Properties: + PolicyDocument: + Statement: + - Effect: DENY + Action: s3:Put* + Resource: arn:aws:s3:::bar diff --git a/tests/unit/customizations/cloudformation/modules/sub-expect.yaml b/tests/unit/customizations/cloudformation/modules/sub-expect.yaml new file mode 100644 index 000000000000..79cb95080ccc --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/sub-expect.yaml @@ -0,0 +1,21 @@ +Resources: + MyBucket1: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + X: + Fn::Sub: ${Foo} + Y: + - Fn::Sub: noparent0-${Foo} + - Fn::Sub: noparent1-${Foo} + Z: + - Fn::Sub: ${Foo} + - Ref: MyBucket2 + - ZZ: + ZZZ: + ZZZZ: + Fn::Sub: ${Foo} + MyBucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: bar diff --git a/tests/unit/customizations/cloudformation/modules/sub-module.yaml b/tests/unit/customizations/cloudformation/modules/sub-module.yaml new file mode 100644 index 000000000000..8d4937f4aefc --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/sub-module.yaml @@ -0,0 +1,23 @@ +Parameters: + Name: + Type: String + SubName: + Type: String +Resources: + Bucket1: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub ${Name} + X: !Sub "${SubName}" + Y: + - !Sub noparent0-${SubName} + - !Sub noparent1-${SubName} + Z: + - !Ref SubName + - !Ref Bucket2 + - ZZ: + ZZZ: + ZZZZ: !Sub "${SubName}" + Bucket2: + Type: AWS::S3::Bucket + diff --git a/tests/unit/customizations/cloudformation/modules/sub-template.yaml b/tests/unit/customizations/cloudformation/modules/sub-template.yaml new file mode 100644 index 000000000000..feeb263bfcad --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/sub-template.yaml @@ -0,0 +1,11 @@ +Resources: + My: + Type: LocalModule + Source: ./sub-module.yaml + Properties: + Name: foo + SubName: !Sub ${Foo} + Overrides: + Bucket2: + Properties: + BucketName: bar diff --git a/tests/unit/customizations/cloudformation/modules/subnet-module.yaml b/tests/unit/customizations/cloudformation/modules/subnet-module.yaml new file mode 100644 index 000000000000..2291bd6b0c7e --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/subnet-module.yaml @@ -0,0 +1,59 @@ +Parameters: + + AZSelection: + Type: Number + + SubnetCidrBlock: + Type: String + +Resources: + + Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: !Select [!Ref AZSelection, Fn::GetAZs: !Ref "AWS::Region"] + CidrBlock: !Ref SubnetCidrBlock + MapIpOnLaunch: true + VpcId: !Ref VPC + + RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + + RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: !Ref SubnetRouteTable + SubnetId: !Ref Subnet + + DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + RouteTableId: !Ref SubnetRouteTable + DependsOn: VPCGW + + EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + + NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: !GetAtt SubnetEIP.AllocationId + SubnetId: !Ref Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + diff --git a/tests/unit/customizations/cloudformation/modules/type-expect.yaml b/tests/unit/customizations/cloudformation/modules/type-expect.yaml new file mode 100644 index 000000000000..1010e68cf134 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/type-expect.yaml @@ -0,0 +1,16 @@ +Resources: + OtherResource: + Type: AWS::S3::Bucket + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + OverrideMe: def + SubName: foo + ContentBucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: + Fn::GetAtt: + - ContentBucket + - Something diff --git a/tests/unit/customizations/cloudformation/modules/type-module.yaml b/tests/unit/customizations/cloudformation/modules/type-module.yaml new file mode 100644 index 000000000000..cbb5d4bbd90b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/type-module.yaml @@ -0,0 +1,14 @@ +Parameters: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name + OverrideMe: abc + SubName: !Sub ${Name} + Bucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: !GetAtt Bucket.Something diff --git a/tests/unit/customizations/cloudformation/modules/type-template.yaml b/tests/unit/customizations/cloudformation/modules/type-template.yaml new file mode 100644 index 000000000000..e868af732938 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/type-template.yaml @@ -0,0 +1,12 @@ +Resources: + Content: + Type: LocalModule + Source: ./type-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def + OtherResource: + Type: AWS::S3::Bucket diff --git a/tests/unit/customizations/cloudformation/modules/url-template.yaml b/tests/unit/customizations/cloudformation/modules/url-template.yaml new file mode 100644 index 000000000000..febd0aee8ca0 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/url-template.yaml @@ -0,0 +1,14 @@ +Constants: + ModuleSource: https://raw.githubusercontent.com/aws/aws-cli/2f0143bab567386b930322b2b0e845740f7adfd0/tests/unit/customizations/cloudformation/modules +Modules: + Content: + Source: !Sub ${Constant::ModuleSource}/basic-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +Resources: + OtherResource: + Type: AWS::S3::Bucket diff --git a/tests/unit/customizations/cloudformation/modules/vpc-expect.yaml b/tests/unit/customizations/cloudformation/modules/vpc-expect.yaml new file mode 100644 index 000000000000..0d43ced5bd2b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/vpc-expect.yaml @@ -0,0 +1,236 @@ +Resources: + NetworkVPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/16 + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default + NetworkPublicSubnet0Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '0' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.128.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet0RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet0RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPublicSubnet0Subnet + NetworkPublicSubnet0DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPublicSubnet0EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPublicSubnet0NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPublicSubnet0SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPublicSubnet0Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPublicSubnet1Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '1' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.192.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet1RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet1RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPublicSubnet1Subnet + NetworkPublicSubnet1DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPublicSubnet1EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPublicSubnet1NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPublicSubnet1SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPublicSubnet1Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPrivateSubnet0Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '0' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.0.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet0RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet0RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPrivateSubnet0Subnet + NetworkPrivateSubnet0DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPrivateSubnet0EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPrivateSubnet0NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPrivateSubnet0SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPrivateSubnet0Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPrivateSubnet1Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '1' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.64.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet1RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet1RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPrivateSubnet1Subnet + NetworkPrivateSubnet1DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPrivateSubnet1EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPrivateSubnet1NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPrivateSubnet1SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPrivateSubnet1Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation diff --git a/tests/unit/customizations/cloudformation/modules/vpc-module.yaml b/tests/unit/customizations/cloudformation/modules/vpc-module.yaml new file mode 100644 index 000000000000..a33c4e29aada --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/vpc-module.yaml @@ -0,0 +1,37 @@ +Parameters: + + CidrBlock: + Type: String + + PrivateCidrBlocks: + Type: CommaDelimitedList + + PublicCidrBlocks: + Type: CommaDelimitedList + +Resources: + + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: !Ref CidrBlock + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default + + PublicSubnet: + Type: LocalModule + Map: !Ref PublicCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + + PrivateSubnet: + Type: LocalModule + Map: !Ref PrivateCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + diff --git a/tests/unit/customizations/cloudformation/modules/vpc-template.yaml b/tests/unit/customizations/cloudformation/modules/vpc-template.yaml new file mode 100644 index 000000000000..92e512b57d67 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/vpc-template.yaml @@ -0,0 +1,9 @@ +Resources: + + Network: + Type: LocalModule + Source: ./vpc-module.yaml + Properties: + CidrBlock: 10.0.0.0/16 + PrivateCidrBlocks: 10.0.0.0/18,10.0.64.0/18 + PublicCidrBlocks: 10.0.128.0/18,10.0.192.0/18 diff --git a/tests/unit/customizations/cloudformation/test_modules.py b/tests/unit/customizations/cloudformation/test_modules.py new file mode 100644 index 000000000000..bcd81bc9d627 --- /dev/null +++ b/tests/unit/customizations/cloudformation/test_modules.py @@ -0,0 +1,154 @@ +"Tests for module support in the package command" + +# pylint: disable=fixme + +from awscli.testutils import unittest +from awscli.customizations.cloudformation import yamlhelper +from awscli.customizations.cloudformation import modules +from awscli.customizations.cloudformation.parse_sub import SubWord, WordType +from awscli.customizations.cloudformation.parse_sub import parse_sub +from awscli.customizations.cloudformation.module_visitor import Visitor +from awscli.customizations.cloudformation.module_constants import ( + process_constants, + replace_constants, +) + +MODULES = "Modules" +RESOURCES = "Resources" +TYPE = "Type" +LOCAL_MODULE = "LocalModule" + + +class TestPackageModules(unittest.TestCase): + "Module tests" + + def setUp(self): + "Initialize the tests" + + def test_parse_sub(self): + "Test the parse_sub function" + cases = { + "ABC": [SubWord(WordType.STR, "ABC")], + "ABC-${XYZ}-123": [ + SubWord(WordType.STR, "ABC-"), + SubWord(WordType.REF, "XYZ"), + SubWord(WordType.STR, "-123"), + ], + "ABC-${!Literal}-1": [SubWord(WordType.STR, "ABC-${Literal}-1")], + "${ABC}": [SubWord(WordType.REF, "ABC")], + "${ABC.XYZ}": [SubWord(WordType.GETATT, "ABC.XYZ")], + "ABC${AWS::AccountId}XYZ": [ + SubWord(WordType.STR, "ABC"), + SubWord(WordType.AWS, "AccountId"), + SubWord(WordType.STR, "XYZ"), + ], + "BAZ${ABC$XYZ}FOO$BAR": [ + SubWord(WordType.STR, "BAZ"), + SubWord(WordType.REF, "ABC$XYZ"), + SubWord(WordType.STR, "FOO$BAR"), + ], + "${ ABC }": [SubWord(WordType.REF, "ABC")], + "${ ABC }": [SubWord(WordType.REF, "ABC")], + " ABC ": [SubWord(WordType.STR, " ABC ")], + } + + for sub, expect in cases.items(): + words = parse_sub(sub, False) + self.assertEqual( + len(expect), + len(words), + f'"{sub}": words len is {len(words)}, expected {len(expect)}', + ) + for i, w in enumerate(expect): + self.assertEqual( + words[i].t, w.t, f'"{sub}": got {words[i]}, expected {w}' + ) + self.assertEqual( + words[i].w, w.w, f'"{sub}": got {words[i]}, expected {w}' + ) + + # Invalid strings should fail + sub = "${AAA" + with self.assertRaises(Exception, msg=f'"{sub}": should have failed'): + parse_sub(sub, False) + + def test_merge_props(self): + "Test the merge_props function" + + original = {"b": "c", "d": {"e": "f", "i": [1, 2, 3]}} + overrides = {"b": "cc", "d": {"e": "ff", "g": "h", "i": [4, 5]}} + expect = {"b": "cc", "d": {"e": "ff", "g": "h", "i": [1, 2, 3, 4, 5]}} + merged = modules.merge_props(original, overrides) + self.assertEqual(merged, expect) + + def test_main(self): + "Run tests on sample templates that include local modules" + + # The tests are in the modules directory. + # Each test has 3 files: + # test-template.yaml, test-module.yaml, and test-expect.yaml + tests = [ + "basic", + "type", + "sub", + "modinmod", + "output", + "policy", + "vpc", + "map", + "conditional", + "cond-intrinsics", + "example", + "getatt", + "constant", + ] + for test in tests: + base = "unit/customizations/cloudformation/modules" + t = modules.read_source(f"{base}/{test}-template.yaml") + td = yamlhelper.yaml_parse(t) + e = modules.read_source(f"{base}/{test}-expect.yaml") + + constants = process_constants(td) + if constants is not None: + replace_constants(constants, td) + + # Modules section + td = modules.process_module_section(td, base, t) + + # Resources with Type LocalModule + td = modules.process_resources_section(td, base, t, None) + + processed = yamlhelper.yaml_dump(td) + self.assertEqual(e, processed) + + def test_visitor(self): + "Test module_visitor" + + template_dict = {} + resources = {} + template_dict["Resources"] = resources + resources["Bucket"] = { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketName": "foo", + "TestList": [ + "Item0", + {"BucketName": "foo"}, + {"Item2": {"BucketName": "foo"}}, + ], + }, + } + v = Visitor(template_dict, None, "") + + def vf(v): + if isinstance(v.d, str) and v.p is not None: + if v.k == "BucketName": + v.p[v.k] = "bar" + + v.visit(vf) + self.assertEqual( + resources["Bucket"]["Properties"]["BucketName"], "bar" + ) + test_list = resources["Bucket"]["Properties"]["TestList"] + self.assertEqual(test_list[1]["BucketName"], "bar") + self.assertEqual(test_list[2]["Item2"]["BucketName"], "bar")