diff --git a/.gitignore b/.gitignore index 6968dc0f..38441023 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc *.retry +templates/AMI-Template-Instances.cfn.yml templates/chef.json templates/chef-source.json templates/cloudfront.cfn.yml diff --git a/AMI-templates.yml b/AMI-templates.yml new file mode 100644 index 00000000..4c3c23d2 --- /dev/null +++ b/AMI-templates.yml @@ -0,0 +1,39 @@ +--- +- name: Cloudformation Playbook + hosts: local + connection: local + + pre_tasks: + - name: get basic_facts + set_fact: + basic_fact={{ item }} +# CKANSource={{ item.CKANSource }} + when: item.Environment == Environment + with_items: "{{ basic_facts }}" + + - name: set facts to environment from basic_fact + set_fact: "{{ item.key }}={{ item.value }}" + with_dict: "{{ basic_fact }}" + when: basic_fact is defined + + - name: kms alias fact + aws_kms_facts: + filters: + alias: "aws/ssm" + region: "{{ region }}" + register: ssmKeyFacts + + - name: set KMS key from alias + set_fact: + SSMKey: "{{ ssmKeyFacts['keys'][0].key_arn }}" + + - name: Generate Lambda file hash + shell: > + md5sum files/instanceSetupLambda.js | awk '{print substr($1, 1, 20)}' + register: hash_output + - set_fact: + instance_setup_source_hash: "{{ hash_output.stdout_lines[0] }}" + + - include_vars: vars/AMI-template-instances.var.yml + roles: + - ansible_cloudformation diff --git a/files/instanceSetupLambda.js b/files/instanceSetupLambda.js index b0d45019..1992c87b 100644 --- a/files/instanceSetupLambda.js +++ b/files/instanceSetupLambda.js @@ -21,6 +21,12 @@ function recordCompletion(event, success) { exports.handler = async (event) => { const instanceId = event['EC2InstanceId']; + /* + * Possible deployment types include: + * - 'setup': Full end-to-end configuration, from vanilla operating system to live instance. + * - 'deploy': Take a vanilla operating system and set up a warm instance, but do not make it active. + * - 'configure': Take a warm instance and make it active. + */ const deployPhase = 'phase' in event ? event['phase'] : 'setup'; if (!['setup', 'deploy', 'configure'].includes(deployPhase)) { console.log("Invalid deployment phase '" + deployPhase + "', must be one of 'setup', 'deploy', 'configure'"); @@ -95,12 +101,15 @@ exports.handler = async (event) => { } else { recipePrefix = `datashades::${layer}`; } - var runList = `recipe[${recipePrefix}-configure]`; - if (deployPhase !== 'configure') { - runList = `recipe[${recipePrefix}-deploy],${runList}`; + var runList = ""; + if (deployPhase !== 'deploy') { + runList = `recipe[${recipePrefix}-configure]`; } if (deployPhase === 'setup') { - runList = `recipe[${recipePrefix}-setup],${runList}`; + runList = `,${runList}`; + } + if (deployPhase !== 'configure') { + runList = `recipe[${recipePrefix}-setup],recipe[${recipePrefix}-deploy]${runList}`; } await ssm.send(new SendCommandCommand({ diff --git a/templates/AMI-Template-Instances.cfn.yml.j2 b/templates/AMI-Template-Instances.cfn.yml.j2 new file mode 100644 index 00000000..0157e272 --- /dev/null +++ b/templates/AMI-Template-Instances.cfn.yml.j2 @@ -0,0 +1,88 @@ +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Creates archetype instances for generating machine image templates.' + +Parameters: + ApplicationName: + Description: Name of the application (eg OpenData) + Type: String + ApplicationId: + Description: All-lowercase identifier for the application (eg 'opendata') + Type: String + ConstraintDescription: Must contain only lowercase/numeric/hyphen/underscore. + AllowedPattern: '[-_a-z0-9]*' + Environment: + Description: The target production vs non-production environment. + Type: String + Default: DEV + AllowedValues: + - DEV + - TRAINING + - STAGING + - PROD + AppSubnets: + Description: The base name for the exported application layer subnet IDs, eg if the exports are 'PRODMyApplicationAppSubnetA' and 'PRODMyApplicationAppSubnetB', then this would be 'PRODMyApplicationAppSubnet'. Only needed for HA configurations. + Type: String + Default: none + DefaultEC2Key: + Description: Select an existing SSH key + Type: AWS::EC2::KeyPair::KeyName + +Resources: + +{% for layer in ['Batch', 'Web', 'Solr'] %} +{% set disk_size = 100 if layer == 'Solr' else 32 %} + {{ layer }}TemplateInstance: + Type: AWS::EC2::Instance + Properties: + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + DeleteOnTermination: true + VolumeSize: 100 + VolumeType: "gp2" + - DeviceName: "/dev/sdi" + Ebs: + DeleteOnTermination: true + VolumeSize: {{ disk_size }} + VolumeType: "gp2" + IamInstanceProfile: !Ref {% if layer != 'Solr' %}Web{% endif %}InstanceRoleProfile + ImageId: "ami-0d71fe73adf7a9887" + InstanceType: "t3a.small" + KeyName: !Ref DefaultEC2Key + NetworkInterfaces: + - DeviceIndex: 0 + GroupSet: + - Fn::ImportValue: !Sub "${Environment}CKANManagementSG" + - Fn::ImportValue: !Sub "${Environment}CKAN{% if layer == 'Solr' %}Database{% else %}AppAsg{% endif %}SG" + SubnetId: + Fn::ImportValue: !Sub "${AppSubnets}A" + UserData: + Fn::Base64: + Fn::Sub: | + #!/bin/sh + if ! (grep '/mnt/local_data' /etc/fstab >/dev/null); then + mkdir /mnt/local_data + mkfs -t xfs /dev/sdi + echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab + mount -a + fi + if ! (yum install chef); then + for i in `seq 1 5`; do + yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break + sleep 5 + done + fi + REGION="--region ${AWS::Region}" + metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \ + INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \ + aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-ami-template" + FUNCTION_NAME=$(aws ssm get-parameter $REGION --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text) + if (aws --version |grep -o 'aws-cli/[2-9]'); then + PAYLOAD_FORMAT="--cli-binary-format raw-in-base64-out" + fi + aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "deploy"}' /var/log/instance-setup.log.`date '+%s'` + Tags: + - Key: Layer + Value: {{ layer|lower }} +{% endfor %} diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 6e076d2e..71ef6fe5 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -12,7 +12,7 @@ Parameters: ConstraintDescription: Must contain only lowercase/numeric/hyphen/underscore. AllowedPattern: '[-_a-z0-9]*' Environment: - Description: Select a stack version. + Description: The target production vs non-production environment. Type: String Default: STAGING AllowedValues: @@ -246,3 +246,12 @@ Resources: {% endif %} {% endfor %} + +Outputs: + +{% for layer in ['Batch', 'Web', 'Solr'] %} + {{ layer }}LaunchTemplateId: + Value: !Ref {{ layer }}LaunchTemplate + Export: + Name: !Sub "${Environment}${ApplicationName}{{ layer }}LaunchTemplateId" +{% endfor %} diff --git a/vars/AMI-template-instances.var.yml b/vars/AMI-template-instances.var.yml new file mode 100644 index 00000000..9748afb3 --- /dev/null +++ b/vars/AMI-template-instances.var.yml @@ -0,0 +1,31 @@ +--- + +common_stack: &common_stack + state: "{{ state | default('present')}}" + region: "{{ region }}" + disable_rollback: true + template_jinja: "templates/AMI-Template-Instances.cfn.yml.j2" + template: "templates/AMI-Template-Instances.cfn.yml" + template_parameters: &common_stack_template_parameters + ApplicationName: "{{ service_name }}" + ApplicationId: "{{ service_name_lower }}" + Environment: "{{ Environment }}" + AppSubnets: "{{ Environment }}CKANAppSubnet" + LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" + AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" + SSMKey: "{{ SSMKey | default('') }}" + InternalStackZone: "{{ Environment }}CKANPrivateHostedZone" + DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" + tags: &common_stack_tags + Environment: "{{ Environment }}" + Service: "{{ service_name }}" + Division: "{{ Division }}" + Owner: "{{ Owner }}" + Version: "1.0" + +cloudformation_stacks: + - <<: *common_stack + name: "{{ service_name }}-{{ Environment }}-AMI-Template-Instances" + template_parameters: + <<: *common_stack_template_parameters + Environment: "{{ Environment }}"