QOLDEV-833 add template to stand up instances for generating golden AMIs

.gitignore

@@ -1,5 +1,6 @@
 *.pyc
 *.retry
+templates/AMI-Template-Instances.cfn.yml
 templates/chef.json
 templates/chef-source.json
 templates/cloudfront.cfn.yml

AMI-templates.yml

@@ -0,0 +1,39 @@
+---
+- name: Cloudformation Playbook
+  hosts: local
+  connection: local
+
+  pre_tasks:
+    - name: get basic_facts
+      set_fact:
+        basic_fact={{ item }}
+#        CKANSource={{ item.CKANSource }}
+      when: item.Environment == Environment
+      with_items: "{{ basic_facts }}"
+
+    - name: set facts to environment from basic_fact
+      set_fact: "{{ item.key }}={{ item.value }}"
+      with_dict: "{{ basic_fact }}"
+      when: basic_fact is defined
+
+    - name: kms alias fact
+      aws_kms_facts:
+        filters:
+          alias: "aws/ssm"
+        region: "{{ region }}"
+      register: ssmKeyFacts
+
+    - name: set KMS key from alias
+      set_fact:
+        SSMKey: "{{ ssmKeyFacts['keys'][0].key_arn }}"
+
+    - name: Generate Lambda file hash
+      shell: >
+        md5sum files/instanceSetupLambda.js | awk '{print substr($1, 1, 20)}'
+      register: hash_output
+    - set_fact:
+        instance_setup_source_hash: "{{ hash_output.stdout_lines[0] }}"
+
+    - include_vars: vars/AMI-template-instances.var.yml
+  roles:
+    - ansible_cloudformation

files/instanceSetupLambda.js

@@ -21,6 +21,12 @@ function recordCompletion(event, success) {
 
 exports.handler = async (event) => {
   const instanceId = event['EC2InstanceId'];
+  /*
+   * Possible deployment types include:
+   * - 'setup': Full end-to-end configuration, from vanilla operating system to live instance.
+   * - 'deploy': Take a vanilla operating system and set up a warm instance, but do not make it active.
+   * - 'configure': Take a warm instance and make it active.
+   */
   const deployPhase = 'phase' in event ? event['phase'] : 'setup';
   if (!['setup', 'deploy', 'configure'].includes(deployPhase)) {
     console.log("Invalid deployment phase '" + deployPhase + "', must be one of 'setup', 'deploy', 'configure'");
@@ -95,12 +101,15 @@ exports.handler = async (event) => {
   } else {
     recipePrefix = `datashades::${layer}`;
   }
-  var runList = `recipe[${recipePrefix}-configure]`;
-  if (deployPhase !== 'configure') {
-    runList = `recipe[${recipePrefix}-deploy],${runList}`;
+  var runList = "";
+  if (deployPhase !== 'deploy') {
+    runList = `recipe[${recipePrefix}-configure]`;
   }
   if (deployPhase === 'setup') {
-    runList = `recipe[${recipePrefix}-setup],${runList}`;
+    runList = `,${runList}`;
+  }
+  if (deployPhase !== 'configure') {
+    runList = `recipe[${recipePrefix}-setup],recipe[${recipePrefix}-deploy]${runList}`;
   }
 
   await ssm.send(new SendCommandCommand({

templates/AMI-Template-Instances.cfn.yml.j2

@@ -0,0 +1,88 @@
+---
+AWSTemplateFormatVersion: '2010-09-09'
+Description: 'Creates archetype instances for generating machine image templates.'
+
+Parameters:
+  ApplicationName:
+    Description: Name of the application (eg OpenData)
+    Type: String
+  ApplicationId:
+    Description: All-lowercase identifier for the application (eg 'opendata')
+    Type: String
+    ConstraintDescription: Must contain only lowercase/numeric/hyphen/underscore.
+    AllowedPattern: '[-_a-z0-9]*'
+  Environment:
+    Description: The target production vs non-production environment.
+    Type: String
+    Default: DEV
+    AllowedValues:
+      - DEV
+      - TRAINING
+      - STAGING
+      - PROD
+  AppSubnets:
+    Description: The base name for the exported application layer subnet IDs, eg if the exports are 'PRODMyApplicationAppSubnetA' and 'PRODMyApplicationAppSubnetB', then this would be 'PRODMyApplicationAppSubnet'. Only needed for HA configurations.
+    Type: String
+    Default: none
+  DefaultEC2Key:
+    Description: Select an existing SSH key
+    Type: AWS::EC2::KeyPair::KeyName
+
+Resources:
+
+{% for layer in ['Batch', 'Web', 'Solr'] %}
+{% set disk_size = 100 if layer == 'Solr' else 32 %}
+  {{ layer }}TemplateInstance:
+    Type: AWS::EC2::Instance
+    Properties:
+      BlockDeviceMappings:
+        - DeviceName: "/dev/xvda"
+          Ebs:
+            DeleteOnTermination: true
+            VolumeSize: 100
+            VolumeType: "gp2"
+        - DeviceName: "/dev/sdi"
+          Ebs:
+            DeleteOnTermination: true
+            VolumeSize: {{ disk_size }}
+            VolumeType: "gp2"
+      IamInstanceProfile: !Ref {% if layer != 'Solr' %}Web{% endif %}InstanceRoleProfile
+      ImageId: "ami-0d71fe73adf7a9887"
+      InstanceType: "t3a.small"
+      KeyName: !Ref DefaultEC2Key
+      NetworkInterfaces:
+        - DeviceIndex: 0
+          GroupSet:
+            - Fn::ImportValue: !Sub "${Environment}CKANManagementSG"
+            - Fn::ImportValue: !Sub "${Environment}CKAN{% if layer == 'Solr' %}Database{% else %}AppAsg{% endif %}SG"
+          SubnetId:
+            Fn::ImportValue: !Sub "${AppSubnets}A"
+      UserData:
+        Fn::Base64:
+          Fn::Sub: |
+            #!/bin/sh
+            if ! (grep '/mnt/local_data' /etc/fstab >/dev/null); then
+              mkdir /mnt/local_data
+              mkfs -t xfs /dev/sdi
+              echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab
+              mount -a
+            fi
+              if ! (yum install chef); then
+                for i in `seq 1 5`; do
+                  yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break
+                  sleep 5
+                done
+              fi
+              REGION="--region ${AWS::Region}"
+            metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \
+            INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \
+            aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-ami-template"
+            FUNCTION_NAME=$(aws ssm get-parameter $REGION --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text)
+            if (aws --version |grep -o 'aws-cli/[2-9]'); then
+              PAYLOAD_FORMAT="--cli-binary-format raw-in-base64-out"
+            fi
+            aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "deploy"}' /var/log/instance-setup.log.`date '+%s'`
+      Tags:
+        - Key: Layer
+          Value: {{ layer|lower }}
+{% endfor %}

templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2

@@ -12,7 +12,7 @@ Parameters:
     ConstraintDescription: Must contain only lowercase/numeric/hyphen/underscore.
     AllowedPattern: '[-_a-z0-9]*'
   Environment:
-    Description: Select a stack version.
+    Description: The target production vs non-production environment.
     Type: String
     Default: STAGING
     AllowedValues:
@@ -246,3 +246,12 @@ Resources:
 {% endif %}
 
 {% endfor %}
+
+Outputs:
+
+{% for layer in ['Batch', 'Web', 'Solr'] %}
+  {{ layer }}LaunchTemplateId:
+    Value: !Ref {{ layer }}LaunchTemplate
+    Export:
+      Name: !Sub "${Environment}${ApplicationName}{{ layer }}LaunchTemplateId"
+{% endfor %}

vars/AMI-template-instances.var.yml

@@ -0,0 +1,31 @@
+---
+
+common_stack: &common_stack
+  state: "{{ state | default('present')}}"
+  region: "{{ region }}"
+  disable_rollback: true
+  template_jinja: "templates/AMI-Template-Instances.cfn.yml.j2"
+  template: "templates/AMI-Template-Instances.cfn.yml"
+  template_parameters: &common_stack_template_parameters
+    ApplicationName: "{{ service_name }}"
+    ApplicationId: "{{ service_name_lower }}"
+    Environment: "{{ Environment }}"
+    AppSubnets: "{{ Environment }}CKANAppSubnet"
+    LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}"
+    AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}"
+    SSMKey: "{{ SSMKey | default('') }}"
+    InternalStackZone: "{{ Environment }}CKANPrivateHostedZone"
+    DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}"
+  tags: &common_stack_tags
+    Environment: "{{ Environment }}"
+    Service: "{{ service_name }}"
+    Division: "{{ Division }}"
+    Owner: "{{ Owner }}"
+    Version: "1.0"
+
+cloudformation_stacks:
+  - <<: *common_stack
+    name: "{{ service_name }}-{{ Environment }}-AMI-Template-Instances"
+    template_parameters:
+      <<: *common_stack_template_parameters
+      Environment: "{{ Environment }}"