[QOLDEV-833] add AMI creation to build

- Start template instances after defining the stack and plugins, but before launching autoscaling instances
- Stop template instances and generate AMIs from them
- Provide the AMI IDs to the autoscaling instance creation

build-CKAN.sh

@@ -46,6 +46,17 @@ run-deployment () {
   ./chef-deploy.sh datashades::solr-setup,datashades::solr-deploy,datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr  || exit 1
 }
 
+create-amis () {
+  run-playbook "CloudFormation" "vars/AMI-template-instances.var.yml"
+  PARALLEL=1 ./chef-deploy.sh datashades::ckanweb-setup,datashades::ckanweb-deploy $INSTANCE_NAME $ENVIRONMENT & AMI_SETUP_PID=$!
+  wait $AMI_SETUP_PID
+  run-playbook "create-AMI" "layer=Batch" & BATCH_PID=$!
+  run-playbook "create-AMI" "layer=Web" & WEB_PID=$!
+  run-playbook "create-AMI" "layer=Solr"
+  wait $BATCH_PID
+  wait $WEB_PID
+}
+
 run-all-playbooks () {
   run-shared-resource-playbooks
   run-playbook "CloudFormation" "vars/acm.var.yml"
@@ -55,6 +66,7 @@ run-all-playbooks () {
   run-playbook "CloudFormation" "vars/s3_buckets.var.yml"
   run-playbook "CKAN-Stack"
   run-playbook "CloudFormation" "vars/CKAN-extensions.var.yml"
+  create-amis
   run-playbook "CloudFormation" "vars/instances-${INSTANCE_NAME}.var.yml"
   run-playbook "CloudFormation" "vars/cloudfront-lambda-at-edge.var.yml"
   run-playbook "cloudfront"

create-AMI.yml

@@ -0,0 +1,66 @@
+---
+- name: Create Amazon Machine Images
+  hosts: local
+  connection: local
+
+  pre_tasks:
+    - name: get basic_facts
+      set_fact:
+        basic_fact={{ item }}
+      when: item.Environment == Environment
+      with_items: "{{ basic_facts }}"
+
+    - name: set facts to environment from basic_fact
+      set_fact: "{{ item.key }}={{ item.value }}"
+      with_dict: "{{ basic_fact }}"
+      when: basic_fact is defined
+
+  tasks:
+    - name: Get stack name
+      set_fact:
+        stack_name: "{{ item.name }}"
+      when: item.template_parameters.Environment == Environment
+      with_items: "{{ cloudformation_stacks }}"
+
+    - name: Get current stack facts
+      cloudformation_facts:
+        region: "{{ region }}"
+        stack_name: "{{ stack_name }}"
+        stack_resources: true
+      register: opsworks_facts
+
+    - name: Set new instance facts
+      set_fact:
+          InstanceId: "{{ opsworks_facts.stack_outputs[layer + 'TemplateInstanceId'] }}"
+          InstanceName: "{{ Environment }}-{{ service_name_lower }}-{{ layer }}-image"
+
+    - name: Set Timestamp fact
+      set_fact:
+        timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
+    - name: Stop instance
+      ec2:
+        instance_ids:
+          - "{{ InstanceId }}"
+        region: "{{ Region }}"
+        state: stopped
+        wait: True
+
+    - name: Create new ami
+      ec2_ami:
+        instance_id: "{{ InstanceId }}"
+        region: "{{ Region }}"
+        wait: yes
+        name: "{{ InstanceName }}-{{ timestamp }}"
+        description: "Base image for {{ Environment }} {{ service_name }} {{ layer }} layer"
+        tags:
+          Environment: "{{ Environment }}"
+          Service: "{{ service_name }}"
+          Division: "{{ Division }}"
+          Owner: "{{ Owner }}"
+          Version: "1.0"
+      register: new_image
+
+    - name: Set new AmiId fact
+      set_fact:
+        "{{ layer }}AmiId: "{{ new_image.image_id }}"

templates/AMI-Template-Instances.cfn.yml.j2

@@ -67,22 +67,20 @@ Resources:
               echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab
               mount -a
             fi
-              if ! (yum install chef); then
-                for i in `seq 1 5`; do
-                  yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break
-                  sleep 5
-                done
-              fi
-              REGION="--region ${AWS::Region}"
-            metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \
-            INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \
-            aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-ami-template"
-            FUNCTION_NAME=$(aws ssm get-parameter $REGION --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text)
-            if (aws --version |grep -o 'aws-cli/[2-9]'); then
-              PAYLOAD_FORMAT="--cli-binary-format raw-in-base64-out"
+            if ! (yum install chef); then
+              for i in `seq 1 5`; do
+                yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break
+                sleep 5
+              done
             fi
-            aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "deploy"}' /var/log/instance-setup.log.`date '+%s'`
       Tags:
         - Key: Layer
           Value: {{ layer|lower }}
 {% endfor %}
+
+{% for layer in ['Batch', 'Web', 'Solr'] %}
+  {{ layer }}TemplateInstanceId:
+    Value: !Ref {{ layer }}TemplateInstance
+    Export:
+      Name: !Sub "${Environment}${ApplicationName}{{ layer }}TemplateInstanceId"
+{% endfor %}

vars/AMI-template-instances.var.yml

@@ -12,9 +12,6 @@ common_stack: &common_stack
     Environment: "{{ Environment }}"
     AppSubnets: "{{ Environment }}CKANAppSubnet"
     LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}"
-    AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}"
-    SSMKey: "{{ SSMKey | default('') }}"
-    InternalStackZone: "{{ Environment }}CKANPrivateHostedZone"
     DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}"
   tags: &common_stack_tags
     Environment: "{{ Environment }}"

vars/instances-CKANTest.var.yml

@@ -14,9 +14,9 @@ common_stack: &common_stack
     AppSubnets: "{{ Environment }}CKANAppSubnet"
     EnableDataStore: "{{ enable_datastore | default('no') }}"
     DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}"
-    BatchImageId: "{{ ami_al2023 }}"
-    WebImageId: "{{ ami_al2023 }}"
-    SolrImageId: "{{ ami_al2023 }}"
+    BatchImageId: "{{ BatchAmiId | default(ami_al2023) }}"
+    WebImageId: "{{ WebAmiId | default(ami_al2023) }}"
+    SolrImageId: "{{ SolrAmiId | default(ami_al2023) }}"
   tags: &common_stack_tags
     Environment: "{{ Environment }}"
     Service: "{{ service_name }}"

vars/instances-OpenData.var.yml

@@ -16,9 +16,9 @@ common_stack: &common_stack
     AppSubnets: "{{ Environment }}CKANAppSubnet"
     EnableDataStore: "{{ enable_datastore | default('no') }}"
     DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}"
-    BatchImageId: "{{ ami_al2023 }}"
-    WebImageId: "{{ ami_al2023 }}"
-    SolrImageId: "{{ ami_al2023 }}"
+    BatchImageId: "{{ BatchAmiId | default(ami_al2023) }}"
+    WebImageId: "{{ WebAmiId | default(ami_al2023) }}"
+    SolrImageId: "{{ SolrAmiId | default(ami_al2023) }}"
   tags: &common_stack_tags
     Environment: "{{ Environment }}"
     Service: "{{ service_name }}"

vars/instances-Publications.var.yml

@@ -15,9 +15,9 @@ common_stack: &common_stack
     AppSubnets: "{{ Environment }}CKANAppSubnet"
     EnableDataStore: "{{ enable_datastore | default('no') }}"
     DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}"
-    BatchImageId: "{{ ami_al2023 }}"
-    WebImageId: "{{ ami_al2023 }}"
-    SolrImageId: "{{ ami_al2023 }}"
+    BatchImageId: "{{ BatchAmiId | default(ami_al2023) }}"
+    WebImageId: "{{ WebAmiId | default(ami_al2023) }}"
+    SolrImageId: "{{ SolrAmiId | default(ami_al2023) }}"
   tags: &common_stack_tags
     Environment: "{{ Environment }}"
     Service: "{{ service_name }}"