From 9227bcba7f0f13bc401eee0df76fa82a42fa6b89 Mon Sep 17 00:00:00 2001 From: antuarc Date: Thu, 28 Nov 2024 11:58:25 +1000 Subject: [PATCH] [QOLDEV-833] add AMI creation to build - Start template instances after defining the stack and plugins, but before launching autoscaling instances - Stop template instances and generate AMIs from them - Provide the AMI IDs to the autoscaling instance creation --- AMI-templates.yml | 12 +-- build-CKAN.sh | 14 +++ create-AMI.yml | 87 +++++++++++++++++++ inventory/hosts | 3 +- templates/AMI-Template-Instances.cfn.yml.j2 | 29 +++++-- ...ashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 4 +- .../Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 | 10 +++ vars/AMI-template-instances.var.yml | 4 - vars/instances-CKANTest.var.yml | 9 +- vars/instances-OpenData.var.yml | 8 +- vars/instances-Publications.var.yml | 9 +- 11 files changed, 145 insertions(+), 44 deletions(-) create mode 100644 create-AMI.yml diff --git a/AMI-templates.yml b/AMI-templates.yml index 4c3c23d2..7d081a0b 100644 --- a/AMI-templates.yml +++ b/AMI-templates.yml @@ -23,17 +23,7 @@ region: "{{ region }}" register: ssmKeyFacts - - name: set KMS key from alias - set_fact: - SSMKey: "{{ ssmKeyFacts['keys'][0].key_arn }}" - - - name: Generate Lambda file hash - shell: > - md5sum files/instanceSetupLambda.js | awk '{print substr($1, 1, 20)}' - register: hash_output - - set_fact: - instance_setup_source_hash: "{{ hash_output.stdout_lines[0] }}" - - include_vars: vars/AMI-template-instances.var.yml + roles: - ansible_cloudformation diff --git a/build-CKAN.sh b/build-CKAN.sh index d0490fb6..a45b504e 100755 --- a/build-CKAN.sh +++ b/build-CKAN.sh @@ -46,6 +46,16 @@ run-deployment () { ./chef-deploy.sh datashades::solr-setup,datashades::solr-deploy,datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr || exit 1 } +create-amis () { + run-playbook "AMI-templates.var.yml" + run-playbook "create-AMI" "layer=Batch" & BATCH_PID=$! + run-playbook "create-AMI" "layer=Web" & WEB_PID=$! + run-playbook "create-AMI" "layer=Solr" + wait $BATCH_PID + wait $WEB_PID + ANSIBLE_EXTRA_VARS="$ANSIBLE_EXTRA_VARS state=absent" run-playbook "CloudFormation" "vars/AMI-template-instances.var.yml" +} + run-all-playbooks () { run-shared-resource-playbooks run-playbook "CloudFormation" "vars/acm.var.yml" @@ -55,6 +65,10 @@ run-all-playbooks () { run-playbook "CloudFormation" "vars/s3_buckets.var.yml" run-playbook "CKAN-Stack" run-playbook "CloudFormation" "vars/CKAN-extensions.var.yml" + if ! (create-amis); then + ANSIBLE_EXTRA_VARS="$ANSIBLE_EXTRA_VARS state=absent" run-playbook "CloudFormation" "vars/AMI-template-instances.var.yml" + exit 1 + fi run-playbook "CloudFormation" "vars/instances-${INSTANCE_NAME}.var.yml" run-playbook "CloudFormation" "vars/cloudfront-lambda-at-edge.var.yml" run-playbook "cloudfront" diff --git a/create-AMI.yml b/create-AMI.yml new file mode 100644 index 00000000..b6f82126 --- /dev/null +++ b/create-AMI.yml @@ -0,0 +1,87 @@ +--- +- name: Create Amazon Machine Images + hosts: local + connection: local + + pre_tasks: + - name: get basic_facts + set_fact: + basic_fact={{ item }} + when: item.Environment == Environment + with_items: "{{ basic_facts }}" + + - name: set facts to environment from basic_fact + set_fact: "{{ item.key }}={{ item.value }}" + with_dict: "{{ basic_fact }}" + when: basic_fact is defined + + - name: Set Timestamp fact + set_fact: + timestamp: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S%Z') }}" + + tasks: + - name: Get stack name + set_fact: + stack_name: "{{ item.name }}" + when: item.template_parameters.Environment == Environment + with_items: "{{ cloudformation_stacks }}" + + - name: Get current stack facts + cloudformation_facts: + region: "{{ region }}" + stack_name: "{{ stack_name }}" + stack_resources: true + register: opsworks_facts + + - name: Set new instance facts + set_fact: + InstanceId: "{{ opsworks_facts.stack_outputs[layer + 'TemplateInstanceId'] }}" + InstanceName: "{{ Environment }}-{{ service_name_lower }}-{{ layer }}-image" + + - name: Wait for instance configuration + shell: | + STATUS=$(aws ssm list-commands --region {{ region }} --instance-id {{ InstanceId }} + --filter "key=DocumentName,value=AWS-ApplyChefRecipes" --filter "key=InvokedAfter,value={{ timestamp }}" + --query "Commands|[0].Status" --output text) || return 1 + for retry in `seq 1 180`; do + if [ "$STATUS" = "Pending" ] || [ "$STATUS" = "InProgress" ]; then + sleep 20 + STATUS=$(aws ssm list-commands --region {{ region }} --instance-id {{ InstanceId }} + --filter "key=DocumentName,value=AWS-ApplyChefRecipes" --filter "key=InvokedAfter,value={{ timestamp }}" + --query "Commands|[0].Status" --output text) || return 1 + debug "Deployment $DEPLOYMENT_ID: $STATUS" + else + break + fi + done + if [ "$STATUS" != "Success" ]; then + debug "Failed to deploy $DEPLOYMENT_ID, status $STATUS - aborting" + return 1 + fi + + - name: Stop instance + ec2: + instance_ids: + - "{{ InstanceId }}" + region: "{{ Region }}" + state: stopped + wait: True + + - name: Create new ami + ec2_ami: + instance_id: "{{ InstanceId }}" + region: "{{ Region }}" + wait: yes + name: "{{ InstanceName }}-{{ timestamp }}" + description: "Base image for {{ Environment }} {{ service_name }} {{ layer }} layer" + tags: + Environment: "{{ Environment }}" + Service: "{{ service_name }}" + Division: "{{ Division }}" + Owner: "{{ Owner }}" + Version: "1.0" + register: new_image + + - name: Set new AmiId fact + set_fact: + "{{ layer }}AmiId: "{{ new_image.image_id }}" diff --git a/inventory/hosts b/inventory/hosts index 86ee9315..fa6de6dc 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -1,2 +1,3 @@ [local] -localhost region=ap-southeast-2 Owner="Development and Delivery" Division="Qld Online" +# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 +localhost region=ap-southeast-2 Owner="Development and Delivery" Division="Qld Online" base_ami="ami-0e326862c8e74c0fe" diff --git a/templates/AMI-Template-Instances.cfn.yml.j2 b/templates/AMI-Template-Instances.cfn.yml.j2 index 0157e272..05e4a939 100644 --- a/templates/AMI-Template-Instances.cfn.yml.j2 +++ b/templates/AMI-Template-Instances.cfn.yml.j2 @@ -46,8 +46,10 @@ Resources: DeleteOnTermination: true VolumeSize: {{ disk_size }} VolumeType: "gp2" - IamInstanceProfile: !Ref {% if layer != 'Solr' %}Web{% endif %}InstanceRoleProfile - ImageId: "ami-0d71fe73adf7a9887" + IamInstanceProfile: + Fn::ImportValue: !Sub "${Environment}${ApplicationName}{% if layer != 'Solr' %}Web{% endif %}InstanceProfileName" + # Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 + ImageId: "{{ base_ami }}" InstanceType: "t3a.small" KeyName: !Ref DefaultEC2Key NetworkInterfaces: @@ -67,13 +69,13 @@ Resources: echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab mount -a fi - if ! (yum install chef); then - for i in `seq 1 5`; do - yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break - sleep 5 - done - fi - REGION="--region ${AWS::Region}" + if ! (yum install chef); then + for i in `seq 1 5`; do + yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break + sleep 5 + done + fi + REGION="--region ${AWS::Region}" metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \ INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \ aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-ami-template" @@ -86,3 +88,12 @@ Resources: - Key: Layer Value: {{ layer|lower }} {% endfor %} + +Outputs: + +{% for layer in ['Batch', 'Web', 'Solr'] %} + {{ layer }}TemplateInstanceId: + Value: !Ref {{ layer }}TemplateInstance + Export: + Name: !Sub "${Environment}${ApplicationName}{{ layer }}TemplateInstanceId" +{% endfor %} diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 71ef6fe5..5a8fb1af 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -150,8 +150,8 @@ Resources: VolumeSize: {{ disk_size }} VolumeType: "gp2" IamInstanceProfile: - Arn: - Fn::ImportValue: !Sub "${Environment}${ApplicationName}{% if layer != 'Solr' %}Web{% endif %}InstanceProfile" + Name: + Fn::ImportValue: !Sub "${Environment}${ApplicationName}{% if layer != 'Solr' %}Web{% endif %}InstanceProfileName" ImageId: !Ref {{ layer }}ImageId InstanceType: !Ref {{ layer }}EC2Size KeyName: !Ref DefaultEC2Key diff --git a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 index 25f8d6db..f5e6374a 100644 --- a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 @@ -630,6 +630,16 @@ Outputs: Export: Name: !Sub "${Environment}${ApplicationName}WebAlbTargetGroup" + InstanceProfileName: + Value: !Ref InstanceRoleProfile + Export: + Name: !Sub "${Environment}${ApplicationName}InstanceProfileName" + + WebInstanceProfileName: + Value: !Ref WebInstanceRoleProfile + Export: + Name: !Sub "${Environment}${ApplicationName}WebInstanceProfileName" + InstanceProfile: Value: !GetAtt InstanceRoleProfile.Arn Export: diff --git a/vars/AMI-template-instances.var.yml b/vars/AMI-template-instances.var.yml index 9748afb3..a0ff061d 100644 --- a/vars/AMI-template-instances.var.yml +++ b/vars/AMI-template-instances.var.yml @@ -11,10 +11,6 @@ common_stack: &common_stack ApplicationId: "{{ service_name_lower }}" Environment: "{{ Environment }}" AppSubnets: "{{ Environment }}CKANAppSubnet" - LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" - AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" - SSMKey: "{{ SSMKey | default('') }}" - InternalStackZone: "{{ Environment }}CKANPrivateHostedZone" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" tags: &common_stack_tags Environment: "{{ Environment }}" diff --git a/vars/instances-CKANTest.var.yml b/vars/instances-CKANTest.var.yml index 4cc2c7ae..7315448d 100644 --- a/vars/instances-CKANTest.var.yml +++ b/vars/instances-CKANTest.var.yml @@ -1,7 +1,4 @@ --- -# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 -ami_al2023: "ami-0e326862c8e74c0fe" - common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -14,9 +11,9 @@ common_stack: &common_stack AppSubnets: "{{ Environment }}CKANAppSubnet" EnableDataStore: "{{ enable_datastore | default('no') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - BatchImageId: "{{ ami_al2023 }}" - WebImageId: "{{ ami_al2023 }}" - SolrImageId: "{{ ami_al2023 }}" + BatchImageId: "{{ BatchAmiId | default(base_ami) }}" + WebImageId: "{{ WebAmiId | default(base_ami) }}" + SolrImageId: "{{ SolrAmiId | default(base_ami) }}" tags: &common_stack_tags Environment: "{{ Environment }}" Service: "{{ service_name }}" diff --git a/vars/instances-OpenData.var.yml b/vars/instances-OpenData.var.yml index 14029443..b6e68d34 100644 --- a/vars/instances-OpenData.var.yml +++ b/vars/instances-OpenData.var.yml @@ -1,7 +1,5 @@ --- # Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 -ami_al2023: "ami-0e326862c8e74c0fe" - common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -16,9 +14,9 @@ common_stack: &common_stack AppSubnets: "{{ Environment }}CKANAppSubnet" EnableDataStore: "{{ enable_datastore | default('no') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - BatchImageId: "{{ ami_al2023 }}" - WebImageId: "{{ ami_al2023 }}" - SolrImageId: "{{ ami_al2023 }}" + BatchImageId: "{{ BatchAmiId | default(base_ami) }}" + WebImageId: "{{ WebAmiId | default(base_ami) }}" + SolrImageId: "{{ SolrAmiId | default(base_ami) }}" tags: &common_stack_tags Environment: "{{ Environment }}" Service: "{{ service_name }}" diff --git a/vars/instances-Publications.var.yml b/vars/instances-Publications.var.yml index 530b2b94..baf61fb9 100644 --- a/vars/instances-Publications.var.yml +++ b/vars/instances-Publications.var.yml @@ -1,7 +1,4 @@ --- -# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 -ami_al2023: "ami-0e326862c8e74c0fe" - common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -15,9 +12,9 @@ common_stack: &common_stack AppSubnets: "{{ Environment }}CKANAppSubnet" EnableDataStore: "{{ enable_datastore | default('no') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - BatchImageId: "{{ ami_al2023 }}" - WebImageId: "{{ ami_al2023 }}" - SolrImageId: "{{ ami_al2023 }}" + BatchImageId: "{{ BatchAmiId | default(base_ami) }}" + WebImageId: "{{ WebAmiId | default(base_ami) }}" + SolrImageId: "{{ SolrAmiId | default(base_ami) }}" tags: &common_stack_tags Environment: "{{ Environment }}" Service: "{{ service_name }}"