From 4211e8ed587b349dfcc56ae62b70ce6971fc4fb5 Mon Sep 17 00:00:00 2001 From: antuarc Date: Thu, 28 Nov 2024 11:58:25 +1000 Subject: [PATCH] [QOLDEV-833] add AMI creation to build - Start template instances after defining the stack and plugins, but before launching autoscaling instances - Stop template instances and generate AMIs from them - Provide the AMI IDs to the autoscaling instance creation --- AMI-templates.yml | 12 +- build-CKAN.sh | 27 ++++- create-AMI.yml | 105 ++++++++++++++++++ inventory/hosts | 3 +- templates/AMI-Template-Instances.cfn.yml.j2 | 29 +++-- ...ashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 12 +- .../Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 | 10 ++ vars/AMI-template-instances.var.yml | 4 - vars/instances-CKANTest.var.yml | 9 +- vars/instances-OpenData.var.yml | 8 +- vars/instances-Publications.var.yml | 9 +- 11 files changed, 177 insertions(+), 51 deletions(-) create mode 100644 create-AMI.yml diff --git a/AMI-templates.yml b/AMI-templates.yml index 4c3c23d2..7d081a0b 100644 --- a/AMI-templates.yml +++ b/AMI-templates.yml @@ -23,17 +23,7 @@ region: "{{ region }}" register: ssmKeyFacts - - name: set KMS key from alias - set_fact: - SSMKey: "{{ ssmKeyFacts['keys'][0].key_arn }}" - - - name: Generate Lambda file hash - shell: > - md5sum files/instanceSetupLambda.js | awk '{print substr($1, 1, 20)}' - register: hash_output - - set_fact: - instance_setup_source_hash: "{{ hash_output.stdout_lines[0] }}" - - include_vars: vars/AMI-template-instances.var.yml + roles: - ansible_cloudformation diff --git a/build-CKAN.sh b/build-CKAN.sh index d0490fb6..c494048b 100755 --- a/build-CKAN.sh +++ b/build-CKAN.sh @@ -2,14 +2,12 @@ #deploy CKAN base infrastructure #ensure we die if any function fails -set -e +set -ex VARS_FILE="$1" ENVIRONMENT="$2" ANSIBLE_EXTRA_VARS="$ANSIBLE_EXTRA_VARS Environment=$ENVIRONMENT" -set -x - run-playbook () { if [ -z "$2" ]; then unset VARS_FILE_2 @@ -37,13 +35,24 @@ run-shared-resource-playbooks () { run-deployment () { run-playbook "chef-json" - ./chef-deploy.sh datashades::ckanweb-setup,datashades::ckanweb-deploy,datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$! + ./chef-deploy.sh datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$! # Check if the web deployment immediately failed kill -0 $WEB_PID - PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-setup,datashades::ckanbatch-deploy,datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$! + PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$! wait $WEB_PID wait $BATCH_PID - ./chef-deploy.sh datashades::solr-setup,datashades::solr-deploy,datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr || exit 1 + ./chef-deploy.sh datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr || exit 1 +} + +create-amis () { + ANSIBLE_EXTRA_VARS="$ANSIBLE_EXTRA_VARS timestamp=`date -u +%Y-%m-%dT%H:%M:%SZ`" + run-playbook "AMI-templates.yml" + run-playbook "create-AMI" "layer=Batch" & BATCH_PID=$! + run-playbook "create-AMI" "layer=Web" & WEB_PID=$! + run-playbook "create-AMI" "layer=Solr" + wait $BATCH_PID + wait $WEB_PID + run-playbook "AMI-templates.yml" "state=absent" } run-all-playbooks () { @@ -55,6 +64,10 @@ run-all-playbooks () { run-playbook "CloudFormation" "vars/s3_buckets.var.yml" run-playbook "CKAN-Stack" run-playbook "CloudFormation" "vars/CKAN-extensions.var.yml" + if ! (create-amis); then + ANSIBLE_EXTRA_VARS="$ANSIBLE_EXTRA_VARS state=absent" run-playbook "CloudFormation" "vars/AMI-template-instances.var.yml" || exit 1 + exit 1 + fi run-playbook "CloudFormation" "vars/instances-${INSTANCE_NAME}.var.yml" run-playbook "CloudFormation" "vars/cloudfront-lambda-at-edge.var.yml" run-playbook "cloudfront" @@ -78,6 +91,8 @@ if [ $# -ge 3 ]; then PARALLEL=1 ./chef-deploy.sh datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr wait $WEB_PID wait $BATCH_PID + elif [ "$3" = "create-amis" ]; then + create-amis else # run custom playbook run-playbook "$3" "$4" diff --git a/create-AMI.yml b/create-AMI.yml new file mode 100644 index 00000000..cee6e63a --- /dev/null +++ b/create-AMI.yml @@ -0,0 +1,105 @@ +--- +- name: Create Amazon Machine Images + hosts: local + connection: local + vars_files: + - vars/AMI-template-instances.var.yml + + pre_tasks: + - name: get basic_facts + set_fact: + basic_fact={{ item }} + when: item.Environment == Environment + with_items: "{{ basic_facts }}" + + - name: set facts to environment from basic_fact + set_fact: "{{ item.key }}={{ item.value }}" + with_dict: "{{ basic_fact }}" + when: basic_fact is defined + + tasks: + - name: Get stack name + set_fact: + stack_name: "{{ item.name }}" + when: item.template_parameters.Environment == Environment + with_items: "{{ cloudformation_stacks }}" + + - name: Get current stack facts + cloudformation_facts: + region: "{{ region }}" + stack_name: "{{ stack_name }}" + stack_resources: true + register: opsworks_facts + + - name: Set new instance facts + set_fact: + InstanceId: "{{ opsworks_facts.ansible_facts.cloudformation[stack_name].stack_outputs[layer + 'TemplateInstanceId'] }}" + InstanceName: "{{ Environment }}-{{ service_name_lower }}-{{ layer }}-image" + + - name: Wait for instance startup + shell: | + STATUS=$(aws ec2 describe-instances --region {{ region }} --instance-ids {{ InstanceId }} --query "Reservations[].Instances[].State.Name" --output text) || return 1 + echo "Instance {{ InstanceId }} status: $STATUS" >&2 + for retry in `seq 1 6`; do + if [ "$STATUS" = "pending" ]; then + sleep 10 + $(aws ec2 describe-instances --region {{ region }} --instance-ids {{ InstanceId }} --query "Reservations[].Instances[].State.Name" --output text) || return 1 + echo "Instance {{ InstanceId }} status: $STATUS" >&2 + else + break + fi + done + if [ "$STATUS" != "running" ]; then + echo "Failed to start {{ InstanceId }}, status $STATUS - aborting" >&2 + exit 2 + fi + + - name: Wait for instance configuration + shell: | + STATUS=$(aws ssm list-commands --region {{ region }} --instance-id {{ InstanceId }} \ + --filter "key=DocumentName,value=AWS-ApplyChefRecipes" --filter "key=InvokedAfter,value={{ timestamp }}" \ + --query "Commands|[0].Status" --output text) || exit 1 + echo "Deployment $DEPLOYMENT_ID: $STATUS" >&2 + for retry in `seq 1 180`; do + if [ "$STATUS" = "Pending" ] || [ "$STATUS" = "InProgress" ] || [ "$STATUS" = "None" ]; then + sleep 20 + STATUS=$(aws ssm list-commands --region {{ region }} --instance-id {{ InstanceId }} \ + --filter "key=DocumentName,value=AWS-ApplyChefRecipes" --filter "key=InvokedAfter,value={{ timestamp }}" \ + --query "Commands|[0].Status" --output text) || exit 1 + echo "Instance {{ InstanceId }} deployment status: $STATUS" >&2 + else + break + fi + done + if [ "$STATUS" != "Success" ]; then + echo "Failed to deploy $DEPLOYMENT_ID, status $STATUS - aborting" >&2 + exit 2 + fi + + - name: Stop instance + ec2: + instance_ids: + - "{{ InstanceId }}" + region: "{{ region }}" + state: stopped + wait: True + + - name: Create new ami + ec2_ami: + instance_id: "{{ InstanceId }}" + region: "{{ region }}" + wait: yes + name: "{{ InstanceName }}-{{ timestamp | replace(':', '-') }}" + description: "Base image for {{ Environment }} {{ service_name }} {{ layer }} layer" + tags: + Name: "{{ InstanceName }}-{{ timestamp | replace(':', '-') }}" + Environment: "{{ Environment }}" + Service: "{{ service_name }}" + Division: "{{ Division }}" + Owner: "{{ Owner }}" + Version: "1.0" + register: new_image + + - name: Record AMI ID + shell: | + aws ssm put-parameter --type String --name "/config/CKAN/{{ Environment }}/app/{{ service_name_lower }}/{{ layer }}AmiId" --value "{{ new_image.image_id }}" diff --git a/inventory/hosts b/inventory/hosts index 86ee9315..fa6de6dc 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -1,2 +1,3 @@ [local] -localhost region=ap-southeast-2 Owner="Development and Delivery" Division="Qld Online" +# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 +localhost region=ap-southeast-2 Owner="Development and Delivery" Division="Qld Online" base_ami="ami-0e326862c8e74c0fe" diff --git a/templates/AMI-Template-Instances.cfn.yml.j2 b/templates/AMI-Template-Instances.cfn.yml.j2 index 0157e272..05e4a939 100644 --- a/templates/AMI-Template-Instances.cfn.yml.j2 +++ b/templates/AMI-Template-Instances.cfn.yml.j2 @@ -46,8 +46,10 @@ Resources: DeleteOnTermination: true VolumeSize: {{ disk_size }} VolumeType: "gp2" - IamInstanceProfile: !Ref {% if layer != 'Solr' %}Web{% endif %}InstanceRoleProfile - ImageId: "ami-0d71fe73adf7a9887" + IamInstanceProfile: + Fn::ImportValue: !Sub "${Environment}${ApplicationName}{% if layer != 'Solr' %}Web{% endif %}InstanceProfileName" + # Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 + ImageId: "{{ base_ami }}" InstanceType: "t3a.small" KeyName: !Ref DefaultEC2Key NetworkInterfaces: @@ -67,13 +69,13 @@ Resources: echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab mount -a fi - if ! (yum install chef); then - for i in `seq 1 5`; do - yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break - sleep 5 - done - fi - REGION="--region ${AWS::Region}" + if ! (yum install chef); then + for i in `seq 1 5`; do + yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break + sleep 5 + done + fi + REGION="--region ${AWS::Region}" metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \ INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \ aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-ami-template" @@ -86,3 +88,12 @@ Resources: - Key: Layer Value: {{ layer|lower }} {% endfor %} + +Outputs: + +{% for layer in ['Batch', 'Web', 'Solr'] %} + {{ layer }}TemplateInstanceId: + Value: !Ref {{ layer }}TemplateInstance + Export: + Name: !Sub "${Environment}${ApplicationName}{{ layer }}TemplateInstanceId" +{% endfor %} diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 71ef6fe5..4a18c8b8 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -150,8 +150,8 @@ Resources: VolumeSize: {{ disk_size }} VolumeType: "gp2" IamInstanceProfile: - Arn: - Fn::ImportValue: !Sub "${Environment}${ApplicationName}{% if layer != 'Solr' %}Web{% endif %}InstanceProfile" + Name: + Fn::ImportValue: !Sub "${Environment}${ApplicationName}{% if layer != 'Solr' %}Web{% endif %}InstanceProfileName" ImageId: !Ref {{ layer }}ImageId InstanceType: !Ref {{ layer }}EC2Size KeyName: !Ref DefaultEC2Key @@ -182,7 +182,13 @@ Resources: if (aws --version |grep -o 'aws-cli/[2-9]'); then PAYLOAD_FORMAT="--cli-binary-format raw-in-base64-out" fi - aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "setup"}' /var/log/instance-setup.log.`date '+%s'` + MY_AMI_ID=`curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/ami-id` + if [ "$MY_AMI_ID" = "{{ base_ami }}" ]; then + TARGET_PHASE=setup + else + TARGET_PHASE=configure + fi + aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "'$TARGET_PHASE'"}' /var/log/instance-setup.log.`date '+%s'` {{ layer }}ScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup diff --git a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 index 25f8d6db..f5e6374a 100644 --- a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 @@ -630,6 +630,16 @@ Outputs: Export: Name: !Sub "${Environment}${ApplicationName}WebAlbTargetGroup" + InstanceProfileName: + Value: !Ref InstanceRoleProfile + Export: + Name: !Sub "${Environment}${ApplicationName}InstanceProfileName" + + WebInstanceProfileName: + Value: !Ref WebInstanceRoleProfile + Export: + Name: !Sub "${Environment}${ApplicationName}WebInstanceProfileName" + InstanceProfile: Value: !GetAtt InstanceRoleProfile.Arn Export: diff --git a/vars/AMI-template-instances.var.yml b/vars/AMI-template-instances.var.yml index 9748afb3..a0ff061d 100644 --- a/vars/AMI-template-instances.var.yml +++ b/vars/AMI-template-instances.var.yml @@ -11,10 +11,6 @@ common_stack: &common_stack ApplicationId: "{{ service_name_lower }}" Environment: "{{ Environment }}" AppSubnets: "{{ Environment }}CKANAppSubnet" - LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" - AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" - SSMKey: "{{ SSMKey | default('') }}" - InternalStackZone: "{{ Environment }}CKANPrivateHostedZone" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" tags: &common_stack_tags Environment: "{{ Environment }}" diff --git a/vars/instances-CKANTest.var.yml b/vars/instances-CKANTest.var.yml index 4cc2c7ae..b24dc015 100644 --- a/vars/instances-CKANTest.var.yml +++ b/vars/instances-CKANTest.var.yml @@ -1,7 +1,4 @@ --- -# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 -ami_al2023: "ami-0e326862c8e74c0fe" - common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -14,9 +11,9 @@ common_stack: &common_stack AppSubnets: "{{ Environment }}CKANAppSubnet" EnableDataStore: "{{ enable_datastore | default('no') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - BatchImageId: "{{ ami_al2023 }}" - WebImageId: "{{ ami_al2023 }}" - SolrImageId: "{{ ami_al2023 }}" + BatchImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/BatchAmiId') | default(base_ami, True) }}" + WebImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/WebAmiId') | default(base_ami, True) }}" + SolrImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/SolrAmiId') | default(base_ami, True) }}" tags: &common_stack_tags Environment: "{{ Environment }}" Service: "{{ service_name }}" diff --git a/vars/instances-OpenData.var.yml b/vars/instances-OpenData.var.yml index 14029443..111c00ee 100644 --- a/vars/instances-OpenData.var.yml +++ b/vars/instances-OpenData.var.yml @@ -1,7 +1,5 @@ --- # Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 -ami_al2023: "ami-0e326862c8e74c0fe" - common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -16,9 +14,9 @@ common_stack: &common_stack AppSubnets: "{{ Environment }}CKANAppSubnet" EnableDataStore: "{{ enable_datastore | default('no') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - BatchImageId: "{{ ami_al2023 }}" - WebImageId: "{{ ami_al2023 }}" - SolrImageId: "{{ ami_al2023 }}" + BatchImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/BatchAmiId') | default(base_ami, True) }}" + WebImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/WebAmiId') | default(base_ami, True) }}" + SolrImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/SolrAmiId') | default(base_ami, True) }}" tags: &common_stack_tags Environment: "{{ Environment }}" Service: "{{ service_name }}" diff --git a/vars/instances-Publications.var.yml b/vars/instances-Publications.var.yml index 530b2b94..b0abc546 100644 --- a/vars/instances-Publications.var.yml +++ b/vars/instances-Publications.var.yml @@ -1,7 +1,4 @@ --- -# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 -ami_al2023: "ami-0e326862c8e74c0fe" - common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -15,9 +12,9 @@ common_stack: &common_stack AppSubnets: "{{ Environment }}CKANAppSubnet" EnableDataStore: "{{ enable_datastore | default('no') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - BatchImageId: "{{ ami_al2023 }}" - WebImageId: "{{ ami_al2023 }}" - SolrImageId: "{{ ami_al2023 }}" + BatchImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/BatchAmiId') | default(base_ami, True) }}" + WebImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/WebAmiId') | default(base_ami, True) }}" + SolrImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/SolrAmiId') | default(base_ami, True) }}" tags: &common_stack_tags Environment: "{{ Environment }}" Service: "{{ service_name }}"