Skip to content

Commit

Permalink
[QOLDEV-833] add AMI creation to build
Browse files Browse the repository at this point in the history
- Start template instances after defining the stack and plugins, but before launching autoscaling instances
- Stop template instances and generate AMIs from them
- Provide the AMI IDs to the autoscaling instance creation
  • Loading branch information
ThrawnCA committed Nov 28, 2024
1 parent ae74558 commit f0cce6a
Show file tree
Hide file tree
Showing 11 changed files with 155 additions and 50 deletions.
12 changes: 1 addition & 11 deletions AMI-templates.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,7 @@
region: "{{ region }}"
register: ssmKeyFacts

- name: set KMS key from alias
set_fact:
SSMKey: "{{ ssmKeyFacts['keys'][0].key_arn }}"

- name: Generate Lambda file hash
shell: >
md5sum files/instanceSetupLambda.js | awk '{print substr($1, 1, 20)}'
register: hash_output
- set_fact:
instance_setup_source_hash: "{{ hash_output.stdout_lines[0] }}"

- include_vars: vars/AMI-template-instances.var.yml

roles:
- ansible_cloudformation
25 changes: 20 additions & 5 deletions build-CKAN.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ run-playbook () {
if [ -z "$2" ]; then
unset VARS_FILE_2
elif [ -e "$2" ]; then
VARS_FILE_2="--extra-vars @$2"
VARS_FILE_2='--extra-vars @"'"$2"'"'
else
VARS_FILE_2="--extra-vars $2"
VARS_FILE_2='--extra-vars "'"$2"'"'
fi
PLAYBOOK="$1"
if [ ! -e "$PLAYBOOK" ]; then
Expand All @@ -37,13 +37,24 @@ run-shared-resource-playbooks () {

run-deployment () {
run-playbook "chef-json"
./chef-deploy.sh datashades::ckanweb-setup,datashades::ckanweb-deploy,datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$!
./chef-deploy.sh datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$!
# Check if the web deployment immediately failed
kill -0 $WEB_PID
PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-setup,datashades::ckanbatch-deploy,datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$!
PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$!
wait $WEB_PID
wait $BATCH_PID
./chef-deploy.sh datashades::solr-setup,datashades::solr-deploy,datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr || exit 1
./chef-deploy.sh datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr || exit 1
}

create-amis () {
TIMESTAMP=`date +%Y-%m-%dT%H:%M:%S%Z`
run-playbook "AMI-templates.var.yml"
run-playbook "create-AMI" "layer=Batch timestamp=$TIMESTAMP" & BATCH_PID=$!
run-playbook "create-AMI" "layer=Web timestamp=$TIMESTAMP" & WEB_PID=$!
run-playbook "create-AMI" "layer=Solr timestamp=$TIMESTAMP"
wait $BATCH_PID
wait $WEB_PID
ANSIBLE_EXTRA_VARS="$ANSIBLE_EXTRA_VARS state=absent" run-playbook "CloudFormation" "vars/AMI-template-instances.var.yml"
}

run-all-playbooks () {
Expand All @@ -55,6 +66,10 @@ run-all-playbooks () {
run-playbook "CloudFormation" "vars/s3_buckets.var.yml"
run-playbook "CKAN-Stack"
run-playbook "CloudFormation" "vars/CKAN-extensions.var.yml"
if ! (create-amis); then
ANSIBLE_EXTRA_VARS="$ANSIBLE_EXTRA_VARS state=absent" run-playbook "CloudFormation" "vars/AMI-template-instances.var.yml"
exit 1
fi
run-playbook "CloudFormation" "vars/instances-${INSTANCE_NAME}.var.yml"
run-playbook "CloudFormation" "vars/cloudfront-lambda-at-edge.var.yml"
run-playbook "cloudfront"
Expand Down
84 changes: 84 additions & 0 deletions create-AMI.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
---
- name: Create Amazon Machine Images
hosts: local
connection: local

pre_tasks:
- name: get basic_facts
set_fact:
basic_fact={{ item }}
when: item.Environment == Environment
with_items: "{{ basic_facts }}"

- name: set facts to environment from basic_fact
set_fact: "{{ item.key }}={{ item.value }}"
with_dict: "{{ basic_fact }}"
when: basic_fact is defined

tasks:
- name: Get stack name
set_fact:
stack_name: "{{ item.name }}"
when: item.template_parameters.Environment == Environment
with_items: "{{ cloudformation_stacks }}"

- name: Get current stack facts
cloudformation_facts:
region: "{{ region }}"
stack_name: "{{ stack_name }}"
stack_resources: true
register: opsworks_facts

- name: Set new instance facts
set_fact:
InstanceId: "{{ opsworks_facts.stack_outputs[layer + 'TemplateInstanceId'] }}"
InstanceName: "{{ Environment }}-{{ service_name_lower }}-{{ layer }}-image"

- name: Wait for instance configuration
shell: |
STATUS=$(aws ssm list-commands --region {{ region }} --instance-id {{ InstanceId }}
--filter "key=DocumentName,value=AWS-ApplyChefRecipes" --filter "key=InvokedAfter,value={{ timestamp }}"
--query "Commands|[0].Status" --output text) || return 1
for retry in `seq 1 180`; do
if [ "$STATUS" = "Pending" ] || [ "$STATUS" = "InProgress" ]; then
sleep 20
STATUS=$(aws ssm list-commands --region {{ region }} --instance-id {{ InstanceId }}
--filter "key=DocumentName,value=AWS-ApplyChefRecipes" --filter "key=InvokedAfter,value={{ timestamp }}"
--query "Commands|[0].Status" --output text) || return 1
debug "Deployment $DEPLOYMENT_ID: $STATUS"
else
break
fi
done
if [ "$STATUS" != "Success" ]; then
debug "Failed to deploy $DEPLOYMENT_ID, status $STATUS - aborting"
return 1
fi
- name: Stop instance
ec2:
instance_ids:
- "{{ InstanceId }}"
region: "{{ Region }}"
state: stopped
wait: True

- name: Create new ami
ec2_ami:
instance_id: "{{ InstanceId }}"
region: "{{ Region }}"
wait: yes
name: "{{ InstanceName }}-{{ timestamp }}"
description: "Base image for {{ Environment }} {{ service_name }} {{ layer }} layer"
tags:
Environment: "{{ Environment }}"
Service: "{{ service_name }}"
Division: "{{ Division }}"
Owner: "{{ Owner }}"
Version: "1.0"
register: new_image

- name: Record AMI ID
aws_ssm_parameter:
name: "/config/CKAN/{{ Environment }}/app/{{ service_name_lower }}/{{ layer }}AmiId"
value: "{{ new_image.image_id }}"
3 changes: 2 additions & 1 deletion inventory/hosts
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
[local]
localhost region=ap-southeast-2 Owner="Development and Delivery" Division="Qld Online"
# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64
localhost region=ap-southeast-2 Owner="Development and Delivery" Division="Qld Online" base_ami="ami-0e326862c8e74c0fe"
29 changes: 20 additions & 9 deletions templates/AMI-Template-Instances.cfn.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,10 @@ Resources:
DeleteOnTermination: true
VolumeSize: {{ disk_size }}
VolumeType: "gp2"
IamInstanceProfile: !Ref {% if layer != 'Solr' %}Web{% endif %}InstanceRoleProfile
ImageId: "ami-0d71fe73adf7a9887"
IamInstanceProfile:
Fn::ImportValue: !Sub "${Environment}${ApplicationName}{% if layer != 'Solr' %}Web{% endif %}InstanceProfileName"
# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64
ImageId: "{{ base_ami }}"
InstanceType: "t3a.small"
KeyName: !Ref DefaultEC2Key
NetworkInterfaces:
Expand All @@ -67,13 +69,13 @@ Resources:
echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab
mount -a
fi
if ! (yum install chef); then
for i in `seq 1 5`; do
yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break
sleep 5
done
fi
REGION="--region ${AWS::Region}"
if ! (yum install chef); then
for i in `seq 1 5`; do
yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break
sleep 5
done
fi
REGION="--region ${AWS::Region}"
metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \
INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \
aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-ami-template"
Expand All @@ -86,3 +88,12 @@ Resources:
- Key: Layer
Value: {{ layer|lower }}
{% endfor %}

Outputs:

{% for layer in ['Batch', 'Web', 'Solr'] %}
{{ layer }}TemplateInstanceId:
Value: !Ref {{ layer }}TemplateInstance
Export:
Name: !Sub "${Environment}${ApplicationName}{{ layer }}TemplateInstanceId"
{% endfor %}
12 changes: 9 additions & 3 deletions templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,8 @@ Resources:
VolumeSize: {{ disk_size }}
VolumeType: "gp2"
IamInstanceProfile:
Arn:
Fn::ImportValue: !Sub "${Environment}${ApplicationName}{% if layer != 'Solr' %}Web{% endif %}InstanceProfile"
Name:
Fn::ImportValue: !Sub "${Environment}${ApplicationName}{% if layer != 'Solr' %}Web{% endif %}InstanceProfileName"
ImageId: !Ref {{ layer }}ImageId
InstanceType: !Ref {{ layer }}EC2Size
KeyName: !Ref DefaultEC2Key
Expand Down Expand Up @@ -182,7 +182,13 @@ Resources:
if (aws --version |grep -o 'aws-cli/[2-9]'); then
PAYLOAD_FORMAT="--cli-binary-format raw-in-base64-out"
fi
aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "setup"}' /var/log/instance-setup.log.`date '+%s'`
MY_AMI_ID=`curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/ami-id`
if [ "$MY_AMI_ID" = "{{ base_ami }}" ]; then
TARGET_PHASE=setup
else
TARGET_PHASE=configure
fi
aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "'$TARGET_PHASE'"}' /var/log/instance-setup.log.`date '+%s'`

{{ layer }}ScalingGroup:
Type: AWS::AutoScaling::AutoScalingGroup
Expand Down
10 changes: 10 additions & 0 deletions templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -630,6 +630,16 @@ Outputs:
Export:
Name: !Sub "${Environment}${ApplicationName}WebAlbTargetGroup"

InstanceProfileName:
Value: !Ref InstanceRoleProfile
Export:
Name: !Sub "${Environment}${ApplicationName}InstanceProfileName"

WebInstanceProfileName:
Value: !Ref WebInstanceRoleProfile
Export:
Name: !Sub "${Environment}${ApplicationName}WebInstanceProfileName"

InstanceProfile:
Value: !GetAtt InstanceRoleProfile.Arn
Export:
Expand Down
4 changes: 0 additions & 4 deletions vars/AMI-template-instances.var.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,6 @@ common_stack: &common_stack
ApplicationId: "{{ service_name_lower }}"
Environment: "{{ Environment }}"
AppSubnets: "{{ Environment }}CKANAppSubnet"
LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}"
AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}"
SSMKey: "{{ SSMKey | default('') }}"
InternalStackZone: "{{ Environment }}CKANPrivateHostedZone"
DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}"
tags: &common_stack_tags
Environment: "{{ Environment }}"
Expand Down
9 changes: 3 additions & 6 deletions vars/instances-CKANTest.var.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
---
# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64
ami_al2023: "ami-0e326862c8e74c0fe"

common_stack: &common_stack
state: "{{ state | default('present')}}"
region: "{{ region }}"
Expand All @@ -14,9 +11,9 @@ common_stack: &common_stack
AppSubnets: "{{ Environment }}CKANAppSubnet"
EnableDataStore: "{{ enable_datastore | default('no') }}"
DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}"
BatchImageId: "{{ ami_al2023 }}"
WebImageId: "{{ ami_al2023 }}"
SolrImageId: "{{ ami_al2023 }}"
BatchImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/BatchAmiId') | default(base_ami, True) }}"
WebImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/WebAmiId') | default(base_ami, True) }}"
SolrImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/SolrAmiId') | default(base_ami, True) }}"
tags: &common_stack_tags
Environment: "{{ Environment }}"
Service: "{{ service_name }}"
Expand Down
8 changes: 3 additions & 5 deletions vars/instances-OpenData.var.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
---
# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64
ami_al2023: "ami-0e326862c8e74c0fe"

common_stack: &common_stack
state: "{{ state | default('present')}}"
region: "{{ region }}"
Expand All @@ -16,9 +14,9 @@ common_stack: &common_stack
AppSubnets: "{{ Environment }}CKANAppSubnet"
EnableDataStore: "{{ enable_datastore | default('no') }}"
DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}"
BatchImageId: "{{ ami_al2023 }}"
WebImageId: "{{ ami_al2023 }}"
SolrImageId: "{{ ami_al2023 }}"
BatchImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/BatchAmiId') | default(base_ami, True) }}"
WebImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/WebAmiId') | default(base_ami, True) }}"
SolrImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/SolrAmiId') | default(base_ami, True) }}"
tags: &common_stack_tags
Environment: "{{ Environment }}"
Service: "{{ service_name }}"
Expand Down
9 changes: 3 additions & 6 deletions vars/instances-Publications.var.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
---
# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64
ami_al2023: "ami-0e326862c8e74c0fe"

common_stack: &common_stack
state: "{{ state | default('present')}}"
region: "{{ region }}"
Expand All @@ -15,9 +12,9 @@ common_stack: &common_stack
AppSubnets: "{{ Environment }}CKANAppSubnet"
EnableDataStore: "{{ enable_datastore | default('no') }}"
DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}"
BatchImageId: "{{ ami_al2023 }}"
WebImageId: "{{ ami_al2023 }}"
SolrImageId: "{{ ami_al2023 }}"
BatchImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/BatchAmiId') | default(base_ami, True) }}"
WebImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/WebAmiId') | default(base_ami, True) }}"
SolrImageId: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/SolrAmiId') | default(base_ami, True) }}"
tags: &common_stack_tags
Environment: "{{ Environment }}"
Service: "{{ service_name }}"
Expand Down

0 comments on commit f0cce6a

Please sign in to comment.