From 399ce03e783f03f08f064070aa25bc43b4b462d5 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 8 May 2024 15:53:17 +1000 Subject: [PATCH 01/22] [QOLDEV-833] use bare-bones custom AMI - Shared yum packages are preinstalled, 4GB swapfile has been created, Supervisord service enabled. No environment-specific changes have been made. --- templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 28a50a0a..ccedb8e0 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -113,15 +113,15 @@ Parameters: BatchImageId: Description: The Amazon Machine Image ID to use for launching batch instances. Defaults to Amazon Linux 2. Type: String - Default: "ami-03b836d87d294e89e" + Default: "ami-0d71fe73adf7a9887" WebImageId: Description: The Amazon Machine Image ID to use for launching web instances. Defaults to Amazon Linux 2. Type: String - Default: "ami-03b836d87d294e89e" + Default: "ami-0d71fe73adf7a9887" SolrImageId: Description: The Amazon Machine Image ID to use for launching Solr instances. Defaults to Amazon Linux 2. Type: String - Default: "ami-03b836d87d294e89e" + Default: "ami-0d71fe73adf7a9887" DefaultEC2Key: Description: Select an existing SSH key Type: AWS::EC2::KeyPair::KeyName From fb5c54686ba8ff21f992ba1e1816e9cc4ee0c483 Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 17 May 2024 15:56:25 +1000 Subject: [PATCH 02/22] [QOLDEV-819] exclude Solr instances from power management - Recreating Solr instances doesn't preserve the index. TODO Fix index sync so new instances can pick up the latest index. --- templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 59084cb9..997740e7 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -209,7 +209,7 @@ Resources: Value: {{ layer|lower }} PropagateAtLaunch: true -{% if item.tags["PowerManaged"] == "Yes" %} +{% if item.tags["PowerManaged"] == "Yes" and layer != 'Solr' %} {{ layer }}ScalingIn: Type: AWS::AutoScaling::ScheduledAction Properties: From 1b3ef2c8b6a7dda686aca9222a6f0b3a564c461e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 21 May 2024 11:59:27 +1000 Subject: [PATCH 03/22] [QOLDEV-839] drop OpsWorks resources (stack and layers) and documentation references - Can't rename CloudFormation stacks at this point, or else they will be deleted and recreated, which would be unnecessarily disruptive --- README.md | 10 +- ...shades-OpsWorks-CKAN-Extensions.cfn.yml.j2 | 6 +- ...ashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 2 +- .../Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 | 269 +----------------- templates/hosted-zone.cfn.yml | 2 +- vars/CKAN-Stack.var.yml | 1 - 6 files changed, 9 insertions(+), 281 deletions(-) diff --git a/README.md b/README.md index 58a7dc17..31162f8d 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ Author: qol.development@smartservice.qld.gov.au -**Full one click deployment of Datashared AWS OpsWorks Stack via Ansible** +**Full one click deployment of CKAN AWS Stack via Ansible** This stands up the www.data.qld.gov.au and www.publications.qld.gov.au aws stacks using: * SSM * RDS * Redis Cluster -* OpsWorks +* EC2 Autoscaling * Cloudfront with Lambda@Edge * and may more features @@ -25,7 +25,7 @@ This stands up the www.data.qld.gov.au and www.publications.qld.gov.au aws stack For the system to work with updates to the lambda function, you must; * first add a new version to cloudfront-lambdaAtEdge.cfn.yml which references the changed lambda function (there is no need for a new lambda function) * export the new version from said cloudformation template -* update the cloudfront.yml ansible script to load the new version property name. +* update the cloudfront.yml ansible script to load the new version property name. * you can delete previous versions after a successful real, do note that cloudfront will hold onto a lambda function and versions until its 'replication' finishes. **QOL 2019 update** @@ -60,7 +60,7 @@ Common issues during set up are as follows: It's assumed that you: -* have a pretty good working knowledge of AWS, CloudFormation, OpsWorks, and CKAN and its requirements such as Solr and Postgres. Those will be necessary to troubleshoot builds when you haven't provided the correct parameters or some other obstacle gets in your way. +* have a pretty good working knowledge of AWS, CloudFormation, EC2 Autoscaling, and CKAN and its requirements such as Solr and Postgres. Those will be necessary to troubleshoot builds when you haven't provided the correct parameters or some other obstacle gets in your way. * have built, installed and successfully run CKAN manually on some kind of single node configuration. If not, this stack isn't designed to be something to cut your teeth on. It's been designed to be relatively foolproof, but not completely so. * know your way around the Linux command line reasonably well and know how to deal with error logs, dependency conflicts etc. @@ -118,7 +118,7 @@ and automated system maintenance. Our hope and expectation is that it benefits the wider Public Data community and progresses the Open Data ideal. -Current AWS costs for 2 CKAN applications by 4 envirionments is just shy of 3k USD a month. +Current AWS costs for 2 CKAN applications by 4 environments is just shy of 3k USD a month. ## TODO ## Make requirements-dev look up vars/shared-${app}.var.yml and test all environment plugins diff --git a/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 index 1f0cf96f..8d9db161 100644 --- a/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 @@ -1,10 +1,6 @@ --- AWSTemplateFormatVersion: '2010-09-09' -Description: |- - Creates OpsWorks Applications for CKAN Stack extensions. - Current extension list: - Legacy theme - Queensland Government extension +Description: Creates metadata needed to deploy CKAN Stack extensions. Parameters: Environment: diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 997740e7..5f7eafa3 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -1,6 +1,6 @@ --- AWSTemplateFormatVersion: '2010-09-09' -Description: 'Creates instances for OpsWorks CKAN NFS Stack.' +Description: 'Creates server instances for a CKAN Stack.' Parameters: ApplicationName: diff --git a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 index be94f69b..468446d2 100644 --- a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: "Creates an OpsWorks CKAN Stack." +Description: "Creates permissions and metadata for a CKAN stack." Parameters: CookbookURL: @@ -84,13 +84,6 @@ Parameters: Type: String MinLength: 6 MaxLength: 254 - CreateServiceRole: - Description: Required if OpsWorks stacks have never been created in this account. - Type: String - Default: "no" - AllowedValues: - - "yes" - - "no" EnableDataStore: Description: Whether or not to support the DataStore. Type: String @@ -152,42 +145,7 @@ Parameters: Description: Name of the S3 Attachment bucket. Type: String -Conditions: - ProvisionServiceRole: - Fn::Equals: - - "yes" - - !Ref CreateServiceRole - Resources: - OpsWorksServiceRole: - Condition: ProvisionServiceRole - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - - Effect: Allow - Principal: - Service: opsworks.amazonaws.com - Action: sts:AssumeRole - Policies: - - PolicyName: "aws-opsworks-service-policy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - - Action: - - "ec2:*" - - iam:PassRole - - cloudwatch:GetMetricStatistics - - cloudwatch:DescribeAlarms - - "ecs:*" - - "elasticloadbalancing:*" - - "rds:*" - Effect: Allow - Resource: "*" - AttachmentsPolicy: Type: AWS::IAM::ManagedPolicy Properties: @@ -377,7 +335,6 @@ Resources: - !Ref InstancePolicy - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore - - arn:aws:iam::aws:policy/AWSOpsWorksCloudWatchLogs - arn:aws:iam::aws:policy/CloudFrontReadOnlyAccess # for domain name lookups - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy # to write CloudWatch logs - arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess # for domain name lookups @@ -420,7 +377,6 @@ Resources: - !Ref AttachmentsPolicy - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore - - arn:aws:iam::aws:policy/AWSOpsWorksCloudWatchLogs - arn:aws:iam::aws:policy/CloudFrontReadOnlyAccess # for domain name lookups - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy # to write CloudWatch logs - arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess # for domain name lookups @@ -569,194 +525,6 @@ Resources: Type: String Value: !Ref SolrSource - OpsWorksStack: - Type: AWS::OpsWorks::Stack - Properties: - AgentVersion: LATEST - ConfigurationManager: - Name: Chef - Version: 12 - CustomCookbooksSource: - Revision: !Ref CookbookRevision - Type: !Ref CookbookURLType - # SshKey: !Sub "${CookbookSSHKey}\n" - Url: !Ref CookbookURL - DefaultInstanceProfileArn: - Fn::GetAtt: - - InstanceRoleProfile - - Arn - DefaultOs: "Amazon Linux 2" - DefaultSshKeyName: !Ref DefaultEC2Key - DefaultSubnetId: - Fn::ImportValue: !Sub "${AppSubnets}A" - Name: !Sub "${ApplicationName}_${Environment}" - ServiceRoleArn: - Fn::If: - - ProvisionServiceRole - - Fn::GetAtt: - - OpsWorksServiceRole - - Arn - - Fn::Join: - - "" - - - "arn:aws:iam::" - - !Ref "AWS::AccountId" - - ":role/aws-opsworks-service-role" - UseCustomCookbooks: true - UseOpsworksSecurityGroups: false - VpcId: - Fn::ImportValue: !Ref StackVPC - - OpsWorksSolrLayer: - Type: AWS::OpsWorks::Layer - Properties: - CustomRecipes: - Setup: - - "datashades::solr-setup" - Deploy: - - "datashades::solr-deploy" - Configure: - - "datashades::solr-configure" - AutoAssignElasticIps: false - AutoAssignPublicIps: false - CustomSecurityGroupIds: - - Fn::ImportValue: !Ref AdminSG - - Fn::ImportValue: !Ref DatabaseSG - EnableAutoHealing: true - InstallUpdatesOnBoot: true - Name: !Sub "${ApplicationName}-Solr" - Shortname: !Sub "${ApplicationId}-solr" - StackId: !Ref OpsWorksStack - Type: custom - Tags: - - Key: Layer - Value: solr - VolumeConfigurations: - - MountPoint: "/mnt/local_data" - NumberOfDisks: 1 - Size: 32 - - OpsWorksWebLayer: - Type: AWS::OpsWorks::Layer - Properties: - AutoAssignElasticIps: false - AutoAssignPublicIps: false - CustomRecipes: - Setup: - - "datashades::ckanweb-setup" - Deploy: - - "datashades::ckanweb-deploy" - Configure: - - "datashades::ckanweb-configure" - CustomSecurityGroupIds: - - Fn::ImportValue: !Ref AdminSG - - Fn::ImportValue: !Ref AppSG - CustomInstanceProfileArn: - Fn::GetAtt: - - WebInstanceRoleProfile - - Arn - EnableAutoHealing: true - InstallUpdatesOnBoot: true - Name: !Sub "${ApplicationName}-Web" - Shortname: !Sub "${ApplicationId}-web" - StackId: !Ref OpsWorksStack - Type: custom - Tags: - - Key: Layer - Value: web - LoadBasedAutoScaling: - Enable: Yes - UpScaling: - CpuThreshold: 80 - MemoryThreshold: 80 - IgnoreMetricsTime: 20 - InstanceCount: 1 - ThresholdsWaitTime: 5 - DownScaling: - CpuThreshold: 30 - MemoryThreshold: 30 - IgnoreMetricsTime: 20 - InstanceCount: 1 - ThresholdsWaitTime: 10 - VolumeConfigurations: - - MountPoint: "/mnt/local_data" - NumberOfDisks: 1 - Size: 32 - - OpsWorksBatchLayer: - Type: AWS::OpsWorks::Layer - Properties: - AutoAssignElasticIps: false - AutoAssignPublicIps: false - CustomRecipes: - Setup: - - "datashades::ckanbatch-setup" - Deploy: - - "datashades::ckanbatch-deploy" - Configure: - - "datashades::ckanbatch-configure" - CustomSecurityGroupIds: - - Fn::ImportValue: !Ref AdminSG - - Fn::ImportValue: !Ref AppSG - CustomInstanceProfileArn: - Fn::GetAtt: - - WebInstanceRoleProfile - - Arn - EnableAutoHealing: true - InstallUpdatesOnBoot: true - Name: !Sub "${ApplicationName}-Batch" - Shortname: !Sub "${ApplicationId}-batch" - StackId: !Ref OpsWorksStack - Type: custom - Tags: - - Key: Layer - Value: batch - LoadBasedAutoScaling: - Enable: Yes - UpScaling: - CpuThreshold: 80 - IgnoreMetricsTime: 20 - InstanceCount: 1 - ThresholdsWaitTime: 5 - DownScaling: - CpuThreshold: 30 - IgnoreMetricsTime: 20 - InstanceCount: 1 - ThresholdsWaitTime: 10 - VolumeConfigurations: - - MountPoint: "/mnt/local_data" - NumberOfDisks: 1 - Size: 32 - - CKANELB: - Type: AWS::ElasticLoadBalancing::LoadBalancer - Properties: - LoadBalancerName: !Sub "${Environment}-${ApplicationName}ELB" - CrossZone: true - HealthCheck: - Target: "HTTP:80/api/action/status_show" - HealthyThreshold: 2 - UnhealthyThreshold: 6 - Interval: 30 - Timeout: 20 - Listeners: - - LoadBalancerPort: 443 - InstancePort: 80 - Protocol: HTTPS - SSLCertificateId: !Ref ACMCertificateARN - PolicyNames: - - sticky-LB-1hour - LBCookieStickinessPolicy: - - PolicyName: sticky-LB-1hour - CookieExpirationPeriod: 3600 - Scheme: "internet-facing" - SecurityGroups: - - Fn::ImportValue: !Ref AppLBSG - - Fn::ImportValue: !Ref CloudFrontSG - Subnets: - - Fn::ImportValue: !Sub "${WebSubnets}A" - - Fn::ImportValue: !Sub "${WebSubnets}B" - - Fn::ImportValue: !Sub "${WebSubnets}C" - # Application Load Balancer structure is more complex than classic. # The LoadBalancer defines configuration values like logging config and security groups. # The Listener is external-facing, defining the port, certificate, etc, @@ -824,12 +592,6 @@ Resources: VpcId: Fn::ImportValue: !Ref StackVPC - ELBAttachment: - Type: AWS::OpsWorks::ElasticLoadBalancerAttachment - Properties: - ElasticLoadBalancerName: !Ref CKANELB - LayerId: !Ref OpsWorksWebLayer - WebELBDNSName: Type: AWS::Route53::RecordSet Properties: @@ -841,35 +603,6 @@ Resources: DNSName: !GetAtt CKANALB.DNSName Outputs: - StackID: - Value: !Ref OpsWorksStack - Description: OpsWorks Stack Id - Export: - Name: !Sub "${Environment}${ApplicationName}OpsWorksStack" - - SolrLayer: - Value: !Ref OpsWorksSolrLayer - Description: SolrCloud Layer Id - Export: - Name: !Sub "${Environment}${ApplicationName}OpsWorksSolrLayer" - - WebLayer: - Value: !Ref OpsWorksWebLayer - Description: Web Layer Id - Export: - Name: !Sub "${Environment}${ApplicationName}OpsWorksWebLayer" - - BatchLayer: - Value: !Ref OpsWorksBatchLayer - Description: Batch Layer Id - Export: - Name: !Sub "${Environment}${ApplicationName}OpsWorksBatchLayer" - - WebELBName: - Value: !Ref CKANELB - Export: - Name: !Sub "${Environment}${ApplicationName}WebElbName" - WebELBDNSName: Value: !Ref WebELBDNSName Export: diff --git a/templates/hosted-zone.cfn.yml b/templates/hosted-zone.cfn.yml index 95278340..236ea495 100644 --- a/templates/hosted-zone.cfn.yml +++ b/templates/hosted-zone.cfn.yml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: "Creates a private hosted zone to hold an OpsWorks CKAN Stack." +Description: Creates metadata needed to deploy CKAN Stack extensions. Parameters: Environment: diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index f0be263a..149d107f 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -37,7 +37,6 @@ common_stack: &common_stack CloudFrontSG: "{{ Environment }}CKANCloudfrontHTTPSSG" AppSG: "{{ Environment }}CKANAppAsgSG" DatabaseSG: "{{ Environment }}CKANDatabaseSG" - CreateServiceRole: "yes" EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" From 14a9477ff8e12689208b8ccb935741bd13e68ed0 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 27 May 2024 10:58:06 +1000 Subject: [PATCH 04/22] [QOLDEV-867] add autoscaling policy to target 50% CPU utilisation --- .../Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 5f7eafa3..2db9a3a0 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -209,6 +209,16 @@ Resources: Value: {{ layer|lower }} PropagateAtLaunch: true + {{ layer }}DynamicScalingPolicy: + Type: AWS::AutoScaling::ScalingPolicy + Properties: + AutoScalingGroupName: !Ref {{ layer }}ScalingGroup + PolicyType: TargetTrackingScaling + TargetTrackingConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ASGAverageCPUUtilization + TargetValue: 50 + {% if item.tags["PowerManaged"] == "Yes" and layer != 'Solr' %} {{ layer }}ScalingIn: Type: AWS::AutoScaling::ScheduledAction From 5ad191b61cdb9ac08578b2820297c0a5792d72d6 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jun 2024 16:32:29 +1000 Subject: [PATCH 05/22] [QOLDEV-892] handle AWS CLI v2 syntax if present --- .../Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 2db9a3a0..b0114337 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -166,11 +166,15 @@ Resources: echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab mount -a fi + REGION="--region ${AWS::Region}" metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \ INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \ - aws ec2 create-tags --region "${AWS::Region}" --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-$INSTANCE_ID" - FUNCTION_NAME=$(aws ssm get-parameter --region "${AWS::Region}" --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text) - aws lambda invoke --region "${AWS::Region}" --function-name "$FUNCTION_NAME" --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "setup"}' /var/log/instance-setup.log.`date '+%s'` + aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-$INSTANCE_ID" + FUNCTION_NAME=$(aws ssm get-parameter $REGION --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text) + if (aws --version |grep -o 'aws-cli/[2-9]'); then + PAYLOAD_FORMAT="--cli-binary-format raw-in-base64-out" + fi + aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "setup"}' /var/log/instance-setup.log.`date '+%s'` {% if layer == 'Web' %} {% set minInstanceCount = (item.template_parameters['WebEC2Count'] | default('2') | int) - 1 %} From 731b0de33705925b2644467237808a545f4361e4 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jun 2024 16:32:55 +1000 Subject: [PATCH 06/22] [QOLDEV-892] update sandbox to Amazon Linux 2023 --- vars/instances-CKANTest.var.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vars/instances-CKANTest.var.yml b/vars/instances-CKANTest.var.yml index 9f4753ae..2a6e24ed 100644 --- a/vars/instances-CKANTest.var.yml +++ b/vars/instances-CKANTest.var.yml @@ -61,6 +61,9 @@ cloudformation_stacks: SolrEC2Count: 1 WebEC2Size: t3a.micro BatchEC2Size: t3a.micro + BatchImageId: "ami-0e326862c8e74c0fe" + WebImageId: "ami-0e326862c8e74c0fe" + SolrImageId: "ami-0e326862c8e74c0fe" tags: <<: *common_stack_tags PowerManaged: "Yes" From 7a983cd072314168d551c971349c1c4b68394326 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 20 Jun 2024 13:50:28 +1000 Subject: [PATCH 07/22] [QOLDEV-892] install Chef client manually - Automatic install during SSM Run Command sometimes fails without logging the reason --- files/instanceSetupLambda.js | 2 +- templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/files/instanceSetupLambda.js b/files/instanceSetupLambda.js index 22c366e3..f4ce3d63 100644 --- a/files/instanceSetupLambda.js +++ b/files/instanceSetupLambda.js @@ -113,7 +113,7 @@ exports.handler = async (event) => { SourceType: [cookbookType], SourceInfo: [sourceInfo], RunList: [runList], - ChefClientVersion: ["14"], + ChefClientVersion: ["None"], WhyRun: ["False"], ComplianceSeverity: ["None"], ComplianceType: ["Custom:Chef"] diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index b0114337..6995c1ec 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -166,6 +166,7 @@ Resources: echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab mount -a fi + yum install chef || yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/14.15.6/el/7/chef-14.15.6-1.el7.x86_64.rpm" REGION="--region ${AWS::Region}" metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \ INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \ From 74fb30078059cc00579584a414032738b39eb77b Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 21 Jun 2024 16:33:22 +1000 Subject: [PATCH 08/22] [QOLDEV-892] update all DEV environments to Amazon Linux 2023 --- vars/instances-OpenData.var.yml | 3 +++ vars/instances-Publications.var.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/vars/instances-OpenData.var.yml b/vars/instances-OpenData.var.yml index e2a2089b..a50569de 100644 --- a/vars/instances-OpenData.var.yml +++ b/vars/instances-OpenData.var.yml @@ -62,6 +62,9 @@ cloudformation_stacks: WebEC2Count: 1 SolrEC2Size: t3a.small SolrEC2Count: 1 + BatchImageId: "ami-0e326862c8e74c0fe" + WebImageId: "ami-0e326862c8e74c0fe" + SolrImageId: "ami-0e326862c8e74c0fe" tags: <<: *common_stack_tags PowerManaged: "No" diff --git a/vars/instances-Publications.var.yml b/vars/instances-Publications.var.yml index ee82e8e5..1dda5f0f 100644 --- a/vars/instances-Publications.var.yml +++ b/vars/instances-Publications.var.yml @@ -69,6 +69,9 @@ cloudformation_stacks: WebEC2Count: 1 WebEC2Size: t3a.micro BatchEC2Size: t3a.micro + BatchImageId: "ami-0e326862c8e74c0fe" + WebImageId: "ami-0e326862c8e74c0fe" + SolrImageId: "ami-0e326862c8e74c0fe" tags: <<: *common_stack_tags PowerManaged: "Yes" From 4f25afbce46890fabaf656567cad12de0a33273f Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 24 Jun 2024 13:39:26 +1000 Subject: [PATCH 09/22] [QOLDEV-892] retry Chef client installation if RPM database is busy --- templates/3_tier_vpc.yml | 7 +++---- templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 7 ++++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/templates/3_tier_vpc.yml b/templates/3_tier_vpc.yml index d77a1ed7..46340589 100644 --- a/templates/3_tier_vpc.yml +++ b/templates/3_tier_vpc.yml @@ -1168,12 +1168,12 @@ Resources: PrivateRouteTable: Properties: Tags: - - Key: Name - Value: !Sub "${VPCNamePrefix}Vpc-${Environment}-PrivateRoutes" + - Key: Name + Value: !Sub "${VPCNamePrefix}Vpc-${Environment}-PrivateRoutes" VpcId: Ref: VPC Type: AWS::EC2::RouteTable -## private RouteTableB to E + ## private RouteTableB to E PrivateNATGatewayRouteB: Condition: 2PlusAZsNatGateways DependsOn: NATGatewayB @@ -1584,7 +1584,6 @@ Resources: SubnetId: Ref: WebSubnetE Type: AWS::EC2::SubnetRouteTableAssociation - #Allow s3 root for listing as well as get set on folders S3Endpoint: Type: "AWS::EC2::VPCEndpoint" Properties: diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 6995c1ec..5dfc2572 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -166,7 +166,12 @@ Resources: echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab mount -a fi - yum install chef || yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/14.15.6/el/7/chef-14.15.6-1.el7.x86_64.rpm" + if ! (yum install chef); then + for i in `seq 1 5`; do + yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/14.15.6/el/7/chef-14.15.6-1.el7.x86_64.rpm" && break + sleep 5 + done + fi REGION="--region ${AWS::Region}" metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \ INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \ From 03881ce406fed886ed62e624d19f80df7dcead91 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 24 Jun 2024 15:46:46 +1000 Subject: [PATCH 10/22] [QOLDEV-892] update cookbook to support Amazon Linux 2023 --- vars/CKAN-Stack.var.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index 149d107f..91bf224d 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -40,7 +40,7 @@ common_stack: &common_stack EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - CookbookRevision: "{{ CookbookRevision | default('7.0.8') }}" + CookbookRevision: "{{ CookbookRevision | default('7.1.0') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket SolrSource: "{{ solr_url }}" From ee478263f5a34442c9011e37824898691a2a7b87 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 25 Jun 2024 16:14:21 +1000 Subject: [PATCH 11/22] [QOLDEV-867] don't allow deployments to push an ASG below its minimum - If desired capacity is already at minimum, attempting to put a server in standby without replacement will fail. Detect this condition and spawn a replacement. --- chef-deploy.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/chef-deploy.sh b/chef-deploy.sh index da996d31..eb61f943 100755 --- a/chef-deploy.sh +++ b/chef-deploy.sh @@ -147,8 +147,17 @@ deploy () { INSTANCE_STATE=$(aws ec2 describe-instances --filters Name=instance-id,Values=$instance --query "Reservations[].Instances[0].State.Name" --output text) if [ "$INSTANCE_STATE" != "running" ]; then continue; fi if [ "$ASG_NAME" != "" ] && (aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].Instances[?InstanceId=='$instance'].InstanceId" --output text |grep "$instance" >/dev/null); then + # Check if the group is already at minimum capacity + CAPACITIES=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name TRAINING-CKANTest-Web-ASG --query "AutoScalingGroups[0].{min: MinSize, desired: DesiredCapacity}" --output text) + CAPACITY_1=`echo $CAPACITIES | awk '{print $1}'` + CAPACITY_2=`echo $CAPACITIES | awk '{print $2}'` + if [ "$CAPACITY_1" = "$CAPACITY_2" ]; then + DECREMENT_BEHAVIOUR="--no-should-decrement-desired-capacity" + else + DECREMENT_BEHAVIOUR="--should-decrement-desired-capacity" + fi # Instances in standby will not get traffic nor health checks, allowing us to update them without interruption - OUTPUT=$(aws autoscaling enter-standby --auto-scaling-group-name "$ASG_NAME" --should-decrement-desired-capacity --instance-ids $instance --query "Activities[].Description" --output text) + OUTPUT=$(aws autoscaling enter-standby --auto-scaling-group-name "$ASG_NAME" $DECREMENT_BEHAVIOUR --instance-ids $instance --query "Activities[].Description" --output text) debug "$OUTPUT" elif [ "$ELB_NAME" != "" ]; then OUTPUT=$(aws elb deregister-instances-from-load-balancer --load-balancer-name "$ELB_NAME" --instances "$instance" --query "Instances[].InstanceId" --output text) From ff0bac71bb55b34c598a8743568ecaea8e0d86ea Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 25 Jun 2024 16:41:36 +1000 Subject: [PATCH 12/22] [QOLDEV-867] fail deployment fast if initial deployment fails --- build-CKAN.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-CKAN.sh b/build-CKAN.sh index 234a846d..d0490fb6 100755 --- a/build-CKAN.sh +++ b/build-CKAN.sh @@ -38,6 +38,8 @@ run-shared-resource-playbooks () { run-deployment () { run-playbook "chef-json" ./chef-deploy.sh datashades::ckanweb-setup,datashades::ckanweb-deploy,datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$! + # Check if the web deployment immediately failed + kill -0 $WEB_PID PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-setup,datashades::ckanbatch-deploy,datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$! wait $WEB_PID wait $BATCH_PID From 5541e4d43653269df65a0ac2bc96fdd967c4765d Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 26 Jun 2024 10:02:18 +1000 Subject: [PATCH 13/22] [QOLDEV-867] simplify instance count configuration - Set minimum and desired capacities to the same value, now that our deployments can handle that - Drop redundant config that is equivalent to the defaults --- .../Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 10 ++-------- vars/instances-CKANTest.var.yml | 2 -- vars/instances-OpenData.var.yml | 3 +-- vars/instances-Publications.var.yml | 2 -- 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 5dfc2572..200c1fe2 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -182,18 +182,12 @@ Resources: fi aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "setup"}' /var/log/instance-setup.log.`date '+%s'` -{% if layer == 'Web' %} -{% set minInstanceCount = (item.template_parameters['WebEC2Count'] | default('2') | int) - 1 %} -{% else %} -{% set minInstanceCount = 1 %} -{% endif %} - {{ layer }}ScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup Properties: AutoScalingGroupName: !Sub "${Environment}-${ApplicationName}-{{ layer }}-ASG" DesiredCapacity: !Ref {{ layer }}EC2Count - MinSize: {{ minInstanceCount }} + MinSize: !Ref {{ layer }}EC2Count MaxSize: 6 LaunchTemplate: LaunchTemplateId: !Ref {{ layer }}LaunchTemplate @@ -244,7 +238,7 @@ Resources: Properties: AutoScalingGroupName: !Ref {{ layer }}ScalingGroup DesiredCapacity: !Ref {{ layer }}EC2Count - MinSize: {{ minInstanceCount }} + MinSize: !Ref {{ layer }}EC2Count MaxSize: 6 Recurrence: "0 20 * * *" {% endif %} diff --git a/vars/instances-CKANTest.var.yml b/vars/instances-CKANTest.var.yml index 2a6e24ed..50e21d95 100644 --- a/vars/instances-CKANTest.var.yml +++ b/vars/instances-CKANTest.var.yml @@ -24,7 +24,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: PROD - WebEC2Count: 2 tags: <<: *common_stack_tags PowerManaged: "No" @@ -34,7 +33,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: STAGING - WebEC2Count: 2 tags: <<: *common_stack_tags PowerManaged: "Yes" diff --git a/vars/instances-OpenData.var.yml b/vars/instances-OpenData.var.yml index a50569de..b0ffd234 100644 --- a/vars/instances-OpenData.var.yml +++ b/vars/instances-OpenData.var.yml @@ -27,7 +27,7 @@ cloudformation_stacks: <<: *common_stack_template_parameters Environment: PROD SolrEC2Size: t3a.medium - WebEC2Count: 5 #We have been running on 2 auto and 3 manually created instances, time to make it IasC + WebEC2Count: 5 tags: <<: *common_stack_tags PowerManaged: "No" @@ -37,7 +37,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: STAGING - WebEC2Count: 2 SolrEC2Size: t3a.medium tags: <<: *common_stack_tags diff --git a/vars/instances-Publications.var.yml b/vars/instances-Publications.var.yml index 1dda5f0f..96de9ee5 100644 --- a/vars/instances-Publications.var.yml +++ b/vars/instances-Publications.var.yml @@ -25,7 +25,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: PROD - WebEC2Count: 2 SolrEC2Size: t3a.small WebEC2Size: t3a.small BatchEC2Size: t3a.micro @@ -38,7 +37,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: STAGING - WebEC2Count: 2 SolrEC2Size: t3a.small WebEC2Size: t3a.small BatchEC2Size: t3a.micro From 967bef5866c2d4e95381219e611e29afefe1cfcb Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 26 Jun 2024 10:48:51 +1000 Subject: [PATCH 14/22] [QOLDEV-867] capture failed deployments and clean up before exiting --- chef-deploy.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chef-deploy.sh b/chef-deploy.sh index eb61f943..abdc2fd6 100755 --- a/chef-deploy.sh +++ b/chef-deploy.sh @@ -164,8 +164,7 @@ deploy () { debug "Deregistered instance $instance from load balancer $ELB_NAME, resulting registered instances: $OUTPUT" fi DEPLOYMENT_ID=$(aws ssm send-command --document-name "AWS-ApplyChefRecipes" --document-version "\$DEFAULT" --instance-ids $instance --parameters '{'"$CHEF_SOURCE"',"RunList":["'"$RUN_LIST"'"],"JsonAttributesSources":[""],"JsonAttributesContent":[""],"ChefClientVersion":["14"],"ChefClientArguments":[""],"WhyRun":["False"],"ComplianceSeverity":["None"],"ComplianceType":["Custom:Chef"],"ComplianceReportBucket":[""]}' --timeout-seconds 3600 --max-concurrency "50" --max-errors "0" --output-s3-bucket-name "osssio-ckan-web-logs" --output-s3-key-prefix "run_command" --region ap-southeast-2 --query "Command.CommandId" --output text) - wait_for_deployment $DEPLOYMENT_ID - DEPLOYMENT_SUCCESS=$? + wait_for_deployment $DEPLOYMENT_ID || DEPLOYMENT_SUCCESS=$? if [ "$ASG_NAME" != "" ]; then OUTPUT=$(aws autoscaling exit-standby --auto-scaling-group-name "$ASG_NAME" --instance-ids $instance --query "Activities[].Description" --output text) debug "$OUTPUT" From d762ba582bc83b1b645d17af50b378e3bae07dff Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 26 Jun 2024 16:14:02 +1000 Subject: [PATCH 15/22] [QOLDEV-867] set default deployment status to success --- chef-deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/chef-deploy.sh b/chef-deploy.sh index abdc2fd6..868c5fe5 100755 --- a/chef-deploy.sh +++ b/chef-deploy.sh @@ -164,6 +164,7 @@ deploy () { debug "Deregistered instance $instance from load balancer $ELB_NAME, resulting registered instances: $OUTPUT" fi DEPLOYMENT_ID=$(aws ssm send-command --document-name "AWS-ApplyChefRecipes" --document-version "\$DEFAULT" --instance-ids $instance --parameters '{'"$CHEF_SOURCE"',"RunList":["'"$RUN_LIST"'"],"JsonAttributesSources":[""],"JsonAttributesContent":[""],"ChefClientVersion":["14"],"ChefClientArguments":[""],"WhyRun":["False"],"ComplianceSeverity":["None"],"ComplianceType":["Custom:Chef"],"ComplianceReportBucket":[""]}' --timeout-seconds 3600 --max-concurrency "50" --max-errors "0" --output-s3-bucket-name "osssio-ckan-web-logs" --output-s3-key-prefix "run_command" --region ap-southeast-2 --query "Command.CommandId" --output text) + DEPLOYMENT_SUCCESS=0 wait_for_deployment $DEPLOYMENT_ID || DEPLOYMENT_SUCCESS=$? if [ "$ASG_NAME" != "" ]; then OUTPUT=$(aws autoscaling exit-standby --auto-scaling-group-name "$ASG_NAME" --instance-ids $instance --query "Activities[].Description" --output text) From f4e56d52c2642e071724ca1ec73ce176ad000078 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 26 Jun 2024 16:25:29 +1000 Subject: [PATCH 16/22] [QOLDEV-867] add debug message when not decrementing capacity --- chef-deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/chef-deploy.sh b/chef-deploy.sh index 868c5fe5..26a2f2a2 100755 --- a/chef-deploy.sh +++ b/chef-deploy.sh @@ -152,6 +152,7 @@ deploy () { CAPACITY_1=`echo $CAPACITIES | awk '{print $1}'` CAPACITY_2=`echo $CAPACITIES | awk '{print $2}'` if [ "$CAPACITY_1" = "$CAPACITY_2" ]; then + debug "Capacity is at minimum ($CAPACITY_1 = $CAPACITY_2), new instance will be started" DECREMENT_BEHAVIOUR="--no-should-decrement-desired-capacity" else DECREMENT_BEHAVIOUR="--should-decrement-desired-capacity" From f4cbd23b5cc65d40ae596a0e9132b86b3736b1a5 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 26 Jun 2024 16:50:52 +1000 Subject: [PATCH 17/22] [QOLDEV-867] fix ASG name for capacity check --- chef-deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef-deploy.sh b/chef-deploy.sh index 26a2f2a2..665ec5ce 100755 --- a/chef-deploy.sh +++ b/chef-deploy.sh @@ -148,7 +148,7 @@ deploy () { if [ "$INSTANCE_STATE" != "running" ]; then continue; fi if [ "$ASG_NAME" != "" ] && (aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].Instances[?InstanceId=='$instance'].InstanceId" --output text |grep "$instance" >/dev/null); then # Check if the group is already at minimum capacity - CAPACITIES=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name TRAINING-CKANTest-Web-ASG --query "AutoScalingGroups[0].{min: MinSize, desired: DesiredCapacity}" --output text) + CAPACITIES=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].{min: MinSize, desired: DesiredCapacity}" --output text) CAPACITY_1=`echo $CAPACITIES | awk '{print $1}'` CAPACITY_2=`echo $CAPACITIES | awk '{print $2}'` if [ "$CAPACITY_1" = "$CAPACITY_2" ]; then From c508e3854f22bfb6bf9577054dbdb49279d46f48 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 27 Jun 2024 15:17:25 +1000 Subject: [PATCH 18/22] [QOLDEV-902] update Archiver to ensure QA runs on uploaded files --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 5032ba65..807eb5c2 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -119,7 +119,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.17" + version: "2.1.1-qgov.18" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 42a6040c..084c2ed9 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -119,7 +119,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.17" + version: "2.1.1-qgov.18" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" From 8f129c361d52fc78a49aca86569af0267fb62e22 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 27 Jun 2024 16:00:47 +1000 Subject: [PATCH 19/22] [QOLDEV-892] update cookbook to retain support Amazon Linux 2 --- vars/CKAN-Stack.var.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index 91bf224d..1c4901ce 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -40,7 +40,7 @@ common_stack: &common_stack EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - CookbookRevision: "{{ CookbookRevision | default('7.1.0') }}" + CookbookRevision: "{{ CookbookRevision | default('7.1.1') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket SolrSource: "{{ solr_url }}" From 0a38eb64c8a97aed00ee95830ebef7b448912df8 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 28 Jun 2024 13:35:52 +1000 Subject: [PATCH 20/22] [QOLDEV-892] skip putting instances into standby if they're not in service --- chef-deploy.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/chef-deploy.sh b/chef-deploy.sh index 665ec5ce..9cc0f9eb 100755 --- a/chef-deploy.sh +++ b/chef-deploy.sh @@ -146,7 +146,8 @@ deploy () { # double-check that instance is still running INSTANCE_STATE=$(aws ec2 describe-instances --filters Name=instance-id,Values=$instance --query "Reservations[].Instances[0].State.Name" --output text) if [ "$INSTANCE_STATE" != "running" ]; then continue; fi - if [ "$ASG_NAME" != "" ] && (aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].Instances[?InstanceId=='$instance'].InstanceId" --output text |grep "$instance" >/dev/null); then + if [ "$ASG_NAME" != "" ] && (aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].Instances[?InstanceId=='$instance' && LifecycleState=='InService'].InstanceId" --output text |grep "$instance" >/dev/null); then + IN_ASG="true" # Check if the group is already at minimum capacity CAPACITIES=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].{min: MinSize, desired: DesiredCapacity}" --output text) CAPACITY_1=`echo $CAPACITIES | awk '{print $1}'` @@ -167,7 +168,10 @@ deploy () { DEPLOYMENT_ID=$(aws ssm send-command --document-name "AWS-ApplyChefRecipes" --document-version "\$DEFAULT" --instance-ids $instance --parameters '{'"$CHEF_SOURCE"',"RunList":["'"$RUN_LIST"'"],"JsonAttributesSources":[""],"JsonAttributesContent":[""],"ChefClientVersion":["14"],"ChefClientArguments":[""],"WhyRun":["False"],"ComplianceSeverity":["None"],"ComplianceType":["Custom:Chef"],"ComplianceReportBucket":[""]}' --timeout-seconds 3600 --max-concurrency "50" --max-errors "0" --output-s3-bucket-name "osssio-ckan-web-logs" --output-s3-key-prefix "run_command" --region ap-southeast-2 --query "Command.CommandId" --output text) DEPLOYMENT_SUCCESS=0 wait_for_deployment $DEPLOYMENT_ID || DEPLOYMENT_SUCCESS=$? - if [ "$ASG_NAME" != "" ]; then + if [ "$IN_ASG" = "true" ]; then + # reactivate the instance if we put it into standby + # NB If it was in standby before we started, then we will deploy to it + # but leave it in standby. OUTPUT=$(aws autoscaling exit-standby --auto-scaling-group-name "$ASG_NAME" --instance-ids $instance --query "Activities[].Description" --output text) debug "$OUTPUT" elif [ "$ELB_NAME" != "" ]; then From dcf8b7a6dffd65bdd875e4807743c9a29e6fe51d Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 1 Jul 2024 16:13:51 +1000 Subject: [PATCH 21/22] [QOLDEV-892] update Chef client to version 18.x - This appears to work now that we have the XCrypt compatibility library --- templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 200c1fe2..0a2be633 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -113,6 +113,7 @@ Parameters: BatchImageId: Description: The Amazon Machine Image ID to use for launching batch instances. Defaults to Amazon Linux 2. Type: String + # Customised image based on Amazon Linux 2, preinstalling some basics Default: "ami-0d71fe73adf7a9887" WebImageId: Description: The Amazon Machine Image ID to use for launching web instances. Defaults to Amazon Linux 2. @@ -168,7 +169,7 @@ Resources: fi if ! (yum install chef); then for i in `seq 1 5`; do - yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/14.15.6/el/7/chef-14.15.6-1.el7.x86_64.rpm" && break + yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break sleep 5 done fi From f70e685c380b55a119f20b284d00c3c662d6092f Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 1 Jul 2024 16:19:23 +1000 Subject: [PATCH 22/22] [QOLDEV-892] extract AMI ID to a documented variable --- vars/instances-CKANTest.var.yml | 9 ++++++--- vars/instances-OpenData.var.yml | 9 ++++++--- vars/instances-Publications.var.yml | 9 ++++++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/vars/instances-CKANTest.var.yml b/vars/instances-CKANTest.var.yml index 50e21d95..73143638 100644 --- a/vars/instances-CKANTest.var.yml +++ b/vars/instances-CKANTest.var.yml @@ -1,4 +1,7 @@ --- +# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 +ami_al2023: "ami-0e326862c8e74c0fe" + common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -59,9 +62,9 @@ cloudformation_stacks: SolrEC2Count: 1 WebEC2Size: t3a.micro BatchEC2Size: t3a.micro - BatchImageId: "ami-0e326862c8e74c0fe" - WebImageId: "ami-0e326862c8e74c0fe" - SolrImageId: "ami-0e326862c8e74c0fe" + BatchImageId: "{{ ami_al2023 }}" + WebImageId: "{{ ami_al2023 }}" + SolrImageId: "{{ ami_al2023 }}" tags: <<: *common_stack_tags PowerManaged: "Yes" diff --git a/vars/instances-OpenData.var.yml b/vars/instances-OpenData.var.yml index b0ffd234..a721876d 100644 --- a/vars/instances-OpenData.var.yml +++ b/vars/instances-OpenData.var.yml @@ -1,4 +1,7 @@ --- +# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 +ami_al2023: "ami-0e326862c8e74c0fe" + common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -61,9 +64,9 @@ cloudformation_stacks: WebEC2Count: 1 SolrEC2Size: t3a.small SolrEC2Count: 1 - BatchImageId: "ami-0e326862c8e74c0fe" - WebImageId: "ami-0e326862c8e74c0fe" - SolrImageId: "ami-0e326862c8e74c0fe" + BatchImageId: "{{ ami_al2023 }}" + WebImageId: "{{ ami_al2023 }}" + SolrImageId: "{{ ami_al2023 }}" tags: <<: *common_stack_tags PowerManaged: "No" diff --git a/vars/instances-Publications.var.yml b/vars/instances-Publications.var.yml index 96de9ee5..3ede91c9 100644 --- a/vars/instances-Publications.var.yml +++ b/vars/instances-Publications.var.yml @@ -1,4 +1,7 @@ --- +# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 +ami_al2023: "ami-0e326862c8e74c0fe" + common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -67,9 +70,9 @@ cloudformation_stacks: WebEC2Count: 1 WebEC2Size: t3a.micro BatchEC2Size: t3a.micro - BatchImageId: "ami-0e326862c8e74c0fe" - WebImageId: "ami-0e326862c8e74c0fe" - SolrImageId: "ami-0e326862c8e74c0fe" + BatchImageId: "{{ ami_al2023 }}" + WebImageId: "{{ ami_al2023 }}" + SolrImageId: "{{ ami_al2023 }}" tags: <<: *common_stack_tags PowerManaged: "Yes"