diff --git a/README.md b/README.md index 58a7dc17..31162f8d 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ Author: qol.development@smartservice.qld.gov.au -**Full one click deployment of Datashared AWS OpsWorks Stack via Ansible** +**Full one click deployment of CKAN AWS Stack via Ansible** This stands up the www.data.qld.gov.au and www.publications.qld.gov.au aws stacks using: * SSM * RDS * Redis Cluster -* OpsWorks +* EC2 Autoscaling * Cloudfront with Lambda@Edge * and may more features @@ -25,7 +25,7 @@ This stands up the www.data.qld.gov.au and www.publications.qld.gov.au aws stack For the system to work with updates to the lambda function, you must; * first add a new version to cloudfront-lambdaAtEdge.cfn.yml which references the changed lambda function (there is no need for a new lambda function) * export the new version from said cloudformation template -* update the cloudfront.yml ansible script to load the new version property name. +* update the cloudfront.yml ansible script to load the new version property name. * you can delete previous versions after a successful real, do note that cloudfront will hold onto a lambda function and versions until its 'replication' finishes. **QOL 2019 update** @@ -60,7 +60,7 @@ Common issues during set up are as follows: It's assumed that you: -* have a pretty good working knowledge of AWS, CloudFormation, OpsWorks, and CKAN and its requirements such as Solr and Postgres. Those will be necessary to troubleshoot builds when you haven't provided the correct parameters or some other obstacle gets in your way. +* have a pretty good working knowledge of AWS, CloudFormation, EC2 Autoscaling, and CKAN and its requirements such as Solr and Postgres. Those will be necessary to troubleshoot builds when you haven't provided the correct parameters or some other obstacle gets in your way. * have built, installed and successfully run CKAN manually on some kind of single node configuration. If not, this stack isn't designed to be something to cut your teeth on. It's been designed to be relatively foolproof, but not completely so. * know your way around the Linux command line reasonably well and know how to deal with error logs, dependency conflicts etc. @@ -118,7 +118,7 @@ and automated system maintenance. Our hope and expectation is that it benefits the wider Public Data community and progresses the Open Data ideal. -Current AWS costs for 2 CKAN applications by 4 envirionments is just shy of 3k USD a month. +Current AWS costs for 2 CKAN applications by 4 environments is just shy of 3k USD a month. ## TODO ## Make requirements-dev look up vars/shared-${app}.var.yml and test all environment plugins diff --git a/build-CKAN.sh b/build-CKAN.sh index 234a846d..d0490fb6 100755 --- a/build-CKAN.sh +++ b/build-CKAN.sh @@ -38,6 +38,8 @@ run-shared-resource-playbooks () { run-deployment () { run-playbook "chef-json" ./chef-deploy.sh datashades::ckanweb-setup,datashades::ckanweb-deploy,datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$! + # Check if the web deployment immediately failed + kill -0 $WEB_PID PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-setup,datashades::ckanbatch-deploy,datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$! wait $WEB_PID wait $BATCH_PID diff --git a/chef-deploy.sh b/chef-deploy.sh index da996d31..9cc0f9eb 100755 --- a/chef-deploy.sh +++ b/chef-deploy.sh @@ -146,18 +146,32 @@ deploy () { # double-check that instance is still running INSTANCE_STATE=$(aws ec2 describe-instances --filters Name=instance-id,Values=$instance --query "Reservations[].Instances[0].State.Name" --output text) if [ "$INSTANCE_STATE" != "running" ]; then continue; fi - if [ "$ASG_NAME" != "" ] && (aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].Instances[?InstanceId=='$instance'].InstanceId" --output text |grep "$instance" >/dev/null); then + if [ "$ASG_NAME" != "" ] && (aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].Instances[?InstanceId=='$instance' && LifecycleState=='InService'].InstanceId" --output text |grep "$instance" >/dev/null); then + IN_ASG="true" + # Check if the group is already at minimum capacity + CAPACITIES=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].{min: MinSize, desired: DesiredCapacity}" --output text) + CAPACITY_1=`echo $CAPACITIES | awk '{print $1}'` + CAPACITY_2=`echo $CAPACITIES | awk '{print $2}'` + if [ "$CAPACITY_1" = "$CAPACITY_2" ]; then + debug "Capacity is at minimum ($CAPACITY_1 = $CAPACITY_2), new instance will be started" + DECREMENT_BEHAVIOUR="--no-should-decrement-desired-capacity" + else + DECREMENT_BEHAVIOUR="--should-decrement-desired-capacity" + fi # Instances in standby will not get traffic nor health checks, allowing us to update them without interruption - OUTPUT=$(aws autoscaling enter-standby --auto-scaling-group-name "$ASG_NAME" --should-decrement-desired-capacity --instance-ids $instance --query "Activities[].Description" --output text) + OUTPUT=$(aws autoscaling enter-standby --auto-scaling-group-name "$ASG_NAME" $DECREMENT_BEHAVIOUR --instance-ids $instance --query "Activities[].Description" --output text) debug "$OUTPUT" elif [ "$ELB_NAME" != "" ]; then OUTPUT=$(aws elb deregister-instances-from-load-balancer --load-balancer-name "$ELB_NAME" --instances "$instance" --query "Instances[].InstanceId" --output text) debug "Deregistered instance $instance from load balancer $ELB_NAME, resulting registered instances: $OUTPUT" fi DEPLOYMENT_ID=$(aws ssm send-command --document-name "AWS-ApplyChefRecipes" --document-version "\$DEFAULT" --instance-ids $instance --parameters '{'"$CHEF_SOURCE"',"RunList":["'"$RUN_LIST"'"],"JsonAttributesSources":[""],"JsonAttributesContent":[""],"ChefClientVersion":["14"],"ChefClientArguments":[""],"WhyRun":["False"],"ComplianceSeverity":["None"],"ComplianceType":["Custom:Chef"],"ComplianceReportBucket":[""]}' --timeout-seconds 3600 --max-concurrency "50" --max-errors "0" --output-s3-bucket-name "osssio-ckan-web-logs" --output-s3-key-prefix "run_command" --region ap-southeast-2 --query "Command.CommandId" --output text) - wait_for_deployment $DEPLOYMENT_ID - DEPLOYMENT_SUCCESS=$? - if [ "$ASG_NAME" != "" ]; then + DEPLOYMENT_SUCCESS=0 + wait_for_deployment $DEPLOYMENT_ID || DEPLOYMENT_SUCCESS=$? + if [ "$IN_ASG" = "true" ]; then + # reactivate the instance if we put it into standby + # NB If it was in standby before we started, then we will deploy to it + # but leave it in standby. OUTPUT=$(aws autoscaling exit-standby --auto-scaling-group-name "$ASG_NAME" --instance-ids $instance --query "Activities[].Description" --output text) debug "$OUTPUT" elif [ "$ELB_NAME" != "" ]; then diff --git a/files/instanceSetupLambda.js b/files/instanceSetupLambda.js index 22c366e3..f4ce3d63 100644 --- a/files/instanceSetupLambda.js +++ b/files/instanceSetupLambda.js @@ -113,7 +113,7 @@ exports.handler = async (event) => { SourceType: [cookbookType], SourceInfo: [sourceInfo], RunList: [runList], - ChefClientVersion: ["14"], + ChefClientVersion: ["None"], WhyRun: ["False"], ComplianceSeverity: ["None"], ComplianceType: ["Custom:Chef"] diff --git a/templates/3_tier_vpc.yml b/templates/3_tier_vpc.yml index d77a1ed7..46340589 100644 --- a/templates/3_tier_vpc.yml +++ b/templates/3_tier_vpc.yml @@ -1168,12 +1168,12 @@ Resources: PrivateRouteTable: Properties: Tags: - - Key: Name - Value: !Sub "${VPCNamePrefix}Vpc-${Environment}-PrivateRoutes" + - Key: Name + Value: !Sub "${VPCNamePrefix}Vpc-${Environment}-PrivateRoutes" VpcId: Ref: VPC Type: AWS::EC2::RouteTable -## private RouteTableB to E + ## private RouteTableB to E PrivateNATGatewayRouteB: Condition: 2PlusAZsNatGateways DependsOn: NATGatewayB @@ -1584,7 +1584,6 @@ Resources: SubnetId: Ref: WebSubnetE Type: AWS::EC2::SubnetRouteTableAssociation - #Allow s3 root for listing as well as get set on folders S3Endpoint: Type: "AWS::EC2::VPCEndpoint" Properties: diff --git a/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 index 1f0cf96f..8d9db161 100644 --- a/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 @@ -1,10 +1,6 @@ --- AWSTemplateFormatVersion: '2010-09-09' -Description: |- - Creates OpsWorks Applications for CKAN Stack extensions. - Current extension list: - Legacy theme - Queensland Government extension +Description: Creates metadata needed to deploy CKAN Stack extensions. Parameters: Environment: diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 58282209..0a2be633 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -1,6 +1,6 @@ --- AWSTemplateFormatVersion: '2010-09-09' -Description: 'Creates instances for OpsWorks CKAN NFS Stack.' +Description: 'Creates server instances for a CKAN Stack.' Parameters: ApplicationName: @@ -113,15 +113,16 @@ Parameters: BatchImageId: Description: The Amazon Machine Image ID to use for launching batch instances. Defaults to Amazon Linux 2. Type: String - Default: "ami-03b836d87d294e89e" + # Customised image based on Amazon Linux 2, preinstalling some basics + Default: "ami-0d71fe73adf7a9887" WebImageId: Description: The Amazon Machine Image ID to use for launching web instances. Defaults to Amazon Linux 2. Type: String - Default: "ami-03b836d87d294e89e" + Default: "ami-0d71fe73adf7a9887" SolrImageId: Description: The Amazon Machine Image ID to use for launching Solr instances. Defaults to Amazon Linux 2. Type: String - Default: "ami-03b836d87d294e89e" + Default: "ami-0d71fe73adf7a9887" DefaultEC2Key: Description: Select an existing SSH key Type: AWS::EC2::KeyPair::KeyName @@ -166,24 +167,28 @@ Resources: echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab mount -a fi + if ! (yum install chef); then + for i in `seq 1 5`; do + yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break + sleep 5 + done + fi + REGION="--region ${AWS::Region}" metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \ INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \ - aws ec2 create-tags --region "${AWS::Region}" --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-$INSTANCE_ID" - FUNCTION_NAME=$(aws ssm get-parameter --region "${AWS::Region}" --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text) - aws lambda invoke --region "${AWS::Region}" --function-name "$FUNCTION_NAME" --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "setup"}' /var/log/instance-setup.log.`date '+%s'` - -{% if layer == 'Web' %} -{% set minInstanceCount = (item.template_parameters['WebEC2Count'] | default('2') | int) - 1 %} -{% else %} -{% set minInstanceCount = 1 %} -{% endif %} + aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-$INSTANCE_ID" + FUNCTION_NAME=$(aws ssm get-parameter $REGION --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text) + if (aws --version |grep -o 'aws-cli/[2-9]'); then + PAYLOAD_FORMAT="--cli-binary-format raw-in-base64-out" + fi + aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "setup"}' /var/log/instance-setup.log.`date '+%s'` {{ layer }}ScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup Properties: AutoScalingGroupName: !Sub "${Environment}-${ApplicationName}-{{ layer }}-ASG" DesiredCapacity: !Ref {{ layer }}EC2Count - MinSize: {{ minInstanceCount }} + MinSize: !Ref {{ layer }}EC2Count MaxSize: 6 LaunchTemplate: LaunchTemplateId: !Ref {{ layer }}LaunchTemplate @@ -209,7 +214,17 @@ Resources: Value: {{ layer|lower }} PropagateAtLaunch: true -{% if item.tags["PowerManaged"] == "Yes" %} + {{ layer }}DynamicScalingPolicy: + Type: AWS::AutoScaling::ScalingPolicy + Properties: + AutoScalingGroupName: !Ref {{ layer }}ScalingGroup + PolicyType: TargetTrackingScaling + TargetTrackingConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ASGAverageCPUUtilization + TargetValue: 50 + +{% if item.tags["PowerManaged"] == "Yes" and layer != 'Solr' %} {{ layer }}ScalingIn: Type: AWS::AutoScaling::ScheduledAction Properties: @@ -224,7 +239,7 @@ Resources: Properties: AutoScalingGroupName: !Ref {{ layer }}ScalingGroup DesiredCapacity: !Ref {{ layer }}EC2Count - MinSize: {{ minInstanceCount }} + MinSize: !Ref {{ layer }}EC2Count MaxSize: 6 Recurrence: "0 20 * * *" {% endif %} diff --git a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 index be94f69b..468446d2 100644 --- a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: "Creates an OpsWorks CKAN Stack." +Description: "Creates permissions and metadata for a CKAN stack." Parameters: CookbookURL: @@ -84,13 +84,6 @@ Parameters: Type: String MinLength: 6 MaxLength: 254 - CreateServiceRole: - Description: Required if OpsWorks stacks have never been created in this account. - Type: String - Default: "no" - AllowedValues: - - "yes" - - "no" EnableDataStore: Description: Whether or not to support the DataStore. Type: String @@ -152,42 +145,7 @@ Parameters: Description: Name of the S3 Attachment bucket. Type: String -Conditions: - ProvisionServiceRole: - Fn::Equals: - - "yes" - - !Ref CreateServiceRole - Resources: - OpsWorksServiceRole: - Condition: ProvisionServiceRole - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - - Effect: Allow - Principal: - Service: opsworks.amazonaws.com - Action: sts:AssumeRole - Policies: - - PolicyName: "aws-opsworks-service-policy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - - Action: - - "ec2:*" - - iam:PassRole - - cloudwatch:GetMetricStatistics - - cloudwatch:DescribeAlarms - - "ecs:*" - - "elasticloadbalancing:*" - - "rds:*" - Effect: Allow - Resource: "*" - AttachmentsPolicy: Type: AWS::IAM::ManagedPolicy Properties: @@ -377,7 +335,6 @@ Resources: - !Ref InstancePolicy - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore - - arn:aws:iam::aws:policy/AWSOpsWorksCloudWatchLogs - arn:aws:iam::aws:policy/CloudFrontReadOnlyAccess # for domain name lookups - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy # to write CloudWatch logs - arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess # for domain name lookups @@ -420,7 +377,6 @@ Resources: - !Ref AttachmentsPolicy - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore - - arn:aws:iam::aws:policy/AWSOpsWorksCloudWatchLogs - arn:aws:iam::aws:policy/CloudFrontReadOnlyAccess # for domain name lookups - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy # to write CloudWatch logs - arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess # for domain name lookups @@ -569,194 +525,6 @@ Resources: Type: String Value: !Ref SolrSource - OpsWorksStack: - Type: AWS::OpsWorks::Stack - Properties: - AgentVersion: LATEST - ConfigurationManager: - Name: Chef - Version: 12 - CustomCookbooksSource: - Revision: !Ref CookbookRevision - Type: !Ref CookbookURLType - # SshKey: !Sub "${CookbookSSHKey}\n" - Url: !Ref CookbookURL - DefaultInstanceProfileArn: - Fn::GetAtt: - - InstanceRoleProfile - - Arn - DefaultOs: "Amazon Linux 2" - DefaultSshKeyName: !Ref DefaultEC2Key - DefaultSubnetId: - Fn::ImportValue: !Sub "${AppSubnets}A" - Name: !Sub "${ApplicationName}_${Environment}" - ServiceRoleArn: - Fn::If: - - ProvisionServiceRole - - Fn::GetAtt: - - OpsWorksServiceRole - - Arn - - Fn::Join: - - "" - - - "arn:aws:iam::" - - !Ref "AWS::AccountId" - - ":role/aws-opsworks-service-role" - UseCustomCookbooks: true - UseOpsworksSecurityGroups: false - VpcId: - Fn::ImportValue: !Ref StackVPC - - OpsWorksSolrLayer: - Type: AWS::OpsWorks::Layer - Properties: - CustomRecipes: - Setup: - - "datashades::solr-setup" - Deploy: - - "datashades::solr-deploy" - Configure: - - "datashades::solr-configure" - AutoAssignElasticIps: false - AutoAssignPublicIps: false - CustomSecurityGroupIds: - - Fn::ImportValue: !Ref AdminSG - - Fn::ImportValue: !Ref DatabaseSG - EnableAutoHealing: true - InstallUpdatesOnBoot: true - Name: !Sub "${ApplicationName}-Solr" - Shortname: !Sub "${ApplicationId}-solr" - StackId: !Ref OpsWorksStack - Type: custom - Tags: - - Key: Layer - Value: solr - VolumeConfigurations: - - MountPoint: "/mnt/local_data" - NumberOfDisks: 1 - Size: 32 - - OpsWorksWebLayer: - Type: AWS::OpsWorks::Layer - Properties: - AutoAssignElasticIps: false - AutoAssignPublicIps: false - CustomRecipes: - Setup: - - "datashades::ckanweb-setup" - Deploy: - - "datashades::ckanweb-deploy" - Configure: - - "datashades::ckanweb-configure" - CustomSecurityGroupIds: - - Fn::ImportValue: !Ref AdminSG - - Fn::ImportValue: !Ref AppSG - CustomInstanceProfileArn: - Fn::GetAtt: - - WebInstanceRoleProfile - - Arn - EnableAutoHealing: true - InstallUpdatesOnBoot: true - Name: !Sub "${ApplicationName}-Web" - Shortname: !Sub "${ApplicationId}-web" - StackId: !Ref OpsWorksStack - Type: custom - Tags: - - Key: Layer - Value: web - LoadBasedAutoScaling: - Enable: Yes - UpScaling: - CpuThreshold: 80 - MemoryThreshold: 80 - IgnoreMetricsTime: 20 - InstanceCount: 1 - ThresholdsWaitTime: 5 - DownScaling: - CpuThreshold: 30 - MemoryThreshold: 30 - IgnoreMetricsTime: 20 - InstanceCount: 1 - ThresholdsWaitTime: 10 - VolumeConfigurations: - - MountPoint: "/mnt/local_data" - NumberOfDisks: 1 - Size: 32 - - OpsWorksBatchLayer: - Type: AWS::OpsWorks::Layer - Properties: - AutoAssignElasticIps: false - AutoAssignPublicIps: false - CustomRecipes: - Setup: - - "datashades::ckanbatch-setup" - Deploy: - - "datashades::ckanbatch-deploy" - Configure: - - "datashades::ckanbatch-configure" - CustomSecurityGroupIds: - - Fn::ImportValue: !Ref AdminSG - - Fn::ImportValue: !Ref AppSG - CustomInstanceProfileArn: - Fn::GetAtt: - - WebInstanceRoleProfile - - Arn - EnableAutoHealing: true - InstallUpdatesOnBoot: true - Name: !Sub "${ApplicationName}-Batch" - Shortname: !Sub "${ApplicationId}-batch" - StackId: !Ref OpsWorksStack - Type: custom - Tags: - - Key: Layer - Value: batch - LoadBasedAutoScaling: - Enable: Yes - UpScaling: - CpuThreshold: 80 - IgnoreMetricsTime: 20 - InstanceCount: 1 - ThresholdsWaitTime: 5 - DownScaling: - CpuThreshold: 30 - IgnoreMetricsTime: 20 - InstanceCount: 1 - ThresholdsWaitTime: 10 - VolumeConfigurations: - - MountPoint: "/mnt/local_data" - NumberOfDisks: 1 - Size: 32 - - CKANELB: - Type: AWS::ElasticLoadBalancing::LoadBalancer - Properties: - LoadBalancerName: !Sub "${Environment}-${ApplicationName}ELB" - CrossZone: true - HealthCheck: - Target: "HTTP:80/api/action/status_show" - HealthyThreshold: 2 - UnhealthyThreshold: 6 - Interval: 30 - Timeout: 20 - Listeners: - - LoadBalancerPort: 443 - InstancePort: 80 - Protocol: HTTPS - SSLCertificateId: !Ref ACMCertificateARN - PolicyNames: - - sticky-LB-1hour - LBCookieStickinessPolicy: - - PolicyName: sticky-LB-1hour - CookieExpirationPeriod: 3600 - Scheme: "internet-facing" - SecurityGroups: - - Fn::ImportValue: !Ref AppLBSG - - Fn::ImportValue: !Ref CloudFrontSG - Subnets: - - Fn::ImportValue: !Sub "${WebSubnets}A" - - Fn::ImportValue: !Sub "${WebSubnets}B" - - Fn::ImportValue: !Sub "${WebSubnets}C" - # Application Load Balancer structure is more complex than classic. # The LoadBalancer defines configuration values like logging config and security groups. # The Listener is external-facing, defining the port, certificate, etc, @@ -824,12 +592,6 @@ Resources: VpcId: Fn::ImportValue: !Ref StackVPC - ELBAttachment: - Type: AWS::OpsWorks::ElasticLoadBalancerAttachment - Properties: - ElasticLoadBalancerName: !Ref CKANELB - LayerId: !Ref OpsWorksWebLayer - WebELBDNSName: Type: AWS::Route53::RecordSet Properties: @@ -841,35 +603,6 @@ Resources: DNSName: !GetAtt CKANALB.DNSName Outputs: - StackID: - Value: !Ref OpsWorksStack - Description: OpsWorks Stack Id - Export: - Name: !Sub "${Environment}${ApplicationName}OpsWorksStack" - - SolrLayer: - Value: !Ref OpsWorksSolrLayer - Description: SolrCloud Layer Id - Export: - Name: !Sub "${Environment}${ApplicationName}OpsWorksSolrLayer" - - WebLayer: - Value: !Ref OpsWorksWebLayer - Description: Web Layer Id - Export: - Name: !Sub "${Environment}${ApplicationName}OpsWorksWebLayer" - - BatchLayer: - Value: !Ref OpsWorksBatchLayer - Description: Batch Layer Id - Export: - Name: !Sub "${Environment}${ApplicationName}OpsWorksBatchLayer" - - WebELBName: - Value: !Ref CKANELB - Export: - Name: !Sub "${Environment}${ApplicationName}WebElbName" - WebELBDNSName: Value: !Ref WebELBDNSName Export: diff --git a/templates/hosted-zone.cfn.yml b/templates/hosted-zone.cfn.yml index 95278340..236ea495 100644 --- a/templates/hosted-zone.cfn.yml +++ b/templates/hosted-zone.cfn.yml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: "Creates a private hosted zone to hold an OpsWorks CKAN Stack." +Description: Creates metadata needed to deploy CKAN Stack extensions. Parameters: Environment: diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index f0be263a..1c4901ce 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -37,11 +37,10 @@ common_stack: &common_stack CloudFrontSG: "{{ Environment }}CKANCloudfrontHTTPSSG" AppSG: "{{ Environment }}CKANAppAsgSG" DatabaseSG: "{{ Environment }}CKANDatabaseSG" - CreateServiceRole: "yes" EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - CookbookRevision: "{{ CookbookRevision | default('7.0.8') }}" + CookbookRevision: "{{ CookbookRevision | default('7.1.1') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket SolrSource: "{{ solr_url }}" diff --git a/vars/instances-CKANTest.var.yml b/vars/instances-CKANTest.var.yml index 9f4753ae..73143638 100644 --- a/vars/instances-CKANTest.var.yml +++ b/vars/instances-CKANTest.var.yml @@ -1,4 +1,7 @@ --- +# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 +ami_al2023: "ami-0e326862c8e74c0fe" + common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -24,7 +27,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: PROD - WebEC2Count: 2 tags: <<: *common_stack_tags PowerManaged: "No" @@ -34,7 +36,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: STAGING - WebEC2Count: 2 tags: <<: *common_stack_tags PowerManaged: "Yes" @@ -61,6 +62,9 @@ cloudformation_stacks: SolrEC2Count: 1 WebEC2Size: t3a.micro BatchEC2Size: t3a.micro + BatchImageId: "{{ ami_al2023 }}" + WebImageId: "{{ ami_al2023 }}" + SolrImageId: "{{ ami_al2023 }}" tags: <<: *common_stack_tags PowerManaged: "Yes" diff --git a/vars/instances-OpenData.var.yml b/vars/instances-OpenData.var.yml index e2a2089b..a721876d 100644 --- a/vars/instances-OpenData.var.yml +++ b/vars/instances-OpenData.var.yml @@ -1,4 +1,7 @@ --- +# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 +ami_al2023: "ami-0e326862c8e74c0fe" + common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -27,7 +30,7 @@ cloudformation_stacks: <<: *common_stack_template_parameters Environment: PROD SolrEC2Size: t3a.medium - WebEC2Count: 5 #We have been running on 2 auto and 3 manually created instances, time to make it IasC + WebEC2Count: 5 tags: <<: *common_stack_tags PowerManaged: "No" @@ -37,7 +40,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: STAGING - WebEC2Count: 2 SolrEC2Size: t3a.medium tags: <<: *common_stack_tags @@ -62,6 +64,9 @@ cloudformation_stacks: WebEC2Count: 1 SolrEC2Size: t3a.small SolrEC2Count: 1 + BatchImageId: "{{ ami_al2023 }}" + WebImageId: "{{ ami_al2023 }}" + SolrImageId: "{{ ami_al2023 }}" tags: <<: *common_stack_tags PowerManaged: "No" diff --git a/vars/instances-Publications.var.yml b/vars/instances-Publications.var.yml index ee82e8e5..3ede91c9 100644 --- a/vars/instances-Publications.var.yml +++ b/vars/instances-Publications.var.yml @@ -1,4 +1,7 @@ --- +# Amazon Linux 2023: al2023-ami-2023.4.20240611.0-kernel-6.1-x86_64 +ami_al2023: "ami-0e326862c8e74c0fe" + common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" @@ -25,7 +28,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: PROD - WebEC2Count: 2 SolrEC2Size: t3a.small WebEC2Size: t3a.small BatchEC2Size: t3a.micro @@ -38,7 +40,6 @@ cloudformation_stacks: template_parameters: <<: *common_stack_template_parameters Environment: STAGING - WebEC2Count: 2 SolrEC2Size: t3a.small WebEC2Size: t3a.small BatchEC2Size: t3a.micro @@ -69,6 +70,9 @@ cloudformation_stacks: WebEC2Count: 1 WebEC2Size: t3a.micro BatchEC2Size: t3a.micro + BatchImageId: "{{ ami_al2023 }}" + WebImageId: "{{ ami_al2023 }}" + SolrImageId: "{{ ami_al2023 }}" tags: <<: *common_stack_tags PowerManaged: "Yes" diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 5032ba65..807eb5c2 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -119,7 +119,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.17" + version: "2.1.1-qgov.18" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 42a6040c..084c2ed9 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -119,7 +119,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.17" + version: "2.1.1-qgov.18" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}"