From 4dea021cac77a1cd1ad49b8c81aa477d46617d99 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 3 Apr 2024 16:01:14 +1000 Subject: [PATCH 1/6] [QOLDEV-545] refactor stack template - Restore roles to their original state, split off parameter store values instead --- CKAN-extensions.yml | 23 -- build-CKAN.sh | 2 +- instance-roles.yml | 37 ---- ...shades-OpsWorks-CKAN-Extensions.cfn.yml.j2 | 60 ++++-- .../Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 | 201 +++++++++++++----- templates/instance-roles.cfn.yml | 192 ----------------- vars/CKAN-Stack.var.yml | 1 + vars/instance-roles.var.yml | 21 -- 8 files changed, 197 insertions(+), 340 deletions(-) delete mode 100644 CKAN-extensions.yml delete mode 100644 instance-roles.yml delete mode 100644 templates/instance-roles.cfn.yml delete mode 100644 vars/instance-roles.var.yml diff --git a/CKAN-extensions.yml b/CKAN-extensions.yml deleted file mode 100644 index 1d1e4b14..00000000 --- a/CKAN-extensions.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: Cloudformation Playbook - hosts: local - connection: local - vars_files: - - vars/CKAN-extensions.var.yml - pre_tasks: - - name: get basic_facts - set_fact: - basic_fact={{ item }} - when: item.Environment == Environment - with_items: "{{ basic_facts }}" - - - name: set facts to environment from basic_fact - set_fact: "{{ item.key }}={{ item.value }}" - with_dict: "{{ basic_fact }}" - when: basic_fact is defined - - - name: set EnvExtensions - set_fact: EnvExtensions={{ extensions[Environment] }} - - roles: - - ansible_cloudformation diff --git a/build-CKAN.sh b/build-CKAN.sh index f26b8d1e..5571d37a 100755 --- a/build-CKAN.sh +++ b/build-CKAN.sh @@ -53,8 +53,8 @@ run-all-playbooks () { run-playbook "database-config" fi run-playbook "CloudFormation" "vars/s3_buckets.var.yml" - run-playbook "instance-roles" run-playbook "CKAN-Stack" + run-playbook "CloudFormation" "vars/CKAN-extensions.var.yml" run-playbook "CloudFormation" "vars/instances-${INSTANCE_NAME}.var.yml" run-playbook "CloudFormation" "vars/cloudfront-lambda-at-edge.var.yml" run-playbook "cloudfront" diff --git a/instance-roles.yml b/instance-roles.yml deleted file mode 100644 index 90709b50..00000000 --- a/instance-roles.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Create IAM roles for CKAN servers - hosts: local - connection: local - - pre_tasks: - - name: get basic_facts - set_fact: - basic_fact={{ item }} - when: item.Environment == Environment - with_items: "{{ basic_facts }}" - - - name: set facts to environment from basic_fact - set_fact: "{{ item.key }}={{ item.value }}" - with_dict: "{{ basic_fact }}" - when: basic_fact is defined - - - set_fact: - acm_stack_name: "{{ service_name }}-{{ Environment }}-acm" - - - name: kms alias fact - aws_kms_facts: - filters: - alias: "aws/ssm" - region: "{{ region }}" - register: ssmKeyFacts - - - debug: - msg: "{{ ssmKeyFacts }}" - - - name: set KMS key from alias - set_fact: - SSMKey: "{{ ssmKeyFacts['keys'][0].key_arn }}" - - - include_vars: vars/instance-roles.var.yml - roles: - - ansible_cloudformation diff --git a/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 index 8df177cc..312b6552 100644 --- a/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2 @@ -28,25 +28,49 @@ Parameters: ConstraintDescription: Must contain only lowercase/numeric/hyphen/underscore. AllowedPattern: '[-_a-z0-9]*' - Resources: - {% for key in EnvExtensions %} + PluginListParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_app_names" + Type: String + Value: "{{ extensions[Environment] | join(',') }}" + +{% for plugin in extensions[Environment] %} + {{ plugin }}AppNameParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/name" + Type: String + Value: "{{ extensions[Environment][plugin].name }}" + + {{ plugin }}AppShortNameParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/shortname" + Type: String + Value: "{{ extensions[Environment][plugin].shortname }}" - {{ key }}: - Type: AWS::OpsWorks::App + {{ plugin }}AppSourceTypeParameter: + Type: AWS::SSM::Parameter Properties: - AppSource: - Type: "{{ EnvExtensions[key].type }}" - Url: "{{ EnvExtensions[key].url }}" - {% if EnvExtensions[key].version is defined %}Revision: "{{ EnvExtensions[key].version }}"{% endif %} - #new line required for jinja to not merge groups - Description: {{ EnvExtensions[key].description }} - EnableSsl: !!bool false - Name: "{{ EnvExtensions[key].name }}" - Shortname: "{{ EnvExtensions[key].shortname }}" - StackId: - Fn::ImportValue: !Ref OpsWorksStack - Type: "other" - - {% endfor %} + Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/app_source/type" + Type: String + Value: git + + {{ plugin }}AppSourceURLParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/app_source/url" + Type: String + Value: "{{ extensions[Environment][plugin].url }}" + + {{ plugin }}AppSourceRevisionParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/app_source/revision" + Type: String + Value: "{{ extensions[Environment][plugin].version }}" +{% endfor %} + diff --git a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 index 8a65f850..af44540e 100644 --- a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 @@ -136,6 +136,9 @@ Parameters: ACMCertificateARN: Description: ARN for certificate to be used by the application load balancer. Type: String + SSMKey: + Type: String + Description: KMS key for alias/aws/ssm GTMContainerId: Description: Google Tag Manager container ID Type: String @@ -186,21 +189,167 @@ Resources: Effect: Allow Resource: "*" + SmtpRelayPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + Description: Provides access to retrieve SSM parameters related to the AWS SMTP relay, including KMS decryption, and send emails. + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: ses:SendRawEmail + Resource: "*" + - Effect: Allow + Action: ssm:DescribeParameters + Resource: "*" + - Effect: Allow + Action: + - ssm:GetParameters + - ssm:GetParameter + - ssm:GetParametersByPath + Resource: + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/smtpRelay" + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/smtpRelay/*" + - Effect: Allow + Action: kms:Decrypt + Resource: !Ref SSMKey + + AttachmentsPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + Description: Provides access to s3 attachments bucket. + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - s3:AbortMultipartUpload + - s3:ListMultipartUploadParts + - s3:ListBucket + - s3:*Upload* + - s3:GetObject + - s3:GetObjectAcl + - s3:GetObjectVersion + - s3:PutObject + - s3:PutObjectAcl + - s3:PutObjectTagging + - s3:DeleteObject + - s3:DeleteObjectTagging + - s3:GetBucketAcl + - s3:GetBucketCORS + - s3:GetBucketPolicy + Resource: + - !Sub "arn:aws:s3:::${AttachmentsBucketName}" + - !Sub "arn:aws:s3:::${AttachmentsBucketName}/*" + - Effect: Allow + Action: + - s3:ListAllMyBuckets + Resource: + - !Sub "arn:aws:s3:::" + + EBSTaggingPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + Description: Allows EC2 instances to tag their EBS root volumes (eg for automatic backups) + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: ec2:CreateTags + Resource: "*" + + InstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: !Sub "${ApplicationName}-${Environment}-CustomPolicy" + PolicyDocument: + Version: "2012-10-17" + Statement: + # allow the service layer to update internal DNS + - Effect: Allow + Action: + - route53:ChangeResourceRecordSets + Resource: + Fn::Join: + - "" + - - "arn:aws:route53:::hostedzone/" + - Fn::ImportValue: !Ref InternalStackZone + # allow the servers to retrieve a subset of SSM Parameter Store values + - Effect: Allow + Action: + - ssm:GetParameter + - ssm:GetParameters + - ssm:GetParametersByPath + Resource: + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/*" + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/common/*" + ManagedPolicyArns: + - !Ref EBSTaggingPolicy + - !Ref SmtpRelayPolicy + - !Ref AttachmentsPolicy + - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess + - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM + - arn:aws:iam::aws:policy/AWSOpsWorksCloudWatchLogs + - arn:aws:iam::aws:policy/CloudFrontReadOnlyAccess # for domain name lookups + - arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess # for domain name lookups + InstanceRoleProfile: Type: AWS::IAM::InstanceProfile Properties: Path: / Roles: - - Fn::ImportValue: - !Sub "${Environment}${ApplicationName}InstanceRole" + - !Ref InstanceRole + + WebInstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: !Sub "${ApplicationName}-${Environment}-Web-CustomPolicy" + PolicyDocument: + Version: "2012-10-17" + Statement: + # allow the web layer to retrieve a subset of SSM Parameter Store values + - Effect: Allow + Action: + - ssm:GetParameter + - ssm:GetParameters + - ssm:GetParametersByPath + Resource: + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/app/${ApplicationId}/" + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/app/${ApplicationId}/*" + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/db/${ApplicationId}_*" + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/common/*" + ManagedPolicyArns: + - !Ref EBSTaggingPolicy + - !Ref SmtpRelayPolicy + - !Ref AttachmentsPolicy + - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess + - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM + - arn:aws:iam::aws:policy/AWSOpsWorksCloudWatchLogs + - arn:aws:iam::aws:policy/CloudFrontReadOnlyAccess # for domain name lookups + - arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess # for domain name lookups WebInstanceRoleProfile: Type: AWS::IAM::InstanceProfile Properties: Path: / Roles: - - Fn::ImportValue: - !Sub "${Environment}${ApplicationName}WebInstanceRole" + - !Ref WebInstanceRole # Populate SSM Parameter Store with all the information needed to deploy. # This should allow a blank instance to set itself up based only on the tags applied to it. @@ -332,50 +481,6 @@ Resources: Type: String Value: !Ref SolrSource - PluginListParameter: - Type: AWS::SSM::Parameter - Properties: - Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_app_names" - Type: String - Value: "{{ extensions[Environment] | join(',') }}" - -{% for plugin in extensions[Environment] %} - {{ plugin }}AppNameParameter: - Type: AWS::SSM::Parameter - Properties: - Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/name" - Type: String - Value: "{{ extensions[Environment][plugin].name }}" - - {{ plugin }}AppShortNameParameter: - Type: AWS::SSM::Parameter - Properties: - Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/shortname" - Type: String - Value: "{{ extensions[Environment][plugin].shortname }}" - - {{ plugin }}AppSourceTypeParameter: - Type: AWS::SSM::Parameter - Properties: - Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/app_source/type" - Type: String - Value: git - - {{ plugin }}AppSourceURLParameter: - Type: AWS::SSM::Parameter - Properties: - Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/app_source/url" - Type: String - Value: "{{ extensions[Environment][plugin].url }}" - - {{ plugin }}AppSourceRevisionParameter: - Type: AWS::SSM::Parameter - Properties: - Name: !Sub "/config/CKAN/${Environment}/app/${ApplicationId}/plugin_apps/{{ plugin }}/app_source/revision" - Type: String - Value: "{{ extensions[Environment][plugin].version }}" -{% endfor %} - OpsWorksStack: Type: AWS::OpsWorks::Stack Properties: diff --git a/templates/instance-roles.cfn.yml b/templates/instance-roles.cfn.yml deleted file mode 100644 index c9388b6d..00000000 --- a/templates/instance-roles.cfn.yml +++ /dev/null @@ -1,192 +0,0 @@ -AWSTemplateFormatVersion: 2010-09-09 -Description: "Creates IAM roles with permissions needed for CKAN servers" - -Parameters: - ApplicationName: - Description: Name of the application (ie. GI or Services, etc) - Type: String - ApplicationId: - Description: All-lowercase identifier for the application (eg 'gi', 'services', etc) - Type: String - ConstraintDescription: Must contain only lowercase/numeric/hyphen/underscore. - AllowedPattern: '[-_a-z0-9]*' - Environment: - Description: Select a stack version. - Type: String - Default: STAGING - AllowedValues: - - DEV - - TRAINING - - STAGING - - PROD - InternalStackZone: - Description: Exported name of the Route53 hosted zone ID. - Type: String - SSMKey: - Type: String - Description: KMS key for alias/aws/ssm - AttachmentsBucketName: - Description: Name of the S3 Attachment bucket. - Type: String - -Resources: - SmtpRelayPolicy: - Type: AWS::IAM::ManagedPolicy - Properties: - Description: Provides access to retrieve SSM parameters related to the AWS SMTP relay, including KMS decryption, and send emails. - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: ses:SendRawEmail - Resource: "*" - - Effect: Allow - Action: ssm:DescribeParameters - Resource: "*" - - Effect: Allow - Action: - - ssm:GetParameters - - ssm:GetParameter - - ssm:GetParametersByPath - Resource: - - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/smtpRelay" - - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/smtpRelay/*" - - Effect: Allow - Action: kms:Decrypt - Resource: !Ref SSMKey - - AttachmentsPolicy: - Type: AWS::IAM::ManagedPolicy - Properties: - Description: Provides access to s3 attachments bucket. - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - s3:AbortMultipartUpload - - s3:ListMultipartUploadParts - - s3:ListBucket - - s3:*Upload* - - s3:GetObject - - s3:GetObjectAcl - - s3:GetObjectVersion - - s3:PutObject - - s3:PutObjectAcl - - s3:PutObjectTagging - - s3:DeleteObject - - s3:DeleteObjectTagging - - s3:GetBucketAcl - - s3:GetBucketCORS - - s3:GetBucketPolicy - Resource: - - !Sub "arn:aws:s3:::${AttachmentsBucketName}" - - !Sub "arn:aws:s3:::${AttachmentsBucketName}/*" - - Effect: Allow - Action: - - s3:ListAllMyBuckets - Resource: - - !Sub "arn:aws:s3:::" - - EBSTaggingPolicy: - Type: AWS::IAM::ManagedPolicy - Properties: - Description: Allows EC2 instances to tag their EBS root volumes (eg for automatic backups) - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: ec2:CreateTags - Resource: "*" - - InstanceRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: ec2.amazonaws.com - Action: sts:AssumeRole - Policies: - - PolicyName: !Sub "${ApplicationName}-${Environment}-InstancePolicy" - PolicyDocument: - Version: "2012-10-17" - Statement: - # allow the service layer to update internal DNS - - Effect: Allow - Action: - - route53:ChangeResourceRecordSets - Resource: - Fn::Join: - - "" - - - "arn:aws:route53:::hostedzone/" - - Fn::ImportValue: !Ref InternalStackZone - # allow the servers to retrieve a subset of SSM Parameter Store values - - Effect: Allow - Action: - - ssm:GetParameter - - ssm:GetParameters - - ssm:GetParametersByPath - Resource: - - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/*" - - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/common/*" - ManagedPolicyArns: - - !Ref EBSTaggingPolicy - - !Ref SmtpRelayPolicy - - !Ref AttachmentsPolicy - - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess - - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM - - arn:aws:iam::aws:policy/AWSOpsWorksCloudWatchLogs - - arn:aws:iam::aws:policy/CloudFrontReadOnlyAccess # for domain name lookups - - arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess # for domain name lookups - - WebInstanceRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: ec2.amazonaws.com - Action: sts:AssumeRole - Policies: - - PolicyName: !Sub "${ApplicationName}-${Environment}-WebInstancePolicy" - PolicyDocument: - Version: "2012-10-17" - Statement: - # allow the web layer to retrieve a subset of SSM Parameter Store values - - Effect: Allow - Action: - - ssm:GetParameter - - ssm:GetParameters - - ssm:GetParametersByPath - Resource: - - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/app/${ApplicationId}/" - - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/app/${ApplicationId}/*" - - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/db/${ApplicationId}_*" - - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/common/*" - ManagedPolicyArns: - - !Ref EBSTaggingPolicy - - !Ref SmtpRelayPolicy - - !Ref AttachmentsPolicy - - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess - - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM - - arn:aws:iam::aws:policy/AWSOpsWorksCloudWatchLogs - - arn:aws:iam::aws:policy/CloudFrontReadOnlyAccess # for domain name lookups - - arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess # for domain name lookups - -Outputs: - InstanceRole: - Value: !Ref InstanceRole - Description: Resource identifier for the global instance role - Export: - Name: !Sub "${Environment}${ApplicationName}InstanceRole" - - WebInstanceRole: - Value: !Ref WebInstanceRole - Description: Resource identifier for the web instance role - Export: - Name: !Sub "${Environment}${ApplicationName}WebInstanceRole" diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index c5014078..0bac8dc1 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -39,6 +39,7 @@ common_stack: &common_stack DatabaseSG: "{{ Environment }}CKANDatabaseSG" CreateServiceRole: "yes" EnableDataStore: "{{ enable_datastore | default('no') }}" + SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" CookbookRevision: "{{ CookbookRevision | default('7.0.3') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" diff --git a/vars/instance-roles.var.yml b/vars/instance-roles.var.yml deleted file mode 100644 index 8616140e..00000000 --- a/vars/instance-roles.var.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- - -cloudformation_stacks: - - name: "{{ service_name }}-{{ Environment }}-instance-roles" - state: "{{ state | default('present')}}" - region: "{{ region }}" - disable_rollback: true - template: "templates/instance-roles.cfn.yml" - template_parameters: - ApplicationName: "{{ service_name }}" - ApplicationId: "{{ service_name | lower }}" - Environment: "{{ Environment }}" - InternalStackZone: "{{ Environment }}CKANPrivateHostedZone" - SSMKey: "{{ SSMKey | default('') }}" - AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket - tags: - Environment: "{{ Environment }}" - Service: "{{ service_name }}" - Division: "{{ Division }}" - Owner: "{{ Owner }}" - Version: "1.0" From 9c54ed549a9e8fbe6ff8760f51e7eac9c03ba2c0 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 4 Apr 2024 09:33:06 +1000 Subject: [PATCH 2/6] [QOLDEV-545] update cookbook - Fix Recline previews - Manage Solr via Supervisord instead of initd or systemd, for greater consistency - Use Parameter Store values instead of custom JSON, so we can run commands on server start without an active deployment --- vars/CKAN-Stack.var.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index 0bac8dc1..c0bb77a6 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -41,7 +41,7 @@ common_stack: &common_stack EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - CookbookRevision: "{{ CookbookRevision | default('7.0.3') }}" + CookbookRevision: "{{ CookbookRevision | default('7.0.4') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket SolrSource: "{{ solr_url }}" From 0662007e927209212d3683066cd26099d4ffbe30 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 4 Apr 2024 13:53:52 +1000 Subject: [PATCH 3/6] [QOLSVC-5373] update Resource Type Validation plugin - Treat PPT and PPTX files as interchangeable --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- vars/shared-Publications.var.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 09dd8c0f..1daa07f2 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -135,7 +135,7 @@ extensions: description: "CKAN Extension for checking uploaded resource types" type: "git" url: "https://github.com/qld-gov-au/ckanext-resource-type-validation.git" - version: "1.0.9" + version: "1.0.10" CKANExtCSRFFilter: &CKANExtCSRFFilter name: "ckanext-csrf-filter-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index ccca0107..326006c5 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -135,7 +135,7 @@ extensions: description: "CKAN Extension for checking uploaded resource types" type: "git" url: "https://github.com/qld-gov-au/ckanext-resource-type-validation.git" - version: "1.0.9" + version: "1.0.10" CKANExtCSRFFilter: &CKANExtCSRFFilter name: "ckanext-csrf-filter-{{ Environment }}" diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index ab332bf6..a076a452 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -46,7 +46,7 @@ extensions: description: "CKAN Extension for checking uploaded resource types" type: "git" url: "https://github.com/qld-gov-au/ckanext-resource-type-validation.git" - version: "1.0.9" + version: "1.0.10" CKANExtCSRFFilter: &CKANExtCSRFFilter name: "ckanext-csrf-filter-{{ Environment }}" From d7e88c18c3163413d92323f06e019831b6d205c9 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 4 Apr 2024 14:51:10 +1000 Subject: [PATCH 4/6] [QOLSVC-5373] pin 'typer' library to avoid version conflict - Newer 'typer' wants Click 8+ but the Validation plugin isn't ready --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0e80947f..160db40d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,3 +9,4 @@ mock pyOpenSSL>=23.0.0 pytest-ckan selenium<4.10 +typer<0.12 From dc86f95cce59a620d477578c342a49caf5a523ba Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 5 Apr 2024 10:04:51 +1000 Subject: [PATCH 5/6] [QOLSVC-5373] update cookbook to pin 'typer' on live systems --- vars/CKAN-Stack.var.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index 9d70b6c0..cefdcc37 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -41,7 +41,7 @@ common_stack: &common_stack EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - CookbookRevision: "{{ CookbookRevision | default('7.0.4') }}" + CookbookRevision: "{{ CookbookRevision | default('7.0.5') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket SolrSource: "{{ solr_url }}" From 061509821fcc625d7d3c4f10bb253206b4478418 Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 5 Apr 2024 10:16:51 +1000 Subject: [PATCH 6/6] [QOLDEV-749] update CKAN fork to enable en_AU locale in Data Tables --- vars/CKAN-Stack.var.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index 9d70b6c0..0fb28f48 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -5,7 +5,7 @@ NonProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdNonProduction' ProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdProduction', region=region) }}" solr_url: "http://archive.apache.org/dist/lucene/solr/8.11.2/solr-8.11.2.zip" -ckan_tag: "ckan-2.10.4-qgov.2" +ckan_tag: "ckan-2.10.4-qgov.3" ckan_qgov_branch: "qgov-master-2.10.4" common_stack: &common_stack