From 1492e33bf62cfe04fb8eaeee361a5d7e05a4e10f Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 13 Jul 2022 12:18:01 +1000 Subject: [PATCH 01/40] [QOL-9055] update Geoscience Harvester to fix Py3 compatibility --- vars/shared-CKANTest.var.yml | 6 +++++- vars/shared-OpenData.var.yml | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 398a0df5..78f84a09 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -111,7 +111,7 @@ extensions: description: "CKAN Extension for Data Qld Harvesting Enhancements for Geoscience" type: "git" url: "https://github.com/qld-gov-au/ckanext-harvester-data-qld-geoscience" - version: "v0.0.5" + version: "v0.0.6" CKANExtReport: &CKANExtReport name: "ckanext-report-{{ Environment }}" @@ -224,6 +224,10 @@ extensions: <<: *CKANExtCSRFFilter version: "main" + CKANExtHarvestDataQldGeoScience: + <<: *CKANExtHarvestDataQldGeoScience + version: "master" + basic_facts: - Environment: TRAINING PublicStackZoneTLD: "{{ public_tld }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 13ac4f5f..35374673 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -111,8 +111,7 @@ extensions: description: "CKAN Extension for Data Qld Harvesting Enhancements" type: "git" url: "https://github.com/qld-gov-au/ckanext-harvester-data-qld-geoscience" - version: "v0.0.5" - + version: "v0.0.6" CKANExtReport: &CKANExtReport name: "ckanext-report-{{ Environment }}" @@ -225,6 +224,10 @@ extensions: <<: *CKANExtCSRFFilter version: "develop" + CKANExtHarvestDataQldGeoScience: + <<: *CKANExtHarvestDataQldGeoScience + version: "develop" + basic_facts: - Environment: PROD PublicStackZoneTLD: "{{ public_tld }}" From d64bf3eb4628d3a40e97f31b76e6deaf4e6cfc1d Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 13 Jul 2022 14:34:12 +1000 Subject: [PATCH 02/40] [QOL-9055] update QA plugin to fix Py3 compatibility --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 78f84a09..1147f833 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -135,7 +135,7 @@ extensions: description: "CKAN Extension for Quality Assurance" type: "git" url: "https://github.com/qld-gov-au/ckanext-qa.git" - version: "2.0.3-qgov.4" + version: "2.0.3-qgov.5" CKANExtResourceTypeValidation: &CKANExtResourceTypeValidation name: "ckanext-resource-type-validation-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 35374673..0e6f2ce9 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -135,7 +135,7 @@ extensions: description: "CKAN Extension for Quality Assurance" type: "git" url: "https://github.com/qld-gov-au/ckanext-qa.git" - version: "2.0.3-qgov.4" + version: "2.0.3-qgov.5" CKANExtResourceTypeValidation: &CKANExtResourceTypeValidation name: "ckanext-resource-type-validation-{{ Environment }}" From f5bdca8dcae976b9d8508a4e2052d31b8622e649 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 13 Jul 2022 15:15:53 +1000 Subject: [PATCH 03/40] [QOL-9055] update Data Requests to fix Py3 compatibility - Also fix a bug when moving to Flask, so editing a data request will redirect properly afterward --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 1147f833..43b07ae7 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -87,7 +87,7 @@ extensions: description: "CKAN Extension for Data Requests" type: "git" url: "https://github.com/qld-gov-au/ckanext-datarequests.git" - version: "2.2.1-qgov" + version: "2.2.1-qgov.2" CKANExtYTP: &CKANExtYTP name: "ckanext-ytp-comments-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 0e6f2ce9..98cf36e6 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -87,7 +87,7 @@ extensions: description: "CKAN Extension for Data Requests" type: "git" url: "https://github.com/qld-gov-au/ckanext-datarequests.git" - version: "2.2.1-qgov" + version: "2.2.1-qgov.2" CKANExtYTP: &CKANExtYTP name: "ckanext-ytp-comments-{{ Environment }}" From 313c1f67dedac40301fc89d20c498dd8369def6b Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 13 Jul 2022 20:59:17 +1000 Subject: [PATCH 04/40] [QOL-9055] update resource type validation to fix Py3 compatibility --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- vars/shared-Publications.var.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 43b07ae7..af35bdac 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -143,7 +143,7 @@ extensions: description: "CKAN Extension for checking uploaded resource types" type: "git" url: "https://github.com/qld-gov-au/ckanext-resource-type-validation.git" - version: "1.0.2" + version: "1.0.3" CKANExtCSRFFilter: &CKANExtCSRFFilter name: "ckanext-csrf-filter-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 98cf36e6..97c47f81 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -143,7 +143,7 @@ extensions: description: "CKAN Extension for checking uploaded resource types" type: "git" url: "https://github.com/qld-gov-au/ckanext-resource-type-validation.git" - version: "1.0.2" + version: "1.0.3" CKANExtCSRFFilter: &CKANExtCSRFFilter name: "ckanext-csrf-filter-{{ Environment }}" diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index f8ebb217..d9f407f3 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -46,7 +46,7 @@ extensions: description: "CKAN Extension for checking uploaded resource types" type: "git" url: "https://github.com/qld-gov-au/ckanext-resource-type-validation.git" - version: "1.0.2" + version: "1.0.3" CKANExtCSRFFilter: &CKANExtCSRFFilter name: "ckanext-csrf-filter-{{ Environment }}" From 52fa07f204793b885834cf5314c1a8f96d7c5156 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 14 Jul 2022 23:44:36 +1000 Subject: [PATCH 05/40] [QOL-9055] drop duplicate configs --- .docker/test-OpenData.ini | 2 +- vars/shared-CKANTest.var.yml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.docker/test-OpenData.ini b/.docker/test-OpenData.ini index 7971196e..2b91fcf6 100644 --- a/.docker/test-OpenData.ini +++ b/.docker/test-OpenData.ini @@ -97,7 +97,7 @@ ckan.redis.url = redis://redis:6379 # Add ``datapusher`` to enable DataPusher # Add ``resource_proxy`` to enable resource proxying and get around the # same origin policy -ckan.plugins = stats text_view image_view recline_view datastore data_qld scheming_datasets validation dcat qa archiver report data_qld_reporting data_qld_google_analytics harvester_data_qld_geoscience harvest qgovext ytp_comments datarequests xloader csrf_filter resource_type_validation +ckan.plugins = stats text_view image_view recline_view datastore data_qld data_qld_google_analytics scheming_datasets validation dcat qa archiver report harvester_data_qld_geoscience harvest qgovext ytp_comments datarequests xloader csrf_filter resource_type_validation ssm_config odi_certificates # Define which views should be created by default # (plugins must be loaded in ckan.plugins) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index af35bdac..f20b98ac 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -224,10 +224,6 @@ extensions: <<: *CKANExtCSRFFilter version: "main" - CKANExtHarvestDataQldGeoScience: - <<: *CKANExtHarvestDataQldGeoScience - version: "master" - basic_facts: - Environment: TRAINING PublicStackZoneTLD: "{{ public_tld }}" From 8a3c9d1e6217151cc7de71e88d77b70e963e60ed Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 15 Jul 2022 09:19:46 +1000 Subject: [PATCH 06/40] [QOL-9055] update YTP Comments to better handle Python 3 --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index f20b98ac..628012cc 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.2" + version: "2.5.0-qgov.3" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 97c47f81..18346346 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.2" + version: "2.5.0-qgov.3" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" From 57040da12f0f774f3e605b95e2533f6d933d83a1 Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 15 Jul 2022 14:35:54 +1000 Subject: [PATCH 07/40] [QOL-9055] update Archiver to remain Py2 compatible --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 628012cc..6d75ff9f 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -127,7 +127,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.8" + version: "2.1.1-qgov.9" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 18346346..b2857997 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -127,7 +127,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.8" + version: "2.1.1-qgov.9" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" From 45eb8b7042e5f12b2004a1f96adb772bcf304f0d Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 26 Jul 2022 09:35:22 +1000 Subject: [PATCH 08/40] [QOL-9055] update Validation to be ready for Python 3 --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 6d75ff9f..94b69a05 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -63,7 +63,7 @@ extensions: description: "CKAN Extension for validating Data Packages" type: "git" url: "https://github.com/qld-gov-au/ckanext-validation.git" - version: "v0.0.8-qgov.4" + version: "v0.0.8-qgov.5" CKANExtDataQld: &CKANExtDataQld name: "ckanext-data-qld-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index b2857997..c8f5036c 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -63,7 +63,7 @@ extensions: description: "CKAN Extension for validating Data Packages" type: "git" url: "https://github.com/qld-gov-au/ckanext-validation.git" - version: "v0.0.8-qgov.4" + version: "v0.0.8-qgov.5" CKANExtDataQld: &CKANExtDataQld name: "ckanext-data-qld-{{ Environment }}" From e29e3014a9ab0b747c4fa64e24746d67754954c1 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 26 Jul 2022 11:10:16 +1000 Subject: [PATCH 09/40] [DC-32816] update CKAN fork to patch sqlparse - This was applied to 2.8 and upstream 2.10, but was overlooked for 2.9 --- vars/CKAN-Stack.var.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index d625c46e..fa722f9e 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -5,7 +5,7 @@ NonProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdNonProduction' ProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdProduction', region=region) }}" solr7: "http://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.zip" -ckan_tag: "ckan-2.9.5-qgov.7" +ckan_tag: "ckan-2.9.5-qgov.8" ckan_qgov_branch: "qgov-master-2.9.5" common_stack: &common_stack From 79fc3c2b4b5e53fd3945e0d36ab4d7874d51b1c1 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 26 Jul 2022 11:24:22 +1000 Subject: [PATCH 10/40] [QOL-9055] update QGOV extension to handle Python 3 --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- vars/shared-Publications.var.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 94b69a05..898cd87a 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -23,7 +23,7 @@ extensions: description: "CKAN Extension for Queensland Government Open Data" type: "git" url: "https://github.com/qld-gov-au/ckan-ex-qgov.git" - version: "5.0.2" + version: "5.0.3" CKANExtS3Filestore: &CKANExtS3Filestore name: "ckanext-s3filestore-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index c8f5036c..a76b8a7e 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -23,7 +23,7 @@ extensions: description: "CKAN Extension for Queensland Government Open Data" type: "git" url: "https://github.com/qld-gov-au/ckan-ex-qgov.git" - version: "5.0.2" + version: "5.0.3" CKANExtS3Filestore: &CKANExtS3Filestore name: "ckanext-s3filestore-{{ Environment }}" diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index d9f407f3..0b240864 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -14,7 +14,7 @@ extensions: description: "CKAN Extension for Queensland Government sites" type: "git" url: "https://github.com/qld-gov-au/ckan-ex-qgov.git" - version: "5.0.2" + version: "5.0.3" CKANExtS3Filestore: &CKANExtS3Filestore name: "ckanext-s3filestore-{{ Environment }}" From e82e1df5122e088cc0fbb4a49a08df1197de997e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 26 Jul 2022 11:26:20 +1000 Subject: [PATCH 11/40] [QOL-9055] update QGOV extension to preferred URL --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- vars/shared-Publications.var.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 898cd87a..f6ee393a 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -22,7 +22,7 @@ extensions: shortname: "ckanext-qgov" description: "CKAN Extension for Queensland Government Open Data" type: "git" - url: "https://github.com/qld-gov-au/ckan-ex-qgov.git" + url: "https://github.com/qld-gov-au/ckanext-qgov.git" version: "5.0.3" CKANExtS3Filestore: &CKANExtS3Filestore diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index a76b8a7e..9dddf109 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -22,7 +22,7 @@ extensions: shortname: "ckanext-qgov" description: "CKAN Extension for Queensland Government Open Data" type: "git" - url: "https://github.com/qld-gov-au/ckan-ex-qgov.git" + url: "https://github.com/qld-gov-au/ckanext-qgov.git" version: "5.0.3" CKANExtS3Filestore: &CKANExtS3Filestore diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index 0b240864..1a969b10 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -13,7 +13,7 @@ extensions: shortname: "ckanext-qgov" description: "CKAN Extension for Queensland Government sites" type: "git" - url: "https://github.com/qld-gov-au/ckan-ex-qgov.git" + url: "https://github.com/qld-gov-au/ckanext-qgov.git" version: "5.0.3" CKANExtS3Filestore: &CKANExtS3Filestore From 3519bebb96514bf9f202a64019e358088ca80f89 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 26 Jul 2022 13:48:23 +1000 Subject: [PATCH 12/40] [QOL-9055] switch Harvester extension to 'master' branch on test-dev - This matches the other plugins. 'develop' branch is for dev.data.qld.gov.au and dev.publications, 'master'/'main' branch is for test-dev.data.qld.gov.au --- vars/shared-CKANTest.var.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index f6ee393a..7729043e 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -186,7 +186,7 @@ extensions: CKANExtHarvestDataQldGeoScience: <<: *CKANExtHarvestDataQldGeoScience - version: "develop" + version: "master" CKANExtODICertificates: <<: *CKANExtODICertificates From e832ae91fee0a57da88521b6dab59775d901acfc Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 26 Jul 2022 13:53:16 +1000 Subject: [PATCH 13/40] [QOL-9055] update YTP Comments to handle Python 3 --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 7729043e..dfc67d65 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.3" + version: "2.5.0-qgov.4" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 9dddf109..a408d72a 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.3" + version: "2.5.0-qgov.4" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" From 752b2dcfaf56c2a9b886817ee0470c92eefe6102 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 27 Jul 2022 10:07:06 +1000 Subject: [PATCH 14/40] [QOL-9055] update S3 Filestore to fix redirects --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- vars/shared-Publications.var.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index f8949525..497881bd 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -31,7 +31,7 @@ extensions: description: "CKAN Extension to keep uploaded files in S3" type: "git" url: "https://github.com/qld-gov-au/ckanext-s3filestore.git" - version: "0.7.7-qgov.2" + version: "0.7.7-qgov.3" CKANExtSSMConfig: &CKANExtSSMConfig name: "ckanext-ssm-config-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index f44ed6e1..ddc507ad 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -31,7 +31,7 @@ extensions: description: "CKAN Extension to keep uploaded files in S3" type: "git" url: "https://github.com/qld-gov-au/ckanext-s3filestore.git" - version: "0.7.7-qgov.2" + version: "0.7.7-qgov.3" CKANExtSSMConfig: &CKANExtSSMConfig name: "ckanext-ssm-config-{{ Environment }}" diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index 60c37366..318735b7 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -22,7 +22,7 @@ extensions: description: "CKAN Extension to keep uploaded files in S3" type: "git" url: "https://github.com/qld-gov-au/ckanext-s3filestore.git" - version: "0.7.7-qgov.2" + version: "0.7.7-qgov.3" CKANExtSSMConfig: &CKANExtSSMConfig name: "ckanext-ssm-config-{{ Environment }}" From 0f10ef5fcf104c2b83bdfc95f895f0590b5f9a70 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 27 Jul 2022 10:56:54 +1000 Subject: [PATCH 15/40] [QOL-9055] update CSRF filter to fix aborts --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- vars/shared-Publications.var.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 497881bd..8665788e 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -151,7 +151,7 @@ extensions: description: "CKAN Extension for preventing Cross-Site Request Forgery attacks" type: "git" url: "https://github.com/qld-gov-au/ckanext-csrf-filter.git" - version: "1.1.4" + version: "1.1.5" PROD: <<: *default_extensions diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index ddc507ad..a9f81afb 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -151,7 +151,7 @@ extensions: description: "CKAN Extension for preventing Cross-Site Request Forgery attacks" type: "git" url: "https://github.com/qld-gov-au/ckanext-csrf-filter.git" - version: "1.1.4" + version: "1.1.5" PROD: <<: *default_extensions diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index 318735b7..fd368939 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -54,7 +54,7 @@ extensions: description: "CKAN Extension for preventing Cross-Site Request Forgery attacks" type: "git" url: "https://github.com/qld-gov-au/ckanext-csrf-filter.git" - version: "1.1.4" + version: "1.1.5" PROD: <<: *default_extensions From 8613f31fe45cd1904b543fe2fab8e56ab22d9ed5 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 27 Jul 2022 13:36:20 +1000 Subject: [PATCH 16/40] [QOL-9055] update YTP Comments to fix aborts and define needed Webassets bundles --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 8665788e..de6f7987 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.4" + version: "2.5.0-qgov.5" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index a9f81afb..571e810e 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.4" + version: "2.5.0-qgov.5" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" From 05f58d3794b00596546a1abbf50592bd468c5cb0 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 27 Jul 2022 14:22:54 +1000 Subject: [PATCH 17/40] [QOL-9055] update Publications theme to handle Py3 --- vars/shared-Publications.var.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index fd368939..45b44759 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -38,7 +38,7 @@ extensions: type: "git" description: "CKAN Extension for Queensland Government Publications theme" url: "https://github.com/qld-gov-au/ckanext-publications-qld-theme.git" - version: "1.1.5" + version: "1.1.6" CKANExtResourceTypeValidation: &CKANExtResourceTypeValidation name: "ckanext-resource-type-validation-{{ Environment }}" From e1c34ac79c89540b01824cb4b4867a36ea7919ff Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 27 Jul 2022 14:24:37 +1000 Subject: [PATCH 18/40] [QOL-9055] run tests on both Py2 and Py3 --- .../{Dockerfile.ckan => Dockerfile-py2.ckan} | 3 +- .docker/Dockerfile-py3.ckan | 53 +++++++++++++++++++ .docker/scripts/create-test-data.sh | 18 +++---- .docker/scripts/extract-id.py | 5 ++ .docker/scripts/init-ext.sh | 24 ++++++++- .docker/test-OpenData.ini | 2 + .github/workflows/test-staging.yml | 6 ++- .github/workflows/test.yml | 6 ++- docker-compose.yml | 2 +- 9 files changed, 101 insertions(+), 18 deletions(-) rename .docker/{Dockerfile.ckan => Dockerfile-py2.ckan} (96%) create mode 100644 .docker/Dockerfile-py3.ckan create mode 100644 .docker/scripts/extract-id.py diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile-py2.ckan similarity index 96% rename from .docker/Dockerfile.ckan rename to .docker/Dockerfile-py2.ckan index b664b2fc..8dd1a6da 100644 --- a/.docker/Dockerfile.ckan +++ b/.docker/Dockerfile-py2.ckan @@ -2,10 +2,11 @@ FROM amazeeio/python:2.7-ckan-21.8.0 ARG SITE_URL=http://ckan:3000/ ARG CKAN_REPO=qld-gov-au/ckan -ARG CKAN_VERSION=ckan-2.8.8-qgov.5 +ARG CKAN_VERSION=ckan-2.9.5-qgov.8 ARG VARS_TYPE ARG DEPLOY_ENV ENV VARS_TYPE="${VARS_TYPE}" +ENV PYTHON_VERSION=py2 ENV SITE_URL="${SITE_URL}" ENV DEPLOY_ENV="${DEPLOY_ENV}" ENV VENV_DIR=/app/ckan/default diff --git a/.docker/Dockerfile-py3.ckan b/.docker/Dockerfile-py3.ckan new file mode 100644 index 00000000..57ce91e3 --- /dev/null +++ b/.docker/Dockerfile-py3.ckan @@ -0,0 +1,53 @@ +FROM amazeeio/python:3.8-22.7.0 + +ARG SITE_URL=http://ckan:3000/ +ARG CKAN_REPO=qld-gov-au/ckan +ARG CKAN_VERSION=ckan-2.9.5-qgov.8 +ARG VARS_TYPE +ARG DEPLOY_ENV +ENV VARS_TYPE="${VARS_TYPE}" +ENV PYTHON_VERSION=py3 +ENV SITE_URL="${SITE_URL}" +ENV DEPLOY_ENV="${DEPLOY_ENV}" +ENV VENV_DIR=/app/ckan/default +ENV APP_DIR=/app +ENV CKAN_INI=/app/ckan/default/production.ini + +WORKDIR "${APP_DIR}" + +ENV DOCKERIZE_VERSION v0.6.1 +RUN apk add --no-cache curl build-base git libxml2-dev libxslt-dev postgresql postgresql-client postgresql-dev \ + && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ + && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ + && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz + +# Install CKAN. + +RUN mkdir -p ${VENV_DIR} \ + && virtualenv ${VENV_DIR} \ + && . ${VENV_DIR}/bin/activate \ + && pip install setuptools==44.1.0 \ + && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \ + && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \ + && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ + && deactivate \ + && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ + && fix-permissions ${APP_DIR}/ckan + +COPY .docker/test-$VARS_TYPE.ini $CKAN_INI + +COPY . ${APP_DIR}/ + +COPY .docker/scripts ${APP_DIR}/scripts + +COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/ + +COPY vars/shared-$VARS_TYPE.var.yml $APP_DIR/scripts/extensions.yml + +RUN chmod +x ${APP_DIR}/scripts/*.sh ${VENV_DIR}/bin/ckan_cli + +# Init current extension. +RUN ${APP_DIR}/scripts/init-ext.sh + +ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"] +CMD ["/app/scripts/serve.sh"] diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index e910a764..47bca0f1 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -6,15 +6,14 @@ set -e set -x CKAN_ACTION_URL=http://ckan:3000/api/action +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi -CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" -CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" -CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" - add_user_if_needed () { echo "Adding user '$2' ($1) with email address [$3]" ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\ @@ -26,7 +25,6 @@ add_user_if_needed () { add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" ckan_cli sysadmin add "${CKAN_USER_NAME}" -# We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') if [ "$API_KEY" = "None" ]; then echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..." @@ -46,7 +44,7 @@ add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost add_user_if_needed test_org_member "Test Member" test_org_member@localhost -echo "Creating ${TEST_ORG_TITLE} Organisation:" +echo "Creating ${TEST_ORG_TITLE} organisation:" TEST_ORG=$( \ curl -LsH "Authorization: ${API_KEY}" \ @@ -54,9 +52,9 @@ TEST_ORG=$( \ ${CKAN_ACTION_URL}/organization_create ) -TEST_ORG_ID=$(echo $TEST_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') +TEST_ORG_ID=$(echo $TEST_ORG | python $APP_DIR/scripts/extract-id.py) -echo "Assigning test users to ${TEST_ORG_TITLE} Organisation:" +echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):" curl -LsH "Authorization: ${API_KEY}" \ --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \ @@ -174,7 +172,7 @@ DR_ORG=$( \ ${CKAN_ACTION_URL}/organization_create ) -DR_ORG_ID=$(echo $DR_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') +DR_ORG_ID=$(echo $DR_ORG | python $APP_DIR/scripts/extract-id.py) echo "Assigning test users to ${DR_ORG_TITLE} Organisation:" @@ -228,7 +226,7 @@ REPORT_ORG=$( \ ${CKAN_ACTION_URL}/organization_create ) -REPORT_ORG_ID=$(echo $REPORT_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') +REPORT_ORG_ID=$(echo $REPORT_ORG | python $APP_DIR/scripts/extract-id.py) echo "Assigning test users to ${REPORT_ORG_TITLE} Organisation:" diff --git a/.docker/scripts/extract-id.py b/.docker/scripts/extract-id.py new file mode 100644 index 00000000..ae9cf79d --- /dev/null +++ b/.docker/scripts/extract-id.py @@ -0,0 +1,5 @@ +# encoding: utf-8 +import json +import sys + +print(json.loads(sys.stdin.read())['result']['id']) diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh index 50d4035a..561f009d 100755 --- a/.docker/scripts/init-ext.sh +++ b/.docker/scripts/init-ext.sh @@ -4,6 +4,26 @@ # set -e +install_requirements () { + PROJECT_DIR="$1" + for filename in requirements-$PYTHON_VERSION.txt requirements.txt pip-requirements.txt; do + if [ -f "$PROJECT_DIR/$filename" ]; then + pip install -r "$PROJECT_DIR/$filename" + return 0 + fi + done +} + +install_dev_requirements () { + PROJECT_DIR="$1" + for filename in dev-requirements-$PYTHON_VERSION.txt requirements-dev-$PYTHON_VERSION.txt requirements-dev.txt dev-requirements.txt; do + if [ -f "$PROJECT_DIR/$filename" ]; then + pip install -r "$PROJECT_DIR/$filename" + return 0 + fi + done +} + if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi @@ -11,8 +31,8 @@ pip install -r "requirements.txt" pip install -r "requirements-dev.txt" EXTENSIONS_FILE=$APP_DIR/scripts/extensions.yml python $(dirname $0)/generate-ext-requirements.py pip install --force-reinstall -r "/tmp/requirements-ext.txt" -for req in `ls $VENV_DIR/src/ckanext-*/requirements.txt`; do - pip install -r "$req" +for extension in . `ls -d $VENV_DIR/src/ckanext-*`; do + install_requirements $extension done if [ "$VENV_DIR" != "" ]; then diff --git a/.docker/test-OpenData.ini b/.docker/test-OpenData.ini index 2b91fcf6..93bba37b 100644 --- a/.docker/test-OpenData.ini +++ b/.docker/test-OpenData.ini @@ -142,6 +142,8 @@ ckan.storage_path = /app/filestore #ckan.datapusher.url = http://127.0.0.1:8800/ #ckan.datapusher.assume_task_stale_after = 3600 +ckanext.xloader.jobs_db.uri = postgresql://ckan:ckan@postgres/ckan?sslmode=disable + # Resource Proxy settings # Preview size limit, default: 1MB #ckan.resource_proxy.max_file_size = 1048576 diff --git a/.github/workflows/test-staging.yml b/.github/workflows/test-staging.yml index ab859c83..3a64676e 100644 --- a/.github/workflows/test-staging.yml +++ b/.github/workflows/test-staging.yml @@ -13,16 +13,18 @@ jobs: strategy: matrix: application: [OpenData, Publications] + python-version: [py2, py3] target_environment: [STAGING] fail-fast: true - name: ${{ matrix.application }} ${{ matrix.target_environment }} infrastructure build + name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} infrastructure build runs-on: ubuntu-latest container: integratedexperts/ci-builder env: VARS_TYPE: ${{ matrix.application }} DEPLOY_ENV: ${{ matrix.target_environment }} CKAN_REPO: qld-gov-au/ckan + PYTHON_VERSION: ${{ matrix.python-version }} steps: - uses: actions/checkout@v2 @@ -45,6 +47,6 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: ${{ matrix.application }} ${{ matrix.target_environment }} screenshots + name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} screenshots path: /tmp/artifacts/behave/screenshots timeout-minutes: 3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed40b786..5dd86798 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,16 +10,18 @@ jobs: strategy: matrix: application: [OpenData, Publications] + python-version: [py2, py3] target_environment: [DEV] fail-fast: false - name: ${{ matrix.application }} ${{ matrix.target_environment }} infrastructure build + name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} infrastructure build runs-on: ubuntu-latest container: integratedexperts/ci-builder env: VARS_TYPE: ${{ matrix.application }} DEPLOY_ENV: ${{ matrix.target_environment }} CKAN_REPO: qld-gov-au/ckan + PYTHON_VERSION: ${{ matrix.python-version }} steps: - uses: actions/checkout@v2 @@ -42,6 +44,6 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: ${{ matrix.application }} ${{ matrix.target_environment }} screenshots + name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} screenshots path: /tmp/artifacts/behave/screenshots timeout-minutes: 1 diff --git a/docker-compose.yml b/docker-compose.yml index 878e1e6d..be0306f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: ckan: build: context: . - dockerfile: .docker/Dockerfile.ckan + dockerfile: .docker/Dockerfile-${PYTHON_VERSION}.ckan args: SITE_URL: "http://${PROJECT}.docker.amazee.io" VARS_TYPE: From be9eba5abf60e384807badf1d627b63cf1980b61 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 27 Jul 2022 15:57:31 +1000 Subject: [PATCH 19/40] [QOL-9055] update QA to handle Python 3 --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index de6f7987..1ea38ca5 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -135,7 +135,7 @@ extensions: description: "CKAN Extension for Quality Assurance" type: "git" url: "https://github.com/qld-gov-au/ckanext-qa.git" - version: "2.0.3-qgov.5" + version: "2.0.3-qgov.6" CKANExtResourceTypeValidation: &CKANExtResourceTypeValidation name: "ckanext-resource-type-validation-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 571e810e..9a3201ec 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -135,7 +135,7 @@ extensions: description: "CKAN Extension for Quality Assurance" type: "git" url: "https://github.com/qld-gov-au/ckanext-qa.git" - version: "2.0.3-qgov.5" + version: "2.0.3-qgov.6" CKANExtResourceTypeValidation: &CKANExtResourceTypeValidation name: "ckanext-resource-type-validation-{{ Environment }}" From 3751bc0dca628747e08ea4252e2bda8ddb399194 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 28 Jul 2022 08:57:29 +1000 Subject: [PATCH 20/40] [QOL-9055] update QGOV extension to fix redirects --- test/features/data_qld_theme.feature | 5 ----- test/features/data_usability_rating.feature | 1 + test/features/datarequest.feature | 4 ++-- test/features/engagement_reporting.feature | 2 +- test/features/groups.feature | 8 ++++---- test/features/organisations.feature | 8 ++++---- test/features/resource_freshness.feature | 1 + test/features/search_facets.feature | 2 +- test/features/user_creation.feature | 8 ++++++++ test/features/users.feature | 14 +++++++------- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- vars/shared-Publications.var.yml | 2 +- 13 files changed, 32 insertions(+), 27 deletions(-) diff --git a/test/features/data_qld_theme.feature b/test/features/data_qld_theme.feature index e88077cb..e65eb598 100644 --- a/test/features/data_qld_theme.feature +++ b/test/features/data_qld_theme.feature @@ -85,11 +85,6 @@ Feature: Theme customisations Then I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=json') and contains(string(), 'JSON')]" Then I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=xml') and contains(string(), 'XML')]" - @Publications - Scenario: Homepage 'Publish in the Gazettes' link is correct - When I go to homepage - Then I should see an element with xpath "//a[@href='https://www.forgov.qld.gov.au/information-and-communication-technology/communication-and-publishing/website-and-digital-publishing/queensland-government-gazette/publish-in-the-gazette' and string()='Publish in the Gazettes']" - @Publications Scenario: Menu items are present and correct When I go to "/dataset" diff --git a/test/features/data_usability_rating.feature b/test/features/data_usability_rating.feature index 7cd5633f..f23e81fb 100644 --- a/test/features/data_usability_rating.feature +++ b/test/features/data_usability_rating.feature @@ -17,4 +17,5 @@ Feature: Data usability rating | TXT | txt_resource.txt | 1 | | XLS | xls_resource.xls | 2 | | CSV | csv_resource.csv | 3 | + | JSON | csv_resource.csv | 3 | | RDF | rdf_resource.rdf | 4 | diff --git a/test/features/datarequest.feature b/test/features/datarequest.feature index 1782185c..952abc7d 100644 --- a/test/features/datarequest.feature +++ b/test/features/datarequest.feature @@ -60,7 +60,7 @@ Feature: Data Request When I log in and create a datarequest And I press the element with xpath "//a[contains(string(), 'Close')]" And I select "Requestor initiated closure" from "close_circumstance" - And I press the element with xpath "//button[contains(@class, 'btn-danger')]" + And I press the element with xpath "//button[contains(@class, 'btn-danger') and contains(string(), 'Close data request')]" Then I should see an element with xpath "//i[contains(@class, 'icon-lock')]" And I should not see an element with xpath "//a[contains(string(), 'Close')]" @@ -69,7 +69,7 @@ Feature: Data Request When I log in and create a datarequest And I press the element with xpath "//a[contains(string(), 'Close')]" And I select "Requestor initiated closure" from "close_circumstance" - And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close']" + And I press the element with xpath "//button[contains(@class, 'btn-danger') and contains(string(), 'Close data request')]" Then I should see an element with xpath "//a[contains(string(), 'Re-open')]" When I press the element with xpath "//a[contains(string(), 'Re-open')]" Then I should see an element with xpath "//i[contains(@class, 'icon-unlock')]" diff --git a/test/features/engagement_reporting.feature b/test/features/engagement_reporting.feature index 33e54c79..3fd6bb79 100644 --- a/test/features/engagement_reporting.feature +++ b/test/features/engagement_reporting.feature @@ -67,7 +67,7 @@ Feature: Engagement Reporting And I press the element with xpath "//a[contains(string(), 'Close')]" And I select "To be released as open data at a later date" from "close_circumstance" And I fill in "approx_publishing_date" with "01/01/1970" - And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close']" + And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close' and contains(string(), 'Close data request')]" And I should see an element with xpath "//i[contains(@class, 'icon-lock')]" When I go to my reports page diff --git a/test/features/groups.feature b/test/features/groups.feature index 9333bbe1..f48b0f51 100644 --- a/test/features/groups.feature +++ b/test/features/groups.feature @@ -5,7 +5,7 @@ Feature: Group APIs Given "" as the persona When I log in And I view the "silly-walks" group API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "group_admin"') and contains(string(), '"name": "walker"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "group_admin"') and contains(string(), '"name": "walker"')]" Examples: Admins | Persona | @@ -16,7 +16,7 @@ Feature: Group APIs Given "" as the persona When I log in And I view the "silly-walks" group API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" Examples: Non-admin users | Persona | @@ -27,7 +27,7 @@ Feature: Group APIs Scenario: Group membership is not accessible anonymously Given "Unauthenticated" as the persona When I view the "silly-walks" group API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" @unauthenticated Scenario: Group overview is accessible to everyone @@ -38,4 +38,4 @@ Feature: Group APIs And I should see an element with xpath "//a[contains(@href, '/group/silly-walks')]" When I view the "silly-walks" group API "not including" users - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "silly-walks"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "silly-walks"')]" diff --git a/test/features/organisations.feature b/test/features/organisations.feature index 60ed3574..e8c2490d 100644 --- a/test/features/organisations.feature +++ b/test/features/organisations.feature @@ -5,7 +5,7 @@ Feature: Organization APIs Given "" as the persona When I log in And I view the "department-of-health" organisation API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "organisation_admin"') and contains(string(), '"name": "editor"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "organisation_admin"') and contains(string(), '"name": "editor"')]" Examples: Admins | Persona | @@ -16,7 +16,7 @@ Feature: Organization APIs Given "" as the persona When I log in And I view the "department-of-health" organisation API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" Examples: Non-admin users | Persona | @@ -28,7 +28,7 @@ Feature: Organization APIs Scenario: Organisation membership is not accessible anonymously Given "Unauthenticated" as the persona When I view the "department-of-health" organisation API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" @unauthenticated Scenario: Organisation overview is accessible to everyone @@ -39,4 +39,4 @@ Feature: Organization APIs And I should see an element with xpath "//a[contains(@href, '/organization/department-of-health')]" When I view the "department-of-health" organisation API "not including" users - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "department-of-health"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "department-of-health"')]" diff --git a/test/features/resource_freshness.feature b/test/features/resource_freshness.feature index f9451662..888d5551 100644 --- a/test/features/resource_freshness.feature +++ b/test/features/resource_freshness.feature @@ -30,6 +30,7 @@ Feature: Resource freshness Scenario Outline: As a user with editing privileges, when I set a 'monthly' update frequently, I should still be able to update the dataset via the API Given "" as the persona When I log in + Then I should be able to patch dataset "test-dataset" via the API And I go to "/dataset/edit/test-dataset" And I select "monthly" from "update_frequency" Then I should see "Next update due" diff --git a/test/features/search_facets.feature b/test/features/search_facets.feature index a16bbe5e..c594a09e 100644 --- a/test/features/search_facets.feature +++ b/test/features/search_facets.feature @@ -2,6 +2,6 @@ Feature: Search facets Scenario: When I go to the dataset list page, I can see the 'Data Portals' facet - When I go to "/dataset" + When I go to dataset page Then I should see "Data Portals" And I should see an element with xpath "//a[contains(@href, '?dataset_type=dataset')]/span[contains(@class, 'item-label') and contains(string(), 'data.qld.gov.au')]" diff --git a/test/features/user_creation.feature b/test/features/user_creation.feature index eda029ab..e29db5b8 100644 --- a/test/features/user_creation.feature +++ b/test/features/user_creation.feature @@ -1,6 +1,14 @@ @user_creation Feature: User creation + Scenario: SysAdmin can create 'Excluded display name words' in ckan admin config + Given "SysAdmin" as the persona + When I log in + Then I go to "/ckan-admin/config" + Then I should see "Excluded display name words" + Then I fill in "ckanext.data_qld.excluded_display_name_words" with "gov" + And I press the element with xpath "//button[contains(@class, 'btn-primary')]" + Scenario: SysAdmin create a new user to the site. Given "SysAdmin" as the persona When I log in diff --git a/test/features/users.feature b/test/features/users.feature index dfe44d3c..39236cd1 100644 --- a/test/features/users.feature +++ b/test/features/users.feature @@ -31,7 +31,7 @@ Feature: User APIs Given "" as the persona When I log in And I go to the user list API - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "admin"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "admin"')]" Examples: Admins | Persona | @@ -43,19 +43,19 @@ Feature: User APIs Given "Publisher" as the persona When I log in And I go to the user list API - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" @unauthenticated Scenario: User list is not accessible anonymously Given "Unauthenticated" as the persona When I go to the user list API - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'requires an authenticated user')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'requires an authenticated user')]" Scenario Outline: User detail is accessible to admins Given "" as the persona When I log in And I go to the "admin" user API - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "admin"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "admin"')]" Examples: Admins | Persona | @@ -67,19 +67,19 @@ Feature: User APIs Given "Publisher" as the persona When I log in And I go to the "editor" user API - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "editor"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "editor"')]" Scenario: Non-self user detail is not accessible to non-admins Given "Publisher" as the persona When I log in And I go to the "admin" user API - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" @unauthenticated Scenario: User detail is not accessible anonymously Given "Unauthenticated" as the persona When I go to the "editor" user API - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" Scenario Outline: User profile page is accessible to admins Given "" as the persona diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 1ea38ca5..b9a71687 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -23,7 +23,7 @@ extensions: description: "CKAN Extension for Queensland Government Open Data" type: "git" url: "https://github.com/qld-gov-au/ckanext-qgov.git" - version: "5.0.3" + version: "5.0.4" CKANExtS3Filestore: &CKANExtS3Filestore name: "ckanext-s3filestore-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 9a3201ec..0aeba4c6 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -23,7 +23,7 @@ extensions: description: "CKAN Extension for Queensland Government Open Data" type: "git" url: "https://github.com/qld-gov-au/ckanext-qgov.git" - version: "5.0.3" + version: "5.0.4" CKANExtS3Filestore: &CKANExtS3Filestore name: "ckanext-s3filestore-{{ Environment }}" diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index 45b44759..7ab3ebc6 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -14,7 +14,7 @@ extensions: description: "CKAN Extension for Queensland Government sites" type: "git" url: "https://github.com/qld-gov-au/ckanext-qgov.git" - version: "5.0.3" + version: "5.0.4" CKANExtS3Filestore: &CKANExtS3Filestore name: "ckanext-s3filestore-{{ Environment }}" From b953f34f2274e80c1e99fc1787c3181d2737fc0c Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 28 Jul 2022 11:13:26 +1000 Subject: [PATCH 21/40] [QOL-9055] update Archiver to fix Py3 compatibility - Drop obsolete Celery dependency that's complicating things. Only CKAN 2.6 or less needs it. --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index b9a71687..a387762c 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -127,7 +127,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.9" + version: "2.1.1-qgov.10" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 0aeba4c6..626a7175 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -127,7 +127,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.9" + version: "2.1.1-qgov.10" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" From d804c8a533f38b3c369854381c0cb0ad0648988e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 28 Jul 2022 11:58:53 +1000 Subject: [PATCH 22/40] [QOL-9055] update DCAT to fix Py3 compatibility --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index a387762c..9c433d66 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -47,7 +47,7 @@ extensions: description: "CKAN Extension for DCAT" type: "git" url: "https://github.com/ckan/ckanext-dcat.git" - version: "v1.1.3" + version: "v1.2.0" CKANExtScheming: &CKANExtScheming name: "ckanext-scheming-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 626a7175..46136cf0 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -47,7 +47,7 @@ extensions: description: "CKAN Extension for DCAT" type: "git" url: "https://github.com/ckan/ckanext-dcat.git" - version: "v1.1.3" + version: "v1.2.0" CKANExtScheming: &CKANExtScheming name: "ckanext-scheming-{{ Environment }}" From ff0df6cf1701a2a39e62d840eff91bc9f3ca5c8e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 28 Jul 2022 12:15:17 +1000 Subject: [PATCH 23/40] [QOL-9055] install libffi-dev for Py3 testing - This is needed to support ckanext-harvest --- .docker/Dockerfile-py3.ckan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/Dockerfile-py3.ckan b/.docker/Dockerfile-py3.ckan index 57ce91e3..926e178d 100644 --- a/.docker/Dockerfile-py3.ckan +++ b/.docker/Dockerfile-py3.ckan @@ -16,7 +16,7 @@ ENV CKAN_INI=/app/ckan/default/production.ini WORKDIR "${APP_DIR}" ENV DOCKERIZE_VERSION v0.6.1 -RUN apk add --no-cache curl build-base git libxml2-dev libxslt-dev postgresql postgresql-client postgresql-dev \ +RUN apk add --no-cache curl build-base git libffi-dev libxml2-dev libxslt-dev postgresql postgresql-client postgresql-dev \ && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz From b83411994895fd0dbea5ef9f25f6586afcdfdfa0 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 28 Jul 2022 12:55:49 +1000 Subject: [PATCH 24/40] [QOL-9055] update YTP Comments to fix Webassets path --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 9c433d66..ee6f6fbd 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.5" + version: "2.5.0-qgov.6" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 46136cf0..68db25cd 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.5" + version: "2.5.0-qgov.6" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" From 97d6693794a4fe890c8ef17aa3cdd8b66e19e09d Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 29 Jul 2022 10:05:57 +1000 Subject: [PATCH 25/40] [QOL-9055] update YTP Comments to fix data request comment notifications --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index ee6f6fbd..10d62472 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.6" + version: "2.5.0-qgov.7" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 68db25cd..89f12480 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.6" + version: "2.5.0-qgov.7" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" From 03339ec21bd0b591fe58b645f1e1c4d92276c764 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 29 Jul 2022 10:16:00 +1000 Subject: [PATCH 26/40] [QOL-9055] update Archiver to fix Py3 compatibility - Drop obsolete Kombu dependency --- .docker/scripts/init-ext.sh | 28 ++++++++++++++-------------- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh index 561f009d..807dd275 100755 --- a/.docker/scripts/init-ext.sh +++ b/.docker/scripts/init-ext.sh @@ -5,20 +5,21 @@ set -e install_requirements () { - PROJECT_DIR="$1" - for filename in requirements-$PYTHON_VERSION.txt requirements.txt pip-requirements.txt; do - if [ -f "$PROJECT_DIR/$filename" ]; then - pip install -r "$PROJECT_DIR/$filename" + PROJECT_DIR=$1 + shift + # Identify the best match requirements file, ignore the others. + # If there is one specific to our Python version, use that. + for filename_pattern in "$@"; do + filename="$PROJECT_DIR/${filename_pattern}-$PYTHON_VERSION.txt" + if [ -f "$filename" ]; then + pip install -r "$filename" return 0 fi done -} - -install_dev_requirements () { - PROJECT_DIR="$1" - for filename in dev-requirements-$PYTHON_VERSION.txt requirements-dev-$PYTHON_VERSION.txt requirements-dev.txt dev-requirements.txt; do - if [ -f "$PROJECT_DIR/$filename" ]; then - pip install -r "$PROJECT_DIR/$filename" + for filename_pattern in "$@"; do + filename="$PROJECT_DIR/$filename_pattern.txt" + if [ -f "$filename" ]; then + pip install -r "$filename" return 0 fi done @@ -27,12 +28,11 @@ install_dev_requirements () { if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi -pip install -r "requirements.txt" -pip install -r "requirements-dev.txt" +install_requirements . dev-requirements requirements-dev EXTENSIONS_FILE=$APP_DIR/scripts/extensions.yml python $(dirname $0)/generate-ext-requirements.py pip install --force-reinstall -r "/tmp/requirements-ext.txt" for extension in . `ls -d $VENV_DIR/src/ckanext-*`; do - install_requirements $extension + install_requirements $extension requirements pip-requirements done if [ "$VENV_DIR" != "" ]; then diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 10d62472..d9b94b94 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -127,7 +127,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.10" + version: "2.1.1-qgov.11" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 89f12480..4932249f 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -127,7 +127,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.10" + version: "2.1.1-qgov.11" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" From 78baf828308c7d0a41e06565d2b9b90fe3d757b9 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 29 Jul 2022 10:26:37 +1000 Subject: [PATCH 27/40] [QOL-9055] update Report to ensure Flask views are returned --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index d9b94b94..c8895080 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -119,7 +119,7 @@ extensions: description: "CKAN Extension for Reporting" type: "git" url: "https://github.com/qld-gov-au/ckanext-report.git" - version: "0.3" + version: "0.3.1-qgov.1" CKANExtArchiver: &CKANExtArchiver name: "ckanext-archiver-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 4932249f..b102cf12 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -119,7 +119,7 @@ extensions: description: "CKAN Extension for Reporting" type: "git" url: "https://github.com/qld-gov-au/ckanext-report.git" - version: "0.3" + version: "0.3.1-qgov.1" CKANExtArchiver: &CKANExtArchiver name: "ckanext-archiver-{{ Environment }}" From 7aa19350dc255dcb3a13843cdcd5d3945c54247a Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 29 Jul 2022 10:35:24 +1000 Subject: [PATCH 28/40] [QOL-9055] pin test pyyaml version for Py2/3 compatibility --- .docker/scripts/init-ext.sh | 2 +- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh index 807dd275..19565d2c 100755 --- a/.docker/scripts/init-ext.sh +++ b/.docker/scripts/init-ext.sh @@ -28,12 +28,12 @@ install_requirements () { if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi -install_requirements . dev-requirements requirements-dev EXTENSIONS_FILE=$APP_DIR/scripts/extensions.yml python $(dirname $0)/generate-ext-requirements.py pip install --force-reinstall -r "/tmp/requirements-ext.txt" for extension in . `ls -d $VENV_DIR/src/ckanext-*`; do install_requirements $extension requirements pip-requirements done +install_requirements . dev-requirements requirements-dev if [ "$VENV_DIR" != "" ]; then deactivate diff --git a/requirements-dev.txt b/requirements-dev.txt index 3b4638f6..59d7c960 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,6 +8,6 @@ flake8==3.8.3 mock pika>=1.1.0,<1.3.0 pytest-ckan -PyYAML>=1.0 +pyyaml==5.4.1 setuptools==44.1.1 splinter>=0.13.0,<0.17 From 6456b1c7308a234bcbcf6feb0dcb9ee8abdb10a8 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 29 Jul 2022 10:50:54 +1000 Subject: [PATCH 29/40] [QOL-9055] install requirements both before and after extensions - Install before so we get priority when a version range is specified. Install again after to apply our pins --- .docker/scripts/init-ext.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh index 19565d2c..86d16993 100755 --- a/.docker/scripts/init-ext.sh +++ b/.docker/scripts/init-ext.sh @@ -28,6 +28,7 @@ install_requirements () { if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi +install_requirements . dev-requirements requirements-dev EXTENSIONS_FILE=$APP_DIR/scripts/extensions.yml python $(dirname $0)/generate-ext-requirements.py pip install --force-reinstall -r "/tmp/requirements-ext.txt" for extension in . `ls -d $VENV_DIR/src/ckanext-*`; do From 9009161ab7ca1243e81278615d877815b18b313a Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 29 Jul 2022 14:16:00 +1000 Subject: [PATCH 30/40] [QOL-9055] update Data Requests for Python 3 --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index c8895080..c0c4ae48 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -87,7 +87,7 @@ extensions: description: "CKAN Extension for Data Requests" type: "git" url: "https://github.com/qld-gov-au/ckanext-datarequests.git" - version: "2.2.1-qgov.2" + version: "2.2.1-qgov.3" CKANExtYTP: &CKANExtYTP name: "ckanext-ytp-comments-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index b102cf12..09596a7c 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -87,7 +87,7 @@ extensions: description: "CKAN Extension for Data Requests" type: "git" url: "https://github.com/qld-gov-au/ckanext-datarequests.git" - version: "2.2.1-qgov.2" + version: "2.2.1-qgov.3" CKANExtYTP: &CKANExtYTP name: "ckanext-ytp-comments-{{ Environment }}" From 32616fa626db4a4205c3cc2e5ba4273af05fdfde Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 2 Aug 2022 09:56:31 +1000 Subject: [PATCH 31/40] [QOL-9055] point CKAN_INI at our test config instead of copying the file --- .docker/Dockerfile-py2.ckan | 4 +--- .docker/Dockerfile-py3.ckan | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.docker/Dockerfile-py2.ckan b/.docker/Dockerfile-py2.ckan index 8dd1a6da..f176df1a 100644 --- a/.docker/Dockerfile-py2.ckan +++ b/.docker/Dockerfile-py2.ckan @@ -11,7 +11,7 @@ ENV SITE_URL="${SITE_URL}" ENV DEPLOY_ENV="${DEPLOY_ENV}" ENV VENV_DIR=/app/ckan/default ENV APP_DIR=/app -ENV CKAN_INI=/app/ckan/default/production.ini +ENV CKAN_INI=.docker/test-$VARS_TYPE.ini WORKDIR "${APP_DIR}" @@ -35,8 +35,6 @@ RUN . ${VENV_DIR}/bin/activate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ && fix-permissions ${APP_DIR}/ckan -COPY .docker/test-$VARS_TYPE.ini $CKAN_INI - COPY . ${APP_DIR}/ COPY .docker/scripts ${APP_DIR}/scripts diff --git a/.docker/Dockerfile-py3.ckan b/.docker/Dockerfile-py3.ckan index 926e178d..7d8dd6cb 100644 --- a/.docker/Dockerfile-py3.ckan +++ b/.docker/Dockerfile-py3.ckan @@ -11,7 +11,7 @@ ENV SITE_URL="${SITE_URL}" ENV DEPLOY_ENV="${DEPLOY_ENV}" ENV VENV_DIR=/app/ckan/default ENV APP_DIR=/app -ENV CKAN_INI=/app/ckan/default/production.ini +ENV CKAN_INI=.docker/test-$VARS_TYPE.ini WORKDIR "${APP_DIR}" @@ -34,8 +34,6 @@ RUN mkdir -p ${VENV_DIR} \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ && fix-permissions ${APP_DIR}/ckan -COPY .docker/test-$VARS_TYPE.ini $CKAN_INI - COPY . ${APP_DIR}/ COPY .docker/scripts ${APP_DIR}/scripts From 6b10d995aa3faf4ac51458541ba2381efe1131ab Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 2 Aug 2022 09:57:22 +1000 Subject: [PATCH 32/40] [QOL-9055] use SRC_DIR instead of VENV_DIR when possible - simpler and more consistent with standard CKAN test containers --- .docker/Dockerfile-py2.ckan | 12 +++++++----- .docker/Dockerfile-py3.ckan | 6 ++++-- .docker/scripts/init-ext.sh | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.docker/Dockerfile-py2.ckan b/.docker/Dockerfile-py2.ckan index f176df1a..d3874141 100644 --- a/.docker/Dockerfile-py2.ckan +++ b/.docker/Dockerfile-py2.ckan @@ -10,6 +10,7 @@ ENV PYTHON_VERSION=py2 ENV SITE_URL="${SITE_URL}" ENV DEPLOY_ENV="${DEPLOY_ENV}" ENV VENV_DIR=/app/ckan/default +ENV SRC_DIR=/app/ckan/default/src ENV APP_DIR=/app ENV CKAN_INI=.docker/test-$VARS_TYPE.ini @@ -26,11 +27,12 @@ RUN apk add --no-cache curl build-base postgresql-client \ RUN . ${VENV_DIR}/bin/activate \ && pip install setuptools==36.1 \ && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \ - && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \ - && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \ - pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \ - pip install -r "${VENV_DIR}/src/ckan/requirements.txt") \ - && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ + && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${SRC_DIR}/ckan/requirements.txt" \ + && ((test -f "${SRC_DIR}/ckan/requirements-py2.txt" && \ + pip install -r "${SRC_DIR}/ckan/requirements-py2.txt") || \ + pip install -r "${SRC_DIR}/ckan/requirements.txt") \ + && pip install -r "${SRC_DIR}/ckan/dev-requirements.txt" \ + && ln -s "${SRC_DIR}/ckan/who.ini" "${VENV_DIR}/who.ini" \ && deactivate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ && fix-permissions ${APP_DIR}/ckan diff --git a/.docker/Dockerfile-py3.ckan b/.docker/Dockerfile-py3.ckan index 7d8dd6cb..4eeccf1c 100644 --- a/.docker/Dockerfile-py3.ckan +++ b/.docker/Dockerfile-py3.ckan @@ -10,6 +10,7 @@ ENV PYTHON_VERSION=py3 ENV SITE_URL="${SITE_URL}" ENV DEPLOY_ENV="${DEPLOY_ENV}" ENV VENV_DIR=/app/ckan/default +ENV SRC_DIR=/app/ckan/default/src ENV APP_DIR=/app ENV CKAN_INI=.docker/test-$VARS_TYPE.ini @@ -28,8 +29,9 @@ RUN mkdir -p ${VENV_DIR} \ && . ${VENV_DIR}/bin/activate \ && pip install setuptools==44.1.0 \ && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \ - && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \ - && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ + && pip install -r "${SRC_DIR}/ckan/requirements.txt" \ + -r "${SRC_DIR}/ckan/dev-requirements.txt" \ + && ln -s "${SRC_DIR}/ckan/who.ini" "${VENV_DIR}/who.ini" \ && deactivate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ && fix-permissions ${APP_DIR}/ckan diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh index 86d16993..d6fda09a 100755 --- a/.docker/scripts/init-ext.sh +++ b/.docker/scripts/init-ext.sh @@ -31,7 +31,7 @@ fi install_requirements . dev-requirements requirements-dev EXTENSIONS_FILE=$APP_DIR/scripts/extensions.yml python $(dirname $0)/generate-ext-requirements.py pip install --force-reinstall -r "/tmp/requirements-ext.txt" -for extension in . `ls -d $VENV_DIR/src/ckanext-*`; do +for extension in . `ls -d $SRC_DIR/ckanext-*`; do install_requirements $extension requirements pip-requirements done install_requirements . dev-requirements requirements-dev From 0c2d5d2f0c7975df6ace04bec622c9489e469f11 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 2 Aug 2022 10:34:36 +1000 Subject: [PATCH 33/40] [QOL-9055] fix Docker container paths - ensure who.ini is alongside test.ini, try to avoid using .docker at all inside the container --- .ahoy.yml | 2 +- .docker/Dockerfile-py2.ckan | 6 ++++-- .docker/Dockerfile-py3.ckan | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.ahoy.yml b/.ahoy.yml index 2f88b115..99af6a52 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -86,7 +86,7 @@ commands: usage: Install a site. cmd: | ahoy title "Installing a fresh site" - ahoy cli "./scripts/init.sh" + ahoy cli '$APP_DIR/scripts/init.sh' clean: usage: Remove containers and all build files. diff --git a/.docker/Dockerfile-py2.ckan b/.docker/Dockerfile-py2.ckan index d3874141..5d1960cb 100644 --- a/.docker/Dockerfile-py2.ckan +++ b/.docker/Dockerfile-py2.ckan @@ -12,7 +12,7 @@ ENV DEPLOY_ENV="${DEPLOY_ENV}" ENV VENV_DIR=/app/ckan/default ENV SRC_DIR=/app/ckan/default/src ENV APP_DIR=/app -ENV CKAN_INI=.docker/test-$VARS_TYPE.ini +ENV CKAN_INI=/app/test.ini WORKDIR "${APP_DIR}" @@ -32,13 +32,15 @@ RUN . ${VENV_DIR}/bin/activate \ pip install -r "${SRC_DIR}/ckan/requirements-py2.txt") || \ pip install -r "${SRC_DIR}/ckan/requirements.txt") \ && pip install -r "${SRC_DIR}/ckan/dev-requirements.txt" \ - && ln -s "${SRC_DIR}/ckan/who.ini" "${VENV_DIR}/who.ini" \ + && ln -s "${SRC_DIR}/ckan/who.ini" "$APP_DIR/who.ini" \ && deactivate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ && fix-permissions ${APP_DIR}/ckan COPY . ${APP_DIR}/ +COPY .docker/test-$VARS_TYPE.ini ${APP_DIR}/test.ini + COPY .docker/scripts ${APP_DIR}/scripts COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/ diff --git a/.docker/Dockerfile-py3.ckan b/.docker/Dockerfile-py3.ckan index 4eeccf1c..f61340e6 100644 --- a/.docker/Dockerfile-py3.ckan +++ b/.docker/Dockerfile-py3.ckan @@ -12,7 +12,7 @@ ENV DEPLOY_ENV="${DEPLOY_ENV}" ENV VENV_DIR=/app/ckan/default ENV SRC_DIR=/app/ckan/default/src ENV APP_DIR=/app -ENV CKAN_INI=.docker/test-$VARS_TYPE.ini +ENV CKAN_INI=/app/test.ini WORKDIR "${APP_DIR}" @@ -31,13 +31,15 @@ RUN mkdir -p ${VENV_DIR} \ && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \ && pip install -r "${SRC_DIR}/ckan/requirements.txt" \ -r "${SRC_DIR}/ckan/dev-requirements.txt" \ - && ln -s "${SRC_DIR}/ckan/who.ini" "${VENV_DIR}/who.ini" \ + && ln -s "${SRC_DIR}/ckan/who.ini" "$APP_DIR/who.ini" \ && deactivate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ && fix-permissions ${APP_DIR}/ckan COPY . ${APP_DIR}/ +COPY .docker/test-$VARS_TYPE.ini ${APP_DIR}/test.ini + COPY .docker/scripts ${APP_DIR}/scripts COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/ From ebece617c3b5a8f38ae1e21ddbf0e91d7fc68563 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 2 Aug 2022 10:35:12 +1000 Subject: [PATCH 34/40] [QOL-9055] fix JSON resource for QA testing --- test/features/data_usability_rating.feature | 2 +- test/fixtures/json_resource.json | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/json_resource.json diff --git a/test/features/data_usability_rating.feature b/test/features/data_usability_rating.feature index f23e81fb..7f69dd2a 100644 --- a/test/features/data_usability_rating.feature +++ b/test/features/data_usability_rating.feature @@ -17,5 +17,5 @@ Feature: Data usability rating | TXT | txt_resource.txt | 1 | | XLS | xls_resource.xls | 2 | | CSV | csv_resource.csv | 3 | - | JSON | csv_resource.csv | 3 | + | JSON | json_resource.json | 3 | | RDF | rdf_resource.rdf | 4 | diff --git a/test/fixtures/json_resource.json b/test/fixtures/json_resource.json new file mode 100644 index 00000000..1e0ca3d4 --- /dev/null +++ b/test/fixtures/json_resource.json @@ -0,0 +1,3 @@ +{ + "foo": "baz" +} From eaba542110471bfc181ba08839c266c14c772195 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 2 Aug 2022 10:35:38 +1000 Subject: [PATCH 35/40] [QOL-9055] clean up docs --- .env | 6 +++--- .flake8 | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 169401ae..126de260 100644 --- a/.env +++ b/.env @@ -15,11 +15,11 @@ PROJECT="ckan-qld-infrastructure" # Docker Compose project name. All containers will have this name. COMPOSE_PROJECT_NAME="$PROJECT" -# Flag to allow code linting failures. +# Flag to allow code linting failures. 0=enforce, 1=ignore ALLOW_LINT_FAIL=0 -# Flag to allow unit tests failures. +# Flag to allow unit tests failures. 0=enforce, 1=ignore ALLOW_UNIT_FAIL=0 -# Flag to allow BDD tests failures. +# Flag to allow BDD tests failures. 0=enforce, 1=ignore ALLOW_BDD_FAIL=0 diff --git a/.flake8 b/.flake8 index 27ae6526..8412d46e 100644 --- a/.flake8 +++ b/.flake8 @@ -10,6 +10,7 @@ format = pylint # Show the source of errors. show_source = True +statistics = True max-complexity = 10 max-line-length = 127 From a114b245f6727e9b6a7e2b3a65f90a0d7d734148 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 2 Aug 2022 10:36:03 +1000 Subject: [PATCH 36/40] [QOL-9055] fix base64 decoding in email testing to handle Py3 --- test/features/datarequest.feature | 2 +- test/features/steps/steps.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/features/datarequest.feature b/test/features/datarequest.feature index 952abc7d..2e3c1da8 100644 --- a/test/features/datarequest.feature +++ b/test/features/datarequest.feature @@ -17,7 +17,7 @@ Feature: Data Request Scenario: Data requests submitted without a description will produce an error message Given "SysAdmin" as the persona When I log in and go to the data requests page - And I press the element with xpath "//a[contains(@class, 'btn-primary')]" + And I press the element with xpath "//a[contains(@class, 'btn-primary') and contains(string(), 'Add data request')]" And I fill in "title" with "Test data request" And I press the element with xpath "//button[contains(@class, 'btn-primary')]" Then I should see an element with the css selector "div.error-explanation.alert.alert-error" within 2 seconds diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 50ba609c..c313e15a 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -7,6 +7,7 @@ import quopri import requests import uuid +import six @step(u'I get the current URL') @@ -97,6 +98,7 @@ def add_resource(context, name, url): And I fill in "name" with "{name}" And I fill in "description" with "description" And I fill in "size" with "1024" if present + And I execute the script "document.getElementById('field-format').value='HTML'" And I press the element with xpath "//form[contains(@class, 'resource-form')]//button[contains(@class, 'btn-primary')]" """.format(name=name, url=url)) @@ -244,7 +246,11 @@ def filter_contents(mail): payload_bytes = quopri.decodestring(payload) if len(payload_bytes) > 0: payload_bytes += b'=' # do fix the padding error issue - decoded_payload = payload_bytes.decode('base64') + if six.PY2: + decoded_payload = payload_bytes.decode('base64') + else: + import base64 + decoded_payload = six.ensure_text(base64.b64decode(six.ensure_binary(payload_bytes))) print('decoded_payload: ', decoded_payload) return text in decoded_payload and (not text2 or text2 in decoded_payload) From 4ba931ea074252f783153eac7dd6a4f91c3e4459 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 2 Aug 2022 13:35:10 +1000 Subject: [PATCH 37/40] [QOL-9055] drop unnecessary test step --- .docker/Dockerfile-py2.ckan | 5 ++--- .docker/Dockerfile-py3.ckan | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.docker/Dockerfile-py2.ckan b/.docker/Dockerfile-py2.ckan index 5d1960cb..fc06fbfd 100644 --- a/.docker/Dockerfile-py2.ckan +++ b/.docker/Dockerfile-py2.ckan @@ -27,7 +27,6 @@ RUN apk add --no-cache curl build-base postgresql-client \ RUN . ${VENV_DIR}/bin/activate \ && pip install setuptools==36.1 \ && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \ - && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${SRC_DIR}/ckan/requirements.txt" \ && ((test -f "${SRC_DIR}/ckan/requirements-py2.txt" && \ pip install -r "${SRC_DIR}/ckan/requirements-py2.txt") || \ pip install -r "${SRC_DIR}/ckan/requirements.txt") \ @@ -37,9 +36,9 @@ RUN . ${VENV_DIR}/bin/activate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ && fix-permissions ${APP_DIR}/ckan -COPY . ${APP_DIR}/ +COPY .docker/test-$VARS_TYPE.ini $CKAN_INI -COPY .docker/test-$VARS_TYPE.ini ${APP_DIR}/test.ini +COPY . ${APP_DIR}/ COPY .docker/scripts ${APP_DIR}/scripts diff --git a/.docker/Dockerfile-py3.ckan b/.docker/Dockerfile-py3.ckan index f61340e6..1245c241 100644 --- a/.docker/Dockerfile-py3.ckan +++ b/.docker/Dockerfile-py3.ckan @@ -36,9 +36,9 @@ RUN mkdir -p ${VENV_DIR} \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ && fix-permissions ${APP_DIR}/ckan -COPY . ${APP_DIR}/ +COPY .docker/test-$VARS_TYPE.ini $CKAN_INI -COPY .docker/test-$VARS_TYPE.ini ${APP_DIR}/test.ini +COPY . ${APP_DIR}/ COPY .docker/scripts ${APP_DIR}/scripts From 1d521bce757b9b442e70e66061a034e819454b9f Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 2 Aug 2022 14:14:49 +1000 Subject: [PATCH 38/40] [QOL-9055] update Data Qld extension to handle Py3 --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index c0c4ae48..6cdc8f19 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -71,7 +71,7 @@ extensions: description: "CKAN Extension for Queensland Government Open Data" type: "git" url: "https://github.com/qld-gov-au/ckanext-data-qld.git" - version: "6.3.4" + version: "6.3.5" CKANExtODICertificates: &CKANExtODICertificates name: "ckanext-odi-certificates-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 09596a7c..8ecf52d0 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -71,7 +71,7 @@ extensions: description: "CKAN Extension for Queensland Government Open Data" type: "git" url: "https://github.com/qld-gov-au/ckanext-data-qld.git" - version: "6.3.4" + version: "6.3.5" CKANExtODICertificates: &CKANExtODICertificates name: "ckanext-odi-certificates-{{ Environment }}" From 3ce5f240f7dd9e3dfbba1f09e398a55a403e3c7b Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 3 Aug 2022 14:04:37 +1000 Subject: [PATCH 39/40] [QOL-9055] update cookbook to handle Amazon Linux 2 --- vars/CKAN-Stack.var.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index fa722f9e..b610a485 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -45,7 +45,7 @@ common_stack: &common_stack EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - CookbookRevision: "{{ CookbookRevision | default('5.2.14') }}" + CookbookRevision: "{{ CookbookRevision | default('5.3.0') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket SolrSource: "{{ solr7 }}" From 7e19a91bce15a643d0ccf932cf12ccf235ea115e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 3 Aug 2022 15:04:12 +1000 Subject: [PATCH 40/40] [QOL-9055] update YTP Comments to fix follow/mute actions --- vars/shared-CKANTest.var.yml | 2 +- vars/shared-OpenData.var.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index 6cdc8f19..67845e9e 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.7" + version: "2.5.0-qgov.8" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 8ecf52d0..876f3fae 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.7" + version: "2.5.0-qgov.8" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}"