diff --git a/.ahoy.yml b/.ahoy.yml index 73f07649..f6147171 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -160,9 +160,10 @@ commands: ahoy cli "behave -k ${*:-test/features} --tags=-unauthenticated --tags=-smoke --tags=-OpenData --tags=-Publications" \ || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] else - # run tests with the specified tag - ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG --tags=-OpenData --tags=-Publications" \ - || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] + # run shared or app-specific tests with the specific tag + (ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG --tags=-OpenData --tags=-Publications" \ + && ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG --tags=$VARS_TYPE" \ + ) || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] fi ahoy stop-mailmock ahoy stop-ckan-job-workers diff --git a/.docker/test-OpenData.ini b/.docker/test-OpenData.ini index d14521ca..c0e4f9cd 100644 --- a/.docker/test-OpenData.ini +++ b/.docker/test-OpenData.ini @@ -96,7 +96,7 @@ ckan.redis.url = redis://redis:6379 ## Plugins Settings ckan.plugins = - stats resource_proxy text_view webpage_view recline_grid_view image_view recline_view recline_graph_view recline_map_view + stats resource_proxy text_view webpage_view image_view data_qld data_qld_google_analytics dcat structured_data validation diff --git a/.docker/test-Publications.ini b/.docker/test-Publications.ini index 5efde570..9530d0a8 100644 --- a/.docker/test-Publications.ini +++ b/.docker/test-Publications.ini @@ -93,7 +93,7 @@ ckan.redis.url = redis://redis:6379 ## Plugins Settings ckan.plugins = - stats text_view image_view recline_view + stats text_view image_view publications_qld_theme resource_type_validation qgovext diff --git a/.gitignore b/.gitignore index 0f996fa2..6968dc0f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,12 @@ *.pyc *.retry +templates/chef.json +templates/chef-source.json templates/cloudfront.cfn.yml templates/security_groups.cfn.yml templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml +templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml templates/*.sql .idea *.log diff --git a/build-CKAN.sh b/build-CKAN.sh index 83f20961..e6a451aa 100755 --- a/build-CKAN.sh +++ b/build-CKAN.sh @@ -36,20 +36,14 @@ run-shared-resource-playbooks () { } run-deployment () { - PARALLEL=1 ./opsworks-deploy.sh update_custom_cookbooks $STACK_NAME - ./opsworks-deploy.sh setup $STACK_NAME ${INSTANCE_SHORTNAME}-web & WEB_PID=$! - PARALLEL=1 ./opsworks-deploy.sh setup $STACK_NAME ${INSTANCE_SHORTNAME}-batch & BATCH_PID=$! + run-playbook "chef-json" + ./chef-deploy.sh datashades::ckanweb-setup,datashades::ckanweb-deploy $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$! + PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-setup,datashades::ckanbatch-deploy,datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$! wait $WEB_PID + PARALLEL=1 ./chef-deploy.sh datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web wait $BATCH_PID - STOPPED_INSTANCES=$(./opsworks-deploy.sh get_stopped_instances $STACK_NAME ${INSTANCE_SHORTNAME}-solr) - if [ "$STOPPED_INSTANCES" = "" ]; then - ./opsworks-deploy.sh execute_recipes $STACK_NAME ${INSTANCE_SHORTNAME}-solr datashades::solr-deploy || exit 1 - else - ./opsworks-deploy.sh start $STACK_NAME ${INSTANCE_SHORTNAME}-solr - ./opsworks-deploy.sh setup $STACK_NAME ${INSTANCE_SHORTNAME}-solr - ./opsworks-deploy.sh stop $STACK_NAME ${INSTANCE_SHORTNAME}-solr "$STOPPED_INSTANCES" - fi - PARALLEL=1 ./opsworks-deploy.sh configure $STACK_NAME + ./chef-deploy.sh datashades::solr-setup,datashades::solr-deploy $INSTANCE_NAME $ENVIRONMENT solr || exit 1 + PARALLEL=1 ./chef-deploy.sh datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr } run-all-playbooks () { @@ -60,7 +54,6 @@ run-all-playbooks () { fi run-playbook "CloudFormation" "vars/s3_buckets.var.yml" run-playbook "CKAN-Stack" - run-playbook "CKAN-extensions" run-playbook "CloudFormation" "vars/instances-${INSTANCE_NAME}.var.yml" run-playbook "CloudFormation" "vars/cloudfront-lambda-at-edge.var.yml" run-playbook "cloudfront" @@ -71,10 +64,19 @@ if [ $# -ge 3 ]; then if [ "$3" = "deploy" ]; then run-deployment elif [ "$3" = "setup" ]; then - PARALLEL=1 ./opsworks-deploy.sh setup $STACK_NAME + run-playbook "chef-json" + PARALLEL=1 ./chef-deploy.sh datashades::ckanweb-setup,datashades::ckanweb-deploy,datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$! + PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-setup,datashades::ckanbatch-deploy,datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$! + PARALLEL=1 ./chef-deploy.sh datashades::solr-setup,datashades::solr-deploy,datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr + wait $WEB_PID + wait $BATCH_PID elif [ "$3" = "configure" ]; then - PARALLEL=1 ./opsworks-deploy.sh update_custom_cookbooks $STACK_NAME - PARALLEL=1 ./opsworks-deploy.sh configure $STACK_NAME + run-playbook "chef-json" + PARALLEL=1 ./chef-deploy.sh datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$! + PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$! + PARALLEL=1 ./chef-deploy.sh datashades::solr-configure $INSTANCE_NAME $ENVIRONMENT solr + wait $WEB_PID + wait $BATCH_PID else # run custom playbook run-playbook "$3" "$4" diff --git a/chef-deploy.sh b/chef-deploy.sh new file mode 100755 index 00000000..4069c7fe --- /dev/null +++ b/chef-deploy.sh @@ -0,0 +1,104 @@ +#!/bin/sh + +set -e + +debug () { + echo "$1" >&2 +} + +if [ "$#" -lt 3 ]; then + debug "Usage: [PARALLEL=true] $0 [environment eg DEV] [layer eg web] [recipe name eg datashades::ckanweb-deploy]" + debug "When PARALLEL is true, deployments will run on all servers in the stack/layer simultaneously. This is suitable for low-impact commands like 'configure'." + debug "When PARALLEL is false or not set, deployments will run on each server sequentially. This is slower, but safer, avoiding race conditions and allowing disruptive commands like 'setup' without downtime." + exit 1 +fi + +COMMAND="$1" +SERVICE="$2" +ENVIRONMENT="$3" +LAYER_NAME="$4" +RUN_LIST="recipe[$(echo $COMMAND | sed 's/,/],recipe[/g')]" + +# Retrieve generated custom JSON +CHEF_SOURCE=$(cat templates/chef-source.json | tr -d '\n') +CUSTOM_JSON=$(cat templates/chef.json | tr -d '\n' | sed 's/"/\\"/g') + +REGION_SNIPPET="--region ap-southeast-2" +MESSAGE="[$COMMAND on $SERVICE $ENVIRONMENT $LAYER_NAME]" +debug "$MESSAGE: commencing..." + +wait_for_deployment () { + DEPLOYMENT_ID=$1 + debug "$MESSAGE: waiting for deployment $DEPLOYMENT_ID..." + STATUS=$(aws ssm list-commands $REGION_SNIPPET --command-id $DEPLOYMENT_ID --query "Commands|[0].Status" --output text) || exit 1 + for retry in `seq 1 180`; do + if [ "$STATUS" = "Pending" ] || [ "$STATUS" = "InProgress" ]; then + sleep 10 + STATUS=$(aws ssm list-commands $REGION_SNIPPET --command-id $DEPLOYMENT_ID --query "Commands|[0].Status" --output text) || exit 1 + debug "Deployment $DEPLOYMENT_ID: $STATUS" + else + break + fi + done + if [ "$STATUS" != "Success" ]; then + debug "Failed to deploy $DEPLOYMENT_ID, status $STATUS - aborting" + exit 1 + fi +} + +find_load_balancer () { + # Finds the first load balancer whose tags all match the deployment, + # and outputs its identifying name + for lb_name in `aws elb describe-load-balancers --query "LoadBalancerDescriptions[].LoadBalancerName" --output text`; do + LB_TAGS=$(aws elb describe-tags --load-balancer-name $lb_name --query "TagDescriptions[].Tags[].{Key: Key, Value: Value}[?Key=='Environment' || Key=='Service' || Key == 'Layer']" --output text) + if (echo $LB_TAGS | grep -E "(\s|^)Environment\s+$ENVIRONMENT(\s|$)" >/dev/null 2>&1) \ + && (echo $LB_TAGS | grep -E "(\s|^)Service\s+$SERVICE(\s|$)" >/dev/null 2>&1) \ + && (echo $LB_TAGS | grep -E "(\s|^)Layer\s+$LAYER_NAME(\s|$)" >/dev/null 2>&1); then + echo "$lb_name" + fi + done +} + +deploy () { + for truthy in `echo "y t true T 1" |xargs echo`; do + if [ "$PARALLEL" = "$truthy" ]; then + debug "$MESSAGE: Parallel enabled, will deploy to all target instances simultaneously" + PARALLEL=true + fi + done + + INSTANCE_IDENTIFIER_SNIPPET="--filters Name=tag:Environment,Values=$ENVIRONMENT Name=tag:Service,Values=$SERVICE Name=instance-state-name,Values=running" + if [ "$LAYER_NAME" != "" ]; then + INSTANCE_IDENTIFIER_SNIPPET="${INSTANCE_IDENTIFIER_SNIPPET} Name=tag:Layer,Values=$LAYER_NAME" + fi + INSTANCE_IDS=$(aws ec2 describe-instances $REGION_SNIPPET $INSTANCE_IDENTIFIER_SNIPPET --query "Reservations[].Instances[].InstanceId" --output text) || exit 1 + if [ "$INSTANCE_IDS" = "" ]; then + debug "No eligible instance(s) in $SERVICE $ENVIRONMENT $LAYER_NAME" + exit 1 + else + debug "Target instance(s) in $SERVICE $ENVIRONMENT $LAYER_NAME: $INSTANCE_IDS" + fi + if [ "$PARALLEL" = "true" ]; then + DEPLOYMENT_ID=$(aws ssm send-command --document-name "AWS-ApplyChefRecipes" --document-version "\$DEFAULT" --instance-ids $INSTANCE_IDS --parameters '{'"$CHEF_SOURCE"',"RunList":["'"$RUN_LIST"'"],"JsonAttributesSources":[""],"JsonAttributesContent":["'"$CUSTOM_JSON"'"],"ChefClientVersion":["14"],"ChefClientArguments":[""],"WhyRun":["False"],"ComplianceSeverity":["None"],"ComplianceType":["Custom:Chef"],"ComplianceReportBucket":[""]}' --timeout-seconds 3600 --max-concurrency "50" --max-errors "0" --output-s3-bucket-name "osssio-ckan-web-logs" --output-s3-key-prefix "run_command" --region ap-southeast-2 --query "Command.CommandId" --output text) + wait_for_deployment $DEPLOYMENT_ID || exit 1 + else + ELB_NAME=$(find_load_balancer) + for instance in $INSTANCE_IDS; do + if [ "$ELB_NAME" != "" ]; then + OUTPUT=$(aws elb deregister-instances-from-load-balancer --load-balancer-name "$ELB_NAME" --instances "$instance" --query "Instances[].InstanceId" --output text) + debug "Deregistered instance $instance from load balancer $ELB_NAME, resulting registered instances: $OUTPUT" + fi + DEPLOYMENT_ID=$(aws ssm send-command --document-name "AWS-ApplyChefRecipes" --document-version "\$DEFAULT" --instance-ids $instance --parameters '{'"$CHEF_SOURCE"',"RunList":["'"$RUN_LIST"'"],"JsonAttributesSources":[""],"JsonAttributesContent":["'"$CUSTOM_JSON"'"],"ChefClientVersion":["14"],"ChefClientArguments":[""],"WhyRun":["False"],"ComplianceSeverity":["None"],"ComplianceType":["Custom:Chef"],"ComplianceReportBucket":[""]}' --timeout-seconds 3600 --max-concurrency "50" --max-errors "0" --output-s3-bucket-name "osssio-ckan-web-logs" --output-s3-key-prefix "run_command" --region ap-southeast-2 --query "Command.CommandId" --output text) + wait_for_deployment $DEPLOYMENT_ID + DEPLOYMENT_SUCCESS=$? + if [ "$ELB_NAME" != "" ]; then + OUTPUT=$(aws elb register-instances-with-load-balancer --load-balancer-name "$ELB_NAME" --instances "$instance" --query "Instances[].InstanceId" --output text) + debug "Registered instance with load balancer $ELB_NAME, resulting registered instances: $OUTPUT" + fi + if [ "$DEPLOYMENT_SUCCESS" != "0" ]; then exit 1; fi + done + debug "$MESSAGE: success" + fi +} + +deploy diff --git a/chef-json.yml b/chef-json.yml new file mode 100644 index 00000000..9dbc3c97 --- /dev/null +++ b/chef-json.yml @@ -0,0 +1,64 @@ +--- +- name: Generate Chef custom JSON + hosts: localhost + connection: local + vars_files: + - vars/CKAN-Stack.var.yml + pre_tasks: + + - name: get basic_facts + set_fact: + basic_fact={{ item }} + when: item.Environment == Environment + with_items: "{{ basic_facts }}" + + - name: set facts to environment from basic_fact + set_fact: "{{ item.key }}={{ item.value }}" + with_dict: "{{ basic_fact }}" + when: basic_fact is defined + + - name: Identify cloudformation stack + set_fact: + Stack_Name: "{{ item.name }}" + state: "{{ item.state }}" + template_parameters: "{{ item.template_parameters }}" + when: item.template_parameters.Environment == Environment + with_items: "{{ cloudformation_stacks }}" + + - name: Describe cloudformation stack + cloudformation_facts: + region: "{{ region }}" + stack_name: "{{ Stack_Name }}" + stack_resources: true + register: stack + + - name: set facts to environment from stack parameters + set_fact: "{{ item.key }}={{ item.value }}" + with_dict: "{{ stack.ansible_facts.cloudformation[Stack_Name].stack_parameters }}" + + - name: Identify hosted zone cloudformation stack + set_fact: + Zone_Stack_Name: "CKAN-{{ Environment }}-hosted-zone" + + - name: Describe hosted zone cloudformation stack + cloudformation_facts: + region: "{{ region }}" + stack_name: "{{ Zone_Stack_Name }}" + stack_resources: true + register: hosted_zone + + - name: set internal TLD from hosted zone + set_fact: + InternalHostedZoneTLD: "{{ hosted_zone.ansible_facts.cloudformation[Zone_Stack_Name].stack_outputs.ZoneTLD }}" + + tasks: + + - name: Prepare chef source JSON + template: + src: chef-source.json.j2 + dest: templates/chef-source.json + + - name: Prepare custom JSON + template: + src: chef.json.j2 + dest: templates/chef.json diff --git a/library/ec2_vpc_peer_describe.py b/library/ec2_vpc_peer_describe.py index 1ae16d89..96284a1b 100644 --- a/library/ec2_vpc_peer_describe.py +++ b/library/ec2_vpc_peer_describe.py @@ -215,7 +215,7 @@ def main(): client = boto3_conn(module, conn_type='client', resource='ec2', region=region, endpoint=ec2_url, **aws_connect_kwargs) except botocore.exceptions.NoCredentialsError as e: module.fail_json(msg="Can't authorize connection - "+str(e)) - except Exception, e: + except Exception as e: module.fail_json(msg='Failed to connect to VPC: %s' % str(e)) if peer_vpc_id is None: @@ -232,4 +232,4 @@ def main(): from ansible.module_utils.basic import * from ansible.module_utils.ec2 import * -main() \ No newline at end of file +main() diff --git a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 similarity index 92% rename from templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml rename to templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 index 9f4270c3..6c750201 100644 --- a/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml +++ b/templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2 @@ -109,25 +109,6 @@ Parameters: DefaultEC2Key: Description: Select an existing SSH key Type: AWS::EC2::KeyPair::KeyName - AdminUserGroup: - Description: Valid Linux group name - Type: String - Default: link - CKANAdminPW: - NoEcho: true - Description: Password must be at least 8 characters up to a maximum of 64 characters in length. - Type: String - MinLength: 8 - MaxLength: 64 - AllowedPattern: "[-a-zA-Z0-9_!@#$%^&*()|]*" - ConstraintDescription: Must be 8 - 64 characters. - CKANAdminEmail: - Description: Valid Email Address - Type: String - AdminPubKeyBucket: - Description: S3 Bucket URL - Type: String - Default: s3://link-opsworks/keys/ StackVPC: Description: The exported name of the existing VPC ID. Type: String @@ -155,23 +136,6 @@ Parameters: ACMCertificateARN: Description: ARN for certificate to be used by the application load balancer. Type: String - PGDBUser: - Description: Account name for (non-superuser) Postgres DB access. - Type: String - PGDBPassword: - NoEcho: true - Description: Password must be at least 8 alphanumeric characters up to a maximum of 64 characters in length. - Type: String - MinLength: 8 - MaxLength: 64 - AllowedPattern: "[a-zA-Z0-9]*" - ConstraintDescription: Must be 8 - 64 alphanumeric characters. - BeakerSecret: - NoEcho: true - Description: Secret key used by CKAN to hash login cookies. - Type: String - MinLength: 32 - ConstraintDescription: Must be at least 32 characters. SSMKey: Type: String Description: KMS key for alias/aws/ssm @@ -318,6 +282,14 @@ Resources: - "" - - "arn:aws:route53:::hostedzone/" - Fn::ImportValue: !Ref InternalStackZone + # allow the servers to retrieve a subset of SSM Parameter Store values + - Effect: Allow + Action: + - ssm:GetParameter + - ssm:GetParameters + - ssm:GetParametersByPath + Resource: + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/common/*" ManagedPolicyArns: - !Ref EBSTaggingPolicy - !Ref SmtpRelayPolicy @@ -360,6 +332,7 @@ Resources: - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/app/${ApplicationId}/" - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/app/${ApplicationId}/*" - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/db/${ApplicationId}_*" + - !Sub "arn:aws:ssm:*:*:parameter/config/CKAN/${Environment}/common/*" ManagedPolicyArns: - !Ref EBSTaggingPolicy - !Ref SmtpRelayPolicy @@ -393,24 +366,35 @@ Resources: datashades: ckan_web: title: !Sub "${ApplicationTitle} | Queensland Government" + site_domain: !Ref SiteDomain email_domain: !Ref EmailDomain - adminpw: !Ref CKANAdminPW - adminemail: !Ref CKANAdminEmail - beaker_secret: !Ref BeakerSecret - dbuser: !Ref PGDBUser - dbname: !Ref ApplicationName - dsuser: !Sub "${PGDBUser}_datastore" - dsname: !Sub "${ApplicationName}_datastore" dsenable: !Ref EnableDataStore endpoint: / google: gtm_container_id: !Ref GTMContainerId analytics_id: !Ref AnalyticsId - jumpbox: - bucket: !Ref AdminPubKeyBucket - usergroup: !Ref AdminUserGroup - postgres: - password: !Ref PGDBPassword + ckan_app: + name: !Sub "${ApplicationName}-${Environment}" + shortname: !Sub "${ApplicationId}-${Environment}" + app_source: + type: git + url: !Ref CKANSource + revision: !Ref CKANRevision + plugin_apps: +{% for plugin in extensions[Environment] %} + - name: "{{ extensions[Environment][plugin].name }}" + shortname: "{{ extensions[Environment][plugin].shortname }}" + app_source: + type: "{{ extensions[Environment][plugin].type }}" + url: "{{ extensions[Environment][plugin].url }}" + revision: "{{ extensions[Environment][plugin].version}}" +{% endfor %} + solr_app: + name: !Sub "${ApplicationName}-${Environment}-solr" + shortname: !Sub "${ApplicationId}-${Environment}-solr" + app_source: + type: archive + url: !Ref SolrSource redis: hostname: Fn::ImportValue: !Ref CacheAddress @@ -471,6 +455,9 @@ Resources: Shortname: !Sub "${ApplicationId}-solr" StackId: !Ref OpsWorksStack Type: custom + Tags: + - Key: Layer + Value: solr VolumeConfigurations: - MountPoint: "/mnt/local_data" NumberOfDisks: 1 @@ -503,6 +490,9 @@ Resources: Shortname: !Sub "${ApplicationId}-web" StackId: !Ref OpsWorksStack Type: custom + Tags: + - Key: Layer + Value: web LoadBasedAutoScaling: Enable: Yes UpScaling: @@ -549,6 +539,9 @@ Resources: Shortname: !Sub "${ApplicationId}-batch" StackId: !Ref OpsWorksStack Type: custom + Tags: + - Key: Layer + Value: batch LoadBasedAutoScaling: Enable: Yes UpScaling: diff --git a/templates/cache.cfn.yml b/templates/cache.cfn.yml index 4e1edb44..9b55d47e 100644 --- a/templates/cache.cfn.yml +++ b/templates/cache.cfn.yml @@ -68,3 +68,13 @@ Resources: NumCacheNodes: !Ref NumCacheNodes VpcSecurityGroupIds: - Fn::ImportValue: !Sub "${Environment}${Platform}DatabaseSG" + + EndpointAddress: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "/config/CKAN/${Environment}/common/cache_address" + Type: String + Value: + Fn::GetAtt: + - CacheCluster + - RedisEndpoint.Address diff --git a/templates/chef-source.json.j2 b/templates/chef-source.json.j2 new file mode 100644 index 00000000..3944f596 --- /dev/null +++ b/templates/chef-source.json.j2 @@ -0,0 +1,2 @@ +{% set ref_type = 'tags' if CookbookRevision | regex_search('^[0-9.]{5}') else 'remotes/origin' %} +"SourceType":["Git"],"SourceInfo":["{\"repository\":\"{{ CookbookURL }}\",\"getOptions\":\"branch:refs/{{ ref_type }}/{{ CookbookRevision }}\"}"] diff --git a/templates/chef.json.j2 b/templates/chef.json.j2 new file mode 100644 index 00000000..26549c6a --- /dev/null +++ b/templates/chef.json.j2 @@ -0,0 +1,44 @@ +{"datashades": { + "public_tld": "{{ PublicTLD }}", + "tld": "{{ InternalHostedZoneTLD }}", + "app_id": "{{ ApplicationId }}", + "version": "{{ Environment }}", + "ckan_web": { + "title": "{{ ApplicationTitle }}", + "site_domain": "{{ SiteDomain }}", + "email_domain": "{{ EmailDomain }}", + "dsenable": "{{ EnableDataStore }}", + "endpoint": "/", + "ckan_app": { + "name": "{{ ApplicationName }}-{{ Environment }}", + "shortname": "{{ ApplicationId }}-{{ Environment }}", + "app_source": { + "type": "git", + "url": "{{ CKANSource }}", + "revision": "{{ CKANRevision }}" + } + }, + "plugin_apps": [ +{% for plugin in extensions[Environment] %} + {% if loop.index != 1 %},{% endif %} + { + "name": "{{ extensions[Environment][plugin].name }}", + "shortname": "{{ extensions[Environment][plugin].shortname }}", + "app_source": { + "type": "{{ extensions[Environment][plugin].type }}", + "url": "{{ extensions[Environment][plugin].url }}", + "revision": "{{ extensions[Environment][plugin].version}}" + } + } +{% endfor %} + ] + }, + "solr_app": { + "name": "{{ ApplicationName }}-{{ Environment }}-solr", + "shortname": "{{ ApplicationId }}-{{ Environment }}-solr", + "app_source": { + "type": "archive", + "url": "{{ SolrSource }}" + } + } +}} diff --git a/test/features/resources.feature b/test/features/resources.feature index 5ec06a61..bdf4b498 100644 --- a/test/features/resources.feature +++ b/test/features/resources.feature @@ -50,6 +50,6 @@ Feature: Resource UI Scenario: Link resource with missing or invalid protocol should use HTTP Given "SysAdmin" as the persona When I log in - And I create a dataset and resource with key-value parameters "title=Non-HTTP resource" and "name=Non-HTTP link::url=git+https://github.com/ckan/ckan.git" + And I create a dataset and resource with key-value parameters "notes=Testing invalid link protocol" and "name=Non-HTTP link::url=git+https://github.com/ckan/ckan.git" And I press "Non-HTTP link" Then I should see "http://git+https://github.com/ckan/ckan.git" diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index 1e4bdf7b..0bac8dc1 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -5,14 +5,15 @@ NonProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdNonProduction' ProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdProduction', region=region) }}" solr_url: "http://archive.apache.org/dist/lucene/solr/8.11.2/solr-8.11.2.zip" -ckan_tag: "ckan-2.10.3-qgov.5" -ckan_qgov_branch: "qgov-master-2.10.3" +ckan_tag: "ckan-2.10.4-qgov.1" +ckan_qgov_branch: "qgov-master-2.10.4" common_stack: &common_stack state: "{{ state | default('present')}}" region: "{{ region }}" disable_rollback: true template: "templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml" + template_jinja: "templates/Datashades-OpsWorks-CKAN-Stack.cfn.yml.j2" template_parameters: &common_stack_template_parameters ApplicationName: "{{ service_name }}" ApplicationId: "{{ service_name | lower }}" @@ -24,16 +25,11 @@ common_stack: &common_stack EmailDomain: "{{ RootDomain }}" SiteDomain: "{{ SiteDomain }}" RootDomain: "{{ RootDomain }}" - ACMCertificateARN: "{{ ACMCertificateARNForDefaultRegion }}" + ACMCertificateARN: "{{ ACMCertificateARNForDefaultRegion | default('') }}" CacheAddress: "{{ Environment }}CKANCacheClusterAddress" StackVPC: "CKANVpc-{{ Environment }}" InternalStackZoneTLD: "{{ Environment }}CKANPrivateTLD" InternalStackZone: "{{ Environment }}CKANPrivateHostedZone" - CKANAdminEmail: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/admin_email', region=region) }}" - CKANAdminPW: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/admin_password', region=region) }}" - PGDBUser: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/db/' + service_name_lower + '_user', region=region) }}" - PGDBPassword: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/db/' + service_name_lower + '_password', region=region) }}" - BeakerSecret: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/beaker_secret', region=region) }}" WebSubnets: "{{ Environment }}CKANWebSubnet" AppSubnets: "{{ Environment }}CKANAppSubnet" AdminSG: "{{ Environment }}CKANManagementSG" @@ -45,7 +41,7 @@ common_stack: &common_stack EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - CookbookRevision: "{{ CookbookRevision | default('6.1.21') }}" + CookbookRevision: "{{ CookbookRevision | default('7.0.3') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket SolrSource: "{{ solr_url }}" diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index e95b0ec8..087c5505 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -15,7 +15,7 @@ extensions: description: "CKAN Express Loader Extension" type: "git" url: "https://github.com/qld-gov-au/ckanext-xloader.git" - version: "1.0.1-qgov.11" + version: "1.0.1-qgov.12" CKANExtQGOV: &CKANExtQGOV name: "ckanext-qgov-{{ Environment }}" @@ -87,7 +87,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.16" + version: "2.5.0-qgov.17" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index d43e2295..c93aba8f 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -15,7 +15,7 @@ extensions: description: "CKAN Express Loader Extension" type: "git" url: "https://github.com/qld-gov-au/ckanext-xloader.git" - version: "1.0.1-qgov.11" + version: "1.0.1-qgov.12" CKANExtQGOV: &CKANExtQGOV name: "ckanext-qgov-{{ Environment }}" @@ -87,7 +87,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.16" + version: "2.5.0-qgov.17" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}"