diff --git a/.gitignore b/.gitignore index 6968dc0f..38441023 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc *.retry +templates/AMI-Template-Instances.cfn.yml templates/chef.json templates/chef-source.json templates/cloudfront.cfn.yml diff --git a/AMI-templates.yml b/AMI-templates.yml new file mode 100644 index 00000000..4c3c23d2 --- /dev/null +++ b/AMI-templates.yml @@ -0,0 +1,39 @@ +--- +- name: Cloudformation Playbook + hosts: local + connection: local + + pre_tasks: + - name: get basic_facts + set_fact: + basic_fact={{ item }} +# CKANSource={{ item.CKANSource }} + when: item.Environment == Environment + with_items: "{{ basic_facts }}" + + - name: set facts to environment from basic_fact + set_fact: "{{ item.key }}={{ item.value }}" + with_dict: "{{ basic_fact }}" + when: basic_fact is defined + + - name: kms alias fact + aws_kms_facts: + filters: + alias: "aws/ssm" + region: "{{ region }}" + register: ssmKeyFacts + + - name: set KMS key from alias + set_fact: + SSMKey: "{{ ssmKeyFacts['keys'][0].key_arn }}" + + - name: Generate Lambda file hash + shell: > + md5sum files/instanceSetupLambda.js | awk '{print substr($1, 1, 20)}' + register: hash_output + - set_fact: + instance_setup_source_hash: "{{ hash_output.stdout_lines[0] }}" + + - include_vars: vars/AMI-template-instances.var.yml + roles: + - ansible_cloudformation diff --git a/files/instanceSetupLambda.js b/files/instanceSetupLambda.js index b0d45019..1992c87b 100644 --- a/files/instanceSetupLambda.js +++ b/files/instanceSetupLambda.js @@ -21,6 +21,12 @@ function recordCompletion(event, success) { exports.handler = async (event) => { const instanceId = event['EC2InstanceId']; + /* + * Possible deployment types include: + * - 'setup': Full end-to-end configuration, from vanilla operating system to live instance. + * - 'deploy': Take a vanilla operating system and set up a warm instance, but do not make it active. + * - 'configure': Take a warm instance and make it active. + */ const deployPhase = 'phase' in event ? event['phase'] : 'setup'; if (!['setup', 'deploy', 'configure'].includes(deployPhase)) { console.log("Invalid deployment phase '" + deployPhase + "', must be one of 'setup', 'deploy', 'configure'"); @@ -95,12 +101,15 @@ exports.handler = async (event) => { } else { recipePrefix = `datashades::${layer}`; } - var runList = `recipe[${recipePrefix}-configure]`; - if (deployPhase !== 'configure') { - runList = `recipe[${recipePrefix}-deploy],${runList}`; + var runList = ""; + if (deployPhase !== 'deploy') { + runList = `recipe[${recipePrefix}-configure]`; } if (deployPhase === 'setup') { - runList = `recipe[${recipePrefix}-setup],${runList}`; + runList = `,${runList}`; + } + if (deployPhase !== 'configure') { + runList = `recipe[${recipePrefix}-setup],recipe[${recipePrefix}-deploy]${runList}`; } await ssm.send(new SendCommandCommand({ diff --git a/templates/AMI-Template-Instances.cfn.yml.j2 b/templates/AMI-Template-Instances.cfn.yml.j2 new file mode 100644 index 00000000..0157e272 --- /dev/null +++ b/templates/AMI-Template-Instances.cfn.yml.j2 @@ -0,0 +1,88 @@ +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Creates archetype instances for generating machine image templates.' + +Parameters: + ApplicationName: + Description: Name of the application (eg OpenData) + Type: String + ApplicationId: + Description: All-lowercase identifier for the application (eg 'opendata') + Type: String + ConstraintDescription: Must contain only lowercase/numeric/hyphen/underscore. + AllowedPattern: '[-_a-z0-9]*' + Environment: + Description: The target production vs non-production environment. + Type: String + Default: DEV + AllowedValues: + - DEV + - TRAINING + - STAGING + - PROD + AppSubnets: + Description: The base name for the exported application layer subnet IDs, eg if the exports are 'PRODMyApplicationAppSubnetA' and 'PRODMyApplicationAppSubnetB', then this would be 'PRODMyApplicationAppSubnet'. Only needed for HA configurations. + Type: String + Default: none + DefaultEC2Key: + Description: Select an existing SSH key + Type: AWS::EC2::KeyPair::KeyName + +Resources: + +{% for layer in ['Batch', 'Web', 'Solr'] %} +{% set disk_size = 100 if layer == 'Solr' else 32 %} + {{ layer }}TemplateInstance: + Type: AWS::EC2::Instance + Properties: + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + DeleteOnTermination: true + VolumeSize: 100 + VolumeType: "gp2" + - DeviceName: "/dev/sdi" + Ebs: + DeleteOnTermination: true + VolumeSize: {{ disk_size }} + VolumeType: "gp2" + IamInstanceProfile: !Ref {% if layer != 'Solr' %}Web{% endif %}InstanceRoleProfile + ImageId: "ami-0d71fe73adf7a9887" + InstanceType: "t3a.small" + KeyName: !Ref DefaultEC2Key + NetworkInterfaces: + - DeviceIndex: 0 + GroupSet: + - Fn::ImportValue: !Sub "${Environment}CKANManagementSG" + - Fn::ImportValue: !Sub "${Environment}CKAN{% if layer == 'Solr' %}Database{% else %}AppAsg{% endif %}SG" + SubnetId: + Fn::ImportValue: !Sub "${AppSubnets}A" + UserData: + Fn::Base64: + Fn::Sub: | + #!/bin/sh + if ! (grep '/mnt/local_data' /etc/fstab >/dev/null); then + mkdir /mnt/local_data + mkfs -t xfs /dev/sdi + echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab + mount -a + fi + if ! (yum install chef); then + for i in `seq 1 5`; do + yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break + sleep 5 + done + fi + REGION="--region ${AWS::Region}" + metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \ + INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \ + aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-ami-template" + FUNCTION_NAME=$(aws ssm get-parameter $REGION --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text) + if (aws --version |grep -o 'aws-cli/[2-9]'); then + PAYLOAD_FORMAT="--cli-binary-format raw-in-base64-out" + fi + aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "deploy"}' /var/log/instance-setup.log.`date '+%s'` + Tags: + - Key: Layer + Value: {{ layer|lower }} +{% endfor %} diff --git a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 index 6e076d2e..71ef6fe5 100644 --- a/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 +++ b/templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2 @@ -12,7 +12,7 @@ Parameters: ConstraintDescription: Must contain only lowercase/numeric/hyphen/underscore. AllowedPattern: '[-_a-z0-9]*' Environment: - Description: Select a stack version. + Description: The target production vs non-production environment. Type: String Default: STAGING AllowedValues: @@ -246,3 +246,12 @@ Resources: {% endif %} {% endfor %} + +Outputs: + +{% for layer in ['Batch', 'Web', 'Solr'] %} + {{ layer }}LaunchTemplateId: + Value: !Ref {{ layer }}LaunchTemplate + Export: + Name: !Sub "${Environment}${ApplicationName}{{ layer }}LaunchTemplateId" +{% endfor %} diff --git a/templates/database.cfn.yml b/templates/database.cfn.yml index c0ef58ca..669759c3 100644 --- a/templates/database.cfn.yml +++ b/templates/database.cfn.yml @@ -27,6 +27,7 @@ Outputs: Value: !Ref Database Description: RDS Instance Id + Parameters: DBAllocatedStorage: ConstraintDescription: must be between 5 and 1024Gb. @@ -46,6 +47,13 @@ Parameters: ConstraintDescription: must select a valid database instance type. Description: Database instance class Type: String + DBMaxParallelWorkersPerGather: + Type: Number + Default: -1 + Description: "Specify the max parallel workers per gather (0 - 100), or -1 for no setting." + MinValue: -1 + MaxValue: 100 + DBName: AllowedPattern: '[a-zA-Z][a-zA-Z0-9]*' ConstraintDescription: must begin with a letter and contain only 1-64 alphanumeric @@ -116,6 +124,8 @@ Parameters: StackZone: Description: Exported name of the Route53 hosted zone ID. Type: String +Conditions: + IsMaxParallelWorkersPerGatherSet: !Not [!Equals [!Ref DBMaxParallelWorkersPerGather, "-1"]] Resources: Database: @@ -177,6 +187,14 @@ Resources: Description: Setup parameters for Queensland Government databases Family: !Sub "${DBEngine}${DBEngineVersion}" Parameters: - datestyle: "ISO, DMY" - rds.adaptive_autovacuum: 1 - rds.force_ssl: 0 + Fn::If: + - IsMaxParallelWorkersPerGatherSet + - datestyle: "ISO, DMY" + rds.adaptive_autovacuum: 1 + rds.force_ssl: 0 + max_parallel_workers_per_gather: !Ref DBMaxParallelWorkersPerGather + #ELSE + - datestyle: "ISO, DMY" + rds.adaptive_autovacuum: 1 + rds.force_ssl: 0 + diff --git a/templates/waf_web_acl.cfn.yml b/templates/waf_web_acl.cfn.yml index df4adbac..c52a2a60 100644 --- a/templates/waf_web_acl.cfn.yml +++ b/templates/waf_web_acl.cfn.yml @@ -21,8 +21,20 @@ Resources: MetricName: "WafWebACL" SampledRequestsEnabled: true Rules: - - Name: !Sub "${Environment}-${Platform}-XSSRule" + - Name: !Sub "${Environment}-${Platform}-RateLimitRule" Priority: 1 + Action: + Block: {} + Statement: + RateBasedStatement: + AggregateKeyType: "IP" + EvaluationWindowSec: 300 + Limit: 5000 + VisibilityConfig: + CloudWatchMetricsEnabled: true + MetricName: "RateLimitRule" + - Name: !Sub "${Environment}-${Platform}-XSSRule" + Priority: 2 Action: Block: {} Statement: diff --git a/test/features/xloader.feature b/test/features/xloader.feature index 5af78684..a39096c9 100644 --- a/test/features/xloader.feature +++ b/test/features/xloader.feature @@ -7,11 +7,12 @@ Feature: XLoader When I log in And I create a dataset and resource with key-value parameters "notes=Testing XLoader" and "name=test-csv-resource::url=https://people.sc.fsu.edu/~jburkardt/data/csv/addresses.csv::format=CSV" # Wait for XLoader to run - And I wait for 10 seconds And I press "test-csv-resource" + And I reload page every 3 seconds until I see an element with xpath "//*[contains(string(), 'DataStore')]" but not more than 6 times Then I should see "DataStore" When I press "DataStore" + And I reload page every 3 seconds until I see an element with xpath "//*[contains(string(), 'Express Load completed')]" but not more than 6 times Then I should see "Express Load completed" And I should see "Data Schema" And I should see "Data Dictionary" diff --git a/vars/AMI-template-instances.var.yml b/vars/AMI-template-instances.var.yml new file mode 100644 index 00000000..9748afb3 --- /dev/null +++ b/vars/AMI-template-instances.var.yml @@ -0,0 +1,31 @@ +--- + +common_stack: &common_stack + state: "{{ state | default('present')}}" + region: "{{ region }}" + disable_rollback: true + template_jinja: "templates/AMI-Template-Instances.cfn.yml.j2" + template: "templates/AMI-Template-Instances.cfn.yml" + template_parameters: &common_stack_template_parameters + ApplicationName: "{{ service_name }}" + ApplicationId: "{{ service_name_lower }}" + Environment: "{{ Environment }}" + AppSubnets: "{{ Environment }}CKANAppSubnet" + LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" + AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" + SSMKey: "{{ SSMKey | default('') }}" + InternalStackZone: "{{ Environment }}CKANPrivateHostedZone" + DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" + tags: &common_stack_tags + Environment: "{{ Environment }}" + Service: "{{ service_name }}" + Division: "{{ Division }}" + Owner: "{{ Owner }}" + Version: "1.0" + +cloudformation_stacks: + - <<: *common_stack + name: "{{ service_name }}-{{ Environment }}-AMI-Template-Instances" + template_parameters: + <<: *common_stack_template_parameters + Environment: "{{ Environment }}" diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index 6940e7ac..9e9e2e58 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -5,7 +5,7 @@ NonProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdNonProduction' ProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdProduction', region=region) }}" solr_url: "https://archive.apache.org/dist/lucene/solr/8.11.2/solr-8.11.2.zip" -ckan_tag: "ckan-2.10.5-qgov.3" +ckan_tag: "ckan-2.10.5-qgov.4" ckan_qgov_branch: "qgov-master-2.10.5" common_stack: &common_stack @@ -40,7 +40,7 @@ common_stack: &common_stack EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - CookbookRevision: "{{ CookbookRevision | default('7.2.1') }}" + CookbookRevision: "{{ CookbookRevision | default('8.0.0') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket SolrSource: "{{ solr_url }}" diff --git a/vars/database.var.yml b/vars/database.var.yml index e664ded0..8c8820b8 100644 --- a/vars/database.var.yml +++ b/vars/database.var.yml @@ -32,6 +32,7 @@ cloudformation_stacks: <<: *common_stack_template_parameters DBAllocatedStorage: "600" DBClass: db.m5.large + DBMaxParallelWorkersPerGather: 0 #increase if have more than 4 cpu core StorageEncrypted: "True" PreferredMaintenanceWindow: "sat:17:00-sat:17:30" Environment: PROD @@ -46,6 +47,7 @@ cloudformation_stacks: <<: *common_stack_template_parameters DBAllocatedStorage: '300' DBClass: db.t3.large + DBMaxParallelWorkersPerGather: 0 #increase if have more than 4 cpu core StorageEncrypted: "True" Environment: STAGING MultiAZ: "true" @@ -59,6 +61,7 @@ cloudformation_stacks: <<: *common_stack_template_parameters DBAllocatedStorage: "100" DBClass: db.t3.medium + DBMaxParallelWorkersPerGather: 0 #increase if have more than 4 cpu core StorageEncrypted: "False" Environment: TRAINING MultiAZ: "false" @@ -73,6 +76,7 @@ cloudformation_stacks: DBEngineVersion: 15 DBAllocatedStorage: "200" DBClass: db.t3.medium + DBMaxParallelWorkersPerGather: 0 #increase if have more than 4 cpu core StorageEncrypted: "False" Environment: DEV MultiAZ: "false" diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index ab1ea01e..57e92ae7 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -14,7 +14,7 @@ extensions: description: "CKAN Express Loader Extension" type: "git" url: "https://github.com/qld-gov-au/ckanext-xloader.git" - version: "1.0.1-qgov.14" + version: "1.0.1-qgov.16" CKANExtQGOV: &CKANExtQGOV name: "ckanext-qgov-{{ Environment }}" @@ -166,7 +166,7 @@ extensions: description: "CKAN Extension for single sign-on capability" type: "git" url: "https://github.com/qld-gov-au/ckanext-oidc-pkce" - version: "0.4.0-qgov.1" + version: "0.4.0-qgov.2" PROD: <<: *default_extensions diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index 943fbeb3..e3c3afcd 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -14,7 +14,7 @@ extensions: description: "CKAN Express Loader Extension" type: "git" url: "https://github.com/qld-gov-au/ckanext-xloader.git" - version: "1.0.1-qgov.14" + version: "1.0.1-qgov.16" CKANExtQGOV: &CKANExtQGOV name: "ckanext-qgov-{{ Environment }}" @@ -166,7 +166,7 @@ extensions: description: "CKAN Extension for single sign-on capability" type: "git" url: "https://github.com/qld-gov-au/ckanext-oidc-pkce" - version: "0.4.0-qgov.1" + version: "0.4.0-qgov.2" PROD: <<: *default_extensions diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index f25803c3..7ed418a0 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -62,7 +62,7 @@ extensions: description: "CKAN Extension for single sign-on capability" type: "git" url: "https://github.com/qld-gov-au/ckanext-oidc-pkce" - version: "0.4.0-qgov.1" + version: "0.4.0-qgov.2" PROD: <<: *default_extensions