Merge pull request #46 from docksal/develop

Release 3.0.0

.gitignore

@@ -1 +1,5 @@
 */build/
+.vscode
+
+# GCP ignores.
+gcp-deployment-manager/local-override.env

.travis.yml

@@ -22,9 +22,10 @@ install:
 
 script:
   - cd aws-cloudformation && ./scripts/bash2yaml startup-${TEMPLATE_TYPE}.sh tpl-${TEMPLATE_TYPE}.yaml
-  - mkdir -p ${LOCAL_DIR} && mv -f template.yaml ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml
+  - aws cloudformation package --template-file template.yaml --s3-bucket null --output-template-file package.json --use-json
+  - mkdir -p ${LOCAL_DIR} && mv -f package.json ${LOCAL_DIR}/${TEMPLATE_TYPE}.json
   - tmp=$(basename $(mktemp -u))
-  - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml s3://${S3_BUCKET}/${tmp} --acl public-read
+  - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.json s3://${S3_BUCKET}/${tmp} --acl public-read
   - res=$(aws cloudformation validate-template --region ${AWS_REGION} --template-url https://${S3_BUCKET}.s3.amazonaws.com/${tmp} || echo "error")
   - aws s3 rm s3://${S3_BUCKET}/${tmp}
   - if [ "${res}" == "error" ]; then exit 1; fi

aws-cloudformation/README.md

@@ -36,7 +36,7 @@ If you have an existing AWS account (with billing and an SSH key pair), just cli
 **WARNING:** if you have an existing sandbox server created before Dec 31, 2019 (v1), **DO NOT UPGRADE**.  
 See [v2.0.0](https://github.com/docksal/sandbox-server/releases/tag/v2.0.0) release notes.
 
-[![Launch Basic Stack](https://cdn.rawgit.com/buildkite/cloudformation-launch-stack-button-svg/master/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=docksal-sandbox-server&templateURL=https://s3.us-east-2.amazonaws.com/docksal-aws-templates/sandbox-server/v2/basic.yaml)
+[![Launch Basic Stack](https://cdn.rawgit.com/buildkite/cloudformation-launch-stack-button-svg/master/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=docksal-sandbox-server&templateURL=https://s3.us-east-2.amazonaws.com/docksal-aws-templates/sandbox-server/v3/basic.json)
 
 You will be prompted for:
 
@@ -60,20 +60,24 @@ If you have an existing AWS account (with billing and an SSH key pair), just cli
 **WARNING:** if you have an existing sandbox server created before Dec 31, 2019 (v1), **DO NOT UPGRADE**.  
 See [v2.0.0](https://github.com/docksal/sandbox-server/releases/tag/v2.0.0) release notes.
 
-[![Launch Advanced Stack](https://cdn.rawgit.com/buildkite/cloudformation-launch-stack-button-svg/master/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=docksal-sandbox-server&templateURL=https://s3.us-east-2.amazonaws.com/docksal-aws-templates/sandbox-server/v2/advanced.yaml)
+[![Launch Advanced Stack](https://cdn.rawgit.com/buildkite/cloudformation-launch-stack-button-svg/master/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=docksal-sandbox-server&templateURL=https://s3.us-east-2.amazonaws.com/docksal-aws-templates/sandbox-server/v3/advanced.json)
 
-You will be prompted for few required and optional settings.
+You will be prompted for a few required and optional settings.
 
 - Basic: Required
   - Resource type (`ec2` vs `spot`)
-  - Instance type
+  - Instance type (primary)
+  - Instance type 2 (spot only)
+  - Instance type 3 (spot only)
   - SSH key
   - Availability zone
-- VPC/Network: Optional**
+- VPC/Network: Optional
   - VPC ID
   - Subnet ID
   - Elastic IP
-  - Access from
+  - Access from CIDR 1
+  - Access from CIDR 2
+  - Access from CIDR 3
 - Storage: Optional
   - Persistent data volume
   - Enable artifacts bucket
@@ -85,6 +89,8 @@ You will be prompted for few required and optional settings.
 - LetsEncrypt settings: Optional
   - Sandbox domain name
   - LetsEncrypt configuration
+- Docksal settings: Optional
+  - Docksal version
 
 Once provisioned, the IP address of the server will be printed in the **Outputs** section in CloudFormation (`<external-ip>`). 
 

aws-cloudformation/scripts/find-amis

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # ubuntu official owner 099720109477
-latest_ami="ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-????????"
+latest_ami="ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-????????"
 # latest_ami="ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20191002"
 
 AWS_REGION=$(aws ec2 describe-regions --output text | awk '{print $3}' | xargs)

aws-cloudformation/startup-advanced.sh

@@ -33,7 +33,6 @@ MOUNT_POINT="/data"
 BUILD_USER="build-agent"
 BUILD_USER_UID="1100"
 BUILD_USER_HOME="/home/${BUILD_USER}"
-DOCKSAL_VERSION="master"
 PROJECT_INACTIVITY_TIMEOUT="0.5h"
 PROJECT_DANGLING_TIMEOUT="168h"
 PROJECTS_ROOT="${BUILD_USER_HOME}/builds"
@@ -159,6 +158,7 @@ do
   sleep 5
 done
 
+# get stack parameters
 export EIP=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Outputs[?OutputKey==`IPAddress`].OutputValue' --output text)
 export VOLUME_ID=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`ExistingDataVolume`].ParameterValue' --output text)
 export GITHUB_TOKEN=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`GitHubToken`].ParameterValue' --output text)
@@ -167,6 +167,8 @@ export GITHUB_TEAM_SLUG=$(aws cloudformation describe-stacks --stack-name=${STAC
 export LETSENCRYPT_DOMAIN=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`LetsEncryptDomain`].ParameterValue' --output text)
 export LETSENCRYPT_CONFIG=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`LetsEncryptConfig`].ParameterValue' --output text)
 export ARTIFACTS_S3_BUCKET=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Outputs[?OutputKey==`ArtifactsBucket`].OutputValue' --output text)
+export DOCKSAL_VERSION=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`DocksalVersion`].ParameterValue' --output text)
+export DOCKSAL_VERSION=${DOCKSAL_VERSION:-"master"}
 
 # attach/detach elastic ip
 if [[ "${EIP}" != "${ATTACHED_IP}" ]]
@@ -325,6 +327,9 @@ then
     sed -i "s|^BACKUP_SSH_PUBLIC_KEY=\".*\"|BACKUP_SSH_PUBLIC_KEY=\"${BACKUP_SSH_PUBLIC_KEY}\"|g" /usr/local/bin/ssh-rake
     chmod +x /usr/local/bin/ssh-rake
     /usr/local/bin/ssh-rake install
+    # Remove ec2-instance-connect as it brakes ssh-rake
+    # See https://github.com/aws/aws-ec2-instance-connect-config/issues/19
+    apt-get purge ec2-instance-connect -y
 fi
 
 if [[ "${old_stack_md5sum}" != "${stack_md5sum}" ]]
@@ -385,4 +390,3 @@ fi
 
 su - build-agent -c "fin system reset"
 echo "${stack_md5sum}" >/root/stack_last_update
-

aws-cloudformation/tpl-advanced.yaml

@@ -1,4 +1,4 @@
-AWSTemplateFormatVersion: 2010-09-09
+AWSTemplateFormatVersion: "2010-09-09"
 Description: Sandbox server template (advanced)
 
 Parameters:
@@ -16,35 +16,65 @@ Parameters:
       - spot
   InstanceType:
     Description: |
-      EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM.
-      Instances with < 2GB RAM are not recommended and should only be used for testing the CloudFormation template.
+      EC2 instance type (default: t3.small = 2 vCPU, 2GB RAM).
+      t3 (Intel) / t3a (AMD) family instances provide burstable CPU performance and are generally the best choise for a sandbox server.
+      t3/t3a: CPU/RAM ratio is variable from 1/1 to 1/4 based on instance size (t3.large = 2 vCPU, 8GB RAM).
+      c5: CPU/RAM ratio is 1/2 (c5.large = 2 vCPU, 4GB RAM).
+      m5: CPU/RAM ratio is 1/4 (m5.large = 2 vCPU, 8GB RAM).
+      r5: CPU/RAM ratio is 1/8 (r5.large = 2 vCPU, 16GB RAM).
     Type: String
-    # Instances with < 2GB RAM should only be used for testing the CloudFormation template
-    # t3.small: 2 vCPU, 2GB RAM
     Default: t3.small
     AllowedValues:
-      - t3.nano
-      - t3.micro
       - t3.small
       - t3.medium
       - t3.large
       - t3.xlarge
       - t3.2xlarge
+      - t3a.small
+      - t3a.medium
+      - t3a.large
+      - t3a.xlarge
+      - t3a.2xlarge
+      - c5.large
+      - c5.xlarge
+      - c5.2xlarge
+      - c5.4xlarge
+      - m5.large
+      - m5.xlarge
+      - m5.2xlarge
+      - m5.4xlarge
+      - r5.large
+      - r5.xlarge
+      - r5.2xlarge
+  InstanceType2:
+    Description: |
+      Additional EC2 instance type (spot mode only). AWS will pick the cheapest available option for spot instance.
+      IMPORTANT: "Instance Type"/"Instance Type 2" must be unique, overwise stack creation/update will fail.
+      Recomendation: Use a "t3" instance for the primary option and a matching size "t3a" instance type here.
+    Type: String
+    Default: t3.medium
+    AllowedValues:
+      - t3.small
+      - t3.medium
+      - t3.large
+      - t3.xlarge
+      - t3.2xlarge
+      - t3a.small
+      - t3a.medium
+      - t3a.large
+      - t3a.xlarge
+      - t3a.2xlarge
+      - c5.large
+      - c5.xlarge
+      - c5.2xlarge
+      - c5.4xlarge
       - m5.large
       - m5.xlarge
       - m5.2xlarge
       - m5.4xlarge
-      - m5d.large
-      - m5d.xlarge
-      - m5d.2xlarge
-      - m5d.4xlarge
       - r5.large
       - r5.xlarge
       - r5.2xlarge
-      - r5d.large
-      - r5d.xlarge
-      - r5d.2xlarge
-    ConstraintDescription: "Must be a valid EC2 instance type"
   KeyName:
     Description: "Name of an existing EC2 KeyPair to enable SSH access to the instance"
     Type: "AWS::EC2::KeyPair::KeyName"
@@ -128,6 +158,11 @@ Parameters:
     Description: "Set name for the artifacts bucket. Leave empty to have the bucket name automatically generated."
     Type: String
     Default: ""
+  DocksalVersion:
+    Description: |
+      Specify the version of Docksal to install (e.g., v1.14.0). Leave empty to get the latest stable version (master).
+    Type: String
+    Default: ""
 
 Metadata:
   AWS::CloudFormation::Interface:
@@ -137,6 +172,7 @@ Metadata:
         Parameters:
           - ResourceType
           - InstanceType
+          - InstanceType2
           - KeyName
           - ManualAZ
       - Label:
@@ -165,11 +201,17 @@ Metadata:
         Parameters:
           - LetsEncryptDomain
           - LetsEncryptConfig
+      - Label:
+          default: "Docksal settings: Optional"
+        Parameters:
+          - DocksalVersion
     ParameterLabels:
       ResourceType:
         default: "Resource type"
       InstanceType:
-        default: "Instance type"
+        default: "Instance type (primary)"
+      InstanceType2:
+        default: "Instance type 2 (spot only)"
       KeyName:
         default: "SSH key"
       ManualAZ:
@@ -181,11 +223,11 @@ Metadata:
       ExistingEIP:
         default: "Elastic IP"
       AccessFrom1:
-        default: "Access from cidr 1"
+        default: "Access from CIDR 1"
       AccessFrom2:
-        default: "Access from cidr 2"
+        default: "Access from CIDR 2"
       AccessFrom3:
-        default: "Access from cidr 3"
+        default: "Access from CIDR 3"
       ExistingDataVolume:
         default: "Persistent data volume"
       EnableArtifactsBucket:
@@ -202,41 +244,43 @@ Metadata:
         default: "Sandbox domain name"
       LetsEncryptConfig:
         default: "LetsEncrypt configuration"
+      DocksalVersion:
+        default: "Docksal version"
 
 Mappings:
   Region2AMI:
     eu-north-1:
-      AMI: ami-005bc7d72deb72a3d
+      AMI: ami-0d4e2b57f569e9daa
     ap-south-1:
-      AMI: ami-0245841fc4b40e22f
+      AMI: ami-0c5b1a88222ac79cb
     eu-west-3:
-      AMI: ami-0b70d1460d5c7a299
+      AMI: ami-0e60c6afa19d896ee
     eu-west-2:
-      AMI: ami-00622b440d92e55c0
+      AMI: ami-0917237b4e71c5759
     eu-west-1:
-      AMI: ami-04c58523038d79132
+      AMI: ami-0dad359ff462124ca
     ap-northeast-2:
-      AMI: ami-02b4a5559ce53a570
+      AMI: ami-0de407404c33d1671
     ap-northeast-1:
-      AMI: ami-0f6b4f4104d26f399
+      AMI: ami-0c1ac8728ef7f87a4
     sa-east-1:
-      AMI: ami-049f5d88d2d436431
+      AMI: ami-001c16a3a4f5d85f1
     ca-central-1:
-      AMI: ami-0972a0d3135cf1fc0
+      AMI: ami-08a6203f59f9df866
     ap-southeast-1:
-      AMI: ami-07febfdfb4080320e
+      AMI: ami-0b8cf0f359b1335e1
     ap-southeast-2:
-      AMI: ami-04a0f7552cff370ba
+      AMI: ami-0a1a4d97d4af3009b
     eu-central-1:
-      AMI: ami-09356619876445425
+      AMI: ami-05c26ae4789875080
     us-east-1:
-      AMI: ami-00a208c7cdba991ea
+      AMI: ami-068663a3c619dd892
     us-east-2:
-      AMI: ami-059d836af932792c3
+      AMI: ami-0e84e211558a022c0
     us-west-1:
-      AMI: ami-0f42d8c4eb586ccf7
+      AMI: ami-075fd582acf0c0128
     us-west-2:
-      AMI: ami-0a7d051a1c4b54f65
+      AMI: ami-09dd2e08d601bff67
 
 Conditions:
   CreateEC2: !Equals [!Ref ResourceType, "ec2"]
@@ -557,7 +601,8 @@ Resources:
         TargetCapacity: 1
         IamFleetRole: !GetAtt FleetRole.Arn
         LaunchSpecifications:
-          - InstanceType: !Ref InstanceType
+          - &instance-profile
+            InstanceType: !Ref InstanceType
             IamInstanceProfile:
               Arn:
                 Fn::GetAtt:
@@ -586,10 +631,13 @@ Resources:
                 - - |
                 # add to the end of init script echo line with template variable values, for recreate instance on every variable change
                   - !Sub
-                    - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig} >/dev/null
+                    - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig}-${DocksalVersion} >/dev/null
                     - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ]
                       eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ]
 
+          - <<: *instance-profile
+            InstanceType: !Ref InstanceType2
+
   ec2Instance:
     Type: 'AWS::EC2::Instance'
     Condition: CreateEC2
@@ -616,7 +664,7 @@ Resources:
           - - |
           # add to the end of init script echo line with template variable values, for recreate instance on every variable change
             - !Sub
-              - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig} >/dev/null
+              - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig}-${DocksalVersion} >/dev/null
               - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ]
                 eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ]