From a8729847f13825ab413886070fada4d111dcda66 Mon Sep 17 00:00:00 2001 From: Sergey Zabolotny Date: Tue, 5 May 2020 10:39:44 +0300 Subject: [PATCH 01/25] Switch to ubuntu 20.04LTS amis --- aws-cloudformation/scripts/find-amis | 2 +- aws-cloudformation/tpl-advanced.yaml | 32 ++++++++++++++-------------- aws-cloudformation/tpl-basic.yaml | 32 ++++++++++++++-------------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/aws-cloudformation/scripts/find-amis b/aws-cloudformation/scripts/find-amis index e0fe1a9..32929df 100755 --- a/aws-cloudformation/scripts/find-amis +++ b/aws-cloudformation/scripts/find-amis @@ -1,7 +1,7 @@ #!/bin/bash # ubuntu official owner 099720109477 -latest_ami="ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-????????" +latest_ami="ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-????????" # latest_ami="ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20191002" AWS_REGION=$(aws ec2 describe-regions --output text | awk '{print $3}' | xargs) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index f6b8ed9..7e086f7 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -206,37 +206,37 @@ Metadata: Mappings: Region2AMI: eu-north-1: - AMI: ami-005bc7d72deb72a3d + AMI: ami-0d4e2b57f569e9daa ap-south-1: - AMI: ami-0245841fc4b40e22f + AMI: ami-0c5b1a88222ac79cb eu-west-3: - AMI: ami-0b70d1460d5c7a299 + AMI: ami-0e60c6afa19d896ee eu-west-2: - AMI: ami-00622b440d92e55c0 + AMI: ami-0917237b4e71c5759 eu-west-1: - AMI: ami-04c58523038d79132 + AMI: ami-0dad359ff462124ca ap-northeast-2: - AMI: ami-02b4a5559ce53a570 + AMI: ami-0de407404c33d1671 ap-northeast-1: - AMI: ami-0f6b4f4104d26f399 + AMI: ami-0c1ac8728ef7f87a4 sa-east-1: - AMI: ami-049f5d88d2d436431 + AMI: ami-001c16a3a4f5d85f1 ca-central-1: - AMI: ami-0972a0d3135cf1fc0 + AMI: ami-08a6203f59f9df866 ap-southeast-1: - AMI: ami-07febfdfb4080320e + AMI: ami-0b8cf0f359b1335e1 ap-southeast-2: - AMI: ami-04a0f7552cff370ba + AMI: ami-0a1a4d97d4af3009b eu-central-1: - AMI: ami-09356619876445425 + AMI: ami-05c26ae4789875080 us-east-1: - AMI: ami-00a208c7cdba991ea + AMI: ami-068663a3c619dd892 us-east-2: - AMI: ami-059d836af932792c3 + AMI: ami-0e84e211558a022c0 us-west-1: - AMI: ami-0f42d8c4eb586ccf7 + AMI: ami-075fd582acf0c0128 us-west-2: - AMI: ami-0a7d051a1c4b54f65 + AMI: ami-09dd2e08d601bff67 Conditions: CreateEC2: !Equals [!Ref ResourceType, "ec2"] diff --git a/aws-cloudformation/tpl-basic.yaml b/aws-cloudformation/tpl-basic.yaml index 2ef432d..18fb04a 100644 --- a/aws-cloudformation/tpl-basic.yaml +++ b/aws-cloudformation/tpl-basic.yaml @@ -65,37 +65,37 @@ Metadata: Mappings: Region2AMI: eu-north-1: - AMI: ami-005bc7d72deb72a3d + AMI: ami-0d4e2b57f569e9daa ap-south-1: - AMI: ami-0245841fc4b40e22f + AMI: ami-0c5b1a88222ac79cb eu-west-3: - AMI: ami-0b70d1460d5c7a299 + AMI: ami-0e60c6afa19d896ee eu-west-2: - AMI: ami-00622b440d92e55c0 + AMI: ami-0917237b4e71c5759 eu-west-1: - AMI: ami-04c58523038d79132 + AMI: ami-0dad359ff462124ca ap-northeast-2: - AMI: ami-02b4a5559ce53a570 + AMI: ami-0de407404c33d1671 ap-northeast-1: - AMI: ami-0f6b4f4104d26f399 + AMI: ami-0c1ac8728ef7f87a4 sa-east-1: - AMI: ami-049f5d88d2d436431 + AMI: ami-001c16a3a4f5d85f1 ca-central-1: - AMI: ami-0972a0d3135cf1fc0 + AMI: ami-08a6203f59f9df866 ap-southeast-1: - AMI: ami-07febfdfb4080320e + AMI: ami-0b8cf0f359b1335e1 ap-southeast-2: - AMI: ami-04a0f7552cff370ba + AMI: ami-0a1a4d97d4af3009b eu-central-1: - AMI: ami-09356619876445425 + AMI: ami-05c26ae4789875080 us-east-1: - AMI: ami-00a208c7cdba991ea + AMI: ami-068663a3c619dd892 us-east-2: - AMI: ami-059d836af932792c3 + AMI: ami-0e84e211558a022c0 us-west-1: - AMI: ami-0f42d8c4eb586ccf7 + AMI: ami-075fd582acf0c0128 us-west-2: - AMI: ami-0a7d051a1c4b54f65 + AMI: ami-09dd2e08d601bff67 Resources: InstanceSecurityGroup: From 3bcab95db197e3fd372ff66a8f84a121e489e260 Mon Sep 17 00:00:00 2001 From: Sergey Zabolotny Date: Tue, 2 Jun 2020 17:52:57 +0300 Subject: [PATCH 02/25] Multitype EC2 support in spot mode --- aws-cloudformation/tpl-advanced.yaml | 136 ++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 1 deletion(-) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index f6b8ed9..bb2e37a 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -16,7 +16,7 @@ Parameters: - spot InstanceType: Description: | - EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM. + Main EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM. Instances with < 2GB RAM are not recommended and should only be used for testing the CloudFormation template. Type: String # Instances with < 2GB RAM should only be used for testing the CloudFormation template @@ -44,6 +44,68 @@ Parameters: - r5d.large - r5d.xlarge - r5d.2xlarge + InstanceType2: + Description: | + Additional EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM. + Instances with < 2GB RAM are not recommended and should only be used for testing the CloudFormation template. + Will be used cheapest EC2 instance. Option applicable in spot mode only + Type: String + # Instances with < 2GB RAM should only be used for testing the CloudFormation template + # t3.small: 2 vCPU, 2GB RAM + Default: t3.small + AllowedValues: + - t3.nano + - t3.micro + - t3.small + - t3.medium + - t3.large + - t3.xlarge + - t3.2xlarge + - m5.large + - m5.xlarge + - m5.2xlarge + - m5.4xlarge + - m5d.large + - m5d.xlarge + - m5d.2xlarge + - m5d.4xlarge + - r5.large + - r5.xlarge + - r5.2xlarge + - r5d.large + - r5d.xlarge + - r5d.2xlarge + InstanceType3: + Description: | + Additional EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM. + Instances with < 2GB RAM are not recommended and should only be used for testing the CloudFormation template. + Will be used cheapest EC2 instance. Option applicable in spot mode only + Type: String + # Instances with < 2GB RAM should only be used for testing the CloudFormation template + # t3.small: 2 vCPU, 2GB RAM + Default: t3.small + AllowedValues: + - t3.nano + - t3.micro + - t3.small + - t3.medium + - t3.large + - t3.xlarge + - t3.2xlarge + - m5.large + - m5.xlarge + - m5.2xlarge + - m5.4xlarge + - m5d.large + - m5d.xlarge + - m5d.2xlarge + - m5d.4xlarge + - r5.large + - r5.xlarge + - r5.2xlarge + - r5d.large + - r5d.xlarge + - r5d.2xlarge ConstraintDescription: "Must be a valid EC2 instance type" KeyName: Description: "Name of an existing EC2 KeyPair to enable SSH access to the instance" @@ -137,6 +199,8 @@ Metadata: Parameters: - ResourceType - InstanceType + - InstanceType2 + - InstanceType3 - KeyName - ManualAZ - Label: @@ -170,6 +234,10 @@ Metadata: default: "Resource type" InstanceType: default: "Instance type" + InstanceType2: + default: "Instance type" + InstanceType3: + default: "Instance type" KeyName: default: "SSH key" ManualAZ: @@ -590,6 +658,72 @@ Resources: - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ] eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ] + - InstanceType: !Ref InstanceType2 + IamInstanceProfile: + Arn: + Fn::GetAtt: + - spotInstanceProfile + - Arn + Placement: + AvailabilityZone: !Ref ManualAZ + TagSpecifications: + - ResourceType: instance + Tags: + - Key: StackId + Value: !Ref AWS::StackId + ImageId: !FindInMap [Region2AMI, !Ref "AWS::Region", AMI] + KeyName: !Ref KeyName + SecurityGroups: + - GroupId: !GetAtt InstanceSecurityGroup.GroupId + BlockDeviceMappings: + # Root volume + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: '8' + SubnetId: !If [ SubnetIdDefined, !Ref SubnetId, !Ref "AWS::NoValue" ] + UserData: !Base64 + 'Fn::Join': + - '' + - - | + # add to the end of init script echo line with template variable values, for recreate instance on every variable change + - !Sub + - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig} >/dev/null + - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ] + eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ] + + - InstanceType: !Ref InstanceType3 + IamInstanceProfile: + Arn: + Fn::GetAtt: + - spotInstanceProfile + - Arn + Placement: + AvailabilityZone: !Ref ManualAZ + TagSpecifications: + - ResourceType: instance + Tags: + - Key: StackId + Value: !Ref AWS::StackId + ImageId: !FindInMap [Region2AMI, !Ref "AWS::Region", AMI] + KeyName: !Ref KeyName + SecurityGroups: + - GroupId: !GetAtt InstanceSecurityGroup.GroupId + BlockDeviceMappings: + # Root volume + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: '8' + SubnetId: !If [ SubnetIdDefined, !Ref SubnetId, !Ref "AWS::NoValue" ] + UserData: !Base64 + 'Fn::Join': + - '' + - - | + # add to the end of init script echo line with template variable values, for recreate instance on every variable change + - !Sub + - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig} >/dev/null + - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ] + eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ] + ec2Instance: Type: 'AWS::EC2::Instance' Condition: CreateEC2 From 54756a62c389287453fa1fb4f3136accc59a34d8 Mon Sep 17 00:00:00 2001 From: Sergey Zabolotny Date: Thu, 4 Jun 2020 13:13:11 +0300 Subject: [PATCH 03/25] Use anchors in template, switch to json package format --- .travis.yml | 5 +- aws-cloudformation/tpl-advanced.yaml | 80 ++++------------------------ aws-cloudformation/tpl-basic.yaml | 2 +- scripts/upload-to-s3.sh | 12 ++--- 4 files changed, 21 insertions(+), 78 deletions(-) diff --git a/.travis.yml b/.travis.yml index fbbdae4..2c4bd95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,10 @@ install: script: - cd aws-cloudformation && ./scripts/bash2yaml startup-${TEMPLATE_TYPE}.sh tpl-${TEMPLATE_TYPE}.yaml - - mkdir -p ${LOCAL_DIR} && mv -f template.yaml ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml + - aws cloudformation package --template-file template.yaml --s3-bucket null --output-template-file package.json --use-json + - mkdir -p ${LOCAL_DIR} && mv -f package.json ${LOCAL_DIR}/${TEMPLATE_TYPE}.json - tmp=$(basename $(mktemp -u)) - - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml s3://${S3_BUCKET}/${tmp} --acl public-read + - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.json s3://${S3_BUCKET}/${tmp} --acl public-read - res=$(aws cloudformation validate-template --region ${AWS_REGION} --template-url https://${S3_BUCKET}.s3.amazonaws.com/${tmp} || echo "error") - aws s3 rm s3://${S3_BUCKET}/${tmp} - if [ "${res}" == "error" ]; then exit 1; fi diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index bb2e37a..9ff9114 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -1,4 +1,4 @@ -AWSTemplateFormatVersion: 2010-09-09 +AWSTemplateFormatVersion: "2010-09-09" Description: Sandbox server template (advanced) Parameters: @@ -49,10 +49,11 @@ Parameters: Additional EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM. Instances with < 2GB RAM are not recommended and should only be used for testing the CloudFormation template. Will be used cheapest EC2 instance. Option applicable in spot mode only + IMPORTANT!!! Instance types InstanceType, InstanceType2 and InstanceType3 should differs, overwise stack creation/update will fail Type: String # Instances with < 2GB RAM should only be used for testing the CloudFormation template # t3.small: 2 vCPU, 2GB RAM - Default: t3.small + Default: t3.medium AllowedValues: - t3.nano - t3.micro @@ -80,10 +81,11 @@ Parameters: Additional EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM. Instances with < 2GB RAM are not recommended and should only be used for testing the CloudFormation template. Will be used cheapest EC2 instance. Option applicable in spot mode only + IMPORTANT!!! Instance types InstanceType, InstanceType2 and InstanceType3 should differs, overwise stack creation/update will fail Type: String # Instances with < 2GB RAM should only be used for testing the CloudFormation template # t3.small: 2 vCPU, 2GB RAM - Default: t3.small + Default: t3.large AllowedValues: - t3.nano - t3.micro @@ -625,7 +627,8 @@ Resources: TargetCapacity: 1 IamFleetRole: !GetAtt FleetRole.Arn LaunchSpecifications: - - InstanceType: !Ref InstanceType + - &instance-profile + InstanceType: !Ref InstanceType IamInstanceProfile: Arn: Fn::GetAtt: @@ -658,71 +661,10 @@ Resources: - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ] eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ] - - InstanceType: !Ref InstanceType2 - IamInstanceProfile: - Arn: - Fn::GetAtt: - - spotInstanceProfile - - Arn - Placement: - AvailabilityZone: !Ref ManualAZ - TagSpecifications: - - ResourceType: instance - Tags: - - Key: StackId - Value: !Ref AWS::StackId - ImageId: !FindInMap [Region2AMI, !Ref "AWS::Region", AMI] - KeyName: !Ref KeyName - SecurityGroups: - - GroupId: !GetAtt InstanceSecurityGroup.GroupId - BlockDeviceMappings: - # Root volume - - DeviceName: /dev/sda1 - Ebs: - VolumeSize: '8' - SubnetId: !If [ SubnetIdDefined, !Ref SubnetId, !Ref "AWS::NoValue" ] - UserData: !Base64 - 'Fn::Join': - - '' - - - | - # add to the end of init script echo line with template variable values, for recreate instance on every variable change - - !Sub - - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig} >/dev/null - - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ] - eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ] - - - InstanceType: !Ref InstanceType3 - IamInstanceProfile: - Arn: - Fn::GetAtt: - - spotInstanceProfile - - Arn - Placement: - AvailabilityZone: !Ref ManualAZ - TagSpecifications: - - ResourceType: instance - Tags: - - Key: StackId - Value: !Ref AWS::StackId - ImageId: !FindInMap [Region2AMI, !Ref "AWS::Region", AMI] - KeyName: !Ref KeyName - SecurityGroups: - - GroupId: !GetAtt InstanceSecurityGroup.GroupId - BlockDeviceMappings: - # Root volume - - DeviceName: /dev/sda1 - Ebs: - VolumeSize: '8' - SubnetId: !If [ SubnetIdDefined, !Ref SubnetId, !Ref "AWS::NoValue" ] - UserData: !Base64 - 'Fn::Join': - - '' - - - | - # add to the end of init script echo line with template variable values, for recreate instance on every variable change - - !Sub - - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig} >/dev/null - - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ] - eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ] + - <<: *instance-profile + InstanceType: !Ref InstanceType2 + - <<: *instance-profile + InstanceType: !Ref InstanceType3 ec2Instance: Type: 'AWS::EC2::Instance' diff --git a/aws-cloudformation/tpl-basic.yaml b/aws-cloudformation/tpl-basic.yaml index 2ef432d..5735e7b 100644 --- a/aws-cloudformation/tpl-basic.yaml +++ b/aws-cloudformation/tpl-basic.yaml @@ -1,4 +1,4 @@ -AWSTemplateFormatVersion: 2010-09-09 +AWSTemplateFormatVersion: "2010-09-09" Description: Sandbox server template (basic) Parameters: diff --git a/scripts/upload-to-s3.sh b/scripts/upload-to-s3.sh index b61682e..193ad9b 100755 --- a/scripts/upload-to-s3.sh +++ b/scripts/upload-to-s3.sh @@ -42,15 +42,15 @@ is_pr && exit 0 # Upload templates if is_edge; then - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml s3://${S3_BUCKET}/${EDGE_UPLOAD_DIR}/${TEMPLATE_TYPE}.yaml --acl public-read + aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.json s3://${S3_BUCKET}/${EDGE_UPLOAD_DIR}/${TEMPLATE_TYPE}.json --acl public-read elif is_stable; then - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml s3://${S3_BUCKET}/${STABLE_UPLOAD_DIR}/${TEMPLATE_TYPE}.yaml --acl public-read + aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.json s3://${S3_BUCKET}/${STABLE_UPLOAD_DIR}/${TEMPLATE_TYPE}.json --acl public-read elif is_release; then # Provide all semver version options (major, major.minor, major.minor.hotfix) - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml s3://${S3_BUCKET}/${RELEASE_UPLOAD_DIR_MAJOR}/${TEMPLATE_TYPE}.yaml --acl public-read - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml s3://${S3_BUCKET}/${RELEASE_UPLOAD_DIR_MINOR}/${TEMPLATE_TYPE}.yaml --acl public-read - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml s3://${S3_BUCKET}/${RELEASE_UPLOAD_DIR_HOTFIX}/${TEMPLATE_TYPE}.yaml --acl public-read + aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.json s3://${S3_BUCKET}/${RELEASE_UPLOAD_DIR_MAJOR}/${TEMPLATE_TYPE}.json --acl public-read + aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.json s3://${S3_BUCKET}/${RELEASE_UPLOAD_DIR_MINOR}/${TEMPLATE_TYPE}.json --acl public-read + aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.json s3://${S3_BUCKET}/${RELEASE_UPLOAD_DIR_HOTFIX}/${TEMPLATE_TYPE}.json --acl public-read else # upload templates for every branch - aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml s3://${S3_BUCKET}/${BRANCH_UPLOAD_DIR}/${TEMPLATE_TYPE}.yaml --acl public-read + aws s3 cp ${LOCAL_DIR}/${TEMPLATE_TYPE}.json s3://${S3_BUCKET}/${BRANCH_UPLOAD_DIR}/${TEMPLATE_TYPE}.json --acl public-read fi From 6547c5dc99aae6b87af19f6fc26c718b8f8dcbf1 Mon Sep 17 00:00:00 2001 From: "Les Peabody (BOS-GEN)" Date: Fri, 26 Jun 2020 17:41:17 -0400 Subject: [PATCH 04/25] Alter GCP implementation. - Ensures instance has a static IP. - Allows you to configure machine size, disk size, and preemption. - Uses managed instance groups in case of the machine being preemptible, it will always attempt to bring the machine back online automatically. - Adds schemas for the resource templates, which provide reasonable default values. - Adds a Makefile to make it easy to create, update, and delete the sandbox infrastructure from you local system. --- .gitignore | 4 + gcp-deployment-manager/Docksal.jinja | 88 ----------- gcp-deployment-manager/Makefile | 127 ++++++++++++++++ gcp-deployment-manager/README.md | 138 ++++++++++++++---- gcp-deployment-manager/config.yaml | 17 --- gcp-deployment-manager/project.env | 18 +++ gcp-deployment-manager/startup.sh | 116 --------------- .../templates/docksal_disk.jinja | 12 ++ .../templates/docksal_disk.jinja.schema | 17 +++ .../templates/docksal_group.py | 35 +++++ .../templates/docksal_group.py.schema | 22 +++ .../templates/docksal_template.jinja | 56 +++++++ .../templates/docksal_template.jinja.schema | 43 ++++++ .../templates/docksal_vm.jinja | 80 ++++++++++ .../templates/docksal_vm.jinja.schema | 40 +++++ gcp-deployment-manager/templates/startup.sh | 128 ++++++++++++++++ 16 files changed, 689 insertions(+), 252 deletions(-) delete mode 100644 gcp-deployment-manager/Docksal.jinja create mode 100644 gcp-deployment-manager/Makefile delete mode 100644 gcp-deployment-manager/config.yaml create mode 100644 gcp-deployment-manager/project.env delete mode 100755 gcp-deployment-manager/startup.sh create mode 100644 gcp-deployment-manager/templates/docksal_disk.jinja create mode 100644 gcp-deployment-manager/templates/docksal_disk.jinja.schema create mode 100644 gcp-deployment-manager/templates/docksal_group.py create mode 100644 gcp-deployment-manager/templates/docksal_group.py.schema create mode 100644 gcp-deployment-manager/templates/docksal_template.jinja create mode 100644 gcp-deployment-manager/templates/docksal_template.jinja.schema create mode 100644 gcp-deployment-manager/templates/docksal_vm.jinja create mode 100644 gcp-deployment-manager/templates/docksal_vm.jinja.schema create mode 100755 gcp-deployment-manager/templates/startup.sh diff --git a/.gitignore b/.gitignore index e341201..4d029f1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ */build/ +.vscode + +# GCP ignores. +gcp-deployment-manager/local-override.env diff --git a/gcp-deployment-manager/Docksal.jinja b/gcp-deployment-manager/Docksal.jinja deleted file mode 100644 index 7b6ee8e..0000000 --- a/gcp-deployment-manager/Docksal.jinja +++ /dev/null @@ -1,88 +0,0 @@ -{# - -Docksal Sandbox Server template - -#} - -{% set project = env["project"] %} -{% set deployment = env["deployment"] %} -{% set instanceName = "%s-vm" % deployment %} -{% set zone = properties["zone"] %} -{% set machineType = properties["machineType"] %} -{% set dataDisk = "%s-data" % deployment %} -{% set dataDiskSizeGb = properties["dataDiskSizeGb"] %} - -resources: - -# Firewall -resources: -- name: {{ deployment }}-firewall - type: compute.v1.firewall - properties: - network: https://www.googleapis.com/compute/v1/projects/{{ project }}/global/networks/default - sourceRanges: ["0.0.0.0/0"] - allowed: - - IPProtocol: TCP - ports: - - "80" - - "443" - - "22" - -# Data disk -- type: compute.v1.disk - name: {{ dataDisk }} - properties: - zone: {{ zone }} - sizeGb: {{ dataDiskSizeGb }} - # Disk type is a full URI. Example uses pd-standard, but pd-ssd can be used as well - type: https://www.googleapis.com/compute/v1/projects/{{ project }}/zones/{{ zone }}/diskTypes/pd-standard - -# VM instance -- type: compute.v1.instance - name: {{ instanceName }} - properties: - zone: {{ zone }} - machineType: https://www.googleapis.com/compute/v1/projects/{{ project }}/zones/{{ zone }}/machineTypes/{{ machineType }} - - disks: - - deviceName: {{ dataDisk }}-boot - type: PERSISTENT - boot: true - autoDelete: true - initializeParams: - sourceImage: https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-minimal-1804-bionic-v20191113 - - deviceName: {{ dataDisk }}-data - type: PERSISTENT - source: $(ref.{{ dataDisk }}.selfLink) - autoDelete: true - - networkInterfaces: - - network: https://www.googleapis.com/compute/v1/projects/{{ project }}/global/networks/default - # Access Config required to give the instance a public IP address - accessConfigs: - - name: External NAT - type: ONE_TO_ONE_NAT - - metadata: - items: - {% for key, value in properties['metadata-from-file'].iteritems() %} - - key: {{ key }} - value: | - {{ imports[value]|indent(10) }} - {% endfor %} - -outputs: - - name: deployment - value: {{ deployment }} - - name: project - value: {{ project }} - - name: vmId - value: $(ref.{{ instanceName }}.id) - - name: vmExternalIP - value: $(ref.{{ instanceName }}.networkInterfaces[0].accessConfigs[0].natIP) - - name: vmInternalIP - value: $(ref.{{ instanceName }}.networkInterfaces[0].networkIP) - - name: vmName - value: {{ instanceName }} - - name: vmSelfLink - value: $(ref.{{ instanceName }}.selfLink) diff --git a/gcp-deployment-manager/Makefile b/gcp-deployment-manager/Makefile new file mode 100644 index 0000000..477cc9e --- /dev/null +++ b/gcp-deployment-manager/Makefile @@ -0,0 +1,127 @@ +# Default project variables. +include project.env +# Project-specific overrides. These should be version controlled. +PROJECT_OVERRIDES:=project-override.env +# Untracked local overrides file, for development purposes. +LOCAL_OVERRIDES:=local-override.env +DOCKER_ENV_FILE:=--env-file=project.env + +ifneq ("$(wildcard $(PROJECT_OVERRIDES))","") + include $(PROJECT_OVERRIDES) + DOCKER_ENV_FILE+=--env-file=$(PROJECT_OVERRIDES) +endif + +ifneq ("$(wildcard $(LOCAL_OVERRIDES))","") + include $(LOCAL_OVERRIDES) + DOCKER_ENV_FILE+=--env-file=$(LOCAL_OVERRIDES) +endif + +export + +login: + docker run -ti --name gcloud-config google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) gcloud auth login + +guard-%: + @ if [ "${${*}}" = "" ]; then \ + echo "Environment variable $* not set"; \ + exit 1; \ + fi + +bash: + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + -w /opt/deployment \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) /bin/bash + +clean-container: + docker rm -v gcloud-config + +create: + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud deployment-manager deployments create $(DISK_DEPLOYMENT) \ + --template=/opt/deployment/templates/docksal_disk.jinja \ + --properties="diskSizeGb:$(DISK_SIZE),zone:'$(ZONE)'" + + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud deployment-manager deployments create $(TEMPLATE_DEPLOYMENT) \ + --template=/opt/deployment/templates/docksal_template.jinja \ + --properties="templateId:'$(TEMPLATE_ID)',region:'$(REGION)',machineType:'$(DEFAULT_MACHINE_SIZE)',preemptive:$(PREEMPTIVE)" + + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud deployment-manager deployments create $(GROUP_DEPLOYMENT) \ + --template=/opt/deployment/templates/docksal_group.py \ + --properties="templateId:'$(TEMPLATE_ID)',zone:'$(ZONE)'" + +update: + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud deployment-manager deployments update $(DISK_DEPLOYMENT) \ + --template=/opt/deployment/templates/docksal_disk.jinja \ + --properties="diskSizeGb:$(DISK_SIZE)" + + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud deployment-manager deployments update $(TEMPLATE_DEPLOYMENT) \ + --template=/opt/deployment/templates/docksal_template.jinja \ + --properties="templateId:'$(TEMPLATE_ID)',region:'$(REGION)',machineType:'$(DEFAULT_MACHINE_SIZE)',preemptive:$(PREEMPTIVE)" \ + --delete-policy=abandon + + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud deployment-manager deployments update $(GROUP_DEPLOYMENT) \ + --template=/opt/deployment/templates/docksal_group.py \ + --properties="templateId:'$(TEMPLATE_ID)',zone:'$(ZONE)'" + + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud compute instance-groups managed rolling-action start-update $(GROUP_NAME) \ + --version template=$(TEMPLATE_ID) \ + --zone $(ZONE) + +delete: + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud deployment-manager deployments delete $(GROUP_DEPLOYMENT) --quiet + + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud deployment-manager deployments delete $(TEMPLATE_DEPLOYMENT) --quiet + + docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud deployment-manager deployments delete $(DISK_DEPLOYMENT) --quiet diff --git a/gcp-deployment-manager/README.md b/gcp-deployment-manager/README.md index cc92621..b6f6616 100644 --- a/gcp-deployment-manager/README.md +++ b/gcp-deployment-manager/README.md @@ -2,10 +2,10 @@ This is a Docksal Sandbox Server template for Google Cloud Deployment Manager. -Google Cloud Deployment Manager is an infrastructure management service that makes it simple to create, deploy, -and manage Google Cloud Platform resources. With Deployment Manager, you can create a static or dynamic template -that describes the configuration of your Google Cloud environment and then use Deployment Manager to create these -resources as a single deployment. +Google Cloud Deployment Manager is an infrastructure management service that makes it simple to create, deploy, +and manage Google Cloud Platform resources. With Deployment Manager, you can create a static or dynamic template +that describes the configuration of your Google Cloud environment and then use Deployment Manager to create these +resources as a single or collection of deployments. For an overview of Deployment Manager, see https://cloud.google.com/deployment-manager/docs. @@ -17,20 +17,13 @@ For an overview of Deployment Manager, see https://cloud.google.com/deployment-m 1. [Enable the Deployment Manager and Compute APIs](https://console.cloud.google.com/flows/enableapi?apiid=deploymentmanager,compute_component). -1. Clone this repo +1. Setup SSH keys. - Clone this repository locally or with [Cloud Shell](https://cloud.google.com/shell/). - With Cloud Shell, you can manage your GCP project and resources without installing anything. +1. Proceed with Local setup below. - [![Open in Cloud Shell](http://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/open?git_repo=https%3A%2F%2Fgithub.com%2Fdocksal%2Fsandbox-server&page=editor) +1. Deploy the sandbox. -1. Configure `gcloud` tool to use your project. Replace `` with the project ID - - ``` - gcloud config set project - ``` - -## Set up SSH keys in the GCP project +### Set up SSH keys in the GCP project Generate a new SSH key pair: @@ -40,38 +33,121 @@ Replace `` with something meaningful to identify the key, e.g. `docksal **Note**: The `-C build-agent@docksal-sandbox` part is important. -GCP uses the comment in the key to map the key to a Linux user. It will update the `build-agent` user's +GCP uses the comment in the key to map the key to a Linux user. It will update the `build-agent` user's `~/.ssh/authorized_keys` automatically, when you follow the steps below. View and copy the public key: cat ~/.ssh/.pub - **Note**: they key is a single line. If using Cloud Shell, it may break it into multiple lines. If that is the case, you will have yo manually fix the string to be one line. - Use the copied string to set a [project-wide](https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys#project-wide) public SSH key on GCP. -## Deploy the sandbox server +### Local Setup + +You must have Docker installed. Local setup uses the image `google/cloud-sdk`. See the [image's Docker Hub page](https://hub.docker.com/r/google/cloud-sdk) for additional details on using this image. + +1. Run `make login`. This will ask you to copy/paste a URL into your browser and copy/paste a token. See [`gcloud auth login`](https://cloud.google.com/sdk/gcloud/reference/auth/login) for additional details. + +1. Run `make bash` for container CLI and run `gcloud config set project `. Future `make` commands will automatically run with the context of your project. + +### Deploying the sandbox + +After deployment you will have the following resources requisitioned (resources you pay for denoted by ($)): + +- VM instance (defaults to preemptible) ($) +- Standard disk (150GB default) ($) +- Static IP address ($) +- Managed instance group +- VM instance template + +Assuming you have already logged in and set the project as directed above, you simply need to run the following to setup your sandbox: + + make create + +Then, in your GCP Console: + +1. navigate to Compute Engine > VM instances. +1. Find the IP address assigned to your VM. +1. Log into your VM via `build-agent@x.x.x.x`, where x.x.x.x is the IP identified in step 2. +1. Run `fin system status`. If the Docksal system containers all appear as `Up` then everything is ready to go. +1. If the Docksal system status is either not returned, or you get an error of some kind, you will want to: log out, wait a few minutes, then log back in. Run `fin system status` again and it should come back as up. + +**NOTE**: You will want to wait ~5 minutes after running `make create` to ensure the VM has had time to fully assemble itself. This is only true for first time setups. + +## Managing user access to the machine + +You can associate multiple SSH keys with the project to control user access, see [working with SSH keys](https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys) for details on how to add and remove SSH keys from the project. + +Project SSH keys, and their associated user, are automatically added to all VM instances in your project. If you add an SSH key to the project metadata with user id `user-a`, then after the machine has been provisioned, or restarted, that user will be able to login to the machine via `ssh user-a@x.x.x.x` where `x.x.x.x` is the sandbox VM's IP address. + +### Running commands as `build-agent` + +Sandboxes are stored in `/home/build-agent/builds`. Therefore, if you need to interact with a sandbox, you must be the user `build-agent`. All users with project-level SSH keys have passwordless sudo access. So, if you are logging into the machine as a user besides `build-agent`, to interact with the builds you must do the following: + +1. Run `ssh my-user-id@x.x.x.x`. +1. Run `sudo -s` to assume root. +1. Run `su - build-agent`. + +## Customizing VM instance properties + +Reasonable defaults are assigned to the sandbox server out of the box: + +| Property | Value | +|--------------|---------------| +| Machine size | n1-standard-2 | +| Disk size | 150GB | +| Preemptible | true | +| Region | us-east4 | +| Zone | us-east4-c | + +These properties are represented in `project.env`. You may override these values in one of two files: `project-override.env` and `local-override.env`. `project-override.env` is intended to be a version controlled file. `local-override.env` is, conversely, not version controlled. The files are included in the `Makefile` in the following order: + +1. project.env +1. project-override.env +1. local-override.env + +This project supports the customization of the following variables: + +1. `ZONE` - See [Regions and Zones](https://cloud.google.com/compute/docs/regions-zones) for acceptable values. +1. `REGION` - See [Regions and Zones](https://cloud.google.com/compute/docs/regions-zones) for acceptable values. +1. `DEFAULT_MACHINE_SIZE` - See [Machine Types](https://cloud.google.com/compute/docs/machine-types) for the various types of machines you can use. We have found that the `n1-standard` series provides optimum computing power for sandbox purposes. +1. `DISK_SIZE` - the size of the machine in GB (integer only). +1. `TEMPLATE_ID` - unique identifier for a template. If any other properties of the template changes, you MUST change the template ID as well. You will most frequently need to change this if you decide to toggle the instance's preemptive setting. +1. `PREEMPTIVE` - (boolean) true or false. Determines whether or not a machine should be preemptive. Preemptive machines are far cheaper, and the uptime is usually adequate enough as to not provide disruptions to your daily operations. See [Preemptible VM instances](https://cloud.google.com/compute/docs/instances/preemptible) for more information. + +**NOTE:** Never surround the values of overridden variables with quotes. Reference `project.env` for the proper way to format values. + +### Examples + +Lets say you need a bigger disk and a bigger machine than comes standard. You'll want: + +1. Add `project-override.env` and set its contents to: + + DEFAULT_MACHINE_SIZE=n1-standard-4 + DISK_SIZE=250 -Navigate to the `gcp-deployment-manager` folder: +1. Run `make create`. That will create a sandbox server with machine type `n1-standard-4` instead of `n1-standard-2` and a disk with size `250GB` instead of `150GB`. - cd gcp-deployment-manager +If you have already created the sandbox instance and you need to upsize the machine because you are running more active sandboxes, you should: -Launch the deployment: +1. Update/create `project-override.env` to include: - gcloud deployment-manager deployments create docksal-sandbox-server --config config.yaml + - A bigger machine size. + - A DIFFERENT value for TEMPLATE_ID. This is because the machine size is a property of the instance template, and GCP does not allow updating properties of a template. A NEW template must be created with the new values. The `Makefile` handles this for you automatically, but you DO need to set a `TEMPLATE_ID` value that is different from what was last deployed. -In the output of the command you'll find the server public IP address: + Note that, any updates to the following variables requires you also update the current value of `TEMPLATE_ID` to something different: - OUTPUTS VALUE - ip + - ZONE + - PREEMPTIVE + - DEFAULT_MACHINE_SIZE -There is a startup script that will do server provisioning. It will take 2-5 minutes from this point. +1. With the above in mind, lets say the current machine type is `n1-standard-4`, and you want to upsize it to `n1-standard-8` to get more processors and memory. Update `project-override.env` to look like: -You can now proceed to [Access the sandbox server](/README.md#server-access) + DEFAULT_MACHINE_SIZE=n1-standard-8 + TEMPLATE_ID=custom-docksal-sandbox-template-002 -## Delete the sandbox server +1. Run `make update`. If you forget to change `TEMPLATE_ID` to a new, unique, value, then `make update` will error. -To delete the deployment and all the resources that were created: +1. Observe the GCP Console and verify that your machine size has been updated. If there is a sandbox site present on the server, visit its sandbox URL to verify that the site is still functioning. - gcloud deployment-manager deployments delete docksal-sandbox-server +The machine instance is intended to be stateless, but the disk is stateful. So, you can recreate the machine instance as many times as you'd like, but it will keep using the same data disk which preserves all of the existing sandboxes and settings in the `build-agent` home directory. diff --git a/gcp-deployment-manager/config.yaml b/gcp-deployment-manager/config.yaml deleted file mode 100644 index 02506c7..0000000 --- a/gcp-deployment-manager/config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -imports: -- path: Docksal.jinja -- path: startup.sh - -resources: -- name: Docksal - type: Docksal.jinja - properties: - zone: us-east4-c - machineType: n1-standard-2 - dataDiskSizeGb: 100 - metadata-from-file: - startup-script: startup.sh - -outputs: -- name: ip - value: $(ref.Docksal.vmExternalIP) diff --git a/gcp-deployment-manager/project.env b/gcp-deployment-manager/project.env new file mode 100644 index 0000000..85d8950 --- /dev/null +++ b/gcp-deployment-manager/project.env @@ -0,0 +1,18 @@ +# Project constants. DO NOT OVERRIDE. +CLOUD_SDK_IMAGE_VERSION="298.0.0-alpine" +DISK_DEPLOYMENT=docksal-sandbox-disk +GROUP_DEPLOYMENT=docksal-sandbox-group +GROUP_NAME=mig-docksal-group-py +TEMPLATE_DEPLOYMENT=docksal-sandbox-template + +# Overridable properties. +# NOTE: If changing PREEMPTIVE, ZONE, or DEFAULT_MACHINE_SIZE then you MUST +# also change TEMPLATE_ID to be something different as well. We recommend +# chaning it to e.g. custom-docksal-sandbox-template-002. Always increment the +# last number as a matter of convention. +ZONE=us-east4-c +REGION=us-east4 +DEFAULT_MACHINE_SIZE=n1-standard-2 +DISK_SIZE=150 +TEMPLATE_ID=docksal-sandbox-template-001 +PREEMPTIVE=true diff --git a/gcp-deployment-manager/startup.sh b/gcp-deployment-manager/startup.sh deleted file mode 100755 index 4459a08..0000000 --- a/gcp-deployment-manager/startup.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -# This is a startup script for a Docksal Sandbox server in GCP. -# It installs and configures Docksal on a bare Ubuntu machine (tested with Ubuntu 18.04 Minimal). -# -# The startup script log can be views via "gcloud compute ssh vm-sandbox-test -- tail -f /var/log/syslog" - -set -x # Print commands -set -e # Fail on errors - -# Persistent disk settings -DATA_DISK="/dev/sdb" -MOUNT_POINT="/data" -BUILD_USER="build-agent" -BUILD_USER_UID="1100" -BUILD_USER_HOME="/home/${BUILD_USER}" -DATA_BUILD_USER_HOME="${MOUNT_POINT}${BUILD_USER_HOME}" -DOCKSAL_VERSION="master" -PROJECT_INACTIVITY_TIMEOUT="0.5h" -PROJECT_DANGLING_TIMEOUT="168h" -PROJECTS_ROOT="${BUILD_USER_HOME}/builds" - -# Create build-agent user with no-password sudo access -# Forcing the uid to avoid race conditions with GCP creating project level users at the same time. -# (Otherwise, we may run into something like "useradd: UID 1001 is not unique") -if [[ "$(id -u ${BUILD_USER})" != "${BUILD_USER_UID}" ]]; then - adduser --disabled-password --gecos "" --uid ${BUILD_USER_UID} ${BUILD_USER} - usermod -aG sudo ${BUILD_USER} - echo "${BUILD_USER} ALL=(ALL) NOPASSWD:ALL" >/etc/sudoers.d/101-${BUILD_USER} -fi - -# Mount the persistent data disk if it was attached -if lsblk ${DATA_DISK} &>/dev/null; then - echo "Using persistent disk: ${DATA_DISK} for data storage: ${MOUNT_POINT}" - - # Format the disk if necessary - if [[ $(lsblk -f ${DATA_DISK}) != *ext4* ]]; then - mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard ${DATA_DISK} - fi - - # Mount the data disk - mkdir -p ${MOUNT_POINT} - cp /etc/fstab /etc/fstab.backup - # Write disk mount to /etc/fstab (so that it persists on reboots) - # Equivalent of `mount /dev/sdb /mnt/data` - echo "${DATA_DISK} ${MOUNT_POINT} ext4 defaults,nofail 0 2" | tee -a /etc/fstab - mount -a - - # Move BUILD_USER_HOME to the data disk - # E.g. /home/build-agent => /mnt/data/home/build-agent - if [[ ! -d ${DATA_BUILD_USER_HOME} ]]; then - mkdir -p $(dirname ${DATA_BUILD_USER_HOME}) - mv ${BUILD_USER_HOME} $(dirname ${DATA_BUILD_USER_HOME}) - else - rm -rf ${BUILD_USER_HOME} - fi - ln -s ${DATA_BUILD_USER_HOME} ${BUILD_USER_HOME} - - # Symlink /var/lib/docker (should not yet exist when this script runs) to the data volume - mkdir -p ${MOUNT_POINT}/var/lib/docker - ln -s ${MOUNT_POINT}/var/lib/docker /var/lib/docker -else - echo "WARNING: data volume not found. Using instance-only storage" -fi - -# Create the projects/builds directory -mkdir -p ${PROJECTS_ROOT} - -# SSH settings: ensure ~/.ssh exists for the build user -mkdir -p ${BUILD_USER_HOME}/.ssh - -# SSH settings: authorized_keys -# If ~/.ssh/authorized_keys does not exist for the build user, reuse the one from the default user account (ubuntu) -if [[ ! -f "${BUILD_USER_HOME}/.ssh/authorized_keys" ]]; then - cp "/home/ubuntu/.ssh/authorized_keys" "${BUILD_USER_HOME}/.ssh/authorized_keys" - chown ${BUILD_USER}:${BUILD_USER} "${BUILD_USER_HOME}/.ssh/authorized_keys" -fi - -# SSH settings: disable the host key check -if [[ ! -f "${BUILD_USER_HOME}/.ssh/config" ]]; then - tee "${BUILD_USER_HOME}/.ssh/config" < + description: | + Disk deployment meant to be used with Docksal sandbox infrastructure. + +additionalProperties: false + +properties: + zone: + type: string + description: The zone which this resource resides. + default: us-east4-c + diskSizeGb: + type: integer + description: The size of the disk, in gigabytes (GB). + default: 150 diff --git a/gcp-deployment-manager/templates/docksal_group.py b/gcp-deployment-manager/templates/docksal_group.py new file mode 100644 index 0000000..5f2cd97 --- /dev/null +++ b/gcp-deployment-manager/templates/docksal_group.py @@ -0,0 +1,35 @@ +def generate_config(context): + properties = context.properties + resource_name = 'mig-' + context.env['name'] + project = context.env['project'] + zone = properties['zone'] + template_id = properties['templateId'] + instance_template = 'projects/' + project + '/global/instanceTemplates/' + template_id + + outputs = [] + resources = [{ + 'name': resource_name, + 'type': 'gcp-types/compute-v1:instanceGroupManagers', + 'properties': { + 'instanceTemplate': instance_template, + 'name': resource_name, + 'zone': zone, + 'targetSize': 1, + 'updatePolicy': { + 'maxSurge': { + 'calculated': 0, + 'fixed': 0 + }, + 'maxUnavailable': { + 'calculated': 1, + 'fixed': 1 + }, + 'minReadySec': 0, + 'minimalAction': 'REPLACE', + 'replacementMethod': 'RECREATE', + 'type': 'PROACTIVE' + } + } + }] + + return {'resources': resources, 'outputs': outputs} diff --git a/gcp-deployment-manager/templates/docksal_group.py.schema b/gcp-deployment-manager/templates/docksal_group.py.schema new file mode 100644 index 0000000..49b3ac9 --- /dev/null +++ b/gcp-deployment-manager/templates/docksal_group.py.schema @@ -0,0 +1,22 @@ +info: + title: Docksal Managed Instance Group + author: Les Peabody + description: | + Created a MIG meant to control a singular, preemptible VM. + +additionalProperties: false + +required: + - templateId + +properties: + zone: + type: string + description: The zone which this MIG occupies. + default: us-east4-c + templateId: + type: string + description: | + The current instance template resource name that should be used to + generate the sandbox virtual machine. + default: docksal-sandbox-template-001 diff --git a/gcp-deployment-manager/templates/docksal_template.jinja b/gcp-deployment-manager/templates/docksal_template.jinja new file mode 100644 index 0000000..6e1cb89 --- /dev/null +++ b/gcp-deployment-manager/templates/docksal_template.jinja @@ -0,0 +1,56 @@ +{% set staticIpName = "static-ip-%s" % env["deployment"] %} +{% set project = env["project"] %} +{% set region = properties["region"] %} +{% set templateId = properties["templateId"] %} +{% set deployment = env["deployment"] %} +{% set preemptive = properties["preemptive"] %} + +resources: +- name: {{ staticIpName }} + properties: + addressType: EXTERNAL + description: | + The static IP address used for VMs in the deployment + {{ deployment }}. + region: {{ region }} + resourceType: addresses + type: gcp-types/compute-v1:addresses +- name: {{ templateId }} + properties: + name: {{ templateId }} + properties: + machineType: {{ properties["machineType"] }} + disks: + - deviceName: boot + boot: true + autoDelete: true + initializeParams: + sourceImage: projects/ubuntu-os-cloud/global/images/ubuntu-1804-bionic-v20200610 + - deviceName: docksal-sandbox-disk-data + source: docksal-sandbox-disk + {% if properties["startup"] or properties["docksalDnsDomain"] %} + metadata: + items: + {% if properties["startup"] %} + - key: startup-script + value: | + {{ imports[properties["startup"]]|indent(12) }} + {% endif %} + {% if properties["docksalDnsDomain"] %} + - key: DOCKSAL_DNS_DOMAIN + value: {{ properties["docksalDnsDomain"] }} + {% endif %} + {% endif %} + networkInterfaces: + - network: https://www.googleapis.com/compute/v1/projects/{{ project }}/global/networks/default + accessConfigs: + - type: ONE_TO_ONE_NAT + name: "External NAT" + natIP: $(ref.{{ staticIpName }}.address) + scheduling: + preemptible: {{ properties["preemptive"] }} + tags: + items: + - http-server + - https-server + type: gcp-types/compute-v1:instanceTemplates diff --git a/gcp-deployment-manager/templates/docksal_template.jinja.schema b/gcp-deployment-manager/templates/docksal_template.jinja.schema new file mode 100644 index 0000000..fbf9639 --- /dev/null +++ b/gcp-deployment-manager/templates/docksal_template.jinja.schema @@ -0,0 +1,43 @@ +info: + title: Docksal Instance Template + author: Les Peabody + description: | + Creates a Docksal VM instance template and static IP resource. + +additionalProperties: false + +imports: +- path: startup.sh + name: startup.sh + +properties: + region: + type: string + description: The region which this resource resides. + default: us-east4 + machineType: + type: string + description: | + The type of machine instance to use. Defaults to n1-standard-2. + default: n1-standard-2 + startup: + type: string + description: | + The name of the script processed every time the server starts up. + default: startup.sh + docksalDnsDomain: + type: string + description: | + A custom domain that points at the sandbox server's IP address. + dataDiskName: + type: string + description: The name of the disk containing instance data. + default: docksal-sandbox-disk + templateId: + type: string + description: The resource name attached to the instance template. + default: docksal-sandbox-template-001 + preemptive: + type: boolean + description: Whether or not the machine should be preemptible. Preemptive instances are cheaper overall. + default: true diff --git a/gcp-deployment-manager/templates/docksal_vm.jinja b/gcp-deployment-manager/templates/docksal_vm.jinja new file mode 100644 index 0000000..08b417b --- /dev/null +++ b/gcp-deployment-manager/templates/docksal_vm.jinja @@ -0,0 +1,80 @@ +{# + +Docksal VM + +Note: Ensure that the docksal-sandbox-disk deployment has already run or this +will fail. + +#} + +{% set project = env["project"] %} +{% set deployment = env["deployment"] %} +{% set instanceName = "%s-vm" % deployment %} +{% set zone = properties["zone"] %} +{% set region = properties["region"] %} +{% set machineType = properties["machineType"] %} +{% set staticIpName = "static-ip-%s" % env["deployment"] %} +{% set dataDiskName = properties["dataDiskName"] %} +{% set bootDiskId = "%s-boot" % deployment %} + +resources: + +# Static IP +- name: {{ staticIpName }} + properties: + addressType: EXTERNAL + description: | + The static IP address used for VMs in the deployment + {{ deployment }}. + region: {{ region }} + resourceType: addresses + type: gcp-types/compute-v1:addresses + +# VM instance +- type: compute.v1.instance + name: {{ instanceName }} + properties: + zone: {{ zone }} + machineType: https://www.googleapis.com/compute/v1/projects/{{ project }}/zones/{{ zone }}/machineTypes/{{ machineType }} + tags: + items: + - http-server + - https-server + + disks: + - deviceName: {{ bootDiskId }} + type: PERSISTENT + boot: true + autoDelete: true + initializeParams: + sourceImage: projects/ubuntu-os-cloud/global/images/ubuntu-minimal-1804-bionic-v20200610 + diskName: {{ bootDiskId }} + - deviceName: {{ dataDiskName }}-data + source: projects/{{ project }}/zones/{{ zone }}/disks/{{ dataDiskName }} + + networkInterfaces: + - network: https://www.googleapis.com/compute/v1/projects/{{ project }}/global/networks/default + # Access Config required to give the instance a public IP address + accessConfigs: + - name: External IP + natIP: $(ref.{{ staticIpName }}.address) + + metadata: + items: + {% for key, value in properties['metadata-from-file'].items() %} + - key: {{ key }} + value: | + {{ imports[value]|indent(10) }} + {% endfor %} + +outputs: + - name: deployment + value: {{ deployment }} + - name: project + value: {{ project }} + - name: vmId + value: $(ref.{{ instanceName }}.id) + - name: vmName + value: {{ instanceName }} + - name: vmSelfLink + value: $(ref.{{ instanceName }}.selfLink) diff --git a/gcp-deployment-manager/templates/docksal_vm.jinja.schema b/gcp-deployment-manager/templates/docksal_vm.jinja.schema new file mode 100644 index 0000000..44d57ad --- /dev/null +++ b/gcp-deployment-manager/templates/docksal_vm.jinja.schema @@ -0,0 +1,40 @@ +info: + title: Docksal VM Instance + author: Les Peabody + description: | + Creates a Docksal VM instance and static IP resource. + +additionalProperties: false + +imports: +- path: startup.sh + name: startup.sh + +properties: + region: + type: string + description: The region which this resource resides. + default: us-east4 + zone: + type: string + description: The zone which this resource resides. + default: us-east4-c + machineType: + type: string + description: | + The type of machine instance to use. Defaults to n1-standard-2. + default: n1-standard-2 + metadata-from-file: + type: object + additionalProperties: false + default: + startup-script: startup.sh + properties: + startup-script: + type: string + description: The name of the script to be imported. + default: startup.sh + dataDiskName: + type: string + description: The name of the disk containing instance data. + default: docksal-sandbox-disk diff --git a/gcp-deployment-manager/templates/startup.sh b/gcp-deployment-manager/templates/startup.sh new file mode 100755 index 0000000..7e66a33 --- /dev/null +++ b/gcp-deployment-manager/templates/startup.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +# This is a startup script for a Docksal Sandbox server in GCP. +# It installs and configures Docksal on a bare Ubuntu machine (tested with Ubuntu 18.04 Minimal). +# +# The startup script log can be views via "gcloud compute ssh vm-sandbox-test -- tail -f /var/log/syslog" + +set -x # Print commands +set -e # Fail on errors + +# Persistent disk settings +DATA_DISK="/dev/sdb" +MOUNT_POINT="/data" +BUILD_USER="build-agent" +BUILD_USER_UID="1100" +BUILD_USER_HOME="/home/${BUILD_USER}" +DATA_BUILD_USER_HOME="${MOUNT_POINT}${BUILD_USER_HOME}" +DOCKSAL_VERSION="master" +PROJECT_INACTIVITY_TIMEOUT="0.5h" +PROJECT_DANGLING_TIMEOUT="168h" +PROJECTS_ROOT="${BUILD_USER_HOME}/builds" + +set_docksal_dns_domain() { + local base_metadata="http://metadata.google.internal/computeMetadata/v1/instance/attributes/" + local dns_attr="DOCKSAL_DNS_DOMAIN" + local flavor_header="Metadata-Flavor: Google" + local metadata=$(curl "$base_metadata" -H "$flavor_header" 2>/dev/null) + if [[ "$metadata" == *"$dns_attr"* ]] ; then + # We assume that if the attribute is present in the metadata directory then + # it has an accompanying value. + local dns_attr_value=$(curl "${base_metadata}${dns_attr}" -H "$flavor_header" 2>/dev/null) + sed -i "/^$dns_attr/d" "/home/$BUILD_USER/.docksal/docksal.env" + echo "$dns_attr=\"$dns_attr_value\"" | tee -a "/home/$BUILD_USER/.docksal/docksal.env" >/dev/null + fi +} + +apt update +apt install -y vim + +# Mount the persistent data disk if it was attached +if lsblk ${DATA_DISK} &>/dev/null; then + echo "Using persistent disk: ${DATA_DISK} for data storage: ${MOUNT_POINT}" + + # Format the disk if necessary + if [[ $(lsblk -f ${DATA_DISK}) != *ext4* ]]; then + mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard ${DATA_DISK} + fi + + # Mount the data disk + mkdir -p ${MOUNT_POINT} + cp /etc/fstab /etc/fstab.backup + # Write disk mount to /etc/fstab (so that it persists on reboots) + # Equivalent of `mount /dev/sdb /mnt/data` + echo "${DATA_DISK} ${MOUNT_POINT} ext4 defaults,nofail 0 2" | tee -a /etc/fstab + mount -a + + # Move BUILD_USER_HOME to the data disk + # E.g. /home/build-agent => /mnt/data/home/build-agent + if [[ ! -d ${DATA_BUILD_USER_HOME} ]]; then + mkdir -p $(dirname ${DATA_BUILD_USER_HOME}) + mv ${BUILD_USER_HOME} $(dirname ${DATA_BUILD_USER_HOME}) + else + rm -rf ${BUILD_USER_HOME} + fi + ln -s ${DATA_BUILD_USER_HOME} ${BUILD_USER_HOME} + + # Symlink /var/lib/docker (should not yet exist when this script runs) to the data volume + mkdir -p ${MOUNT_POINT}/var/lib/docker + ln -s ${MOUNT_POINT}/var/lib/docker /var/lib/docker +else + echo "WARNING: data volume not found. Using instance-only storage" +fi + +# Create the projects/builds directory +mkdir -p ${PROJECTS_ROOT} + +# SSH settings: ensure ~/.ssh exists for the build user +mkdir -p ${BUILD_USER_HOME}/.ssh + +# SSH settings: authorized_keys +# If ~/.ssh/authorized_keys does not exist for the build user, reuse the one from the default user account (ubuntu) +if [[ ! -f "${BUILD_USER_HOME}/.ssh/authorized_keys" ]]; then + cp "/home/ubuntu/.ssh/authorized_keys" "${BUILD_USER_HOME}/.ssh/authorized_keys" + chown ${BUILD_USER}:${BUILD_USER} "${BUILD_USER_HOME}/.ssh/authorized_keys" +fi + +# SSH settings: disable the host key check +if [[ ! -f "${BUILD_USER_HOME}/.ssh/config" ]]; then + tee "${BUILD_USER_HOME}/.ssh/config" < Date: Fri, 26 Jun 2020 18:01:28 -0400 Subject: [PATCH 05/25] No quotes. --- gcp-deployment-manager/project.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcp-deployment-manager/project.env b/gcp-deployment-manager/project.env index 85d8950..61d1f74 100644 --- a/gcp-deployment-manager/project.env +++ b/gcp-deployment-manager/project.env @@ -1,5 +1,5 @@ # Project constants. DO NOT OVERRIDE. -CLOUD_SDK_IMAGE_VERSION="298.0.0-alpine" +CLOUD_SDK_IMAGE_VERSION=298.0.0-alpine DISK_DEPLOYMENT=docksal-sandbox-disk GROUP_DEPLOYMENT=docksal-sandbox-group GROUP_NAME=mig-docksal-group-py From 6ac293ceeb665bd91f0a8b5027353baea0283912 Mon Sep 17 00:00:00 2001 From: "Les Peabody (BOS-GEN)" Date: Fri, 26 Jun 2020 18:18:56 -0400 Subject: [PATCH 06/25] Use a constant for the command. Much cleaner. --- gcp-deployment-manager/Makefile | 78 ++++++++------------------------- 1 file changed, 18 insertions(+), 60 deletions(-) diff --git a/gcp-deployment-manager/Makefile b/gcp-deployment-manager/Makefile index 477cc9e..d08eaa2 100644 --- a/gcp-deployment-manager/Makefile +++ b/gcp-deployment-manager/Makefile @@ -5,6 +5,7 @@ PROJECT_OVERRIDES:=project-override.env # Untracked local overrides file, for development purposes. LOCAL_OVERRIDES:=local-override.env DOCKER_ENV_FILE:=--env-file=project.env +# GCLOUD:=docker run -it --rm --volumes-from gcloud-config -v `pwd`:/opt/deployment $(DOCKER_ENV_FILE) google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) gcloud ifneq ("$(wildcard $(PROJECT_OVERRIDES))","") include $(PROJECT_OVERRIDES) @@ -16,6 +17,13 @@ ifneq ("$(wildcard $(LOCAL_OVERRIDES))","") DOCKER_ENV_FILE+=--env-file=$(LOCAL_OVERRIDES) endif +GCLOUD:=docker run -it --rm \ + --volumes-from gcloud-config \ + -v `pwd`:/opt/deployment \ + $(DOCKER_ENV_FILE) \ + google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ + gcloud + export login: @@ -39,89 +47,39 @@ clean-container: docker rm -v gcloud-config create: - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud deployment-manager deployments create $(DISK_DEPLOYMENT) \ + $(GCLOUD) deployment-manager deployments create $(DISK_DEPLOYMENT) \ --template=/opt/deployment/templates/docksal_disk.jinja \ --properties="diskSizeGb:$(DISK_SIZE),zone:'$(ZONE)'" - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud deployment-manager deployments create $(TEMPLATE_DEPLOYMENT) \ + $(GCLOUD) deployment-manager deployments create $(TEMPLATE_DEPLOYMENT) \ --template=/opt/deployment/templates/docksal_template.jinja \ --properties="templateId:'$(TEMPLATE_ID)',region:'$(REGION)',machineType:'$(DEFAULT_MACHINE_SIZE)',preemptive:$(PREEMPTIVE)" - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud deployment-manager deployments create $(GROUP_DEPLOYMENT) \ + $(GCLOUD) deployment-manager deployments create $(GROUP_DEPLOYMENT) \ --template=/opt/deployment/templates/docksal_group.py \ --properties="templateId:'$(TEMPLATE_ID)',zone:'$(ZONE)'" update: - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud deployment-manager deployments update $(DISK_DEPLOYMENT) \ + $(GCLOUD) deployment-manager deployments update $(DISK_DEPLOYMENT) \ --template=/opt/deployment/templates/docksal_disk.jinja \ --properties="diskSizeGb:$(DISK_SIZE)" - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud deployment-manager deployments update $(TEMPLATE_DEPLOYMENT) \ + $(GCLOUD) deployment-manager deployments update $(TEMPLATE_DEPLOYMENT) \ --template=/opt/deployment/templates/docksal_template.jinja \ --properties="templateId:'$(TEMPLATE_ID)',region:'$(REGION)',machineType:'$(DEFAULT_MACHINE_SIZE)',preemptive:$(PREEMPTIVE)" \ --delete-policy=abandon - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud deployment-manager deployments update $(GROUP_DEPLOYMENT) \ + $(GCLOUD) deployment-manager deployments update $(GROUP_DEPLOYMENT) \ --template=/opt/deployment/templates/docksal_group.py \ --properties="templateId:'$(TEMPLATE_ID)',zone:'$(ZONE)'" - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud compute instance-groups managed rolling-action start-update $(GROUP_NAME) \ + $(GCLOUD) compute instance-groups managed rolling-action start-update $(GROUP_NAME) \ --version template=$(TEMPLATE_ID) \ --zone $(ZONE) delete: - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud deployment-manager deployments delete $(GROUP_DEPLOYMENT) --quiet + $(GCLOUD) deployment-manager deployments delete $(GROUP_DEPLOYMENT) --quiet - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud deployment-manager deployments delete $(TEMPLATE_DEPLOYMENT) --quiet + $(GCLOUD) deployment-manager deployments delete $(TEMPLATE_DEPLOYMENT) --quiet - docker run -it --rm \ - --volumes-from gcloud-config \ - -v `pwd`:/opt/deployment \ - $(DOCKER_ENV_FILE) \ - google/cloud-sdk:$(CLOUD_SDK_IMAGE_VERSION) \ - gcloud deployment-manager deployments delete $(DISK_DEPLOYMENT) --quiet + $(GCLOUD) deployment-manager deployments delete $(DISK_DEPLOYMENT) --quiet From a7e6c76967c0044f517c0d22c1f849e1222a06fc Mon Sep 17 00:00:00 2001 From: "Les Peabody (BOS-GEN)" Date: Fri, 26 Jun 2020 18:19:06 -0400 Subject: [PATCH 07/25] This can be a one-liner. --- gcp-deployment-manager/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gcp-deployment-manager/README.md b/gcp-deployment-manager/README.md index b6f6616..d27d58c 100644 --- a/gcp-deployment-manager/README.md +++ b/gcp-deployment-manager/README.md @@ -85,8 +85,7 @@ Project SSH keys, and their associated user, are automatically added to all VM i Sandboxes are stored in `/home/build-agent/builds`. Therefore, if you need to interact with a sandbox, you must be the user `build-agent`. All users with project-level SSH keys have passwordless sudo access. So, if you are logging into the machine as a user besides `build-agent`, to interact with the builds you must do the following: 1. Run `ssh my-user-id@x.x.x.x`. -1. Run `sudo -s` to assume root. -1. Run `su - build-agent`. +1. Run `sudo su - build-agent` to assume root. ## Customizing VM instance properties From fcd30c052bb55a57fa94e1955ba828195567fad0 Mon Sep 17 00:00:00 2001 From: "Les Peabody (BOS-GEN)" Date: Fri, 26 Jun 2020 18:32:22 -0400 Subject: [PATCH 08/25] Align server with latest ubuntu version. --- gcp-deployment-manager/templates/docksal_template.jinja | 2 +- gcp-deployment-manager/templates/startup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gcp-deployment-manager/templates/docksal_template.jinja b/gcp-deployment-manager/templates/docksal_template.jinja index 6e1cb89..51d6c2a 100644 --- a/gcp-deployment-manager/templates/docksal_template.jinja +++ b/gcp-deployment-manager/templates/docksal_template.jinja @@ -25,7 +25,7 @@ resources: boot: true autoDelete: true initializeParams: - sourceImage: projects/ubuntu-os-cloud/global/images/ubuntu-1804-bionic-v20200610 + sourceImage: https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-minimal-2004-focal-v20200610 - deviceName: docksal-sandbox-disk-data source: docksal-sandbox-disk {% if properties["startup"] or properties["docksalDnsDomain"] %} diff --git a/gcp-deployment-manager/templates/startup.sh b/gcp-deployment-manager/templates/startup.sh index 7e66a33..411776b 100755 --- a/gcp-deployment-manager/templates/startup.sh +++ b/gcp-deployment-manager/templates/startup.sh @@ -15,7 +15,7 @@ BUILD_USER="build-agent" BUILD_USER_UID="1100" BUILD_USER_HOME="/home/${BUILD_USER}" DATA_BUILD_USER_HOME="${MOUNT_POINT}${BUILD_USER_HOME}" -DOCKSAL_VERSION="master" +DOCKSAL_VERSION="develop" PROJECT_INACTIVITY_TIMEOUT="0.5h" PROJECT_DANGLING_TIMEOUT="168h" PROJECTS_ROOT="${BUILD_USER_HOME}/builds" From 0ee454a441ef0c74e56094e5b37283ce12d3401d Mon Sep 17 00:00:00 2001 From: "Les Peabody (BOS-GEN)" Date: Fri, 26 Jun 2020 19:42:07 -0400 Subject: [PATCH 09/25] Supply docksal version as a project setting. --- gcp-deployment-manager/Makefile | 4 ++-- gcp-deployment-manager/project.env | 1 + gcp-deployment-manager/templates/docksal_template.jinja | 3 +++ .../templates/docksal_template.jinja.schema | 4 ++++ gcp-deployment-manager/templates/startup.sh | 7 ++++++- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/gcp-deployment-manager/Makefile b/gcp-deployment-manager/Makefile index d08eaa2..af6c83e 100644 --- a/gcp-deployment-manager/Makefile +++ b/gcp-deployment-manager/Makefile @@ -53,7 +53,7 @@ create: $(GCLOUD) deployment-manager deployments create $(TEMPLATE_DEPLOYMENT) \ --template=/opt/deployment/templates/docksal_template.jinja \ - --properties="templateId:'$(TEMPLATE_ID)',region:'$(REGION)',machineType:'$(DEFAULT_MACHINE_SIZE)',preemptive:$(PREEMPTIVE)" + --properties="templateId:'$(TEMPLATE_ID)',region:'$(REGION)',machineType:'$(DEFAULT_MACHINE_SIZE)',preemptive:$(PREEMPTIVE),docksalVersion:'$(DOCKSAL_VERSION)'" $(GCLOUD) deployment-manager deployments create $(GROUP_DEPLOYMENT) \ --template=/opt/deployment/templates/docksal_group.py \ @@ -66,7 +66,7 @@ update: $(GCLOUD) deployment-manager deployments update $(TEMPLATE_DEPLOYMENT) \ --template=/opt/deployment/templates/docksal_template.jinja \ - --properties="templateId:'$(TEMPLATE_ID)',region:'$(REGION)',machineType:'$(DEFAULT_MACHINE_SIZE)',preemptive:$(PREEMPTIVE)" \ + --properties="templateId:'$(TEMPLATE_ID)',region:'$(REGION)',machineType:'$(DEFAULT_MACHINE_SIZE)',preemptive:$(PREEMPTIVE),docksalVersion:'$(DOCKSAL_VERSION)'" \ --delete-policy=abandon $(GCLOUD) deployment-manager deployments update $(GROUP_DEPLOYMENT) \ diff --git a/gcp-deployment-manager/project.env b/gcp-deployment-manager/project.env index 61d1f74..90a2bb3 100644 --- a/gcp-deployment-manager/project.env +++ b/gcp-deployment-manager/project.env @@ -4,6 +4,7 @@ DISK_DEPLOYMENT=docksal-sandbox-disk GROUP_DEPLOYMENT=docksal-sandbox-group GROUP_NAME=mig-docksal-group-py TEMPLATE_DEPLOYMENT=docksal-sandbox-template +DOCKSAL_VERSION=develop # Overridable properties. # NOTE: If changing PREEMPTIVE, ZONE, or DEFAULT_MACHINE_SIZE then you MUST diff --git a/gcp-deployment-manager/templates/docksal_template.jinja b/gcp-deployment-manager/templates/docksal_template.jinja index 51d6c2a..e214532 100644 --- a/gcp-deployment-manager/templates/docksal_template.jinja +++ b/gcp-deployment-manager/templates/docksal_template.jinja @@ -4,6 +4,7 @@ {% set templateId = properties["templateId"] %} {% set deployment = env["deployment"] %} {% set preemptive = properties["preemptive"] %} +{% set docksalVersion = properties["docksalVersion"] %} resources: - name: {{ staticIpName }} @@ -31,6 +32,8 @@ resources: {% if properties["startup"] or properties["docksalDnsDomain"] %} metadata: items: + - key: docksalVersion + value: {{ properties["docksalVersion"] }} {% if properties["startup"] %} - key: startup-script value: | diff --git a/gcp-deployment-manager/templates/docksal_template.jinja.schema b/gcp-deployment-manager/templates/docksal_template.jinja.schema index fbf9639..7e9c231 100644 --- a/gcp-deployment-manager/templates/docksal_template.jinja.schema +++ b/gcp-deployment-manager/templates/docksal_template.jinja.schema @@ -41,3 +41,7 @@ properties: type: boolean description: Whether or not the machine should be preemptible. Preemptive instances are cheaper overall. default: true + docksalVersion: + type: string + description: The version of Docksal to install on the sandbox server. + default: develop diff --git a/gcp-deployment-manager/templates/startup.sh b/gcp-deployment-manager/templates/startup.sh index 411776b..ce6429b 100755 --- a/gcp-deployment-manager/templates/startup.sh +++ b/gcp-deployment-manager/templates/startup.sh @@ -8,6 +8,11 @@ set -x # Print commands set -e # Fail on errors +# Helper function for getting metadata. +get_metadata() { + curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/$1 -H "Metadata-Flavor: Google" 2>/dev/null +} + # Persistent disk settings DATA_DISK="/dev/sdb" MOUNT_POINT="/data" @@ -15,7 +20,7 @@ BUILD_USER="build-agent" BUILD_USER_UID="1100" BUILD_USER_HOME="/home/${BUILD_USER}" DATA_BUILD_USER_HOME="${MOUNT_POINT}${BUILD_USER_HOME}" -DOCKSAL_VERSION="develop" +DOCKSAL_VERSION="$(get_metadata docksalVersion)" PROJECT_INACTIVITY_TIMEOUT="0.5h" PROJECT_DANGLING_TIMEOUT="168h" PROJECTS_ROOT="${BUILD_USER_HOME}/builds" From c59f0253c80087f389047f55ea436f0cc0470ce1 Mon Sep 17 00:00:00 2001 From: "Les Peabody (BOS-GEN)" Date: Fri, 26 Jun 2020 19:43:38 -0400 Subject: [PATCH 10/25] Categorize as overridable. --- gcp-deployment-manager/project.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcp-deployment-manager/project.env b/gcp-deployment-manager/project.env index 90a2bb3..54972d6 100644 --- a/gcp-deployment-manager/project.env +++ b/gcp-deployment-manager/project.env @@ -4,7 +4,6 @@ DISK_DEPLOYMENT=docksal-sandbox-disk GROUP_DEPLOYMENT=docksal-sandbox-group GROUP_NAME=mig-docksal-group-py TEMPLATE_DEPLOYMENT=docksal-sandbox-template -DOCKSAL_VERSION=develop # Overridable properties. # NOTE: If changing PREEMPTIVE, ZONE, or DEFAULT_MACHINE_SIZE then you MUST @@ -17,3 +16,4 @@ DEFAULT_MACHINE_SIZE=n1-standard-2 DISK_SIZE=150 TEMPLATE_ID=docksal-sandbox-template-001 PREEMPTIVE=true +DOCKSAL_VERSION=develop From 2d64af62f5337b89879807a2fd42d6a501c0ac36 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 1 Jul 2020 16:22:10 -0700 Subject: [PATCH 11/25] Dropped t3.nano and t3.micro EC2 type options - t3.nano and t3.micro have less than 2GB of RAM, which is not enough to run a sandbox server anyway. - Updated parameter descriptions. --- aws-cloudformation/tpl-advanced.yaml | 26 +++++--------------------- aws-cloudformation/tpl-basic.yaml | 4 ---- 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index 9ff9114..86c4ff6 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -15,16 +15,10 @@ Parameters: - ec2 - spot InstanceType: - Description: | - Main EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM. - Instances with < 2GB RAM are not recommended and should only be used for testing the CloudFormation template. + Description: "EC2 instance type (default: t3.small = 2 vCPU, 2GB RAM)" Type: String - # Instances with < 2GB RAM should only be used for testing the CloudFormation template - # t3.small: 2 vCPU, 2GB RAM Default: t3.small AllowedValues: - - t3.nano - - t3.micro - t3.small - t3.medium - t3.large @@ -46,17 +40,11 @@ Parameters: - r5d.2xlarge InstanceType2: Description: | - Additional EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM. - Instances with < 2GB RAM are not recommended and should only be used for testing the CloudFormation template. - Will be used cheapest EC2 instance. Option applicable in spot mode only - IMPORTANT!!! Instance types InstanceType, InstanceType2 and InstanceType3 should differs, overwise stack creation/update will fail + Additional EC2 instance type (spot mode only). AWS will pick the cheapest available option for spot instance. + IMPORTANT: InstanceType, InstanceType2, and InstanceType3 must be unique, overwise stack creation/update will fail. Type: String - # Instances with < 2GB RAM should only be used for testing the CloudFormation template - # t3.small: 2 vCPU, 2GB RAM Default: t3.medium AllowedValues: - - t3.nano - - t3.micro - t3.small - t3.medium - t3.large @@ -78,13 +66,9 @@ Parameters: - r5d.2xlarge InstanceType3: Description: | - Additional EC2 instance type. Default: t3.small = 2 vCPU, 2GB RAM. - Instances with < 2GB RAM are not recommended and should only be used for testing the CloudFormation template. - Will be used cheapest EC2 instance. Option applicable in spot mode only - IMPORTANT!!! Instance types InstanceType, InstanceType2 and InstanceType3 should differs, overwise stack creation/update will fail + Additional EC2 instance type (spot mode only). AWS will pick the cheapest available option for spot instance. + IMPORTANT: InstanceType, InstanceType2, and InstanceType3 must be unique, overwise stack creation/update will fail. Type: String - # Instances with < 2GB RAM should only be used for testing the CloudFormation template - # t3.small: 2 vCPU, 2GB RAM Default: t3.large AllowedValues: - t3.nano diff --git a/aws-cloudformation/tpl-basic.yaml b/aws-cloudformation/tpl-basic.yaml index 5735e7b..15d20ba 100644 --- a/aws-cloudformation/tpl-basic.yaml +++ b/aws-cloudformation/tpl-basic.yaml @@ -5,12 +5,8 @@ Parameters: InstanceType: Description: "EC2 instance type (default: t3.small = 2 vCPU, 2GB RAM)" Type: String - # Instances with < 2GB RAM should only be used for testing the CloudFormation template - # t3.small: 2 vCPU, 2GB RAM Default: t3.small AllowedValues: - - t3.nano - - t3.micro - t3.small - t3.medium - t3.large From 42ed83740654917558e96ff07caaec0ca944839b Mon Sep 17 00:00:00 2001 From: Sergey Zabolotny Date: Thu, 16 Jul 2020 11:39:34 +0300 Subject: [PATCH 12/25] Adding the ability to select Docksal version --- aws-cloudformation/startup-advanced.sh | 3 +++ aws-cloudformation/tpl-advanced.yaml | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/aws-cloudformation/startup-advanced.sh b/aws-cloudformation/startup-advanced.sh index 9caa3bb..46845e9 100755 --- a/aws-cloudformation/startup-advanced.sh +++ b/aws-cloudformation/startup-advanced.sh @@ -159,6 +159,7 @@ do sleep 5 done +# get stack parameters export EIP=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Outputs[?OutputKey==`IPAddress`].OutputValue' --output text) export VOLUME_ID=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`ExistingDataVolume`].ParameterValue' --output text) export GITHUB_TOKEN=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`GitHubToken`].ParameterValue' --output text) @@ -167,6 +168,8 @@ export GITHUB_TEAM_SLUG=$(aws cloudformation describe-stacks --stack-name=${STAC export LETSENCRYPT_DOMAIN=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`LetsEncryptDomain`].ParameterValue' --output text) export LETSENCRYPT_CONFIG=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`LetsEncryptConfig`].ParameterValue' --output text) export ARTIFACTS_S3_BUCKET=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Outputs[?OutputKey==`ArtifactsBucket`].OutputValue' --output text) +USER_DEFINED_DOCKSAL_VERSION=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`DocksalVersion`].ParameterValue' --output text) +DOCKSAL_VERSION=${USER_DEFINED_DOCKSAL_VERSION:-${DOCKSAL_VERSION}} # attach/detach elastic ip if [[ "${EIP}" != "${ATTACHED_IP}" ]] diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index 86c4ff6..87d24d5 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -176,6 +176,11 @@ Parameters: Description: "Set name for the artifacts bucket. Leave empty to have the bucket name automatically generated." Type: String Default: "" + DocksalVersion: + Description: | + Docksal version. Leave empty for stable release + Type: String + Default: "" Metadata: AWS::CloudFormation::Interface: @@ -215,6 +220,10 @@ Metadata: Parameters: - LetsEncryptDomain - LetsEncryptConfig + - Label: + default: "Docksal settings: Optional" + Parameters: + - DocksalVersion ParameterLabels: ResourceType: default: "Resource type" @@ -256,6 +265,8 @@ Metadata: default: "Sandbox domain name" LetsEncryptConfig: default: "LetsEncrypt configuration" + DocksalVersion: + default: "Docksal version" Mappings: Region2AMI: From f6bb1ad1665bfab56f16be4d4d387f2328ff6635 Mon Sep 17 00:00:00 2001 From: Sergey Zabolotny Date: Thu, 16 Jul 2020 12:02:17 +0300 Subject: [PATCH 13/25] Recreate instance when Docksal version changed --- aws-cloudformation/tpl-advanced.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index 87d24d5..ad3141a 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -687,7 +687,7 @@ Resources: - - | # add to the end of init script echo line with template variable values, for recreate instance on every variable change - !Sub - - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig} >/dev/null + - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig}-${DocksalVersion} >/dev/null - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ] eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ] From bd34c7fd329041f09da0f0b3d190801e5cb7b572 Mon Sep 17 00:00:00 2001 From: Sergey Zabolotny Date: Thu, 16 Jul 2020 12:07:50 +0300 Subject: [PATCH 14/25] Recreate spot instance when Docksal version changed --- aws-cloudformation/tpl-advanced.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index ad3141a..3ba4323 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -652,7 +652,7 @@ Resources: - - | # add to the end of init script echo line with template variable values, for recreate instance on every variable change - !Sub - - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig} >/dev/null + - echo ${eip}-${ExistingDataVolume}-${s3name}-${GitHubToken}-${GitHubOrgName}-${GitHubTeamSlug}-${LetsEncryptDomain}-${LetsEncryptConfig}-${DocksalVersion} >/dev/null - s3name: !If [ ArtifactsEnabled, !Ref ArtifactsBucket, "" ] eip: !If [ ExistingEIPNotDefined, !Ref EIP, !Ref ExistingEIP ] From 1076cba9bdcc6eb5f75fe9b5bd663db859ea842c Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 16 Jul 2020 10:07:37 -0700 Subject: [PATCH 15/25] Update aws-cloudformation/tpl-advanced.yaml --- aws-cloudformation/tpl-advanced.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index 3ba4323..21d19d0 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -178,7 +178,7 @@ Parameters: Default: "" DocksalVersion: Description: | - Docksal version. Leave empty for stable release + Specify the version of Docksal to install (e.g., v1.14.0). Leave empty to get the latest stable version (master). Type: String Default: "" From dcc12eeafd4c6a9413408f24cc0f448cc60c632c Mon Sep 17 00:00:00 2001 From: Sergey Zabolotny Date: Fri, 17 Jul 2020 08:43:56 +0300 Subject: [PATCH 16/25] Remove unnecessary variable --- aws-cloudformation/startup-advanced.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/aws-cloudformation/startup-advanced.sh b/aws-cloudformation/startup-advanced.sh index 46845e9..480b813 100755 --- a/aws-cloudformation/startup-advanced.sh +++ b/aws-cloudformation/startup-advanced.sh @@ -33,7 +33,6 @@ MOUNT_POINT="/data" BUILD_USER="build-agent" BUILD_USER_UID="1100" BUILD_USER_HOME="/home/${BUILD_USER}" -DOCKSAL_VERSION="master" PROJECT_INACTIVITY_TIMEOUT="0.5h" PROJECT_DANGLING_TIMEOUT="168h" PROJECTS_ROOT="${BUILD_USER_HOME}/builds" @@ -168,8 +167,8 @@ export GITHUB_TEAM_SLUG=$(aws cloudformation describe-stacks --stack-name=${STAC export LETSENCRYPT_DOMAIN=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`LetsEncryptDomain`].ParameterValue' --output text) export LETSENCRYPT_CONFIG=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`LetsEncryptConfig`].ParameterValue' --output text) export ARTIFACTS_S3_BUCKET=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Outputs[?OutputKey==`ArtifactsBucket`].OutputValue' --output text) -USER_DEFINED_DOCKSAL_VERSION=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`DocksalVersion`].ParameterValue' --output text) -DOCKSAL_VERSION=${USER_DEFINED_DOCKSAL_VERSION:-${DOCKSAL_VERSION}} +export DOCKSAL_VERSION=$(aws cloudformation describe-stacks --stack-name=${STACK_ID} --query 'Stacks[*].Parameters[?ParameterKey==`DocksalVersion`].ParameterValue' --output text) +export DOCKSAL_VERSION=${DOCKSAL_VERSION:-"master"} # attach/detach elastic ip if [[ "${EIP}" != "${ATTACHED_IP}" ]] From 354aa9d76f21f2258e64ffd50ecc083f51603979 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 21 Jul 2020 08:19:16 -0700 Subject: [PATCH 17/25] AWS: Updated Instance Type labels --- aws-cloudformation/tpl-advanced.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index d156ef5..385ed8b 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -41,7 +41,7 @@ Parameters: InstanceType2: Description: | Additional EC2 instance type (spot mode only). AWS will pick the cheapest available option for spot instance. - IMPORTANT: InstanceType, InstanceType2, and InstanceType3 must be unique, overwise stack creation/update will fail. + IMPORTANT: "Instance Type"/"Instance Type 2"/"Instance Type 3" must be unique, overwise stack creation/update will fail. Type: String Default: t3.medium AllowedValues: @@ -67,7 +67,7 @@ Parameters: InstanceType3: Description: | Additional EC2 instance type (spot mode only). AWS will pick the cheapest available option for spot instance. - IMPORTANT: InstanceType, InstanceType2, and InstanceType3 must be unique, overwise stack creation/update will fail. + IMPORTANT: "Instance Type"/"Instance Type 2"/"Instance Type 3" must be unique, overwise stack creation/update will fail. Type: String Default: t3.large AllowedValues: @@ -228,11 +228,11 @@ Metadata: ResourceType: default: "Resource type" InstanceType: - default: "Instance type" + default: "Instance type (primary)" InstanceType2: - default: "Instance type" + default: "Instance type 2 (spot only)" InstanceType3: - default: "Instance type" + default: "Instance type 3 (spot only)" KeyName: default: "SSH key" ManualAZ: From 7dc8f02015dfafdf08441093a84da7c7fd352bcb Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 21 Jul 2020 08:20:02 -0700 Subject: [PATCH 18/25] AWS: Updated Launch Stack button links for v3 --- aws-cloudformation/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws-cloudformation/README.md b/aws-cloudformation/README.md index 31eef8a..7fe6cc9 100644 --- a/aws-cloudformation/README.md +++ b/aws-cloudformation/README.md @@ -36,7 +36,7 @@ If you have an existing AWS account (with billing and an SSH key pair), just cli **WARNING:** if you have an existing sandbox server created before Dec 31, 2019 (v1), **DO NOT UPGRADE**. See [v2.0.0](https://github.com/docksal/sandbox-server/releases/tag/v2.0.0) release notes. -[![Launch Basic Stack](https://cdn.rawgit.com/buildkite/cloudformation-launch-stack-button-svg/master/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=docksal-sandbox-server&templateURL=https://s3.us-east-2.amazonaws.com/docksal-aws-templates/sandbox-server/v2/basic.yaml) +[![Launch Basic Stack](https://cdn.rawgit.com/buildkite/cloudformation-launch-stack-button-svg/master/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=docksal-sandbox-server&templateURL=https://s3.us-east-2.amazonaws.com/docksal-aws-templates/sandbox-server/v3/basic.json) You will be prompted for: @@ -60,16 +60,16 @@ If you have an existing AWS account (with billing and an SSH key pair), just cli **WARNING:** if you have an existing sandbox server created before Dec 31, 2019 (v1), **DO NOT UPGRADE**. See [v2.0.0](https://github.com/docksal/sandbox-server/releases/tag/v2.0.0) release notes. -[![Launch Advanced Stack](https://cdn.rawgit.com/buildkite/cloudformation-launch-stack-button-svg/master/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=docksal-sandbox-server&templateURL=https://s3.us-east-2.amazonaws.com/docksal-aws-templates/sandbox-server/v2/advanced.yaml) +[![Launch Advanced Stack](https://cdn.rawgit.com/buildkite/cloudformation-launch-stack-button-svg/master/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=docksal-sandbox-server&templateURL=https://s3.us-east-2.amazonaws.com/docksal-aws-templates/sandbox-server/v3/advanced.json) -You will be prompted for few required and optional settings. +You will be prompted for a few required and optional settings. - Basic: Required - Resource type (`ec2` vs `spot`) - Instance type - SSH key - Availability zone -- VPC/Network: Optional** +- VPC/Network: Optional - VPC ID - Subnet ID - Elastic IP From 0fb85a9f2fc3c08bdc867dc8f7f3cc39083ab76f Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 21 Jul 2020 08:27:59 -0700 Subject: [PATCH 19/25] AWS: Updated parameter labels in advanced template --- aws-cloudformation/tpl-advanced.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index 385ed8b..e3e0acc 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -244,11 +244,11 @@ Metadata: ExistingEIP: default: "Elastic IP" AccessFrom1: - default: "Access from cidr 1" + default: "Access from CIDR 1" AccessFrom2: - default: "Access from cidr 2" + default: "Access from CIDR 2" AccessFrom3: - default: "Access from cidr 3" + default: "Access from CIDR 3" ExistingDataVolume: default: "Persistent data volume" EnableArtifactsBucket: From f130670347333533c4df84f9cc8e9f968e55211e Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 21 Jul 2020 08:33:03 -0700 Subject: [PATCH 20/25] AWS: Updated README.md Updated parameter list for advanced stack --- aws-cloudformation/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/aws-cloudformation/README.md b/aws-cloudformation/README.md index 7fe6cc9..506d18e 100644 --- a/aws-cloudformation/README.md +++ b/aws-cloudformation/README.md @@ -66,14 +66,18 @@ You will be prompted for a few required and optional settings. - Basic: Required - Resource type (`ec2` vs `spot`) - - Instance type + - Instance type (primary) + - Instance type 2 (spot only) + - Instance type 3 (spot only) - SSH key - Availability zone - VPC/Network: Optional - VPC ID - Subnet ID - Elastic IP - - Access from + - Access from CIDR 1 + - Access from CIDR 2 + - Access from CIDR 3 - Storage: Optional - Persistent data volume - Enable artifacts bucket @@ -85,6 +89,8 @@ You will be prompted for a few required and optional settings. - LetsEncrypt settings: Optional - Sandbox domain name - LetsEncrypt configuration +- Docksal settings: Optional + - Docksal version Once provisioned, the IP address of the server will be printed in the **Outputs** section in CloudFormation (``). From 178c3d07f0e20fae5b10a9c4b0d6cb05909da3ae Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 21 Jul 2020 12:18:47 -0700 Subject: [PATCH 21/25] AWS: Dropped InstanceType3 option 3 types can be confusing/difficult to choose from. 2 should be enough to form a viable spot fleet. --- aws-cloudformation/tpl-advanced.yaml | 34 ---------------------------- 1 file changed, 34 deletions(-) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index e3e0acc..01dca16 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -64,35 +64,6 @@ Parameters: - r5d.large - r5d.xlarge - r5d.2xlarge - InstanceType3: - Description: | - Additional EC2 instance type (spot mode only). AWS will pick the cheapest available option for spot instance. - IMPORTANT: "Instance Type"/"Instance Type 2"/"Instance Type 3" must be unique, overwise stack creation/update will fail. - Type: String - Default: t3.large - AllowedValues: - - t3.nano - - t3.micro - - t3.small - - t3.medium - - t3.large - - t3.xlarge - - t3.2xlarge - - m5.large - - m5.xlarge - - m5.2xlarge - - m5.4xlarge - - m5d.large - - m5d.xlarge - - m5d.2xlarge - - m5d.4xlarge - - r5.large - - r5.xlarge - - r5.2xlarge - - r5d.large - - r5d.xlarge - - r5d.2xlarge - ConstraintDescription: "Must be a valid EC2 instance type" KeyName: Description: "Name of an existing EC2 KeyPair to enable SSH access to the instance" Type: "AWS::EC2::KeyPair::KeyName" @@ -191,7 +162,6 @@ Metadata: - ResourceType - InstanceType - InstanceType2 - - InstanceType3 - KeyName - ManualAZ - Label: @@ -231,8 +201,6 @@ Metadata: default: "Instance type (primary)" InstanceType2: default: "Instance type 2 (spot only)" - InstanceType3: - default: "Instance type 3 (spot only)" KeyName: default: "SSH key" ManualAZ: @@ -658,8 +626,6 @@ Resources: - <<: *instance-profile InstanceType: !Ref InstanceType2 - - <<: *instance-profile - InstanceType: !Ref InstanceType3 ec2Instance: Type: 'AWS::EC2::Instance' From fd6aa2b3fc68383e03fac6232fe736cd56e6f955 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 21 Jul 2020 12:22:54 -0700 Subject: [PATCH 22/25] AWS: Updated list of instance types - Added description for different instance families - Added t3a and c5 options - Dropped m5d and r5d options --- aws-cloudformation/tpl-advanced.yaml | 43 +++++++++++++++++----------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/aws-cloudformation/tpl-advanced.yaml b/aws-cloudformation/tpl-advanced.yaml index 01dca16..7946c21 100644 --- a/aws-cloudformation/tpl-advanced.yaml +++ b/aws-cloudformation/tpl-advanced.yaml @@ -15,7 +15,13 @@ Parameters: - ec2 - spot InstanceType: - Description: "EC2 instance type (default: t3.small = 2 vCPU, 2GB RAM)" + Description: | + EC2 instance type (default: t3.small = 2 vCPU, 2GB RAM). + t3 (Intel) / t3a (AMD) family instances provide burstable CPU performance and are generally the best choise for a sandbox server. + t3/t3a: CPU/RAM ratio is variable from 1/1 to 1/4 based on instance size (t3.large = 2 vCPU, 8GB RAM). + c5: CPU/RAM ratio is 1/2 (c5.large = 2 vCPU, 4GB RAM). + m5: CPU/RAM ratio is 1/4 (m5.large = 2 vCPU, 8GB RAM). + r5: CPU/RAM ratio is 1/8 (r5.large = 2 vCPU, 16GB RAM). Type: String Default: t3.small AllowedValues: @@ -24,24 +30,27 @@ Parameters: - t3.large - t3.xlarge - t3.2xlarge + - t3a.small + - t3a.medium + - t3a.large + - t3a.xlarge + - t3a.2xlarge + - c5.large + - c5.xlarge + - c5.2xlarge + - c5.4xlarge - m5.large - m5.xlarge - m5.2xlarge - m5.4xlarge - - m5d.large - - m5d.xlarge - - m5d.2xlarge - - m5d.4xlarge - r5.large - r5.xlarge - r5.2xlarge - - r5d.large - - r5d.xlarge - - r5d.2xlarge InstanceType2: Description: | Additional EC2 instance type (spot mode only). AWS will pick the cheapest available option for spot instance. - IMPORTANT: "Instance Type"/"Instance Type 2"/"Instance Type 3" must be unique, overwise stack creation/update will fail. + IMPORTANT: "Instance Type"/"Instance Type 2" must be unique, overwise stack creation/update will fail. + Recomendation: Use a "t3" instance for the primary option and a matching size "t3a" instance type here. Type: String Default: t3.medium AllowedValues: @@ -50,20 +59,22 @@ Parameters: - t3.large - t3.xlarge - t3.2xlarge + - t3a.small + - t3a.medium + - t3a.large + - t3a.xlarge + - t3a.2xlarge + - c5.large + - c5.xlarge + - c5.2xlarge + - c5.4xlarge - m5.large - m5.xlarge - m5.2xlarge - m5.4xlarge - - m5d.large - - m5d.xlarge - - m5d.2xlarge - - m5d.4xlarge - r5.large - r5.xlarge - r5.2xlarge - - r5d.large - - r5d.xlarge - - r5d.2xlarge KeyName: Description: "Name of an existing EC2 KeyPair to enable SSH access to the instance" Type: "AWS::EC2::KeyPair::KeyName" From ad9efba0a443a43ad0131d04e5383a07c9dd2245 Mon Sep 17 00:00:00 2001 From: "Les Peabody (BOS-GEN)" Date: Wed, 22 Jul 2020 10:05:22 -0400 Subject: [PATCH 23/25] Remove unused docksal_vm, revert to master. docksal_vm was a leftover from the older basic vs advanced workflow, prior to consolidating to an instance template strategy. --- gcp-deployment-manager/project.env | 4 +- .../templates/docksal_vm.jinja | 80 ------------------- .../templates/docksal_vm.jinja.schema | 40 ---------- 3 files changed, 2 insertions(+), 122 deletions(-) delete mode 100644 gcp-deployment-manager/templates/docksal_vm.jinja delete mode 100644 gcp-deployment-manager/templates/docksal_vm.jinja.schema diff --git a/gcp-deployment-manager/project.env b/gcp-deployment-manager/project.env index 54972d6..551b914 100644 --- a/gcp-deployment-manager/project.env +++ b/gcp-deployment-manager/project.env @@ -14,6 +14,6 @@ ZONE=us-east4-c REGION=us-east4 DEFAULT_MACHINE_SIZE=n1-standard-2 DISK_SIZE=150 -TEMPLATE_ID=docksal-sandbox-template-001 +TEMPLATE_ID=docksal-sandbox-template-002 PREEMPTIVE=true -DOCKSAL_VERSION=develop +DOCKSAL_VERSION=master diff --git a/gcp-deployment-manager/templates/docksal_vm.jinja b/gcp-deployment-manager/templates/docksal_vm.jinja deleted file mode 100644 index 08b417b..0000000 --- a/gcp-deployment-manager/templates/docksal_vm.jinja +++ /dev/null @@ -1,80 +0,0 @@ -{# - -Docksal VM - -Note: Ensure that the docksal-sandbox-disk deployment has already run or this -will fail. - -#} - -{% set project = env["project"] %} -{% set deployment = env["deployment"] %} -{% set instanceName = "%s-vm" % deployment %} -{% set zone = properties["zone"] %} -{% set region = properties["region"] %} -{% set machineType = properties["machineType"] %} -{% set staticIpName = "static-ip-%s" % env["deployment"] %} -{% set dataDiskName = properties["dataDiskName"] %} -{% set bootDiskId = "%s-boot" % deployment %} - -resources: - -# Static IP -- name: {{ staticIpName }} - properties: - addressType: EXTERNAL - description: | - The static IP address used for VMs in the deployment - {{ deployment }}. - region: {{ region }} - resourceType: addresses - type: gcp-types/compute-v1:addresses - -# VM instance -- type: compute.v1.instance - name: {{ instanceName }} - properties: - zone: {{ zone }} - machineType: https://www.googleapis.com/compute/v1/projects/{{ project }}/zones/{{ zone }}/machineTypes/{{ machineType }} - tags: - items: - - http-server - - https-server - - disks: - - deviceName: {{ bootDiskId }} - type: PERSISTENT - boot: true - autoDelete: true - initializeParams: - sourceImage: projects/ubuntu-os-cloud/global/images/ubuntu-minimal-1804-bionic-v20200610 - diskName: {{ bootDiskId }} - - deviceName: {{ dataDiskName }}-data - source: projects/{{ project }}/zones/{{ zone }}/disks/{{ dataDiskName }} - - networkInterfaces: - - network: https://www.googleapis.com/compute/v1/projects/{{ project }}/global/networks/default - # Access Config required to give the instance a public IP address - accessConfigs: - - name: External IP - natIP: $(ref.{{ staticIpName }}.address) - - metadata: - items: - {% for key, value in properties['metadata-from-file'].items() %} - - key: {{ key }} - value: | - {{ imports[value]|indent(10) }} - {% endfor %} - -outputs: - - name: deployment - value: {{ deployment }} - - name: project - value: {{ project }} - - name: vmId - value: $(ref.{{ instanceName }}.id) - - name: vmName - value: {{ instanceName }} - - name: vmSelfLink - value: $(ref.{{ instanceName }}.selfLink) diff --git a/gcp-deployment-manager/templates/docksal_vm.jinja.schema b/gcp-deployment-manager/templates/docksal_vm.jinja.schema deleted file mode 100644 index 44d57ad..0000000 --- a/gcp-deployment-manager/templates/docksal_vm.jinja.schema +++ /dev/null @@ -1,40 +0,0 @@ -info: - title: Docksal VM Instance - author: Les Peabody - description: | - Creates a Docksal VM instance and static IP resource. - -additionalProperties: false - -imports: -- path: startup.sh - name: startup.sh - -properties: - region: - type: string - description: The region which this resource resides. - default: us-east4 - zone: - type: string - description: The zone which this resource resides. - default: us-east4-c - machineType: - type: string - description: | - The type of machine instance to use. Defaults to n1-standard-2. - default: n1-standard-2 - metadata-from-file: - type: object - additionalProperties: false - default: - startup-script: startup.sh - properties: - startup-script: - type: string - description: The name of the script to be imported. - default: startup.sh - dataDiskName: - type: string - description: The name of the disk containing instance data. - default: docksal-sandbox-disk From 0e93c94ddb3005e998833e81b56fafe98c8d518d Mon Sep 17 00:00:00 2001 From: "Les Peabody (BOS-GEN)" Date: Wed, 22 Jul 2020 10:10:45 -0400 Subject: [PATCH 24/25] Update Ubuntu 18.04 reference to 20.04. --- gcp-deployment-manager/templates/startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcp-deployment-manager/templates/startup.sh b/gcp-deployment-manager/templates/startup.sh index ce6429b..55ddc6d 100755 --- a/gcp-deployment-manager/templates/startup.sh +++ b/gcp-deployment-manager/templates/startup.sh @@ -1,7 +1,7 @@ #!/bin/bash # This is a startup script for a Docksal Sandbox server in GCP. -# It installs and configures Docksal on a bare Ubuntu machine (tested with Ubuntu 18.04 Minimal). +# It installs and configures Docksal on a bare Ubuntu machine (tested with Ubuntu 20.04 Minimal). # # The startup script log can be views via "gcloud compute ssh vm-sandbox-test -- tail -f /var/log/syslog" From ea06ff0307f839c840c06e5c25ef87c847d37095 Mon Sep 17 00:00:00 2001 From: Sergey Zabolotny <37826765+sergey-zabolotny@users.noreply.github.com> Date: Thu, 23 Jul 2020 20:47:58 +0300 Subject: [PATCH 25/25] Remove ec2-instance-connect package (#49) Co-authored-by: Leonid Makarov --- aws-cloudformation/startup-advanced.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aws-cloudformation/startup-advanced.sh b/aws-cloudformation/startup-advanced.sh index 480b813..e69f79e 100755 --- a/aws-cloudformation/startup-advanced.sh +++ b/aws-cloudformation/startup-advanced.sh @@ -327,6 +327,9 @@ then sed -i "s|^BACKUP_SSH_PUBLIC_KEY=\".*\"|BACKUP_SSH_PUBLIC_KEY=\"${BACKUP_SSH_PUBLIC_KEY}\"|g" /usr/local/bin/ssh-rake chmod +x /usr/local/bin/ssh-rake /usr/local/bin/ssh-rake install + # Remove ec2-instance-connect as it brakes ssh-rake + # See https://github.com/aws/aws-ec2-instance-connect-config/issues/19 + apt-get purge ec2-instance-connect -y fi if [[ "${old_stack_md5sum}" != "${stack_md5sum}" ]] @@ -387,4 +390,3 @@ fi su - build-agent -c "fin system reset" echo "${stack_md5sum}" >/root/stack_last_update -