mardizzone/POS-1058: run devnets behind VPN and Bastillion (#155)

* [WIP] dev: add: enable connection only when using VPN and Bastillion

* [WIP] dev: chg: remove some logs

* [WIP] dev: chg: use dns instead of IPs

* [WIP] dev: chg: remove exit

* [WIP] dev: chg: remove eips from cidr blocks

* dev: chg: use dns ips as hosts ips

* dev: chg: add check on files length while removing keystores

* dev: chg: replace ips with dnsIps

.gitignore

@@ -3,6 +3,7 @@ test
 .DS_Store
 devnet
 .env
+*.tfvars
 .terraform*
 terraform.tfstate
 terraform.tfstate.backup

README.md

@@ -27,6 +27,9 @@ To use the `express-cli` you have to execute the following steps.
   by running `nvm use` from the root folder
 - install `express-cli` and `matic-cli` locally with command `npm i`
 - generate a keypair on AWS EC2 and download its certificate locally (`.pem` file)
+- copy `secret.tfvars.example` to `secret.tfvar` with command `cp secret.tfvars.example secret.tfvars` and check the commented file for details
+- **If you are a Polygon employee**, connect to the company VPN
+- modify `secret.tfvar` with addresses of the allowed IPs (as specified in `secret.tfvars.example` file)
 - copy `.env.example` to `.env` with command `cp .env.example .env` and check the heavily commented file for details
 - make sure `PEM_FILE_PATH` points to a correct AWS key certificate, the one you downloaded in the previous steps
 - define the number of nodes (`TF_VAR_VALIDATOR_COUNT` and `TF_VAR_SENTRY_COUNT`) and adjust the `DEVNET_BOR_USERS`

configs/devnet/docker-setup-config.yaml

@@ -16,11 +16,12 @@ blockNumber: '0'
 blockTime: '2'
 numOfValidators: 1
 numOfNonValidators: 1
+numOfArchiveNodes: 0
 ethURL: http://ganache:9545
 ethHostUser: ubuntu
 devnetType: docker
 borDockerBuildContext: https://github.com/maticnetwork/bor.git#develop
 heimdallDockerBuildContext: https://github.com/maticnetwork/heimdall.git#develop
 devnetBorUsers: ubuntu,ubuntu
 devnetBorHosts:
-  - 35.92.248.232
+  - ec2-xx-xxx-xx-xxx.us-west-2.compute.amazonaws.com # use localhost for local deployments

configs/devnet/remote-setup-config.yaml

@@ -16,15 +16,16 @@ blockNumber: '0'
 blockTime: '2'
 numOfValidators: 1
 numOfNonValidators: 1
-ethURL: http://172.20.1.100:9545
+numOfArchiveNodes: 0
+ethURL: http://ec2-xx-xxx-xxx-xx.us-west-2.compute.amazonaws.com:9545
 ethHostUser: ubuntu
 devnetType: remote
 devnetBorHosts:
-  - 172.20.1.100
-  - 172.20.1.101
+  - ec2-xx-xxx-xxx-xx.us-west-2.compute.amazonaws.com
+  - ec2-yy-yyy-yyy-yy.us-west-2.compute.amazonaws.com
 devnetHeimdallHosts:
-  - 172.20.1.100
-  - 172.20.1.101
+  - ec2-xx-xxx-xxx-xx.us-west-2.compute.amazonaws.com
+  - ec2-yy-yyy-yyy-yy.us-west-2.compute.amazonaws.com
 devnetBorUsers:
   - ubuntu
   - ubuntu

main.tf

@@ -58,7 +58,8 @@ resource "aws_security_group" "internet_facing_alb" {
       from_port   = ingress.value
       to_port     = ingress.value
       protocol    = "tcp"
-      cidr_blocks = var.SG_CIDR_BLOCKS
+      cidr_blocks = concat(var.SG_CIDR_BLOCKS, [aws_vpc.My_VPC.cidr_block])
+      self = true
     }
   }
   dynamic "egress" {
@@ -68,7 +69,8 @@ resource "aws_security_group" "internet_facing_alb" {
       from_port   = egress.value
       to_port     = egress.value
       protocol    = "-1"
-      cidr_blocks = var.SG_CIDR_BLOCKS
+      cidr_blocks = var.SG_CIDR_BLOCKS_OUT
+      self = true
     }
   }
   tags = {
@@ -102,7 +104,10 @@ resource "aws_vpc" "My_VPC" {
   }
 }
 
-resource "aws_internet_gateway" "gw" { vpc_id = aws_vpc.My_VPC.id }
+resource "aws_internet_gateway" "gw" {
+  vpc_id = aws_vpc.My_VPC.id
+
+}
 
 resource "aws_route_table" "table" {
   vpc_id = aws_vpc.My_VPC.id
@@ -127,6 +132,10 @@ output "instance_ips" {
   value = aws_eip.eip.*.public_ip
 }
 
+output "instance_dns_ips" {
+  value = aws_eip.eip.*.public_dns
+}
+
 output "instance_ids" {
   value = aws_instance.app_server.*.id
 }

secret.tfvars.example

@@ -0,0 +1,7 @@
+# 1. Copy this file to secret.tfvar (e.g. `cp secret.tfvars.example secret.tfvars`)
+# 2. Modify secret.tfvars by replacing the following with the list of IPs you want to allow
+# 3. For Polygon employees, these would be Bastillion and VPN IP addresses
+# 4. If you don't know what are Bastillion and VPN IP addresses, please ask to the PoS team (ref. [email protected])
+# NOTE for Polygon employees: Do NOT use "0.0.0.0/0" for security reasons, otherwise the admin port will be open to the Internet!
+
+SG_CIDR_BLOCKS=["1.2.3.4/0", "5.6.7.8/0"]

src/express/commands/destroy.js

@@ -7,7 +7,7 @@ const shell = require('shelljs')
 export async function terraformDestroy() {
   console.log('📍Executing terraform destroy...')
   require('dotenv').config({ path: `${process.cwd()}/.env` })
-  shell.exec('terraform destroy -auto-approve', {
+  shell.exec('terraform destroy -auto-approve -var-file=./secret.tfvars', {
     env: {
       ...process.env
     }

src/express/commands/init.js

@@ -12,6 +12,9 @@ export async function terraformInit() {
 
   shell.exec(`mkdir -p ./deployments/devnet-${nextDevnetId}`)
   shell.exec(`cp ./.env ./deployments/devnet-${nextDevnetId}/.env`)
+  shell.exec(
+    `cp ./secret.tfvars ./deployments/devnet-${nextDevnetId}/secret.tfvars`
+  )
   shell.exec(`cp ./main.tf ./deployments/devnet-${nextDevnetId}/main.tf`)
   shell.exec(
     `cp ./variables.tf ./deployments/devnet-${nextDevnetId}/variables.tf`

src/express/commands/start.js

@@ -22,7 +22,7 @@ const shell = require('shelljs')
 async function terraformApply(devnetId) {
   console.log('📍Executing terraform apply...')
   shell.exec(
-    `terraform -chdir=../../deployments/devnet-${devnetId} apply -auto-approve`,
+    `terraform -chdir=../../deployments/devnet-${devnetId} apply -auto-approve -var-file=./secret.tfvars`,
     {
       env: {
         ...process.env
@@ -395,9 +395,9 @@ export async function start() {
 
   await terraformApply(devnetId)
   const tfOutput = await terraformOutput()
-  const ips = JSON.parse(tfOutput).instance_ips.value.toString()
+  const dnsIps = JSON.parse(tfOutput).instance_dns_ips.value.toString()
   const ids = JSON.parse(tfOutput).instance_ids.value.toString()
-  process.env.DEVNET_BOR_HOSTS = ips
+  process.env.DEVNET_BOR_HOSTS = dnsIps
   process.env.INSTANCES_IDS = ids
 
   await validateConfigs()
@@ -421,15 +421,15 @@ export async function start() {
   console.log('📍Waiting 30s for the VMs to initialize...')
   await timer(30000)
 
-  await installRequiredSoftwareOnRemoteMachines(ips, devnetType, devnetId)
+  await installRequiredSoftwareOnRemoteMachines(dnsIps, devnetType, devnetId)
 
-  await prepareMaticCLI(ips, devnetType, devnetId)
+  await prepareMaticCLI(dnsIps, devnetType, devnetId)
 
-  await eventuallyCleanupPreviousDevnet(ips, devnetType, devnetId)
+  await eventuallyCleanupPreviousDevnet(dnsIps, devnetType, devnetId)
 
   if (devnetType === 'docker') {
-    await runDockerSetupWithMaticCLI(ips, devnetId)
+    await runDockerSetupWithMaticCLI(dnsIps, devnetId)
   } else {
-    await runRemoteSetupWithMaticCLI(ips, devnetId)
+    await runRemoteSetupWithMaticCLI(dnsIps, devnetId)
   }
 }

src/setup/devnet/index.js

@@ -892,12 +892,11 @@ export class Devnet {
               'keystore'
             )
             fs.readdir(keystoreDir, async (err, files) => {
-              if (err) throw err
-
-              for (let j = 1; j < files.length; j++) {
-                await fs.unlink(path.join(keystoreDir, files[j]), err => {
-                  if (err) throw err
-                })
+              if (err) console.log(err) // harmless
+              if (files) {
+                for (let j = 1; j < files.length; j++) {
+                  await fs.unlink(path.join(keystoreDir, files[j]))
+                }
               }
             })
             await timer(2000)

variables.tf

@@ -67,6 +67,11 @@ variable "REGION" {
 }
 
 variable "SG_CIDR_BLOCKS" {
+  description = "Contains allowed IPs. Please, set them into secret.tfvars (example available at secret.tfvars.example)"
+  sensitive = true
+}
+
+variable "SG_CIDR_BLOCKS_OUT" {
   default = ["0.0.0.0/0"]
 }