diff --git a/.gitignore b/.gitignore index 2fc7f49083f..57c30467f00 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ Thumbs.db .idea .fleet + # Folders _obj _test diff --git a/e2e/terraform/.terraform.lock.hcl b/e2e/terraform/.terraform.lock.hcl deleted file mode 100644 index 5cef68f560c..00000000000 --- a/e2e/terraform/.terraform.lock.hcl +++ /dev/null @@ -1,177 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "4.10.0" - hashes = [ - "h1:3zeyl8QwNYPXRD4b++0Vo9nBcsL3FXT+DT3x/KJNKB0=", - "h1:F9BjbxBhuo1A/rP318IUrkW3TAh29i6UC18qwhzCs6c=", - "h1:S6xGPRL08YEuBdemiYZyIBf/YwM4OCvzVuaiuU6kLjc=", - "h1:pjPLizna1qa/CZh7HvLuQ73YmqaunLXatyOqzF2ePEI=", - "zh:0a2a7eabfeb7dbb17b7f82aff3fa2ba51e836c15e5be4f5468ea44bd1299b48d", - "zh:23409c7205d13d2d68b5528e1c49e0a0455d99bbfec61eb0201142beffaa81f7", - "zh:3adad2245d97816f3919778b52c58fb2de130938a3e9081358bfbb72ec478d9a", - "zh:5bf100aba6332f24b1ffeae7536d5d489bb907bf774a06b95f2183089eaf1a1a", - "zh:63c3a24c0c229a1d3390e6ea2454ba4d8ace9b94e086bee1dbdcf665ae969e15", - "zh:6b76f5ffd920f0a750da3a4ff1d00eab18d9cd3731b009aae3df4135613bad4d", - "zh:8cd6b1e6b51e8e9bbe2944bb169f113d20d1d72d07ccd1b7b83f40b3c958233e", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:c5c31f58fb5bd6aebc6c662a4693640ec763cb3399cce0b592101cf24ece1625", - "zh:cc485410be43d6ad95d81b9e54cc4d2117aadf9bf5941165a9df26565d9cce42", - "zh:cebb89c74b6a3dc6780824b1d1e2a8d16a51e75679e14ad0b830d9f7da1a3a67", - "zh:e7dc427189cb491e1f96e295101964415cbf8630395ee51e396d2a811f365237", - ] -} - -provider "registry.terraform.io/hashicorp/external" { - version = "2.2.2" - hashes = [ - "h1:/Qsdu8SIXbfANKJFs1UTAfvcomJUalOd3uDZvj3jixA=", - "h1:BKQ5f5ijzeyBSnUr+j0wUi+bYv6KBQVQNDXNRVEcfJE=", - "h1:VUkgcWvCliS0HO4kt7oEQhFD2gcx/59XpwMqxfCU1kE=", - "h1:e7RpnZ2PbJEEPnfsg7V0FNwbfSk0/Z3FdrLsXINBmDY=", - "zh:0b84ab0af2e28606e9c0c1289343949339221c3ab126616b831ddb5aaef5f5ca", - "zh:10cf5c9b9524ca2e4302bf02368dc6aac29fb50aeaa6f7758cce9aa36ae87a28", - "zh:56a016ee871c8501acb3f2ee3b51592ad7c3871a1757b098838349b17762ba6b", - "zh:719d6ef39c50e4cffc67aa67d74d195adaf42afcf62beab132dafdb500347d39", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7fbfc4d37435ac2f717b0316f872f558f608596b389b895fcb549f118462d327", - "zh:8ac71408204db606ce63fe8f9aeaf1ddc7751d57d586ec421e62d440c402e955", - "zh:a4cacdb06f114454b6ed0033add28006afa3f65a0ea7a43befe45fc82e6809fb", - "zh:bb5ce3132b52ae32b6cc005bc9f7627b95259b9ffe556de4dad60d47d47f21f0", - "zh:bb60d2976f125ffd232a7ccb4b3f81e7109578b23c9c6179f13a11d125dca82a", - "zh:f9540ecd2e056d6e71b9ea5f5a5cf8f63dd5c25394b9db831083a9d4ea99b372", - "zh:ffd998b55b8a64d4335a090b6956b4bf8855b290f7554dd38db3302de9c41809", - ] -} - -provider "registry.terraform.io/hashicorp/hcp" { - version = "0.26.0" - hashes = [ - "h1:B5O/NawTnKPdUgUlGP/mM2ybv0RcLvVJVOcrivDdFnI=", - "h1:C0KoYT09Ff91pE5KzrFrISCE5wQyJaJnxPdA0SXDOzI=", - "h1:f4IwCK9heo5F+k+nRFY/fzG18DesbBcqRL8F4WsKh7Q=", - "h1:fCHcXVlT/MoAqvIUjFyJqtGrz+ebHNCcR1YM2ZSRPxE=", - "zh:0fa82a384b25a58b65523e0ea4768fa1212b1f5cfc0c9379d31162454fedcc9d", - "zh:6fa5415dbac9c8d20026772dd5aee7dd3ac541e9d86827d0b70bc752472ec76c", - "zh:7490212c32339153165aec1dcef063804aac0d3f1cfbdfd3d04d7a60c29b0f40", - "zh:792e8fbe630159105801a471c46c988d94636637c1e5cdb725956cab4e664c87", - "zh:9e460a3e4735ff24f2fc1c445fce54e4ed596c8dc97f683f5cefa93fb2be9b14", - "zh:a124e8366fdf10d17a0b2860151beb00e12d8c33860fcc661547d0239138d3fb", - "zh:a9b9cb4d077f8d8bcc22c813aea820c224228807f34e2e3716d30c84ce63c53a", - "zh:aae6a8e87c6c64bb33311ef658993a5cc8398aac8dcb2c18953bd9e96a2e0011", - "zh:dc2e83b8f4ca2d4aa2e0b5cc98b9c298c1cf5c583d323320c85d4f06f8f4b43c", - "zh:e17b1c7ef80c3507c892d343282c61dc58ab45978481ee004843f1746f6b791c", - "zh:ee35efe2628aca5f259f3fee8db15accfdced1a5530f01c8a23f59e5ed5dcb7a", - "zh:f8173393330eb376b7357f8271d1c75e0850905dceb32ce482af58e112894278", - ] -} - -provider "registry.terraform.io/hashicorp/local" { - version = "2.2.2" - hashes = [ - "h1:5UYW2wJ320IggrzLt8tLD6MowePqycWtH1b2RInHZkE=", - "h1:BVEZnjtpWxKPG9OOQh4dFa1z5pwMO/uuzYtu6AR2LyM=", - "h1:S6nf97sybBugc8FtrOSPXaynEKx0gO6Oktu6KJzvdDU=", - "h1:SjDyZXIUHEQzZe10VjhlhZq2a9kgQB6tmqJcpq2BeWg=", - "zh:027e4873c69da214e2fed131666d5de92089732a11d096b68257da54d30b6f9d", - "zh:0ba2216e16cfb72538d76a4c4945b4567a76f7edbfef926b1c5a08d7bba2a043", - "zh:1fee8f6aae1833c27caa96e156cf99a681b6f085e476d7e1b77d285e21d182c1", - "zh:2e8a3e72e877003df1c390a231e0d8e827eba9f788606e643f8e061218750360", - "zh:719008f9e262aa1523a6f9132adbe9eee93c648c2981f8359ce41a40e6425433", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9a70fdbe6ef955c4919a4519caca116f34c19c7ddedd77990fbe4f80fe66dc84", - "zh:abc412423d670cbb6264827fa80e1ffdc4a74aff3f19ba6a239dd87b85b15bec", - "zh:ae953a62c94d2a2a0822e5717fafc54e454af57bd6ed02cd301b9786765c1dd3", - "zh:be0910bdf46698560f9e86f51a4ff795c62c02f8dc82b2b1dab77a0b3a93f61e", - "zh:e58f9083b7971919b95f553227adaa7abe864fce976f0166cf4d65fc17257ff2", - "zh:ff4f77cbdbb22cc98182821c7ef84dce16298ab0e997d5c7fae97247f7a4bcb0", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.1.1" - hashes = [ - "h1:71sNUDvmiJcijsvfXpiLCz0lXIBSsEJjMxljt7hxMhw=", - "h1:Pctug/s/2Hg5FJqjYcTM0kPyx3AoYK1MpRWO0T9V2ns=", - "h1:YvH6gTaQzGdNv+SKTZujU1O0bO+Pw6vJHOPhqgN8XNs=", - "h1:ZD4wyZ0KJzt5s2mD0xD7paJlVONNicLvZKdgtezz02I=", - "zh:063466f41f1d9fd0dd93722840c1314f046d8760b1812fa67c34de0afcba5597", - "zh:08c058e367de6debdad35fc24d97131c7cf75103baec8279aba3506a08b53faf", - "zh:73ce6dff935150d6ddc6ac4a10071e02647d10175c173cfe5dca81f3d13d8afe", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:8fdd792a626413502e68c195f2097352bdc6a0df694f7df350ed784741eb587e", - "zh:976bbaf268cb497400fd5b3c774d218f3933271864345f18deebe4dcbfcd6afa", - "zh:b21b78ca581f98f4cdb7a366b03ae9db23a73dfa7df12c533d7c19b68e9e72e5", - "zh:b7fc0c1615dbdb1d6fd4abb9c7dc7da286631f7ca2299fb9cd4664258ccfbff4", - "zh:d1efc942b2c44345e0c29bc976594cb7278c38cfb8897b344669eafbc3cddf46", - "zh:e356c245b3cd9d4789bab010893566acace682d7db877e52d40fc4ca34a50924", - "zh:ea98802ba92fcfa8cf12cbce2e9e7ebe999afbf8ed47fa45fc847a098d89468b", - "zh:eff8872458806499889f6927b5d954560f3d74bf20b6043409edf94d26cd906f", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.1.2" - hashes = [ - "h1:5A5VsY5wNmOZlupUcLnIoziMPn8htSZBXbP3lI7lBEM=", - "h1:9A6Ghjgad0KjJRxa6nPo8i8uFvwj3Vv0wnEgy49u+24=", - "h1:JF+aiOtS0G0ffbBdk1qfj7IrT39y/GZh/yl2IhqcIVM=", - "h1:hxN/z2AVJkF2ei7bfevJdD1B0WfyABxxk9j1zzLsLRk=", - "zh:0daceba867b330d3f8e2c5dc895c4291845a78f31955ce1b91ab2c4d1cd1c10b", - "zh:104050099efd30a630741f788f9576b19998e7a09347decbec3da0b21d64ba2d", - "zh:173f4ef3fdf0c7e2564a3db0fac560e9f5afdf6afd0b75d6646af6576b122b16", - "zh:41d50f975e535f968b3f37170fb07937c15b76d85ba947d0ce5e5ff9530eda65", - "zh:51a5038867e5e60757ed7f513dd6a973068241190d158a81d1b69296efb9cb8d", - "zh:6432a568e97a5a36cc8aebca5a7e9c879a55d3bc71d0da1ab849ad905f41c0be", - "zh:6bac6501394b87138a5e17c9f3a41e46ff7833ad0ba2a96197bb7787e95b641c", - "zh:6c0a7f5faacda644b022e7718e53f5868187435be6d000786d1ca05aa6683a25", - "zh:74c89de3fa6ef3027efe08f8473c2baeb41b4c6cee250ba7aeb5b64e8c79800d", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:b29eabbf0a5298f0e95a1df214c7cfe06ea9bcf362c63b3ad2f72d85da7d4685", - "zh:e891458c7a61e5b964e09616f1a4f87d0471feae1ec04cc51776e7dec1a3abce", - ] -} - -provider "registry.terraform.io/hashicorp/tls" { - version = "3.3.0" - hashes = [ - "h1:A4xOtHhD4jCmn4nO1xCTk2Nl5IP5JpjicjF+Fuu2ZFQ=", - "h1:Uf8HqbZjYn8pKB0og2H9A8IXIKtHT+o8BE3+fjtO1ZQ=", - "h1:oitTcxYGyDvHuNsjPJUi00a+AT0k+TWgNsGUSM2CV/E=", - "h1:xx/b39Q9FVZSlDc97rlDmQ9dNaaxFFyVzP9kV+47z28=", - "zh:16140e8cc880f95b642b6bf6564f4e98760e9991864aacc8e21273423571e561", - "zh:16338b8457759c97fdd73153965d6063b037f2954fd512e569fcdc42b7fef743", - "zh:348bd44b7cd0c6d663bba36cecb474c17635a8f22b02187d034b8e57a8729c5a", - "zh:3832ac73c2335c0fac26138bacbd18160efaa3f06c562869acc129e814e27f86", - "zh:756d1e60690d0164eee9c93b498b4c8beabbfc1d8b7346cb6d2fa719055089d6", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:93b911bcddba8dadc5339edb004c8019c230ea67477c73c4f741c236dd9511b1", - "zh:c0c4e5742e8ac004c507540423db52af3f44b8ec04443aa8e14669340819344f", - "zh:c78296a1dff8ccd5d50203aac353422fc18d425072ba947c88cf5b46de7d32d2", - "zh:d7143f444e0f7e6cd67fcaf080398b4f1487cf05de3e0e79af6c14e22812e38b", - "zh:e600ac76b118816ad72132eee4c22ab5fc044f67c3babc54537e1fc1ad53d295", - "zh:fca07af5f591e12d2dc178a550da69a4847bdb34f8180a5b8e04fde6b528cf99", - ] -} - -provider "registry.terraform.io/hashicorp/vault" { - version = "3.4.1" - hashes = [ - "h1:HIjd/7KktGO5E/a0uICbIanUj0Jdd0j8aL/r+QxFhAs=", - "h1:X8P4B/zB97Dtj21qp0Rrswlz92WYCA5C59jpYGZeQuc=", - "h1:dXJBo807u69+Uib2hjoBQ68G2+nGXcNZeq/THVyQQVc=", - "h1:oow6cAwKiFpJBBWKsDqNmwZIrFTWWvoeIbqs+vyUDE0=", - "zh:1eb8370a1846e34e2bcc4d11eece5733735784a8eab447bbed3cfd822101b577", - "zh:2df3989327cea68b2167514b7ebddc67b09340f00bbf3fa85df03c97adfb9d25", - "zh:3dd1e317264f574985e856296deef71a76464918bf0566eb0d7f6389ea0586bd", - "zh:9750861f2822482aa608ea5a52b385bc42b2e1f2511094e6a975412618c4495d", - "zh:9b940e7f78975d29a4d0a116cf43c0bc1cb03bec4ad8d34887d64e6e60bacb9e", - "zh:9cb6e7ad2a62529d35dacd20695d49c2f02230cb785d46178cc10f4ec80e5a51", - "zh:a12718689bbcb37bcbb9132c18bffd354fad8ab5c8cb89cec1a0ee85c65b8cb7", - "zh:a6e38afacca1af4fab04a9f2dc49b8295eb462db68bdc7451352d0f950f804f8", - "zh:d6e0e994d51b9e07d5713d4796381f9e129e9de962e79caae2b7055f6f68297e", - "zh:ea4bbef7a1bb2553db473fa304c93845674167b61e8c9677107a96c8c696da12", - "zh:f985a8b7f4ef7d1eba9cef7d99997ee9c4a54ffe76dab7fa8b1fdec2a9edca7e", - ] -} diff --git a/e2e/terraform/Makefile b/e2e/terraform/Makefile index 10b4eda7884..14d7124c2a4 100644 --- a/e2e/terraform/Makefile +++ b/e2e/terraform/Makefile @@ -7,7 +7,7 @@ CONSUL_LICENSE_PATH ?= custom.tfvars: echo 'nomad_local_binary = "$(PKG_PATH)"' > custom.tfvars echo 'volumes = false' >> custom.tfvars - echo 'client_count_ubuntu_jammy_amd64 = 3' >> custom.tfvars + echo 'client_count_linux = 3' >> custom.tfvars echo 'client_count_windows_2016_amd64 = 0' >> custom.tfvars echo 'consul_license = "$(shell cat $(CONSUL_LICENSE_PATH))"' >> custom.tfvars echo 'nomad_license = "$(shell cat $(NOMAD_LICENSE_PATH))"' >> custom.tfvars diff --git a/e2e/terraform/README.md b/e2e/terraform/README.md index a5785654557..12a9e119628 100644 --- a/e2e/terraform/README.md +++ b/e2e/terraform/README.md @@ -51,7 +51,7 @@ Linux clients or Windows clients. region = "us-east-1" instance_type = "t2.medium" server_count = "3" -client_count_ubuntu_jammy_amd64 = "4" +client_count_linux = "4" client_count_windows_2016_amd64 = "1" ``` diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf index f6e84ef5e23..ac2a4e84550 100644 --- a/e2e/terraform/main.tf +++ b/e2e/terraform/main.tf @@ -1,34 +1,12 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - provider "aws" { region = var.region } -data "aws_caller_identity" "current" { -} - -resource "random_pet" "e2e" { -} - -resource "random_password" "windows_admin_password" { - length = 20 - special = true - override_special = "_%@" -} +module "provision-infra" { + source = "./provision-infra" -locals { - random_name = "${var.name}-${random_pet.e2e.id}" -} - -# Generates keys to use for provisioning and access -module "keys" { - name = local.random_name - path = "${path.root}/keys" - source = "mitchellh/dynamic-keys/aws" - version = "v2.0.0" -} - -data "aws_kms_alias" "e2e" { - name = "alias/${var.aws_kms_alias}" -} + server_count = var.client_count_linux + client_count_linux = var.client_count_linux + client_count_windows_2016_amd64 = var.client_count_windows_2016_amd64 + nomad_local_binary = var.nomad_local_binary +} \ No newline at end of file diff --git a/e2e/terraform/outputs.tf b/e2e/terraform/outputs.tf index 0441906b132..9ebe44b55fa 100644 --- a/e2e/terraform/outputs.tf +++ b/e2e/terraform/outputs.tf @@ -2,43 +2,19 @@ # SPDX-License-Identifier: BUSL-1.1 output "servers" { - value = aws_instance.server.*.public_ip + value = module.provision-infra.servers } output "linux_clients" { - value = aws_instance.client_ubuntu_jammy_amd64.*.public_ip + value = module.provision-infra.linux_clients } output "windows_clients" { - value = aws_instance.client_windows_2016_amd64.*.public_ip + value = module.provision-infra.windows_clients } output "message" { - value = <&2 echo "got task dir: $task_dir; message: $message" + + mkdir -p "$task_dir" + # and write something to a file we can check in the test. + echo "$message" > "$task_dir/victory" +} + +# run the appropriate CNI command +case "$CNI_COMMAND" in + VERSION) version ; exit ;; + ADD) add ;; +esac + +# bogus reply so nomad doesn't error +cat < /tmp/resolv.conf +nameserver 127.0.0.1 +nameserver $DOCKER_BRIDGE_IP_ADDRESS +EOF +cp /tmp/resolv.conf /etc/resolv.conf + +# need to get the interface for dnsmasq config so that we can +# accomodate both "predictable" and old-style interface names +IFACE=$(ip route | grep default | awk '{print $5}') + +cat < /tmp/dnsmasq +port=53 +resolv-file=/var/run/dnsmasq/resolv.conf +bind-interfaces +interface=docker0 +interface=lo +interface=$IFACE +listen-address=127.0.0.1 +server=/consul/127.0.0.1#8600 +EOF +cp /tmp/dnsmasq /etc/dnsmasq.d/default + +# need to get the AWS DNS address from the VPC... +# this is pretty hacky but will work for any typical case +MAC=$(curl -s --fail http://169.254.169.254/latest/meta-data/mac) +CIDR_BLOCK=$(curl -s --fail "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC/vpc-ipv4-cidr-block") +VPC_DNS_ROOT=$(echo "$CIDR_BLOCK" | cut -d'.' -f1-3) +echo "nameserver ${VPC_DNS_ROOT}.2" > /tmp/dnsmasq-resolv.conf +cp /tmp/dnsmasq-resolv.conf /var/run/dnsmasq/resolv.conf + +/usr/sbin/dnsmasq --test diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq b/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq new file mode 100644 index 00000000000..42b06f6e5b8 --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq @@ -0,0 +1,8 @@ +port=53 +resolv-file=/var/run/dnsmasq/resolv.conf +bind-interfaces +interface=docker0 +interface=lo +interface=eth0 +listen-address=127.0.0.1 +server=/consul/127.0.0.1#8600 diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq.service b/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq.service new file mode 100644 index 00000000000..93b7c97e3f5 --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq.service @@ -0,0 +1,37 @@ +[Unit] +Description=dnsmasq - A lightweight DHCP and caching DNS server +Requires=network.target +Wants=nss-lookup.target +Before=nss-lookup.target +After=network.target +After=docker.service + +[Service] +Type=forking +PIDFile=/run/dnsmasq/dnsmasq.pid + +# Configure our hosts and resolver file with info from the host, +# then test the resulting config file before starting +ExecStartPre=/usr/local/bin/dnsconfig.sh + +# (from upstream) +# We run dnsmasq via the /etc/init.d/dnsmasq script which acts as a +# wrapper picking up extra configuration files and then execs dnsmasq +# itself, when called with the "systemd-exec" function. +ExecStart=/etc/init.d/dnsmasq systemd-exec + +# (from upstream) +# The systemd-*-resolvconf functions configure (and deconfigure) +# resolvconf to work with the dnsmasq DNS server. They're called like +# this to get correct error handling (ie don't start-resolvconf if the +# dnsmasq daemon fails to start. +ExecStartPost=/etc/init.d/dnsmasq systemd-start-resolvconf + +# We need to tell docker to pick up the changes +ExecStartPost=/bin/systemctl restart docker + +ExecStop=/etc/init.d/dnsmasq systemd-stop-resolvconf +ExecReload=/bin/kill -HUP $MAINPID + +[Install] +WantedBy=multi-user.target diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/nomad.service b/e2e/terraform/packer/ubuntu-jammy-arm64/nomad.service new file mode 100644 index 00000000000..8490fc9c8e8 --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/nomad.service @@ -0,0 +1,21 @@ +[Unit] +Description=Nomad Agent +Requires=network-online.target +After=network-online.target +StartLimitIntervalSec=0 +StartLimitBurst=3 + +[Service] +ExecReload=/bin/kill -HUP $MAINPID +ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d +EnvironmentFile=-/etc/nomad.d/.environment +KillMode=process +KillSignal=SIGINT +LimitNOFILE=65536 +LimitNPROC=infinity +TasksMax=infinity +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/setup.sh b/e2e/terraform/packer/ubuntu-jammy-arm64/setup.sh new file mode 100755 index 00000000000..4129326d18b --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/setup.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# setup script for Ubuntu Linux 22.04. Assumes that Packer has placed +# build-time config files at /tmp/linux + +set -xeuo pipefail + +NOMAD_PLUGIN_DIR=/opt/nomad/plugins/ + +mkdir_for_root() { + sudo mkdir -p "$1" + sudo chmod 755 "$1" +} + +# Disable interactive apt prompts +export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections + +mkdir_for_root /opt +mkdir_for_root /opt/bin # for envoy +mkdir_for_root /srv/data # for host volumes +mkdir_for_root /opt/cni/bin +mkdir_for_root /opt/cni/config + +# Dependencies +sudo apt-get update +sudo apt-get upgrade -y +sudo apt-get install -y \ + software-properties-common \ + dnsmasq unzip tree redis-tools jq curl tmux awscli nfs-common \ + apt-transport-https ca-certificates gnupg2 stress + +# Install hc-install +curl -o /tmp/hc-install.zip https://releases.hashicorp.com/hc-install/0.9.0/hc-install_0.9.0_linux_arm64.zip +sudo unzip -d /usr/local/bin /tmp/hc-install.zip + +# Disable the firewall +sudo ufw disable || echo "ufw not installed" + +echo "Install HashiCorp apt repositories" +wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + +echo "Installing Docker apt repositories" +sudo install -m 0755 -d /etc/apt/keyrings +curl --insecure -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg +echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +echo "Refresh apt with third party repositories" +sudo apt-get update + +echo "Install Consul and Nomad" +sudo apt-get install -y \ + consul-enterprise \ + nomad + +# Note: neither service will start on boot because we haven't enabled +# the systemd unit file and we haven't uploaded any configuration +# files for Consul and Nomad + +echo "Configure Consul" +mkdir_for_root /etc/consul.d +mkdir_for_root /opt/consul +sudo mv /tmp/linux/consul.service /etc/systemd/system/consul.service + +echo "Configure Nomad" +mkdir_for_root /etc/nomad.d +mkdir_for_root /opt/nomad +mkdir_for_root $NOMAD_PLUGIN_DIR +sudo mv /tmp/linux/nomad.service /etc/systemd/system/nomad.service + +echo "Installing third-party tools" + +# Docker +echo "Installing Docker CE" +sudo apt-get install -y docker-ce docker-ce-cli + +# Java +echo "Installing Java" +sudo apt-get install -y openjdk-17-jdk-headless + +# CNI +echo "Installing CNI plugins" +wget -q -O - \ + https://github.com/containernetworking/plugins/releases/download/v1.0.0/cni-plugins-linux-arm64-v1.0.0.tgz \ + | sudo tar -C /opt/cni/bin -xz + +echo "Installing consul-cni plugin" +sudo hc-install install --path /opt/cni/bin --version 1.5.1 consul-cni + +echo "Installing custom test plugins" +# for .conf and .json config tests +sudo mv /tmp/linux/cni/loopback.* /opt/cni/config/ +# cni_args test plugin and network config +sudo mv /tmp/linux/cni/cni_args.conflist /opt/cni/config/ +sudo mv /tmp/linux/cni/cni_args.sh /opt/cni/bin/ + +# Podman +echo "Installing Podman" +sudo apt-get -y install podman catatonit + +echo "Installing Podman Driver" +sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version 0.5.0 nomad-driver-podman + +# Pledge +echo "Installing Pledge Driver" +curl -k -fsSL -o /tmp/pledge-driver.tar.gz https://github.com/shoenig/nomad-pledge-driver/releases/download/v0.3.0/nomad-pledge-driver_0.3.0_linux_amd64.tar.gz +curl -k -fsSL -o /tmp/pledge https://github.com/shoenig/nomad-pledge-driver/releases/download/pledge-1.8.com/pledge-1.8.com +tar -C /tmp -xf /tmp/pledge-driver.tar.gz +sudo mv /tmp/nomad-pledge-driver ${NOMAD_PLUGIN_DIR} +sudo mv /tmp/pledge /usr/local/bin +sudo chmod +x /usr/local/bin/pledge + +# Exec2 +echo "Installing Exec2 Driver" +sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version v0.1.0-alpha.2 nomad-driver-exec2 +sudo chmod +x ${NOMAD_PLUGIN_DIR}/nomad-driver-exec2 + +# Envoy +echo "Installing Envoy" +sudo curl -s -S -L -o /opt/bin/envoy https://github.com/envoyproxy/envoy/releases/download/v1.29.4/envoy-1.29.4-linux-x86_64 +sudo chmod +x /opt/bin/envoy + +# ECS +if [ -a "/tmp/linux/nomad-driver-ecs" ]; then + echo "Installing nomad-driver-ecs" + sudo install --mode=0755 --owner=ubuntu /tmp/linux/nomad-driver-ecs "$NOMAD_PLUGIN_DIR" +else + echo "nomad-driver-ecs not found: skipping install" +fi + +echo "Configuring dnsmasq" + +# disable systemd stub resolver +sudo sed -i 's|#DNSStubListener=yes|DNSStubListener=no|g' /etc/systemd/resolved.conf + +# disable systemd-resolved and configure dnsmasq to forward local requests to +# consul. the resolver files need to dynamic configuration based on the VPC +# address and docker bridge IP, so those will be rewritten at boot time. +sudo systemctl disable systemd-resolved.service +sudo systemctl stop systemd-resolved.service +sudo mv /tmp/linux/dnsmasq /etc/dnsmasq.d/default +sudo chown root:root /etc/dnsmasq.d/default + +# this is going to be overwritten at provisioning time, but we need something +# here or we can't fetch binaries to do the provisioning +echo 'nameserver 8.8.8.8' > /tmp/resolv.conf +sudo mv /tmp/resolv.conf /etc/resolv.conf + +sudo mv /tmp/linux/dnsmasq.service /etc/systemd/system/dnsmasq.service +sudo mv /tmp/linux/dnsconfig.sh /usr/local/bin/dnsconfig.sh +sudo chmod +x /usr/local/bin/dnsconfig.sh +sudo systemctl daemon-reload + +echo "Updating boot parameters" + +# enable cgroup_memory and swap +sudo sed -i 's/GRUB_CMDLINE_LINUX="[^"]*/& cgroup_enable=memory swapaccount=1/' /etc/default/grub +sudo update-grub \ No newline at end of file diff --git a/e2e/terraform/compute.tf b/e2e/terraform/provision-infra/compute.tf similarity index 84% rename from e2e/terraform/compute.tf rename to e2e/terraform/provision-infra/compute.tf index 9788ae17dbf..491f76d661a 100644 --- a/e2e/terraform/compute.tf +++ b/e2e/terraform/provision-infra/compute.tf @@ -3,6 +3,7 @@ locals { ami_prefix = "nomad-e2e-v3" + ubuntu_instance_name = "ubuntu-jammy-${var.instance_architecture}" } resource "aws_instance" "server" { @@ -22,18 +23,18 @@ resource "aws_instance" "server" { } } -resource "aws_instance" "client_ubuntu_jammy_amd64" { - ami = data.aws_ami.ubuntu_jammy_amd64.image_id +resource "aws_instance" "client_ubuntu_jammy" { + ami = data.aws_ami.ubuntu_jammy.image_id instance_type = var.instance_type key_name = module.keys.key_name vpc_security_group_ids = [aws_security_group.clients.id] # see also the secondary ENI - count = var.client_count_ubuntu_jammy_amd64 + count = var.client_count_linux iam_instance_profile = data.aws_iam_instance_profile.nomad_e2e_cluster.name availability_zone = var.availability_zone # Instance tags tags = { - Name = "${local.random_name}-client-ubuntu-jammy-amd64-${count.index}" + Name = "${local.random_name}-client-ubuntu-jammy-${count.index}" ConsulAutoJoin = "auto-join-${local.random_name}" User = data.aws_caller_identity.current.arn } @@ -100,10 +101,30 @@ data "aws_ami" "ubuntu_jammy_amd64" { values = ["Ubuntu"] } - filter { +/* filter { name = "tag:BuilderSha" values = [data.external.packer_sha.result["sha"]] + } */ +} + +data "aws_ami" "ubuntu_jammy" { + most_recent = true + owners = ["self"] + + filter { + name = "name" + values = ["${local.ami_prefix}-${local.ubuntu_instance_name}-*"] } + + filter { + name = "tag:OS" + values = ["Ubuntu"] + } + +/* filter { + name = "tag:BuilderSha" + values = [data.external.packer_sha.result["sha"]] + } */ } data "aws_ami" "windows_2016_amd64" { diff --git a/e2e/terraform/consul-clients.tf b/e2e/terraform/provision-infra/consul-clients.tf similarity index 83% rename from e2e/terraform/consul-clients.tf rename to e2e/terraform/provision-infra/consul-clients.tf index 2d6501c9600..d3fdfc762f1 100644 --- a/e2e/terraform/consul-clients.tf +++ b/e2e/terraform/provision-infra/consul-clients.tf @@ -35,12 +35,12 @@ resource "tls_locally_signed_cert" "consul_agents" { resource "local_sensitive_file" "consul_agents_key" { content = tls_private_key.consul_agents.private_key_pem - filename = "uploads/shared/consul.d/agent_cert.key.pem" + filename = "${path.module}/provision-nomad/uploads/shared/consul.d/agent_cert.key.pem" } resource "local_sensitive_file" "consul_agents_cert" { content = tls_locally_signed_cert.consul_agents.cert_pem - filename = "uploads/shared/consul.d/agent_cert.pem" + filename = "${path.module}/provision-nomad/uploads/shared/consul.d/agent_cert.pem" } # Consul tokens for the Consul agents @@ -52,7 +52,7 @@ resource "local_sensitive_file" "consul_agent_config_file" { token = "${random_uuid.consul_agent_token.result}" autojoin_value = "auto-join-${local.random_name}" }) - filename = "uploads/shared/consul.d/clients.hcl" + filename = "${path.module}/provision-nomad/uploads/shared/consul.d/clients.hcl" file_permission = "0600" } @@ -66,7 +66,7 @@ resource "local_sensitive_file" "nomad_client_config_for_consul" { client_service_name = "client-${local.random_name}" server_service_name = "server-${local.random_name}" }) - filename = "uploads/shared/nomad.d/client-consul.hcl" + filename = "${path.module}/provision-nomad/uploads/shared/nomad.d/client-consul.hcl" file_permission = "0600" } @@ -76,6 +76,6 @@ resource "local_sensitive_file" "nomad_server_config_for_consul" { client_service_name = "client-${local.random_name}" server_service_name = "server-${local.random_name}" }) - filename = "uploads/shared/nomad.d/server-consul.hcl" + filename = "${path.module}/provision-nomad/uploads/shared/nomad.d/server-consul.hcl" file_permission = "0600" } diff --git a/e2e/terraform/consul-servers.tf b/e2e/terraform/provision-infra/consul-servers.tf similarity index 85% rename from e2e/terraform/consul-servers.tf rename to e2e/terraform/provision-infra/consul-servers.tf index a0fbfdb98e2..62aa7b83b75 100644 --- a/e2e/terraform/consul-servers.tf +++ b/e2e/terraform/provision-infra/consul-servers.tf @@ -21,7 +21,7 @@ resource "local_sensitive_file" "consul_server_config_file" { nomad_token = "${random_uuid.consul_token_for_nomad.result}" autojoin_value = "auto-join-${local.random_name}" }) - filename = "uploads/shared/consul.d/servers.hcl" + filename = "${path.module}/provision-nomad/uploads/shared/consul.d/servers.hcl" file_permission = "0600" } @@ -59,12 +59,12 @@ resource "tls_locally_signed_cert" "consul_server" { resource "local_sensitive_file" "consul_server_key" { content = tls_private_key.consul_server.private_key_pem - filename = "uploads/shared/consul.d/server_cert.key.pem" + filename = "${path.module}/provision-nomad/uploads/shared/consul.d/server_cert.key.pem" } resource "local_sensitive_file" "consul_server_cert" { content = tls_locally_signed_cert.consul_server.cert_pem - filename = "uploads/shared/consul.d/server_cert.pem" + filename = "${path.module}/provision-nomad/uploads/shared/consul.d/server_cert.pem" } # if consul_license is unset, it'll be a harmless empty license file @@ -72,7 +72,7 @@ resource "local_sensitive_file" "consul_environment" { content = templatefile("${path.module}/provision-nomad/etc/consul.d/.environment", { license = var.consul_license }) - filename = "uploads/shared/consul.d/.environment" + filename = "${path.module}/provision-nomad/uploads/shared/consul.d/.environment" file_permission = "0600" } @@ -97,23 +97,23 @@ resource "null_resource" "upload_consul_server_configs" { } provisioner "file" { - source = "keys/tls_ca.crt" + source = "${path.root}/keys/tls_ca.crt" destination = "/tmp/consul_ca.pem" } provisioner "file" { - source = "uploads/shared/consul.d/.environment" + source = "${path.module}/provision-nomad/uploads/shared/consul.d/.environment" destination = "/tmp/.consul_environment" } provisioner "file" { - source = "uploads/shared/consul.d/server_cert.pem" + source = "${path.module}/provision-nomad/uploads/shared/consul.d/server_cert.pem" destination = "/tmp/consul_cert.pem" } provisioner "file" { - source = "uploads/shared/consul.d/server_cert.key.pem" + source = "${path.module}/provision-nomad/uploads/shared/consul.d/server_cert.key.pem" destination = "/tmp/consul_cert.key.pem" } provisioner "file" { - source = "uploads/shared/consul.d/servers.hcl" + source = "${path.module}/provision-nomad/uploads/shared/consul.d/servers.hcl" destination = "/tmp/consul_server.hcl" } provisioner "file" { @@ -166,10 +166,10 @@ resource "null_resource" "bootstrap_consul_acls" { depends_on = [null_resource.install_consul_server_configs] provisioner "local-exec" { - command = "./scripts/bootstrap-consul.sh" + command = "${path.module}/scripts/bootstrap-consul.sh" environment = { CONSUL_HTTP_ADDR = "https://${aws_instance.consul_server.public_ip}:8501" - CONSUL_CACERT = "keys/tls_ca.crt" + CONSUL_CACERT = "${path.root}/keys/tls_ca.crt" CONSUL_HTTP_TOKEN = "${random_uuid.consul_initial_management_token.result}" CONSUL_AGENT_TOKEN = "${random_uuid.consul_agent_token.result}" NOMAD_CLUSTER_CONSUL_TOKEN = "${random_uuid.consul_token_for_nomad.result}" diff --git a/e2e/terraform/ecs-task.json b/e2e/terraform/provision-infra/ecs-task.json similarity index 100% rename from e2e/terraform/ecs-task.json rename to e2e/terraform/provision-infra/ecs-task.json diff --git a/e2e/terraform/ecs.tf b/e2e/terraform/provision-infra/ecs.tf similarity index 100% rename from e2e/terraform/ecs.tf rename to e2e/terraform/provision-infra/ecs.tf diff --git a/e2e/terraform/ecs.tftpl b/e2e/terraform/provision-infra/ecs.tftpl similarity index 100% rename from e2e/terraform/ecs.tftpl rename to e2e/terraform/provision-infra/ecs.tftpl diff --git a/e2e/terraform/hcp_vault.tf b/e2e/terraform/provision-infra/hcp_vault.tf similarity index 95% rename from e2e/terraform/hcp_vault.tf rename to e2e/terraform/provision-infra/hcp_vault.tf index 4bfbe9efcb3..cd71f497c40 100644 --- a/e2e/terraform/hcp_vault.tf +++ b/e2e/terraform/provision-infra/hcp_vault.tf @@ -48,6 +48,6 @@ resource "local_sensitive_file" "nomad_config_for_vault" { namespace = var.hcp_vault_namespace role = "nomad-tasks-${local.random_name}" }) - filename = "uploads/shared/nomad.d/vault.hcl" + filename = "${path.module}/provision-nomad/uploads/shared/nomad.d/vault.hcl" file_permission = "0600" } diff --git a/e2e/terraform/iam.tf b/e2e/terraform/provision-infra/iam.tf similarity index 100% rename from e2e/terraform/iam.tf rename to e2e/terraform/provision-infra/iam.tf diff --git a/e2e/terraform/provision-infra/main.tf b/e2e/terraform/provision-infra/main.tf new file mode 100644 index 00000000000..64e52be0848 --- /dev/null +++ b/e2e/terraform/provision-infra/main.tf @@ -0,0 +1,30 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +data "aws_caller_identity" "current" { +} + +resource "random_pet" "e2e" { +} + +resource "random_password" "windows_admin_password" { + length = 20 + special = true + override_special = "_%@" +} + +locals { + random_name = "${var.name}-${random_pet.e2e.id}" +} + +# Generates keys to use for provisioning and access +module "keys" { + name = local.random_name + path = "${path.root}/keys" + source = "mitchellh/dynamic-keys/aws" + version = "v2.0.0" +} + +data "aws_kms_alias" "e2e" { + name = "alias/${var.aws_kms_alias}" +} diff --git a/e2e/terraform/network.tf b/e2e/terraform/provision-infra/network.tf similarity index 97% rename from e2e/terraform/network.tf rename to e2e/terraform/provision-infra/network.tf index 79330e0aab8..774da56bffb 100644 --- a/e2e/terraform/network.tf +++ b/e2e/terraform/provision-infra/network.tf @@ -207,9 +207,9 @@ resource "aws_network_interface" "clients_secondary" { subnet_id = data.aws_subnet.secondary.id security_groups = [aws_security_group.clients_secondary.id] - count = var.client_count_ubuntu_jammy_amd64 + count = var.client_count_linux attachment { - instance = aws_instance.client_ubuntu_jammy_amd64[count.index].id + instance = aws_instance.client_ubuntu_jammy[count.index].id device_index = 1 } } diff --git a/e2e/terraform/nomad-acls.tf b/e2e/terraform/provision-infra/nomad-acls.tf similarity index 89% rename from e2e/terraform/nomad-acls.tf rename to e2e/terraform/provision-infra/nomad-acls.tf index b5cce557f3e..5e106ac58a5 100644 --- a/e2e/terraform/nomad-acls.tf +++ b/e2e/terraform/provision-infra/nomad-acls.tf @@ -14,16 +14,16 @@ resource "null_resource" "bootstrap_nomad_acls" { command = "./scripts/bootstrap-nomad.sh" environment = { NOMAD_ADDR = "https://${aws_instance.server.0.public_ip}:4646" - NOMAD_CACERT = "keys/tls_ca.crt" - NOMAD_CLIENT_CERT = "keys/tls_api_client.crt" - NOMAD_CLIENT_KEY = "keys/tls_api_client.key" + NOMAD_CACERT = "${path.root}/keys/tls_ca.crt" + NOMAD_CLIENT_CERT = "${path.root}/keys/tls_api_client.crt" + NOMAD_CLIENT_KEY = "${path.root}/keys/tls_api_client.key" } } } data "local_sensitive_file" "nomad_token" { depends_on = [null_resource.bootstrap_nomad_acls] - filename = "${path.module}/keys/nomad_root_token" + filename = "${path.root}/keys/nomad_root_token" } # push the token out to the servers for humans to use. diff --git a/e2e/terraform/nomad.tf b/e2e/terraform/provision-infra/nomad.tf similarity index 91% rename from e2e/terraform/nomad.tf rename to e2e/terraform/provision-infra/nomad.tf index 0c7719b9362..03ddcb25f06 100644 --- a/e2e/terraform/nomad.tf +++ b/e2e/terraform/provision-infra/nomad.tf @@ -31,16 +31,16 @@ module "nomad_server" { # TODO: split out the different Linux targets (ubuntu, centos, arm, etc.) when # they're available -module "nomad_client_ubuntu_jammy_amd64" { +module "nomad_client_ubuntu_jammy" { source = "./provision-nomad" - depends_on = [aws_instance.client_ubuntu_jammy_amd64] - count = var.client_count_ubuntu_jammy_amd64 + depends_on = [aws_instance.client_ubuntu_jammy] + count = var.client_count_linux platform = "linux" arch = "linux_amd64" role = "client" index = count.index - instance = aws_instance.client_ubuntu_jammy_amd64[count.index] + instance = aws_instance.client_ubuntu_jammy[count.index] nomad_local_binary = count.index < length(var.nomad_local_binary_client_ubuntu_jammy_amd64) ? var.nomad_local_binary_client_ubuntu_jammy_amd64[count.index] : var.nomad_local_binary diff --git a/e2e/terraform/provision-infra/outputs.tf b/e2e/terraform/provision-infra/outputs.tf new file mode 100644 index 00000000000..c50436ef857 --- /dev/null +++ b/e2e/terraform/provision-infra/outputs.tf @@ -0,0 +1,61 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +output "servers" { + value = aws_instance.server.*.public_ip +} + +output "linux_clients" { + value = aws_instance.client_ubuntu_jammy.*.public_ip +} + +output "windows_clients" { + value = aws_instance.client_windows_2016_amd64.*.public_ip +} + +output "message" { + value = <&2 + exit 1 +fi + +# Create the BINARY_PATH directory +mkdir -p "$BINARY_PATH" + +# Unzip the file +unzip -o "$LOCAL_ZIP" -d "$BINARY_PATH" + +# Check if the file was unzipped +if [ $? -eq 0 ]; then + echo "File unzipped successfully to $BINARY_PATH" +else + echo "Error unzipping file." >&2 + exit 1 +fi + +# Remove the zipped file +rm "$LOCAL_ZIP" \ No newline at end of file diff --git a/enos/modules/fetch_artifactory/variables.tf b/enos/modules/fetch_artifactory/variables.tf new file mode 100644 index 00000000000..b1438adcc76 --- /dev/null +++ b/enos/modules/fetch_artifactory/variables.tf @@ -0,0 +1,55 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +variable "artifactory_username" { + type = string + description = "The username to use when connecting to artifactory" + default = null +} + +variable "artifactory_token" { + type = string + description = "The token to use when connecting to artifactory" + default = null + sensitive = true +} + +variable "artifactory_host" { + type = string + description = "The artifactory host to search for Nomad artifacts" + default = "https://artifactory.hashicorp.engineering/artifactory" +} + +variable "artifactory_repo" { + type = string + description = "The artifactory repo to search for Nomad artifacts" + default = "hashicorp-crt-staging-local*" +} + +variable "edition" { + type = string + description = "The edition of the binary to search, it can be either CE or ENT" +} + +variable "os" { + type = string + description = "The operative system the binary is needed for" + default = "linux" +} + +variable "product_version" { + description = "The version of Nomad we are testing" + type = string + default = null +} + +variable "arch" { + description = "The artifactory path to search for Nomad artifacts" + type = string +} + +variable "binary_path" { + description = "The path to donwload and unzip the binary" + type = string + default = "/home/ubuntu/nomad" +}