diff --git a/e2e/terraform/Makefile b/e2e/terraform/Makefile index 10b4eda7884..14d7124c2a4 100644 --- a/e2e/terraform/Makefile +++ b/e2e/terraform/Makefile @@ -7,7 +7,7 @@ CONSUL_LICENSE_PATH ?= custom.tfvars: echo 'nomad_local_binary = "$(PKG_PATH)"' > custom.tfvars echo 'volumes = false' >> custom.tfvars - echo 'client_count_ubuntu_jammy_amd64 = 3' >> custom.tfvars + echo 'client_count_linux = 3' >> custom.tfvars echo 'client_count_windows_2016_amd64 = 0' >> custom.tfvars echo 'consul_license = "$(shell cat $(CONSUL_LICENSE_PATH))"' >> custom.tfvars echo 'nomad_license = "$(shell cat $(NOMAD_LICENSE_PATH))"' >> custom.tfvars diff --git a/e2e/terraform/README.md b/e2e/terraform/README.md index a5785654557..12a9e119628 100644 --- a/e2e/terraform/README.md +++ b/e2e/terraform/README.md @@ -51,7 +51,7 @@ Linux clients or Windows clients. region = "us-east-1" instance_type = "t2.medium" server_count = "3" -client_count_ubuntu_jammy_amd64 = "4" +client_count_linux = "4" client_count_windows_2016_amd64 = "1" ``` diff --git a/e2e/terraform/compute.tf b/e2e/terraform/compute.tf index af4254825a9..675e8b3c0db 100644 --- a/e2e/terraform/compute.tf +++ b/e2e/terraform/compute.tf @@ -3,6 +3,7 @@ locals { ami_prefix = "nomad-e2e-v3" + ubuntu_instance_name = "ubuntu-jammy-${var.instance_architecture}" } resource "aws_instance" "server" { @@ -22,18 +23,18 @@ resource "aws_instance" "server" { } } -resource "aws_instance" "client_ubuntu_jammy_amd64" { - ami = data.aws_ami.ubuntu_jammy_amd64.image_id +resource "aws_instance" "client_ubuntu_jammy" { + ami = data.aws_ami.ubuntu_jammy.image_id instance_type = var.instance_type key_name = module.keys.key_name vpc_security_group_ids = [aws_security_group.clients.id] # see also the secondary ENI - count = var.client_count_ubuntu_jammy_amd64 + count = var.client_count_linux iam_instance_profile = data.aws_iam_instance_profile.nomad_e2e_cluster.name availability_zone = var.availability_zone # Instance tags tags = { - Name = "${local.random_name}-client-ubuntu-jammy-amd64-${count.index}" + Name = "${local.random_name}-client-ubuntu-jammy-${count.index}" ConsulAutoJoin = "auto-join-${local.random_name}" User = data.aws_caller_identity.current.arn } @@ -102,6 +103,29 @@ data "aws_ami" "ubuntu_jammy_amd64" { name = "tag:BuilderSha" values = [data.external.packer_sha.result["sha"]] } +<<<<<<< Updated upstream +======= +} + +data "aws_ami" "ubuntu_jammy" { + most_recent = true + owners = ["self"] + + filter { + name = "name" + values = ["${local.ami_prefix}-${ubuntu_instance_name}-*"] + } + + filter { + name = "tag:OS" + values = ["Ubuntu"] + } + + filter { + name = "tag:BuilderSha" + values = [data.external.packer_sha.result["sha"]] + } +>>>>>>> Stashed changes } data "aws_ami" "windows_2016_amd64" { diff --git a/e2e/terraform/network.tf b/e2e/terraform/network.tf index 79330e0aab8..774da56bffb 100644 --- a/e2e/terraform/network.tf +++ b/e2e/terraform/network.tf @@ -207,9 +207,9 @@ resource "aws_network_interface" "clients_secondary" { subnet_id = data.aws_subnet.secondary.id security_groups = [aws_security_group.clients_secondary.id] - count = var.client_count_ubuntu_jammy_amd64 + count = var.client_count_linux attachment { - instance = aws_instance.client_ubuntu_jammy_amd64[count.index].id + instance = aws_instance.client_ubuntu_jammy[count.index].id device_index = 1 } } diff --git a/e2e/terraform/nomad.tf b/e2e/terraform/nomad.tf index 0c7719b9362..03ddcb25f06 100644 --- a/e2e/terraform/nomad.tf +++ b/e2e/terraform/nomad.tf @@ -31,16 +31,16 @@ module "nomad_server" { # TODO: split out the different Linux targets (ubuntu, centos, arm, etc.) when # they're available -module "nomad_client_ubuntu_jammy_amd64" { +module "nomad_client_ubuntu_jammy" { source = "./provision-nomad" - depends_on = [aws_instance.client_ubuntu_jammy_amd64] - count = var.client_count_ubuntu_jammy_amd64 + depends_on = [aws_instance.client_ubuntu_jammy] + count = var.client_count_linux platform = "linux" arch = "linux_amd64" role = "client" index = count.index - instance = aws_instance.client_ubuntu_jammy_amd64[count.index] + instance = aws_instance.client_ubuntu_jammy[count.index] nomad_local_binary = count.index < length(var.nomad_local_binary_client_ubuntu_jammy_amd64) ? var.nomad_local_binary_client_ubuntu_jammy_amd64[count.index] : var.nomad_local_binary diff --git a/e2e/terraform/outputs.tf b/e2e/terraform/outputs.tf index 43d760c0393..be5f5c5225e 100644 --- a/e2e/terraform/outputs.tf +++ b/e2e/terraform/outputs.tf @@ -6,7 +6,7 @@ output "servers" { } output "linux_clients" { - value = aws_instance.client_ubuntu_jammy_amd64.*.public_ip + value = aws_instance.client_ubuntu_jammy.*.public_ip } output "windows_clients" { @@ -31,7 +31,7 @@ ssh into servers with: ssh into clients with: -%{for ip in aws_instance.client_ubuntu_jammy_amd64.*.public_ip~} +%{for ip in aws_instance.client_ubuntu_jammy.*.public_ip~} ssh -i keys/${local.random_name}.pem ubuntu@${ip} %{endfor~} %{for ip in aws_instance.client_windows_2016_amd64.*.public_ip~} diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64.pkr.hcl b/e2e/terraform/packer/ubuntu-jammy-arm64.pkr.hcl new file mode 100644 index 00000000000..3af70cc8262 --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64.pkr.hcl @@ -0,0 +1,59 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +variable "build_sha" { + type = string + description = "the revision of the packer scripts building this image" +} + +locals { + timestamp = regex_replace(timestamp(), "[- TZ:]", "") + distro = "ubuntu-jammy-22.04-arm64-server-*" + version = "v3" +} + +source "amazon-ebs" "latest_ubuntu_jammy" { + ami_name = "nomad-e2e-${local.version}-ubuntu-jammy-arm64-${local.timestamp}" + iam_instance_profile = "packer_build" // defined in nomad-e2e repo + instance_type = "m7a.large" + region = "us-east-1" + ssh_username = "ubuntu" + ssh_interface = "public_ip" + + source_ami_filter { + filters = { + architecture = "arm64" + "block-device-mapping.volume-type" = "gp2" + name = "ubuntu/images/hvm-ssd/${local.distro}" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] // Canonical + } + + tags = { + OS = "Ubuntu" + Version = "Jammy" + BuilderSha = var.build_sha + } +} + +build { + sources = ["source.amazon-ebs.latest_ubuntu_jammy"] + + provisioner "file" { + destination = "/tmp/linux" + source = "./ubuntu-jammy-arm64" + } + + // cloud-init modifies the apt sources, so we need to wait + // before running our setup + provisioner "shell-local" { + inline = ["sleep 30"] + } + + provisioner "shell" { + script = "./ubuntu-jammy-arm64/setup.sh" + } +} diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/cni/cni_args.conflist b/e2e/terraform/packer/ubuntu-jammy-arm64/cni/cni_args.conflist new file mode 100644 index 00000000000..ae119111d78 --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/cni/cni_args.conflist @@ -0,0 +1,9 @@ +{ + "cniVersion": "1.0.0", + "name": "cni_args", + "plugins": [ + { + "type": "cni_args.sh" + } + ] +} diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/cni/cni_args.sh b/e2e/terraform/packer/ubuntu-jammy-arm64/cni/cni_args.sh new file mode 100755 index 00000000000..bcdf0d05153 --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/cni/cni_args.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -euo pipefail + +# things are prefixed with "Fancy*" because this is a fancy plugin. +# CNI_ARGS='IgnoreUnknown=true;FancyTaskDir=/tmp/cni_args;FancyMessage=hiiii;Another=whatever' +# what we need to do: +# 1. read CNI_ARGS environment variable +# * write to a file named $FancyTaskDir/victory +# 2. write CNI-spec json to stdout for Nomad to read + +# https://github.com/containernetworking/cni/blob/main/SPEC.md#version-success +function version() { + cat <&2 echo "got task dir: $task_dir; message: $message" + + mkdir -p "$task_dir" + # and write something to a file we can check in the test. + echo "$message" > "$task_dir/victory" +} + +# run the appropriate CNI command +case "$CNI_COMMAND" in + VERSION) version ; exit ;; + ADD) add ;; +esac + +# bogus reply so nomad doesn't error +cat < /tmp/resolv.conf +nameserver 127.0.0.1 +nameserver $DOCKER_BRIDGE_IP_ADDRESS +EOF +cp /tmp/resolv.conf /etc/resolv.conf + +# need to get the interface for dnsmasq config so that we can +# accomodate both "predictable" and old-style interface names +IFACE=$(ip route | grep default | awk '{print $5}') + +cat < /tmp/dnsmasq +port=53 +resolv-file=/var/run/dnsmasq/resolv.conf +bind-interfaces +interface=docker0 +interface=lo +interface=$IFACE +listen-address=127.0.0.1 +server=/consul/127.0.0.1#8600 +EOF +cp /tmp/dnsmasq /etc/dnsmasq.d/default + +# need to get the AWS DNS address from the VPC... +# this is pretty hacky but will work for any typical case +MAC=$(curl -s --fail http://169.254.169.254/latest/meta-data/mac) +CIDR_BLOCK=$(curl -s --fail "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC/vpc-ipv4-cidr-block") +VPC_DNS_ROOT=$(echo "$CIDR_BLOCK" | cut -d'.' -f1-3) +echo "nameserver ${VPC_DNS_ROOT}.2" > /tmp/dnsmasq-resolv.conf +cp /tmp/dnsmasq-resolv.conf /var/run/dnsmasq/resolv.conf + +/usr/sbin/dnsmasq --test diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq b/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq new file mode 100644 index 00000000000..42b06f6e5b8 --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq @@ -0,0 +1,8 @@ +port=53 +resolv-file=/var/run/dnsmasq/resolv.conf +bind-interfaces +interface=docker0 +interface=lo +interface=eth0 +listen-address=127.0.0.1 +server=/consul/127.0.0.1#8600 diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq.service b/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq.service new file mode 100644 index 00000000000..93b7c97e3f5 --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/dnsmasq.service @@ -0,0 +1,37 @@ +[Unit] +Description=dnsmasq - A lightweight DHCP and caching DNS server +Requires=network.target +Wants=nss-lookup.target +Before=nss-lookup.target +After=network.target +After=docker.service + +[Service] +Type=forking +PIDFile=/run/dnsmasq/dnsmasq.pid + +# Configure our hosts and resolver file with info from the host, +# then test the resulting config file before starting +ExecStartPre=/usr/local/bin/dnsconfig.sh + +# (from upstream) +# We run dnsmasq via the /etc/init.d/dnsmasq script which acts as a +# wrapper picking up extra configuration files and then execs dnsmasq +# itself, when called with the "systemd-exec" function. +ExecStart=/etc/init.d/dnsmasq systemd-exec + +# (from upstream) +# The systemd-*-resolvconf functions configure (and deconfigure) +# resolvconf to work with the dnsmasq DNS server. They're called like +# this to get correct error handling (ie don't start-resolvconf if the +# dnsmasq daemon fails to start. +ExecStartPost=/etc/init.d/dnsmasq systemd-start-resolvconf + +# We need to tell docker to pick up the changes +ExecStartPost=/bin/systemctl restart docker + +ExecStop=/etc/init.d/dnsmasq systemd-stop-resolvconf +ExecReload=/bin/kill -HUP $MAINPID + +[Install] +WantedBy=multi-user.target diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/nomad.service b/e2e/terraform/packer/ubuntu-jammy-arm64/nomad.service new file mode 100644 index 00000000000..8490fc9c8e8 --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/nomad.service @@ -0,0 +1,21 @@ +[Unit] +Description=Nomad Agent +Requires=network-online.target +After=network-online.target +StartLimitIntervalSec=0 +StartLimitBurst=3 + +[Service] +ExecReload=/bin/kill -HUP $MAINPID +ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d +EnvironmentFile=-/etc/nomad.d/.environment +KillMode=process +KillSignal=SIGINT +LimitNOFILE=65536 +LimitNPROC=infinity +TasksMax=infinity +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/e2e/terraform/packer/ubuntu-jammy-arm64/setup.sh b/e2e/terraform/packer/ubuntu-jammy-arm64/setup.sh new file mode 100755 index 00000000000..4129326d18b --- /dev/null +++ b/e2e/terraform/packer/ubuntu-jammy-arm64/setup.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# setup script for Ubuntu Linux 22.04. Assumes that Packer has placed +# build-time config files at /tmp/linux + +set -xeuo pipefail + +NOMAD_PLUGIN_DIR=/opt/nomad/plugins/ + +mkdir_for_root() { + sudo mkdir -p "$1" + sudo chmod 755 "$1" +} + +# Disable interactive apt prompts +export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections + +mkdir_for_root /opt +mkdir_for_root /opt/bin # for envoy +mkdir_for_root /srv/data # for host volumes +mkdir_for_root /opt/cni/bin +mkdir_for_root /opt/cni/config + +# Dependencies +sudo apt-get update +sudo apt-get upgrade -y +sudo apt-get install -y \ + software-properties-common \ + dnsmasq unzip tree redis-tools jq curl tmux awscli nfs-common \ + apt-transport-https ca-certificates gnupg2 stress + +# Install hc-install +curl -o /tmp/hc-install.zip https://releases.hashicorp.com/hc-install/0.9.0/hc-install_0.9.0_linux_arm64.zip +sudo unzip -d /usr/local/bin /tmp/hc-install.zip + +# Disable the firewall +sudo ufw disable || echo "ufw not installed" + +echo "Install HashiCorp apt repositories" +wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + +echo "Installing Docker apt repositories" +sudo install -m 0755 -d /etc/apt/keyrings +curl --insecure -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg +echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +echo "Refresh apt with third party repositories" +sudo apt-get update + +echo "Install Consul and Nomad" +sudo apt-get install -y \ + consul-enterprise \ + nomad + +# Note: neither service will start on boot because we haven't enabled +# the systemd unit file and we haven't uploaded any configuration +# files for Consul and Nomad + +echo "Configure Consul" +mkdir_for_root /etc/consul.d +mkdir_for_root /opt/consul +sudo mv /tmp/linux/consul.service /etc/systemd/system/consul.service + +echo "Configure Nomad" +mkdir_for_root /etc/nomad.d +mkdir_for_root /opt/nomad +mkdir_for_root $NOMAD_PLUGIN_DIR +sudo mv /tmp/linux/nomad.service /etc/systemd/system/nomad.service + +echo "Installing third-party tools" + +# Docker +echo "Installing Docker CE" +sudo apt-get install -y docker-ce docker-ce-cli + +# Java +echo "Installing Java" +sudo apt-get install -y openjdk-17-jdk-headless + +# CNI +echo "Installing CNI plugins" +wget -q -O - \ + https://github.com/containernetworking/plugins/releases/download/v1.0.0/cni-plugins-linux-arm64-v1.0.0.tgz \ + | sudo tar -C /opt/cni/bin -xz + +echo "Installing consul-cni plugin" +sudo hc-install install --path /opt/cni/bin --version 1.5.1 consul-cni + +echo "Installing custom test plugins" +# for .conf and .json config tests +sudo mv /tmp/linux/cni/loopback.* /opt/cni/config/ +# cni_args test plugin and network config +sudo mv /tmp/linux/cni/cni_args.conflist /opt/cni/config/ +sudo mv /tmp/linux/cni/cni_args.sh /opt/cni/bin/ + +# Podman +echo "Installing Podman" +sudo apt-get -y install podman catatonit + +echo "Installing Podman Driver" +sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version 0.5.0 nomad-driver-podman + +# Pledge +echo "Installing Pledge Driver" +curl -k -fsSL -o /tmp/pledge-driver.tar.gz https://github.com/shoenig/nomad-pledge-driver/releases/download/v0.3.0/nomad-pledge-driver_0.3.0_linux_amd64.tar.gz +curl -k -fsSL -o /tmp/pledge https://github.com/shoenig/nomad-pledge-driver/releases/download/pledge-1.8.com/pledge-1.8.com +tar -C /tmp -xf /tmp/pledge-driver.tar.gz +sudo mv /tmp/nomad-pledge-driver ${NOMAD_PLUGIN_DIR} +sudo mv /tmp/pledge /usr/local/bin +sudo chmod +x /usr/local/bin/pledge + +# Exec2 +echo "Installing Exec2 Driver" +sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version v0.1.0-alpha.2 nomad-driver-exec2 +sudo chmod +x ${NOMAD_PLUGIN_DIR}/nomad-driver-exec2 + +# Envoy +echo "Installing Envoy" +sudo curl -s -S -L -o /opt/bin/envoy https://github.com/envoyproxy/envoy/releases/download/v1.29.4/envoy-1.29.4-linux-x86_64 +sudo chmod +x /opt/bin/envoy + +# ECS +if [ -a "/tmp/linux/nomad-driver-ecs" ]; then + echo "Installing nomad-driver-ecs" + sudo install --mode=0755 --owner=ubuntu /tmp/linux/nomad-driver-ecs "$NOMAD_PLUGIN_DIR" +else + echo "nomad-driver-ecs not found: skipping install" +fi + +echo "Configuring dnsmasq" + +# disable systemd stub resolver +sudo sed -i 's|#DNSStubListener=yes|DNSStubListener=no|g' /etc/systemd/resolved.conf + +# disable systemd-resolved and configure dnsmasq to forward local requests to +# consul. the resolver files need to dynamic configuration based on the VPC +# address and docker bridge IP, so those will be rewritten at boot time. +sudo systemctl disable systemd-resolved.service +sudo systemctl stop systemd-resolved.service +sudo mv /tmp/linux/dnsmasq /etc/dnsmasq.d/default +sudo chown root:root /etc/dnsmasq.d/default + +# this is going to be overwritten at provisioning time, but we need something +# here or we can't fetch binaries to do the provisioning +echo 'nameserver 8.8.8.8' > /tmp/resolv.conf +sudo mv /tmp/resolv.conf /etc/resolv.conf + +sudo mv /tmp/linux/dnsmasq.service /etc/systemd/system/dnsmasq.service +sudo mv /tmp/linux/dnsconfig.sh /usr/local/bin/dnsconfig.sh +sudo chmod +x /usr/local/bin/dnsconfig.sh +sudo systemctl daemon-reload + +echo "Updating boot parameters" + +# enable cgroup_memory and swap +sudo sed -i 's/GRUB_CMDLINE_LINUX="[^"]*/& cgroup_enable=memory swapaccount=1/' /etc/default/grub +sudo update-grub \ No newline at end of file diff --git a/e2e/terraform/tls_client.tf b/e2e/terraform/tls_client.tf index 9a5e48c3f70..65cb6a250b4 100644 --- a/e2e/terraform/tls_client.tf +++ b/e2e/terraform/tls_client.tf @@ -56,7 +56,7 @@ resource "tls_self_signed_cert" "self_signed" { organization = "HashiCorp, Inc." } - ip_addresses = toset(aws_instance.client_ubuntu_jammy_amd64.*.public_ip) + ip_addresses = toset(aws_instance.client_ubuntu_jammy.*.public_ip) validity_period_hours = 720 allowed_uses = [ diff --git a/e2e/terraform/variables.tf b/e2e/terraform/variables.tf index ca2a64fd6f4..a14cc0f7a78 100644 --- a/e2e/terraform/variables.tf +++ b/e2e/terraform/variables.tf @@ -21,12 +21,17 @@ variable "instance_type" { default = "t3a.medium" } +variable "instance_architecture" { + description = "The architecture for the AWS instance type to use for both clients and servers." + default = "amd64" +} + variable "server_count" { description = "The number of servers to provision." default = "3" } -variable "client_count_ubuntu_jammy_amd64" { +variable "client_count_linux" { description = "The number of Ubuntu clients to provision." default = "4" } diff --git a/enos/enos-quality.hcl b/enos/enos-quality.hcl index eeb8dfeced2..6e160dd4dd6 100644 --- a/enos/enos-quality.hcl +++ b/enos/enos-quality.hcl @@ -13,6 +13,14 @@ quality "nomad_nodes_status" { description = "A GET call to /v1/nodes returns the correct number of clients and they are all eligible and ready" } +quality "nomad_node_eligibility" { + description = "A GET call to /v1/node/:node-id returns the same node.SchedulingEligibility before and after a server upgrade" +} + +quality "nomad_node_metadata" { + description = "A GET call to /v1/node/:node-id returns the same node.Meta for each server before and after a server upgrade" +} + quality "nomad_job_status" { description = "A GET call to /v1/jobs returns the correct number of jobs and they are all running" } @@ -25,7 +33,6 @@ quality "nomad_reschedule_alloc" { description = "A POST / PUT call to /v1/allocation/:alloc_id/stop results in the stopped allocation being rescheduled" } - quality "nomad_restore_snapshot" { description = "A node can be restored from a snapshot built on a previous version" } @@ -33,3 +40,8 @@ quality "nomad_restore_snapshot" { quality "nomad_allocs_status" { description = "A GET call to /v1/allocs returns the correct number of allocations and they are all running" } + +quality "nomad_alloc_reconect" { + description = "A GET call to /v1/alloc/:alloc_id will return the same alloc.CreateTime for each allocation before and after a client upgrade" +} + diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index 2fc4919ed04..1fde2b60ef1 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -8,59 +8,65 @@ scenario "upgrade" { EOF matrix { - arch = ["amd64", "arm64"] + arch = ["amd64"] + #arch = ["amd64", "arm64"] //service_discovery = ["consul", "nomad"] - edition = ["ce", "ent"] - os = ["linux", "windows"] + #edition = ["ce", "ent"] + edition = ["ce"] + os = ["linux"] + #os = ["linux", "windows"] - exclude { + /* exclude { os = ["windows"] arch = ["arm64"] - } + } */ } locals { - cluster_name = "upgrade-testing-cluster-${matrix.os}-${matrix.arch}" - ubuntu_count = matrix.os == "linux" ? 4 : 0 + cluster_name = "upgrade-testing-cluster-${matrix.os}-${matrix.arch}-${matrix.edition}" + linux_count = matrix.os == "linux" ? 4 : 0 windows_count = matrix.os == "windows" ? 4 : 0 + arch = matrix.arch } - step "copy_initial_binary" { + step "copy_initial_binary" { description = <<-EOF Determine which Nomad artifact we want to use for the scenario, depending on the 'arch', 'edition' and 'os' EOF + module = module.build_artifactory variables { artifactory_username = var.artifactory_username artifactory_token = var.artifactory_token - arch = matrix.arch + arch = local.arch edition = matrix.edition product_version = var.product_version os = matrix.os - local_path = var.binary_local_path + binary_path = "${var.nomad_local_binary}/${matrix.os}-${matrix.arch}-${matrix.edition}/nomad" } - } + } - step "provision_cluster" { + /* step "provision_cluster" { + // depends_on = [step.copy_initial_binary] description = <<-EOF Using the binary from the previous step, provision a Nomad cluster using the e2e EOF - depends_on = [step.copy_initial_binary] - module = module.provision_cluster variables { name = local.cluster_name nomad_local_binary = step.copy_initial_binary.nomad_local_binary - client_count_ubuntu_jammy_amd64 = local.ubuntu_count + client_count_linux = local.linux_count client_count_windows_2016_amd64 = local.windows_count nomad_license = var.nomad_license consul_license = var.consul_license - volumes = false + volumes = false + instance_architecture = matrix.arch } } + */ /* step "run_new_workloads" { description = <<-EOF diff --git a/enos/enos-vars.hcl b/enos/enos-vars.hcl index fdbda644b9e..9e51ec53342 100644 --- a/enos/enos-vars.hcl +++ b/enos/enos-vars.hcl @@ -29,7 +29,7 @@ variable "binary_local_path" { # Variables for the provision_cluster module variable "nomad_local_binary" { description = "The path to a local binary to provision" - default = "" + #default = "/Users/juanita.delacuestamorales/nomad" #TODO! } variable "nomad_license" { diff --git a/enos/modules/fetch_artifactory/locals.tf b/enos/modules/fetch_artifactory/locals.tf index 7f4b7492fc0..a93d74fb002 100644 --- a/enos/modules/fetch_artifactory/locals.tf +++ b/enos/modules/fetch_artifactory/locals.tf @@ -19,4 +19,5 @@ locals { } artifact_name = "nomad_${local.artifact_version}${local.package_extensions[var.arch][var.os]}" + artifact_zip = "nomad_${local.artifact_version}${local.package_extensions[var.arch][var.os]}.zip" } \ No newline at end of file diff --git a/enos/modules/fetch_artifactory/main.tf b/enos/modules/fetch_artifactory/main.tf index b413f6b238b..f4e045031ad 100644 --- a/enos/modules/fetch_artifactory/main.tf +++ b/enos/modules/fetch_artifactory/main.tf @@ -26,6 +26,7 @@ resource "enos_local_exec" "install_binary" { URL = data.enos_artifactory_item.nomad.results[0].url BINARY_PATH = var.binary_path TOKEN = var.artifactory_token + LOCAL_ZIP = local.artifact_zip } scripts = [abspath("${path.module}/scripts/install.sh")] diff --git a/enos/modules/fetch_artifactory/outputs.tf b/enos/modules/fetch_artifactory/outputs.tf index e335d514d3a..7d2259ea418 100644 --- a/enos/modules/fetch_artifactory/outputs.tf +++ b/enos/modules/fetch_artifactory/outputs.tf @@ -1,8 +1,8 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 -output "local_binary" { - value = var.binary_path +output "nomad_local_binary" { + value = "${var.binary_path}/${var.os}-${var.arch}-${var.edition}/nomad" description = "Path where the binary will be placed" } @@ -12,4 +12,5 @@ output "vault_artifactory_release" { url = data.enos_artifactory_item.nomad.results[0].url sha256 = data.enos_artifactory_item.nomad.results[0].sha256 } -} \ No newline at end of file +} +