vSphere ubuntu (#59)

* add vsphere init * reuse existing scritps * add persistent admin user * add vm template post processor * selinux fixes, rhel8/9 flexibility, config updates * add FIPS and skip chown of non-existent dirs on agent nodes * remove misleading variables from example vars * remove unused vars * add ubuntu * add vars examples, clean up cloudinit * update expample vars, handle ubuntu version for stigs, add root login allowed for provisioning, update ssh timeout, cloud-init boot fix * remove old example file, remove http_ks * add clean cloud init to vsphere * Whitespace and tweaking default variables. * add retires for install-rke2 * add retries to aws to resolve race condition * diable package updates and upgrades as part of cloudinit --------- Co-authored-by: Justin Oursler <[email protected]>
defenseunicorns · Aug 28, 2024 · 2fecc1c · 2fecc1c
1 parent d1f109e
commit 2fecc1c
Show file tree

Hide file tree

Showing 10 changed files with 132 additions and 36 deletions.
diff --git a/packer/aws/aws.pkr.hcl b/packer/aws/aws.pkr.hcl
@@ -72,6 +72,7 @@ build {
     execute_command = "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}"
     script          = "../scripts/rke2-install.sh"
     timeout         = "15m"
+    max_retries = 3 # Occationally first-attempt will fail, potentially due to the restart mandated by os-stig.sh
   }
 
   provisioner "shell" {

diff --git a/packer/scripts/os-stig.sh b/packer/scripts/os-stig.sh
@@ -3,12 +3,12 @@ set -e
 
 # Detect distro, ubuntu or rhel supported
 DISTRO=$( cat /etc/os-release | tr [:upper:] [:lower:] | grep -Poi '(ubuntu|rhel)' | uniq )
+VERSION=$( cat /etc/os-release | grep -Poi '^version="[0-9]+\.[0-9]+' | cut -d\" -f2 | cut -d. -f1 )
 
 # Pull Ansible STIGs from https://public.cyber.mil/stigs/supplemental-automation-content/
 mkdir -p /tmp/ansible && chmod 700 /tmp/ansible && cd /tmp/ansible
 if [[ $DISTRO == "rhel" ]]; then
   # Determine which stigs to apply based on RHEL version
-  VERSION=$( cat /etc/os-release | grep -Poi '^version="[0-9]+\.[0-9]+' | cut -d\" -f2 | cut -d. -f1 )
   if [[ ${VERSION} -eq 9 ]] ; then
     curl -L -o ansible.zip https://dl.dod.cyber.mil/wp-content/uploads/stigs/zip/U_RHEL_9_V1R2_STIG_Ansible.zip
   elif [[ ${VERSION} -eq 8 ]]; then
@@ -18,7 +18,15 @@ if [[ $DISTRO == "rhel" ]]; then
     exit 1
   fi
 elif [[ $DISTRO == "ubuntu" ]]; then
-  curl -L -o ansible.zip https://dl.dod.cyber.mil/wp-content/uploads/stigs/zip/U_CAN_Ubuntu_20-04_LTS_V1R11_STIG_Ansible.zip
+  if [[ ${VERSION} -eq 22 ]] ; then
+    # Currently there is no stig available for Ubuntu 22.04
+    curl -L -o ansible.zip https://dl.dod.cyber.mil/wp-content/uploads/stigs/zip/U_CAN_Ubuntu_20-04_LTS_V1R11_STIG_Ansible.zip
+  elif [[ ${VERSION} -eq 20 ]]; then
+    curl -L -o ansible.zip https://dl.dod.cyber.mil/wp-content/uploads/stigs/zip/U_CAN_Ubuntu_20-04_LTS_V1R11_STIG_Ansible.zip
+  else
+    echo "Unrecognized Ubuntu version, exiting"
+    exit 1
+  fi
 fi
 unzip ansible.zip
 unzip *-ansible.zip

diff --git a/packer/vsphere/example.pkrvars.hcl b/packer/vsphere/example.pkrvars.hcl
diff --git a/packer/vsphere/http_ks/uds_ks.pkrtpl → packer/vsphere/http/uds_ks.pkrtpl b/packer/vsphere/http_ks/uds_ks.pkrtpl → packer/vsphere/http/uds_ks.pkrtpl
diff --git a/packer/vsphere/http/uds_meta_data.pkrtpl b/packer/vsphere/http/uds_meta_data.pkrtpl
diff --git a/packer/vsphere/http/uds_user_data.pkrtpl b/packer/vsphere/http/uds_user_data.pkrtpl
@@ -0,0 +1,52 @@
+#cloud-config
+autoinstall:
+  version: 1
+  locale: en_US
+  keyboard:
+    layout: us
+    variant: ''
+  storage:
+    layout:
+      name: lvm
+      match:
+       path: /dev/sda
+  ssh:
+    install-server: yes
+  package_update: false
+  package_upgrade: false
+  packages:
+    - htop
+    - tmux
+    - whois
+    - dnsutils
+    - jq
+    - open-vm-tools
+    - unzip
+    - libopenscap8
+    - zfsutils-linux
+    - apt-offline
+    - iptables
+    - open-iscsi
+    - nfs-common
+    - vlock
+    - chrony
+  user-data:
+    users:
+      - name: root
+        lock_passwd: false
+        hashed_passwd: "${root_password}"
+        ssh_redirect_user: false
+
+      - name: ${persistent_admin_username}
+        groups: sudo
+        shell: /bin/bash
+        lock_passwd: false
+        sudo: ALL=(ALL) NOPASSWD:ALL
+        hashed_passwd: "${persistent_admin_password}"
+        ssh_redirect_user: false
+    ssh_pwauth: True
+    disable_root: false
+    preserve_hostname: true
+    runcmd:
+      - sed -i -e '/^[#]*PermitRootLogin/s/^.*$/PermitRootLogin yes/' /etc/ssh/sshd_config
+      - systemctl restart ssh
diff --git a/packer/vsphere/rhel-example.pkrvars.hcl b/packer/vsphere/rhel-example.pkrvars.hcl
@@ -0,0 +1,4 @@
+linux_distro = "rhel"
+uds_iso_filepath = "rhel-9.4-x86_64-dvd/rhel-9.4-x86_64-dvd.iso"
+rhsm_username = "hellorhsm"
+rhsm_password = "mypassword" 
diff --git a/packer/vsphere/ubuntu-example.pkrvars.hcl b/packer/vsphere/ubuntu-example.pkrvars.hcl
@@ -0,0 +1,6 @@
+linux_distro = "ubuntu"
+uds_iso_filepath = "ubuntu-22.04.4-live-server-amd64/ubuntu-22.04.4-live-server-amd64.iso"
+uds_os_type = "ubuntu64Guest"
+ubuntu_pro_token = "cooltoken"
+# Ubuntu requires a longer timeout due to cloudinit steps executed during boot process
+ssh_timeout = "10m"
diff --git a/packer/vsphere/variables.pkr.hcl b/packer/vsphere/variables.pkr.hcl
@@ -42,7 +42,7 @@ variable "uds_datacenter_name" {
 
 variable "uds_datastore_name" {
   type        = string
-  default     = "192.168.10.3-ds"
+  default     = "esxi1-ds"
   description = "Name of the vSphere Datacenter to use for the Packer build"
 }
 
@@ -66,11 +66,21 @@ variable "uds_packer_vm_shutdown_command" {
 
 variable "http_directory" {
   type = string
-  default = "http_ks"
+  default = "http"
   description = "Name of the local directory containing the kickstart file to be used for booting the UDS nodes"
 }
 
-variable "boot_command" {
+variable "ubuntu_boot_command" {
+  type = list(string)
+  default = [
+    "e<wait><down><down><down><end>",
+    " autoinstall 'ds=nocloud-net;s=http://{{ .HTTPIP }}:{{ .HTTPPort }}/cloud-init/'",
+    "<wait><F10><wait>"
+  ]
+  description = "Boot command to execute on Ubuntu build VM"
+}
+
+variable "rhel_boot_command" {
   type = list(string)
   default = [
     "<up>",
@@ -81,7 +91,7 @@ variable "boot_command" {
     "fips=1",
     "<enter>"
   ]
-  description = "Boot command to execute on the build VM"
+  description = "Boot command to execute on RHEL build VM"
 }
 
 variable "http_ip" {
@@ -104,7 +114,7 @@ variable "uds_datastore_cluster_name" {
 
 variable "uds_content_library_name" {
   type        = string
-  default     = "UDS_CL-192.168.10.3"
+  default     = "UDS_CL-esxi1"
   description = "Content library storing iso used for UDS node build"
 }
 
@@ -167,7 +177,7 @@ variable "vm_disk_configurations" {
     }
   ))
   default = [{
-    disk_size      = 20000
+    disk_size      = 50000
     disk_controller_index = 0
     disk_thin_provisioned = false
     disk_eagerly_scrub = false
@@ -188,6 +198,16 @@ variable "k8s_distro" {
   description = "The Kubernetes distribution being installed"
 }
 
+variable "linux_distro" {
+  type = string
+  default = "rhel"
+  validation {
+    condition = contains(["rhel", "ubuntu"], var.linux_distro)
+    error_message = "Must be either 'rhel' or 'ubuntu'."
+  }
+  description = "The Linux distribution used as the image base OS"
+}
+
 variable "uds_packer_vm_name" {
   type = string
   default = "uds_node"
@@ -226,12 +246,14 @@ variable "persistent_admin_password" {
 
 variable "rhsm_username" {
   type        = string
+  default     = "unused_default"
   description = "The username to register with Redhat Subscription Manager"
   sensitive   = true
 }
 
 variable "rhsm_password" {
   type        = string
+  default     = "unused_default"
   description = "The password to register with Redhat Subscription Manager"
   sensitive   = true
 }
@@ -266,3 +288,9 @@ variable "vm_template_folder" {
   default = ""
   description = "The name of the vSphere VM Template folder in which to place the VM Template generated by the build"
 }
+
+variable "ssh_timeout" {
+  type = string
+  default = "5m"
+  description = "Amount of time to wait for SSH connection to be established for provisioner steps"
+} 
diff --git a/packer/vsphere/vsphere.pkr.hcl b/packer/vsphere/vsphere.pkr.hcl
@@ -13,21 +13,27 @@ packer {
 }
 
 locals {
-  vm_name = "${var.uds_packer_vm_name}_${var.k8s_distro}"
+  vm_name = "${var.uds_packer_vm_name}_${var.linux_distro}_${var.k8s_distro}"
   uds_content_library_item_description = var.uds_content_library_item_description != null ? var.uds_content_library_item_description : local.vm_name
-  shutdown_command = var.uds_packer_vm_shutdown_command == "" ? "sudo su -c \"shutdown -P now\"" : var.uds_packer_vm_shutdown_command 
+  shutdown_command = var.uds_packer_vm_shutdown_command == "" ? "sudo su -c \"shutdown -P now\"" : var.uds_packer_vm_shutdown_command
   http_content = {
-    "/uds.ks" = templatefile("${abspath(path.root)}/http_ks/uds_ks.pkrtpl", { 
-      root_password = bcrypt(var.root_password) 
+    "/uds.ks" = templatefile("${abspath(path.root)}/http/uds_ks.pkrtpl", {
+      root_password = bcrypt(var.root_password)
       rhsm_username = var.rhsm_username
       rhsm_password = var.rhsm_password
       persistent_admin_username = var.persistent_admin_username
       persistent_admin_password = bcrypt(var.persistent_admin_password)
     })
+    "/cloud-init/user-data" = templatefile("${abspath(path.root)}/http/uds_user_data.pkrtpl", {
+      root_password = bcrypt(var.root_password)
+      persistent_admin_username = var.persistent_admin_username
+      persistent_admin_password = bcrypt(var.persistent_admin_password)
+    })
+    "/cloud-init/meta-data" = templatefile("${abspath(path.root)}/http/uds_meta_data.pkrtpl", {})
   }
 }
 
-source "vsphere-iso" "rke2-rhel-base" {
+source "vsphere-iso" "rke2-base" {
   # vSphere connection
   vcenter_server      = var.vsphere_server
   username            = var.vsphere_username
@@ -68,12 +74,12 @@ source "vsphere-iso" "rke2-rhel-base" {
   # Temporary VM guest OS
   iso_paths = ["${var.uds_content_library_name}/${var.uds_iso_filepath}"]
   guest_os_type = var.uds_os_type
-  
+
   # Temporary VM boot configuration
-  boot_command = var.boot_command
+  boot_command = var.linux_distro == "ubuntu" ? var.ubuntu_boot_command : var.rhel_boot_command
   http_content = local.http_content
-  http_ip = var.http_ip != null ? var.http_ip : "" 
-  
+  http_ip = var.http_ip != null ? var.http_ip : ""
+
   # Temporary VM shutdown configuration
   shutdown_timeout = var.uds_packer_vm_shutdown_timeout
   shutdown_command = local.shutdown_command
@@ -100,12 +106,13 @@ source "vsphere-iso" "rke2-rhel-base" {
   # Communicator
   communicator = "ssh"
   ssh_username = "root"
-  ssh_password = var.root_password 
+  ssh_timeout  = var.ssh_timeout
+  ssh_password = var.root_password
 }
 
 build {
 
-  sources = ["source.vsphere-iso.rke2-rhel-base"]
+  sources = ["source.vsphere-iso.rke2-base"]
 
   provisioner "shell" {
     execute_command = "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}"
@@ -132,6 +139,7 @@ build {
     execute_command = "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}"
     script          = "../scripts/rke2-install.sh"
     timeout         = "15m"
+    max_retries = 3 # Occationally first-attempt will fail, potentially due to the restart mandated by os-stig.sh
   }
 
   provisioner "shell" {
@@ -166,6 +174,12 @@ build {
     timeout         = "15m"
   }
 
+  provisioner "shell" {
+    execute_command = "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}"
+    script          = "../scripts/cleanup-cloud-init.sh"
+    timeout         = "15m"
+  }
+
   post-processors {
     post-processor "vsphere-template" {
       host                = var.vsphere_server