tests(e2e): Configure AD before starting test suite (#828)

tests(e2e): Configure AD before starting test suite

.github/workflows/e2e-build-images.yaml

@@ -71,8 +71,8 @@ jobs:
       - name: Set up SSH key
         run: |
           mkdir -p ~/.ssh
-          echo "${{ secrets.AZURE_SSH_KEY }}" > ~/.ssh/id_rsa-e2e
-          chmod 600 ~/.ssh/id_rsa-e2e
+          echo "${{ secrets.AZURE_SSH_KEY }}" > ~/.ssh/adsys-e2e.pem
+          chmod 600 ~/.ssh/adsys-e2e.pem
       - name: Check if template needs to be created
         id: check-vm-template
         run: |
@@ -100,7 +100,7 @@ jobs:
       - name: Build base VM
         if: steps.check-vm-template.outputs.image-version != ''
         run: |
-          go run ./e2e/cmd/build_base_image/01_prepare_base_vm --vm-image ${{ steps.check-vm-template.outputs.image-version }} --codename ${{ matrix.codename }} --ssh-key ~/.ssh/id_rsa-e2e
+          go run ./e2e/cmd/build_base_image/01_prepare_base_vm --vm-image ${{ steps.check-vm-template.outputs.image-version }} --codename ${{ matrix.codename }}
       - name: Create template version
         if: steps.check-vm-template.outputs.image-version != ''
         run: |

.github/workflows/e2e-tests.yaml

@@ -66,8 +66,8 @@ jobs:
       - name: Set up SSH key
         run: |
           mkdir -p ~/.ssh
-          echo "${{ secrets.AZURE_SSH_KEY }}" > ~/.ssh/id_rsa-e2e
-          chmod 600 ~/.ssh/id_rsa-e2e
+          echo "${{ secrets.AZURE_SSH_KEY }}" > ~/.ssh/adsys-e2e.pem
+          chmod 600 ~/.ssh/adsys-e2e.pem
       - name: Build adsys deb
         run: |
           go run ./e2e/cmd/provision_resources/00_build_adsys_deb --codename ${{ matrix.codename }}
@@ -80,7 +80,7 @@ jobs:
           key: ${{ secrets.VPN_KEY }}
       - name: Provision client VM
         run: |
-          go run ./e2e/cmd/provision_resources/01_provision_client --ssh-key ~/.ssh/id_rsa-e2e
+          go run ./e2e/cmd/provision_resources/01_provision_client
       - name: Deprovision resources
         if: ${{ always() }}
         run: |

.gitignore

@@ -40,6 +40,7 @@ debian/adsys-windows
 
 # E2E testing
 inventory.yaml
+e2e/assets/gpo/**/*.pol
 
 # GitHub CI temporary files
 node_modules

e2e/assets/gpo/computers/GPT.INI

@@ -0,0 +1,3 @@
+[General]
+Version=42
+displayName=New Group Policy Object

e2e/assets/gpo/users-admins/GPT.INI

@@ -0,0 +1,3 @@
+[General]
+Version=42
+displayName=New Group Policy Object

e2e/assets/gpo/users/GPT.INI

@@ -0,0 +1,3 @@
+[General]
+Version=42
+displayName=New Group Policy Object

e2e/cmd/provision_resources/02_provision_ad/main.go

@@ -0,0 +1,119 @@
+// Package main provides a script to prepare OU and GPO configuration on the
+// domain controller, converting XML GPOs to binary POL format and staging them
+// in the SYSVOL share.
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/ubuntu/adsys/e2e/internal/command"
+	"github.com/ubuntu/adsys/e2e/internal/inventory"
+	"github.com/ubuntu/adsys/e2e/internal/remote"
+	"github.com/ubuntu/adsys/e2e/scripts"
+)
+
+var sshKey string
+
+func main() {
+	os.Exit(run())
+}
+
+func run() int {
+	cmd := command.New(action,
+		command.WithValidateFunc(validate),
+		command.WithStateTransition(inventory.ClientProvisioned, inventory.ADProvisioned),
+	)
+	cmd.Usage = fmt.Sprintf(`go run ./%s [options]
+
+Prepare OU and GPO configuration on the domain controller.
+
+The AD password must be set in the AD_PASSWORD environment variable.
+
+This script will:
+ - convert XML GPOs in the e2e/gpo directory to POL format
+ - upload the GPO structure to the domain controller
+ - upload & run a PowerShell script to the domain controller responsible for creating the required resources`, filepath.Base(os.Args[0]))
+
+	return cmd.Execute(context.Background())
+}
+
+func validate(_ context.Context, cmd *command.Command) error {
+	var err error
+	sshKey, err = command.ValidateAndExpandPath(cmd.Inventory.SSHKeyPath, command.DefaultSSHKeyPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func action(ctx context.Context, cmd *command.Command) error {
+	gpoDir, err := scripts.GPODir()
+	if err != nil {
+		return err
+	}
+	scriptsDir, err := scripts.Dir()
+	if err != nil {
+		return err
+	}
+
+	// Convert XML GPOs to POL format
+	// #nosec G204: this is only for tests, under controlled args
+	out, err := exec.CommandContext(ctx, "python3", filepath.Join(scriptsDir, "xml_to_pol.py"), gpoDir).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to convert GPOs to POL format: %w\n%s", err, out)
+	}
+	log.Debugf("xml_to_pol.py output:\n%s", out)
+
+	// Establish remote connection
+	client, err := remote.NewClient(inventory.DomainControllerIP, "localadmin", sshKey)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	// Recursively upload the GPO structure to the domain controller
+	if err := filepath.Walk(gpoDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// We only need to copy files
+		if info.IsDir() {
+			return nil
+		}
+
+		// Get the relative path of the file
+		relPath, err := filepath.Rel(gpoDir, path)
+		if err != nil {
+			return err
+		}
+
+		// Upload the file
+		remotePath := filepath.Join("C:", "Temp", cmd.Inventory.Hostname, relPath)
+		if err := client.Upload(path, remotePath); err != nil {
+			return err
+		}
+
+		return nil
+	}); err != nil {
+		return fmt.Errorf("failed to upload GPOs to domain controller: %w", err)
+	}
+
+	// Upload the PowerShell script to the domain controller
+	if err := client.Upload(filepath.Join(scriptsDir, "prepare-ad.ps1"), filepath.Join("C:", "Temp", cmd.Inventory.Hostname)); err != nil {
+		return err
+	}
+
+	// Run the PowerShell script
+	if _, err := client.Run(ctx, fmt.Sprintf("powershell.exe -ExecutionPolicy Bypass -File %s -hostname %s", filepath.Join("C:", "Temp", cmd.Inventory.Hostname, "prepare-ad.ps1"), cmd.Inventory.Hostname)); err != nil {
+		return fmt.Errorf("error running the PowerShell script: %w", err)
+	}
+
+	return nil
+}

e2e/internal/command/command.go

@@ -18,7 +18,7 @@ import (
 
 const (
 	// DefaultSSHKeyPath is the default path to the SSH private key.
-	DefaultSSHKeyPath = "~/.ssh/id_rsa"
+	DefaultSSHKeyPath = "~/.ssh/adsys-e2e.pem"
 )
 
 type cmdFunc func(context.Context, *Command) error

e2e/internal/inventory/inventory.go

@@ -8,8 +8,13 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-// DefaultPath is the default path to the inventory file.
-const DefaultPath = "inventory.yaml"
+const (
+	// DefaultPath is the default path to the inventory file.
+	DefaultPath = "inventory.yaml"
+
+	// DomainControllerIP is the IP address of the domain controller.
+	DomainControllerIP = "10.1.0.4"
+)
 
 // Inventory represents the contents of an inventory file.
 type Inventory struct {

e2e/internal/remote/remote.go

@@ -168,6 +168,16 @@ func (c Client) Upload(localPath string, remotePath string) error {
 		remotePath = filepath.Join(remotePath, filepath.Base(localPath))
 	}
 
+	// Check if the parent directory structure exists, create it if not
+	parentDir := filepath.Dir(remotePath)
+	if _, err := ftp.Stat(parentDir); err != nil && errors.Is(err, os.ErrNotExist) {
+		log.Debugf("Creating directory %q on remote host %q", parentDir, c.RemoteAddr().String())
+		if err := ftp.MkdirAll(parentDir); err != nil {
+			return fmt.Errorf("failed to create directory %q on remote host %q: %w", parentDir, c.RemoteAddr().String(), err)
+		}
+	}
+
+	// Create the remote file
 	remote, err := ftp.Create(remotePath)
 	if err != nil {
 		return err

e2e/scripts/prepare-ad.ps1

@@ -0,0 +1,71 @@
+# Description: Prepare the domain controller for E2E testing
+#
+# The script takes a single argument, the hostname of the Linux client to be tested.
+# It creates the following OU structure, together with GPOs and users:
+#  DC=warthogs,DC=biz
+#  └── $hostname
+#      ├── users <──────── linked to $hostname-users-gpo
+#      │   ├── admins <─── linked to $hostname-admins-gpo
+#      │   │   └── 👤 $hostname-adm
+#      │   └── 👤 $hostname-usr
+#      ├── computers <──── linked to $hostname-computers-gpo
+#      │   └── 💻 $hostname
+#      └── out-of-tree
+#
+# The script assumes the GPO data is stored in the same directory - this is the
+# case when ran via the ./cmd/provision_resources/02_provision_ad command.
+#
+# The script is not idempotent, it will fail if any resources already exist.
+param (
+    [string]$hostname
+)
+
+# Uncomment to dry run the script
+# $WhatIfPreference = $true
+
+# Stop on first error
+$ErrorActionPreference = "Stop"
+
+# Create parent OU
+$parentOUPath = "DC=warthogs,DC=biz"
+New-ADOrganizationalUnit -Name $hostname -Path $parentOUPath -ProtectedFromAccidentalDeletion $false
+
+$organizationalUnits = @{
+    'users' = "OU=${hostname},${parentOUPath}"
+    'computers' = "OU=${hostname},${parentOUPath}"
+    'admins' = "OU=users,OU=${hostname},${parentOUPath}"
+    'out-of-tree' = "OU=${hostname},${parentOUPath}"
+}
+
+# Create child OUs
+foreach ($ou in $organizationalUnits.GetEnumerator()) {
+    New-ADOrganizationalUnit -Name $ou.Key -Path $ou.Value -ProtectedFromAccidentalDeletion $false
+}
+
+# Prepare GPOs
+# POL files are stored in the same directory as this script
+$gpoPaths = 'users', 'users-admins', 'computers'
+foreach ($gpoPath in $gpoPaths) {
+    $targetOU = $gpoPath.split('-')[-1]
+    $targetOUPath = $organizationalUnits[$targetOU]
+
+    $gpoName = "$hostname-$targetOU-gpo"
+    $gpo = New-GPO -Name $gpoName -Comment $hostname
+
+    # Copy path to SYSVOL
+    $sourceDir = Join-Path -Path $PSScriptRoot -ChildPath $gpoPath
+    $destinationDir = "\\warthogs.biz\SYSVOL\warthogs.biz\Policies\{$($gpo.Id)}"
+    Copy-Item -Path "$sourceDir\*" -Destination $destinationDir -Recurse -Force
+
+    # Link GPO to OU
+    New-GPLink -Name $gpoName -Target "OU=${targetOU},${targetOUPath}" -LinkEnabled Yes
+}
+
+# Create users
+$password = ConvertTo-SecureString -String 'supersecretpassword' -AsPlainText -Force
+New-ADUser -Name "${hostname}-usr" -Path "OU=users,$($organizationalUnits['users'])" -AccountPassword $password -Enabled $true
+New-ADUser -Name "${hostname}-adm" -Path "OU=admins,$($organizationalUnits['admins'])" -AccountPassword $password -Enabled $true
+
+# Move machine to computers OU
+$identity = Get-ADComputer -Identity $hostname
+Move-ADObject -Identity $identity -TargetPath "OU=computers,$($organizationalUnits['computers'])"

e2e/scripts/scripts.go

@@ -16,6 +16,15 @@ func Dir() (string, error) {
 	return filepath.Dir(currentFile), nil
 }
 
+// GPODir returns the directory containing the GPOs.
+func GPODir() (string, error) {
+	adsysRootDir, err := RootDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(adsysRootDir, "e2e", "assets", "gpo"), nil
+}
+
 // RootDir returns the root directory of the project.
 func RootDir() (string, error) {
 	currentDir, err := Dir()

e2e/scripts/xml_to_pol.py

@@ -1,10 +1,13 @@
 import argparse
+import base64
 import logging
 import os
-import re
+import types
 
 import xml.etree.ElementTree as etree
 
+from samba.dcerpc import preg
+from samba.dcerpc import misc
 from samba.gp_parse.gp_pol import GPPolParser
 from pathlib import Path
 
@@ -65,7 +68,58 @@ def convert_to_xml(pol_file):
     parser.write_xml(xml_file)
 
 def convert_to_pol(xml_file):
+    # This is a hack to pick up an unreleased Samba fix for properly parsing
+    # empty MULTI_SZ values
+    def _load_xml(self, root):
+        self.pol_file = preg.file()
+        self.pol_file.header.signature = root.attrib['signature']
+        self.pol_file.header.version = int(root.attrib['version'])
+        self.pol_file.num_entries = int(root.attrib['num_entries'])
+
+        entries = []
+        for e in root.findall('Entry'):
+            entry = preg.entry()
+            entry_type = int(e.attrib['type'])
+
+            entry.type = entry_type
+
+            entry.keyname = e.find('Key').text
+            value_name = e.find('ValueName').text
+            if value_name is None:
+                value_name = ''
+
+            entry.valuename = value_name
+
+            if misc.REG_MULTI_SZ == entry_type:
+                values = [x.text for x in e.findall('Value')]
+                if values == [None]:
+                    data = u'\x00'
+                else:
+                    data = u'\x00'.join(values) + u'\x00\x00'
+                entry.data = data.encode('utf-16le')
+            elif (misc.REG_NONE == entry_type):
+                pass
+            elif (misc.REG_SZ == entry_type or
+                  misc.REG_EXPAND_SZ == entry_type):
+                string_val = e.find('Value').text
+                if string_val is None:
+                    string_val = ''
+                entry.data = string_val
+            elif (misc.REG_DWORD == entry_type or
+                  misc.REG_DWORD_BIG_ENDIAN == entry_type or
+                  misc.REG_QWORD == entry_type):
+                entry.data = int(e.find('Value').text)
+            else: # REG UNKNOWN or REG_BINARY
+                entry.data = base64.b64decode(e.find('Value').text)
+
+            entries.append(entry)
+
+        self.pol_file.entries = entries
+
     parser = GPPolParser()
+
+    # Override load_xml method with our custom one
+    parser.load_xml = types.MethodType(_load_xml, parser)
     with open(xml_file, 'r') as f:
         xml_data = f.read()
     parser.load_xml(etree.fromstring(xml_data.strip()))