From 397daffb1b4809ad673a388c096a396c625f73d2 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Mon, 6 Nov 2023 10:32:58 +0100 Subject: [PATCH] Better docs --- .github/workflows/build_and_deploy.yml | 19 ++++--- README.md | 70 +++++++++++++++++++++----- 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 415a9fa..e32a1c9 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -40,28 +40,35 @@ jobs: files_yaml: | symlinks: - '*/*.hcl' - - name: Get image directories based on changed templates + - name: Get image directories based on changed symlinks # For every changed template in an directory, collect all / directories that contain a symlink to the template. # For instance, if ubuntu/src-ubunut.pkr.hcl changes, find all subdirectories of 'ubuntu' that contain links to that file # -> the result would be e.g. 'ubuntu/focal ubuntu/jammy ubuntu/focal_desktop' id: get-symlink-dirs working-directory: "${{ github.repository }}" run: | + # Loop over each changed /*.hcl file for file in ${{ steps.changed-symlink-yaml.outputs.symlinks_all_changed_files }}; do dir=$(dirname $file) - links=$(find "$dir" -type l) + links=$(find "$dir" -type l) # Find all symlinks under the current $file's directory for link in $links; do + # If this link has $file as a target, add the link's directory to the array of changed template directories if [[ $(readlink -f "$link") == $(realpath "$file") ]]; then CHANGED="$CHANGED $(dirname $link)" fi done done + # Store the results echo changed_dirs="$CHANGED" >> "$GITHUB_OUTPUT" - id: set-images-matrix - run: | # Add together the directories found in the changed-images-yaml and get-symlink-dirs steps into a single matrix - ALL_DIRS="${{ steps.changed-images-yaml.outputs.images_all_changed_files }} ${{ steps.get-symlink-dirs.outputs.changed_dirs }}" - MATRIX=($(echo "$ALL_DIRS" | xargs -n 1 | sort -u)) # Construct a unique array out of ALL_DIRS - MATRIX=$(printf '%s\n' "${MATRIX[@]}" | jq -R . | jq -c -s .) # Construct a valid json array out of the bash array + run: | + # First add together the directories found in the changed-images-yaml and changed-templates-yaml steps into a single array + ALL_DIRS="${{ steps.changed-images-yaml.outputs.images_all_changed_files }} ${{ steps.get-changed-templates.outputs.changed_dirs }}" + # Now construct a unique array out of ALL_DIRS + MATRIX=($(echo "$ALL_DIRS" | xargs -n 1 | sort -u)) + # Next create a valid json array out of the bash array using jq + MATRIX=$(printf '%s\n' "${MATRIX[@]}" | jq -R . | jq -c -s .) + # Store the results echo images=$MATRIX >> $GITHUB_OUTPUT echo matrix={\"images\": $MATRIX} >> $GITHUB_OUTPUT outputs: diff --git a/README.md b/README.md index aa3c9be..6425f76 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,10 @@ A set of scripts and [Packer](https://www.packer.io/) image definitions for buil Provided are: -* `ubuntu20/src-workspace.pkr.hcl`: the Packer template for an Ubuntu 20.04 workspace * `pack.sh`: a script that wraps Packer to generate a container or virtual machine image. +* templates for various Ubuntu flavours in the `ubuntu` directory -Using this template, you can build local containers or virtual machines that are as close to a workspace as it would be deployed on Research Cloud as possible! The generated containers -and VMs are ready in a matter of a few minutes. +Using these templates, you can build local containers or virtual machines that are as close to a workspace as it would be deployed on Research Cloud as possible! The generated containers and VMs are ready in a matter of a few minutes. ## Requirements @@ -19,17 +18,17 @@ and VMs are ready in a matter of a few minutes. ## Usage +To create the default `ubuntu/focal` image: + `./pack.sh [docker,vagrant]` For example, `./pack.sh docker` will generate a Docker container based on the template, while `./pack.sh vagrant` will create a vagrant VM based on the same. -The default name of the generated image is `src-basic-workspace`. So after running `./pack.sh docker`, you can see: +To select a different image, use the `IMG` environment variable: -``` -$ docker image list -REPOSITORY TAG IMAGE ID CREATED SIZE -src-basic-workspace latest c4c4496e23b5 6 minutes ago 608MB -``` +`IMG=ubuntu/jammy ./pack.sh docker` + +Valid values for `IMG` are paths to directories containing the Packer templates, of the form `/` (for example, `ubuntu/jammy`). ### Architecture @@ -37,12 +36,21 @@ By default, images will be built for the `amd64` architecture. Set the `ARCH` en `ARCH=arm64 ./pack.sh docker` +### Image outputs + +The default name and tag of the generated image is of the form `src-test-workspace:os_version`. So after running `./pack.sh docker`, you can see e.g.: + +``` +$ docker image list +REPOSITORY TAG IMAGE ID CREATED SIZE +ghcr.io/utrechtuniversity/src-test-workspace ubuntu_jammy 41bbc80e345f 5 days ago 438MB +``` + # How it works -The template uses a basic OS container (for the Docker target) or VM image (for the Vagrant target) to build the workspace on top of. At present, only Ubuntu 20.04 (focal) sources are -provided. +The template uses a basic OS container (for the Docker target) or VM image (for the Vagrant target) to build the workspace on top of. -On top of the base Linux image, Packer will execute a number of Ansible playbooks. These are the three basic components provided by SURF that need to be present on any SRC workspace: +On top of the base Linux image, Packer will execute a number of shell scripts, and more importantly Ansible playbooks. These are the three basic components provided by SURF that need to be present on any SRC workspace: 1. SRC-OS 1. SRC-CO @@ -64,7 +72,7 @@ This component is an Ansible playbook that: * connects to the workspace * installs ansible on the workspace -* calls the locally installed `ansible-playbook` to execute third-party (e.g. our) components +* calls the locally installed `ansible-playbook` to execute third-party components The Packer template will execute the SRC-External component once, with the `dummy-external-plugin` component. The latter does not do anything except a ping operation, but running the SRC-External playbook on the workspace ensures that Ansible is installed in exactly the same way and using exactly the same version as on Research Cloud. @@ -88,3 +96,39 @@ Packer will use the most recent version of the source OS images available on Doc Docker does not allow modifying `/etc/hosts` or `/etc/hostname`. The SRC-OS component tries to do just that. To work around this issue, this repository utilizes a [fork](https://github.com/UtrechtUniversity/src-plugin-os/tree/3afd56eb7f4e5ad53d2e91b35920205384cbe6f6) of the component that adds tags to the tasks that attempt to modify the `/etc/host*` files. Tasks with these tags are skipped by ansible (as defined in the Packer template). + +# Templates + +The `pack.sh` scripts expects the templates to live in this repository, with the following directory structure: + +``` +os +├── version1 +│   ├── src-os.pkr.hcl +│   └── variables.auto.pkrvars.hcl +├── version2 +│   ├── src-os.pkr.hcl +│   └── variables.auto.pkrvars.hcl +``` + +For example, for the Ubuntu templates: + +``` +ubuntu +├── focal +│   ├── src-ubuntu.pkr.hcl -> ../src-ubuntu.pkr.hcl +│   └── variables.auto.pkrvars.hcl +├── focal_desktop +│   ├── src-ubuntu.pkr.hcl -> ../src-ubuntu.pkr.hcl +│   └── variables.auto.pkrvars.hcl +├── jammy +│   ├── src-ubuntu.pkr.hcl -> ../src-ubuntu.pkr.hcl +│   └── variables.auto.pkrvars.hcl +└── src-ubuntu.pkr.hcl +``` + +As you can see, for the Ubuntu packages, the templates in `ubuntu//` actually just symlink the same template file `ubuntu/src-ubuntu.pkr.hcl`. Customization for each version is achieved using the `variables.auto.pkrvars.hcl` file in each `ubuntu/` directory. + +### CI + +The CI build-and-deploy task will run the `pack.sh` script on each template directory (`/`) that was changed in the pushed commit. It also detects if a symlinked template file is changed, and then run `pack.sh` on each template that relies on it. So for instance, if `ubuntu/src-ubuntu.pkr.hcl`, the build is run for all the Ubuntu templates (`ubuntu/focal`, `ubuntu/jammy`, `ubuntu_focal-desktop`). \ No newline at end of file