deploy: ebe7b65

coderefinery · Oct 7, 2024 · 8427c6c · 8427c6c
commit 8427c6c
Show file tree

Hide file tree

Showing 245 changed files with 59,001 additions and 0 deletions.
diff --git a/.buildinfo b/.buildinfo
@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: e4dd0cebedc9134d87f3c4373472ea75
+tags: d77d1c0d9ca2f4c8421862c7c5a0d620
diff --git a/.nojekyll b/.nojekyll
diff --git a/_downloads/1349acf23fb6c8d5adb6f0cd27df2cec/puhti-ompi.def b/_downloads/1349acf23fb6c8d5adb6f0cd27df2cec/puhti-ompi.def
@@ -0,0 +1,70 @@
+Bootstrap: docker
+From: rockylinux:{{ OS_VERSION }}
+
+%arguments
+
+  NPROCS=4
+  OPENMPI_VERSION=4.1.4rc1
+  OSU_MICRO_BENCHMARKS_VERSION=7.4
+  GCC_VERSION=9
+  UCX_VERSION=1.13.0
+  OS_NAME=rhel
+  OS_VERSION=8.6
+  OFED_VERSION=5.6-2.0.9.0
+
+%post
+
+  ### Install OpenMPI dependencies
+
+  # Base tools and newer gcc version
+
+  dnf install -y dnf-plugins-core epel-release
+  dnf config-manager  --set-enabled powertools
+  dnf install -y make gdb wget numactl-devel which
+  dnf -y install gcc-toolset-{{ GCC_VERSION }}
+  source /opt/rh/gcc-toolset-{{ GCC_VERSION }}/enable
+
+  # Enable Mellanox OFED rpm repo
+
+  wget https://www.mellanox.com/downloads/ofed/RPM-GPG-KEY-Mellanox
+  rpm --import RPM-GPG-KEY-Mellanox
+  rm RPM-GPG-KEY-Mellanox
+  cd /etc/yum.repos.d/
+  wget https://linux.mellanox.com/public/repo/mlnx_ofed/{{ OFED_VERSION }}/{{ OS_NAME }}{{ OS_VERSION }}/mellanox_mlnx_ofed.repo
+  cd /
+
+  # Install network library components
+
+  dnf -y install rdma-core ucx-ib-{{ UCX_VERSION }} ucx-devel-{{ UCX_VERSION }} ucx-knem-{{ UCX_VERSION }} ucx-cma-{{ UCX_VERSION }} ucx-rdmacm-{{ UCX_VERSION }}
+
+  ### Install OpenMPI
+
+  dnf -y install openmpi-{{ OPENMPI_VERSION }}
+
+  ### Build example application
+
+  export OMPI_DIR=/usr/mpi/gcc/openmpi-{{ OPENMPI_VERSION }}
+  export PATH="$OMPI_DIR/bin:$PATH"
+  export LD_LIBRARY_PATH="$OMPI_DIR/lib:$LD_LIBRARY_PATH"
+
+  # Build osu benchmarks
+  cd /opt
+  wget -q http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}.tar.gz
+  tar xf osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}.tar.gz
+  cd osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}
+  ./configure --prefix=/opt/osu-micro-benchmarks CC=mpicc CFLAGS=-O3
+  make -j{{ NPROCS }}
+  make install
+  cd ..
+  rm -rf osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }} osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}.tar.gz
+
+%environment
+  export OMPI_DIR=/usr/mpi/gcc/openmpi-{{ OPENMPI_VERSION }}
+  export PATH="$OMPI_DIR/bin:$PATH"
+  export LD_LIBRARY_PATH="$OMPI_DIR/lib:$LD_LIBRARY_PATH"
+  export MANPATH="$OMPI_DIR/share/man:$MANPATH"
+
+%runscript
+  source /opt/rh/gcc-toolset-{{ GCC_VERSION }}/enable
+  /opt/osu-micro-benchmarks/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_bw
+
diff --git a/_downloads/34449fcadec2c9575cd060dce996227a/triton-ompi.def b/_downloads/34449fcadec2c9575cd060dce996227a/triton-ompi.def
@@ -0,0 +1,56 @@
+Bootstrap: docker
+From: ubuntu:latest
+
+%arguments
+
+  NPROCS=4
+  OPENMPI_VERSION=4.1.6
+  OSU_MICRO_BENCHMARKS_VERSION=7.4
+
+%post
+
+  ### Install OpenMPI dependencies
+
+  apt-get update
+  apt-get install -y wget bash gcc gfortran g++ make file bzip2 ca-certificates libucx-dev
+
+  ### Build OpenMPI
+
+  OPENMPI_VERSION_SHORT=$(echo {{ OPENMPI_VERSION }} | cut -f 1-2 -d '.')
+  cd /opt
+  mkdir ompi
+  wget -q https://download.open-mpi.org/release/open-mpi/v${OPENMPI_VERSION_SHORT}/openmpi-{{ OPENMPI_VERSION }}.tar.bz2
+  tar -xvf openmpi-{{ OPENMPI_VERSION }}.tar.bz2
+  # Compile and install
+  cd openmpi-{{ OPENMPI_VERSION }}
+  ./configure --prefix=/opt/ompi --with-ucx=/usr
+  make -j{{ NPROCS }}
+  make install
+  cd ..
+  rm -rf openmpi-{{ OPENMPI_VERSION }} openmpi-{{ OPENMPI_VERSION }}.tar.bz2
+
+  ### Build example application
+
+  export OMPI_DIR=/opt/ompi
+  export PATH="$OMPI_DIR/bin:$PATH"
+  export LD_LIBRARY_PATH="$OMPI_DIR/lib:$LD_LIBRARY_PATH"
+
+  # Build osu benchmarks
+  cd /opt
+  wget -q http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}.tar.gz
+  tar xf osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}.tar.gz
+  cd osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}
+  ./configure --prefix=/opt/osu-micro-benchmarks CC=/opt/ompi/bin/mpicc CFLAGS=-O3
+  make -j{{ NPROCS }}
+  make install
+  cd ..
+  rm -rf osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }} osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}.tar.gz
+
+%environment
+  export OMPI_DIR=/opt/ompi
+  export PATH="$OMPI_DIR/bin:$PATH"
+  export LD_LIBRARY_PATH="$OMPI_DIR/lib:$LD_LIBRARY_PATH"
+  export MANPATH="$OMPI_DIR/share/man:$MANPATH"
+
+%runscript
+  /opt/osu-micro-benchmarks/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_bw
diff --git a/_downloads/c7c7f903ce66845641c48c1015bb49c9/lumi-mpich.def b/_downloads/c7c7f903ce66845641c48c1015bb49c9/lumi-mpich.def
@@ -0,0 +1,39 @@
+bootstrap: docker
+from: ubuntu:latest
+
+%arguments
+
+  NPROCS=4
+  MPICH_VERSION=3.1.4
+  OSU_MICRO_BENCHMARKS_VERSION=7.4
+
+%post
+
+  ### Install OpenMPI dependencies
+
+  apt-get update
+  apt-get install -y file g++ gcc gfortran make gdb strace wget ca-certificates --no-install-recommends
+
+  # Build MPICH
+
+  wget -q http://www.mpich.org/static/downloads/{{ MPICH_VERSION }}/mpich-{{ MPICH_VERSION }}.tar.gz
+  tar xf mpich-{{ MPICH_VERSION }}.tar.gz
+  cd mpich-{{ MPICH_VERSION }}
+  ./configure --disable-fortran --enable-fast=all,O3 --prefix=/usr
+  make -j{{ NPROCS }}
+  make install
+  ldconfig
+
+  # Build osu benchmarks
+  cd /opt
+  wget -q http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}.tar.gz
+  tar xf osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}.tar.gz
+  cd osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}
+  ./configure --prefix=/opt/osu-micro-benchmarks CC=mpicc CFLAGS=-O3
+  make -j{{ NPROCS }}
+  make install
+  cd ..
+  rm -rf osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }} osu-micro-benchmarks-{{ OSU_MICRO_BENCHMARKS_VERSION }}.tar.gz
+
+%runscript
+  /opt/osu-micro-benchmarks/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_bw
diff --git a/_images/app_run_example.png b/_images/app_run_example.png
diff --git a/_images/apptainer_example.png b/_images/apptainer_example.png
diff --git a/_images/bind_example.png b/_images/bind_example.png
diff --git a/_images/build_example.png b/_images/build_example.png
diff --git a/_images/containerized_application.png b/_images/containerized_application.png
diff --git a/_images/default_mounts.png b/_images/default_mounts.png
diff --git a/_images/dockerfile_python_image.png b/_images/dockerfile_python_image.png
diff --git a/_images/exec_example.png b/_images/exec_example.png
diff --git a/_images/image_explanation.png b/_images/image_explanation.png
diff --git a/_images/image_explanation_apptainer.png b/_images/image_explanation_apptainer.png
diff --git a/_images/instance_example.png b/_images/instance_example.png
diff --git a/_images/mpi_install_structure.png b/_images/mpi_install_structure.png
diff --git a/_images/mpi_job_structure.png b/_images/mpi_job_structure.png
diff --git a/_images/mpi_job_structure_hybrid.png b/_images/mpi_job_structure_hybrid.png
diff --git a/_images/normal_application.png b/_images/normal_application.png
diff --git a/_images/nv_example.png b/_images/nv_example.png
diff --git a/_images/pull_example.png b/_images/pull_example.png
diff --git a/_images/rocm_example.png b/_images/rocm_example.png
diff --git a/_images/run_example.png b/_images/run_example.png
diff --git a/_images/shell_example.png b/_images/shell_example.png
diff --git a/_sources/basics_running_containers.rst.txt b/_sources/basics_running_containers.rst.txt
@@ -0,0 +1,182 @@
+Basics of running containers
+============================
+
+.. objectives::
+
+   * Learn what methods are available to run a container
+
+
+Different ways you can run your containers
+------------------------------------------
+
+You can run your container in multiple different ways:
+
+- You can run the container as a single application
+- You can get an interactive terminal in the container
+- You can run an application from within the container
+
+
+We'll cover these ways in individual sections below.
+
+
+How to read Apptainer commands?
+-------------------------------
+
+Apptainer's commands can sometimes have multiple flags
+which can make it hard to parse the command line.
+
+A typical apptainer command has:
+
+1. ``apptainer`` is at the start because we use the
+   Apptainer client to run the commands.
+2. A subcommand (like ``pull``, ``build``, ``exec``, ``run`` etc.)
+   that tells which Apptainer feature we want to use.
+3. Often after the subcommand we have the name of the image we
+   want to use.
+4. At the end we might have additional commands that Apptainer
+   ignores but are to be digested by the application that the container itself is running.
+
+For legibility reasons many commands in this lesson are highlighted
+in this format:
+
+.. figure:: img/apptainer_example.png
+   :width: 100%
+
+   Figure 1: How apptainer-commands are highlighted
+
+
+For the comprehensive list of client commands, see
+`Apptainer CLI reference page <https://apptainer.org/docs/user/latest/cli.html>`__.
+
+
+Obtaining a container from container registry
+---------------------------------------------
+
+For these examples let's use a
+`Python image from Docker Hub <https://hub.docker.com/_/python>`__.
+
+For now let's just consider it as an application we want
+to use and that this application is available in Docker Hub.
+
+Let's create the container image with the following command:
+
+.. code-block:: console
+
+   $ apptainer pull python.sif docker://python
+
+We'll talk about container images and image building later on.
+For now it is enough to know that the file ``python.sif`` contains
+a Python installation.
+
+Running the container
+---------------------
+
+Running the container means that we run some specific program
+that the image creator has set to be the default program for this image.
+
+When we run the container we will execute a so-called entry script
+or runscript that contains commands specified by the image creators.
+
+.. warning::
+
+   Remember to only run containers from sources that you trust.
+
+Let's run the Python container image:
+
+.. code-block:: console
+
+   $ apptainer run python.sif
+
+.. admonition:: Expected result
+   :class: dropdown
+
+   We get a Python interpreter running in the container:
+
+   .. code-block:: console
+
+      $ apptainer run python.sif
+      Python 3.12.3 (main, Apr 10 2024, 14:35:18) [GCC 12.2.0] on linux
+      Type "help", "copyright", "credits" or "license" for more information.
+      >>>
+
+
+You should read the command line like this:
+
+.. figure:: img/run_example.png
+   :scale: 50%
+
+   Figure 2: A simple run command
+
+This syntax might seem self-evident, but in more complex use cases we will give
+additional flags to various parts of the command.
+
+
+Launching an interactive shell in the container
+-----------------------------------------------
+
+We can launch an interactive shell in a container with
+``apptainer shell``-command:
+
+.. code-block:: console
+
+   $ apptainer shell python.sif
+
+.. admonition:: Expected result
+   :class: dropdown
+
+   We get a ``bash`` terminal running in the container:
+
+   .. code-block:: console
+
+      $ apptainer shell python.sif
+      Apptainer> echo $SHELL
+      /bin/bash
+
+You should read the command line like this:
+
+.. figure:: img/shell_example.png
+   :scale: 50%
+
+   Figure 3: Launch a shell in a container
+
+
+Running a single program from the container
+-------------------------------------------
+
+We can launch a single program in a container with
+``apptainer exec``-command.
+
+The Python container is based on a Debian Linux distribution.
+Let's check the Debian version in the container:
+
+.. code-block:: console
+
+   $ apptainer exec python.sif cat /etc/debian_version
+
+.. admonition:: Expected result
+   :class: dropdown
+
+   With the command we print the contents of the
+   ``/etc/debian_version``-file:
+
+   .. code-block:: console
+
+      $ apptainer exec python.sif cat /etc/debian_version
+      12.5
+
+You should read the command line like this:
+
+.. figure:: img/exec_example.png
+   :scale: 50%
+
+   Figure 4: Launch a program in a container
+
+
+Review of this session
+----------------------
+
+.. admonition:: Key points to remember
+
+   - You can run the containers default program/application/set-of-commands with ``apptainer run my_image.sif``
+   - You can start an interactive shell in a container with ``apptainer shell my_image.sif``
+   - You can execute a single program in a container with ``apptainer exec my_image.sif my_program arg1 arg2``