From ac8c01f3224d2a8a2fc9ce4e78ef59f01c5858a1 Mon Sep 17 00:00:00 2001 From: Matthias Wolf Date: Wed, 11 Dec 2024 12:33:41 +0100 Subject: [PATCH] Update README, make multiscale-run a container _for_ multiscale-run (#8) Julia is too painful, thus this is not a container containing multiscale-run any longer, but one that can be used to install multiscale-run. Currently disabled, as STEPS can't be built with a modern GCC. --- .github/actions/build_container/action.yaml | 10 + .github/workflows/spacktainer.yaml | 10 +- .gitlab-ci.yml | 220 ------------------ README.md | 140 +++++++---- .../amd64/multiscale-run/spack.yaml | 21 +- .../amd64/neurodamus-neocortex/spack.yaml | 2 +- 6 files changed, 135 insertions(+), 268 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.github/actions/build_container/action.yaml b/.github/actions/build_container/action.yaml index 34ad796..2216468 100644 --- a/.github/actions/build_container/action.yaml +++ b/.github/actions/build_container/action.yaml @@ -21,12 +21,21 @@ inputs: description: Extra args to pass to buildah required: false default: '' + DOCKERHUB_USER: + description: Username for Dockerhub authentication + required: true + DOCKERHUB_PASSWORD: + description: Password for Dockerhub authentication + required: true GHCR_USER: description: Username for GHCR authentication required: true GHCR_TOKEN: description: Token for GHCR authentication required: true + GHCR_PATH: + description: Path for the GitHub container registry + required: true SPACK_DEPLOYMENT_KEY_PUB: description: Public key for spack deployments required: true @@ -49,6 +58,7 @@ runs: echo "${{ inputs.SPACK_DEPLOYMENT_KEY_PUB }}" > ${{ inputs.BUILD_PATH }}/key.pub echo "${{ inputs.SPACK_DEPLOYMENT_KEY_PRIVATE }}" > ${{ inputs.BUILD_PATH }}/key aws ecr get-login-password --region us-east-1 | buildah login --username AWS --password-stdin ${{ inputs.AWS_ECR_URL }} + buildah login --username ${{ inputs.DOCKERHUB_USER }} --password ${{ inputs.DOCKERHUB_PASSWORD }} docker.io buildah login --username ${{ inputs.GHCR_USER }} --password ${{ inputs.GHCR_TOKEN }} ghcr.io # This is written like that in case $BUILDAH_EXTRA_ARGS has args that require spaces, # which is tricky with shell variable expansion. Similar to Kaniko, see also: diff --git a/.github/workflows/spacktainer.yaml b/.github/workflows/spacktainer.yaml index 7e5152f..e438357 100644 --- a/.github/workflows/spacktainer.yaml +++ b/.github/workflows/spacktainer.yaml @@ -28,6 +28,8 @@ jobs: --label ch.epfl.bbpgitlab.ci-commit-branch="$GITHUB_REF_NAME" --build-arg SPACK_BRANCH=develop # ' --label org.opencontainers.image.created="$CI_JOB_STARTED_AT"' + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} SPACK_DEPLOYMENT_KEY_PUB: ${{ secrets.SPACK_DEPLOYMENT_KEY_PUB }} SPACK_DEPLOYMENT_KEY_PRIVATE: ${{ secrets.SPACK_DEPLOYMENT_KEY_PRIVATE }} runtime-container-job: @@ -56,6 +58,8 @@ jobs: --label ch.epfl.bbpgitlab.ci-commit-branch="$GITHUB_REF_NAME" --build-arg SPACK_BRANCH=develop # ' --label org.opencontainers.image.created="$CI_JOB_STARTED_AT"' + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} SPACK_DEPLOYMENT_KEY_PUB: ${{ secrets.SPACK_DEPLOYMENT_KEY_PUB }} SPACK_DEPLOYMENT_KEY_PRIVATE: ${{ secrets.SPACK_DEPLOYMENT_KEY_PRIVATE }} spacktainer-build-job: @@ -64,10 +68,10 @@ jobs: spacktainer: - appositionizer - brain-indexer - - brayns + # - brayns - connectome-manipulator - functionalizer - - multiscale-run + # - multiscale-run - neurodamus-hippocampus - neurodamus-neocortex - system-benchmarks @@ -111,5 +115,7 @@ jobs: --build-arg MIRROR_AUTH_ARG="\"--s3-access-key-id='${{ secrets.AWS_CACHE_ACCESS_KEY_ID }} --s3-access-key-secret=${{ secrets.AWS_CACHE_SECRET_ACCESS_KEY }}'\"" # ' --label org.opencontainers.image.created="$CI_JOB_STARTED_AT"' + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} SPACK_DEPLOYMENT_KEY_PUB: ${{ secrets.SPACK_DEPLOYMENT_KEY_PUB }} SPACK_DEPLOYMENT_KEY_PRIVATE: ${{ secrets.SPACK_DEPLOYMENT_KEY_PRIVATE }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 7cfccba..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,220 +0,0 @@ ---- -workflow: - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_PIPELINE_SOURCE == "web" -.rules: - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_PIPELINE_SOURCE == "web" -.no-merge-request-rules: - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: never - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "web" - -stages: - - unit tests - - generate - - build base containers - - build cache - - build spacktainer containers - - cleanup - -variables: - SPACK_BRANCH: develop - -generate base pipeline: - image: ubuntu:22.04 - stage: generate - needs: [] - variables: - SINGULARITY_VERSION: 4.0.2 - S3CMD_VERSION: 2.3.0 - script: - - apt-get update && apt-get install -y ca-certificates git podman python3 python3-pip - skopeo - - pip install --upgrade pip setuptools - - pip install -e ./job_creator - - jc create-jobs --singularity-version ${SINGULARITY_VERSION} --s3cmd-version ${S3CMD_VERSION} - artifacts: - when: always - paths: - - generated_pipeline.yaml - - spacktainer.yaml - - merged_spack*yaml - - job_creator.log - - spacktainer - extends: .rules -base containers and pipeline generation: - stage: build base containers - needs: - - job: generate base pipeline - artifacts: true - trigger: - include: - - artifact: generated_pipeline.yaml - job: generate base pipeline - strategy: depend - extends: .rules -gather child artifacts: - stage: build cache - needs: [base containers and pipeline generation] - image: ubuntu:22.04 - script: - - apt-get update && apt-get install -y ca-certificates git python3 python3-pip - unzip - - pip install --upgrade pip setuptools - - pip install -e ./get_artifacts - - ga -P ${CI_PIPELINE_ID} -t ${GITLAB_API_TOKEN} - - for zipfile in spack.artifacts*zip; do - - unzip ${zipfile} -d ${zipfile%%.zip} - - rm -f ${zipfile} - - architecture=$(echo ${zipfile##spack.artifacts.} | sed 's/.zip//') - - echo ${architecture}=yes >> build.env - - done - - for zipfile in spacktainer.artifacts*zip; do - - unzip ${zipfile} -d ${zipfile%%.zip} - - rm -f ${zipfile} - - done - - cat build.env - artifacts: - when: always - paths: [spack.artifacts.*, spacktainer.artifacts.*] - reports: - dotenv: build.env - extends: .rules -populate buildcache for amd64: - stage: build cache - needs: - - job: gather child artifacts - artifacts: true - trigger: - include: - - artifact: spack.artifacts.amd64/spack_pipeline.yaml - job: gather child artifacts - strategy: depend - extends: .rules -populate buildcache for arm64: - rules: - - exists: - - container_definitions/arm64 - stage: build cache - needs: - - job: gather child artifacts - artifacts: true - trigger: - include: - - artifact: spack.artifacts.arm64/spack_pipeline.yaml - job: gather child artifacts - strategy: depend - extends: .rules -build spacktainers for amd64: - stage: build spacktainer containers - needs: - - job: gather child artifacts - artifacts: true - - populate buildcache for amd64 - trigger: - include: - - artifact: spacktainer.artifacts.amd64/artifacts.amd64/spacktainer_pipeline.yaml - job: gather child artifacts - strategy: depend - extends: .rules -build spacktainers for arm64: - rules: - - exists: - - container_definitions/arm64 - stage: build spacktainer containers - needs: - - job: gather child artifacts - artifacts: true - - populate buildcache for arm64 - trigger: - include: - - artifact: spacktainer.artifacts.arm64/artifacts.arm64/spacktainer_pipeline.yaml - job: gather child artifacts - strategy: depend - extends: .rules -test spackitor: - stage: unit tests - image: python:3.10-buster - script: - - pip install './spackitor[test]' - - coverage run -m pytest --junit-xml=unittests.xml -vs spackitor/tests - - coverage report - - coverage xml - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: [spackitor/**/*] - coverage: /TOTAL.*\s+(\d+%)$/ - artifacts: - when: always - reports: - junit: unittests.xml - coverage_report: - coverage_format: cobertura - path: coverage.xml -cleanup deleted branch containers from the bucket: - stage: cleanup - image: bbpgitlab.epfl.ch:5050/hpc/spacktainers/singularitah:latest - extends: .no-merge-request-rules - before_script: - - sed -i 's/^access_key.*/access_key='${AWS_INFRASTRUCTURE_ACCESS_KEY_ID}'/' /root/.s3cfg - - sed -i 's/^secret_key.*/secret_key='${AWS_INFRASTRUCTURE_SECRET_ACCESS_KEY}'/' - /root/.s3cfg - - let length=$(($(echo $(expr index ${HTTP_PROXY:7} :)) - 1)) - - PROXY_HOST=${HTTP_PROXY:7:${length}} - - PROXY_PORT=${HTTP_PROXY:$((7+${length}+1))} - - sed -i 's/^proxy_host.*/proxy_host='${PROXY_HOST}'/' /root/.s3cfg - - sed -i 's/^proxy_port.*/proxy_port='${PROXY_PORT}'/' /root/.s3cfg - - cat /root/.s3cfg - script: - - git fetch - - git branch -r | sed 's|origin/||' - - branches=($(git branch -r | sed 's|origin/||' | awk '{print $NF}')) - - echo "Existing branches are ${branches[*]}" - - for container in $(s3cmd ls s3://sboinfrastructureassets/containers/ | awk '{print - $4}'); do - - container_branch=$(echo $container | awk -F'__' '{print $3}') - - echo "Container ${container} was built with branch ${container_branch}" - - if [[ -z "${container_branch}" ]] ; then - - echo "no branch for ${container} - skipping" - - continue - - fi - - if [[ ! " ${branches[*]} " =~ " ${container_branch} " ]]; then - - echo "${container_branch} not found in existing branches" - - s3cmd rm ${container} - - else - - echo "The branch still exists, it can stay" - - fi - - done -cleanup deleted branch containers from bb5: - stage: cleanup - tags: [bb5_map] - variables: - CONTAINER_ROOT: /gpfs/bbp.cscs.ch/ssd/containers/hpc/spacktainers - extends: .no-merge-request-rules - script: - - module load unstable git - - git fetch - - git branch -r | sed 's|origin/||' - - branches=($(git branch -r | sed 's|origin/||' | awk '{print $NF}')) - - echo "Existing branches are ${branches[*]}" - - for container in $(ls ${CONTAINER_ROOT}); do - - container_branch=$(echo $container | awk -F'__' '{print $3}' | sed 's/.sif//') - - echo "Checking container ${container} with branch ${container_branch}" - - if [[ -z "${container_branch}" ]] ; then - - echo "no branch for ${container} - skipping" - - continue - - fi - - if [[ ! " ${branches[*]} " =~ " ${container_branch} " ]]; then - - echo "${container_branch} not found in existing branches" - - rm -f ${CONTAINER_ROOT}/${container} - - else - - echo "The branch still exists, it can stay" - - fi - - done diff --git a/README.md b/README.md index 09ee518..c73fc56 100644 --- a/README.md +++ b/README.md @@ -15,29 +15,6 @@ The only files you should have to edit as an end-user are located in the `contai In both cases, the filename will be used as the name of your container. In case of a YAML file, the container version will be derived from the first package in your spec. In case of a def file, the version will be the same as the tag on docker hub. -## Adding extra files to your containers - -Create a folder under `spacktainer/files` to hold your container's files. Make sure to use your container's name to keep everything somewhat orderly. -In your container definition file, add a `spacktainer` section with a `files` key. This key holds a list of `source:target` filepairs (note that there is no space between source and target!) -Source is specified starting from the level below `spacktainer`; in the example below the folder structure would look like this: -``` -spacktainer/files -└── my-awesome-container - ├── some_folder - │   ├── brilliant_hack.patch - │   ├── readme.txt - │   ├── ugly_hack.patch - │   └── useless_but_if_we_delete_it_everything_breaks.jpg - └── script.sh - -``` - -``` -spack: - specs: - - my-awesome-package -``` - # Developer documentation ## Build Order @@ -66,26 +43,6 @@ Folders of note are: * container_definitions: this is where users will define their containers * runtime: base container that contains everything needed to run the spack-built environment -## job_creator - -The main entrypoints can be found, unsurprisingly, in the `__main__.py` file. This is where the `click` commands are defined. - -`architectures.py` contains the configuration for different architectures: what bucket should be used for the Spack package cache, which tag should be applied for the gitlab jobs, in which variables is the authentication defined, etc - -`ci_objects.py` contains helper object that can be used to define gitlab jobs and workflows. These will take care of architecture-specific behaviour (such as setting/unsetting the proxy, setting AWS variables, ...) - -`containers.py` holds everything related to generating container jobs: classes that define the base containers (former Spacktainerizer, Singularitah) as well as the spacktainers (formerly Spackah) and custom containers. It also contains the methods that use these classes and return a workflow with only the required jobs. - -`job_templates.py` holds job definition templates as python dictionaries. - -`logging_config.py` should be self-explanatory - -`packages.py` holds everything related to package-building jobs. Here you'll find the methods that generate the workflows for building the job that runs `spack ci generate` as well as the job that processes the output. - -`spack_template.py` contains the spack.yaml template that will be merged with the user's container config to generate the spack.yaml that will be used to build the container - -`utils.py` contains utility functions for reading/writing yaml, getting the multiarch job for a container, ... - ## Pulling images with Apptainer, Podman, or Sarus Make sure you have your AWS credentials set up. Then identify the image you want to run. @@ -121,6 +78,10 @@ Get a login token from AWS: [secret] ``` +**Note that all images are also available from the GitHub Container Registry (GHCR), i.e., +via `apptainer pull docker://ghcr.io/bluebrain/spack-neurodamus-neocortex`. The URL has to +be all lowercase, and the download will work without login.** + ### Pulling with Apptainer (or Singularity) Pull from the registry, logging in at the same time with the `AWS` username and token from @@ -131,6 +92,9 @@ above: The resulting `neurodamus-neocortex.sif` file is the container and can be copied to a better storage location as desired. +NB: `apptainer` and `singularity` may, in almost all circumstances, be treated +interchangeably. + ### Pulling with Podman Log into the registry, using `AWS` as the username: @@ -149,6 +113,78 @@ Everything in Sarus goes into one command: ❯ sarus pull --login -u AWS 130659266700.dkr.ecr.us-east-1.amazonaws.com/spacktainers/neurodamus-neocortex ``` +## Running the containers + +**To have the best results possible, it is imperative that when running on clusters, +`srun` is called with `srun --mpi=pmi2` to have the MPI in the container interface +properly with the MPI on the cluster.** + +With all container runtime engines, directories that contain the data to be processed and +output directories have to be mounted inside the container. +The following two examples show how to do this on either CSCS or SCITAS. + +Also note that the Python paths given in these examples may vary between containers. It +is best to first run a container interactively and make sure that `init.py` is properly +located. + +### Running with Sarus + +The following is a submission file that shows how to run Neurodamus on Eiger at CSCS. +Note the invocation of Sarus with `sarus run --mpi` _as well as_ `srun --mpi=pmi2`. This +also shows how to mount data directories: +``` +#!/bin/bash +#SBATCH --account=g156 +#SBATCH --partition=normal +#SBATCH --nodes=16 +#SBATCH --tasks-per-node=128 +#SBATCH --exclusive +##SBATCH --mem=0 +#SBATCH --constraint=mc +#SBATCH --time=04:00:00 + +WORKDIR=/project/g156/magkanar/ticket +DATADIR=/project/g156/NSETM-1948-extract-hex-O1/O1_data_physiology/ +IMAGE=130659266700.dkr.ecr.us-east-1.amazonaws.com/spacktainers/neurodamus-neocortex + +srun --mpi=pmi2 \ + sarus run --mpi \ + --env=HDF5_USE_FILE_LOCKING=FALSE \ + --workdir=${PWD} \ + --mount=type=bind,source=${DATADIR},destination=${DATADIR} \ + --mount=type=bind,source=${PWD},destination=${PWD} \ + ${IMAGE} \ + special -mpi -python /opt/view/lib/python3.13/site-packages/neurodamus/data/init.py --configFile=${PWD}/simulation_config_O1_full.json --enable-shm=OFF --coreneuron-direct-mode +``` + +### Running with Apptainer + +The following is a submission file that shows how to run Neurodamus on Jed from SCITAS. +Note that the `/work` directory with data was mounted with `-B /work/lnmc`: +``` +#!/bin/zsh +# +#SBATCH --nodes 2 +#SBATCH --qos parallel +#SBATCH --tasks-per-node 72 +#SBATCH --time 2:00:0 + +CONFIG=/work/lnmc/barrel_cortex/mouse-simulations/simulation_config.json +OUTPUT=/work/lnmc/matthias_test_delete_me_please/apptainah + +srun --mpi=pmi2 \ + apptainer run -B /work/lnmc/ -B ${HOME} /work/lnmc/software/containers/spack-neurodamus-neocortex_latest.sif \ + special \ + -mpi \ + -python \ + /opt/view/lib/python3.13/site-packages/neurodamus/data/init.py \ + --configFile=${HOME}/simulation_config.json \ + --enable-shm=OFF \ + --coreneuron-direct-mode \ + --output-path=${OUTPUT} \ + --lb-mode=WholeCell +``` + ## Reproducing GitHub Action builds containerized First the `builder` and `runtime` containers need to be built locally, with corresponding tags: @@ -217,6 +253,24 @@ COPY --from=builder /etc/debian_version /etc/debian_version ``` This will still require a local GPG key pair to sign packages! +### Converting images to Singularity SIF locally + +To convert images to Singularity locally, it seems simplest to first start a local +Docker registry: +``` +❯ podman container run -dt -p 5000:5000 --name registry docker.io/library/registry:2 +``` + +Then build, tag, and upload a Dockerfile to the registry: +``` +❯ podman build -v $PWD:/src . -t localhost:5000/td +❯ podman push localhost:5000/td +``` +The image from the registry can now be converted: +``` +❯ SINGULARITY_NOHTTPS=1 singularity pull docker://localhost:5000/td:latest +``` + ## Reproducing GitHub Action builds locally (outside a container) Prerequisites needed to try the container building locally: diff --git a/container_definitions/amd64/multiscale-run/spack.yaml b/container_definitions/amd64/multiscale-run/spack.yaml index 1e52b19..27d803d 100644 --- a/container_definitions/amd64/multiscale-run/spack.yaml +++ b/container_definitions/amd64/multiscale-run/spack.yaml @@ -1,7 +1,24 @@ ---- spack: - specs: [py-multiscale-run] + specs: + - "gmsh@4:" + - "py-astrovascpy@0.1.5:" + - "py-bluepysnap@2:" + - "py-libsonata@0.1.20:" + - "py-mpi4py@3:" + - "py-notebook@6:" + - "py-numpy@1.22:" + - "py-pandas@1.4:" + - "py-psutil@5.8:" + - "py-pyyaml@5:" + - "py-scipy@1.11.1:" + - "py-simpleeval@0.9.13:" + - "py-tqdm@4.65:" + - "py-trimesh@3:" + # - "steps@5" - fails to compile with modern GCC 🤷 packages: + gmsh: + require: + - ~fltk # Adds more load to compile petsc: require: - ~hypre~superlu-dist # Hypre fails to compile, SuperLU the same diff --git a/container_definitions/amd64/neurodamus-neocortex/spack.yaml b/container_definitions/amd64/neurodamus-neocortex/spack.yaml index f167916..1fda9d1 100644 --- a/container_definitions/amd64/neurodamus-neocortex/spack.yaml +++ b/container_definitions/amd64/neurodamus-neocortex/spack.yaml @@ -1,7 +1,7 @@ spack: specs: - neurodamus-models model=neocortex ~plasticity+coreneuron+caliper - - py-neurodamus@git.73f26480a3181c7653fd4123a06119c60753f8e9 + - py-neurodamus packages: all: providers: