initial

scripts/Benchmarks/generate_benchmark_scripts.jl

@@ -3,17 +3,10 @@ Pkg.activate()
 using Mustache
 
 # These are generated by solving
-# Minimise X + Y + Z s.t. X*Y*Z=i*48 for X,Y,Z∈ℤ⁺ (See bottom)
-mesh_partitions = [(4, 4, 3),(4, 4, 6),(4, 6, 6),(4, 6, 8),(5, 6, 8),(6, 6, 8),(7, 6, 8),(6, 8, 8),(6, 9, 8),(10, 6, 8),
-  (11, 6, 8),(9, 8, 8),(13, 6, 8),(12, 7, 8),(10, 9, 8),(12, 8, 8),(6, 8, 17),(12, 9, 8),(6, 8, 19),
-  (10, 12, 8),(14, 9, 8),(11, 12, 8),(6, 8, 23),(12, 12, 8),(10, 10, 12),(13, 12, 8),(12, 12, 9),(12, 14, 8),
-  (6, 8, 29),(10, 12, 12),(6, 8, 31),(8, 12, 16),(11, 12, 12),(8, 12, 17),(10, 12, 14),(12, 12, 12),(6, 8, 37),
-  (8, 12, 19),(13, 12, 12),(10, 12, 16),(6, 8, 41),(12, 12, 14),(6, 8, 43),(11, 12, 16),(12, 12, 15),(8, 12, 23),
-  (6, 8, 47),(12, 12, 16),(12, 14, 14),(10, 15, 16),(12, 12, 17),(12, 13, 16),(6, 8, 53),(12, 12, 18),(11, 15, 16),
-  (12, 14, 16),(12, 12, 19),(8, 12, 29),(6, 8, 59),(12, 15, 16),(6, 8, 61),(8, 12, 31),(12, 14, 18),(12, 16, 16),
-  (13, 15, 16),(11, 16, 18),(6, 8, 67),(12, 16, 17),(12, 12, 23),(14, 15, 16),(6, 8, 71),(12, 16, 18),(6, 8, 73),
-  (8, 12, 37),(15, 15, 16),(12, 16, 19),(11, 16, 21),(13, 16, 18),(6, 8, 79),(15, 16, 16),(12, 18, 18),(8, 12, 41),
-  (6, 8, 83),(14, 16, 18),(15, 16, 17),(8, 12, 43),(12, 12, 29),(12, 16, 22),(6, 8, 89),(15, 16, 18)]
+# Minimise X + Y + Z s.t. X*Y*Z=i*128 for X,Y,Z∈ℤ⁺ (See bottom)
+mesh_partitions = [(4,4,8),(4,8,8),(6,8,8),(8,8,8),(10,8,8),(12,8,8),(14,8,8),
+  (16,8,8),(12,12,8),(8,10,16),(16,8,11),(8,12,16),(16,8,13),(8,14,16),
+  (10,12,16),(16,16,8),(8,16,17),(12,12,16),(8,16,19),(16,16,10)]
 
 @assert Int.(prod.(mesh_partitions)/48) == collect(1:length(mesh_partitions))
 
@@ -69,13 +62,13 @@ function generate_jobs(template,phys_type,ndof_per_node,bmark_types)
 end
 
 dir_name= "GridapTopOpt";
-job_output_path = "$(ENV["SCRATCH"])/$dir_name/scripts/Benchmarks/jobs-gadi/";
+job_output_path = "$(ENV["MYSCRATCH"])/$dir_name/scripts/Benchmarks/jobs-setonix/";
 mkpath(job_output_path);
 
 # SETUP PARAMETERS
 cputype=""; # Not needed here
-gb_per_node = 192 # GB
-cps_per_node = 48
+gb_per_node = 256 # GB
+cps_per_node = 128
 wallhr = 3 ; # Hours (Note may want to manually change some afterwards)
 
 nreps = 10; # Number of benchmark repetitions
@@ -84,7 +77,7 @@ fe_order= 1;
 verbose= 1;
 write_dir = "\$SCRATCH/$dir_name/results/benchmarks/"
 
-parts = [(2,2,2);(3,3,3);mesh_partitions[[1,2,3,5]];mesh_partitions[8:5:23]]
+parts = mesh_partitions[1:4]
 strong_dof=(100+1)^3*3; # Number of dofs for strong scaling
 
 # Phys type and number of dofs per node, and what to benchmark
@@ -115,6 +108,7 @@ for jobs in jobs_by_phys
 end
 
 # Mathematica code:
-#   F[N_]:=Minimize[{X+Y+Z,X*Y*Z==N*48, X>0, Y>0, Z>0,X\[Element] Integers,Y\[Element] Integers,Z\[Element] Integers},{X,Y,Z}]
-#   For[i=1,i<=50,i++,X[i]=F[i]];
-#   For[i=1,i<=90,i++,Print[{X[i][[2]][[1]][[2]],X[i][[2]][[2]][[2]],X[i][[2]][[3]][[2]]}]]
+# F[N_]:=Minimize[{X+Y+Z,X*Y*Z==N*128, X>0, Y>0, Z>0,X\[Element] Integers,Y\[Element] Integers,Z\[Element] Integers},{X,Y,Z}]
+# nn = 20;
+# For[i=1,i<=nn,i++,X[i]=F[i]];
+# For[i=1,i<=nn,i++,Print[{X[i][[2]][[1]][[2]],X[i][[2]][[2]][[2]],X[i][[2]][[3]][[2]]}]]

scripts/Benchmarks/hpc-enviroments-setonix/README.md

@@ -0,0 +1,55 @@
+# Setonix environment
+
+It is recommended that the [Setonix documentation](https://pawsey.atlassian.net/wiki/spaces/US/pages/51925226/Setonix+Guides) is first read before usage. The setup is quite different to Gadi and uses Slurm.
+
+## Setup
+
+Three environment variables are required for everything to work:
+
+  - `PROJECT`: Project name. For instance, `bt62` for Santi's project.
+  - `P4EST_VERSION`: Version used for P4est. For instance, `2.8.5`.
+  - `PETSC_VERSION`: Version used for PETSc. For instance, `3.19.5`.
+
+These variables are setup like in `load-configs.sh`, and you should probably have them in your `.bashrc`. Although not compulsory, we also recommend setting up your Julia depot like in `load-configs.sh` or you might run with memory problems in your home folder when compiling.
+
+After these variables have been setup, you can load the necessary modules for MPI with the scripts `load-intel.sh` (Intel MPI) or `load-ompi.sh` (OpenMPI).
+
+We also provide scripts `install-p4est.sh` and `install-petsc.sh` that using the given variables installs the selected configuration for teh libraries in your home directory `$HOME/bin/library/version-mpiversion`.
+
+To load julia, you might add `module load julia/X.Y.Z` to your `.bashrc` or setup your own binaries in `$HOME/bin/julia/X.Y.Z` and then add it to your path.
+
+## MPICH Usage
+
+When running multi-node jobs the following enviroment variables must be set:
+```bash
+export MPICH_OFI_STARTUP_CONNECT=1
+export MPICH_OFI_VERBOSE=1
+```
+
+Executing an MPI application is then done via
+```bash
+srun -N $SLURM_JOB_NUM_NODES -n $SLURM_NTASKS julia ...
+```
+where `SLURM_JOB_NUM_NODES` and `SLURM_NTASKS` are set via
+```bash
+#SBATCH --ntasks={{:ncpus}}
+#SBATCH --ntasks-per-node=128
+#SBATCH --exclusive
+```
+
+Here we exclusively use full nodes. There is another example of shared nodes in `TestEnvironment/job.sh`. See this [link](https://pawsey.atlassian.net/wiki/spaces/US/pages/51927426/Example+Slurm+Batch+Scripts+for+Setonix+on+CPU+Compute+Nodes#ExampleSlurmBatchScriptsforSetonixonCPUComputeNodes-Exclusiveaccesstothenode.1) for further info.
+
+## Interactive jobs
+These can be launched with the `salloc` command. E.g.,
+```
+salloc -p debug --nodes=1 --ntasks=1 --cpus-per-task=1 --mem=16G
+```
+
+## Misc (PBS -> Slurm)
+- Submit a batch job: `qsub` -> `sbatch`
+- Submit an interactive job: `qsub -I` -> `salloc`
+- Delete a job: `qdel <job id>` -> `scancel <job id>`
+- Job status: `qstat` -> `squeue`
+- Hold a job: `qhold <job id>` -> `scontrol hold <job id>`
+- Release a job: `qrls <job id>` -> `scontrol release <job id>`
+- Cluster status: `qstat -B` -> `sinfo`

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/.gitignore

@@ -0,0 +1,3 @@
+Manifest.toml
+LocalPreferences.toml
+TestEnvironment.so

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/Project.toml

@@ -0,0 +1,16 @@
+name = "TestEnvironment"
+uuid = "98cfc141-7c22-4daa-8504-b7d7df96482b"
+authors = ["JordiManyer <[email protected]>"]
+version = "0.1.0"
+
+[deps]
+DrWatson = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
+Gridap = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e"
+GridapDistributed = "f9701e48-63b3-45aa-9a63-9bc6c271f355"
+GridapGmsh = "3025c34a-b394-11e9-2a55-3fee550c04c8"
+GridapP4est = "c2c8e14b-f5fd-423d-9666-1dd9ad120af9"
+GridapPETSc = "bcdc36c2-0c3e-11ea-095a-c9dadae499f1"
+PartitionedArrays = "5a9dfac6-5c52-46f7-8278-5e2210713be9"
+
+[extras]
+MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/compile/compile.jl

@@ -0,0 +1,5 @@
+using PackageCompiler
+
+create_sysimage([:TestEnvironment],
+  sysimage_path=joinpath(@__DIR__,"..","TestEnvironment.so"),
+  precompile_execution_file=joinpath(@__DIR__,"warmup.jl"))

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/compile/compile.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#PBS -P bt62
+#PBS -q normal
+#PBS -l walltime=01:00:00
+#PBS -l ncpus=1
+#PBS -l mem=16gb
+#PBS -N build
+#PBS -l wd
+
+source $PBS_O_WORKDIR/modules.sh
+
+julia --project=$PBS_O_WORKDIR $PBS_O_WORKDIR/compile/compile.jl

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/compile/warmup.jl

@@ -0,0 +1,4 @@
+
+using TestEnvironment
+main_poisson(;nprocs=(1,1))
+main_p4est(;nprocs=1)

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/data/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/job.sh

@@ -0,0 +1,25 @@
+#!/bin/bash --login
+#SBATCH --account=pawsey1076
+#SBATCH --partition=work
+#SBATCH --ntasks=4
+#SBATCH --ntasks-per-node=4
+#SBATCH --cpus-per-task=1
+#SBATCH --mem=8G
+#SBATCH --time=00:20:00
+
+# Load Julia/MPI/PETSc enviroment variables
+source $SLURM_SUBMIT_DIR/../load-configs.sh
+source $SLURM_SUBMIT_DIR/../load-crey-mpich.sh
+
+# Set MPI related environment variables. (Not all need to be set)
+# Main variables for multi-node jobs (activate for multinode jobs)
+# export MPICH_OFI_STARTUP_CONNECT=1
+# export MPICH_OFI_VERBOSE=1
+#Ask MPI to provide useful runtime information (activate if debugging)
+#export MPICH_ENV_DISPLAY=1
+#export MPICH_MEMORY_REPORT=1
+
+srun -N $SLURM_JOB_NUM_NODES -n $SLURM_NTASKS julia --project=$PBS_O_WORKDIR -J$PBS_O_WORKDIR/TestEnvironment.so -e'
+  using TestEnvironment;
+  main_poisson(;nprocs=(2,2),ncells=(100,100))
+'

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/src/TestEnvironment.jl

@@ -0,0 +1,18 @@
+module TestEnvironment
+
+using DrWatson
+using Gridap, GridapDistributed, GridapPETSc, GridapGmsh, GridapP4est
+using PartitionedArrays
+
+export OPTIONS_CG_JACOBI, OPTIONS_CG_AMG, OPTIONS_MUMPS, OPTIONS_NEUTON_MUMPS, OPTIONS_MINRES
+export main_poisson, main_p4est
+
+const OPTIONS_CG_JACOBI = "-pc_type jacobi -ksp_type cg -ksp_converged_reason -ksp_rtol 1.0e-10"
+const OPTIONS_CG_AMG = "-pc_type gamg -ksp_type cg -ksp_converged_reason -ksp_rtol 1.0e-10"
+const OPTIONS_MUMPS = "-pc_type lu -ksp_type preonly -ksp_converged_reason -pc_factor_mat_solver_type mumps"
+const OPTIONS_MINRES = "-ksp_type minres -ksp_converged_reason -ksp_rtol 1.0e-10"
+
+include("poisson_driver.jl")
+include("p4est_driver.jl")
+
+end # module TestEnvironment

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/src/p4est_driver.jl

@@ -0,0 +1,18 @@
+
+function main_p4est(;nprocs)
+  with_mpi() do distribute
+    main_p4est(distribute,nprocs)
+  end
+end
+
+function main_p4est(distribute,nprocs)
+  ranks = distribute(LinearIndices((prod(nprocs),)))
+
+  GridapP4est.with(ranks) do
+    domain = (0,1,0,1)
+    coarse_model = CartesianDiscreteModel(domain,(2,2))
+    model = OctreeDistributedDiscreteModel(ranks,coarse_model,2)
+    rmodel, _ = refine(model)
+    dmodel, _ = redistribute(model)
+  end
+end

scripts/Benchmarks/hpc-enviroments-setonix/TestEnvironment/src/poisson_driver.jl

@@ -0,0 +1,36 @@
+
+function main_poisson(;nprocs,                  # Number of processors
+                      ncells=(20,20),           # Number of cells
+                      options=OPTIONS_CG_JACOBI # PETSc solver options
+                      )
+  with_mpi() do distribute
+    main_poisson(distribute,nprocs,ncells,options)
+  end
+end
+
+function main_poisson(distribute,nprocs,ncells,options)
+  ranks = distribute(LinearIndices((prod(nprocs),)))
+
+  GridapPETSc.with(args=split(options)) do
+    domain = (0,1,0,1)
+    model = CartesianDiscreteModel(ranks,nprocs,domain,ncells)
+
+    order = 2
+    u(x) = (x[1]+x[2])^order
+    f(x) = -Δ(u,x)
+    reffe = ReferenceFE(lagrangian,Float64,order)
+    V = TestFESpace(model,reffe,dirichlet_tags="boundary")
+    U = TrialFESpace(V,u)
+    Ω = Triangulation(model)
+    dΩ = Measure(Ω,2*order)
+    a(u,v) = ∫( ∇(v)⋅∇(u) )dΩ
+    l(v) = ∫( v*f )dΩ
+    op = AffineFEOperator(a,l,U,V)
+
+    solver = PETScLinearSolver()
+    uh = solve(solver,op)
+
+    output_file = datadir("poisson")
+    writevtk(Ω,output_file,cellfields=["uh"=>uh,"grad_uh"=>∇(uh)])
+  end
+end

scripts/Benchmarks/hpc-enviroments-setonix/install-p4est.sh

@@ -0,0 +1,24 @@
+# Install p4est from sources
+# Requires two environment variables to be setup: 
+#   - P4EST_VERSION   :: P4est version
+#   - MPI_VERSION     :: MPI version and library
+#   - CONFIGURE_FLAGS :: --with-xxx style flags for compiler configuration
+
+CURR_DIR=$(pwd)
+PACKAGE=p4est
+INSTALL_ROOT=$HOME/bin
+P4EST_INSTALL=$INSTALL_ROOT/$PACKAGE/$P4EST_VERSION-$MPI_VERSION
+TAR_FILE=$PACKAGE-$P4EST_VERSION.tar.gz
+URL="https://github.com/p4est/p4est.github.io/raw/master/release"
+ROOT_DIR=/tmp
+SOURCES_DIR=$ROOT_DIR/$PACKAGE-$P4EST_VERSION
+BUILD_DIR=$SOURCES_DIR/build
+wget -q $URL/$TAR_FILE -O $ROOT_DIR/$TAR_FILE
+mkdir -p $SOURCES_DIR
+tar xzf $ROOT_DIR/$TAR_FILE -C $SOURCES_DIR --strip-components=1
+cd $SOURCES_DIR
+./configure --prefix=$P4EST_INSTALL $CONFIGURE_FLAGS --without-blas --without-lapack --enable-mpi --disable-dependency-tracking
+make --quiet
+make --quiet install
+rm -rf $ROOT_DIR/$TAR_FILE $SOURCES_DIR
+cd $CURR_DIR

scripts/Benchmarks/hpc-enviroments-setonix/install-petsc.sh

@@ -0,0 +1,27 @@
+# Install PETSc
+# Requires two environment variables to be setup:
+#   - PETSC_VERSION   :: PETSc version
+#   - MPI_VERSION     :: MPI version and library
+#   - CONFIGURE_FLAGS :: --with-xxx style flags to indicate compilers
+
+CURR_DIR=$(pwd)
+PACKAGE=petsc
+INSTALL_ROOT=$HOME/bin
+PETSC_INSTALL=$INSTALL_ROOT/$PACKAGE/$PETSC_VERSION-$MPI_VERSION
+TAR_FILE=$PACKAGE-$PETSC_VERSION.tar.gz
+URL="https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/"
+ROOT_DIR=/tmp
+SOURCES_DIR=$ROOT_DIR/$PACKAGE-$PETSC_VERSION
+BUILD_DIR=$SOURCES_DIR/build
+wget -q $URL/$TAR_FILE -O $ROOT_DIR/$TAR_FILE
+mkdir -p $SOURCES_DIR
+tar xzf $ROOT_DIR/$TAR_FILE -C $SOURCES_DIR --strip-components=1
+cd $SOURCES_DIR
+./configure --prefix=$PETSC_INSTALL $CONFIGURE_FLAGS \
+    --download-mumps --download-scalapack --download-parmetis --download-metis \
+    --download-ptscotch --with-debugging --with-x=0 --with-shared=1 \
+    --with-mpi=1 --with-64-bit-indices
+make
+make install
+rm -rf $ROOT_DIR/$TAR_FILE $SOURCES_DIR
+cd $CURR_DIR

scripts/Benchmarks/hpc-enviroments-setonix/load-configs.sh

@@ -0,0 +1,6 @@
+export P4EST_VERSION='2.8.5'
+export PETSC_VERSION='3.19.5'
+export PROJECT="pawsey1076"
+
+export SCRATCH=$MYSCRATCH
+export JULIA_DEPOT_PATH="$MYSCRATCH/.julia"

scripts/Benchmarks/hpc-enviroments-setonix/load-crey-mpich.sh

@@ -0,0 +1,14 @@
+module purge
+module load cray-mpich
+
+export P4EST_VERSION='2.8.5'
+export PETSC_VERSION='3.19.5'
+export PROJECT="pawsey1076"
+
+export MPI_VERSION="CRAY-MPICH-$CRAY_MPICH_VERSION"
+export JULIA_MPI_PATH=$CRAY_MPICH_DIR
+
+export JULIA_PETSC_LIBRARY="$HOME/bin/petsc/$PETSC_VERSION-$MPI_VERSION/lib/libpetsc"
+export P4EST_ROOT_DIR="$HOME/bin/p4est/$P4EST_VERSION-$MPI_VERSION"
+
+export OMP_NUM_THREADS=1

scripts/Benchmarks/hpc-enviroments-setonix/other/install-p4est-github.sh

@@ -0,0 +1,30 @@
+# This script is specifically to build P4est from Github sources. 
+# This is necessary when installing P4est 2.2 with newer versions of Intel compilers. 
+# 
+# Steps: 
+#   - The key is that we require a newer version of autotools, which is unfortunately 
+#     not available in Gadi. 
+#     Install instructions [here](https://docs.open-mpi.org/en/v5.0.x/developers/gnu-autotools.html).
+#   - Source the typical modules for IntelMPI.
+#   - Add the newly installed libraries to your path, i.e run 
+#        `export PATH=/path-to-autotools/local/bin:$PATH`
+#   - Run this script
+
+CURR_DIR=$(pwd)
+PACKAGE=p4est
+INSTALL_ROOT=$HOME/bin
+P4EST_INSTALL=$INSTALL_ROOT/$PACKAGE/$P4EST_VERSION-$MPI_VERSION
+ROOT_DIR=$INSTALL_ROOT/tmp
+SOURCES_DIR=$ROOT_DIR/$PACKAGE-$P4EST_VERSION
+BUILD_DIR=$SOURCES_DIR/build
+
+git clone [email protected]:cburstedde/p4est.git $SOURCES_DIR
+cd $SOURCES_DIR
+git checkout v2.2
+git submodule init && git submodule update
+./bootstrap
+./configure --prefix=$P4EST_INSTALL $CONFIGURE_FLAGS --without-blas --without-lapack --enable-mpi --disable-dependency-tracking
+make
+make install
+rm -rf $ROOT_DIR/$TAR_FILE $SOURCES_DIR
+cd $CURR_DIR