diff --git a/include/io/outputs.h b/include/io/outputs.h index 330f36a6..b6471cca 100644 --- a/include/io/outputs.h +++ b/include/io/outputs.h @@ -134,10 +134,7 @@ class Outputs : public platypus::NamedFieldsMap } // Write out summary of last timestep to console - void WriteConsoleSummary(int _my_rank, double t) - { - // logger.info("step {}, \tt = {}", _cycle, t); - } + void WriteConsoleSummary(int _my_rank, double t) {} // Initialize GLVis sockets and fields void InitializeGLVis(int _my_rank) diff --git a/include/problem/MFEMProblem.h b/include/problem/MFEMProblem.h index d5875da6..2d7f692c 100644 --- a/include/problem/MFEMProblem.h +++ b/include/problem/MFEMProblem.h @@ -171,6 +171,7 @@ class MFEMProblem : public ExternalProblem std::string _input_mesh; std::string _formulation_name; int _order; + mfem::Device _device; platypus::Coefficients _coefficients; platypus::InputParameters _solver_options; diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh new file mode 100755 index 00000000..b7293447 --- /dev/null +++ b/scripts/build-platypus-csd3-ampere.sh @@ -0,0 +1,267 @@ +#!/bin/bash +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --time=04:00:00 +#SBATCH --mail-type=none +#SBATCH -p ampere +#SBATCH -A ukaea-ap001-GPU +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:1 +#SBATCH --output=platypus_gpu_build.%j.out +#SBATCH --error=platypus_gpu_build.%j.err + +## WARNING: THIS SCRIPT WILL UNINSTALL ALL SPACK MODULES ASSOCIATED WITH +## THE ARCHITECTURE DEFINED IN THE ARCH VARIABLE. IF YOU DO NOT WISH TO DO +## THAT, COMMENT OUT THE SPACK UNINSTALL LINE BEFORE SUBMITTING THE SCRIPT +## -> UNINSTALL LINE IN THE install_spack_deps() FUNCTION +ARCH="linux-rocky8-zen" + +export compile_cores=32 + +load_modules() { + + # Load modules + + # shellcheck source=/dev/null + . /etc/profile.d/modules.sh # Leave this line (enables the module command) + module purge + module load rhel8/slurm + module use /usr/local/software/spack/spack-modules/rocky8-a100-20230831/linux-rocky8-zen3 +} + +set_paths() { + + USER=$(whoami) + BUILD_PREFIX=platypus_gpu + BUILD_DIR_NAME=${BUILD_PREFIX}_build + + ROOT_PATH=/home/${USER}/rds/rds-ukaea-ap001/${USER} + BUILD_PATH=${ROOT_PATH}/${BUILD_DIR_NAME} + + echo "Building in ${BUILD_PATH}" + mkdir -p "${BUILD_PATH}" || { + echo "Failed to create ${BUILD_PATH}" + exit 1 + } + + cd "${BUILD_PATH}" || exit 1 + +} + +check_cuda_version() { + + if grep -q "cuda@11.4" "${SPACK_ROOT}"/etc/spack/defaults/packages.yaml; then + echo "Ampere CUDA module found in spack." + else + echo "Ampere CUDA module not found in spack. Adding to packages.yaml." + CUDA_STR=$' cuda:\n externals:\n - spec: "cuda@11.4"\n prefix: /usr/local/software/cuda/11.4\n buildable: False' + echo "${CUDA_STR}" >> "${SPACK_ROOT}"/etc/spack/defaults/packages.yaml + fi + + # shellcheck source=/dev/null + . "${SPACK_ROOT}"/share/spack/setup-env.sh + +} + +check_spack() { + + cd "${ROOT_PATH}" || exit 1 + + if [ "$(command -v spack)" ]; then + echo "Spack command detected. Using pre-loaded spack." + elif [ -f "${ROOT_PATH}"/spack/share/spack/setup-env.sh ]; then + echo "Spack detected in root directory. Loading." + # shellcheck source=/dev/null + . spack/share/spack/setup-env.sh + else + echo "No spack detected. Building from source." + git clone --depth=100 https://github.com/spack/spack.git + # shellcheck source=/dev/null + . spack/share/spack/setup-env.sh + fi + + check_cuda_version + +} + +install_spack_deps() { + + # Cleaning up everything to start with a new environment + spack uninstall -ay arch=${ARCH} + spack clean -ab + + echo "Installing Petsc..." + # Spack's petsc doesn't like openmpi, but it works with mpich + spack install petsc +cuda cuda_arch=80 +fortran +hdf5 +hypre +metis +mpi \ + ^mpich +cuda cuda_arch=80 \ + ^hdf5 +cxx +fortran +hl +mpi +shared \ + ^hypre +mpi +shared +cuda cuda_arch=80 +superlu-dist +cublas +gpu-aware-mpi \ + ^superlu-dist +cuda cuda_arch=80 +parmetis +shared + spack load petsc arch=${ARCH} + + echo "Installing SLEPc..." + spack install slepc +cuda cuda_arch=80 + spack load slepc arch=${ARCH} + + echo "Installing netcdf..." + spack install netcdf-c +parallel-netcdf + spack load netcdf-c arch=${ARCH} + + echo "Installing ninja..." + spack install ninja + spack load ninja arch=${ARCH} + + echo "Adding python modules..." + + spack install py-pyaml + spack load py-pyaml arch=${ARCH} + + spack install py-jinja2 + spack load py-jinja2 arch=${ARCH} + + spack install py-packaging + spack load py-packaging arch=${ARCH} + + spack install py-setuptools + spack load py-setuptools arch=${ARCH} + +} + +install_gslib() { + + echo "Installing gslib..." + cd "${BUILD_PATH}" || exit 1 + git clone https://github.com/Nek5000/gslib.git + cd gslib || exit 1 + make CC=mpicc CFLAGS="-O2 -fPIC" -j"$compile_cores" +} + +install_mfem() { + + export CXX=mpic++ + export CC=mpicc + export F90=mpif90 + export F77=mpif77 + export FC=mpif90 + SLU_DIR=$(spack find --format "{prefix}" superlu-dist) + export SLU_DIR + + # Build MFEM + cd "${BUILD_PATH}" || exit 1 + git clone https://github.com/mfem/mfem.git + cd mfem || exit 1 + # This is just until MFEM merges Edward's changes. Without this, GPU build crashes! + git checkout EdwardPalmer99/add-missing-header-to-exodus-writer-fix + mkdir build + cd build || exit 1 + echo "Building MFEM" + cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=YES \ + -DMFEM_USE_OPENMP=NO \ + -DMFEM_THREAD_SAFE=YES \ + -DMFEM_ENABLE_EXAMPLES=YES \ + -DMFEM_ENABLE_MINIAPPS=YES \ + -DMFEM_USE_MPI=YES \ + -DMFEM_USE_CUDA=YES \ + -DCUDA_ARCH=sm_80 \ + -DMFEM_USE_METIS_5=YES \ + -DMFEM_USE_SUPERLU=YES \ + -DMFEM_USE_NETCDF=YES \ + -DMFEM_USE_GSLIB=YES \ + -DGSLIB_DIR="${BUILD_PATH}/gslib/build" \ + -DSuperLUDist_DIR="${SLU_DIR}" \ + -DSuperLUDist_VERSION_OK=YES + + if [ $? -eq 2 ]; then + echo "MFEM config failed" + exit 1 + fi + + make -j"$compile_cores" + + if [ $? -eq 2 ]; then + echo "MFEM build failed" + exit 1 + fi + + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${BUILD_PATH}/mfem/build:${BUILD_PATH}/mfem/build/miniapps/common +} + +install_moose() { + + # Some of the variables needed + export MOOSE_JOBS=$compile_cores + export LIBMESH_JOBS=$compile_cores + export METHOD="opt" + SLEPC_DIR=$(spack find --format "{prefix}" slepc arch=${ARCH}) + export SLEPC_DIR + + cd "${BUILD_PATH}" || exit 1 + git clone https://github.com/idaholab/moose + cd moose || exit 1 + + echo "Building libmesh..." + ./scripts/update_and_rebuild_libmesh.sh --with-mpi + if [ $? -eq 2 ]; then + echo "libmesh build failed" + exit 1 + fi + + echo "Building WASP..." + ./scripts/update_and_rebuild_wasp.sh + if [ $? -eq 2 ]; then + echo "WASP build failed" + exit 1 + fi + + ./configure --with-derivative-size=200 + if [ $? -eq 2 ]; then + echo "MOOSE configure failed" + exit 1 + fi + + cd framework || exit 1 + make -j"$compile_cores" + if [ $? -eq 2 ]; then + echo "MOOSE framework build failed" + exit 1 + fi + + cd ../modules || exit 1 + make -j"$compile_cores" + if [ $? -eq 2 ]; then + echo "MOOSE modules build failed" + exit 1 + fi + + # This takes very long! Only run the tests if you really need to! + #cd ../test || exit 1 + #make -j"$compile_cores" + #if [ $? -eq 2 ]; then + # echo "MOOSE test build failed" + # exit 1 + #fi + + #./run_tests -j"$compile_cores" +} + +install_platypus() { + + cd "${BUILD_PATH}" || exit 1 + + echo "Building platypus..." + git clone https://github.com/aurora-multiphysics/platypus.git + cd platypus || exit 1 + make -j"$compile_cores" + +} + +load_modules +set_paths +check_spack +install_spack_deps +install_gslib +install_mfem +install_moose +install_platypus diff --git a/src/problem/MFEMProblem.C b/src/problem/MFEMProblem.C index 35112535..faa6f450 100644 --- a/src/problem/MFEMProblem.C +++ b/src/problem/MFEMProblem.C @@ -12,6 +12,7 @@ MFEMProblem::validParams() "Number of timesteps between successive write outs of data collections to file."); params.addParam( "use_glvis", false, "Attempt to open GLVis ports to display variables during simulation"); + params.addParam("device", "cpu", "Run app on the chosen device."); return params; } @@ -23,6 +24,8 @@ MFEMProblem::MFEMProblem(const InputParameters & params) _outputs(), _exec_params() { + _device.Configure(getParam("device")); + _device.Print(std::cout); } MFEMProblem::~MFEMProblem() {} diff --git a/src/problem_builders/problem_builder_base.C b/src/problem_builders/problem_builder_base.C index 6e9676c7..a2f8dd55 100644 --- a/src/problem_builders/problem_builder_base.C +++ b/src/problem_builders/problem_builder_base.C @@ -13,7 +13,6 @@ Problem::~Problem() void ProblemBuilder::SetMesh(std::shared_ptr pmesh) { - // logger.info("Setting Mesh"); GetProblem()->_pmesh = pmesh; GetProblem()->_comm = pmesh->GetComm(); MPI_Comm_size(pmesh->GetComm(), &(GetProblem()->_num_procs)); @@ -23,35 +22,30 @@ ProblemBuilder::SetMesh(std::shared_ptr pmesh) void ProblemBuilder::SetFESpaces(platypus::FESpaces & fespaces) { - // logger.info("Setting FE Spaces"); GetProblem()->_fespaces = fespaces; } void ProblemBuilder::SetGridFunctions(platypus::GridFunctions & gridfunctions) { - // logger.info("Setting GridFunctions"); GetProblem()->_gridfunctions = gridfunctions; } void ProblemBuilder::SetBoundaryConditions(platypus::BCMap & bc_map) { - // logger.info("Setting Boundary Conditions"); GetProblem()->_bc_map = bc_map; } void ProblemBuilder::SetOutputs(platypus::Outputs & outputs) { - // logger.info("Setting Outputs"); GetProblem()->_outputs = outputs; } void ProblemBuilder::SetSolverOptions(platypus::InputParameters & solver_options) { - // logger.info("Setting Solver Options"); GetProblem()->_solver_options = solver_options; } @@ -70,14 +64,12 @@ ProblemBuilder::SetJacobianSolver(std::shared_ptr jacobian_solver) void ProblemBuilder::SetCoefficients(platypus::Coefficients & coefficients) { - // logger.info("Setting Coefficients"); GetProblem()->_coefficients = coefficients; } void ProblemBuilder::AddFESpace(std::string fespace_name, std::string fec_name, int vdim, int ordering) { - // logger.info("Adding {} FE Space to problem", fespace_name); if (GetProblem()->_fespaces.Has(fespace_name)) { const std::string error_message = "A fespace with the name " + fespace_name + diff --git a/test/tests/unit/kernels/diffusion_mfem.i b/test/tests/unit/kernels/diffusion_mfem.i index d960cde4..250c073a 100644 --- a/test/tests/unit/kernels/diffusion_mfem.i +++ b/test/tests/unit/kernels/diffusion_mfem.i @@ -7,6 +7,7 @@ [Problem] type = MFEMProblem use_glvis = true + device = "cpu" [] [Formulation]