diff --git a/Project.toml b/Project.toml index 17c02a2adac..4b7b69af93b 100644 --- a/Project.toml +++ b/Project.toml @@ -66,7 +66,7 @@ Makie = "0.19" MuladdMacro = "0.2.2" Octavian = "0.3.5" OffsetArrays = "1.3" -P4est = "0.4" +P4est = "0.4.9" Polyester = "0.7.5" PrecompileTools = "1.1" Printf = "1" @@ -84,7 +84,7 @@ StaticArrays = "1" StrideArrays = "0.1.18" StructArrays = "0.6" SummationByPartsOperators = "0.5.41" -T8code = "0.4.1" +T8code = "0.4.3" TimerOutputs = "0.5" Triangulate = "2.0" TriplotBase = "0.1" diff --git a/docs/src/parallelization.md b/docs/src/parallelization.md index fa6fc1a5d32..3cce7c381b2 100644 --- a/docs/src/parallelization.md +++ b/docs/src/parallelization.md @@ -53,30 +53,43 @@ a system-provided MPI installation with Trixi.jl can be found in the following s ### [Using a system-provided MPI installation](@id parallel_system_MPI) -When using Trixi.jl with a system-provided MPI backend the underlying -[`p4est`](https://github.com/cburstedde/p4est) and [`t8code`](https://github.com/DLR-AMR/t8code) -libraries need to be compiled with the same MPI installation. Therefore, you also need to -use system-provided `p4est` and `t8code` installations (for notes on how to install `p4est` -and `t8code` see e.g. [here](https://github.com/cburstedde/p4est/blob/master/README) and -[here](https://github.com/DLR-AMR/t8code/wiki/Installation), use the configure option -`--enable-mpi`). Note that `t8code` already comes with a `p4est` installation, so it suffices -to install `t8code`. In addition, [P4est.jl](https://github.com/trixi-framework/P4est.jl) and -[T8code.jl](https://github.com/DLR-AMR/T8code.jl) need to be configured to use the custom -installations. Follow the steps described -[here](https://github.com/DLR-AMR/T8code.jl/blob/main/README.md#installation) and -[here](https://github.com/trixi-framework/P4est.jl/blob/main/README.md#installation) for the -configuration. The paths that point to `libp4est.so` (and potentially to `libsc.so`) need to be -the same for P4est.jl and T8code.jl. This could e.g. be `libp4est.so` that usually can be found -in `lib/` or `local/lib/` in the installation directory of `t8code`. -In total, in your active Julia project you should have a LocalPreferences.toml file with sections -`[MPIPreferences]`, `[T8code]` and `[P4est]` as well as an entry `MPIPreferences` in your -Project.toml to use a custom MPI installation. A `LocalPreferences.toml` file +When using Trixi.jl with a system-provided MPI backend, the underlying +[`p4est`](https://github.com/cburstedde/p4est), [`t8code`](https://github.com/DLR-AMR/t8code) +and [`HDF5`](https://github.com/HDFGroup/hdf5) libraries need to be compiled with the same MPI +installation. If you want to use `p4est` (via the `P4estMesh`) or `t8code` (via the `T8codeMesh`) +from Trixi.jl, you also need to use system-provided `p4est` or `t8code` installations +(for notes on how to install `p4est` and `t8code` see, e.g., [here](https://github.com/cburstedde/p4est/blob/master/README) +and [here](https://github.com/DLR-AMR/t8code/wiki/Installation), use the configure option +`--enable-mpi`). Otherwise, there will be warnings that no preference is set for P4est.jl and +T8code.jl that can be ignored if you do not use these libraries from Trixi.jl. Note that +`t8code` already comes with a `p4est` installation, so it suffices to install `t8code`. +In order to use system-provided `p4est` and `t8code` installations, [P4est.jl](https://github.com/trixi-framework/P4est.jl) +and [T8code.jl](https://github.com/DLR-AMR/T8code.jl) need to be configured to use the custom +installations. Follow the steps described [here](https://github.com/DLR-AMR/T8code.jl/blob/main/README.md#installation) and +[here](https://github.com/trixi-framework/P4est.jl/blob/main/README.md#installation). +for the configuration. The paths that point to `libp4est.so` (and potentially to `libsc.so`) need to be +the same for P4est.jl and T8code.jl. This could, e.g., be `libp4est.so` that usually can be found +in `lib/` or `local/lib/` in the installation directory of `t8code`. Note that the `T8codeMesh`, however, +does not support MPI yet. +The preferences for [HDF5.jl](https://github.com/JuliaIO/HDF5.jl) always need to be set, even if you +do not want to use `HDF5` from Trixi.jl, see also https://github.com/JuliaIO/HDF5.jl/issues/1079. +To set the preferences for HDF5.jl, follow the instructions described +[here](https://trixi-framework.github.io/Trixi.jl/stable/parallelization/#Using-parallel-input-and-output). + +In total, in your active Julia project you should have a `LocalPreferences.toml` file with sections +`[MPIPreferences]`, `[T8code]` (only needed if `T8codeMesh` is used), `[P4est]` (only needed if +`P4estMesh` is used), and `[HDF5]` as well as an entry `MPIPreferences` in your +`Project.toml` to use a custom MPI installation. A `LocalPreferences.toml` file created as described above might look something like the following: ```toml [HDF5] libhdf5 = "/usr/lib/x86_64-linux-gnu/hdf5/openmpi/libhdf5.so" libhdf5_hl = "/usr/lib/x86_64-linux-gnu/hdf5/openmpi/libhdf5_hl.so" +[HDF5_jll] +libhdf5_hl_path = "/usr/lib/x86_64-linux-gnu/hdf5/openmpi/libhdf5_hl.so" +libhdf5_path = "/usr/lib/x86_64-linux-gnu/hdf5/openmpi/libhdf5.so" + [MPIPreferences] __clear__ = ["preloads_env_switch"] _format = "1.0" @@ -97,6 +110,22 @@ libsc = "/home/mschlott/hackathon/libtrixi/t8code/install/lib/libsc.so" libt8 = "/home/mschlott/hackathon/libtrixi/t8code/install/lib/libt8.so" ``` +This file is created with the following sequence of commands: +```julia +julia> using MPIPreferences +julia> MPIPreferences.use_system_binary() +``` +Restart the Julia REPL +```julia +julia> using P4est +julia> P4est.set_library_p4est!("/home/mschlott/hackathon/libtrixi/t8code/install/lib/libp4est.so") +julia> P4est.set_library_sc!("/home/mschlott/hackathon/libtrixi/t8code/install/lib/libsc.so") +julia> using T8code +julia> T8code.set_libraries_path!("/home/mschlott/hackathon/libtrixi/t8code/install/lib/") +julia> using HDF5 +julia> HDF5.API.set_libraries!("/usr/lib/x86_64-linux-gnu/hdf5/openmpi/libhdf5.so", "/usr/lib/x86_64-linux-gnu/hdf5/openmpi/libhdf5_hl.so") +``` +After the preferences are set, restart the Julia REPL again. ### [Usage](@id parallel_usage) @@ -218,7 +247,7 @@ julia> HDF5.API.set_libraries!("/path/to/your/libhdf5.so", "/path/to/your/libhdf ``` For more information see also the [documentation of HDF5.jl](https://juliaio.github.io/HDF5.jl/stable/mpi/). In total, you should -have a file called LocalPreferences.toml in the project directory that contains a section +have a file called `LocalPreferences.toml` in the project directory that contains a section `[MPIPreferences]`, a section `[HDF5]` with entries `libhdf5` and `libhdf5_hl`, a section `[P4est]` with the entry `libp4est` as well as a section `[T8code]` with the entries `libt8`, `libp4est` and `libsc`. diff --git a/src/auxiliary/p4est.jl b/src/auxiliary/p4est.jl index 968af339cbd..0b826254129 100644 --- a/src/auxiliary/p4est.jl +++ b/src/auxiliary/p4est.jl @@ -13,14 +13,20 @@ This function will check if `p4est` is already initialized and if yes, do nothing, thus it is safe to call it multiple times. """ function init_p4est() - p4est_package_id = P4est.package_id() - if p4est_package_id >= 0 - return nothing + # Only initialize p4est if P4est.jl can be used + if P4est.preferences_set_correctly() + p4est_package_id = P4est.package_id() + if p4est_package_id >= 0 + return nothing + end + + # Initialize `p4est` with log level ERROR to prevent a lot of output in AMR simulations + p4est_init(C_NULL, SC_LP_ERROR) + else + @warn "Preferences for P4est.jl are not set correctly. Until fixed, using `P4estMesh` will result in a crash. " * + "See also https://trixi-framework.github.io/Trixi.jl/stable/parallelization/#parallel_system_MPI" end - # Initialize `p4est` with log level ERROR to prevent a lot of output in AMR simulations - p4est_init(C_NULL, SC_LP_ERROR) - return nothing end diff --git a/src/auxiliary/t8code.jl b/src/auxiliary/t8code.jl index 37cb782bb93..bd781b21c1e 100644 --- a/src/auxiliary/t8code.jl +++ b/src/auxiliary/t8code.jl @@ -7,34 +7,40 @@ is already initialized and if yes, do nothing, thus it is safe to call it multiple times. """ function init_t8code() - t8code_package_id = t8_get_package_id() - if t8code_package_id >= 0 - return nothing - end + # Only initialize t8code if T8code.jl can be used + if T8code.preferences_set_correctly() + t8code_package_id = t8_get_package_id() + if t8code_package_id >= 0 + return nothing + end - # Initialize the sc library, has to happen before we initialize t8code. - let catch_signals = 0, print_backtrace = 0, log_handler = C_NULL - T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, log_handler, - T8code.Libt8.SC_LP_ERROR) - end + # Initialize the sc library, has to happen before we initialize t8code. + let catch_signals = 0, print_backtrace = 0, log_handler = C_NULL + T8code.Libt8.sc_init(mpi_comm(), catch_signals, print_backtrace, log_handler, + T8code.Libt8.SC_LP_ERROR) + end - if T8code.Libt8.p4est_is_initialized() == 0 - # Initialize `p4est` with log level ERROR to prevent a lot of output in AMR simulations - T8code.Libt8.p4est_init(C_NULL, T8code.Libt8.SC_LP_ERROR) - end + if T8code.Libt8.p4est_is_initialized() == 0 + # Initialize `p4est` with log level ERROR to prevent a lot of output in AMR simulations + T8code.Libt8.p4est_init(C_NULL, T8code.Libt8.SC_LP_ERROR) + end - # Initialize t8code with log level ERROR to prevent a lot of output in AMR simulations. - t8_init(T8code.Libt8.SC_LP_ERROR) - - if haskey(ENV, "TRIXI_T8CODE_SC_FINALIZE") - # Normally, `sc_finalize` should always be called during shutdown of an - # application. It checks whether there is still un-freed memory by t8code - # and/or T8code.jl and throws an exception if this is the case. For - # production runs this is not mandatory, but is helpful during - # development. Hence, this option is only activated when environment - # variable TRIXI_T8CODE_SC_FINALIZE exists. - @warn "T8code.jl: sc_finalize will be called during shutdown of Trixi.jl." - MPI.add_finalize_hook!(T8code.Libt8.sc_finalize) + # Initialize t8code with log level ERROR to prevent a lot of output in AMR simulations. + t8_init(T8code.Libt8.SC_LP_ERROR) + + if haskey(ENV, "TRIXI_T8CODE_SC_FINALIZE") + # Normally, `sc_finalize` should always be called during shutdown of an + # application. It checks whether there is still un-freed memory by t8code + # and/or T8code.jl and throws an exception if this is the case. For + # production runs this is not mandatory, but is helpful during + # development. Hence, this option is only activated when environment + # variable TRIXI_T8CODE_SC_FINALIZE exists. + @warn "T8code.jl: sc_finalize will be called during shutdown of Trixi.jl." + MPI.add_finalize_hook!(T8code.Libt8.sc_finalize) + end + else + @warn "Preferences for T8code.jl are not set correctly. Until fixed, using `T8codeMesh` will result in a crash. " * + "See also https://trixi-framework.github.io/Trixi.jl/stable/parallelization/#parallel_system_MPI" end return nothing