From a9519132fe3505185d5d123f94fc167746dd7677 Mon Sep 17 00:00:00 2001 From: Johannes Blaschke Date: Thu, 19 Oct 2023 12:52:16 -0700 Subject: [PATCH 1/5] deal with vendor=cray but no gtl --- lib/MPIPreferences/src/MPIPreferences.jl | 11 ++++++-- lib/MPIPreferences/src/parse_cray_cc.jl | 35 +++++++++++++++++++----- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl index 0a4b9cf68..4fdb53d59 100644 --- a/lib/MPIPreferences/src/MPIPreferences.jl +++ b/lib/MPIPreferences/src/MPIPreferences.jl @@ -170,11 +170,18 @@ function use_system_binary(; preloads = [] preloads_env_switch = nothing cclibs = [] - if vendor === nothing + if isnothing(vendor) elseif vendor == "cray" cray_pe = CrayParser.analyze_cray_cc() library_names = [cray_pe.libmpi] - preloads = [cray_pe.libgtl] + # if there is no preload, then set preloads to "nothing" instead of + # "[nothing]" -- the later of which would cause an error when trying to + # dump as toml + if cray_pe.libgtl == nothing + preloads = nothing + else + preloads = [cray_pe.libgtl] + end preloads_env_switch = cray_pe.gtl_env_switch cclibs = cray_pe.cclibs else diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl index f167e6fcc..32f5b96dc 100644 --- a/lib/MPIPreferences/src/parse_cray_cc.jl +++ b/lib/MPIPreferences/src/parse_cray_cc.jl @@ -6,38 +6,59 @@ reduce(f::Function)::Function = Base.Fix1(Base.reduce, f) struct CrayPE libmpi::String - libgtl::String + libgtl::Union{String, Nothing} cclibs::Vector{String} gtl_env_switch::String - - CrayPE(mpi_dl::T, gtl_dl::T, cclibs::Vector{T}) where T <:AbstractString = new( + + CrayPE( + mpi_dl::T, + gtl_dl::T, + cclibs::Vector{T} + ) where T <:AbstractString = new( "lib" * mpi_dl * ".so", # Assuming Linux -- CrayPE is only avaialbe for linux anyway "lib" * gtl_dl * ".so", cclibs, "MPICH_GPU_SUPPORT_ENABLED" ) + CrayPE( + mpi_dl::T, + gtl_dl::Nothing, + cclibs::Vector{T} + ) where T <:AbstractString = new( + "lib" * mpi_dl * ".so", # Assuming Linux -- CrayPE is only avaialbe for linux anyway + nothing, + cclibs, + "MPICH_GPU_SUPPORT_ENABLED" + ) end const libmpi_prefix = "mpi_" const libgtl_prefix = "mpi_gtl_" +function only_or_nothing(iter) + if length(iter) == 0 + return nothing + end + only(iter) +end + function cray_mpi(libs) x = libs |> - filter(x-> startswith(x, libmpi_prefix)) |> + filter(x-> startswith(x, libmpi_prefix)) |> filter(x->!startswith(x, libgtl_prefix)) return only(x) end function cray_gtl(libs) x = libs |> - filter(x->startswith(x, libmpi_prefix)) |> + filter(x->startswith(x, libmpi_prefix)) |> filter(x->startswith(x, libgtl_prefix)) - return only(x) + return only_or_nothing(x) end function other_libs(libs) x = libs |> - filter(x->!startswith(x, libmpi_prefix)) |> + filter(x->!startswith(x, libmpi_prefix)) |> filter(x->!startswith(x, libgtl_prefix)) return x end From 3017bb47831cd445b145c68c33178e87b48b15d9 Mon Sep 17 00:00:00 2001 From: Johannes Blaschke Date: Fri, 20 Oct 2023 02:31:07 -0700 Subject: [PATCH 2/5] bump preferences version --- lib/MPIPreferences/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MPIPreferences/Project.toml b/lib/MPIPreferences/Project.toml index 4ad8d8c0e..af65e09f7 100644 --- a/lib/MPIPreferences/Project.toml +++ b/lib/MPIPreferences/Project.toml @@ -1,7 +1,7 @@ name = "MPIPreferences" uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" authors = [] -version = "0.1.9" +version = "0.1.10" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" From c72875dff5c77c26aa079de09752813c20110536 Mon Sep 17 00:00:00 2001 From: Johannes Blaschke Date: Fri, 20 Oct 2023 02:43:48 -0700 Subject: [PATCH 3/5] fix style --- lib/MPIPreferences/src/MPIPreferences.jl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl index 4fdb53d59..847a14d21 100644 --- a/lib/MPIPreferences/src/MPIPreferences.jl +++ b/lib/MPIPreferences/src/MPIPreferences.jl @@ -177,11 +177,7 @@ function use_system_binary(; # if there is no preload, then set preloads to "nothing" instead of # "[nothing]" -- the later of which would cause an error when trying to # dump as toml - if cray_pe.libgtl == nothing - preloads = nothing - else - preloads = [cray_pe.libgtl] - end + preloads = isnothing(cray_pe.libgtl) ? nothing : [cray_pe.libgtl] preloads_env_switch = cray_pe.gtl_env_switch cclibs = cray_pe.cclibs else From c344a8c951aaa948d86121f2f37288012f37e67b Mon Sep 17 00:00:00 2001 From: Johannes Blaschke Date: Sat, 4 Nov 2023 18:41:23 -0700 Subject: [PATCH 4/5] build basic test with cuda --- lib/MPIPreferences/src/parse_cray_cc.jl | 9 +++++++- lib/MPIPreferences/test/Project.toml | 5 ++++ lib/MPIPreferences/test/runtests.jl | 23 +++++++++++++++++++ .../test/test_data/LocalPreferences_cuda.toml | 9 ++++++++ .../test/test_data/cc_opts_cuda.txt | 1 + lib/MPIPreferences/test/test_libmpi.jl | 3 +++ 6 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 lib/MPIPreferences/test/Project.toml create mode 100644 lib/MPIPreferences/test/runtests.jl create mode 100644 lib/MPIPreferences/test/test_data/LocalPreferences_cuda.toml create mode 100644 lib/MPIPreferences/test/test_data/cc_opts_cuda.txt create mode 100644 lib/MPIPreferences/test/test_libmpi.jl diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl index 32f5b96dc..5654049bf 100644 --- a/lib/MPIPreferences/src/parse_cray_cc.jl +++ b/lib/MPIPreferences/src/parse_cray_cc.jl @@ -64,7 +64,14 @@ function other_libs(libs) end function analyze_cray_cc() - cray_opts = readchomp(Cmd(["cc", "--cray-print-opts=all"])) + opts_file = get(ENV, "JULIA_MPI_CC_OPTS_FILE", "") + cray_opts = "" + if ! isempty(opts_file) + @warn "Loading Cray parameters from file at: JULIA_MPI_CC_OPTS_FILE=$(opts_file)" + cray_opts = readchomp(opts_file) + else + cray_opts = readchomp(Cmd(["cc", "--cray-print-opts=all"])) + end ld_paths = SubString{String}[] libs = SubString{String}[] diff --git a/lib/MPIPreferences/test/Project.toml b/lib/MPIPreferences/test/Project.toml new file mode 100644 index 000000000..cfd31377a --- /dev/null +++ b/lib/MPIPreferences/test/Project.toml @@ -0,0 +1,5 @@ +[deps] +MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/lib/MPIPreferences/test/runtests.jl b/lib/MPIPreferences/test/runtests.jl new file mode 100644 index 000000000..3aefea869 --- /dev/null +++ b/lib/MPIPreferences/test/runtests.jl @@ -0,0 +1,23 @@ +using Pkg, Test, MPIPreferences + +project_path = dirname(Pkg.project().path) +test_dir = @__DIR__ +test_data = joinpath(@__DIR__, "test_data") + +cmd(f) = `$(Base.julia_cmd()) --startup-file=no --project=$(project_path) $(joinpath(test_dir, f))` + +withenv("JULIA_MPI_CC_OPTS_FILE" => joinpath(test_data, "cc_opts_cuda.txt")) do + MPIPreferences.use_system_binary(vendor="cray"; mpiexec="srun") + + local_pref_1 = joinpath(project_path, "JuliaLocalPreferences.toml") + local_pref_2 = joinpath(project_path, "LocalPreferences.toml") + pref_file = isfile(local_pref_1) ? local_pref_1 : local_pref_2 + + genereated_settings = readchomp(pref_file) + expected_settings = readchomp( + joinpath(test_data, "LocalPreferences_cuda.toml") + ) + + @test genereated_settings == expected_settings + run(cmd("test_libmpi.jl")) +end \ No newline at end of file diff --git a/lib/MPIPreferences/test/test_data/LocalPreferences_cuda.toml b/lib/MPIPreferences/test/test_data/LocalPreferences_cuda.toml new file mode 100644 index 000000000..bf1b9844d --- /dev/null +++ b/lib/MPIPreferences/test/test_data/LocalPreferences_cuda.toml @@ -0,0 +1,9 @@ +[MPIPreferences] +_format = "1.1" +abi = "MPICH" +binary = "system" +cclibs = ["cupti", "cudart", "cuda", "sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"] +libmpi = "libmpi_gnu_91.so" +mpiexec = "srun" +preloads = ["libmpi_gtl_cuda.so"] +preloads_env_switch = "MPICH_GPU_SUPPORT_ENABLED" diff --git a/lib/MPIPreferences/test/test_data/cc_opts_cuda.txt b/lib/MPIPreferences/test/test_data/cc_opts_cuda.txt new file mode 100644 index 000000000..aa6a30d87 --- /dev/null +++ b/lib/MPIPreferences/test/test_data/cc_opts_cuda.txt @@ -0,0 +1 @@ +-I/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/include -I/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/include -I/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/nvvm/include -I/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/CUPTI/include -I/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/Debugger/include -I/opt/cray/pe/dsmml/0.2.2/dsmml//include -I/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/include -L/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/lib -L/opt/cray/pe/mpich/8.1.25/gtl/lib -L/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/lib -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/lib64/stubs -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/nvvm/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/CUPTI/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/Debugger/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/math_libs/11.7/lib64 -L/opt/cray/pe/dsmml/0.2.2/dsmml//lib -L/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/lib64 -Wl,--as-needed,-lcupti,-lcudart,--no-as-needed -lcuda -Wl,--as-needed,-lmpi_gnu_91,--no-as-needed -lmpi_gtl_cuda -Wl,--as-needed,-lsci_gnu_82_mpi,--no-as-needed -Wl,--as-needed,-lsci_gnu_82,--no-as-needed -ldl -Wl,--as-needed,-ldsmml,--no-as-needed -lxpmem diff --git a/lib/MPIPreferences/test/test_libmpi.jl b/lib/MPIPreferences/test/test_libmpi.jl new file mode 100644 index 000000000..250a3cdfe --- /dev/null +++ b/lib/MPIPreferences/test/test_libmpi.jl @@ -0,0 +1,3 @@ +using Test, MPI + +@test MPI.libmpi == "libmpi_gnu_91.so" \ No newline at end of file From a1459737acea0bc4df0446f1c338dbcd509453b3 Mon Sep 17 00:00:00 2001 From: Johannes Blaschke Date: Sat, 4 Nov 2023 19:13:10 -0700 Subject: [PATCH 5/5] add non-gpu tests --- lib/MPIPreferences/test/runtests.jl | 43 ++++++++++++++++--- .../test/test_data/LocalPreferences_cpu.toml | 9 ++++ .../test/test_data/cc_opts_cpu.txt | 1 + lib/MPIPreferences/test/test_gtl_preload.jl | 7 +++ 4 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 lib/MPIPreferences/test/test_data/LocalPreferences_cpu.toml create mode 100644 lib/MPIPreferences/test/test_data/cc_opts_cpu.txt create mode 100644 lib/MPIPreferences/test/test_gtl_preload.jl diff --git a/lib/MPIPreferences/test/runtests.jl b/lib/MPIPreferences/test/runtests.jl index 3aefea869..72c7e895b 100644 --- a/lib/MPIPreferences/test/runtests.jl +++ b/lib/MPIPreferences/test/runtests.jl @@ -4,9 +4,9 @@ project_path = dirname(Pkg.project().path) test_dir = @__DIR__ test_data = joinpath(@__DIR__, "test_data") -cmd(f) = `$(Base.julia_cmd()) --startup-file=no --project=$(project_path) $(joinpath(test_dir, f))` +cmd(f; a="") = `$(Base.julia_cmd()) --startup-file=no --project=$(project_path) $(joinpath(test_dir, f)) $(a)` -withenv("JULIA_MPI_CC_OPTS_FILE" => joinpath(test_data, "cc_opts_cuda.txt")) do +function test_generated_prefs(suffix) MPIPreferences.use_system_binary(vendor="cray"; mpiexec="srun") local_pref_1 = joinpath(project_path, "JuliaLocalPreferences.toml") @@ -15,9 +15,42 @@ withenv("JULIA_MPI_CC_OPTS_FILE" => joinpath(test_data, "cc_opts_cuda.txt")) do genereated_settings = readchomp(pref_file) expected_settings = readchomp( - joinpath(test_data, "LocalPreferences_cuda.toml") + joinpath(test_data, "LocalPreferences_$(suffix).toml") ) @test genereated_settings == expected_settings - run(cmd("test_libmpi.jl")) -end \ No newline at end of file + r = run(cmd("test_libmpi.jl")) + @test success(r) +end + +function test_existing_prefs(suffix) + local_pref_1 = joinpath(project_path, "JuliaLocalPreferences.toml") + local_pref_2 = joinpath(project_path, "LocalPreferences.toml") + + # ensure any previous tests have been cleaned up + if isfile(local_pref_1) rm(local_pref_1) end + if isfile(local_pref_2) rm(local_pref_2) end + + cp( + joinpath(test_data, "LocalPreferences_$(suffix).toml"), + joinpath(project_path, "LocalPreferences.toml") + ) + + r = run(cmd("test_libmpi.jl")) + @test success(r) +end + +withenv("JULIA_MPI_CC_OPTS_FILE" => joinpath(test_data, "cc_opts_cuda.txt")) do + test_generated_prefs("cuda") +end + +withenv("JULIA_MPI_CC_OPTS_FILE" => joinpath(test_data, "cc_opts_cpu.txt")) do + test_generated_prefs("cpu") +end + +test_existing_prefs("cuda") +r = run(cmd("test_gtl_preload.jl"; a="cuda")) +@test success(r) +test_existing_prefs("cpu") +r = run(cmd("test_gtl_preload.jl"; a="cpu")) +@test success(r) \ No newline at end of file diff --git a/lib/MPIPreferences/test/test_data/LocalPreferences_cpu.toml b/lib/MPIPreferences/test/test_data/LocalPreferences_cpu.toml new file mode 100644 index 000000000..a6949d610 --- /dev/null +++ b/lib/MPIPreferences/test/test_data/LocalPreferences_cpu.toml @@ -0,0 +1,9 @@ +[MPIPreferences] +__clear__ = ["preloads"] +_format = "1.1" +abi = "MPICH" +binary = "system" +cclibs = ["sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"] +libmpi = "libmpi_gnu_91.so" +mpiexec = "srun" +preloads_env_switch = "MPICH_GPU_SUPPORT_ENABLED" diff --git a/lib/MPIPreferences/test/test_data/cc_opts_cpu.txt b/lib/MPIPreferences/test/test_data/cc_opts_cpu.txt new file mode 100644 index 000000000..0452a61be --- /dev/null +++ b/lib/MPIPreferences/test/test_data/cc_opts_cpu.txt @@ -0,0 +1 @@ +-I/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/include -I/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/include -I/opt/cray/pe/dsmml/0.2.2/dsmml//include -I/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/include -L/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/lib -L/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/lib -L/opt/cray/pe/dsmml/0.2.2/dsmml//lib -L/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/lib64 -Wl,--as-needed,-lmpi_gnu_91,--no-as-needed -Wl,--as-needed,-lsci_gnu_82_mpi,--no-as-needed -Wl,--as-needed,-lsci_gnu_82,--no-as-needed -ldl -Wl,--as-needed,-ldsmml,--no-as-needed -lxpmem diff --git a/lib/MPIPreferences/test/test_gtl_preload.jl b/lib/MPIPreferences/test/test_gtl_preload.jl new file mode 100644 index 000000000..938fe3862 --- /dev/null +++ b/lib/MPIPreferences/test/test_gtl_preload.jl @@ -0,0 +1,7 @@ +using Test, MPIPreferences + +if ARGS[1] == "cuda" + @test only(MPIPreferences.System.preloads) == "libmpi_gtl_cuda.so" +elseif ARGS[1] == "cpu" + @test isnothing(MPIPreferences.System.preloads) +end \ No newline at end of file