From 5b57d67da3b629f62f496f70d7dfbc8e3221c28c Mon Sep 17 00:00:00 2001 From: Jerry Watkins Date: Tue, 11 Oct 2022 13:59:26 -0400 Subject: [PATCH] Update LandIce summit scripts --- .../do-cmake-albany-summit-cuda-gcc-slfad16 | 0 .../do-cmake-albany-summit-serial-gcc-slfad16 | 0 .../summit}/do-cmake-trilinos-summit-cuda-gcc | 10 +- .../do-cmake-trilinos-summit-serial-gcc | 9 +- .../LandIce => LandIce/summit}/nvcc_wrapper70 | 102 +++++++++++++++--- .../old/do-cmake-albany-summit-gcc-cuda-sfad8 | 0 .../do-cmake-albany-summit-gcc-serial-sfad8 | 0 .../old/do-cmake-trilinos-summit-gcc-cuda | 0 .../old/do-cmake-trilinos-summit-gcc-serial | 0 .../summit}/old/nvcc_wrapper70 | 0 .../summit}/old/summit_gcc_modules.sh | 0 .../summit}/summit_gcc_modules.sh | 6 +- 12 files changed, 102 insertions(+), 25 deletions(-) rename doc/{summit/LandIce => LandIce/summit}/do-cmake-albany-summit-cuda-gcc-slfad16 (100%) rename doc/{summit/LandIce => LandIce/summit}/do-cmake-albany-summit-serial-gcc-slfad16 (100%) rename doc/{summit/LandIce => LandIce/summit}/do-cmake-trilinos-summit-cuda-gcc (94%) rename doc/{summit/LandIce => LandIce/summit}/do-cmake-trilinos-summit-serial-gcc (95%) rename doc/{summit/LandIce => LandIce/summit}/nvcc_wrapper70 (80%) rename doc/{summit/LandIce => LandIce/summit}/old/do-cmake-albany-summit-gcc-cuda-sfad8 (100%) rename doc/{summit/LandIce => LandIce/summit}/old/do-cmake-albany-summit-gcc-serial-sfad8 (100%) rename doc/{summit/LandIce => LandIce/summit}/old/do-cmake-trilinos-summit-gcc-cuda (100%) rename doc/{summit/LandIce => LandIce/summit}/old/do-cmake-trilinos-summit-gcc-serial (100%) rename doc/{summit/LandIce => LandIce/summit}/old/nvcc_wrapper70 (100%) rename doc/{summit/LandIce => LandIce/summit}/old/summit_gcc_modules.sh (100%) rename doc/{summit/LandIce => LandIce/summit}/summit_gcc_modules.sh (80%) diff --git a/doc/summit/LandIce/do-cmake-albany-summit-cuda-gcc-slfad16 b/doc/LandIce/summit/do-cmake-albany-summit-cuda-gcc-slfad16 similarity index 100% rename from doc/summit/LandIce/do-cmake-albany-summit-cuda-gcc-slfad16 rename to doc/LandIce/summit/do-cmake-albany-summit-cuda-gcc-slfad16 diff --git a/doc/summit/LandIce/do-cmake-albany-summit-serial-gcc-slfad16 b/doc/LandIce/summit/do-cmake-albany-summit-serial-gcc-slfad16 similarity index 100% rename from doc/summit/LandIce/do-cmake-albany-summit-serial-gcc-slfad16 rename to doc/LandIce/summit/do-cmake-albany-summit-serial-gcc-slfad16 diff --git a/doc/summit/LandIce/do-cmake-trilinos-summit-cuda-gcc b/doc/LandIce/summit/do-cmake-trilinos-summit-cuda-gcc similarity index 94% rename from doc/summit/LandIce/do-cmake-trilinos-summit-cuda-gcc rename to doc/LandIce/summit/do-cmake-trilinos-summit-cuda-gcc index 1d80488009..37e4eba65d 100755 --- a/doc/summit/LandIce/do-cmake-trilinos-summit-cuda-gcc +++ b/doc/LandIce/summit/do-cmake-trilinos-summit-cuda-gcc @@ -40,8 +40,8 @@ cmake \ \ -DTPL_ENABLE_Netcdf:BOOL=ON \ -DTPL_Netcdf_PARALLEL:BOOL=ON \ - -DTPL_Netcdf_INCLUDE_DIRS:FILEPATH=${OLCF_NETCDF_ROOT}/include \ - -DTPL_Netcdf_LIBRARY_DIRS:FILEPATH=${OLCF_NETCDF_ROOT}/lib \ + -DTPL_Netcdf_INCLUDE_DIRS:FILEPATH=${OLCF_NETCDF_C_ROOT}/include \ + -DTPL_Netcdf_LIBRARY_DIRS:FILEPATH=${OLCF_NETCDF_C_ROOT}/lib \ \ -DTPL_ENABLE_Pnetcdf:STRING=ON \ -DTPL_Pnetcdf_INCLUDE_DIRS:FILEPATH=${OLCF_PARALLEL_NETCDF_ROOT}/include \ @@ -70,6 +70,7 @@ cmake \ -DKokkos_ARCH_POWER9=ON \ -DKokkos_ARCH_VOLTA70=ON \ -DTPL_ENABLE_Thrust:BOOL=ON \ + -DThrust_INCLUDE_DIRS:FILEPATH=${OLCF_CUDA_ROOT}/include \ -DTPL_ENABLE_CUDA:BOOL=ON \ -DTPL_ENABLE_CUSPARSE:BOOL=ON \ \ @@ -81,7 +82,6 @@ cmake \ -DPhalanx_KOKKOS_DEVICE_TYPE:STRING=CUDA \ -DSacado_ENABLE_COMPLEX:BOOL=OFF \ -DTeuchos_ENABLE_COMPLEX:BOOL=OFF \ - -DTpetra_ENABLE_Kokkos_Refactor:BOOL=ON \ -DTpetra_ASSUME_CUDA_AWARE_MPI:BOOL=ON \ -DTpetra_INST_INT_LONG_LONG:BOOL=ON \ -DTpetra_INST_INT_INT:BOOL=OFF \ @@ -144,8 +144,8 @@ cmake \ -D Kokkos_ENABLE_LIBDL:BOOL=ON \ -D Trilinos_ENABLE_PanzerDofMgr:BOOL=ON \ -D Trilinos_ENABLE_ShyLU_DDFROSch:BOOL=ON \ - -D Tpetra_ENABLE_DEPRECATED_CODE:BOOL=ON \ - -D Xpetra_ENABLE_DEPRECATED_CODE:BOOL=ON \ + -D Tpetra_ENABLE_DEPRECATED_CODE:BOOL=OFF \ + -D Xpetra_ENABLE_DEPRECATED_CODE:BOOL=OFF \ \ ${TRILINOS_DIR} diff --git a/doc/summit/LandIce/do-cmake-trilinos-summit-serial-gcc b/doc/LandIce/summit/do-cmake-trilinos-summit-serial-gcc similarity index 95% rename from doc/summit/LandIce/do-cmake-trilinos-summit-serial-gcc rename to doc/LandIce/summit/do-cmake-trilinos-summit-serial-gcc index 2fbd30ddff..ca0ef0e248 100755 --- a/doc/summit/LandIce/do-cmake-trilinos-summit-serial-gcc +++ b/doc/LandIce/summit/do-cmake-trilinos-summit-serial-gcc @@ -39,8 +39,8 @@ cmake \ \ -DTPL_ENABLE_Netcdf:BOOL=ON \ -DTPL_Netcdf_PARALLEL:BOOL=ON \ - -DTPL_Netcdf_INCLUDE_DIRS:FILEPATH=${OLCF_NETCDF_ROOT}/include \ - -DTPL_Netcdf_LIBRARY_DIRS:FILEPATH=${OLCF_NETCDF_ROOT}/lib \ + -DTPL_Netcdf_INCLUDE_DIRS:FILEPATH=${OLCF_NETCDF_C_ROOT}/include \ + -DTPL_Netcdf_LIBRARY_DIRS:FILEPATH=${OLCF_NETCDF_C_ROOT}/lib \ \ -DTPL_ENABLE_Pnetcdf:STRING=ON \ -DTPL_Pnetcdf_INCLUDE_DIRS:FILEPATH=${OLCF_PARALLEL_NETCDF_ROOT}/include \ @@ -78,7 +78,6 @@ cmake \ -DPhalanx_KOKKOS_DEVICE_TYPE:STRING=SERIAL \ -DSacado_ENABLE_COMPLEX:BOOL=OFF \ -DTeuchos_ENABLE_COMPLEX:BOOL=OFF \ - -DTpetra_ENABLE_Kokkos_Refactor:BOOL=ON \ -DTpetra_ASSUME_CUDA_AWARE_MPI:BOOL=OFF \ -DTpetra_INST_INT_LONG_LONG:BOOL=ON \ -DTpetra_INST_INT_INT:BOOL=OFF \ @@ -141,8 +140,8 @@ cmake \ -D Kokkos_ENABLE_LIBDL:BOOL=ON \ -D Trilinos_ENABLE_PanzerDofMgr:BOOL=ON \ -D Trilinos_ENABLE_ShyLU_DDFROSch:BOOL=ON \ - -D Tpetra_ENABLE_DEPRECATED_CODE:BOOL=ON \ - -D Xpetra_ENABLE_DEPRECATED_CODE:BOOL=ON \ + -D Tpetra_ENABLE_DEPRECATED_CODE:BOOL=OFF \ + -D Xpetra_ENABLE_DEPRECATED_CODE:BOOL=OFF \ \ ${TRILINOS_DIR} diff --git a/doc/summit/LandIce/nvcc_wrapper70 b/doc/LandIce/summit/nvcc_wrapper70 similarity index 80% rename from doc/summit/LandIce/nvcc_wrapper70 rename to doc/LandIce/summit/nvcc_wrapper70 index eb87d9cc1f..c4db068de5 100755 --- a/doc/summit/LandIce/nvcc_wrapper70 +++ b/doc/LandIce/summit/nvcc_wrapper70 @@ -96,10 +96,10 @@ replace_pragma_ident=0 first_xcompiler_arg=1 # Allow for setting temp dir without setting TMPDIR in parent (see https://docs.olcf.ornl.gov/systems/summit_user_guide.html#setting-tmpdir-causes-jsm-jsrun-errors-job-state-flip-flop) -if [[ ! -z ${NVCC_WRAPPER_TMPDIR+x} ]]; then +if [[ -z ${NVCC_WRAPPER_TMPDIR+x} ]]; then temp_dir=${TMPDIR:-/tmp} else - temp_dir=${NVCC_WRAPPER_TMPDIR+x} + temp_dir=${NVCC_WRAPPER_TMPDIR} fi # optimization flag added as a command-line argument @@ -152,8 +152,13 @@ do # Ensure we only have one optimization flag because NVCC doesn't allow multiple -O*) if [ -n "$optimization_flag" ]; then - echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-O*), only the last is used because nvcc can only accept a single optimization setting." - shared_args=${shared_args/ $optimization_flag/} + if [ "$1" = "$optimization_flag" ]; then + # Silently consume duplicates of the same argument + shift + continue + fi + echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-O*), only the last is used because nvcc can only accept a single optimization setting." + shared_args=${shared_args/ $optimization_flag/} fi if [ "$1" = "-O" ]; then optimization_flag="-O2" @@ -222,21 +227,92 @@ do fi ;; #Handle known nvcc args - --dryrun|--verbose|--keep|--keep-dir*|-G|-lineinfo|-expt-extended-lambda|-expt-relaxed-constexpr|--resource-usage|-Xptxas*|--fmad=*|--use_fast_math|--Wext-lambda-captures-this|-Wext-lambda-captures-this) + --dryrun|--verbose|--keep|--source-in-ptx|-src-in-ptx|--keep-dir*|-G|-lineinfo|-expt-extended-lambda|-expt-relaxed-constexpr|--resource-usage|--fmad=*|--use_fast_math|--Wext-lambda-captures-this|-Wext-lambda-captures-this) cuda_args="$cuda_args $1" ;; #Handle more known nvcc args - --expt-extended-lambda|--expt-relaxed-constexpr|--Wno-deprecated-gpu-targets|-Wno-deprecated-gpu-targets) + --expt-extended-lambda|--expt-relaxed-constexpr|--Wno-deprecated-gpu-targets|-Wno-deprecated-gpu-targets|-allow-unsupported-compiler|--allow-unsupported-compiler) cuda_args="$cuda_args $1" ;; #Handle known nvcc args that have an argument - -maxrregcount=*|--maxrregcount=*) + -maxrregcount=*|--maxrregcount=*|-time=*|-Xptxas=*) cuda_args="$cuda_args $1" ;; - -maxrregcount|--default-stream|-Xnvlink|--fmad|-cudart|--cudart|-include) + -maxrregcount|--default-stream|-Xnvlink|--fmad|-cudart|--cudart|-include|-time|-Xptxas) cuda_args="$cuda_args $1 $2" shift ;; + # Handle Werror. Note, we must differentiate between the ones going to nvcc and the host compiler + # --Werror kind,... OR --Werror=kind,... <- always to nvcc + --Werror) + cuda_args="$cuda_args $1 $2" + shift + ;; + --Werror=*) + cuda_args="$cuda_args $1" + ;; + # -Werror kind,... where kind is one of {all-warnings, cross-execution-space-call, reorder, default-stream-launch, missing-launch-bounds, ext-lambda-captures-this, deprecated-declarations} <- goes to nvcc + # -Werror not followed by any kind as mentioned above goes to host compiler without any arguments + -Werror) + if [ $# -gt 1 ]; then + IFS="," read -r -a kinds <<< "$2" + first_kind=${kinds[0]} + # check if the first kind is one of the allowed ones, then this must be an nvcc list so put all of them to the cuda compiler + case $first_kind in + all-warnings|cross-execution-space-call|reorder|default-stream-launch|missing-launch-bounds|ext-lambda-captures-this|deprecated-declarations) + cuda_args="$cuda_args $1 $2" + shift + ;; + *) + if [ $first_xcompiler_arg -eq 1 ]; then + xcompiler_args="$1" + first_xcompiler_arg=0 + else + xcompiler_args="$xcompiler_args,$1" + fi + ;; + esac + fi + ;; + # -Werror=kind,... will be split into two parts, those kinds that belong to nvcc (see above) go there, while all others go towards the host compiler + -Werror=*) + kinds_str="${1:8}" # strip -Werror= + IFS="," read -r -a kinds <<< ${kinds_str} + first_werror_cuda=1 + first_werror_host=1 + xcompiler_args_werror= + # loop over all kinds that are sparated via ',' + for kind in "${kinds[@]}" + do + case ${kind} in + all-warnings|cross-execution-space-call|reorder|default-stream-launch|missing-launch-bounds|ext-lambda-captures-this|deprecated-declarations) + if [ $first_werror_cuda -ne 0 ]; then + cuda_args="$cuda_args -Werror=" + first_werror_cuda=0 + else + cuda_args="$cuda_args," + fi + cuda_args="$cuda_args$kind" + ;; + *) + if [ $first_werror_host -eq 0 ]; then + xcompiler_args_werror="${xcompiler_args_werror}," + fi + first_werror_host=0 + xcompiler_args_werror="$xcompiler_args_werror-Werror=$kind" + ;; + esac + done + if [ $first_werror_host -eq 0 ]; then + if [ $first_xcompiler_arg -eq 1 ]; then + xcompiler_args="$xcompiler_args_werror" + first_xcompiler_arg=0 + else + xcompiler_args="$xcompiler_args,$xcompiler_args_werror" + fi + fi + ;; + # End of Werror handling #Handle unsupported standard flags --std=c++1y|-std=c++1y|--std=gnu++1y|-std=gnu++1y|--std=c++1z|-std=c++1z|--std=gnu++1z|-std=gnu++1z|--std=c++2a|-std=c++2a) fallback_std_flag="-std=c++14" @@ -305,7 +381,7 @@ do -std=c++98|--std=c++98) ;; #strip of pedantic because it produces endless warnings about #LINE added by the preprocessor - -pedantic|-Wpedantic|-ansi) + -pedantic|-pedantic-errors|-Wpedantic|-ansi) ;; #strip of -Woverloaded-virtual to avoid "cc1: warning: command line option ‘-Woverloaded-virtual’ is valid for C++/ObjC++ but not for C" -Woverloaded-virtual) @@ -552,14 +628,14 @@ if [ $host_only -eq 1 ]; then $host_command elif [ -n "$nvcc_depfile_command" ]; then if [ "$NVCC_WRAPPER_SHOW_COMMANDS_BEING_RUN" == "1" ] ; then - echo "$nvcc_command && $nvcc_depfile_command" + echo "TMPDIR=${temp_dir} $nvcc_command && TMPDIR=${temp_dir} $nvcc_depfile_command" fi - $nvcc_command && $nvcc_depfile_command + TMPDIR=${temp_dir} $nvcc_command && TMPDIR=${temp_dir} $nvcc_depfile_command else if [ "$NVCC_WRAPPER_SHOW_COMMANDS_BEING_RUN" == "1" ] ; then - echo "$nvcc_command" + echo "TMPDIR=${temp_dir} $nvcc_command" fi - $nvcc_command + TMPDIR=${temp_dir} $nvcc_command fi error_code=$? diff --git a/doc/summit/LandIce/old/do-cmake-albany-summit-gcc-cuda-sfad8 b/doc/LandIce/summit/old/do-cmake-albany-summit-gcc-cuda-sfad8 similarity index 100% rename from doc/summit/LandIce/old/do-cmake-albany-summit-gcc-cuda-sfad8 rename to doc/LandIce/summit/old/do-cmake-albany-summit-gcc-cuda-sfad8 diff --git a/doc/summit/LandIce/old/do-cmake-albany-summit-gcc-serial-sfad8 b/doc/LandIce/summit/old/do-cmake-albany-summit-gcc-serial-sfad8 similarity index 100% rename from doc/summit/LandIce/old/do-cmake-albany-summit-gcc-serial-sfad8 rename to doc/LandIce/summit/old/do-cmake-albany-summit-gcc-serial-sfad8 diff --git a/doc/summit/LandIce/old/do-cmake-trilinos-summit-gcc-cuda b/doc/LandIce/summit/old/do-cmake-trilinos-summit-gcc-cuda similarity index 100% rename from doc/summit/LandIce/old/do-cmake-trilinos-summit-gcc-cuda rename to doc/LandIce/summit/old/do-cmake-trilinos-summit-gcc-cuda diff --git a/doc/summit/LandIce/old/do-cmake-trilinos-summit-gcc-serial b/doc/LandIce/summit/old/do-cmake-trilinos-summit-gcc-serial similarity index 100% rename from doc/summit/LandIce/old/do-cmake-trilinos-summit-gcc-serial rename to doc/LandIce/summit/old/do-cmake-trilinos-summit-gcc-serial diff --git a/doc/summit/LandIce/old/nvcc_wrapper70 b/doc/LandIce/summit/old/nvcc_wrapper70 similarity index 100% rename from doc/summit/LandIce/old/nvcc_wrapper70 rename to doc/LandIce/summit/old/nvcc_wrapper70 diff --git a/doc/summit/LandIce/old/summit_gcc_modules.sh b/doc/LandIce/summit/old/summit_gcc_modules.sh similarity index 100% rename from doc/summit/LandIce/old/summit_gcc_modules.sh rename to doc/LandIce/summit/old/summit_gcc_modules.sh diff --git a/doc/summit/LandIce/summit_gcc_modules.sh b/doc/LandIce/summit/summit_gcc_modules.sh similarity index 80% rename from doc/summit/LandIce/summit_gcc_modules.sh rename to doc/LandIce/summit/summit_gcc_modules.sh index acb6a955fd..1fe728747d 100644 --- a/doc/summit/LandIce/summit_gcc_modules.sh +++ b/doc/LandIce/summit/summit_gcc_modules.sh @@ -6,9 +6,11 @@ module unload xl module load gcc/9.1.0 module load spectrum-mpi/10.4.0.3-20210112 module load git cmake -module load boost/1.76.0 +#module load boost/1.76.0 +module load boost/1.77.0 module load hdf5/1.10.7 -module load netcdf-c/4.8.0 +#module load netcdf-c/4.8.0 +module load netcdf-c/4.8.1 module load netcdf-fortran/4.4.5 module load parallel-netcdf/1.12.2 module load netlib-lapack/3.9.1