-
Notifications
You must be signed in to change notification settings - Fork 90
Shannon
Glen Hansen edited this page Nov 8, 2015
·
9 revisions
Shannon is integrated by Appro
- Current configuration installed 9/2014
- 22 compute nodes
- Each node has two 8 core Sandy Bridge Xeon E5-2670 processors @ 2.6 GHz
- Each node has 128 GB - DDR3 memory 1600 MHz
- 6 nodes have 2x NVIDIA K20x (Kepler), 12 have 4x K20x, 4 have 4x K40m, and 9 have 2x K80s
- 2688 732MHz cores
- Mellanox QDR Infiniband
Here is the module load sequence used
module purge
module load cuda/7.5.7
module load gcc/4.9.0
export CUDA_LAUNCH_BLOCKING=1
-
Version 3.1.4
cd /home/gahanse/src/ wget http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz tar -xvf mpich-3.1.4.tar.gz cd mpich-3.1.4 ./configure --prefix=/home/gahanse/gcc-4.9.0/mpich-3.1.4 make -j 8 make install
These install directions assume that a base install directory of /home/gahanse/gcc-4.9.0/mpich-3.1.4 is used
-
Version 1.2.8
cd /home/gahanse/src wget http://zlib.net/zlib-1.2.8.tar.gz tar -xvf zlib-1.2.8.tar.gz cd zlib-1.2.8 CC=mpicc CFLAGS="-O3" ./configure --prefix=/home/gahanse/gcc-4.9.0/mpich-3.1.4 make install
-
Version 1.9
./configure --prefix=/home/gahanse/gcc-4.9.0/mpich-3.1.4 CC=mpicc CXX=mpicxx
-
Version 1.8.13
cd /home/gahanse/src/ wget http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.13.tar.bz2 tar -xvf hdf5-1.8.13.tar.gz cd hdf5-1.8.13 ./configure CC=mpicc FC=mpif90 CXX=mpicxx CXXFLAGS="-fPIC -O3" CFLAGS="-fPIC -O3" FCFLAGS="-fPIC -O3" \ --enable-parallel --with-zlib=/home/gahanse/gcc-4.9.0/mpich-3.1.4 --prefix=/home/gahanse/gcc-4.9.0/mpich-3.1.4 make -j 8 make install
-
Version 4.3.3.1
cd /home/gahanse/src/ wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.1.tar.gz tar -xvf netcdf-4.3.3.1.tar.gz cd netcdf-4.3.3.1 -
Edit the file
include/netcdf.h. Make the following edits near line 228:#define NC_MAX_DIMS 65536 /* max dimensions per file */ #define NC_MAX_ATTRS 8192 #define NC_MAX_VARS 524288 /* max variables per file */ #define NC_MAX_NAME 256 #define NC_MAX_VAR_DIMS 8 /* max per variable dimensions */ -
Now, configure build:
./configure --prefix=/home/gahanse/gcc-4.9.0/mpich-3.1.4 CC=mpicc FC=mpifort CXX=mpicxx \ CFLAGS="-I/home/gahanse/gcc-4.9.0/mpich-3.1.4/include -fpic -O3" \ CXXFLAGS="-I/home/gahanse/gcc-4.9.0/mpich-3.1.4/include -fPIC -O3" \ LDFLAGS=-L/home/gahanse/gcc-4.9.0/mpich-3.1.4/lib \ FCFLAGS="-fPIC -O3" --disable-fsync --disable-cdmremote --disable-dap --disable-doxygen --enable-netcdf-4 make -j 8 make install
-
Version 1.59.0
cd /home/gahanse/src/ wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.gz/download tar -xvf boost_1_59_0.tar.gz cd boost_1_59_0 -
Build boost with the commands
echo "using gcc : 4.9.0 : /home/projects/gcc/4.9.0/bin/g++ ;" > ~/user-config.jam echo "using mpi : /home/gahanse/gcc-4.9.0/mpich-3.1.4/bin/mpicxx ;" >> ~/user-config.jam ./bootstrap.sh \ --with-libraries=signals,regex,filesystem,system,mpi,serialization,thread,program_options,exception \ --prefix=/home/gahanse/gcc-4.9.0/mpich-3.1.4 ./b2 -j 8 ./b2 -j 8 install
-
Version 4.0.3
cd /home/gahanse/src/ wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz tar -xvf parmetis-4.0.3.tar.gz cd parmetis-4.0.3 -
Edit
metis.hto use anIDXTYPEWIDTHof 64cd metis/include vi metis.h #define IDXTYPEWIDTH 64 #define REALTYPEWIDTH 64 -
Run CMake to build
parmetiscd /home/gahanse/src/parmetis-4.0.3 cd build export metis=/home/gahanse/src/parmetis-4.0.3/metis cmake \ -DCMAKE_INSTALL_PREFIX=/home/gahanse/gcc-4.9.0/mpich-3.1.4 \ -DMETIS_PATH=$metis \ -DGKLIB_PATH=$metis/GKlib \ -DCMAKE_C_COMPILER=mpicc \ -DCMAKE_CXX_COMPILER=mpicxx \ -DMPI_INCLUDE_PATH=/home/gahanse/gcc-4.9.0/mpich-3.1.4/include \ -DCMAKE_C_FLAGS="-fPIC -O3" \ -DCMAKE_CXX_FLAGS="-fPIC -O3" \ .. make -j 8 make install cp libmetis/libmetis.a /home/gahanse/gcc-4.9.0/mpich-3.1.4/lib cp $metis/include/metis.h /home/gahanse/gcc-4.9.0/mpich-3.1.4/include
-
Download Trilinos
cd /home/gahanse/src git clone [email protected]:/git/Trilinos -
Download SCOREC
cd /home/gahanse/src/Trilinos git clone [email protected]:SCOREC/core.git SCOREC -
Configure the Trilinos build
mkdir build cd build ./do-configure -
Example
do-configurescript executed above#!/bin/sh # export TRILINOS_HOME=/ascldap/users/gahanse/src/Trilinos BUILD_DIR=`pwd` INSTALL_DIR=/ascldap/users/gahanse/trilinos/MPI_REL BOOST_DIR=/ascldap/users/gahanse LIB_DIR=$BOOST_DIR MPI_BASE_DIR=/opt/openmpi-1.4-intel NETCDF=/ascldap/users/gahanse HDFDIR=/ascldap/users/gahanse PARMETISDIR=$BOOST_DIR SUPERLUDIR=$BOOST_DIR/SuperLU_4.3 MKL_PATH=/opt/intel-12.1 LABLAS_LIBRARIES="-L$MKL_PATH/lib/intel64 -Wl,--start-group $MKL_PATH/mkl/lib/intel64/libmkl_intel_lp64.a $MKL_PATH/mkl/lib/intel64/libmkl_core.a $MKL_PATH/mkl/lib/intel64/libmkl_sequential.a -Wl,--end-group -lpthread" cmake \ -Wno-dev \ -D Trilinos_CONFIGURE_OPTIONS_FILE:FILEPATH=$TRILINOS_HOME/sampleScripts/AlbanySettings.cmake \ -D Trilinos_EXTRA_REPOSITORIES:STRING="SCOREC" \ -D Trilinos_ENABLE_SCOREC:BOOL=ON \ -D SCOREC_DISABLE_STRONG_WARNINGS:BOOL=ON \ -D CMAKE_BUILD_TYPE:STRING=NONE \ -D CMAKE_CXX_FLAGS:STRING="-O3 -xAVX -DNDEBUG" \ -D CMAKE_C_FLAGS:STRING="-O3 -xAVX -DNDEBUG" \ -D CMAKE_Fortran_FLAGS:STRING="-O3 -xAVX -DNDEBUG" \ -D Rythmos_ENABLE_DEBUG:BOOL=ON \ \ -D TPL_ENABLE_MPI:BOOL=ON \ -D MPI_BASE_DIR:PATH=$MPI_BASE_DIR \ \ -D TPL_ENABLE_Pthread:BOOL=ON \ \ -D TPL_ENABLE_Boost:BOOL=ON \ -D TPL_ENABLE_BoostLib:BOOL=ON \ -D TPL_ENABLE_BoostAlbLib:BOOL=ON \ -D Boost_INCLUDE_DIRS:PATH=$BOOST_DIR/include \ -D Boost_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \ -D BoostLib_INCLUDE_DIRS:PATH=$BOOST_DIR/include \ -D BoostLib_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \ -D BoostAlbLib_INCLUDE_DIRS:PATH=$BOOST_DIR/include \ -D BoostAlbLib_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \ \ -D TPL_ENABLE_Netcdf:STRING=ON \ -D Netcdf_INCLUDE_DIRS:PATH=$NETCDF/include \ -D Netcdf_LIBRARY_DIRS:PATH=$NETCDF/lib \ -D TPL_ENABLE_HDF5:STRING=ON \ -D HDF5_INCLUDE_DIRS:PATH="${HDFDIR}/include" \ -D TPL_HDF5_LIBRARIES:PATH="${HDFDIR}/lib/libhdf5_hl.a;${HDFDIR}/lib/libhdf5.a;${HDFDIR}/lib/libz.a" \ -D Trilinos_EXTRA_LINK_FLAGS="-L${HDFDIR}/lib -lhdf5_hl -lhdf5 -lz -lm" \ \ -D TPL_ENABLE_Zlib:STRING=ON \ -D Zlib_INCLUDE_DIRS:PATH=$HDFDIR/include \ -D TPL_Zlib_LIBRARIES:PATH="$HDFDIR/lib/libz.a" \ \ -D TPL_ENABLE_BLAS:STRING=ON \ -D TPL_ENABLE_LAPACK:STRING=ON \ -D TPL_BLAS_LIBRARIES:STRING="$LABLAS_LIBRARIES" \ -D TPL_LAPACK_LIBRARIES:STRING="$LABLAS_LIBRARIES" \ \ -D TPL_ENABLE_ParMETIS:STRING=ON \ -D ParMETIS_INCLUDE_DIRS:PATH="${PARMETISDIR}/include" \ -D ParMETIS_LIBRARY_DIRS:PATH="${PARMETISDIR}/lib" \ \ -D CMAKE_INSTALL_PREFIX:PATH=$INSTALL_DIR \ \ -D Trilinos_ENABLE_TriKota:BOOL=OFF \ -D DAKOTA_ENABLE_TESTS:BOOL=OFF \ \ $EXTRA_ARGS \ ${TRILINOS_HOME} exit 0 -
Build Trilinos
make -j 8 make install
-
Clone the repo
cd /ascldap/users/gahanse/src git clone [email protected]:gahansen/Albany.git cd Albany -
Configure the Albany build
mkdir build cd build ./do-configure -
CMake configure script
./do-configure:#!/bin/sh export ALB_64BIT_INT=ON export ALB_EPETRA=OFF export ALB_SEE=OFF export ALB_ENABLE_SCOREC=ON export ALB_ENABLE_LCM=ON export ALB_ENABLE_LAME=ON export ALB_ENABLE_HYDRIDE=OFF export ALB_ENABLE_QCAD=OFF export ALB_ENABLE_MOR=OFF export ALB_ENABLE_ASCR=OFF export ALB_ENABLE_AERAS=OFF export ALB_ENABLE_FELIX=OFF export ALB_ENABLE_AlbanyCI=OFF export ALB_DEBUG=OFF export ALB_ENABLE_CHECK_FPE=ON export ALB_ENABLE_SPECULATIVE=OFF export ALB_ENABLE_SG_MP=OFF export ALB_ENABLE_PERF_TESTS=ON export TRILINOS_INSTALL_DIR=/ascldap/users/gahanse/trilinos/MPI_REL export ALB_LAME_INC_DIR=/ascldap/users/gahanse/src/lame-4.24.1/lame/include export ALB_LAME_LIB_DIR=/ascldap/users/gahanse/src/lame-4.24.1/lame/build_intel export ALB_VERBOSE=OFF cmake \ -D ALBANY_TRILINOS_DIR:FILEPATH="$TRILINOS_INSTALL_DIR" \ -D ENABLE_LCM:BOOL=${ALB_ENABLE_LCM} \ -D ENABLE_AERAS:BOOL=${ALB_ENABLE_AERAS} \ -D ENABLE_QCAD:BOOL=${ALB_ENABLE_QCAD} \ -D ENABLE_HYDRIDE:BOOL=${ALB_ENABLE_HYDRIDE} \ -D ENABLE_LCM_SPECULATIVE:BOOL=${ALB_ENABLE_SPECULATIVE} \ -D ENABLE_LAME:BOOL=${ALB_ENABLE_LAME} \ -D CMAKE_VERBOSE_MAKEFILE:BOOL=${ALB_VERBOSE} \ -D ENABLE_DEBUGGING:BOOL=${ALB_DEBUG} \ -D ENABLE_CHECK_FPE:BOOL=${ALB_ENABLE_CHECK_FPE} \ -D ENABLE_SCOREC:BOOL=${ALB_ENABLE_SCOREC} \ -D ENABLE_FELIX:BOOL=${ALB_ENABLE_FELIX} \ -D ENABLE_MOR:BOOL=${ALB_ENABLE_MOR} \ -D ENABLE_ALBANY_CI:BOOL=${ALB_ENABLE_AlbanyCI} \ -D ENABLE_ASCR:BOOL=${ALB_ENABLE_ASCR} \ -D ENABLE_SG_MP:BOOL=${ALB_ENABLE_SG_MP} \ -D ENABLE_PERFORMANCE_TESTS:BOOL=${ALB_ENABLE_PERF_TESTS} \ -D LAME_INCLUDE_DIR:FILEPATH="$ALB_LAME_INC_DIR" \ -D LAME_LIBRARY_DIR:FILEPATH="$ALB_LAME_LIB_DIR" \ -D ALBANY_CTEST_TIMEOUT:INTEGER=70 \ -D ENABLE_64BIT_INT:BOOL=${ALB_64BIT_INT} \ -D ENABLE_ALBANY_EPETRA_EXE:BOOL=${ALB_EPETRA} \ -D ENABLE_SEE:BOOL=${ALB_SEE} \ -D ALBANY_MPI_OPTIONS:BOOL=ON \ -D ALBANY_MPI_EXEC:STRING="mpiexec" \ -D ALBANY_MPI_EXEC_NUMPROCS_FLAG:STRING="-n" \ -D ALBANY_MPI_EXEC_MAX_NUMPROCS:STRING="4" \ -D ALBANY_MPI_LEADING_OPTIONS:STRING="" \ \ .. -
Build Albany
make -j 8 make install