nansencenter · sgao22006 · Oct 10, 2024 · Oct 10, 2024
diff --git a/README.betzy b/README.betzy
@@ -1,16 +1,18 @@
 !+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-1) Recommended .bashrc content for BETZY
+1) Recommended environment for BETZY
 
-ml purge
-ml load ESMF/8.3.0-iomkl-2022a
-ml load UDUNITS/2.2.28-GCCcore-11.3.0
-ml load Python/3.10.4-GCCcore-11.3.0
-ml load GSL/2.7-intel-compilers-2022.1.0
-ml load FFTW/3.3.10-GCC-11.3.0
-ml load CMake/3.23.1-GCCcore-11.3.0
+   ml purge
+   ml load ESMF/8.3.0-iomkl-2022a
+   ml load UDUNITS/2.2.28-GCCcore-11.3.0
+   ml load Python/3.10.4-GCCcore-11.3.0
+   ml load GSL/2.7-intel-compilers-2022.1.0
+   ml load FFTW/3.3.10-GCC-11.3.0
+   ml load CMake/3.23.1-GCCcore-11.3.0
 
-ulimit -s 2000000   ## if needed
+   ulimit -s 2000000   ## if needed
 
+   execute the following line to get the environment (adjust the path if needed): 
+      source ~/NERSC-HYCOM-CICE/betzy_env.sh  
 
 2) Compiling hycom_all under NERSC-HYCOM-CICE/hycom/hycom_ALL/hycom_2.2.72_ALL
 

diff --git a/README.fram b/README.fram
@@ -1,14 +1,16 @@
 !+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-1) Recommended .bashrc content for FRAM
-
-ml purge
-module load Python/3.10.4-GCCcore-11.3.0
-module load ESMF/8.3.0-intel-2022a
-module load FFTW/3.3.10-GCC-11.3.0
-module load UDUNITS/2.2.28-GCCcore-11.3.0
-module load CMake/3.23.1-GCCcore-11.3.0
-module load intel/2022a
-
+1) Recommended environment for FRAM
+
+   ml purge
+   module load Python/3.10.4-GCCcore-11.3.0
+   module load ESMF/8.3.0-intel-2022a
+   module load FFTW/3.3.10-GCC-11.3.0
+   module load UDUNITS/2.2.28-GCCcore-11.3.0
+   module load CMake/3.23.1-GCCcore-11.3.0
+   module load intel/2022a
+
+   execute the following line to get the environment (adjust the path if needed):
+      source ~/NERSC_HYCOM_CICE/fram_env.sh
 
 2) Compiling hycom_all under NERSC-HYCOM-CICE/hycom/hycom_ALL/hycom_2.2.72_ALL
 

diff --git a/TP2a0.10/expt_01.0/srjob.sh b/TP2a0.10/expt_01.0/srjob.sh
@@ -1,101 +1,64 @@
 #!/bin/bash -l
 
-#SBATCH --account=nn9481k
-
-#SBATCH -J  TP2a010
-
-#SBATCH -N 10   # number of nodes
-#SBATCH -n 22   # number of nodes
-## Each compute node has 24 cores (See more details in section Hardware on Sisu User Guide).
-
-##SBATCH -p small_long
-## Choose a suitable queue <test,small,large>
-## How to check queue limits: scontrol show part <queue name>
-## for example: scontrol show part small
-
-## System message output file
-#SBATCH -o log/HYCICE.%J.out
-
-## System error message file
-#SBATCH -e log/HYCICE.%J.err
-
-## How long job takes, wallclock time hh:mm:ss
-#SBATCH -t 00:30:00
-
-#SBATCH  --mail-type=END
-#SBATCH [email protected]
-
-## option: -n (total number of mpi processes)
-## option: -N (number of mpi processes per compute node)
-## option: -S (number of mpi processes per NUMA node)
-## option: -ss (allocate memory only from a local NUMA node)
-
-## Calculate the total number of cores and store it in variable ncores
-##(( ncores = SLURM_NNODES * 24 ))
-##(( ncores = SLURM_NNODES * 32 ))  #on Fram
-
-module load NCL/6.6.2-intel-2018b
-module load FFTW/3.3.8-intel-2018b
-module load Python/2.7.15-intel-2018b
-
-
-export NMPI=220
+#SBATCH --account=nn9481k 
+#SBATCH --job-name=TP2a010
+#SBATCH --time="00:06:00"
+#SBATCH --nodes=4 # number of nodes
+#SBATCH --ntasks=504 # number of cores
+#SBATCH --mail-type=END
+
+#SBATCH -o log/HY_CICE.out
+#SBATCH -e log/HY_CICE.err
+
+#         
+export NMPI=504
 export SLURM_SUBMIT_DIR=$(pwd)
 # Enter directory from where the job was submitted
-cd $SLURM_SUBMIT_DIR       ||  { echo "Could not go to dir $SLURM_SUBMIT_DIR  "; exit 1; }
+cd $SLURM_O_WORKDIR       ||  { echo "Could not go to dir $SLURM_O_WORKDIR  "; exit 1; }
 
 # ------------------- Fetch Environment ------------------------------
 # -------- these are needed in preprocess scripts---------------------
 echo "SLURM_JOBID    = $SLURM_JOBID     "
-echo "SLURM_SUBMIT_DIR= $SLURM_SUBMIT_DIR "
+echo "SLURM_JOBNAME  = $SLURM_JOBNAME   "
+echo "SLURM_SUBMIT_DIR= $SLURM_SUBMIT_KDIR "
+echo "SLURM_TASKNUM  = $SLURM_TASKNUM "
+echo "SLURM_NUM_PPN  = $SLURM_NUM_PPN "
 [ -z "$NOMP" ] && NOMP=0
 
 # Enter directory from where the job was submitted
 cd $SLURM_SUBMIT_DIR       ||  { echo "Could not go to dir $SLURM_O_WORKDIR  "; exit 1; }
 
-
 # Initialize environment (sets Scratch dir ($S), Data dir $D ++ )
 source ../REGION.src  || { echo "Could not source ../REGION.src "; exit 1; }
 source ./EXPT.src  || { echo "Could not source EXPT.src"; exit 1; }
-echo "NMPI =$NMPI (Number of MPI tasks needed for running job) "
-
-
-for ik in `seq 1 1` ; do
-  if [ $ik -eq 0 ] ; then
-    START="2001-12-25T00:00:00"
-    END="2002-01-01T00:00:00"
-    INITFLG="--init"
-    cp -f ice_in.0 ice_in
-  else
-    INITFLG=""
-    cp -f ice_in.1 ice_in
-    if [ $ik -eq 1 ] ; then
-      START="2001-12-25T00:00:00"
-      END="2002-01-02T00:00:00"
-    else 
-      START="2010-01-02T00:00:00"
-      END="2016-01-02T00:00:00"
-    fi
-  fi
-  echo "Start time in pbsjob.sh: $START"
-  echo "End   time in pbsjob.sh: $END"
+source $NHCROOT/environment/betzy_env.sh || { echo "Could not source betzy_env.sh "; exit 1; }
+#source $NHCROOT/environment/fram_env.sh || { echo "Could not source fram_env.sh "; exit 1; }
 
-  # Generate atmospheric forcing :
-  atmo_synoptic.sh erai $START $END 
-
-  # Transfer data files to scratch - must be in "expt_XXX" dir for this script
-  expt_preprocess.sh $START $END $INITFLG        ||  { echo "Preprocess had fatal errors "; exit 1; }
-
-  # Enter Scratch/run dir and Run model
-  cd $S  ||  { echo "Could not go to dir $S  "; exit 1; }
-  ##aprun -n $NMPI -m 500M ./hycom_cice  > ../log/hycom.%J.out 2>&1
-  srun --mpi=pmi2 -n $NMPI --cpu_bind=cores ./hycom_cice  
-
-  # Cleanup and move data files to data directory - must be in "expt_XXX" dir for this script
-  cd $P     ||  { echo "Could not go to dir $P  "; exit 1; }
-  expt_postprocess.sh 
+echo "NMPI =$NMPI (Number of MPI tasks needed for running job) "
 
-done
+START="1990-01-01T00:00:00"
+END="1990-01-05T00:00:00"
+INITFLG="--init"
+#INITFLG=""
+echo "Start time in pbsjob.sh: $START"
+echo "End   time in pbsjob.sh: $END"
+# Generate atmospheric forcing :
+#atmo_synoptic.sh erai+all $START $END 
+../bin/atmo_synoptic.sh era5+lw $START $END
+
+# Transfer data files to scratch - must be in "expt_XXX" dir for this script
+# expt_preprocess.sh $START $END $INITFLG        ||  { echo "Preprocess had fatal errors "; exit 2; }
+../bin/expt_preprocess.sh $START $END $INITFLG        ||  { echo "Preprocess had fatal errors "; exit 1; }
+
+# Enter Scratch/run dir and Run model
+cd $S  ||  { echo "Could not go to dir $S  "; exit 1; }
+#srun -n $NMPI ./hycom_cice  > ../log/hycom.${SLURM_JOBID}.out 2>&1
+srun -n $NMPI --cpu_bind=cores ./hycom_cice 
+
+# Cleanup and move data files to data directory - must be in "expt_XXX" dir for this script
+cd $P     ||  { echo "Could not go to dir $P  "; exit 1; }
+expt_postprocess.sh 
 
 exit $?
 
+
diff --git a/TP5a0.06/expt_01.1/EXPT.src b/TP5a0.06/expt_01.1/EXPT.src
@@ -1,4 +1,4 @@
-#!/binn/bash
+#!/bin/bash
 #
 # Sets environment for this experiment, also makes the scratch (S) and
 # data (D) directories if not already present
@@ -20,7 +20,7 @@ mydir=$(cd $(dirname ${BASH_SOURCE}) && pwd)
 unset -v X E T V K P D S
 X="01.1"                # X based on dir name (expt_01.1)
 E="011"                 # E is X without "."
-T="08"                                                           # Topography version
+T="05"                                                           # Topography version
 export V=2.2.98                                                  # hycom version              
 #export K=`grep "'kdm   ' =" blk* | awk '{printf("%03d", $1)}'`   # get kdm from blkdat
 export K=`grep "'kdm   ' =" $mydir/blkdat.input | awk '{printf("%03d", $1)}'`   # get kdm from blkdat
@@ -33,6 +33,7 @@ export S=$P/SCRATCH                  # Scratch area
 export SIGVER=1   # Version of equation of state (this is 7-term sigma 2). Must not cause conflict with thflag in blkdat.input
 export NMPI=336
 export MXBLCKS=2  # The maximal number of ice blocks relates to the MPI cores distribution, and could be recommened by the error message in a model run.
+export COMPILE_BIOMODEL="no" # Turn ECOSMO on with "yes"
 
 # Consistency check. Ensures expt dir ends in expt_X
 #echo $tmp

diff --git a/TP5a0.06/topo/cice_grid.nc b/TP5a0.06/topo/cice_grid.nc
diff --git a/bin/create_ref_case.sh b/bin/create_ref_case.sh
@@ -131,7 +131,7 @@ if [ ${iceclim} -eq 1 ]; then
       ml load matplotlib/3.5.2-intel-2022a
       ${BINDIR}/Grid_Bathy/cice_kmt.py regional.depth.a
    fi
-   prg=${BINDIR}ice_climatology/extract_clim_iceh_update.sh
+   prg=${BINDIR}ice_climatology/extract_clim_iceh.sh
    ${prg} ${EDIR}
    cd ${EDIR}
    [ -r ice_clim ] && rm -rf ice_clim

diff --git a/bin/expt_preprocess.sh b/bin/expt_preprocess.sh
@@ -288,9 +288,10 @@ ${pget} $BASEDIR/topo/regional.grid.a regional.grid.a || tellerror "no grid file
 ${pget} $BASEDIR/topo/regional.grid.b regional.grid.b || tellerror "no grid file regional.grid.a" 
 ${pget} $BASEDIR/topo/depth_${R}_${T}.a regional.depth.a || tellerror "no topo file depth_${R}_${T}.a" 
 ${pget} $BASEDIR/topo/depth_${R}_${T}.b regional.depth.b || tellerror "no topo file depth_${R}_${T}.b" 
-${pget} $BASEDIR/topo/kmt_${R}_${T}.nc cice_kmt.nc     || tellerror "no kmt file $BASEDIR/topo/kmt_${R}_${T}.nc "
-${pget} $BASEDIR/topo/cice_grid.nc cice_grid.nc        || tellerror "no cice grid file $BASEDIR/topo/cice_grid.nc "
-
+if [ $ICEFLG != 0 ] ; then
+   ${pget} $BASEDIR/topo/kmt_${R}_${T}.nc cice_kmt.nc     || tellerror "no kmt file $BASEDIR/topo/kmt_${R}_${T}.nc "
+   ${pget} $BASEDIR/topo/cice_grid.nc cice_grid.nc        || tellerror "no cice grid file $BASEDIR/topo/cice_grid.nc "
+fi
 
 if [ "$SSTRLX" -eq 3 ] ; then
    [ -f  $CLMDIR/seatmp.a ] || tellerror "File $CLMDIR/seatmp.a does not exist"

diff --git a/bin/ice_climatology/extract_clim_iceh.sh b/bin/ice_climatology/extract_clim_iceh.sh
@@ -87,7 +87,7 @@ if [ -s ${Fini} -a -s ${Ftmpmask} ]; then
    ml load CDO/2.0.6-gompi-2022a
    cdo remapbil,${Ftmpmask} ${Fini} ${Fout}
 
-   ml load NCO/5.1.3-iimpi-2022a
+   ml load NCO/5.1.9-iomkl-2022a
    echo "Defaulting the values for different masks ... "
    ncks -v ${Vars} ${Fout} ${Ftmp}  
    ncrename -h -O -v siconc,aice_raw -v sithick,hi_raw ${Ftmp}  

diff --git a/contributors.txt b/contributors.txt
@@ -6,3 +6,4 @@ Roshin Raj
 Annette Samuelsen
 Achref Othmani
 Till Rasmussen
+Shuang Gao
diff --git a/environment/betzy_env.sh b/environment/betzy_env.sh
@@ -0,0 +1,10 @@
+# Environment for compiling/running HYCOM-CICE on betzy
+ml purge
+ml load ESMF/8.3.0-iomkl-2022a
+ml load UDUNITS/2.2.28-GCCcore-11.3.0
+ml load Python/3.10.4-GCCcore-11.3.0
+ml load GSL/2.7-intel-compilers-2022.1.0
+ml load FFTW/3.3.10-GCC-11.3.0
+ml load CMake/3.23.1-GCCcore-11.3.0
+
+ulimit -s 2000000
diff --git a/environment/fram_env.sh b/environment/fram_env.sh
@@ -0,0 +1,8 @@
+# Environment for compiling/running HYCOM-CICE on fram 
+ml purge
+module load Python/3.10.4-GCCcore-11.3.0
+module load ESMF/8.3.0-intel-2022a
+module load FFTW/3.3.10-GCC-11.3.0
+module load UDUNITS/2.2.28-GCCcore-11.3.0
+module load CMake/3.23.1-GCCcore-11.3.0
+module load intel/2022a