Skip to content

Commit

Permalink
Merge branch 'develop' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
aerorahul authored Jul 2, 2024
2 parents e271417 + 11943e3 commit 9d85f3d
Show file tree
Hide file tree
Showing 6 changed files with 55 additions and 16 deletions.
40 changes: 34 additions & 6 deletions env/GAEA.env
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,52 @@ step=$1
export launcher="srun -l --export=ALL"
export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"

export OMP_STACKSIZE=2048000
export NTHSTACK=1024000000

ulimit -s unlimited
ulimit -a

if [[ "${step}" = "fcst" ]]; then
if [[ "${step}" = "waveinit" ]]; then

export CFP_MP="YES"
if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
export wavempexec=${launcher}
export wave_mpmd=${mpmd_opt}

ppn="npe_node_${step}_${RUN}"
[[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
nprocs="npe_${step}_${RUN}"
[[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"
elif [[ "${step}" = "fcst" ]]; then

if [[ "${CDUMP}" =~ "gfs" ]]; then
nprocs="npe_${step}_gfs"
ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
else
nprocs="npe_${step}"
ppn="npe_node_${step}"
fi
(( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
(( ntasks = nnodes*${!ppn} ))
# With ESMF threading, the model wants to use the full node
export APRUN_UFS="${launcher} -n ${ntasks}"
unset nprocs ppn nnodes ntasks


elif [[ "${step}" = "atmos_products" ]]; then

export USE_CFP="YES" # Use MPMD for downstream product generation
export USE_CFP="YES" # Use MPMD for downstream product generation on Hera

elif [[ "${step}" = "oceanice_products" ]]; then

nth_max=$((npe_node_max / npe_node_oceanice_products))

export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"

elif [[ "${step}" = "fit2obs" ]]; then

nth_max=$((npe_node_max / npe_node_fit2obs))

export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
[[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
export MPIRUN="${launcher} -n ${npe_fit2obs} --cpus-per-task=${NTHREADS_FIT2OBS}"

fi
1 change: 1 addition & 0 deletions parm/config/gfs/config.base
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export QUEUE_SERVICE="@QUEUE_SERVICE@"
export PARTITION_BATCH="@PARTITION_BATCH@"
export PARTITION_SERVICE="@PARTITION_SERVICE@"
export RESERVATION="@RESERVATION@"
export CLUSTERS="@CLUSTERS@"

# Project to use in mass store:
export HPSS_PROJECT="@HPSS_PROJECT@"
Expand Down
5 changes: 5 additions & 0 deletions parm/config/gfs/config.resources.GAEA
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,8 @@ case ${step} in
;;

esac

# shellcheck disable=SC2312
for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
unset "${mem_var}"
done
2 changes: 1 addition & 1 deletion sorc/link_workflow.sh
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ case "${machine}" in
"hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;;
"jet") FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix" ;;
"s4") FIX_DIR="/data/prod/glopara/fix" ;;
"gaea") FIX_DIR="/gpfs/f5/epic/proj-shared/global/glopara/data/fix" ;;
"gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;;
*)
echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR"
exit 1
Expand Down
21 changes: 12 additions & 9 deletions workflow/hosts/gaea.yaml
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
BASE_GIT: '/gpfs/f5/epic/proj-shared/global/glopara/data/git'
DMPDIR: '/gpfs/f5/epic/proj-shared/global/glopara/data/dump'
BASE_CPLIC: '/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR/prototype_ICs'
PACKAGEROOT: '/gpfs/f5/epic/proj-shared/global/glopara/data/nwpara'
COMROOT: '/gpfs/f5/epic/proj-shared/global/glopara/data/com'
BASE_GIT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git'
DMPDIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/dump'
BASE_CPLIC: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDIR/prototype_ICs'
PACKAGEROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/nwpara'
COMROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/com'
COMINsyn: '${COMROOT}/gfs/prod/syndat'
HOMEDIR: '/gpfs/f5/epic/scratch/${USER}'
STMP: '/gpfs/f5/epic/scratch/${USER}'
PTMP: '/gpfs/f5/epic/scratch/${USER}'
HOMEDIR: '/gpfs/f5/ufs-ard/scratch/${USER}'
STMP: '/gpfs/f5/ufs-ard/scratch/${USER}'
PTMP: '/gpfs/f5/ufs-ard/scratch/${USER}'
NOSCRUB: $HOMEDIR
ACCOUNT: epic
ACCOUNT: ufs-ard
ACCOUNT_SERVICE: ufs-ard
SCHEDULER: slurm
QUEUE: normal
QUEUE_SERVICE: normal
PARTITION_BATCH: batch
PARTITION_SERVICE: batch
RESERVATION: ''
CLUSTERS: 'c5'
CHGRP_RSTPROD: 'NO'
CHGRP_CMD: 'chgrp rstprod'
HPSSARCH: 'NO'
Expand Down
2 changes: 2 additions & 0 deletions workflow/rocoto/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,8 @@ def get_resource(self, task_name):
native = '--export=NONE'
if task_config['RESERVATION'] != "":
native += '' if task_name in Tasks.SERVICE_TASKS else ' --reservation=' + task_config['RESERVATION']
if task_config['CLUSTERS'] != "":
native += ' --clusters=' + task_config['CLUSTERS']

queue = task_config['QUEUE_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_config['QUEUE']

Expand Down

0 comments on commit 9d85f3d

Please sign in to comment.