diff --git a/env/HERA.env b/env/HERA.env index d06d02194c..67dca0ca6c 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -16,7 +16,7 @@ step=$1 export npe_node_max=40 export launcher="srun -l --export=ALL" -export mpmd_opt="--multi-prog --output=${step}.%J.%t.out" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment #export I_MPI_ADJUST_ALLREDUCE=5 @@ -215,6 +215,7 @@ elif [[ "${step}" = "post" ]]; then [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} export APRUN_NP="${launcher} -n ${npe_post}" + export USE_CFP="YES" # Use MPMD for downstream product generation on Hera export NTHREADS_DWN=${nth_dwn:-1} [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} export APRUN_DWN="${launcher} -n ${npe_dwn}" @@ -302,3 +303,4 @@ elif [[ "${step}" = "fit2obs" ]]; then export MPIRUN="${launcher} -n ${npe_fit2obs}" fi + diff --git a/env/JET.env b/env/JET.env index 805963a9b8..0c5d7185d1 100755 --- a/env/JET.env +++ b/env/JET.env @@ -22,7 +22,7 @@ elif [[ "${PARTITION_BATCH}" = "kjet" ]]; then export npe_node_max=40 fi export launcher="srun -l --epilog=/apps/local/bin/report-mem --export=ALL" -export mpmd_opt="--multi-prog --output=${step}.%J.%t.out" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment export OMP_STACKSIZE=2048000 @@ -201,6 +201,7 @@ elif [[ "${step}" = "post" ]]; then [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} export APRUN_NP="${launcher} -n ${npe_post}" + export USE_CFP="YES" # Use MPMD for downstream product generation on Jet export NTHREADS_DWN=${nth_dwn:-1} [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} export APRUN_DWN="${launcher} -n ${npe_dwn}" diff --git a/env/ORION.env b/env/ORION.env index 082587e051..400963d3c8 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -16,7 +16,7 @@ step=$1 export npe_node_max=40 export launcher="srun -l --export=ALL" -export mpmd_opt="--multi-prog --output=${step}.%J.%t.out" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment export MPI_BUFS_PER_PROC=2048 @@ -214,6 +214,7 @@ elif [[ "${step}" = "post" ]]; then [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} export APRUN_NP="${launcher} -n ${npe_post}" + export USE_CFP="YES" # Use MPMD for downstream product generation on Orion export NTHREADS_DWN=${nth_dwn:-1} [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} export APRUN_DWN="${launcher} -n ${npe_dwn}" diff --git a/env/S4.env b/env/S4.env index b957303306..1c81f72f6e 100755 --- a/env/S4.env +++ b/env/S4.env @@ -21,7 +21,7 @@ elif [[ ${PARTITION_BATCH} = "ivy" ]]; then export npe_node_max=20 fi export launcher="srun -l --export=ALL" -export mpmd_opt="--multi-prog --output=${step}.%J.%t.out" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment export OMP_STACKSIZE=2048000 @@ -187,6 +187,7 @@ elif [[ "${step}" = "post" ]]; then [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} export APRUN_NP="${launcher} -n ${npe_post}" + export USE_CFP="YES" # Use MPMD for downstream product generation on S4 export NTHREADS_DWN=${nth_dwn:-1} [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} export APRUN_DWN="${launcher} -n ${npe_dwn}" diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 6f01909f4e..22d65ba0ed 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -198,6 +198,7 @@ elif [[ "${step}" = "post" ]]; then [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} export APRUN_NP="${launcher} -n ${npe_np:-${npe_post}} -ppn ${npe_node_post} --cpu-bind depth --depth ${NTHREADS_NP}" + export USE_CFP="YES" # Use MPMD for downstream product generation on WCOSS2 export NTHREADS_DWN=${nth_dwn:-1} [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} export APRUN_DWN="${launcher} -np ${npe_dwn} ${mpmd_opt}" diff --git a/jobs/JGLOBAL_ATMOS_POST b/jobs/JGLOBAL_ATMOS_POST index 9fe618e891..a0cd8871e1 100755 --- a/jobs/JGLOBAL_ATMOS_POST +++ b/jobs/JGLOBAL_ATMOS_POST @@ -51,7 +51,7 @@ fi for grid in '0p25' '0p50' '1p00'; do prod_dir="COM_ATMOS_GRIB_${grid}" GRID=${grid} YMD=${PDY} HH=${cyc} generate_com -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL" - if [[ ! -d "${prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi + if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi done if [ "${RUN}" = gfs ];then diff --git a/ush/fv3gfs_downstream_nems.sh b/ush/fv3gfs_downstream_nems.sh index 48aacf0f07..c09b9a7d3f 100755 --- a/ush/fv3gfs_downstream_nems.sh +++ b/ush/fv3gfs_downstream_nems.sh @@ -1,294 +1,179 @@ #! /usr/bin/env bash -#----------------------------------------------------------------------- -#-Hui-Ya Chuang, January 2014: First version. -# This script was written to include all new Grib2 GFS downstream post processing in -# EMC's parallel so that EMC can reproduce all operational pgb files as in operations. -# Due to EMC's limited resources, MPMD is used to speed up downstream post processing. -#-Hui-Ya Chuang, September 2015: -# modification to generate select grids on non-WCOSS machines without using MPMD. -# NCEP R&D machines do not have MPMD. -#-Fanglin Yang, September 2015 -# 1. restructured and simplified the script. -# 2. use wgrib2 instead of copygb2 for interpolation. copygb2 is slow and is not -# working for converting grib2 files that are produced by nceppost using nemsio files. -# 3. use wgrib2 to convert pgbm on quarter-degree (no matter gaussian or lat-lon) grid -# to pgbq on quarter-degree lat-lon grid. -# 4. Between using COPYGB2 and WGRIB2, about 50% of the fields are bit-wise identical. -# Others are different at the noise level at spotted points. -#-Fanglin Yang, February 2016 -# 1. For NEMSIO, gfspost1 exceends 6-hour CPU limits on WCOSS. Remove pgrbh and pgrbl -#-Fanglin Yang, March 2017 -# 1. Modified for FV3GFS, using NCEP-NCO standard output name convention -# 2. Add option24 to turn on bitmap in grib2 file (from Wen Meng) -#-Wen Meng, January 2018, add flag PGB1F for turning on/ogg grib1 pgb data at 1.00 deg. generation. -#-Wen Meng, Feburary 2018 -# 1. Add flag PGBS for turning on/off pgb data at 1.0 and 0.5 deg. generation frequency of FHOUT_PGB defined. -#-Wen Meng, October 2019 -# 1. Use bilinear interpolation for LAND. It can trancate land-sea mask as 1 or 0. -#-Wen Meng, November 2019 -# 1. Modify sea icea cover via land-sea mask. -#----------------------------------------------------------------------- +source "${HOMEgfs}/ush/preamble.sh" "${FH}" -source "$HOMEgfs/ush/preamble.sh" "$FH" - -export downset=${downset:-1} -export DATA=${DATA:-/ptmpd2/$LOGNAME/test} -export CNVGRIB=${CNVGRIB:-${grib_util_ROOT}/bin/cnvgrib} -export COPYGB2=${COPYGB2:-${grib_util_ROOT}/bin/copygb} +# Programs used export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} -export GRBINDEX=${GRBINDEX:-${wgrib2_ROOT}/bin/grbindex} -export RUN=${RUN:-"gfs"} -export cycn=$(echo $CDATE |cut -c 9-10) -export TCYC=${TCYC:-".t${cycn}z."} -export PREFIX=${PREFIX:-${RUN}${TCYC}} -export PGB1F=${PGB1F:-"NO"} -export FHOUT_PGB=${FHOUT_PGB:-3} -export PGBS=${PGBS:-"NO"} #YES-- generate 1.00 and 0.50 deg pgb data -export MODICEC=${MODICEC:-$USHgfs/mod_icec.sh} - -#--wgrib2 regrid parameters -export option1=' -set_grib_type same -new_grid_winds earth ' -export option21=' -new_grid_interpolation bilinear -if ' -export option22=":(CRAIN|CICEP|CFRZR|CSNOW|ICSEV):" -export option23=' -new_grid_interpolation neighbor -fi ' -export option24=' -set_bitmap 1 -set_grib_max_bits 16 -if ' -export option25=":(APCP|ACPCP|PRATE|CPRAT):" -export option26=' -set_grib_max_bits 25 -fi -if ' -export option27=":(APCP|ACPCP|PRATE|CPRAT|DZDT):" -export option28=' -new_grid_interpolation budget -fi ' -export grid0p25="latlon 0:1440:0.25 90:721:-0.25" -export grid0p5="latlon 0:720:0.5 90:361:-0.5" -export grid1p0="latlon 0:360:1.0 90:181:-1.0" -export grid2p5="latlon 0:144:2.5 90:73:-2.5" - -unset paramlist paramlistb -if [ $FH -eq -1 ] ; then - #export paramlist=/global/save/Hui-Ya.Chuang/gfs_trunk/sib/fix/global_1x1_paramlist_g2.anl - export paramlist=${paramlist:-$PARMpost/global_1x1_paramlist_g2.anl} - export paramlistb=${paramlistb:-$PARMpost/global_master-catchup_parmlist_g2} - export fhr3=anl - export PGBS=YES -elif [ $FH -eq 0 ] ; then - export paramlist=${paramlist:-$PARMpost/global_1x1_paramlist_g2.f000} - export paramlistb=${paramlistb:-$PARMpost/global_master-catchup_parmlist_g2} - export fhr3=000 - export PGBS=YES +CNVGRIB=${CNVGRIB:-${grib_util_ROOT}/bin/cnvgrib} +GRBINDEX=${GRBINDEX:-${wgrib2_ROOT}/bin/grbindex} + +# Scripts used +GFSDWNSH=${GFSDWNSH:-"${HOMEgfs}/ush/fv3gfs_dwn_nems.sh"} + +# variables used here and in $GFSDWNSH +PGBOUT2=${PGBOUT2:-"master.grib2"} # grib2 file from UPP +FH=$(( ${FH:-0} )) # Forecast hour to process +FHOUT_PGB=${FHOUT_PGB:-3} # Output frequency of GFS PGB file at 1-degree and 0.5 degree +npe_dwn=${npe_dwn:-24} +downset=${downset:-1} +PREFIX=${PREFIX:-"${RUN:-gfs}.t${cyc}z."} +PGBS=${PGBS:-"NO"} # YES - generate 1 and 1/2-degree grib2 data +PGB1F=${PGB1F:-"NO"} # YES - generate 1-degree grib1 data + +# Files used +if (( FH == -1 )); then + fhr3="anl" + PGBS="YES" + paramlista=${paramlist:-"${HOMEgfs}/parm/post/global_1x1_paramlist_g2.anl"} +elif (( FH == 0 )); then + fhr3="f000" + PGBS="YES" + paramlista=${paramlist:-"${HOMEgfs}/parm/post/global_1x1_paramlist_g2.f000"} else - export paramlist=${paramlist:-$PARMpost/global_1x1_paramlist_g2} - export paramlistb=${paramlistb:-$PARMpost/global_master-catchup_parmlist_g2} - export fhr3=$(printf "%03d" ${FH}) + fhr3=$(printf "f%03d" "${FH}") if (( FH%FHOUT_PGB == 0 )); then - export PGBS=YES + PGBS="YES" fi + paramlista=${paramlist:-"${HOMEgfs}/parm/post/global_1x1_paramlist_g2"} fi +paramlistb=${paramlistb:-"${HOMEgfs}/parm/post/global_master-catchup_parmlist_g2"} - -$WGRIB2 $PGBOUT2 | grep -F -f $paramlist | $WGRIB2 -i -grib tmpfile1_$fhr3 $PGBOUT2 +# Get inventory from ${PGBOUT2} that matches patterns from ${paramlista} +# Extract this inventory from ${PGBOUT2} into a smaller tmpfile or tmpfileb based on paramlista or paramlistb +# shellcheck disable=SC2312 +${WGRIB2} "${PGBOUT2}" | grep -F -f "${paramlista}" | ${WGRIB2} -i -grib "tmpfile_${fhr3}" "${PGBOUT2}" export err=$?; err_chk -if [ $downset = 2 ]; then - $WGRIB2 $PGBOUT2 | grep -F -f $paramlistb | $WGRIB2 -i -grib tmpfile2_$fhr3 $PGBOUT2 +# Do the same as above for ${paramlistb} +if (( downset = 2 )); then + # shellcheck disable=SC2312 + ${WGRIB2} "${PGBOUT2}" | grep -F -f "${paramlistb}" | ${WGRIB2} -i -grib "tmpfileb_${fhr3}" "${PGBOUT2}" export err=$?; err_chk fi +# Determine grids once and save them as a string and an array for processing +grid_string="0p25" +if [[ "${PGBS}" = "YES" ]]; then + grid_string="${grid_string}:0p50:1p00" +fi +# Also transform the ${grid_string} into an array for processing +IFS=':' read -ra grids <<< "${grid_string}" + #----------------------------------------------------- -#----------------------------------------------------- -export nset=1 -export totalset=2 -if [ $downset = 1 ]; then totalset=1 ; fi +nproc=${nproc:-${npe_dwn}} #.............................................. -while [ $nset -le $totalset ]; do - #.............................................. - export tmpfile=$(eval echo tmpfile${nset}_${fhr3}) +for (( nset=1 ; nset <= downset ; nset++ )); do + + echo "Begin processing nset = ${nset}" - # split of Grib files to run downstream jobs using MPMD - export ncount=$($WGRIB2 $tmpfile |wc -l) - # export tasks_post=$(eval echo \$tasksp_$nknd) - export nproc=${nproc:-${npe_dwn:-24}} - if [ $nproc -gt $ncount ]; then - echo " *** FATA ERROR: Total number of records in $tmpfile is not right" + # Each set represents a group of files + if (( nset == 1 )); then + grp="" # TODO: this should be "a" when we eventually rename the pressure grib2 files per EE2 convention + elif (( nset == 2 )); then + grp="b" + fi + + # process Grib files to run downstream jobs using MPMD + tmpfile="tmpfile${grp}_${fhr3}" + + # shellcheck disable=SC2312 + ncount=$(${WGRIB2} "${tmpfile}" | wc -l) + if (( nproc > ncount )); then + echo "FATAL ERROR: Total number of records in ${tmpfile} is not right" # No, the no. of records < no. of processors export err=8 err_chk fi - export inv=$(expr $ncount / $nproc) - rm -f $DATA/poescript - export iproc=1 - export end=0 + inv=$(( ncount / nproc )) + rm -f "${DATA}/poescript" - while [ $iproc -le $nproc ] ; do - export start=$(expr ${end} + 1) - export end=$(expr ${start} + ${inv} - 1) - if [[ $end -ge $ncount ]] ;then - export end=$ncount - fi + last=0 + for (( iproc = 1 ; iproc <= nproc ; iproc++ )); do + first=$((last + 1)) + last=$((last + inv)) + if (( last > ncount )); then (( last = ncount )); fi - # if final record of each piece is ugrd, add vgrd - # copygb will only interpolate u and v together - #$WGRIB2 -d $end $tmpfile |grep -i ugrd + # if final record of is u-component, add next record v-component + # if final record is land, add next record icec # grep returns 1 if no match is found, so temporarily turn off exit on non-zero rc set +e - $WGRIB2 -d $end $tmpfile | egrep -i "ugrd|ustm|uflx|u-gwd" - export rc=$? - set_strict - if [[ $rc -eq 0 ]] ; then - export end=$(expr ${end} + 1) - elif [[ $rc -gt 1 ]]; then - echo "FATAL: WGRIB2 failed with error code ${rc}" - exit $rc - fi - # if final record is land, add next record icec - set +e - $WGRIB2 -d $end $tmpfile | egrep -i "land" - export rc=$? + # shellcheck disable=SC2312 + ${WGRIB2} -d "${last}" "${tmpfile}" | grep -E -i "ugrd|ustm|uflx|u-gwd|land" + rc=$? set_strict - if [[ $rc -eq 0 ]] ; then - export end=$(expr ${end} + 1) - elif [[ $rc -gt 1 ]]; then - echo "FATAL: WGRIB2 failed with error code ${rc}" - exit $rc + if (( rc == 0 )); then # Matched the grep + last=$(( last + 1 )) fi - if [ $iproc -eq $nproc ]; then - export end=$ncount + if (( iproc == nproc )); then + last=${ncount} fi - $WGRIB2 $tmpfile -for ${start}:${end} -grib ${tmpfile}_${iproc} + # Break tmpfile into processor specific chunks in preparation for MPMD + ${WGRIB2} "${tmpfile}" -for "${first}":"${last}" -grib "${tmpfile}_${iproc}" export err=$?; err_chk - echo "${GFSDWNSH:-$USHgfs/fv3gfs_dwn_nems.sh} ${tmpfile}_${iproc} $fhr3 $iproc $nset" >> $DATA/poescript + input_file="${tmpfile}_${iproc}" + output_file_prefix="pgb2${grp}file_${fhr3}_${iproc}" + echo "${GFSDWNSH} ${input_file} ${output_file_prefix} ${grid_string}" >> "${DATA}/poescript" # if at final record and have not reached the final processor then write echo's to # poescript for remaining processors - if [[ $end -eq $ncount ]] ;then - while [[ $iproc -lt $nproc ]];do - export iproc=$(expr $iproc + 1) - echo "/bin/echo $iproc" >> $DATA/poescript + if (( last == ncount )); then + for (( pproc = iproc+1 ; pproc < nproc ; pproc++ )); do + echo "/bin/echo ${pproc}" >> "${DATA}/poescript" done break fi - export iproc=$(expr $iproc + 1) - done + done # for (( iproc = 1 ; iproc <= nproc ; iproc++ )); do - date - chmod 775 $DATA/poescript - export MP_PGMMODEL=mpmd - export MP_CMDFILE=$DATA/poescript - launcher=${APRUN_DWN:-"aprun -j 1 -n 24 -N 24 -d 1 cfp"} - if [ $machine = WCOSS2 ] ; then - $launcher $MP_CMDFILE - elif [ $machine = HERA -o $machine = ORION -o $machine = JET -o $machine = S4 ] ; then - if [ -s $DATA/poescript_srun ]; then rm -f $DATA/poescript_srun; fi - touch $DATA/poescript_srun - nm=0 - cat $DATA/poescript | while read line; do - echo "$nm $line" >> $DATA/poescript_srun - nm=$((nm+1)) - done - nm=$(wc -l < $DATA/poescript_srun) - ${launcher:-"srun --export=ALL"} -n $nm --multi-prog $DATA/poescript_srun + # Run with MPMD or serial + if [[ "${USE_CFP:-}" = "YES" ]]; then + "${HOMEgfs}/ush/run_mpmd.sh" "${DATA}/poescript" + export err=$? else - $launcher + chmod 755 "${DATA}/poescript" + bash +x "${DATA}/poescript" 2>&1 mpmd.out + export err=$? fi - export err=$? - if [ $err -ne 0 ]; then sh +x $DATA/poescript ; fi - - date - export iproc=1 - while [ $iproc -le $nproc ]; do - if [ $nset = 1 ]; then - cat pgb2file_${fhr3}_${iproc}_0p25 >> pgb2file_${fhr3}_0p25 - if [ "$PGBS" = "YES" ]; then - cat pgb2file_${fhr3}_${iproc}_0p5 >> pgb2file_${fhr3}_0p5 - cat pgb2file_${fhr3}_${iproc}_1p0 >> pgb2file_${fhr3}_1p0 - if [ "$PGB1F" = 'YES' ]; then - cat pgbfile_${fhr3}_${iproc}_1p0 >> pgbfile_${fhr3}_1p0 - fi - fi - elif [ $nset = 2 ]; then - cat pgb2bfile_${fhr3}_${iproc}_0p25 >> pgb2bfile_${fhr3}_0p25 - if [ "$PGBS" = "YES" ]; then - cat pgb2bfile_${fhr3}_${iproc}_0p5 >> pgb2bfile_${fhr3}_0p5 - cat pgb2bfile_${fhr3}_${iproc}_1p0 >> pgb2bfile_${fhr3}_1p0 - fi - fi - export iproc=$(expr $iproc + 1) + err_chk + + # We are in a loop over downset, save output from mpmd into nset specific output + cat mpmd.out # so we capture output into the main logfile + mv mpmd.out "mpmd_${nset}.out" + + # Concatenate grib files from each processor into a single one + # and clean-up as you go + echo "Concatenating processor specific grib2 files into a single product" + for (( iproc = 1 ; iproc <= nproc ; iproc++ )); do + for grid in "${grids[@]}"; do + cat "pgb2${grp}file_${fhr3}_${iproc}_${grid}" >> "pgb2${grp}file_${fhr3}_${grid}" + rm "pgb2${grp}file_${fhr3}_${iproc}_${grid}" + done + # There is no further use of the processor specific tmpfile; delete it + rm "${tmpfile}_${iproc}" done - date - #Chuang: generate second land mask using bi-linear interpolation and append to the end - # if [ $nset = 1 ]; then - # rm -f land.grb newland.grb newnewland.grb newnewland.grb1 - # $WGRIB2 $tmpfile -match "LAND:surface" -grib land.grb - ##0p25 degree - # $WGRIB2 land.grb -set_grib_type same -new_grid_interpolation bilinear -new_grid_winds earth -new_grid $grid0p25 newland.grb - # $WGRIB2 newland.grb -set_byte 4 11 218 -grib newnewland.grb - # cat ./newnewland.grb >> pgb2file_${fhr3}_0p25 - # $CNVGRIB -g21 newnewland.grb newnewland.grb1 - # cat ./newnewland.grb1 >> pgbfile_${fhr3}_0p25 - ##0p5 degree - # rm -f newland.grb newnewland.grb newnewland.grb1 - # $WGRIB2 land.grb -set_grib_type same -new_grid_interpolation bilinear -new_grid_winds earth -new_grid $grid0p5 newland.grb - # $WGRIB2 newland.grb -set_byte 4 11 218 -grib newnewland.grb - # cat ./newnewland.grb >> pgb2file_${fhr3}_0p5 - #1p0 - # rm -f newland.grb newnewland.grb newnewland.grb1 - # $WGRIB2 land.grb -set_grib_type same -new_grid_interpolation bilinear -new_grid_winds earth -new_grid $grid1p0 newland.grb - # $WGRIB2 newland.grb -set_byte 4 11 218 -grib newnewland.grb - # cat ./newnewland.grb >> pgb2file_${fhr3}_1p0 - # $CNVGRIB -g21 newnewland.grb newnewland.grb1 - # cat ./newnewland.grb1 >> pgbfile_${fhr3}_1p0 - # fi + # Move to COM and index the product grib files + for grid in "${grids[@]}"; do + prod_dir="COM_ATMOS_GRIB_${grid}" + ${NCP} "pgb2${grp}file_${fhr3}_${grid}" "${!prod_dir}/${PREFIX}pgrb2${grp}.${grid}.${fhr3}" + ${WGRIB2} -s "pgb2${grp}file_${fhr3}_${grid}" > "${!prod_dir}/${PREFIX}pgrb2${grp}.${grid}.${fhr3}.idx" + done - if [ $nset = 1 ]; then - if [ $fhr3 = anl ]; then - cp "pgb2file_${fhr3}_0p25" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2.0p25.anl" - ${WGRIB2} -s "pgb2file_${fhr3}_0p25" > "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2.0p25.anl.idx" - if [ "$PGBS" = "YES" ]; then - cp "pgb2file_${fhr3}_0p5" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2.0p50.anl" - cp "pgb2file_${fhr3}_1p0" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2.1p00.anl" - ${WGRIB2} -s "pgb2file_${fhr3}_0p5" > "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2.0p50.anl.idx" - ${WGRIB2} -s "pgb2file_${fhr3}_1p0" > "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2.1p00.anl.idx" - if [ "$PGB1F" = 'YES' ]; then - cp "pgbfile_${fhr3}_1p0" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb.1p00.anl" - ${GRBINDEX} "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb.1p00.anl" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb.1p00.anl.idx" - fi - fi - else - cp "pgb2file_${fhr3}_0p25" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2.0p25.f${fhr3}" - ${WGRIB2} -s "pgb2file_${fhr3}_0p25" > "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2.0p25.f${fhr3}.idx" - if [ "$PGBS" = "YES" ]; then - cp "pgb2file_${fhr3}_0p5" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2.0p50.f${fhr3}" - cp "pgb2file_${fhr3}_1p0" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2.1p00.f${fhr3}" - ${WGRIB2} -s "pgb2file_${fhr3}_0p5" > "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2.0p50.f${fhr3}.idx" - ${WGRIB2} -s "pgb2file_${fhr3}_1p0" > "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2.1p00.f${fhr3}.idx" - if [ "$PGB1F" = 'YES' ]; then - cp "pgbfile_${fhr3}_1p0" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb.1p00.f${fhr3}" - ${GRBINDEX} "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb.1p00.f${fhr3}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb.1p00.f${fhr3}.idx" - fi - fi - fi - elif [ $nset = 2 ]; then - if [ $fhr3 = anl ]; then - cp "pgb2bfile_${fhr3}_0p25" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2b.0p25.anl" - ${WGRIB2} -s "pgb2bfile_${fhr3}_0p25" > "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2b.0p25.anl.idx" - if [ "$PGBS" = "YES" ]; then - cp "pgb2bfile_${fhr3}_0p5" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2b.0p50.anl" - cp "pgb2bfile_${fhr3}_1p0" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2b.1p00.anl" - ${WGRIB2} -s "pgb2bfile_${fhr3}_0p5" > "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2b.0p50.anl.idx" - ${WGRIB2} -s "pgb2bfile_${fhr3}_1p0" > "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2b.1p00.anl.idx" - fi - else - cp "pgb2bfile_${fhr3}_0p25" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2b.0p25.f${fhr3}" - ${WGRIB2} -s "pgb2bfile_${fhr3}_0p25" > "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2b.0p25.f${fhr3}.idx" - if [ "$PGBS" = "YES" ]; then - cp "pgb2bfile_${fhr3}_0p5" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2b.0p50.f${fhr3}" - cp "pgb2bfile_${fhr3}_1p0" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2b.1p00.f${fhr3}" - ${WGRIB2} -s "pgb2bfile_${fhr3}_0p5" > "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2b.0p50.f${fhr3}.idx" - ${WGRIB2} -s "pgb2bfile_${fhr3}_1p0" > "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2b.1p00.f${fhr3}.idx" + # Create supplemental 1-degree grib1 output TODO: who needs 1-degree grib1 product? + # move to COM and index it + if (( nset == 1 )); then + if [[ "${PGBS}" = "YES" ]]; then + if [[ "${PGB1F}" = "YES" ]]; then + ${CNVGRIB} -g21 "pgb2${grp}file_${fhr3}_1p00" "pgb${grp}file_${fhr3}_1p00" + export err=$?; err_chk + ${NCP} "pgb${grp}file_${fhr3}_1p00" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb${grp}.1p00.${fhr3}" + ${GRBINDEX} "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb${grp}.1p00.${fhr3}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb${grp}.1p00.${fhr3}.idx" fi fi fi - export nset=$(expr $nset + 1 ) -done + echo "Finished processing nset = ${nset}" + + +done # for (( nset=1 ; nset <= downset ; nset++ )) exit 0 diff --git a/ush/fv3gfs_dwn_nems.sh b/ush/fv3gfs_dwn_nems.sh index fdc58c68ea..0de295f172 100755 --- a/ush/fv3gfs_dwn_nems.sh +++ b/ush/fv3gfs_dwn_nems.sh @@ -1,109 +1,84 @@ #! /usr/bin/env bash -# this script generates 0.25/0.5/1/2.5 deg pgb files for each small Grib file -# Hui-Ya Chuang 01/2014: First Version -# Fanglin Yang 09/2015: Modified to use WGRIB2 instead of COPYGB2 for interpolation -# Fanglin Yang 02/2016: remove 0.5-deg and 2.5deg output to speed up post -# Fanglin Yang 09/11/2017: add option opt24 to turn on bitmap (from Wen Meng) -# Wen Meng 12/2017: add trim_rh.sh for triming RH values larger than 100. -# Wen Meng 01/2018: add flag PGB1F for turning on/off wgrib1 pgb data at 1.00 deg. generation. -# Wen Meng 02/2018: add flag PGBS for turning on/off pgb data at 1.0 and 0.5 deg. generation. -# Wen Meng 10/2019: Use bilinear interpolation for LAND, It can trancate land-sea mask as 0 or 1. -# Wen Meng 11/2019: Teak sea ice cover via land-sea mask. +# This script takes in a master grib file and creates products at various interpolated resolutions +# Generate 0.25 / 0.5 / 1 degree interpolated grib2 files for each input grib2 file +# trim's RH and tweaks sea-ice cover -source "$HOMEgfs/ush/preamble.sh" -export tmpfile=$1 -export fhr3=$2 -export iproc=$3 -export nset=$4 +source "${HOMEgfs}/ush/preamble.sh" -export CNVGRIB=${CNVGRIB:-${grib_util_ROOT}/bin/cnvgrib} -export COPYGB2=${COPYGB2:-${grib_util_ROOT}/bin/copygb} -export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} -export TRIMRH=${TRIMRH:-$USHgfs/trim_rh.sh} -export MODICEC=${MODICEC:-$USHgfs/mod_icec.sh} +input_file=${1:-"pgb2file_in"} # Input pressure grib2 file +output_file_prefix=${2:-"pgb2file_out"} # Prefix for output grib2 file; the prefix is appended by resolution e.g. _0p25 +grid_string=${3:-"0p25"} # Target grids; e.g. "0p25" or "0p25:0p50"; If multiple, they need to be ":" seperated -export opt1=' -set_grib_type same -new_grid_winds earth ' -export opt21=' -new_grid_interpolation bilinear -if ' -export opt22=":(CSNOW|CRAIN|CFRZR|CICEP|ICSEV):" -export opt23=' -new_grid_interpolation neighbor -fi ' -export opt24=' -set_bitmap 1 -set_grib_max_bits 16 -if ' -export opt25=":(APCP|ACPCP|PRATE|CPRAT):" -export opt26=' -set_grib_max_bits 25 -fi -if ' -export opt27=":(APCP|ACPCP|PRATE|CPRAT|DZDT):" -export opt28=' -new_grid_interpolation budget -fi ' -if [ $machine = "S4" ]; then - export optncpu=' -ncpu 1 ' -fi -export grid0p25="latlon 0:1440:0.25 90:721:-0.25" -export grid0p5="latlon 0:720:0.5 90:361:-0.5" -export grid1p0="latlon 0:360:1.0 90:181:-1.0" -export grid2p5="latlon 0:144:2.5 90:73:-2.5" +WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} -export PGB1F=${PGB1F:-"NO"} -export PGBS=${PGBS:-"NO"} -optncpu=${optncpu:-} +# wgrib2 options for regridding +defaults="-set_grib_type same -set_bitmap 1 -set_grib_max_bits 16" +interp_winds="-new_grid_winds earth" +interp_bilinear="-new_grid_interpolation bilinear" +interp_neighbor="-if :(CSNOW|CRAIN|CFRZR|CICEP|ICSEV): -new_grid_interpolation neighbor -fi" +interp_budget="-if :(APCP|ACPCP|PRATE|CPRAT|DZDT): -new_grid_interpolation budget -fi" +increased_bits="-if :(APCP|ACPCP|PRATE|CPRAT): -set_grib_max_bits 25 -fi" -if [ $nset = 1 ]; then - if [ "$PGBS" = "YES" ]; then - $WGRIB2 $optncpu $tmpfile $opt1 $opt21 $opt22 $opt23 $opt24 $opt25 $opt26 $opt27 $opt28 \ - -new_grid $grid0p25 pgb2file_${fhr3}_${iproc}_0p25 \ - -new_grid $grid1p0 pgb2file_${fhr3}_${iproc}_1p0 \ - -new_grid $grid0p5 pgb2file_${fhr3}_${iproc}_0p5 - export err=$?; err_chk - $TRIMRH pgb2file_${fhr3}_${iproc}_0p25 - $TRIMRH pgb2file_${fhr3}_${iproc}_0p5 - $TRIMRH pgb2file_${fhr3}_${iproc}_1p0 - #tweak sea ice cover - count=$($WGRIB2 $optncpu pgb2file_${fhr3}_${iproc}_0p25 -match "LAND|ICEC" |wc -l) - if [ $count -eq 2 ]; then - $MODICEC pgb2file_${fhr3}_${iproc}_0p25 - $MODICEC pgb2file_${fhr3}_${iproc}_0p5 - $MODICEC pgb2file_${fhr3}_${iproc}_1p0 - fi - #$CNVGRIB -g21 pgb2file_${fhr3}_${iproc}_0p25 pgbfile_${fhr3}_${iproc}_0p25 - if [ "$PGB1F" = 'YES' ]; then - $CNVGRIB -g21 pgb2file_${fhr3}_${iproc}_1p0 pgbfile_${fhr3}_${iproc}_1p0 - export err=$?; err_chk - fi - else - $WGRIB2 $optncpu $tmpfile $opt1 $opt21 $opt22 $opt23 $opt24 $opt25 $opt26 $opt27 $opt28 \ - -new_grid $grid0p25 pgb2file_${fhr3}_${iproc}_0p25 - export err=$?; err_chk - $TRIMRH pgb2file_${fhr3}_${iproc}_0p25 - #tweak sea ice cover - count=$($WGRIB2 $optncpu pgb2file_${fhr3}_${iproc}_0p25 -match "LAND|ICEC" |wc -l) - if [ $count -eq 2 ]; then - $MODICEC pgb2file_${fhr3}_${iproc}_0p25 - fi - fi -elif [ $nset = 2 ]; then - if [ "$PGBS" = "YES" ]; then - $WGRIB2 $optncpu $tmpfile $opt1 $opt21 $opt22 $opt23 $opt24 $opt25 $opt26 $opt27 $opt28 \ - -new_grid $grid0p25 pgb2bfile_${fhr3}_${iproc}_0p25 \ - -new_grid $grid1p0 pgb2bfile_${fhr3}_${iproc}_1p0 \ - -new_grid $grid0p5 pgb2bfile_${fhr3}_${iproc}_0p5 - export err=$?; err_chk - $TRIMRH pgb2bfile_${fhr3}_${iproc}_0p25 - $TRIMRH pgb2bfile_${fhr3}_${iproc}_0p5 - $TRIMRH pgb2bfile_${fhr3}_${iproc}_1p0 - else - $WGRIB2 $optncpu $tmpfile $opt1 $opt21 $opt22 $opt23 $opt24 $opt25 $opt26 $opt27 $opt28 \ - -new_grid $grid0p25 pgb2bfile_${fhr3}_${iproc}_0p25 - export err=$?; err_chk - $TRIMRH pgb2bfile_${fhr3}_${iproc}_0p25 - fi -fi +# interpolated target grids +# shellcheck disable=SC2034 +grid0p25="latlon 0:1440:0.25 90:721:-0.25" +# shellcheck disable=SC2034 +grid0p50="latlon 0:720:0.5 90:361:-0.5" +# shellcheck disable=SC2034 +grid1p00="latlon 0:360:1.0 90:181:-1.0" -#---------------------------------------------------------------------------------------------- -#--Hui-Ya Chuang -# export grid1p0="0 6 0 0 0 0 0 0 360 181 0 0 90000000 0 48 -90000000 359000000 1000000 1000000 0" -# $COPYGB2 -g "${grid1p0}" -i0 -x tmpfile_${FH}_${iproc} pgb2file_${FH}_${iproc}_1p0 -# export grid0p5="0 6 0 0 0 0 0 0 720 361 0 0 90000000 0 48 -90000000 359500000 500000 500000 0" -# $COPYGB2 -g "${grid0p5}" -i0 -x tmpfile_${FH}_${iproc} pgb2file_${FH}_${iproc}_0p5 -# export grid2p5="0 6 0 0 0 0 0 0 144 73 0 0 90000000 0 48 -90000000 357500000 2500000 2500000 0" -# $COPYGB2 -g "${grid2p5}" -i0 -x tmpfile_${FH}_${iproc} pgb2file_${FH}_${iproc}_2p5 -# $CNVGRIB -g21 pgb2file_${fhr3}_${iproc}_1p0 pgbfile_${fhr3}_${iproc}_1p0 -# $CNVGRIB -g21 pgb2file_${fhr3}_${iproc}_2p5 pgbfile_${fhr3}_${iproc}_2p5 -#---------------------------------------------------------------------------------------------- +# Functions used in this script +function trim_rh() { + # trim RH values larger than 100. + local filename=$1 + ${WGRIB2} "${filename}" \ + -not_if ':RH:' -grib "${filename}.new" \ + -if ':RH:' -rpn "10:*:0.5:+:floor:1000:min:10:/" -set_grib_type same \ + -set_scaling -1 0 -grib_out "${filename}.new" + rc=$? + if (( rc == 0 )); then mv "${filename}.new" "${filename}"; fi + return "${rc}" +} + +function mod_icec() { + # modify icec based on land-sea mask + local filename=$1 + ${WGRIB2} "${filename}" \ + -if 'LAND' -rpn 'sto_1' -fi \ + -if 'ICEC' -rpn 'rcl_1:0:==:*' -fi \ + -set_grib_type same \ + -set_scaling same same \ + -grib_out "${filename}.new" + rc=$? + if (( rc == 0 )); then mv "${filename}.new" "${filename}"; fi + return "${rc}" +} + +# Transform the input ${grid_string} into an array for processing +IFS=':' read -ra grids <<< "${grid_string}" + +output_grids="" +for grid in "${grids[@]}"; do + gridopt="grid${grid}" + output_grids="${output_grids} -new_grid ${!gridopt} ${output_file_prefix}_${grid}" +done + +#shellcheck disable=SC2086 +${WGRIB2} "${input_file}" ${defaults} \ + ${interp_winds} \ + ${interp_bilinear} \ + ${interp_neighbor} \ + ${interp_budget} \ + ${increased_bits} \ + ${output_grids} +export err=$?; err_chk + +# trim and mask for all grids +for grid in "${grids[@]}"; do + trim_rh "${output_file_prefix}_${grid}"; export err=$?; err_chk + mod_icec "${output_file_prefix}_${grid}"; export err=$?; err_chk +done + +exit 0 diff --git a/ush/mod_icec.sh b/ush/mod_icec.sh deleted file mode 100755 index 96ccab9075..0000000000 --- a/ush/mod_icec.sh +++ /dev/null @@ -1,21 +0,0 @@ -#! /usr/bin/env bash - -#This script is used for modifing icee via land-sea mask -#Wen Meng 11/2019: First Version - -source "$HOMEgfs/ush/preamble.sh" - -f=$1 - -export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} - -$WGRIB2 ${optncpu:-} $f \ - -if 'LAND' -rpn 'sto_1' -fi \ - -if 'ICEC' -rpn 'rcl_1:0:==:*' -fi \ - -set_grib_type same \ - -set_scaling same same \ - -grib_out $f.new -export err=$?; err_chk -mv $f.new $f - -exit 0 diff --git a/ush/run_mpmd.sh b/ush/run_mpmd.sh new file mode 100755 index 0000000000..352a411312 --- /dev/null +++ b/ush/run_mpmd.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +cmdfile=${1:?"run_mpmd requires an input file containing commands to execute in MPMD mode"} + +# Local MPMD file containing instructions to run in CFP +mpmd_cmdfile="${DATA:-}/mpmd_cmdfile" +if [[ -s "${mpmd_cmdfile}" ]]; then rm -f "${mpmd_cmdfile}"; fi + +if [[ "${launcher:-}" =~ ^srun.* ]]; then # srun-based system e.g. Hera, Orion, etc. + + # Slurm requires a counter in front of each line in the script + # Read the incoming cmdfile and create srun usable cmdfile + nm=0 + # shellcheck disable=SC2312 + while IFS= read -r line; do + echo "${nm} ${line}" >> "${mpmd_cmdfile}" + ((nm=nm+1)) + done < "${cmdfile}" + + nprocs=$(wc -l < "${mpmd_cmdfile}") + set +e + # shellcheck disable=SC2086 + ${launcher:-} ${mpmd_opt:-} -n ${nprocs} "${mpmd_cmdfile}" + rc=$? + set_strict + if (( rc == 0 )); then + out_files=$(find . -name 'mpmd.*.*.out') + fi + +elif [[ "${launcher:-}" =~ ^mpiexec.* ]]; then # mpiexec + + # Redirect output from each process to its own stdout + # Read the incoming cmdfile and create mpiexec usable cmdfile + nm=0 + echo "#!/bin/bash" >> "${mpmd_cmdfile}" + # shellcheck disable=SC2312 + while IFS= read -r line; do + echo "${line} > mpmd.${nm}.out" >> "${mpmd_cmdfile}" + ((nm=nm+1)) + done < "${cmdfile}" + + chmod 755 "${mpmd_cmdfile}" + ${launcher:-} "${mpmd_cmdfile}" + rc=$? + if (( rc == 0 )); then + out_files=$(find . -name 'mpmd.*.out') + fi + +else + + echo "FATAL ERROR: CFP is not usable with launcher: '${launcher:-}'" + rc=1 + +fi + +# On success concatenate processor specific output into a single mpmd.out +if (( rc == 0 )); then + rm -f "${mpmd_cmdfile}" + for file in ${out_files}; do + cat "${file}" >> mpmd.out + rm -f "${file}" + done +fi + +exit "${rc}" diff --git a/ush/trim_rh.sh b/ush/trim_rh.sh deleted file mode 100755 index 5a8903cae6..0000000000 --- a/ush/trim_rh.sh +++ /dev/null @@ -1,18 +0,0 @@ -#! /usr/bin/env bash - -#This is scripts is used to trim RH vaule larger than 100. -# Wen Meng 12/2017: First Version - -source "$HOMEgfs/ush/preamble.sh" - -f=$1 - -export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} - -$WGRIB2 ${optncpu:-} $f -not_if ':RH:' -grib $f.new \ - -if ':RH:' -rpn "10:*:0.5:+:floor:1000:min:10:/" -set_grib_type same \ - -set_scaling -1 0 -grib_out $f.new -export err=$?; err_chk -mv $f.new $f - -exit 0