-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_ICON_R02B06N07_ifsinit
378 lines (312 loc) · 20.6 KB
/
run_ICON_R02B06N07_ifsinit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
#!/bin/ksh
# ICON
#
# ------------------------------------------
# Copyright (C) 2004-2024, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
# Contact information: icon-model.org
# See AUTHORS.TXT for a list of authors
# See LICENSES/ for license information
# SPDX-License-Identifier: BSD-3-Clause
# ------------------------------------------
#=============================================================================
# =====================================
# mistral batch job parameters
#-----------------------------------------------------------------------------
#SBATCH --account=mh0287
##SBATCH --job-name=icon-nwp-test
#SBATCH --partition=compute
#SBATCH --nodes=6
#SBATCH --threads-per-core=2
#SBATCH --exclusive
#SBATCH --time=00:15:00
set -x
# OpenMP settings
export OMP_NUM_THREADS=4
export ICON_THREADS=4
export OMP_SCHEDULE=dynamic,1
export OMP_DYNAMIC="false"
export OMP_STACKSIZE=200M
# MPI variables
# ----------------------------
mpi_root=/opt/mpi/bullxmpi_mlx/1.2.8.3
no_of_nodes=${SLURM_JOB_NUM_NODES:-1}
mpi_procs_pernode=12
((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
START="srun --cpu-freq=2500000 --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --cpu_bind=verbose,cores --distribution=block:block --ntasks=$((no_of_nodes * mpi_procs_pernode)) --ntasks-per-node=${mpi_procs_pernode} --cpus-per-task=${OMP_NUM_THREADS} --propagate=STACK --mpi=openmpi"
cd ${SLURM_SUBMIT_DIR}
basedir=${SLURM_SUBMIT_DIR}/../../
# ----------------------------------------------------------------------------
# path definitions
# ----------------------------------------------------------------------------
# directory with input grids:
GRIDDIR=/pool/data/ICON/grids/public/edzw
# directory with input data
INDATADIR=/pool/data/ICON/grids/private/m222072/initdata_R2B6N7
# directory with external parameter data:
EXTPARDIR=/pool/data/ICON/grids/public/edzw
# base directory for ICON sources and binary:
bindir="${basedir}/build/x86_64-unknown-linux-gnu/bin" # binaries
EXPNAME=icon-nwp-test
# absolute path to directory with plenty of space:
EXPDIR=${basedir}/experiments/${EXPNAME}
# path to model binary, including the executable:
MODEL=$bindir/icon
# ----------------------------------------------------------------------------
# copy input data: grids, external parameters, model
# ----------------------------------------------------------------------------
# the directory for the experiment will be created, if not already there
if [ ! -d $EXPDIR ]; then
mkdir -p $EXPDIR
fi
cd ${EXPDIR}
# grid files
ln -sf $GRIDDIR/icon_grid_0023_R02B05_R.nc iconR2B05_DOM00.nc
ln -sf $GRIDDIR/icon_grid_0023_R02B05_R-grfinfo.nc iconR2B05_DOM00-grfinfo.nc
ln -sf $GRIDDIR/icon_grid_0024_R02B06_G.nc iconR2B06_DOM01.nc
ln -sf $GRIDDIR/icon_grid_0024_R02B06_G-grfinfo.nc iconR2B06_DOM01-grfinfo.nc
ln -sf $GRIDDIR/icon_grid_0028_R02B07_N02.nc iconR2B07_DOM02.nc
ln -sf $GRIDDIR/icon_grid_0028_R02B07_N02-grfinfo.nc iconR2B07_DOM02-grfinfo.nc
ln -sf $EXTPARDIR/icon_extpar_0024_R02B06_G_20150805_tiles.nc extpar_iconR2B06_DOM01.nc
ln -sf $EXTPARDIR/icon_extpar_0028_R02B07_N02_20150805_tiles.nc extpar_iconR2B07_DOM02.nc
# interpolated IFS data
ln -sf $INDATADIR/prepiconR2B06_DOM01_2012062000.nc ifs2icon_R2B06_DOM01.nc
ln -sf $INDATADIR/prepiconR2B07_DOM02_2012062000.nc ifs2icon_R2B07_DOM02.nc
# files needed for radiation
ln -sf ${basedir}/data/ECHAM6_CldOptProps.nc .
ln -sf ${basedir}/data/rrtmg_lw.nc .
# copy binary
cp -p $MODEL icon
# ----------------------------------------------------------------------------
# grid namelist settings
# ----------------------------------------------------------------------------
# the grid parameters
atmo_dyn_grids="iconR2B06_DOM01.nc iconR2B07_DOM02.nc"
atmo_rad_grids="iconR2B05_DOM00.nc"
# reconstruct the grid parameters in namelist form
dynamics_grid_filename=""
for gridfile in ${atmo_dyn_grids}; do
dynamics_grid_filename="${dynamics_grid_filename} '${gridfile}',"
done
radiation_grid_filename=""
for gridfile in ${atmo_rad_grids}; do
radiation_grid_filename="${radiation_grid_filename} '${gridfile}',"
done
# ----------------------------------------------------------------------------
# create ICON master namelist
# ----------------------------------------------------------------------------
cat > icon_master.namelist << EOF
! master_nml: ----------------------------------------------------------------
&master_nml
lrestart = .FALSE. ! .TRUE.=current experiment is resumed
/
! master_model_nml: repeated for each model ----------------------------------
&master_model_nml
model_type = 1 ! identifies which component to run (atmosphere,ocean,...)
model_name = "ATMO" ! character string for naming this component.
model_namelist_filename = "NAMELIST_NWP" ! file name containing the model namelists
model_min_rank = 1 ! start MPI rank for this model
model_max_rank = 65536 ! end MPI rank for this model
model_inc_rank = 1 ! stride of MPI ranks
/
! time_nml: specification of date and time------------------------------------
&time_nml
ini_datetime_string = "2012-06-20T00:00:00Z" ! initial date and time of the simulation
/
EOF
# ----------------------------------------------------------------------------
# model namelists
# ----------------------------------------------------------------------------
# For a complete list see doc/Namelist_overview.pdf
cat > NAMELIST_NWP << EOF
! parallel_nml: MPI parallelization -------------------------------------------
¶llel_nml
nproma = 16 ! loop chunk length
p_test_run = .FALSE. ! .TRUE. means verification run for MPI parallelization
num_io_procs = 1 ! number of I/O processors
num_restart_procs = 0 ! number of restart processors
iorder_sendrecv = 3 ! sequence of MPI send/receive calls
/
! run_nml: general switches ---------------------------------------------------
&run_nml
ltestcase = .FALSE. ! idealized testcase runs
num_lev = 90, 56 ! number of full levels (atm.) for each domain
lvert_nest = .TRUE. ! vertical nesting
nsteps = 120 ! number of time steps of this run
dtime = 360 ! timestep in seconds
ldynamics = .TRUE. ! compute adiabatic dynamic tendencies
ltransport = .TRUE. ! compute large-scale tracer transport
ntracer = 5 ! number of advected tracers
iforcing = 3 ! forcing of dynamics and transport by parameterized processes
msg_level = 12 ! controls how much printout is written during runtime
ltimer = .TRUE. ! timer for monitoring the runtime of specific routines
timers_level = 10 ! performance timer granularity
output = "nml" ! main switch for enabling/disabling components of the model output
/
! diffusion_nml: horizontal (numerical) diffusion ----------------------------
&diffusion_nml
hdiff_order = 5 ! order of nabla operator for diffusion
itype_vn_diffu = 1 ! reconstruction method used for Smagorinsky diffusion
itype_t_diffu = 2 ! discretization of temperature diffusion
hdiff_efdt_ratio = 36.0 ! ratio of e-folding time to time step
hdiff_smag_fac = 0.015 ! scaling factor for Smagorinsky diffusion
lhdiff_vn = .TRUE. ! diffusion on the horizontal wind field
lhdiff_temp = .TRUE. ! diffusion on the temperature field
/
! dynamics_nml: dynamical core -----------------------------------------------
&dynamics_nml
divavg_cntrwgt = 0.50 ! weight of central cell for divergence averaging
lcoriolis = .TRUE. ! Coriolis force
/
! extpar_nml: external data --------------------------------------------------
&extpar_nml
extpar_filename = "<path>extpar_<gridfile>" ! filename of external parameter input file
itopo = 1 ! topography (0:analytical)
n_iter_smooth_topo = 1 ! iterations of topography smoother
heightdiff_threshold = 3000.0 ! height difference between neighb. grid points
/
! initicon_nml: specify read-in of initial state ------------------------------
&initicon_nml
zpbl1 = 500.0 ! bottom height of layer used for gradient computation
zpbl2 = 1000.0 ! top height of layer used for gradient computation
/
! grid_nml: horizontal grid --------------------------------------------------
&grid_nml
dynamics_grid_filename = ${dynamics_grid_filename} ! array of the grid filenames for the dycore
radiation_grid_filename = ${radiation_grid_filename} ! array of the grid filenames for the radiation model
dynamics_parent_grid_id = 0, 1 ! array of the indexes of the parent grid filenames
lredgrid_phys = .TRUE.,.TRUE. ! .true.=radiation is calculated on a reduced grid
lfeedback = .TRUE. ! specifies if feedback to parent grid is performed
ifeedback_type = 2 ! feedback type (incremental/relaxation-based)
/
! gridref_nml: grid refinement and nesting -----------------------------------
&gridref_nml
grf_intmethod_e = 6 ! interpolation method for grid refinement
grf_scalfbk = 2 ! feedback method for dynamical scalar variables
grf_tracfbk = 2 ! feedback method for tracer variables
denom_diffu_v = 150.0 ! Denominator for lateral boundary diffusion of velocity
/
! io_nml: general switches for model I/O -------------------------------------
&io_nml
dt_diag = 21600.0 ! diagnostic integral output interval
dt_checkpoint = 864000.0 ! time interval for writing restart files.
itype_pres_msl = 2 ! method for computation of mean sea level pressure
/
! nonhydrostatic_nml: nonhydrostatic model -----------------------------------
&nonhydrostatic_nml
iadv_rhotheta = 2 ! advection method for rho and rhotheta
ivctype = 2 ! type of vertical coordinate
itime_scheme = 4 ! time integration scheme
exner_expol = 0.333 ! temporal extrapolation of Exner function
vwind_offctr = 0.2 ! off-centering in vertical wind solver
damp_height = 50000.0 ! height at which Rayleigh damping of vertical wind starts
rayleigh_coeff = 0.10 ! Rayleigh damping coefficient
ndyn_substeps = 5 ! number of dynamical core substeps
divdamp_order = 4 ! order of divergence damping
igradp_method = 3 ! discretization of horizontal pressure gradient
l_zdiffu_t = .TRUE. ! specifies computation of Smagorinsky temperature diffusion
thslp_zdiffu = 0.02 ! slope threshold (temperature diffusion)
thhgtd_zdiffu = 125.0 ! threshold of height difference (temperature diffusion)
htop_moist_proc = 22500.0 ! max. height for moist physics
hbot_qvsubstep = 19500.0 ! height above which QV is advected with substepping scheme
/
! nwp_phy_nml: switches for the physics schemes ------------------------------
&nwp_phy_nml
inwp_gscp = 1 ! cloud microphysics and precipitation
inwp_convection = 1 ! convection
inwp_radiation = 1 ! radiation
inwp_cldcover = 1 ! cloud cover scheme for radiation
inwp_turb = 1 ! vertical diffusion and transfer
inwp_satad = 1 ! saturation adjustment
inwp_sso = 1 ! subgrid scale orographic drag
inwp_gwd = 1 ! non-orographic gravity wave drag
inwp_surface = 1 ! surface scheme
latm_above_top = .FALSE.,.TRUE. ! take into account atmosphere above model top for radiation computation
efdt_min_raylfric = 7200.0 ! minimum e-folding time of Rayleigh friction
itype_z0 = 2 ! type of roughness length data
/
! output_nml: specifies an output stream --------------------------------------
&output_nml
filetype = 4 ! output format: 2=GRIB2, 4=NETCDFv2
dom = -1 ! write all domains
output_bounds = 0., 10000000., 10800. ! output: start, end, increment
steps_per_file = 2 ! number of output steps in one output file
mode = 1 ! 1: forecast mode (relative t-axis), 2: climate mode (absolute t-axis)
include_last = .TRUE. ! flag whether to include the last time step
output_filename = 'NWP' ! file name base
output_grid = .TRUE. ! flag whether grid information is added to output.
!
ml_varlist = 'u', 'v', 'w', 'temp', 'pres','topography_c', 'pres_msl',
'qv', 'qc', 'qi', 'qr', 'qs', 'tke',
'tkvm', 'tkvh', 'group:pbl_vars',
'group:precip_vars', 'group:additional_precip_vars',
'group:land_vars', 'group:land_tile_vars',
'group:multisnow_vars'
/
! meteogram_output_nml: meteogram output for specified locations --------------
&meteogram_output_nml
lmeteogram_enabled = .TRUE. ! .TRUE.=meteogram of output variables is desired
n0_mtgrm = 0 ! initial time step for meteogram output
ninc_mtgrm = 10 ! meteogram output interval (in terms of time steps)
ldistributed = .FALSE. ! .FALSE.=Do not separate files for each PE
stationlist_tot = 52.17, 14.12, 'Lindenberg',
51.97, 4.93, 'Cabauw',
-10.08, -61.93, 'LBA_Rondonia',
13.50, 2.5 , 'Niamey',
36.61, -97.49, 'ARM_Southern_Great_Plains',
-71.32, 156.62, 'ARM_North_Slope_of_Alaska_Barrow',
-2.06, 147.43, 'ARM_Tropical_W_Pacific_Manus',
-12.43, 130.89, 'ARM_Tropical_W_Pacific_Darwin',
60.00, 80.00, 'Snow Test Russia',
50.00, 8.6 , 'Frankfurt-Flughafen'
/
! sleve_nml: vertical level specification -------------------------------------
&sleve_nml
min_lay_thckn = 20.0 ! layer thickness of lowermost layer
top_height = 75000.0 ! height of model top
stretch_fac = 0.9 ! stretching factor to vary distribution of model levels
decay_scale_1 = 4000.0 ! decay scale of large-scale topography component
decay_scale_2 = 2500.0 ! decay scale of small-scale topography component
decay_exp = 1.2 ! exponent of decay function
flat_height = 16000.0 ! height above which the coordinate surfaces are flat
/
! radiation_nml: radiation scheme ---------------------------------------------
&radiation_nml
irad_o3 = 7 ! ozone climatology
irad_aero = 6 ! aerosols
albedo_type = 2 ! type of surface albedo
/
! transport_nml: tracer transport ---------------------------------------------
&transport_nml
ivadv_tracer = 3, 3, 3, 3, 3 ! tracer specific method to compute vertical advection
itype_hlimit = 3, 4, 4, 4, 4, 0 ! type of limiter for horizontal transport
ihadv_tracer = 52, 2, 2, 2, 2, 0 ! tracer specific method to compute horizontal advection
llsq_svd = .TRUE. ! use SV decomposition for least squares design matrix
beta_fct = 1.005 ! factor of allowed over-/undershooting in monotonous limiter
/
! turbdiff_nml: turbulent diffusion -------------------------------------------
&turbdiff_nml
tkhmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient
tkmmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient
pat_len = 750.0 !
c_diff = 0.2 !
rat_sea = 0.8 !
/
! lnd_nml: land scheme switches -----------------------------------------------
&lnd_nml
ntiles = 3 ! number of tiles
nlev_snow = 2 ! number of snow layers
lmulti_snow = .FALSE. ! .TRUE. for use of multi-layer snow model
idiag_snowfrac = 2 ! type of snow-fraction diagnosis
lsnowtile = .TRUE. ! .TRUE.=consider snow-covered and snow-free separately
itype_root = 2 ! root density distribution
itype_heatcond = 2 ! type of soil heat conductivity
itype_lndtbl = 2 ! table for associating surface parameters
lseaice = .TRUE. ! .TRUE. for use of sea-ice model
llake = .TRUE. ! .TRUE. for use of lake model
/
EOF
# ----------------------------------------------------------------------------
# run the model!
# ----------------------------------------------------------------------------
echo ${START} ${MODEL}
${START} ${MODEL}