From 5fb52ceb804edf8d2355024b7f06282e403808e3 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 15 Nov 2024 13:39:01 -0500 Subject: [PATCH] zonally averaged case AERmonZ / ta, aka CASE 4 now succeeds. objects assigned None and checked if corresponding cmor axis should bother to be assigned, instead of relying on dimenionality of the data --- fre/cmor/cmor_mixer.py | 140 +++++++++++++++++++++++++++---------- run_test_file_cases.py | 152 +++++++++++++++++++++++------------------ 2 files changed, 192 insertions(+), 100 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 0f4ef243..2de8d570 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -198,24 +198,48 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, check_dataset_for_ocean_grid(ds) - # figure out the dimension names programmatically TODO - # Define lat and lon dimensions - # Assume input file is lat/lon grid - lat = ds["lat"][:] - lon = ds["lon"][:] - lat_bnds = ds["lat_bnds"][:] - lon_bnds = ds["lon_bnds"][:] + # try to read what coordinate(s) we're going to be expecting for the variable + expected_mip_coord_dims=None + try: + expected_mip_coord_dims = proj_table_vars["variable_entry"] [target_var] ["dimensions"] + print( '(rewrite_netcdf_file_var) i am hoping to find data for the following coordinate dimensions:\n' + f' expected_mip_coord_dims = {expected_mip_coord_dims}' ) + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not get expected coordinate dimensions for {target_var}. ' + ' in proj_table_vars file {json_table_config}. \n exc = {exc}') + - ## Define time - #time = ds["time"][:] + ## figure out the coordinate/dimension names programmatically TODO + + # Attempt to read lat coordinates + print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lat, lat_bnds') + lat, lat_bnds = None, None + try: + lat, lat_bnds = ds["lat"][:], ds["lat_bnds"][:] + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not read latitude coordinate. moving on.\n exc = {exc}') + print(f' lat = {lat}') + print(f' lat_bnds = {lat_bnds}') + pass + + # Attempt to read lon coordinates + print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lon, lon_bnds') + lon, lon_bnds = None, None + try: + lon, lon_bnds = ds["lon"][:], ds["lon_bnds"][:] + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not read longitude coordinate. moving on.\n exc = {exc}') + print(f' lon = {lon}') + print(f' lon_bnds = {lon_bnds}') + pass # read in time_coords + units - time_coords = ds["time"][:] + time_coords = ds["time"][:] # out this in a try/except thingy, initializing like others? time_coord_units = ds["time"].units print(f"(rewrite_netcdf_file_var) time_coord_units = {time_coord_units}") # read in time_bnds , if present - time_bnds = [] + time_bnds = [] # shouldnt this be initialized like the others? try: time_bnds = ds["time_bnds"][:] #print(f"(rewrite_netcdf_file_var) time_bnds = {time_bnds}") @@ -226,10 +250,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # read the input variable data, i believe var = ds[target_var][:] - # determine the vertical dimension by looping over netcdf variables - vert_dim = get_vertical_dimension(ds, target_var) - print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") - # grab var_dim var_dim = len(var.shape) print(f"(rewrite_netcdf_file_var) var_dim = {var_dim}, local_var = {local_var}") @@ -238,11 +258,17 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, if var_dim not in [3, 4]: raise ValueError(f"var_dim == {var_dim} != 3 nor 4. stop.") + # determine the vertical dimension by looping over netcdf variables + vert_dim = get_vertical_dimension(ds, target_var) # returns int( 0 ) if not present + print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") + # Check var_dim and vert_dim and assign lev if relevant. # error if vert_dim wrong given var_dim lev = None - if var_dim == 4: - if vert_dim not in [ "plev30", "plev19", "plev8", + #if var_dim == 4: + # if vert_dim not in [ "plev30", "plev19", "plev8", + if vert_dim != 0: + if vert_dim not in [ "plev39", "plev30", "plev19", "plev8", "height2m", "level", "lev", "levhalf"] : raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') lev = ds[vert_dim] @@ -269,8 +295,27 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, units = proj_table_vars["variable_entry"] [target_var] ["units"] print(f"(rewrite_netcdf_file_var) units={units}") - cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") - cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + + # setup cmor latitude axis if relevant + #cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") + #cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + print(f'(rewrite_netcdf_file_var) assigning cmor_lat') + cmor_lat = None + if any( [ lat is None, lat_bnds is None ] ): + print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') + else: + cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") + + # setup cmor longitude axis if relevant + print(f'(rewrite_netcdf_file_var) assigning cmor_lon') + cmor_lon = None + if any( [ lon is None, lon_bnds is None ] ): + print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') + else: + cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + + # setup cmor time axis if relevant + cmor_time = None try: print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" f" coord_vals = \n{time_coords}, \n" @@ -283,20 +328,18 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, " coord_vals = time_coords, units = time_coord_units)") cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) - # initializations + + + # other vertical-axis-relevant initializations save_ps = False ps = None ierr_ap, ierr_b = None, None ips = None - # set axes for 3-dim case - if var_dim == 3: - axes = [cmor_time, cmor_lat, cmor_lon] - print(f"(rewrite_netcdf_file_var) axes = {axes}") - # set axes for 4-dim case - elif var_dim == 4: - - if vert_dim in ["plev30", "plev19", "plev8", "height2m"]: + # set cmor vertical axis if relevant + cmor_lev = None + if lev is not None: + if vert_dim in ["plev39", "plev30", "plev19", "plev8", "height2m"]: cmor_lev = cmor.axis( vert_dim, coord_vals = lev[:], units = lev.units ) @@ -342,15 +385,44 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) ierr_ap after calling cmor_zfactor: {ierr_ap}\n' f'(rewrite_netcdf_file_var) ierr_b after calling cmor_zfactor: {ierr_b}' ) + axis_ids = [] + if cmor_time is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_time to axis_ids list...') + axis_ids.append(cmor_time) + print(f' axis_ids now = {axis_ids}') + if cmor_lat is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lat to axis_ids list...') + axis_ids.append(cmor_lat) + print(f' axis_ids now = {axis_ids}') + if cmor_lon is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lon to axis_ids list...') + axis_ids.append(cmor_lon) + print(f' axis_ids now = {axis_ids}') + ips = cmor.zfactor( zaxis_id = cmor_lev, zfactor_name = "ps", - axis_ids = [cmor_time, cmor_lat, cmor_lon], + axis_ids = axis_ids, #[cmor_time, cmor_lat, cmor_lon], units = "Pa" ) save_ps = True - # assign axes at end of 4-dim case - axes = [cmor_time, cmor_lev, cmor_lat, cmor_lon] - - + + + axes = [] + if cmor_time is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_time to axes list...') + axes.append(cmor_time) + print(f' axes now = {axes}') + if cmor_lev is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lev to axes list...') + axes.append(cmor_lev) + print(f' axes now = {axes}') + if cmor_lat is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lat to axes list...') + axes.append(cmor_lat) + print(f' axes now = {axes}') + if cmor_lon is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lon to axes list...') + axes.append(cmor_lon) + print(f' axes now = {axes}') # read positive attribute and create cmor_var? can this return none? TODO @@ -359,7 +431,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor_var = cmor.variable(target_var, units, axes, positive = positive) # Write the output to disk - #var = ds[target_var][:] #was this ever needed? why? cmor.write(cmor_var, var) if save_ps: if any( [ ips is None, ps is None ] ): @@ -371,7 +442,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor.close(ips, file_name = True, preserve = False) filename = cmor.close(cmor_var, file_name = True, preserve = False) print(f"(rewrite_netcdf_file_var) returned by cmor.close: filename = {filename}") - #cmor.close() ds.close() print('-------------------------- END rewrite_netcdf_file_var call -----\n\n') diff --git a/run_test_file_cases.py b/run_test_file_cases.py index 1518c266..7f3fdb34 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -65,51 +65,80 @@ def run_cmor_RUN(filename, table, opt_var_name): return FOO_return -## 1) SUCCEEDs -## land, Lmon, gr1 -#testfile_land_gr1_Lmon = \ -# '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/pp/land/ts/monthly/5yr/land.005101-005512.lai.nc' -#try: -# some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') -#except: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'land_gr1_Lmon / lai') +# 8) FAIL (no latitude nor longitude coordinates cases) +# atmos, Amon / ch4global +# Result - error, +# File "src/netCDF4/_netCDF4.pyx", line 2519, in netCDF4._netCDF4.Dataset.__getitem__ IndexError: +# lat not found in / +testfile_atmos_scalar_gn_Amon_nolon_nolat = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_scalar/ts/monthly/5yr/' + \ + 'atmos_scalar.197001-197412.ch4global.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') -## 2) SUCCEEDs -## atmos, Amon / cl -#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/atmos_level_cmip/ts/monthly/5yr/atmos_level_cmip.196001-196412.cl.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') +sys.exit() -## 3) SUCCEEDs -## atmos, Amon / mc -#testfile_atmos_level_cmip_gr1_Amon_fullL = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/atmos_level_cmip/ts/monthly/5yr/atmos_level_cmip.195501-195912.mc.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') +# 1) SUCCEEDs +# land, Lmon, gr1 +testfile_land_gr1_Lmon = \ + '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/land/ts/monthly/5yr/' + \ + 'land.005101-005512.lai.nc' +try: + some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') +except: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'land_gr1_Lmon / lai') + + +# 2) SUCCEEDs +# atmos, Amon / cl +testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.196001-196412.cl.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') + + +# 3) SUCCEEDs +# atmos, Amon / mc +testfile_atmos_level_cmip_gr1_Amon_fullL = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.195501-195912.mc.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') + -# 4) FAIL (no longitude coordinate case) +# 4) SUCCEEDs (no longitude coordinate case) # atmos, AERmonZ / ta # just like #1, but lack longitude -# Result - error, File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", line 195, in rewrite_netcdf_file_var lon = ds["lon"][:] File "src/netCDF4/_netCDF4.pyx", line 2519, in netCDF4._netCDF4.Dataset.__getitem__ IndexError: lon not found in / testfile_atmos_gr1_AERmonZ_nolons = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/atmos_plev39_cmip.201001-201412.ta.nc' + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ + 'atmos_plev39_cmip.201001-201412.ta.nc' try: some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') except Exception as exc: @@ -118,19 +147,20 @@ def run_cmor_RUN(filename, table, opt_var_name): pass print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') -sys.exit() -## 5) SUCCEEDS -## ocean, Omon / sos -#testfile_ocean_monthly_1x1deg_gr = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/ocean_monthly_1x1deg/ts/monthly/5yr/ocean_monthly_1x1deg.190001-190412.sos.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') +# 5) SUCCEEDs +# ocean, Omon / sos +testfile_ocean_monthly_1x1deg_gr = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_1x1deg.190001-190412.sos.nc' +try: + some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') @@ -138,7 +168,9 @@ def run_cmor_RUN(filename, table, opt_var_name): ## ocean, Omon / sos ## Result - error, AttributeError: NetCDF: Attempt to define fill value when data already exists. #testfile_ocean_monthly_gn = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/pp/ocean_monthly/ts/monthly/5yr/ocean_monthly.002101-002512.sos.nc' +# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/ocean_monthly/ts/monthly/5yr/' + \ +# 'ocean_monthly.002101-002512.sos.nc' #try: # some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') #except Exception as exc: @@ -153,7 +185,9 @@ def run_cmor_RUN(filename, table, opt_var_name): ## ocean, Omon / so ## Result - identical failure to #6 #testfile_ocean_monthly_z_1x1deg_gr = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/ocean_monthly_z_1x1deg.000101-000512.so.nc' +# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ +# 'ocean_monthly_z_1x1deg.000101-000512.so.nc' #try: # some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') #except Exception as exc: @@ -163,27 +197,15 @@ def run_cmor_RUN(filename, table, opt_var_name): #print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') -# 8) FAIL (no latitude nor longitude coordinates cases) -# atmos, Amon / ch4global -# Result - error, File "src/netCDF4/_netCDF4.pyx", line 2519, in netCDF4._netCDF4.Dataset.__getitem__ IndexError: lat not found in / -testfile_atmos_scalar_gn_Amon_nolon_nolat = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/atmos_scalar/ts/monthly/5yr/atmos_scalar.197001-197412.ch4global.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') - - ## 9) FAIL (4 dimensional data with no vertical) ## Result - error, ## File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", ## line 134, in get_vertical_dimension if not (ds[dim].axis and ds[dim].axis == "Z"): ## AttributeError: NetCDF: Attribute not found #testfile_LUmip_refined_gr1_Emon_landusedim = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/LUmip_refined/ts/monthly/5yr/LUmip_refined.185001-185412.gppLut.nc' +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/LUmip_refined/ts/monthly/5yr/' + \ +# 'LUmip_refined.185001-185412.gppLut.nc' #try: # some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') #except Exception as exc: