Skip to content

Commit

Permalink
Merge pull request #18 from MET-OM/dev_cache
Browse files Browse the repository at this point in the history 
remove cache files & add ValueError for points not found in the model grid
  • Loading branch information
@KonstantinChri
KonstantinChri authored Jul 14, 2024
2 parents 7cfe845 + 2b63eb7 commit 8b108d1
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 37 deletions.
6 changes: 5 additions & 1 deletion metocean_api/ts/aux_funcs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ def distance_2points(lat1, lon1, lat2, lon2):
a = np.sin(dlat / 2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2)**2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
distance = R * c # in km

if distance.min() > 50: # min distance between requested and available grid points above 50 km
raise ValueError("Requested grid point out of model domain!!!")

return distance


Expand Down Expand Up @@ -345,7 +349,7 @@ def create_dataframe(product,ds, lon_near, lat_near,outfile,variable, start_time

if save_nc == True:
ds.to_netcdf(outfile.replace('csv','nc'))

return df

def check_datafile_exists(datafile):
Expand Down
110 changes: 75 additions & 35 deletions metocean_api/ts/read_metno.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from .aux_funcs import *

def NORAC_ts(self, save_csv = False, save_nc = False):
def NORAC_ts(self, save_csv = False, save_nc = False, save_cache =False):
"""
Extract times series of the nearest gird point (lon,lat) from
norac wave hindcast and save it as netcdf.
Expand Down Expand Up @@ -38,14 +38,18 @@ def NORAC_ts(self, save_csv = False, save_nc = False):
#Save in csv format
df = create_dataframe(product=self.product,ds=ds, lon_near=lon_near, lat_near=lat_near, outfile=self.datafile, variable=self.variable[:2], start_time = self.start_time, end_time = self.end_time, save_csv=save_csv,save_nc = save_nc, height=self.height)
ds.close()
#remove temp files
#for i in range(len(date_list)):
# os.remove(tempfile[i])
#remove temp/cache files
if save_cache == False:
for i in range(len(date_list)):
try:
os.remove(tempfile[i])
except PermissionError:
print(f"Skipping deletion of {tempfile[i]} due to PermissionError")

return df


def NORA3_wind_wave_ts(self, save_csv = False, save_nc = False):
def NORA3_wind_wave_ts(self, save_csv = False, save_nc = False, save_cache =False):
"""
Extract times series of the nearest gird point (lon,lat) from
nora3 wind and wave hindcast and save it as netcdf.
Expand Down Expand Up @@ -73,13 +77,17 @@ def NORA3_wind_wave_ts(self, save_csv = False, save_nc = False):
#Save in csv format
df = create_dataframe(product=self.product,ds=ds, lon_near=lon_near, lat_near=lat_near, outfile=self.datafile, variable=self.variable[:2], start_time = self.start_time, end_time = self.end_time, save_csv=save_csv, save_nc = save_nc, height=self.height)
ds.close()
#remove temp files
#for i in range(len(date_list)):
# os.remove(tempfile[i])
#remove temp/cache files
if save_cache == False:
for i in range(len(date_list)):
try:
os.remove(tempfile[i])
except PermissionError:
print(f"Skipping deletion of {tempfile[i]} due to PermissionError")

return df

def NORA3_atm_ts(self, save_csv = False, save_nc = False):
def NORA3_atm_ts(self, save_csv = False, save_nc = False, save_cache =False):
"""
Extract times series of the nearest gird point (lon,lat) from
nora3 atm. hindcast (parameteres exc. wind & waves) and save it as netcdf.
Expand Down Expand Up @@ -108,13 +116,17 @@ def NORA3_atm_ts(self, save_csv = False, save_nc = False):
#Save in csv format
df = create_dataframe(product=self.product,ds=ds, lon_near=lon_near, lat_near=lat_near, outfile=self.datafile, variable=self.variable[:2], start_time = self.start_time, end_time = self.end_time, save_csv=save_csv,save_nc = save_nc, height=self.height)
ds.close()
#remove temp files
#for i in range(len(date_list)):
# os.remove(tempfile[i])
#remove temp/cache files
if save_cache == False:
for i in range(len(date_list)):
try:
os.remove(tempfile[i])
except PermissionError:
print(f"Skipping deletion of {tempfile[i]} due to PermissionError")

return df

def NORA3_atm3hr_ts(self, save_csv = False, save_nc = False):
def NORA3_atm3hr_ts(self, save_csv = False, save_nc = False, save_cache =False):
"""
Extract times series of the nearest grid point (lon,lat) from
nora3 atm. hindcast 3-hour files (parameters fex. wind & temperature) and save it as netcdf.
Expand Down Expand Up @@ -160,13 +172,17 @@ def NORA3_atm3hr_ts(self, save_csv = False, save_nc = False):
#Save in csv format
df = create_dataframe(product=self.product,ds=ds, lon_near=lon_near, lat_near=lat_near, outfile=self.datafile, variable=self.variable[:-2], start_time = self.start_time, end_time = self.end_time, save_csv=save_csv,save_nc=save_nc, height=self.height)
ds.close()
#remove temp files
#for i in range(len(date_list)):
# os.remove(tempfile[i])
#remove temp/cache files
if save_cache == False:
for i in range(len(date_list)):
try:
os.remove(tempfile[i])
except PermissionError:
print(f"Skipping deletion of {tempfile[i]} due to PermissionError")

return df

def NORA3_stormsurge_ts(self, save_csv = False,save_nc = False):
def NORA3_stormsurge_ts(self, save_csv = False,save_nc = False, save_cache =False):
"""
Extract times series of the nearest gird point (lon,lat) from
nora3 sea level dataset and save it as netcdf.
Expand Down Expand Up @@ -197,13 +213,17 @@ def NORA3_stormsurge_ts(self, save_csv = False,save_nc = False):
#Save in csv format
df = create_dataframe(product=self.product,ds=ds, lon_near=lon_near, lat_near=lat_near, outfile=self.datafile, variable=self.variable, start_time = self.start_time, end_time = self.end_time, save_csv=save_csv,save_nc = save_nc, height=self.height)
ds.close()
#remove temp files
#for i in range(len(date_list)):
# os.remove(tempfile[i])
#remove temp/cache files
if save_cache == False:
for i in range(len(date_list)):
try:
os.remove(tempfile[i])
except PermissionError:
print(f"Skipping deletion of {tempfile[i]} due to PermissionError")

return df

def NORKYST800_ts(self, save_csv = False, save_nc = False):
def NORKYST800_ts(self, save_csv = False, save_nc = False, save_cache = False):
"""
Extract times series of the nearest gird point (lon,lat) from
Norkyst800.
Expand Down Expand Up @@ -233,15 +253,19 @@ def NORKYST800_ts(self, save_csv = False, save_nc = False):
#Save in csv format
df = create_dataframe(product=self.product,ds=ds, lon_near=lon_near, lat_near=lat_near, outfile=self.datafile, variable=self.variable[:-2], start_time = self.start_time, end_time = self.end_time, save_csv=save_csv,save_nc = save_nc, height=self.height)
ds.close()
#remove temp files
#for i in range(len(date_list)):
# os.remove(tempfile[i])
#remove temp/cache files
if save_cache == False:
for i in range(len(date_list)):
try:
os.remove(tempfile[i])
except PermissionError:
print(f"Skipping deletion of {tempfile[i]} due to PermissionError")

return df



def NORA3_combined_ts(self, save_csv = True,save_nc = False):
def NORA3_combined_ts(self, save_csv = True,save_nc = False, save_cache =False):
self.variable = ['hs','tp','fpI', 'tm1','tm2','tmp','Pdir','thq', 'hs_sea','tp_sea','thq_sea' ,'hs_swell','tp_swell','thq_swell']
self.product = 'NORA3_wave_sub'
df_wave = NORA3_wind_wave_ts(self, save_csv=True)
Expand Down Expand Up @@ -269,7 +293,7 @@ def NORA3_combined_ts(self, save_csv = True,save_nc = False):

return df

def NorkystDA_surface_ts(self, save_csv = False,save_nc = False):
def NorkystDA_surface_ts(self, save_csv = False,save_nc = False, save_cache =False):
"""
Extract times series of the nearest gird point (lon,lat) from
norkystDA surface dataset and save it as netcdf or csv.
Expand Down Expand Up @@ -298,13 +322,17 @@ def NorkystDA_surface_ts(self, save_csv = False,save_nc = False):
#Save in csv format
df = create_dataframe(product=self.product,ds=ds, lon_near=lon_near, lat_near=lat_near, outfile=self.datafile, variable=self.variable, start_time = self.start_time, end_time = self.end_time, save_csv=save_csv,save_nc = save_nc, height=self.height)
ds.close()
#remove temp files
#for i in range(len(date_list)):
# os.remove(tempfile[i])
#remove temp/cache files
if save_cache == False:
for i in range(len(date_list)):
try:
os.remove(tempfile[i])
except PermissionError:
print(f"Skipping deletion of {tempfile[i]} due to PermissionError")

return df

def NorkystDA_zdepth_ts(self, save_csv = False,save_nc = False):
def NorkystDA_zdepth_ts(self, save_csv = False,save_nc = False, save_cache =False):
"""
Extract times series of the nearest gird point (lon,lat) from
norkystDA surface dataset and save it as netcdf or csv.
Expand Down Expand Up @@ -333,10 +361,14 @@ def NorkystDA_zdepth_ts(self, save_csv = False,save_nc = False):
#Save in csv format
df = create_dataframe(product=self.product,ds=ds, lon_near=lon_near, lat_near=lat_near, outfile=self.datafile, variable=self.variable, start_time = self.start_time, end_time = self.end_time, save_csv=save_csv,save_nc = save_nc, height=self.height, depth = self.depth)
ds.close()
#remove temp files
#for i in range(len(date_list)):
# os.remove(tempfile[i])

#remove temp/cache files
if save_cache == False:
for i in range(len(date_list)):
try:
os.remove(tempfile[i])
except PermissionError:
print(f"Skipping deletion of {tempfile[i]} due to PermissionError")

return df

def tempfile_dir(product,lon,lat, date_list,dirName):
Expand Down Expand Up @@ -366,7 +398,7 @@ def apply_nco(infile,tempfile,opt):
else:
break

def OBS_E39(self, save_csv = False, save_nc = False):
def OBS_E39(self, save_csv = False, save_nc = False, save_cache =False):
"""
Extract times series of metocean E39 observations and save it as netcdf/csv.
"""
Expand All @@ -392,6 +424,14 @@ def OBS_E39(self, save_csv = False, save_nc = False):
self.datafile.replace(self.datafile.split('_')[-4],'lon'+str(np.round(ds.longitude.mean().values,2)))
df = create_dataframe(product=self.product,ds=ds, lon_near=ds.longitude.mean().values, lat_near=ds.latitude.mean().values, outfile=self.datafile, variable=self.variable, start_time = self.start_time, end_time = self.end_time, save_csv=save_csv,save_nc = save_nc, height=self.height)
ds.close()
#remove temp/cache files
if save_cache == False:
for i in range(len(date_list)):
try:
os.remove(tempfile[i])
except PermissionError:
print(f"Skipping deletion of {tempfile[i]} due to PermissionError")

print('Data saved at: ' +self.datafile)

return df
2 changes: 1 addition & 1 deletion version.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
__version__ = "1.1.4"
__version__ = "1.1.5"


def git_describe():
Expand Down

0 comments on commit 8b108d1

Please sign in to comment.