get current UTC timestamps in the recommended way

as per https://docs.python.org/3/library/datetime.html#datetime.datetime.utcnow
+ refactor into common module
(addresses #125 for timeseries products only)

aodntools/timeseries_products/aggregated_timeseries.py

@@ -5,15 +5,15 @@
 import os
 import shutil
 import tempfile
-from datetime import datetime
 
 import numpy as np
 import xarray as xr
 from netCDF4 import Dataset, num2date, stringtochar
 from pkg_resources import resource_filename
 
 from aodntools import __version__
-from aodntools.timeseries_products.common import NoInputFilesError, check_file, in_water
+from aodntools.timeseries_products.common import (NoInputFilesError, check_file, in_water, current_utc_timestamp,
+                                                  TIMESTAMP_FORMAT, DATESTAMP_FORMAT)
 
 TEMPLATE_JSON = resource_filename(__name__, 'aggregated_timeseries_template.json')
 
@@ -305,13 +305,10 @@ def main_aggregator(files_to_agg, var_to_agg, site_code, input_dir='', output_di
         ds['source_file'].setncatts(source_file_attributes(download_url_prefix, opendap_url_prefix))
 
     ## set global attrs
-    timeformat = '%Y-%m-%dT%H:%M:%SZ'
-    file_timeformat = '%Y%m%d'
-
-    time_start = num2date(np.min(TIME[:]), time_units, time_calendar).strftime(timeformat)
-    time_end = num2date(np.max(TIME[:]), time_units, time_calendar).strftime(timeformat)
-    time_start_filename = num2date(np.min(TIME[:]), time_units, time_calendar).strftime(file_timeformat)
-    time_end_filename = num2date(np.max(TIME[:]), time_units, time_calendar).strftime(file_timeformat)
+    time_start = num2date(np.min(TIME[:]), time_units, time_calendar).strftime(TIMESTAMP_FORMAT)
+    time_end = num2date(np.max(TIME[:]), time_units, time_calendar).strftime(TIMESTAMP_FORMAT)
+    time_start_filename = num2date(np.min(TIME[:]), time_units, time_calendar).strftime(DATESTAMP_FORMAT)
+    time_end_filename = num2date(np.max(TIME[:]), time_units, time_calendar).strftime(DATESTAMP_FORMAT)
 
     add_attribute = {
                     'title':                    ("Long Timeseries Velocity Aggregated product: " + var_to_agg + " at " +
@@ -325,8 +322,8 @@ def main_aggregator(files_to_agg, var_to_agg, site_code, input_dir='', output_di
                     'geospatial_lat_max':       np.max(ds['LATITUDE'][:]),
                     'geospatial_lon_min':       np.min(ds['LONGITUDE'][:]),
                     'geospatial_lon_max':       np.max(ds['LONGITUDE'][:]),
-                    'date_created':             datetime.utcnow().strftime(timeformat),
-                    'history':                  datetime.utcnow().strftime(timeformat) + ': Aggregated file created.',
+                    'date_created': current_utc_timestamp(),
+                    'history': current_utc_timestamp() + ': Aggregated file created.',
                     'keywords':                 ', '.join([var_to_agg, 'AGGREGATED']),
                     'rejected_files':           "\n".join(rejected_files),
                     'generating_code_version':  __version__}
@@ -348,7 +345,7 @@ def main_aggregator(files_to_agg, var_to_agg, site_code, input_dir='', output_di
     file_version = 1
     output_name = '_'.join(['IMOS', facility_code, data_code, time_start_filename, site_code, ('FV0'+str(file_version)),
                             (var_to_agg + "-" + product_type),
-                            ('END-'+ time_end_filename), 'C-' + datetime.utcnow().strftime(file_timeformat)]) + '.nc'
+                            ('END-'+ time_end_filename), 'C-' + current_utc_timestamp(DATESTAMP_FORMAT)]) + '.nc'
     ncout_path = os.path.join(output_dir, output_name)
     shutil.move(temp_outfile, os.path.join(output_dir, ncout_path))
 

aodntools/timeseries_products/common.py

@@ -1,6 +1,12 @@
 """Code shared by all timeseries product generating code"""
+from datetime import datetime, timezone
+
 import numpy as np
 
+# Common date/time format strings
+TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+DATESTAMP_FORMAT = '%Y%m%d'
+
 
 class NoInputFilesError(Exception):
     """Exception raised if there are no valid input files to aggregate"""
@@ -183,4 +189,8 @@ def in_water(nc):
     :param nc: xarray dataset
     :return: xarray dataset
     """
-    return nc.where(in_water_index(nc), drop=True)
+    return nc.where(in_water_index(nc), drop=True)
+
+
+def current_utc_timestamp(format=TIMESTAMP_FORMAT):
+    return datetime.now(timezone.utc).strftime(format)

aodntools/timeseries_products/gridded_timeseries.py

@@ -3,14 +3,15 @@
 import argparse
 import os.path
 import json
-from datetime import datetime
+from datetime import datetime, timezone
 
 import xarray as xr
 import pandas as pd
 
 from pkg_resources import resource_filename
 
 from aodntools import __version__
+from aodntools.timeseries_products.common import current_utc_timestamp, TIMESTAMP_FORMAT, DATESTAMP_FORMAT
 import aodntools.timeseries_products.aggregated_timeseries as TStools
 
 
@@ -122,14 +123,12 @@ def generate_netcdf_output_filename(nc, facility_code, data_code, VoI, site_code
     :return: name of the output file
     """
 
-    file_timeformat = '%Y%m%d'
-
     if '_' in VoI:
         VoI = VoI.replace('_', '-')
-    t_start = pd.to_datetime(nc.TIME.min().values).strftime(file_timeformat)
-    t_end = pd.to_datetime(nc.TIME.max().values).strftime(file_timeformat)
+    t_start = pd.to_datetime(nc.TIME.min().values).strftime(DATESTAMP_FORMAT)
+    t_end = pd.to_datetime(nc.TIME.max().values).strftime(DATESTAMP_FORMAT)
 
-    output_name = '_'.join(['IMOS', facility_code, data_code, t_start, site_code, ('FV0'+str(file_version)), (VoI+"-"+product_type), ('END-'+ t_end), 'C-' + datetime.utcnow().strftime(file_timeformat)]) + '.nc'
+    output_name = '_'.join(['IMOS', facility_code, data_code, t_start, site_code, ('FV0'+str(file_version)), (VoI+"-"+product_type), ('END-'+ t_end), 'C-' + current_utc_timestamp(DATESTAMP_FORMAT)]) + '.nc'
 
     return output_name
 
@@ -250,10 +249,9 @@ def grid_variable(input_file, VoI, depth_bins=None, max_separation=50, depth_bin
     for attr in ('geospatial_lat_min', 'geospatial_lat_max', 'geospatial_lon_min', 'geospatial_lon_max', 'site_code',
                  'included_values_flagged_as', 'contributor_name', 'contributor_role', 'contributor_email'):
         VoI_interpolated.attrs[attr] = input_global_attributes[attr]
-    timeformat = '%Y-%m-%dT%H:%M:%SZ'
-    date_start = pd.to_datetime(VoI_interpolated.TIME.values.min()).strftime(timeformat)
-    date_end = pd.to_datetime(VoI_interpolated.TIME.values.max()).strftime(timeformat)
-    date_created = datetime.utcnow().strftime(timeformat)
+    date_start = pd.to_datetime(VoI_interpolated.TIME.values.min()).strftime(TIMESTAMP_FORMAT)
+    date_end = pd.to_datetime(VoI_interpolated.TIME.values.max()).strftime(TIMESTAMP_FORMAT)
+    date_created = current_utc_timestamp()
     VoI_interpolated.attrs.update(global_attribute_dictionary)
     VoI_interpolated.attrs.update({
         'source_file':           input_file,

aodntools/timeseries_products/hourly_timeseries.py

@@ -4,7 +4,6 @@
 import json
 import os.path
 from collections import OrderedDict
-from datetime import datetime
 
 import numpy as np
 import pandas as pd
@@ -14,7 +13,8 @@
 
 from aodntools import __version__
 from aodntools.timeseries_products import aggregated_timeseries as utils
-from aodntools.timeseries_products.common import NoInputFilesError, check_file, get_qc_variable_names, in_water
+from aodntools.timeseries_products.common import (NoInputFilesError, check_file, get_qc_variable_names, in_water,
+                                                  current_utc_timestamp, TIMESTAMP_FORMAT, DATESTAMP_FORMAT)
 
 TEMPLATE_JSON = resource_filename(__name__, 'hourly_timeseries_template.json')
 BINNING_METHOD_JSON = resource_filename(__name__, 'binning_method.json')
@@ -180,8 +180,8 @@ def set_globalattr(nc_aggregated, templatefile, site_code, add_attribute, parame
                 'geospatial_lat_max': nc_aggregated.LATITUDE.values.max(),
                 'geospatial_lon_min': nc_aggregated.LONGITUDE.values.min(),
                 'geospatial_lon_max': nc_aggregated.LONGITUDE.values.max(),
-                'date_created': datetime.utcnow().strftime(timeformat),
-                'history': datetime.utcnow().strftime(timeformat) + ': Hourly aggregated file created.',
+                'date_created': current_utc_timestamp(),
+                'history': current_utc_timestamp() + ': Hourly aggregated file created.',
                 'keywords': ', '.join(parameter_names + ['HOURLY', 'AGGREGATED'])}
     global_metadata.update(agg_attr)
     global_metadata.update(add_attribute)
@@ -259,14 +259,12 @@ def generate_netcdf_output_filename(nc, facility_code, data_code, site_code, pro
     :return: name of the output file
     """
 
-    file_timeformat = '%Y%m%d'
-
-    t_start = pd.to_datetime(nc.TIME.min().values).strftime(file_timeformat)
-    t_end = pd.to_datetime(nc.TIME.max().values).strftime(file_timeformat)
+    t_start = pd.to_datetime(nc.TIME.min().values).strftime(DATESTAMP_FORMAT)
+    t_end = pd.to_datetime(nc.TIME.max().values).strftime(DATESTAMP_FORMAT)
 
     output_name = '_'.join(
         ['IMOS', facility_code, data_code, t_start, site_code, ('FV0' + str(file_version)), product_type,
-         ('END-' + t_end), 'C-' + datetime.utcnow().strftime(file_timeformat)]) + '.nc'
+         ('END-' + t_end), 'C-' + current_utc_timestamp(DATESTAMP_FORMAT)]) + '.nc'
 
     return output_name
 

aodntools/timeseries_products/velocity_aggregated_timeseries.py

@@ -4,15 +4,15 @@
 from  netCDF4 import Dataset, num2date, stringtochar
 import numpy as np
 import json
-from datetime import datetime
 import argparse
 from pkg_resources import resource_filename
 from aodntools import __version__
 
 import xarray as xr
 
 from aodntools.timeseries_products import aggregated_timeseries as utils
-from aodntools.timeseries_products.common import NoInputFilesError, check_velocity_file
+from aodntools.timeseries_products.common import (NoInputFilesError, check_velocity_file, current_utc_timestamp,
+                                                  TIMESTAMP_FORMAT, DATESTAMP_FORMAT)
 
 TEMPLATE_JSON = resource_filename(__name__, 'velocity_aggregated_timeseries_template.json')
 
@@ -188,13 +188,10 @@ def velocity_aggregated(files_to_agg, site_code, input_dir='', output_dir='./',
         ds['source_file'].setncatts(utils.source_file_attributes(download_url_prefix, opendap_url_prefix))
 
     ## set global attrs
-    timeformat = '%Y-%m-%dT%H:%M:%SZ'
-    file_timeformat = '%Y%m%d'
-
-    time_start = num2date(np.min(TIME[:]), time_units, time_calendar).strftime(timeformat)
-    time_end = num2date(np.max(TIME[:]), time_units, time_calendar).strftime(timeformat)
-    time_start_filename = num2date(np.min(TIME[:]), time_units, time_calendar).strftime(file_timeformat)
-    time_end_filename = num2date(np.max(TIME[:]), time_units, time_calendar).strftime(file_timeformat)
+    time_start = num2date(np.min(TIME[:]), time_units, time_calendar).strftime(TIMESTAMP_FORMAT)
+    time_end = num2date(np.max(TIME[:]), time_units, time_calendar).strftime(TIMESTAMP_FORMAT)
+    time_start_filename = num2date(np.min(TIME[:]), time_units, time_calendar).strftime(DATESTAMP_FORMAT)
+    time_end_filename = num2date(np.max(TIME[:]), time_units, time_calendar).strftime(DATESTAMP_FORMAT)
 
     add_attribute = {
                     'title':                    ("Long Timeseries Velocity Aggregated product: " + ', '.join(varlist) + " at " +
@@ -208,8 +205,8 @@ def velocity_aggregated(files_to_agg, site_code, input_dir='', output_dir='./',
                     'geospatial_lat_max':       np.max(ds['LATITUDE']),
                     'geospatial_lon_min':       np.min(ds['LONGITUDE']),
                     'geospatial_lon_max':       np.max(ds['LONGITUDE']),
-                    'date_created':             datetime.utcnow().strftime(timeformat),
-                    'history':                  datetime.utcnow().strftime(timeformat) + ': Aggregated file created.',
+                    'date_created':             current_utc_timestamp(),
+                    'history':                  current_utc_timestamp() + ': Aggregated file created.',
                     'keywords':                 ', '.join(varlist + ['AGGREGATED']),
                     'rejected_files':           "\n".join(bad_files.keys()),
                     'generating_code_version':  __version__
@@ -235,7 +232,7 @@ def velocity_aggregated(files_to_agg, site_code, input_dir='', output_dir='./',
     file_version = 1
     output_name = '_'.join(['IMOS', facility_code, data_code, time_start_filename, site_code, ('FV0'+str(file_version)),
                             ("velocity-"+product_type),
-                            ('END-'+ time_end_filename), 'C-' + datetime.utcnow().strftime(file_timeformat)]) + '.nc'
+                            ('END-'+ time_end_filename), 'C-' + current_utc_timestamp(DATESTAMP_FORMAT)]) + '.nc'
     ncout_path = os.path.join(output_dir, output_name)
     shutil.move(temp_outfile, ncout_path)
 

aodntools/timeseries_products/velocity_hourly_timeseries.py

@@ -3,7 +3,6 @@
 import os
 import shutil
 import tempfile
-from datetime import datetime
 
 import numpy as np
 import pandas as pd
@@ -13,7 +12,8 @@
 
 import aodntools.timeseries_products.aggregated_timeseries as utils
 from aodntools import __version__
-from aodntools.timeseries_products.common import NoInputFilesError, check_velocity_file
+from aodntools.timeseries_products.common import (NoInputFilesError, check_velocity_file, current_utc_timestamp,
+                                                  TIMESTAMP_FORMAT, DATESTAMP_FORMAT)
 
 TEMPLATE_JSON = resource_filename(__name__, 'velocity_hourly_timeseries_template.json')
 QC_FLAG_MAX = 2
@@ -252,13 +252,10 @@ def velocity_hourly_aggregated(files_to_agg, site_code, input_dir='', output_dir
         ds['source_file'].setncatts(utils.source_file_attributes(download_url_prefix, opendap_url_prefix))
 
     ## set global attrs
-    timeformat = '%Y-%m-%dT%H:%M:%SZ'
-    file_timeformat = '%Y%m%d'
-
-    time_start = num2date(np.min(TIME[:]), TIME_UNITS, TIME_CALENDAR).strftime(timeformat)
-    time_end = num2date(np.max(TIME[:]), TIME_UNITS, TIME_CALENDAR).strftime(timeformat)
-    time_start_filename = num2date(np.min(TIME[:]), TIME_UNITS, TIME_CALENDAR).strftime(file_timeformat)
-    time_end_filename = num2date(np.max(TIME[:]), TIME_UNITS, TIME_CALENDAR).strftime(file_timeformat)
+    time_start = num2date(np.min(TIME[:]), TIME_UNITS, TIME_CALENDAR).strftime(TIMESTAMP_FORMAT)
+    time_end = num2date(np.max(TIME[:]), TIME_UNITS, TIME_CALENDAR).strftime(TIMESTAMP_FORMAT)
+    time_start_filename = num2date(np.min(TIME[:]), TIME_UNITS, TIME_CALENDAR).strftime(DATESTAMP_FORMAT)
+    time_end_filename = num2date(np.max(TIME[:]), TIME_UNITS, TIME_CALENDAR).strftime(DATESTAMP_FORMAT)
 
 
     add_attribute = {
@@ -273,8 +270,8 @@ def velocity_hourly_aggregated(files_to_agg, site_code, input_dir='', output_dir
                     'geospatial_lat_max':       np.float64(np.max(ds['LATITUDE'])),
                     'geospatial_lon_min':       np.float64(np.min(ds['LONGITUDE'])),
                     'geospatial_lon_max':       np.float64(np.max(ds['LONGITUDE'])),
-                    'date_created':             datetime.utcnow().strftime(timeformat),
-                    'history':                  datetime.utcnow().strftime(timeformat) + ': Aggregated file created.',
+                    'date_created':             current_utc_timestamp(),
+                    'history':                  current_utc_timestamp() + ': Aggregated file created.',
                     'keywords':                 ', '.join(varlist + ['AGGREGATED']),
                     'rejected_files':           "\n".join(bad_files.keys()),
                     'generating_code_version':  __version__
@@ -306,7 +303,7 @@ def velocity_hourly_aggregated(files_to_agg, site_code, input_dir='', output_dir
     file_version = 2
     output_name = '_'.join(['IMOS', facility_code, data_code, time_start_filename, site_code, ('FV0'+str(file_version)),
                             ("velocity-"+product_type),
-                            ('END-'+ time_end_filename), 'C-' + datetime.utcnow().strftime(file_timeformat)]) + '.nc'
+                            ('END-'+ time_end_filename), 'C-' + current_utc_timestamp(DATESTAMP_FORMAT)]) + '.nc'
     ncout_path = os.path.join(output_dir, output_name)
     shutil.move(temp_outfile, ncout_path)