Update integrated test baseline storage

geos_ats_package/geos_ats/baseline_io.py

@@ -9,14 +9,20 @@
 logger = logging.getLogger( 'geos_ats' )
 
 
-def download_baselines( bucket_name, blob_name, baseline_path, force_redownload=False ):
+def download_baselines( bucket_name: str,
+                        blob_name: str,
+                        baseline_path: str,
+                        force_redownload: bool = False,
+                        ok_delete_old_baselines: bool = False ):
     """
     Download and unpack baselines from GCP to the local machine
 
     Args:
         bucket_name (str): Name of the GCP bucket
         blob_name (str): Name of the baseline blob
         baseline_path (str): Path to unpack the baselines
+        force_redownload (bool): Force re-download baseline files
+        ok_delete_old_baselines (bool): Automatically delete old baseline files if present
     """
     # Setup
     baseline_path = os.path.abspath( os.path.expanduser( baseline_path ) )
@@ -33,6 +39,22 @@ def download_baselines( bucket_name, blob_name, baseline_path, force_redownload=
                 logger.info( 'To re-download these files, run with the force_redownload option' )
                 return
 
+        if not ok_delete_old_baselines:
+            for ii in range( 10 ):
+                print( f'Existing baseline files found: {baseline_path}' )
+                user_input = input( 'Delete old baselines?  [y/n]' )
+                user_input = user_input.strip().lower()
+                if user_input in [ "y", "yes" ]:
+                    logger.debug( 'User chose to delete old baselines' )
+                    break
+                elif user_input in [ "n", "no" ]:
+                    logger.debug( 'User chose to keep old baselines' )
+                    logger.warning( 'Running with out of date baseline files' )
+                    return
+                else:
+                    print( f'Unrecognized option: {user_input}' )
+            raise Exception( 'Failed to parse user options for old baselines' )
+
         logger.info( 'Deleting old baselines...' )
         shutil.rmtree( baseline_path )
 
@@ -44,7 +66,9 @@ def download_baselines( bucket_name, blob_name, baseline_path, force_redownload=
         logger.info( 'Downloading baselines...' )
         tmpdir = tempfile.TemporaryDirectory()
         archive_name = os.path.join( tmpdir.name, 'baselines.tar.gz' )
-        client = storage.Client()
+
+        # Download from GCP
+        client = storage.Client( use_auth_w_custom_endpoint=False )
         bucket = client.bucket( bucket_name )
         blob = bucket.blob( blob_name )
         blob.download_to_filename( archive_name )
@@ -59,7 +83,7 @@ def download_baselines( bucket_name, blob_name, baseline_path, force_redownload=
         logger.error( str( e ) )
 
 
-def upload_baselines( bucket_name, blob_name, baseline_path ):
+def upload_baselines( bucket_name: str, blob_name: str, baseline_path: str ):
     """
     Pack and upload baselines to GCP
 
@@ -91,6 +115,7 @@ def upload_baselines( bucket_name, blob_name, baseline_path ):
         archive_name = os.path.join( tmpdir.name, 'baselines.tar.gz' )
         shutil.make_archive( archive_name, format='gztar', base_dir=baseline_path )
 
+        # Upload to gcp
         logger.info( 'Uploading baseline files...' )
         client = storage.Client()
         bucket = client.bucket( bucket_name )
@@ -113,17 +138,20 @@ def manage_baselines( options ):
     """
     # Check for integrated test yaml file
     test_yaml = ''
-    if options.testYAML:
-        test_yaml = options.testYAML
+    if options.yaml:
+        test_yaml = options.yaml
     else:
-        os.path.join( options.geos_bin_dir ), '..', '..', '.integrated_tests.yaml'
+        test_yaml = os.path.join( options.geos_bin_dir, '..', '..', '.integrated_tests.yaml' )
 
     if not os.path.isfile( test_yaml ):
         raise Exception( f'Could not find the integrated test yaml file: {test_yaml}' )
 
-    test_options = yaml.load( open( test_yaml ) )
+    test_options = {}
+    with open( test_yaml ) as f:
+        test_options = yaml.safe_load( f )
+
     baseline_options = test_options.get( 'baselines', {} )
-    for k in [ 'bucket', 'latest' ]:
+    for k in [ 'bucket', 'baseline' ]:
         if k not in baseline_options:
             raise Exception(
                 f'Required information (baselines/{k}) missing from integrated test yaml file: {test_yaml}' )
@@ -136,14 +164,18 @@ def manage_baselines( options ):
             upload_baselines( baseline_options[ 'bucket' ], upload_name, options.baselineDir )
 
             # Update the test config file
-            baseline_options[ 'latest' ] = upload_name
+            baseline_options[ 'baseline' ] = upload_name
             with open( test_yaml, 'w' ) as f:
                 yaml.dump( baseline_options, f )
             quit()
         else:
             raise Exception( f'Could not find the requested baselines to upload: {options.baselineDir}' )
 
-    download_baselines( baseline_options[ 'bucket' ], baseline_options[ 'latest' ], options.baselineDir )
+    download_baselines( baseline_options[ 'bucket' ],
+                        baseline_options[ 'baseline' ],
+                        options.baselineDir,
+                        force_redownload=options.update_baselines,
+                        ok_delete_old_baselines=options.delete_old_baselines )
 
     # Cleanup
     if not os.path.isdir( options.baselineDir ):

geos_ats_package/geos_ats/command_line_parsers.py

@@ -1,8 +1,5 @@
-import logging
 import argparse
 import os
-import shutil
-from pydoc import locate
 
 action_options = {
     "run": "execute the test cases that previously did not pass.",
@@ -48,7 +45,19 @@ def build_command_line_parser():
 
     parser.add_argument( "-b", "--baselineDir", type=str, help="Root baseline directory" )
 
-    parser.add_argument( "-y", "--testYAML", type=str, help="Baseline bucket name", default='' )
+    parser.add_argument( "-y", "--yaml", type=str, help="Path to YAML config file", default='' )
+
+    parser.add_argument( "-d",
+                         "--delete-old-baselines",
+                         action="store_true",
+                         default=False,
+                         help="Automatically delete old baselines" )
+
+    parser.add_argument( "-u",
+                         "--update-baselines",
+                         action="store_true",
+                         default=False,
+                         help="Force baseline file update" )
 
     action_names = ','.join( action_options.keys() )
     parser.add_argument( "-a", "--action", type=str, default="run", help=f"Test actions options ({action_names})" )
@@ -63,14 +72,6 @@ def build_command_line_parser():
                          default="info",
                          help=f"Log verbosity options ({verbosity_names})" )
 
-    parser.add_argument( "-d",
-                         "--detail",
-                         action="store_true",
-                         default=False,
-                         help="Show detailed action/check options" )
-
-    parser.add_argument( "-i", "--info", action="store_true", default=False, help="Info on various topics" )
-
     parser.add_argument( "-r",
                          "--restartCheckOverrides",
                          nargs='+',
@@ -119,17 +120,20 @@ def parse_command_line_arguments( args ):
     # Check action, check, verbosity items
     check = options.check
     if check not in check_options:
-        print(
-            f"Selected check option ({check}) not recognized.  Try running with --help/--details for more information" )
+        print( f"Selected check option ({check}) not recognized" )
         exit_flag = True
 
     action = options.action
     if action not in action_options:
-        print(
-            f"Selected action option ({action}) not recognized.  Try running with --help/--details for more information"
-        )
+        print( f"Selected action option ({action}) not recognized" )
         exit_flag = True
 
+    if exit_flag:
+        for option_type, details in zip( [ 'action', 'check' ], [ action_options, check_options ] ):
+            print( f'\nAvailable {option_type} options:' )
+            for k, v in details.items():
+                print( f'    {k}:  {v}' )
+
     verbose = options.verbose
     if verbose not in verbose_options:
         print( f"Selected verbose option ({verbose}) not recognized" )
@@ -142,14 +146,6 @@ def parse_command_line_arguments( args ):
     if not options.baselineDir:
         options.baselineDir = options.workingDir
 
-    # Print detailed information
-    if options.detail:
-        for option_type, details in zip( [ 'action', 'check' ], [ action_options, check_options ] ):
-            print( f'\nAvailable {option_type} options:' )
-            for k, v in details.items():
-                print( f'    {k}:  {v}' )
-        exit_flag = True
-
     if exit_flag:
         quit()
 

geos_ats_package/setup.cfg

@@ -18,7 +18,7 @@ install_requires =
     lxml
     tabulate
     pyyaml
-    google-cloud
+    google-cloud-storage
     ats @ https://github.com/LLNL/ATS/archive/refs/tags/7.0.105.tar.gz
 python_requires = >=3.7