diff --git a/docs/geos_ats.rst b/docs/geos_ats.rst index 6928daf..dfbbf85 100644 --- a/docs/geos_ats.rst +++ b/docs/geos_ats.rst @@ -26,6 +26,18 @@ Primary entry point for running integrated tests. Other machine-specific options for ATS can be viewed by running `run_geos_ats --ats help` +Debugging +------------------ + +If for any reason you need to debug the geos_ats package, we recommend that you create a local copy of this entry point in the `build/integratedTests` directory: `geosPythonPackages/geos_ats_package/geos_ats/debug_geos_ats.py`. +This script is designed as a debugger entry point, and will read the autogenerated run script that was built during setup. +To use it, you must either have geos_ats installed in your target python environment, or a copy of geosPythonPackages in the expected location (`/usr/workspace/[username]/geosPythonPackages`). + +We recommend that you use VSCode with the Python extension to debug geos_ats. +To begin the debugging session, you simply need to load the entry script, set any initial breakpoints you desire, then select the Debug run option. +Note that this approach can only be used to debug the python code associated with tests, and not the underlying GEOS tests. + + API ------ diff --git a/docs/index.rst b/docs/index.rst index 90e230d..d72849e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,30 +8,65 @@ Python Tools Python Tools Setup --------------------------------- -The preferred method to setup the GEOSX python tools is to run the following command in the build directory: +The preferred method to setup the GEOS python tools is to run the following command in the build directory: .. code-block:: bash make geosx_python_tools -This will attempt to install the required packages into the python distribution indicated via the `Python3_EXECUTABLE` cmake variable (also used by pygeosx). +The ats setup command also sets up the python tools: -If the user does not have write access for the target python distribution, the installation will attempt to create a new virtual python environment (Note: this requires that the virtualenv package be installed). +.. code-block:: bash + + make ats_environment + + +These will attempt to install the required packages into the python distribution indicated via the `Python3_EXECUTABLE` cmake variable (also used by pygeosx). If any package dependencies are missing, then the install script will attempt to fetch them from the internet using pip. -After installation, these packages will be available for import within the associated python distribution, and a set of console scripts will be available within the GEOSX build bin directory. +After installation, these packages will be available for import within the associated python distribution, and a set of console scripts will be available within the GEOS build bin directory. + + +.. note:: + To re-install or update an installed version of geosPythonTools, you can run the `make geosx_python_tools_clean` and `make geosx_python_tools` commands. + + +Manual Installation +--------------------------------- + +In some cases, you may need to manually install or update geosPythonPackages. +To do this, you can clone a copy of the geosPythonPackages repository and install them using pip: -Alternatively, these packages can be installed manually into a python environment using pip: .. code-block:: bash - cd GEOSX/src/coreComponents/python/modules/geosx_mesh_tools_package - pip install --upgrade . + cd /path/to/store/python/tools + git clone https://github.com/GEOS-DEV/geosPythonPackages.git + + # Install/upgrade geos_ats + cd geosPythonPackages/geos_ats_package + python -m pip install --upgrade . + + +.. note:: + To upgrade an existing installation, the python executable in the above command should correspond to the version you indicated in your host config. If you have previously built the tools, this version will be linked in the build directory: `build_dir/bin/python`. + + +Development & Debugging +--------------------------- + +Be default, the python environment setup commands target the "main" branch of geosPythonTools. +To target another version of the tools, you can set the `GEOS_PYTHON_PACKAGES_BRANCH` cmake variable to the name of another valid branch (or git tag) in the host config file. +In this case, the code will pull the most recent commit of the desired branch when building geosPythonTools. + + +.. note:: + If you are working on significant updates to geosPythonTools, you should open a testing branch in the main GEOS repository that defines the `GEOS_PYTHON_PACKAGES_BRANCH` variable. This will ensure that your changes are tested as part of the GEOS CI. + - cd ../geosx_xml_tools_package - pip install --upgrade . +If you need to debug one of the packages in geosPythonTools, we recommend using VSCode with the Python extension installed. +Some of the packages contain specific entry point scripts that can be used to assist in this process. - # Etc. Packages diff --git a/geos_ats_package/geos_ats/debug_geos_ats.py b/geos_ats_package/geos_ats/debug_geos_ats.py new file mode 100644 index 0000000..5bedcb8 --- /dev/null +++ b/geos_ats_package/geos_ats/debug_geos_ats.py @@ -0,0 +1,37 @@ +""" +Entry point for debugging geos_ats + +To use this script, do the following: +- Setup your ats environment (using the 'make ats_environment' command) +- Create a copy of this script in the build/integratedTests directory +- Debug this script with a tool like vscode + +Note: if you have a copy of the geosPythonPackages repository located +in your user workspace, then this script will attempt to use that version +of geos_ats instead of the one installed in your environment. +""" + +import os +import sys + + +def debug_geos_ats(): + # Check for a copy of geos_ats in the user's workspace to use + # instead of any currently installed in python + user = os.environ.get( 'USER', '' ) + mod_path = f"/usr/workspace/{user}/geosPythonPackages/geos_ats_package" + if os.path.isdir( mod_path ): + sys.path.insert( 0, os.path.join( mod_path ) ) + + # Collect command line args from autogenerated script + fname = os.path.join( os.path.dirname( __file__ ), 'geos_ats.sh' ) + args = open( fname, 'r' ).readlines()[ 1 ].split()[ 1:-1 ] + sys.argv.extend( args ) + + # Run ats + from geos_ats import main + main.main() + + +if __name__ == '__main__': + debug_geos_ats() diff --git a/geos_ats_package/geos_ats/geos_ats_debug.py b/geos_ats_package/geos_ats/geos_ats_debug.py deleted file mode 100644 index 25f368c..0000000 --- a/geos_ats_package/geos_ats/geos_ats_debug.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -import os -import glob -from pathlib import Path - -mod_path = Path( __file__ ).resolve().parents[ 1 ] -sys.path.append( os.path.abspath( mod_path ) ) -from geos_ats import main - - -def debug_geosats( build_root='~/GEOS/build-quartz-gcc@12-release', extra_args=[] ): - # Search for and parse the ats script - build_root = os.path.expanduser( build_root ) - ats_script = os.path.join( build_root, 'integratedTests', 'geos_ats.sh' ) - if not os.path.isfile( ats_script ): - raise InputError( - 'Could not find geos_ats.sh at the expected location... Make sure to run \"make ats_environment\"' ) - - with open( ats_script, 'r' ) as f: - header = f.readline() - ats_args = f.readline().split() - sys.argv.extend( ats_args[ 1:-1 ] ) - sys.argv.extend( extra_args ) - - main.main() - - -if ( __name__ == '__main__' ): - # debug_geosats(extra_args=['-a', 'veryclean']) - # debug_geosats(extra_args=['-a', 'rebaselinefailed']) - debug_geosats() diff --git a/geos_ats_package/geos_ats/helpers/restart_check.py b/geos_ats_package/geos_ats/helpers/restart_check.py index 6172045..ad7c8d5 100644 --- a/geos_ats_package/geos_ats/helpers/restart_check.py +++ b/geos_ats_package/geos_ats/helpers/restart_check.py @@ -6,6 +6,7 @@ import re import argparse import logging +import time from pathlib import Path try: from geos_ats.helpers.permute_array import permuteArray # type: ignore[import] @@ -31,6 +32,36 @@ def write( output, msg ): output.write( msg ) +def is_lfs_pointer( fname ): + res = False + try: + header = str( open( fname, 'rb' ).read( 16 ) ) + if 'Git LFS pointer' in header: + res = True + except Exception: + pass + + return res + + +def load_hdf5( fname, max_wait_time=10, mode='r' ): + file = None + for ii in range( max_wait_time ): + if os.path.isfile( fname ): + try: + file = h5py.File( fname, mode ) + logger.debug( f'Opened file: {fname}' ) + break + except IOError: + logger.warning( f'Failed to open file: {fname} (attempt {ii+1}/{max_wait_time})' ) + if is_lfs_pointer( fname ): + raise Exception( f'Target LFS object is not initialized: {fname}' ) + + time.sleep( 1 ) + + return file + + def h5PathJoin( p1, p2 ): if p1 == "/": return "/" + p2 @@ -79,17 +110,23 @@ def __init__( self, assert ( self.atol >= 0.0 ) def filesDiffer( self ): - try: - with h5py.File( self.file_path, "r" ) as file, h5py.File( self.baseline_path, "r" ) as base_file: - self.file_path = file.filename - self.baseline_path = base_file.filename - self.output.write( "\nRank %s is comparing %s with %s \n" % - ( MPI.COMM_WORLD.Get_rank(), self.file_path, self.baseline_path ) ) - self.compareGroups( file, base_file ) - - except IOError as e: - self.logger.debug( e ) - self.output.write( str( e ) ) + # Check to see if the file is on the disk, and wait in case there is any lag in IO + file = load_hdf5( self.file_path ) + base_file = load_hdf5( self.baseline_path ) + rank = MPI.COMM_WORLD.Get_rank() + self.output.write( f"\nRank {rank} is comparing {self.file_path} with {self.baseline_path} \n" ) + + # Compare the files + if ( file is not None ) and ( base_file is not None ): + self.file_path = file.filename + self.baseline_path = base_file.filename + self.compareGroups( file, base_file ) + + else: + if file is None: + self.output.write( f"\nRank {rank} failed to load target file: {self.file_path}\n" ) + if base_file is None: + self.output.write( f"\nRank {rank} failed to load baseline file: {self.baseline_path}\n" ) self.different = True return self.different diff --git a/geos_ats_package/geos_ats/test_case.py b/geos_ats_package/geos_ats/test_case.py index fbeeb03..9384013 100644 --- a/geos_ats_package/geos_ats/test_case.py +++ b/geos_ats_package/geos_ats/test_case.py @@ -388,10 +388,9 @@ def testRebaseline( self ): if config.rebaseline_ask: while 1: if config.rebaseline_undo: - logger.info( f"Are you sure you want to undo the rebaseline for TestCase '{self.name}'?", - flush=True ) + logger.info( f"Are you sure you want to undo the rebaseline for TestCase '{self.name}'?" ) else: - logger.info( f"Are you sure you want to rebaseline TestCase '{self.name}'?", flush=True ) + logger.info( f"Are you sure you want to rebaseline TestCase '{self.name}'?" ) x = input( '[y/n] ' ) x = x.strip()