diff --git a/geos_ats_package/geos_ats/assets/__init__.py b/geos_ats_package/geos_ats/assets/__init__.py new file mode 100644 index 0000000..3c271e6 --- /dev/null +++ b/geos_ats_package/geos_ats/assets/__init__.py @@ -0,0 +1,24 @@ +import os +from pathlib import Path +import shutil + + +def create_assets_folder( target_dir ): + """ + Create an asset directory for html reports + + Args: + target_dir (str): Path to asset directory + """ + target_dir = os.path.abspath( os.path.expanduser( target_dir ) ) + if os.path.isdir( target_dir ): + return + + os.makedirs( target_dir ) + mod_path = os.path.dirname( os.path.abspath( Path( __file__ ).resolve() ) ) + for f in [ 'sorttable.js', 'style.css' ]: + shutil.copyfile( os.path.join( mod_path, f ), os.path.join( target_dir, f ) ) + + shutil.unpack_archive( os.path.join( mod_path, 'lightbox.zip' ), + os.path.join( target_dir, 'lightbox' ), + format='zip' ) diff --git a/geos_ats_package/geos_ats/assets/lightbox.zip b/geos_ats_package/geos_ats/assets/lightbox.zip new file mode 100644 index 0000000..651a928 Binary files /dev/null and b/geos_ats_package/geos_ats/assets/lightbox.zip differ diff --git a/geos_ats_package/geos_ats/assets/sorttable.js b/geos_ats_package/geos_ats/assets/sorttable.js new file mode 100644 index 0000000..38b0fc6 --- /dev/null +++ b/geos_ats_package/geos_ats/assets/sorttable.js @@ -0,0 +1,495 @@ +/* + SortTable + version 2 + 7th April 2007 + Stuart Langridge, http://www.kryogenix.org/code/browser/sorttable/ + + Instructions: + Download this file + Add to your HTML + Add class="sortable" to any table you'd like to make sortable + Click on the headers to sort + + Thanks to many, many people for contributions and suggestions. + Licenced as X11: http://www.kryogenix.org/code/browser/licence.html + This basically means: do what you want with it. +*/ + + +var stIsIE = /*@cc_on!@*/false; + +sorttable = { + init: function() { + // quit if this function has already been called + if (arguments.callee.done) return; + // flag this function so we don't do the same thing twice + arguments.callee.done = true; + // kill the timer + if (_timer) clearInterval(_timer); + + if (!document.createElement || !document.getElementsByTagName) return; + + sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/; + + forEach(document.getElementsByTagName('table'), function(table) { + if (table.className.search(/\bsortable\b/) != -1) { + sorttable.makeSortable(table); + } + }); + + }, + + makeSortable: function(table) { + if (table.getElementsByTagName('thead').length == 0) { + // table doesn't have a tHead. Since it should have, create one and + // put the first table row in it. + the = document.createElement('thead'); + the.appendChild(table.rows[0]); + table.insertBefore(the,table.firstChild); + } + // Safari doesn't support table.tHead, sigh + if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0]; + + if (table.tHead.rows.length != 1) return; // can't cope with two header rows + + // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as + // "total" rows, for example). This is B&R, since what you're supposed + // to do is put them in a tfoot. So, if there are sortbottom rows, + // for backwards compatibility, move them to tfoot (creating it if needed). + sortbottomrows = []; + for (var i=0; i5' : ' ▴'; + this.appendChild(sortrevind); + return; + } + if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) { + // if we're already sorted by this column in reverse, just + // re-reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted_reverse', + 'sorttable_sorted'); + this.removeChild(document.getElementById('sorttable_sortrevind')); + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + return; + } + + // remove sorttable_sorted classes + theadrow = this.parentNode; + forEach(theadrow.childNodes, function(cell) { + if (cell.nodeType == 1) { // an element + cell.className = cell.className.replace('sorttable_sorted_reverse',''); + cell.className = cell.className.replace('sorttable_sorted',''); + } + }); + sortfwdind = document.getElementById('sorttable_sortfwdind'); + if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); } + sortrevind = document.getElementById('sorttable_sortrevind'); + if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); } + + this.className += ' sorttable_sorted'; + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + + // build an array to sort. This is a Schwartzian transform thing, + // i.e., we "decorate" each row with the actual sort key, + // sort based on the sort keys, and then put the rows back in order + // which is a lot faster because you only do getInnerText once per row + row_array = []; + col = this.sorttable_columnindex; + rows = this.sorttable_tbody.rows; + for (var j=0; j 12) { + // definitely dd/mm + return sorttable.sort_ddmm; + } else if (second > 12) { + return sorttable.sort_mmdd; + } else { + // looks like a date, but we can't tell which, so assume + // that it's dd/mm (English imperialism!) and keep looking + sortfn = sorttable.sort_ddmm; + } + } + } + } + return sortfn; + }, + + getInnerText: function(node) { + // gets the text we want to use for sorting for a cell. + // strips leading and trailing whitespace. + // this is *not* a generic getInnerText function; it's special to sorttable. + // for example, you can override the cell text with a customkey attribute. + // it also gets .value for fields. + + if (!node) return ""; + + hasInputs = (typeof node.getElementsByTagName == 'function') && + node.getElementsByTagName('input').length; + + if (node.getAttribute("sorttable_customkey") != null) { + return node.getAttribute("sorttable_customkey"); + } + else if (typeof node.textContent != 'undefined' && !hasInputs) { + return node.textContent.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.innerText != 'undefined' && !hasInputs) { + return node.innerText.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.text != 'undefined' && !hasInputs) { + return node.text.replace(/^\s+|\s+$/g, ''); + } + else { + switch (node.nodeType) { + case 3: + if (node.nodeName.toLowerCase() == 'input') { + return node.value.replace(/^\s+|\s+$/g, ''); + } + case 4: + return node.nodeValue.replace(/^\s+|\s+$/g, ''); + break; + case 1: + case 11: + var innerText = ''; + for (var i = 0; i < node.childNodes.length; i++) { + innerText += sorttable.getInnerText(node.childNodes[i]); + } + return innerText.replace(/^\s+|\s+$/g, ''); + break; + default: + return ''; + } + } + }, + + reverse: function(tbody) { + // reverse the rows in a tbody + newrows = []; + for (var i=0; i=0; i--) { + tbody.appendChild(newrows[i]); + } + delete newrows; + }, + + /* sort functions + each sort function takes two parameters, a and b + you are comparing a[0] and b[0] */ + sort_numeric: function(a,b) { + aa = parseFloat(a[0].replace(/[^0-9.-]/g,'')); + if (isNaN(aa)) aa = 0; + bb = parseFloat(b[0].replace(/[^0-9.-]/g,'')); + if (isNaN(bb)) bb = 0; + return aa-bb; + }, + sort_alpha: function(a,b) { + if (a[0]==b[0]) return 0; + if (a[0] 0 ) { + var q = list[i]; list[i] = list[i+1]; list[i+1] = q; + swap = true; + } + } // for + t--; + + if (!swap) break; + + for(var i = t; i > b; --i) { + if ( comp_func(list[i], list[i-1]) < 0 ) { + var q = list[i]; list[i] = list[i-1]; list[i-1] = q; + swap = true; + } + } // for + b++; + + } // while(swap) + } +} + +/* ****************************************************************** + Supporting functions: bundled here to avoid depending on a library + ****************************************************************** */ + +// Dean Edwards/Matthias Miller/John Resig + +/* for Mozilla/Opera9 */ +if (document.addEventListener) { + document.addEventListener("DOMContentLoaded", sorttable.init, false); +} + +/* for Internet Explorer */ +/*@cc_on @*/ +/*@if (@_win32) + document.write(" + + + + +


+

Configuration

""" # Notations: @@ -245,7 +208,6 @@ def writeHeader( self, sp, refresh ): else: username = os.getenv( "USER" ) - header += "

GEOS ATS Report

\n

Configuration

\n" table = [ [ 'Test Results', gentime ], [ 'User', username ], [ 'Platform', platform ] ] header += tabulate( table, tablefmt='html' ) header += '\n' @@ -264,33 +226,67 @@ def writeSummary( self, sp ): sp.write( "\n\n

Summary

\n\n" ) table_html = tabulate( table, headers=header, tablefmt='unsafehtml' ) + table_html = table_html.replace( '', f'
' ) sp.write( table_html ) def writeTable( self, sp ): - header = ( "Status", "Name", "TestStep", "Elapsed", "Resources", "Output" ) + header = ( "Status", "Name", "TestStep", "Elapsed", "Resources", "Logs", "Output" ) table = [] table_filt = [] - file_pattern = "{}" + file_pattern = "{}" + image_pattern = "{}" color_pattern = "

{}

" for k, v in self.test_results.items(): status_str = v.status.name status_formatted = color_pattern.format( COLORS[ status_str ], k, status_str ) - step_shortname = v.current_step + step_shortname = v.current_step[ v.current_step.rfind( '_' ) + 1:-1 ] elapsed_formatted = hms( v.elapsed ) + + # Collect files to include in the table output_files = [] for s in v.steps.values(): - if os.path.isfile( s.log ): - output_files.append( file_pattern.format( s.log, os.path.basename( s.log ) ) ) - if os.path.isfile( s.log + '.err' ): - output_files.append( file_pattern.format( s.log + '.err', os.path.basename( s.log + '.err' ) ) ) + for f in [ s.log, s.log + '.err' ]: + if os.path.isfile( f ): + output_files.append( f ) for pattern in s.output: for f in sorted( glob.glob( pattern ) ): - if ( ( 'restart' not in f ) or ( '.restartcheck' in f ) ) and os.path.isfile( f ): - output_files.append( file_pattern.format( f, os.path.basename( f ) ) ) - - row = [ status_formatted, k, step_shortname, elapsed_formatted, v.resources, ', '.join( output_files ) ] + if ( 'restart' not in f ): + output_files.append( f ) + + # Copy files and build links + if output_files: + os.makedirs( os.path.join( self.html_data, v.path ), exist_ok=True ) + + log_links = [] + other_links = [] + for f in output_files: + base_fname = os.path.basename( f ) + copy_fname = os.path.join( self.html_data, v.path, base_fname ) + link_fname = os.path.join( '.', self.html_data_name, v.path, base_fname ) + output_fname = base_fname + if ( '.log' in output_fname ): + log_index = base_fname[ :base_fname.find( '.' ) ] + log_type = ''.join( base_fname.split( '_' )[ -2: ] ) + output_fname = f'{log_index}_{log_type}' + + shutil.copyfile( f, copy_fname ) + if os.stat( f ).st_size: + if '.log' in output_fname: + log_links.append( file_pattern.format( link_fname, output_fname ) ) + elif '.png' in output_fname: + image_caption = os.path.join( k, output_fname[ :-4 ] ) + other_links.append( image_pattern.format( link_fname, image_caption, output_fname ) ) + else: + other_links.append( file_pattern.format( link_fname, output_fname ) ) + + # Write row + row = [ + status_formatted, + k.replace( '_', ' ' ), step_shortname, elapsed_formatted, v.resources, ', '.join( log_links ), + ', '.join( other_links ) + ] if status_str == 'FILTERED': table_filt.append( row ) else: @@ -299,11 +295,13 @@ def writeTable( self, sp ): if len( table ): sp.write( "\n\n

Active Tests

\n\n" ) table_html = tabulate( table, headers=header, tablefmt='unsafehtml' ) + table_html = table_html.replace( '
', '
' ) sp.write( table_html ) if len( table_filt ): sp.write( "\n\n

Filtered Tests

\n\n" ) table_html = tabulate( table_filt, headers=header, tablefmt='unsafehtml' ) + table_html = table_html.replace( '
', '
' ) sp.write( table_html ) def writeFooter( self, sp ): diff --git a/geos_ats_package/geos_ats/test_case.py b/geos_ats_package/geos_ats/test_case.py index 9384013..8bbc1d5 100644 --- a/geos_ats_package/geos_ats/test_case.py +++ b/geos_ats_package/geos_ats/test_case.py @@ -320,7 +320,7 @@ def testCreate( self ): # Setup a new test group atsTest = None - ats.tests.AtsTest.newGroup( priority=priority ) + ats.tests.AtsTest.newGroup( priority=priority, path=self.path ) for stepnum, step in enumerate( self.steps ): np = getattr( step.p, "np", 1 ) ngpu = getattr( step.p, "ngpu", 0 ) diff --git a/geos_ats_package/geos_ats/test_steps.py b/geos_ats_package/geos_ats/test_steps.py index 54f7a2f..7eef286 100644 --- a/geos_ats_package/geos_ats/test_steps.py +++ b/geos_ats_package/geos_ats/test_steps.py @@ -7,7 +7,7 @@ import subprocess import re import logging -from geos_ats import common_utilities +from geos_ats import common_utilities, history from geos_ats.common_utilities import Error, Log from geos_ats.configuration_record import config @@ -509,6 +509,9 @@ def resultPaths( self ): def clean( self ): self._clean( self.resultPaths() ) + def rebaseline( self ): + history.write_baseline_log( os.path.join( self.p.baseline_directory, '.baseline_info' ) ) + ################################################################################ # restartcheck diff --git a/geos_ats_package/setup.cfg b/geos_ats_package/setup.cfg index 2d64f43..6ee5dde 100644 --- a/geos_ats_package/setup.cfg +++ b/geos_ats_package/setup.cfg @@ -9,6 +9,7 @@ license = LGPL-2.1 [options] packages = geos_ats + geos_ats.assets geos_ats.helpers geos_ats.machines install_requires = @@ -17,10 +18,22 @@ install_requires = numpy lxml tabulate + pyyaml + tqdm + requests + GitPython + google-cloud-storage + pip-system-certs ats @ https://github.com/LLNL/ATS/archive/refs/tags/7.0.105.tar.gz python_requires = >=3.7 +[options.package_data] +* = *.js, *.css, *.zip + [options.entry_points] console_scripts = run_geos_ats = geos_ats.main:main setup_ats_environment = geos_ats.environment_setup:main + geos_ats_log_check = geos_ats.helpers.log_check:main + geos_ats_restart_check = geos_ats.helpers.restart_check:main + geos_ats_curve_check = geos_ats.helpers.curve_check:main