diff --git a/src/python/clawutil/data.py b/src/python/clawutil/data.py index dcfd915..276a0df 100644 --- a/src/python/clawutil/data.py +++ b/src/python/clawutil/data.py @@ -14,6 +14,8 @@ from __future__ import absolute_import from __future__ import print_function import os +from inspect import signature + try: from urllib.request import urlopen except ImportError: @@ -458,7 +460,6 @@ def __init__(self, pkg, num_dim): import clawpack.amrclaw.data as amrclaw self.xclawcmd = 'xamr' - self.add_data(ClawInputData(num_dim),'clawdata') self.add_data(amrclaw.AmrclawInputData(self.clawdata),'amrdata') self.add_data(amrclaw.RegionData(num_dim=num_dim),'regiondata') self.add_data(amrclaw.GaugeData(num_dim=num_dim),'gaugedata') @@ -517,16 +518,22 @@ def new_UserData(self,name,fname): return data - def write(self): + def write(self, out_dir = ''): r"""Write out each data objects in datalist """ import clawpack.amrclaw.data as amrclaw for data_object in self.data_list: + # UserData doesn't naturally have an "out_file" parameter + if isinstance(data_object, UserData): + fname = data_object.__fname__ + else: + fname = signature(data_object.write).parameters['out_file'].default + fpath = os.path.join(out_dir,fname) if isinstance(data_object, amrclaw.GaugeData): - data_object.write(self.clawdata.num_eqn, self.clawdata.num_aux) + data_object.write(self.clawdata.num_eqn, self.clawdata.num_aux, out_file=fpath) else: - data_object.write() + data_object.write(out_file=fpath) @@ -841,6 +848,8 @@ def add_param(self,name,value,descr=''): descr_dict = self.__descr__ descr_dict[name] = descr - def write(self,data_source='setrun.py'): - super(UserData,self).write(self.__fname__, data_source) + def write(self, data_source='setrun.py', out_file=None): + if out_file is None: + out_file = self.__fname__ + super(UserData,self).write(out_file, data_source) self.close_data_file() diff --git a/src/python/clawutil/runclaw.py b/src/python/clawutil/runclaw.py index 3f55be3..8d8f4d3 100755 --- a/src/python/clawutil/runclaw.py +++ b/src/python/clawutil/runclaw.py @@ -14,10 +14,23 @@ import sys import glob import shutil +import shlex +import subprocess from clawpack.clawutil.claw_git_status import make_git_status_file +# define an execution error class that returns a +# message as well as the rest of the subprocess exceptions +class ClawExeError(subprocess.CalledProcessError): + def __init__(self, msg, *args, **kwargs): + super().__init__(*args, **kwargs) + self.msg = msg + + def __str__(self): + return self.msg + def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, - rundir=None, print_git_status=False, nohup=False, nice=None): + rundir=None, print_git_status=False, nohup=False, nice=None, + xclawout=subprocess.PIPE, xclawerr=subprocess.PIPE): """ Run the Fortran version of Clawpack using executable xclawcmd, which is typically set to 'xclaw', 'xamr', etc. @@ -45,15 +58,18 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, If type(nice) is int, runs the code using "nice -n " with this nice value so it doesn't hog computer resources. + + xclawout and xclawerr define the locations of stdout and stderr for the + execution of CLAW_EXE. They should be strings to filepaths or open file + objects or ``subprocess.PIPE`` or ``subprocess.STDOUT``. To pass both + to the same file, specify ``xclawout`` as the filepath and + ``xclawerr=subprocess.STDOUT``. """ - from clawpack.clawutil.data import ClawData import os,glob,shutil,time verbose = True - xclawout = None - xclawerr = None - + try: nice = int(nice) except: @@ -81,7 +97,7 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, if outdir is None: outdir = '.' - + if rundir is None: rundir = os.getcwd() rundir = os.path.abspath(rundir) @@ -96,169 +112,121 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, clawdata = ClawData() clawdata.read(os.path.join(rundir,'claw.data'), force=True) restart = clawdata.restart - #print '+++ From claw.data determined restart = %s' % restart - - - #returncode = clawjob.runxclaw() - - if 1: - startdir = os.getcwd() - xdir = os.path.abspath(startdir) - outdir = os.path.abspath(outdir) - rundir = os.path.abspath(rundir) - xclawcmd = os.path.join(xdir,xclawcmd) + xclawcmd = os.path.abspath(xclawcmd) + + if os.path.isfile(outdir): + print("==> runclaw: Error: outdir specified is a file") + return + + if (os.path.isdir(outdir) & (not overwrite)): + # copy the old outdir before possibly overwriting + tm = time.localtime(os.path.getmtime(outdir)) + year = str(tm[0]).zfill(4) + month = str(tm[1]).zfill(2) + day = str(tm[2]).zfill(2) + hour = str(tm[3]).zfill(2) + minute = str(tm[4]).zfill(2) + second = str(tm[5]).zfill(2) + outdir_backup = outdir + '_%s-%s-%s-%s%s%s' \ + % (year,month,day,hour,minute,second) + if verbose: + print("==> runclaw: Directory already exists: ",os.path.split(outdir)[1]) + if restart: + print("==> runclaw: Copying directory to: ",os.path.split(outdir_backup)[1]) + else: + print("==> runclaw: Moving directory to: ",os.path.split(outdir_backup)[1]) + time.sleep(1) + try: - os.chdir(xdir) + shutil.move(outdir,outdir_backup) + if restart: + shutil.copytree(outdir_backup,outdir) except: - raise Exception( "==> runclaw: Cannot change to directory xdir = %s" %xdir) - return - - - if os.path.isfile(outdir): - print("==> runclaw: Error: outdir specified is a file") - return - - if (os.path.isdir(outdir) & (not overwrite)): - # copy the old outdir before possibly overwriting - tm = time.localtime(os.path.getmtime(outdir)) - year = str(tm[0]).zfill(4) - month = str(tm[1]).zfill(2) - day = str(tm[2]).zfill(2) - hour = str(tm[3]).zfill(2) - minute = str(tm[4]).zfill(2) - second = str(tm[5]).zfill(2) - outdir_backup = outdir + '_%s-%s-%s-%s%s%s' \ - % (year,month,day,hour,minute,second) - if verbose: - print("==> runclaw: Directory already exists: ",os.path.split(outdir)[1]) - if restart: - print("==> runclaw: Copying directory to: ",os.path.split(outdir_backup)[1]) - else: - print("==> runclaw: Moving directory to: ",os.path.split(outdir_backup)[1]) - time.sleep(1) - - try: - shutil.move(outdir,outdir_backup) - if restart: - shutil.copytree(outdir_backup,outdir) - except: - print("==> runclaw: Could not move directory... copy already exists?") - - - if (not os.path.isdir(outdir)): - try: - os.mkdir(outdir) - except: - print("Cannot make directory ",outdir) - return + print("==> runclaw: Could not move directory... copy already exists?") + - try: - os.chdir(outdir) - except: - print('==> runclaw: *** Error in runxclaw: cannot move to outdir = ',\ - outdir) - raise + os.makedirs(outdir, exist_ok=True) + + if print_git_status not in [False,'False']: + # create files claw_git_status.txt and claw_git_diffs.txt in + # outdir: + make_git_status_file(outdir=outdir) + + # old fort.* files to be removed for new run? + fortfiles = glob.glob(os.path.join(outdir,'fort.*')) + # also need to remove gauge*.txt output files now that the gauge + # output is no longer in fort.gauge (but don't remove new gauges.data) + gaugefiles = glob.glob(os.path.join(outdir,'gauge*.txt')) + + if (overwrite and (not restart)): + # remove any old versions: + if verbose: + print("==> runclaw: Removing all old fort/gauge files in ", outdir) + for file in fortfiles + gaugefiles: + os.remove(file) + elif restart: + if verbose: + print("==> runclaw: Restart: leaving original fort/gauge files in ", outdir) + else: + # this should never be reached: + # if overwrite==False then outdir has already been moved + if len(fortfiles+gaugefiles) > 1: + print("==> runclaw: *** Remove fort.* and gauge*.txt") + print(" from output directory %s and try again," % outdir) + print(" or use overwrite=True in call to runclaw") + print(" e.g., by setting OVERWRITE = True in Makefile") return - - if print_git_status not in [False,'False']: - # create files claw_git_status.txt and claw_git_diffs.txt in - # outdir: - make_git_status_file() - - # old fort.* files to be removed for new run? - fortfiles = glob.glob(os.path.join(outdir,'fort.*')) - # also need to remove gauge*.txt output files now that the gauge - # output is no longer in fort.gauge (but don't remove new gauges.data) - gaugefiles = glob.glob(os.path.join(outdir,'gauge*.txt')) - - if (overwrite and (not restart)): - # remove any old versions: - if verbose: - print("==> runclaw: Removing all old fort/gauge files in ", outdir) - for file in fortfiles + gaugefiles: - os.remove(file) - elif restart: - if verbose: - print("==> runclaw: Restart: leaving original fort/gauge files in ", outdir) + + datafiles = glob.glob(os.path.join(rundir,'*.data')) + if datafiles == (): + print("==> runclaw: Warning: no data files found in directory ",rundir) + else: + if rundir != outdir: + for file in datafiles: + shutil.copy(file,os.path.join(outdir,os.path.basename(file))) + + # execute command to run fortran program: + if nohup: + # run in nohup mode: + print("\n==> Running in nohup mode, output will be sent to:") + print(" %s/nohup.out" % outdir) + if type(nice) is int: + cmd = "nohup time nice -n %s %s " % (nice,xclawcmd) else: - # this should never be reached: - # if overwrite==False then outdir has already been moved - if len(fortfiles+gaugefiles) > 1: - print("==> runclaw: *** Remove fort.* and gauge*.txt") - print(" from output directory %s and try again," % outdir) - print(" or use overwrite=True in call to runclaw") - print(" e.g., by setting OVERWRITE = True in Makefile") - return - - - try: - os.chdir(rundir) - except: - raise Exception("Cannot change to directory %s" % rundir) - return - - datafiles = glob.glob('*.data') - if datafiles == (): - print("==> runclaw: Warning: no data files found in directory ",rundir) + cmd = "nohup time %s " % xclawcmd + print("\n==> Running with command:\n ", cmd) + else: + if type(nice) is int: + cmd = "nice -n %s %s " % (nice,xclawcmd) else: - if rundir != outdir: - for file in datafiles: - shutil.copy(file,os.path.join(outdir,file)) - - if xclawout: - xclawout = open(xclawout,'wb') - if xclawerr: - xclawerr = open(xclawerr,'wb') - - os.chdir(outdir) - - #print "\nIn directory outdir = ",outdir,"\n" - - # execute command to run fortran program: - - try: - #print "\nExecuting ",xclawcmd, " ... " - #pclaw = subprocess.Popen(xclawcmd,stdout=xclawout,stderr=xclawerr) - #print '+++ pclaw started' - #pclaw.wait() # wait for code to run - #returncode = pclaw.returncode - #print '+++ pclaw done' - - if nohup: - # run in nohup mode: - print("\n==> Running in nohup mode, output will be sent to:") - print(" %s/nohup.out" % outdir) - if type(nice) is int: - cmd = "nohup time nice -n %s %s " % (nice,xclawcmd) - else: - cmd = "nohup time %s " % xclawcmd - print("\n==> Running with command:\n ", cmd) - returncode = os.system(cmd) - else: - if type(nice) is int: - cmd = "nice -n %s %s " % (nice,xclawcmd) - else: - cmd = xclawcmd - print("\n==> Running with command:\n ", cmd) - returncode = os.system(cmd) - - if returncode == 0: - print("\n==> runclaw: Finished executing\n") - else: - print("\n ==> runclaw: *** Runtime error: return code = %s\n "\ - % returncode) - print('==> runclaw: Done executing %s via clawutil.runclaw.py' %\ - xclawcmd) - print('==> runclaw: Output is in ', outdir) - - except: - raise Exception("Could not execute command %s" % xclawcmd) + cmd = xclawcmd + print("\n==> Running with command:\n ", cmd) + + cmd_split = shlex.split(cmd) + if isinstance(xclawout,str): + xclawout = open(xclawout,'w', encoding='utf-8', + buffering=1) + if isinstance(xclawerr,str): + xclawerr = open(xclawerr,'w', encoding='utf-8', + buffering=1) + try: + p = subprocess.run(cmd_split, + cwd=outdir, + stdout=xclawout, + stderr=xclawerr, + encoding='utf-8', + check=True) + except subprocess.CalledProcessError as cpe: + raise ClawExeError('error', cpe.returncode, cpe.cmd, + output=cpe.output, + stderr=cpe.stderr) from cpe - os.chdir(startdir) + print('==> runclaw: Done executing %s via clawutil.runclaw.py' %\ + xclawcmd) + print('==> runclaw: Output is in ', outdir) - if returncode != 0: - print('==> runclaw: *** fortran returncode = ', returncode, ' aborting') + return p #---------------------------------------------------------- diff --git a/src/python/clawutil/test.py b/src/python/clawutil/test.py index bddde4a..45e2eef 100644 --- a/src/python/clawutil/test.py +++ b/src/python/clawutil/test.py @@ -25,6 +25,7 @@ import clawpack.pyclaw.gauges as gauges import clawpack.pyclaw.solution as solution import clawpack.clawutil.claw_git_status as claw_git_status +from clawpack.clawutil import runclaw # Support for WIP decorator from functools import wraps @@ -214,28 +215,20 @@ def write_rundata_objects(self, path=None): if path is None: path = self.temp_path - - orig_path = os.getcwd() - os.chdir(path) - self.rundata.write() - os.chdir(orig_path) + self.rundata.write(out_dir=path) def run_code(self): r"""Run test code given an already compiled executable""" - runclaw_cmd = " ".join(( - "cd %s ;" % self.temp_path, - "python", - "$CLAW/clawutil/src/python/clawutil/runclaw.py", - self.executable_name, - self.temp_path, - "True", - "False", - self.temp_path)) - subprocess.check_call(runclaw_cmd, stdout=self.stdout, - stderr=self.stderr, - shell=True) + runclaw.runclaw(xclawcmd=os.path.join(self.temp_path,self.executable_name), + rundir=self.temp_path, + outdir=self.temp_path, + overwrite=True, + restart=False, + xclawout=self.stdout, + xclawerr=self.stderr) + self.stdout.flush() self.stderr.flush() @@ -260,7 +253,7 @@ def runTest(self, save=False, indices=(2, 3)): # Run code self.run_code() - + # Perform tests # Override this class to perform data checks, as is this class will # simply check that a test runs to completion