From 267410c673002ead58937467853e4495768f4176 Mon Sep 17 00:00:00 2001 From: Kyle Mandli Date: Tue, 9 Jul 2019 21:06:10 -0600 Subject: [PATCH 1/5] Fixed data.py issue but runclaw needs help --- src/python/clawutil/data.py | 23 ++++++++++++++--------- src/python/clawutil/runclaw.py | 5 ++++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/python/clawutil/data.py b/src/python/clawutil/data.py index 276a0df..c3be9c4 100644 --- a/src/python/clawutil/data.py +++ b/src/python/clawutil/data.py @@ -14,7 +14,14 @@ from __future__ import absolute_import from __future__ import print_function import os -from inspect import signature +import sys +import inspect +import tarfile +import zipfile +import string +import six +from six.moves import range +from six.moves import input try: from urllib.request import urlopen @@ -22,14 +29,7 @@ # Fall back to Python 2's urllib2 from urllib2 import urlopen -import tarfile -import zipfile -import string - import numpy as np -import six -from six.moves import range -from six.moves import input # ====================== # Remote file handling @@ -528,7 +528,12 @@ def write(self, out_dir = ''): if isinstance(data_object, UserData): fname = data_object.__fname__ else: - fname = signature(data_object.write).parameters['out_file'].default + if six.PY2: + argspec = inspect.getargspec(data_object.write) + fname = argspec.defaults[0] + else: + argspec = inspect.signature(data_object.write) + fname = argspec.parameters['out_file'].default fpath = os.path.join(out_dir,fname) if isinstance(data_object, amrclaw.GaugeData): data_object.write(self.clawdata.num_eqn, self.clawdata.num_aux, out_file=fpath) diff --git a/src/python/clawutil/runclaw.py b/src/python/clawutil/runclaw.py index 8d8f4d3..6f1cfb0 100755 --- a/src/python/clawutil/runclaw.py +++ b/src/python/clawutil/runclaw.py @@ -145,7 +145,9 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, except: print("==> runclaw: Could not move directory... copy already exists?") - + ### + ### TODO: Modify to work with 2.* + ### os.makedirs(outdir, exist_ok=True) if print_git_status not in [False,'False']: @@ -218,6 +220,7 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, encoding='utf-8', check=True) except subprocess.CalledProcessError as cpe: + #### TODO: REWORK for 2.x raise ClawExeError('error', cpe.returncode, cpe.cmd, output=cpe.output, stderr=cpe.stderr) from cpe From 0260c6b4192977c74640e65d7915d71c600a7976 Mon Sep 17 00:00:00 2001 From: Kyle Mandli Date: Wed, 10 Jul 2019 17:14:09 -0600 Subject: [PATCH 2/5] Correct keyword order bug --- src/python/clawutil/data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python/clawutil/data.py b/src/python/clawutil/data.py index c3be9c4..4b37806 100644 --- a/src/python/clawutil/data.py +++ b/src/python/clawutil/data.py @@ -302,7 +302,7 @@ def close_data_file(self): self._out_file = None - def write(self,out_file,data_source='setrun.py'): + def write(self, out_file, data_source='setrun.py'): r"""Write out all data files in this ClawData object""" # Open data file @@ -530,7 +530,9 @@ def write(self, out_dir = ''): else: if six.PY2: argspec = inspect.getargspec(data_object.write) - fname = argspec.defaults[0] + index = argspec.args.index('out_file') - (len(argspec.args) + - len(argspec.defaults)) + fname = argspec.defaults[index] else: argspec = inspect.signature(data_object.write) fname = argspec.parameters['out_file'].default From e4c0181ada30ef705ce7948f44b3dc8c2524ce58 Mon Sep 17 00:00:00 2001 From: Kyle Mandli Date: Fri, 12 Jul 2019 17:00:20 -0600 Subject: [PATCH 3/5] Fixed subprocess call output --- src/python/clawutil/runclaw.py | 54 +++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/python/clawutil/runclaw.py b/src/python/clawutil/runclaw.py index 6f1cfb0..45c008f 100755 --- a/src/python/clawutil/runclaw.py +++ b/src/python/clawutil/runclaw.py @@ -10,12 +10,17 @@ from __future__ import absolute_import from __future__ import print_function +import six + import os import sys import glob import shutil import shlex import subprocess +import time + +from clawpack.clawutil.data import ClawData from clawpack.clawutil.claw_git_status import make_git_status_file # define an execution error class that returns a @@ -30,7 +35,7 @@ def __str__(self): def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, rundir=None, print_git_status=False, nohup=False, nice=None, - xclawout=subprocess.PIPE, xclawerr=subprocess.PIPE): + xclawout=None, xclawerr=None, verbose=True): """ Run the Fortran version of Clawpack using executable xclawcmd, which is typically set to 'xclaw', 'xamr', etc. @@ -66,14 +71,12 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, ``xclawerr=subprocess.STDOUT``. """ - from clawpack.clawutil.data import ClawData - import os,glob,shutil,time - verbose = True - try: - nice = int(nice) - except: - nice = None + if nice is not None: + try: + nice = int(nice) + except: + nice = None # convert strings passed in from Makefile to boolean: if type(overwrite) is str: @@ -145,10 +148,15 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, except: print("==> runclaw: Could not move directory... copy already exists?") - ### - ### TODO: Modify to work with 2.* - ### - os.makedirs(outdir, exist_ok=True) + if six.PY2: + if (not os.path.isdir(outdir)): + try: + os.mkdir(outdir) + except: + print("Cannot make directory ",outdir) + return + else: + os.makedirs(outdir, exist_ok=True) if print_git_status not in [False,'False']: # create files claw_git_status.txt and claw_git_diffs.txt in @@ -186,7 +194,7 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, else: if rundir != outdir: for file in datafiles: - shutil.copy(file,os.path.join(outdir,os.path.basename(file))) + shutil.copy(file, os.path.join(outdir,os.path.basename(file))) # execute command to run fortran program: if nohup: @@ -206,30 +214,28 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, print("\n==> Running with command:\n ", cmd) cmd_split = shlex.split(cmd) - if isinstance(xclawout,str): + if isinstance(xclawout, str): xclawout = open(xclawout,'w', encoding='utf-8', buffering=1) - if isinstance(xclawerr,str): + if isinstance(xclawerr, str): xclawerr = open(xclawerr,'w', encoding='utf-8', buffering=1) try: - p = subprocess.run(cmd_split, - cwd=outdir, - stdout=xclawout, - stderr=xclawerr, - encoding='utf-8', - check=True) + proc = subprocess.check_call(cmd_split, + cwd=outdir, + stdout=xclawout, + stderr=xclawerr) + except subprocess.CalledProcessError as cpe: - #### TODO: REWORK for 2.x raise ClawExeError('error', cpe.returncode, cpe.cmd, output=cpe.output, - stderr=cpe.stderr) from cpe + stderr=cpe.stderr) print('==> runclaw: Done executing %s via clawutil.runclaw.py' %\ xclawcmd) print('==> runclaw: Output is in ', outdir) - return p + return proc #---------------------------------------------------------- From 51345cfafed4dbd48397f2c77b6226cdb0695977 Mon Sep 17 00:00:00 2001 From: Kyle Mandli Date: Mon, 22 Jul 2019 18:57:27 -0600 Subject: [PATCH 4/5] Add warning that Fortran code failed --- src/python/clawutil/runclaw.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/python/clawutil/runclaw.py b/src/python/clawutil/runclaw.py index 45c008f..6dcb4ae 100755 --- a/src/python/clawutil/runclaw.py +++ b/src/python/clawutil/runclaw.py @@ -227,6 +227,9 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, stderr=xclawerr) except subprocess.CalledProcessError as cpe: + print("") + print("*** FORTRAN EXE FAILED ***") + print("") raise ClawExeError('error', cpe.returncode, cpe.cmd, output=cpe.output, stderr=cpe.stderr) From 6c1e0f072c59a66f7be03258f41c17543dc4b9b9 Mon Sep 17 00:00:00 2001 From: Kyle Mandli Date: Tue, 23 Jul 2019 10:08:29 -0600 Subject: [PATCH 5/5] Incorporate error message into exception --- src/python/clawutil/runclaw.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/python/clawutil/runclaw.py b/src/python/clawutil/runclaw.py index 6dcb4ae..38691dd 100755 --- a/src/python/clawutil/runclaw.py +++ b/src/python/clawutil/runclaw.py @@ -227,10 +227,8 @@ def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, stderr=xclawerr) except subprocess.CalledProcessError as cpe: - print("") - print("*** FORTRAN EXE FAILED ***") - print("") - raise ClawExeError('error', cpe.returncode, cpe.cmd, + exe_error_str = "\n\n*** FORTRAN EXE FAILED ***\n" + raise ClawExeError(exe_error_str, cpe.returncode, cpe.cmd, output=cpe.output, stderr=cpe.stderr)