diff --git a/.gitignore b/.gitignore index b11afb7abb..5e6be3f819 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ venv* v3nv/ tmp/ /src/toil/test/cwl/spec* +/src/toil/test/wdl/wdl-conformance-tests /cwltool_deps/ /docs/generated_rst/ /docker/Dockerfile diff --git a/docs/conf.py b/docs/conf.py index a0b402ebf7..a26d25a07a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -160,7 +160,8 @@ def setup(app): autoapi_dirs = ["../src/toil"] autodoc_typehints = "description" autoapi_keep_files = True -autoapi_ignore = ["*.pyi", "*/test/cwl/spec*/*.py", "*/fake_mpi_run.py", "*/tutorial_*.py", "*/example_*.py", "*/mkFile.py", "*/debugWorkflow.py"] +autoapi_ignore = ["*.pyi", "*/test/cwl/spec*/*.py", "*/fake_mpi_run.py", "*/tutorial_*.py", "*/example_*.py", "*/mkFile.py", "*/debugWorkflow.py", + "*/test/wdl/wdl-conformance-tests/*"] autoapi_options = [ "members", "undoc-members", diff --git a/src/toil/job.py b/src/toil/job.py index 91ca5757d5..efa173db2a 100644 --- a/src/toil/job.py +++ b/src/toil/job.py @@ -40,21 +40,18 @@ NamedTuple, Optional, Sequence, - Set, Tuple, TypeVar, Union, cast, - overload, TypedDict, Literal, + overload, + TypedDict, + Literal, ) from urllib.error import HTTPError from urllib.parse import urlsplit, unquote, urljoin -from configargparse import ArgParser - from toil import memoize -from toil.bus import Names -from toil.lib.compatibility import deprecated import dill from configargparse import ArgParser @@ -81,11 +78,13 @@ from optparse import OptionParser from toil.batchSystems.abstractBatchSystem import ( - BatchJobExitReason, - InsufficientSystemResources, + BatchJobExitReason ) from toil.fileStores.abstractFileStore import AbstractFileStore - from toil.jobStores.abstractJobStore import AbstractJobStore, UnimplementedURLException + from toil.jobStores.abstractJobStore import ( + AbstractJobStore, + UnimplementedURLException, + ) logger = logging.getLogger(__name__) @@ -3793,7 +3792,7 @@ class FileMetadata(NamedTuple): def potential_absolute_uris( uri: str, path: list[str], - importer: WDL.Tree.Document | None = None, + importer: str | None = None, execution_dir: str | None = None, ) -> Iterator[str]: """ @@ -3834,7 +3833,7 @@ def potential_absolute_uris( if importer is not None: # Add the place the imported file came form, to search first. - full_path_list.append(Toil.normalize_uri(importer.pos.abspath)) + full_path_list.append(Toil.normalize_uri(importer)) # Then the current directory. We need to make sure to include a filename component here or it will treat the current directory with no trailing / as a document and relative paths will look 1 level up. # When importing on a worker, the cwd will be a tmpdir and will result in FileNotFoundError after os.path.abspath, so override with the execution dir diff --git a/src/toil/wdl/wdltoil.py b/src/toil/wdl/wdltoil.py index fcf2d4dacb..abd1c0ce45 100755 --- a/src/toil/wdl/wdltoil.py +++ b/src/toil/wdl/wdltoil.py @@ -506,7 +506,7 @@ async def toil_read_source( # We track our own failures for debugging tried = [] - for candidate_uri in potential_absolute_uris(uri, path, importer): + for candidate_uri in potential_absolute_uris(uri, path, importer=importer.pos.abspath if importer else None): # For each place to try in order destination_buffer = io.BytesIO() logger.debug("Fetching %s", candidate_uri)