Skip to content

Commit

Permalink
Add support library helper function that puts output files under Cond…
Browse files Browse the repository at this point in the history
…uctor's task outputs when relevant (#88)

* Add library function to deal with output paths

* Handle string paths, update docs

* Fix compatibility with Python 3.8
  • Loading branch information
geoffxy authored Jul 24, 2023
1 parent b9c0059 commit 99401a2
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 1 deletion.
19 changes: 18 additions & 1 deletion src/conductor/lib/path.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os
import pathlib
from typing import List
from typing import List, Union

from conductor.config import (
DEPS_ENV_VARIABLE_NAME,
Expand Down Expand Up @@ -36,3 +36,20 @@ def get_output_path() -> pathlib.Path:
"being executed by Conductor.".format(OUTPUT_ENV_VARIABLE_NAME)
)
return pathlib.Path(os.environ[OUTPUT_ENV_VARIABLE_NAME])


def in_output_dir(file_path: Union[pathlib.Path, str]) -> pathlib.Path:
"""
If the current script is being run by Conductor, this function amends
`file_path` to make it fall under where Conductor's task outputs should be
stored. Otherwise, this function returns `file_path` unchanged.
This is meant to be useful for scripts that may be run independently of
Conductor. Note that `file_path` should be a relative path.
"""
if OUTPUT_ENV_VARIABLE_NAME in os.environ:
return get_output_path() / file_path
elif not isinstance(file_path, pathlib.Path):
return pathlib.Path(file_path)
else:
return file_path
5 changes: 5 additions & 0 deletions tests/fixture-projects/lib-test/path/COND
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,8 @@ run_command(
name="output_path",
run="python3 output_path.py",
)

run_command(
name="in_output_dir",
run="python3 in_output_dir.py",
)
17 changes: 17 additions & 0 deletions tests/fixture-projects/lib-test/path/in_output_dir.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import os
import pathlib
import conductor.lib as cond


def main():
my_file = pathlib.Path("./test.csv")
file_in_output_dir = cond.in_output_dir(my_file)
assert file_in_output_dir != my_file

output_dir = cond.get_output_path()
# Or use pathlib.Path.is_relative_to() on Python 3.9+.
assert os.path.commonpath([file_in_output_dir, output_dir]) == str(output_dir)


if __name__ == "__main__":
main()
13 changes: 13 additions & 0 deletions tests/lib_path_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,16 @@ def test_get_output_path(tmp_path: pathlib.Path):
result = cond.run("//path:output_path")
# The output_path.py script in lib-test/path/ does the correctness assertions.
assert result.returncode == 0


def test_in_output_dir_non_cond():
file_path = pathlib.Path("./test.csv")
out = condlib.in_output_dir(file_path)
assert out == file_path


def test_in_output_dir(tmp_path: pathlib.Path):
cond = ConductorRunner.from_template(tmp_path, FIXTURE_TEMPLATES["lib-test"])
result = cond.run("//path:in_output_dir")
# The in_output_dir.py script in lib-test/path/ does the correctness assertions.
assert result.returncode == 0
15 changes: 15 additions & 0 deletions website/docs/python-support-library.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,3 +90,18 @@ def main():
if __name__ == "__main__":
main()
```

### `in_output_dir()`

```python
def in_output_dir(file_path: path.Path | str) -> pathlib.Path
```

If the current script is being run by Conductor, this function amends
`file_path` to make it fall under where Conductor's task outputs should be
stored. Otherwise, this function returns `file_path` unchanged (but as a
`pathlib.Path`).

This is meant to be useful for scripts that may be run independently of
Conductor. Note that `file_path` should be a relative path.

0 comments on commit 99401a2

Please sign in to comment.