From 32f86413d44e261276691da9e03a859efdaedee0 Mon Sep 17 00:00:00 2001 From: Adi Date: Mon, 2 Oct 2023 15:32:41 +0530 Subject: [PATCH 1/9] test for flux --- .github/workflows/testflux.yml | 46 ++++++++++++++++++++++++++++++++++ pydra/conftest.py | 2 +- pydra/engine/workers.py | 26 +++++++++---------- 3 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/testflux.yml diff --git a/.github/workflows/testflux.yml b/.github/workflows/testflux.yml new file mode 100644 index 0000000000..8f89b2bc7d --- /dev/null +++ b/.github/workflows/testflux.yml @@ -0,0 +1,46 @@ +name: Flux + +on: + push: + branches: + - master + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start Docker container + run: | + docker run -d --name flux -v `pwd`:/pydra fluxrm/flux-sched:focal-v0.28.0 flux start + sleep 5 # Give the container some time to start + + - name: Run necessary changes in container + run: | + # Install Python 3 (if not already installed) + docker exec flux apt-get update + docker exec flux apt-get install -y python3 + + # Add Python to PATH + docker exec flux echo 'export PATH="/home/fluxuser/.local/bin:$PATH"' >> ~/.bashrc + + # Create an alias for 'python' (pointing to 'python3') + docker exec flux echo 'alias python=python3' >> ~/.bashrc + + # Source the updated .bashrc to apply changes + docker exec flux source ~/.bashrc + + - name: Run pytest + run: | + docker exec flux bash -c "pytest --psij=flux --color=yes -vs -n auto --psij=slurm --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/" + + - name: Upload to codecov + run: | + docker exec flux bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" + + - name: Stop and remove Docker container + run: docker stop flux && docker rm flux diff --git a/pydra/conftest.py b/pydra/conftest.py index 66a1d200fc..aa51422d58 100644 --- a/pydra/conftest.py +++ b/pydra/conftest.py @@ -11,7 +11,7 @@ def pytest_addoption(parser): "--psij", action="store", help="run with psij subtype plugin", - choices=["local", "slurm"], + choices=["local", "slurm", "flux"], ) diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index f56e3a3d1e..962bd4597c 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -913,7 +913,7 @@ def __init__(self, subtype, **kwargs): self.psij = psij # Check if the provided subtype is valid - valid_subtypes = ["local", "slurm"] + valid_subtypes = ["local", "slurm", "flux"] if subtype not in valid_subtypes: raise ValueError( f"Invalid 'subtype' provided. Available options: {', '.join(valid_subtypes)}" @@ -992,7 +992,7 @@ async def exec_psij(self, runnable, rerun=False): with open(file_path, "wb") as file: pickle.dump(runnable._run, file) func_path = absolute_path / "run_pickled.py" - spec = self.make_spec("python", [func_path, file_path]) + spec = self.make_spec("python", [str(func_path), str(file_path)]) else: # it could be tuple that includes pickle files with tasks and inputs cache_dir = runnable[-1].cache_dir file_path_1 = cache_dir / "taskmain.pkl" @@ -1006,9 +1006,9 @@ async def exec_psij(self, runnable, rerun=False): spec = self.make_spec( "python", [ - func_path, - file_path_1, - file_path_2, + str(func_path), + str(file_path_1), + str(file_path_2), ], ) @@ -1016,18 +1016,18 @@ async def exec_psij(self, runnable, rerun=False): spec.arguments.append("--rerun") spec.stdout_path = cache_dir / "demo.stdout" - spec.stderr_path = cache_dir / "demo.stderr" + # spec.stderr_path = cache_dir / "demo.stderr" job = self.make_job(spec, None) jex.submit(job) job.wait() - if spec.stderr_path.stat().st_size > 0: - with open(spec.stderr_path, "r") as stderr_file: - stderr_contents = stderr_file.read() - raise Exception( - f"stderr_path '{spec.stderr_path}' is not empty. Contents:\n{stderr_contents}" - ) + # if spec.stderr_path.stat().st_size > 0: + # with open(spec.stderr_path, "r") as stderr_file: + # stderr_contents = stderr_file.read() + # raise Exception( + # f"stderr_path '{spec.stderr_path}' is not empty. Contents:\n{stderr_contents}" + # ) return @@ -1044,6 +1044,6 @@ def close(self): "sge": SGEWorker, **{ "psij-" + subtype: lambda subtype=subtype: PsijWorker(subtype=subtype) - for subtype in ["local", "slurm"] + for subtype in ["local", "slurm", "flux"] }, } From 4ad29965bcd2b5f276488ec66855de402e536ddd Mon Sep 17 00:00:00 2001 From: Aditya Agarwal <50960175+adi611@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:44:29 +0530 Subject: [PATCH 2/9] Update testflux.yml --- .github/workflows/testflux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testflux.yml b/.github/workflows/testflux.yml index 8f89b2bc7d..832e03f1a1 100644 --- a/.github/workflows/testflux.yml +++ b/.github/workflows/testflux.yml @@ -16,7 +16,7 @@ jobs: - name: Start Docker container run: | - docker run -d --name flux -v `pwd`:/pydra fluxrm/flux-sched:focal-v0.28.0 flux start + docker run -itd --name flux -v `pwd`:/pydra fluxrm/flux-sched:focal-v0.28.0 flux start sleep 5 # Give the container some time to start - name: Run necessary changes in container From 0f9d2b0b3e652861622b655e8daf066e214729dd Mon Sep 17 00:00:00 2001 From: Adi Date: Mon, 2 Oct 2023 22:24:02 +0530 Subject: [PATCH 3/9] flux workflow --- .github/workflows/testflux.yml | 65 ++++++++++++++++------------------ 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/.github/workflows/testflux.yml b/.github/workflows/testflux.yml index 832e03f1a1..4588c492b1 100644 --- a/.github/workflows/testflux.yml +++ b/.github/workflows/testflux.yml @@ -9,38 +9,35 @@ on: jobs: build: runs-on: ubuntu-latest - + permissions: + packages: read + strategy: + fail-fast: false + matrix: + container: ['fluxrm/flux-sched:focal-v0.28.0'] + + container: + image: ${{ matrix.container }} + options: "--platform=linux/amd64 --user root -it --init" + + name: ${{ matrix.container }} steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Start Docker container - run: | - docker run -itd --name flux -v `pwd`:/pydra fluxrm/flux-sched:focal-v0.28.0 flux start - sleep 5 # Give the container some time to start - - - name: Run necessary changes in container - run: | - # Install Python 3 (if not already installed) - docker exec flux apt-get update - docker exec flux apt-get install -y python3 - - # Add Python to PATH - docker exec flux echo 'export PATH="/home/fluxuser/.local/bin:$PATH"' >> ~/.bashrc - - # Create an alias for 'python' (pointing to 'python3') - docker exec flux echo 'alias python=python3' >> ~/.bashrc - - # Source the updated .bashrc to apply changes - docker exec flux source ~/.bashrc - - - name: Run pytest - run: | - docker exec flux bash -c "pytest --psij=flux --color=yes -vs -n auto --psij=slurm --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/" - - - name: Upload to codecov - run: | - docker exec flux bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" - - - name: Stop and remove Docker container - run: docker stop flux && docker rm flux + - name: Make Space + run: | + rm -rf /usr/share/dotnet + rm -rf /opt/ghc + - name: Checkout + uses: actions/checkout@v4 + + - name: Install + run: | + apt-get update && apt-get install -y python3-venv + export PATH=$PWD/bin:$PATH + ln -s /usr/bin/python3 /usr/bin/python + python3 -m pip install --upgrade pip && pip install -e ".[test]" && python -c 'import pydra; print(pydra.__version__)' + pip install -e "git+https://github.com/adi611/psij-python.git@adi611-patch-2#egg=psij-python" + - name: Start Flux and Run Test + run: | + export PATH=$PWD/bin:$PATH + flux start python -V + flux start pytest --psij=flux --color=yes -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra/ From 1c83646a402be2e2d99e31cd8723990f4622cde4 Mon Sep 17 00:00:00 2001 From: Adi Date: Mon, 2 Oct 2023 22:28:21 +0530 Subject: [PATCH 4/9] job build to test --- .github/workflows/testflux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testflux.yml b/.github/workflows/testflux.yml index 4588c492b1..d23aa751b2 100644 --- a/.github/workflows/testflux.yml +++ b/.github/workflows/testflux.yml @@ -7,7 +7,7 @@ on: pull_request: jobs: - build: + test: runs-on: ubuntu-latest permissions: packages: read From e7f7023f91b2514f3ae8fb25c78a6c8563f56ad2 Mon Sep 17 00:00:00 2001 From: Adi Date: Mon, 2 Oct 2023 22:46:41 +0530 Subject: [PATCH 5/9] fix stderr issue with psij --- .github/workflows/testflux.yml | 2 +- pydra/engine/workers.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/testflux.yml b/.github/workflows/testflux.yml index d23aa751b2..26e767ba29 100644 --- a/.github/workflows/testflux.yml +++ b/.github/workflows/testflux.yml @@ -34,7 +34,7 @@ jobs: apt-get update && apt-get install -y python3-venv export PATH=$PWD/bin:$PATH ln -s /usr/bin/python3 /usr/bin/python - python3 -m pip install --upgrade pip && pip install -e ".[test]" && python -c 'import pydra; print(pydra.__version__)' + python -m pip install --upgrade pip && pip install -e ".[test]" && python -c 'import pydra; print(pydra.__version__)' pip install -e "git+https://github.com/adi611/psij-python.git@adi611-patch-2#egg=psij-python" - name: Start Flux and Run Test run: | diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 962bd4597c..aa8c4f30b6 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -1016,18 +1016,18 @@ async def exec_psij(self, runnable, rerun=False): spec.arguments.append("--rerun") spec.stdout_path = cache_dir / "demo.stdout" - # spec.stderr_path = cache_dir / "demo.stderr" + spec.stderr_path = cache_dir / "demo.stderr" job = self.make_job(spec, None) jex.submit(job) job.wait() - # if spec.stderr_path.stat().st_size > 0: - # with open(spec.stderr_path, "r") as stderr_file: - # stderr_contents = stderr_file.read() - # raise Exception( - # f"stderr_path '{spec.stderr_path}' is not empty. Contents:\n{stderr_contents}" - # ) + if spec.stderr_path.stat().st_size > 0: + with open(spec.stderr_path, "r") as stderr_file: + stderr_contents = stderr_file.read() + raise Exception( + f"stderr_path '{spec.stderr_path}' is not empty. Contents:\n{stderr_contents}" + ) return From 26e2361a17f1be2b4453ae7377bf36ebe9211632 Mon Sep 17 00:00:00 2001 From: Adi Date: Tue, 3 Oct 2023 10:46:50 +0530 Subject: [PATCH 6/9] intsall package less --- .github/workflows/testflux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testflux.yml b/.github/workflows/testflux.yml index 26e767ba29..d9fe628dec 100644 --- a/.github/workflows/testflux.yml +++ b/.github/workflows/testflux.yml @@ -29,9 +29,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install + - name: Setup Python run: | - apt-get update && apt-get install -y python3-venv + apt-get update && apt-get install -y python3-venv && apt-get install less export PATH=$PWD/bin:$PATH ln -s /usr/bin/python3 /usr/bin/python python -m pip install --upgrade pip && pip install -e ".[test]" && python -c 'import pydra; print(pydra.__version__)' From 76cb8da60429e84757e858308fb10cb7440e6d5c Mon Sep 17 00:00:00 2001 From: Adi Date: Tue, 3 Oct 2023 12:09:02 +0530 Subject: [PATCH 7/9] remove matrix strategy --- .github/workflows/testflux.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/testflux.yml b/.github/workflows/testflux.yml index d9fe628dec..0ca2d8ee24 100644 --- a/.github/workflows/testflux.yml +++ b/.github/workflows/testflux.yml @@ -11,16 +11,11 @@ jobs: runs-on: ubuntu-latest permissions: packages: read - strategy: - fail-fast: false - matrix: - container: ['fluxrm/flux-sched:focal-v0.28.0'] container: - image: ${{ matrix.container }} + image: fluxrm/flux-sched:focal-v0.28.0 options: "--platform=linux/amd64 --user root -it --init" - name: ${{ matrix.container }} steps: - name: Make Space run: | @@ -36,7 +31,7 @@ jobs: ln -s /usr/bin/python3 /usr/bin/python python -m pip install --upgrade pip && pip install -e ".[test]" && python -c 'import pydra; print(pydra.__version__)' pip install -e "git+https://github.com/adi611/psij-python.git@adi611-patch-2#egg=psij-python" - - name: Start Flux and Run Test + - name: Run pytest run: | export PATH=$PWD/bin:$PATH flux start python -V From b58644d649c33e66ed6fc64ffbde91a8005d0fc8 Mon Sep 17 00:00:00 2001 From: Adi Date: Tue, 3 Oct 2023 12:17:11 +0530 Subject: [PATCH 8/9] make naming consistent --- .github/workflows/{testflux.yml => testpsijflux.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{testflux.yml => testpsijflux.yml} (98%) diff --git a/.github/workflows/testflux.yml b/.github/workflows/testpsijflux.yml similarity index 98% rename from .github/workflows/testflux.yml rename to .github/workflows/testpsijflux.yml index 0ca2d8ee24..fb02b12c07 100644 --- a/.github/workflows/testflux.yml +++ b/.github/workflows/testpsijflux.yml @@ -1,4 +1,4 @@ -name: Flux +name: PSI/J-Flux on: push: From 772d07a6a968c1d4306a9d337aae307a660e34f8 Mon Sep 17 00:00:00 2001 From: Aditya Agarwal <50960175+adi611@users.noreply.github.com> Date: Sat, 21 Oct 2023 14:06:41 +0530 Subject: [PATCH 9/9] use exaworks psij-python repo for pip install --- .github/workflows/testpsijflux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testpsijflux.yml b/.github/workflows/testpsijflux.yml index fb02b12c07..8eadb3fd38 100644 --- a/.github/workflows/testpsijflux.yml +++ b/.github/workflows/testpsijflux.yml @@ -30,7 +30,7 @@ jobs: export PATH=$PWD/bin:$PATH ln -s /usr/bin/python3 /usr/bin/python python -m pip install --upgrade pip && pip install -e ".[test]" && python -c 'import pydra; print(pydra.__version__)' - pip install -e "git+https://github.com/adi611/psij-python.git@adi611-patch-2#egg=psij-python" + pip install -e "git+https://github.com/ExaWorks/psij-python.git@main#egg=psij-python" - name: Run pytest run: | export PATH=$PWD/bin:$PATH