Skip to content

Commit

Permalink
Do not transfer of output files if destination is given (#38)
Browse files Browse the repository at this point in the history
  • Loading branch information
JoschD authored Dec 4, 2023
1 parent 2be7eeb commit 79f4de0
Show file tree
Hide file tree
Showing 7 changed files with 32 additions and 13 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# `pylhc-submitter` Changelog

## Version 2.0.3

- Fixing `job_submitter`: Do not transfer any output files, when the `output_destination` is given.

## Version 2.0.2

- Fixing `job_submitter`: Discovers more invalid URIs.
Expand Down
2 changes: 1 addition & 1 deletion pylhc_submitter/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
__title__ = "pylhc_submitter"
__description__ = "pylhc-submitter contains scripts to simplify the creation and submission of jobs to HTCondor at CERN"
__url__ = "https://github.com/pylhc/submitter"
__version__ = "2.0.2"
__version__ = "2.0.3"
__author__ = "pylhc"
__author_email__ = "[email protected]"
__license__ = "MIT"
Expand Down
2 changes: 1 addition & 1 deletion pylhc_submitter/constants/htcondor.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,4 @@
"nextweek", # 1 w
)

NOTIFICATIONS = ("always", "complete", "error", "never")
NOTIFICATIONS = ("always", "complete", "error", "never")
2 changes: 1 addition & 1 deletion pylhc_submitter/job_submitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,7 @@ def check_opts(opt):

creation = CreationOpts(**{f.name: opt[f.name] for f in fields(CreationOpts)})
runner = RunnerOpts(**{f.name: opt[f.name] for f in fields(RunnerOpts)})
runner.output_dir = None if opt.output_destination else opt.output_dir
runner.output_dir = '""' if opt.output_destination else opt.output_dir # empty string stops htc transfer of files
return creation, runner


Expand Down
6 changes: 3 additions & 3 deletions pylhc_submitter/submitter/htc_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def create_multijob_for_bashfiles(job_df: DataFrame, **kwargs) -> str:
Keyword Args:
output_dir (str): output directory that will be transferred. Defaults to ``None``.
duration (str): max duration of the job. Needs to be one of the ``HTCondor`` Jobflavours.
jobflavour (str): max duration of the job. Needs to be one of the ``HTCondor`` Jobflavours.
Defaults to ``workday``.
group (str): force use of accounting group. Defaults to ``None``.
retries (int): maximum amount of retries. Default to ``3``.
Expand Down Expand Up @@ -257,8 +257,8 @@ def map_kwargs(add_dict: Dict[str, Any]) -> Dict[str, Any]:

# Predefined mappings
htc_map = { # name: mapped_name, choices, default
"duration": ("+JobFlavour", JOBFLAVOURS, "workday"),
"output_dir": ("transfer_output_files", None, None),
"jobflavour": ("+JobFlavour", JOBFLAVOURS, "workday"),
"output_dir": ("transfer_output_files", None, '""'),
"accounting_group": ("+AccountingGroup", None, None),
"max_retries": ("max_retries", None, 3),
"notification": ("notification", NOTIFICATIONS, "error"),
Expand Down
6 changes: 3 additions & 3 deletions pylhc_submitter/submitter/runners.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import logging
import multiprocessing
import subprocess
from dataclasses import dataclass
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, Optional, Tuple

Expand All @@ -31,7 +31,7 @@ class RunnerOpts:
output_dir: Optional[str] = None # Name of the output directory, where jobs store data
ssh: Optional[str] = None # SSH command
dryrun: Optional[bool] = False # Perform only a dry-run, i.e. do all but submit to HTC
htc_arguments: Optional[Dict[str, Any]] = None # Arguments to pass on to htc as keywords
htc_arguments: Optional[Dict[str, Any]] = field(default_factory=dict) # Arguments to pass on to htc as keywords
run_local: Optional[bool] = False # Run jobs locally
num_processes: Optional[int] = 4 # Number of processes to run in parallel (locally)

Expand Down Expand Up @@ -83,7 +83,7 @@ def run_htc(job_df: tfs.TfsDataFrame, opt: RunnerOpts) -> None:
subfile = htc_utils.make_subfile(
opt.working_directory, job_df,
output_dir=opt.output_dir,
duration=opt.jobflavour,
jobflavour=opt.jobflavour,
**opt.htc_arguments
)

Expand Down
23 changes: 19 additions & 4 deletions tests/unit/test_job_submitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import pytest

from pylhc_submitter.job_submitter import main as job_submit
from pylhc_submitter.submitter.iotools import get_server_from_uri, is_eos_uri, uri_to_path
from pylhc_submitter.submitter.iotools import uri_to_path
from pylhc_submitter.utils.environment import on_linux, on_windows

SUBFILE = "queuehtc.sub"
Expand Down Expand Up @@ -41,7 +41,6 @@ def test_output_directory(tmp_path):
)
setup.create_mask()
job_submit(**asdict(setup))
_test_output(setup)


def test_detects_wrong_uri(tmp_path):
Expand All @@ -62,7 +61,6 @@ def test_detects_wrong_uri(tmp_path):
assert "EOS-URI" in str(e)



@run_only_on_linux
def test_job_creation_and_localrun_with_multiline_maskstring(tmp_path):
""" Tests that the jobs are created and can be run locally from a multiline mask-string. """
Expand All @@ -84,6 +82,23 @@ def test_job_creation_and_dryrun(tmp_path, maskfile):
_test_output(setup, post_run=False)


@run_only_on_linux
@pytest.mark.parametrize("maskfile", [True, False])
@pytest.mark.parametrize("destination", [True, False])
def test_more_subfile_content(tmp_path, maskfile, destination):
""" Tests that the jobs are created as dry-run from mask-file and from mask-string. """
setup = InputParameters(
jobflavour="espresso", # change from default
working_directory=tmp_path,
dryrun=True,
output_destination= tmp_path / "my_new_output" / "long_path" if destination else None,
job_output_dir="MyOutputDataHere", # change from default
)
setup.create_mask(as_file=maskfile)
job_submit(**asdict(setup))
_test_subfile_content(setup)


@run_only_on_linux
@pytest.mark.parametrize("maskfile", [True, False])
def test_find_errorneous_percentage_signs(tmp_path, maskfile):
Expand Down Expand Up @@ -243,7 +258,7 @@ def _test_subfile_content(setup: InputParameters):
if setup.output_destination is None:
assert filecontents["transfer_output_files"] == setup.job_output_dir
else:
assert "transfer_output_files" not in filecontents
assert filecontents["transfer_output_files"] == '""'

for key in setup.htc_arguments.keys():
assert filecontents[key] == setup.htc_arguments[key]
Expand Down

0 comments on commit 79f4de0

Please sign in to comment.