diff --git a/requirements_test.txt b/requirements_test.txt index e9c4742..fdb4ef8 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,23 +1,22 @@ wheel -galaxy-util@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/util -galaxy-objectstore@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/objectstore -galaxy-files@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/files -galaxy-data@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/data -galaxy-auth@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/auth -galaxy-selenium@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/selenium -galaxy-data@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/data -galaxy-job-execution@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/job_execution -galaxy-job-metrics@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/job_metrics -galaxy-tool-util@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/tool_util -galaxy-web-stack@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/web_stack -galaxy-config@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/config -galaxy-web-framework@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/web_framework +galaxy-util +galaxy-objectstore +galaxy-files +galaxy-data +galaxy-auth +galaxy-selenium +galaxy-data +galaxy-job-execution +galaxy-job-metrics +galaxy-tool-util +galaxy-web-stack +galaxy-config +galaxy-web-framework -e ".[test]" # The following packages are needed for resubmit integration testing (except galaxy-app) -galaxy-job-execution@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/job_execution +galaxy-job-execution edam-ontology dictobj -galaxy-test-base@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/test_base -galaxy-app@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/app -galaxy-webapps@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/webapps -galaxy-test-driver@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/test_driver +galaxy-test-base +galaxy-app +galaxy-test-driver diff --git a/tests/fixtures/mapping-inheritance.yml b/tests/fixtures/mapping-inheritance.yml index 44ccd24..ac1583e 100644 --- a/tests/fixtures/mapping-inheritance.yml +++ b/tests/fixtures/mapping-inheritance.yml @@ -47,8 +47,20 @@ tools: tophat: inherits: ~ gpus: 5 + kraken2: + scheduling: + require: + - custom-indices + kraken5: + scheduling: + accept: + - pulsar + require: + - k8s-special destinations: + default: + runner: local local: runner: local max_accepted_cores: 4 @@ -61,6 +73,25 @@ destinations: max_accepted_cores: 16 max_accepted_mem: 64 max_accepted_gpus: 5 + params: + docker_extra: extra-args + env: + ABC: def scheduling: prefer: - pulsar + destination_that_inherits_runner_from_default: + max_accepted_cores: 4 + max_accepted_mem: 16 + scheduling: + accept: + - custom-indices + prefer: + - general + destination_that_inherits_everything_from_k8s: + inherits: k8s_environment + scheduling: + accept: + - k8s-special + + diff --git a/tests/test_mapper_inheritance.py b/tests/test_mapper_inheritance.py index d579677..75e8f5c 100644 --- a/tests/test_mapper_inheritance.py +++ b/tests/test_mapper_inheritance.py @@ -94,3 +94,23 @@ def test_map_with_default_rules_in_dedicated_file(self): ] destination = self._map_to_destination(tool, user, datasets=[], tpv_config_files=tpv_config_files) self.assertEqual('--cores=8 --mem=250', destination.params.get('submit_native_specification')) + + def test_destination_inherits_runner_from_default(self): + tool_id = 'kraken2' + user = mock_galaxy.User('benjy', 'benjymouse@vortex.org') + tool = mock_galaxy.Tool(tool_id) + datasets = [mock_galaxy.DatasetAssociation("test", mock_galaxy.Dataset("test.txt", file_size=5*1024**3))] + destination = self._map_to_destination(tool, user, datasets) + self.assertEqual('destination_that_inherits_runner_from_default', destination.id) + self.assertEqual('local', destination.runner) + + def test_general_destination_inheritance(self): + tool_id = 'kraken5' + user = mock_galaxy.User('frankie', 'frankiemouse@vortex.org') + tool = mock_galaxy.Tool(tool_id) + datasets = [mock_galaxy.DatasetAssociation("test", mock_galaxy.Dataset("test.txt", file_size=5*1024**3))] + destination = self._map_to_destination(tool, user, datasets) + self.assertEqual('destination_that_inherits_everything_from_k8s', destination.id) + self.assertTrue('ABC' in [e.get('name') for e in destination.env]) + self.assertEqual('extra-args', destination.params.get('docker_extra')) + self.assertEqual('k8s', destination.runner) diff --git a/tpv/core/mapper.py b/tpv/core/mapper.py index 114ec5f..6d5db57 100644 --- a/tpv/core/mapper.py +++ b/tpv/core/mapper.py @@ -34,11 +34,11 @@ def __compile_tool_regex(self, key): raise def _find_entities_matching_id(self, entity_list, entity_name): - matches = [] - if self.default_inherits: - default_match = entity_list.get(self.default_inherits) - if default_match: - matches.append(default_match) + default_inherits = self.__get_default_inherits(entity_list) + if default_inherits: + matches = [default_inherits] + else: + matches = [] for key in entity_list.keys(): if self.lookup_tool_regex(key).match(entity_name): match = entity_list[key] @@ -54,6 +54,20 @@ def __inherit_matching_entities(self, entity_type, entity_name): matches = self._find_entities_matching_id(entity_list, entity_name) return self.inherit_entities(matches) + def __get_default_inherits(self, entity_list): + if self.default_inherits: + default_match = entity_list.get(self.default_inherits) + if default_match: + return default_match + return None + + def __apply_default_destination_inheritance(self, entity_list): + default_inherits = self.__get_default_inherits(entity_list) + if default_inherits: + return [self.inherit_entities([default_inherits, entity]) for entity in entity_list.values()] + else: + return entity_list.values() + def inherit_entities(self, entities): if entities: return functools.reduce(lambda a, b: b.inherit(a), entities) @@ -72,7 +86,8 @@ def rank(self, entity, destinations, context): def match_and_rank_destinations(self, entity, destinations, context): # At this point, the resource requirements (cores, mem, gpus) are unevaluated. # So temporarily evaluate them so we can match up with a destination. - matches = [dest for dest in destinations.values() if dest.matches(entity.evaluate_resources(context), context)] + matches = [dest for dest in self.__apply_default_destination_inheritance(destinations) + if dest.matches(entity.evaluate_resources(context), context)] return self.rank(entity, matches, context) def to_galaxy_destination(self, destination):