Merge pull request #110 from cat-bro/destination-inheritance-extra-tests

Add two tests around destination inheritance (one fails)

requirements_test.txt

@@ -1,23 +1,22 @@
 wheel
-galaxy-util@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/util
-galaxy-objectstore@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/objectstore
-galaxy-files@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/files
-galaxy-data@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/data
-galaxy-auth@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/auth
-galaxy-selenium@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/selenium
-galaxy-data@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/data
-galaxy-job-execution@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/job_execution
-galaxy-job-metrics@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/job_metrics
-galaxy-tool-util@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/tool_util
-galaxy-web-stack@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/web_stack
-galaxy-config@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/config
-galaxy-web-framework@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/web_framework
+galaxy-util
+galaxy-objectstore
+galaxy-files
+galaxy-data
+galaxy-auth
+galaxy-selenium
+galaxy-data
+galaxy-job-execution
+galaxy-job-metrics
+galaxy-tool-util
+galaxy-web-stack
+galaxy-config
+galaxy-web-framework
 -e ".[test]"
 # The following packages are needed for resubmit integration testing (except galaxy-app)
-galaxy-job-execution@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/job_execution
+galaxy-job-execution
 edam-ontology
 dictobj
-galaxy-test-base@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/test_base
-galaxy-app@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/app
-galaxy-webapps@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/webapps
-galaxy-test-driver@git+https://github.com/galaxyproject/galaxy.git@release_22.05#subdirectory=packages/test_driver
+galaxy-test-base
+galaxy-app
+galaxy-test-driver

tests/fixtures/mapping-inheritance.yml

@@ -47,8 +47,20 @@ tools:
   tophat:
     inherits: ~
     gpus: 5
+  kraken2:
+    scheduling:
+      require:
+        - custom-indices
+  kraken5:
+    scheduling:
+      accept:
+        - pulsar
+      require:
+        - k8s-special
 
 destinations:
+  default:
+    runner: local
   local:
     runner: local
     max_accepted_cores: 4
@@ -61,6 +73,25 @@ destinations:
     max_accepted_cores: 16
     max_accepted_mem: 64
     max_accepted_gpus: 5
+    params:
+      docker_extra: extra-args
+    env:
+      ABC: def
     scheduling:
       prefer:
         - pulsar
+  destination_that_inherits_runner_from_default:
+    max_accepted_cores: 4
+    max_accepted_mem: 16
+    scheduling:
+      accept:
+        - custom-indices
+      prefer:
+        - general
+  destination_that_inherits_everything_from_k8s:
+    inherits: k8s_environment
+    scheduling:
+      accept:
+        - k8s-special
+
+

tests/test_mapper_inheritance.py

@@ -94,3 +94,23 @@ def test_map_with_default_rules_in_dedicated_file(self):
         ]
         destination = self._map_to_destination(tool, user, datasets=[], tpv_config_files=tpv_config_files)
         self.assertEqual('--cores=8 --mem=250', destination.params.get('submit_native_specification'))
+
+    def test_destination_inherits_runner_from_default(self):
+        tool_id = 'kraken2'
+        user = mock_galaxy.User('benjy', '[email protected]')
+        tool = mock_galaxy.Tool(tool_id)
+        datasets = [mock_galaxy.DatasetAssociation("test", mock_galaxy.Dataset("test.txt", file_size=5*1024**3))]
+        destination = self._map_to_destination(tool, user, datasets)
+        self.assertEqual('destination_that_inherits_runner_from_default', destination.id)
+        self.assertEqual('local', destination.runner)
+
+    def test_general_destination_inheritance(self):
+        tool_id = 'kraken5'
+        user = mock_galaxy.User('frankie', '[email protected]')
+        tool = mock_galaxy.Tool(tool_id)
+        datasets = [mock_galaxy.DatasetAssociation("test", mock_galaxy.Dataset("test.txt", file_size=5*1024**3))]
+        destination = self._map_to_destination(tool, user, datasets)
+        self.assertEqual('destination_that_inherits_everything_from_k8s', destination.id)
+        self.assertTrue('ABC' in [e.get('name') for e in destination.env])
+        self.assertEqual('extra-args', destination.params.get('docker_extra'))
+        self.assertEqual('k8s', destination.runner)

tpv/core/mapper.py

@@ -34,11 +34,11 @@ def __compile_tool_regex(self, key):
             raise
 
     def _find_entities_matching_id(self, entity_list, entity_name):
-        matches = []
-        if self.default_inherits:
-            default_match = entity_list.get(self.default_inherits)
-            if default_match:
-                matches.append(default_match)
+        default_inherits = self.__get_default_inherits(entity_list)
+        if default_inherits:
+            matches = [default_inherits]
+        else:
+            matches = []
         for key in entity_list.keys():
             if self.lookup_tool_regex(key).match(entity_name):
                 match = entity_list[key]
@@ -54,6 +54,20 @@ def __inherit_matching_entities(self, entity_type, entity_name):
         matches = self._find_entities_matching_id(entity_list, entity_name)
         return self.inherit_entities(matches)
 
+    def __get_default_inherits(self, entity_list):
+        if self.default_inherits:
+            default_match = entity_list.get(self.default_inherits)
+            if default_match:
+                return default_match
+        return None
+
+    def __apply_default_destination_inheritance(self, entity_list):
+        default_inherits = self.__get_default_inherits(entity_list)
+        if default_inherits:
+            return [self.inherit_entities([default_inherits, entity]) for entity in entity_list.values()]
+        else:
+            return entity_list.values()
+
     def inherit_entities(self, entities):
         if entities:
             return functools.reduce(lambda a, b: b.inherit(a), entities)
@@ -72,7 +86,8 @@ def rank(self, entity, destinations, context):
     def match_and_rank_destinations(self, entity, destinations, context):
         # At this point, the resource requirements (cores, mem, gpus) are unevaluated.
         # So temporarily evaluate them so we can match up with a destination.
-        matches = [dest for dest in destinations.values() if dest.matches(entity.evaluate_resources(context), context)]
+        matches = [dest for dest in self.__apply_default_destination_inheritance(destinations)
+                   if dest.matches(entity.evaluate_resources(context), context)]
         return self.rank(entity, matches, context)
 
     def to_galaxy_destination(self, destination):