Add pydantic support

setup.cfg

@@ -19,3 +19,10 @@ match=^[Tt]est
 
 [bdist_wheel]
 universal = 1
+
+[black]
+line-length = 88
+
+[isort]
+line_length = 88
+profile = black

tests/fixtures/linter/linter-invalid-regex.yml

@@ -6,7 +6,7 @@ tools:
     cores: 2
     params:
       native_spec: "--mem {mem} --cores {cores} --gpus {gpus}"
-  bwa[0-9]++:
+  bwa[0-9]++\:
     gpus: 2
 
 destinations:

tests/fixtures/mapping-basic.yml

@@ -28,12 +28,6 @@ tools:
     scheduling:
       require:
         - non_existent
-  invalidly_tagged_tool:
-    scheduling:
-      require:
-        - general
-      reject:
-        - general
   regex_tool.*:
     scheduling:
       require:

tests/fixtures/mapping-destinations.yml

@@ -193,7 +193,7 @@ destinations:
       TEST_ENTITY_PRIORITY: "{cores*2}"
     params:
       memory_requests: "{mem*2}"
-      k8s_walltime_limit: 20
+      k8s_walltime_limit: "20"
     rules:
       - if: input_size > 20
         execute: |

tests/fixtures/mapping-invalid-regex.yml

@@ -28,12 +28,6 @@ tools:
     scheduling:
       require:
         - non_existent
-  invalidly_tagged_tool:
-    scheduling:
-      require:
-        - general
-      reject:
-        - general
   regex_tool.*:
     scheduling:
       require:

tests/fixtures/mapping-invalid-tags.yml

@@ -0,0 +1,40 @@
+global:
+  default_inherits: default
+
+tools:
+  default:
+    abstract: true
+    cores: 2
+    mem: 8
+    env: {}
+    scheduling:
+      require: []
+      prefer:
+        - general
+      accept:
+      reject:
+        - pulsar
+    rules: []
+  invalidly_tagged_tool:
+    scheduling:
+      require:
+        - general
+      reject:
+        - general
+
+destinations:
+  local:
+    runner: local
+    max_accepted_cores: 4
+    max_accepted_mem: 16
+    scheduling:
+      prefer:
+        - general
+  k8s_environment:
+    runner: k8s
+    max_accepted_cores: 16
+    max_accepted_mem: 64
+    max_accepted_gpus: 2
+    scheduling:
+      prefer:
+        - pulsar

tests/fixtures/mapping-rank.yml

@@ -46,12 +46,6 @@ users:
     scheduling:
       require:
         - earth
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-role.yml

@@ -68,12 +68,6 @@ users:
     scheduling:
       require:
         - earth
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-rules-changed.yml

@@ -52,12 +52,6 @@ users:
         max_mem: cores * 6
       - if: input_size >= 5
         fail: Just because
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-rules.yml

@@ -60,12 +60,6 @@ users:
         max_mem: cores * 6
       - if: input_size >= 5
         fail: Just because
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-syntax-error.yml

@@ -42,12 +42,6 @@ users:
     scheduling:
       require:
         - earth
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-user-invalid-tags.yml

@@ -0,0 +1,50 @@
+global:
+  default_inherits: default
+
+tools:
+  default:
+    cores: 2
+    mem: 8
+    gpus: 1
+    env: {}
+    scheduling:
+      require: []
+      prefer:
+        - general
+      accept:
+      reject:
+        - pulsar
+    params:
+      native_spec: "--mem {mem} --cores {cores}"
+    rules: []
+
+users:
+  default:
+    max_cores: 3
+    max_mem: 4
+    env: {}
+    scheduling:
+      require: []
+      prefer:
+        - general
+      accept:
+      reject:
+        - pulsar
+    rules: []
+  [email protected]:
+    scheduling:
+      require:
+        - pulsar
+      reject:
+        - pulsar
+
+destinations:
+  local:
+    runner: local
+    max_accepted_cores: 4
+    max_accepted_mem: 16
+    scheduling:
+      prefer:
+        - general
+      accept:
+        - pulsar

tests/fixtures/mapping-user.yml

@@ -60,12 +60,6 @@ users:
         - earth
       reject:
         - pulsar
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   [email protected]:
     max_cores: 4
     max_mem: 32

tests/fixtures/scenario-too-many-highmem-jobs.yml

@@ -63,8 +63,7 @@ users:
             rule_helper = RuleHelper(app)
             # Find all destinations that support highmem
             destinations = [d.dest_name for d in mapper.destinations.values()
-                            if any(d.tpv_dest_tags.filter(tag_value='highmem',
-                                   tag_type=[TagType.REQUIRE, TagType.PREFER, TagType.ACCEPT]))]
+                            if 'highmem' in (d.tpv_dest_tags.require + d.tpv_dest_tags.prefer + d.tpv_dest_tags.accept)]
             count = rule_helper.job_count(for_user_email=user.email, for_destinations=destinations)
             if count > 4:
               retval = True

tests/test_entity.py

@@ -37,21 +37,20 @@ def test_all_entities_refer_to_same_loader(self):
         }
         # make sure we are still referring to the same loader after evaluation
         evaluated_entity = gateway.ACTIVE_DESTINATION_MAPPER.match_combine_evaluate_entities(context, tool, user)
-        assert evaluated_entity.loader == original_loader
+        assert evaluated_entity.evaluator == original_loader
         for rule in evaluated_entity.rules:
-            assert rule.loader == original_loader
+            assert rule.evaluator == original_loader
 
     def test_destination_to_dict(self):
-
         tpv_config = os.path.join(os.path.dirname(__file__), 'fixtures/mapping-rule-argument-based.yml')
         loader = TPVConfigLoader.from_url_or_path(tpv_config)
 
         # create a destination
-        destination = loader.destinations["k8s_environment"]
+        destination = loader.config.destinations["k8s_environment"]
         # serialize the destination
-        serialized_destination = destination.to_dict()
+        serialized_destination = destination.dict()
         # deserialize the same destination
-        deserialized_destination = Destination.from_dict(loader, serialized_destination)
+        deserialized_destination = Destination(evaluator=loader, **serialized_destination)
         # make sure the deserialized destination is the same as the original
         self.assertEqual(deserialized_destination, destination)
 
@@ -60,24 +59,23 @@ def test_tool_to_dict(self):
         loader = TPVConfigLoader.from_url_or_path(tpv_config)
 
         # create a tool
-        tool = loader.tools["limbo"]
+        tool = loader.config.tools["limbo"]
         # serialize the tool
-        serialized_destination = tool.to_dict()
+        serialized_tool = tool.dict()
         # deserialize the same tool
-        deserialized_destination = Tool.from_dict(loader, serialized_destination)
+        deserialized_tool = Tool(evaluator=loader, **serialized_tool)
         # make sure the deserialized tool is the same as the original
-        self.assertEqual(deserialized_destination, tool)
+        self.assertEqual(deserialized_tool, tool)
 
     def test_tag_equivalence(self):
-        tag1 = Tag("tag_name", "tag_value", TagType.REQUIRE)
-        tag2 = Tag("tag_name2", "tag_value", TagType.REQUIRE)
-        tag3 = Tag("tag_name", "tag_value1", TagType.REQUIRE)
-        tag4 = Tag("tag_name", "tag_value1", TagType.PREFER)
-        same_as_tag1 = Tag("tag_name", "tag_value", TagType.REQUIRE)
+        tag1 = Tag(value="tag_value", tag_type=TagType.REQUIRE)
+        tag2 = Tag(value="tag_value", tag_type=TagType.REQUIRE)
+        tag3 = Tag(value="tag_value1", tag_type=TagType.REQUIRE)
+        tag4 = Tag(value="tag_value1", tag_type=TagType.PREFER)
+        same_as_tag1 = Tag(value="tag_value", tag_type=TagType.REQUIRE)
 
         self.assertEqual(tag1, tag1)
         self.assertEqual(tag1, same_as_tag1)
-        self.assertNotEqual(tag1, tag2)
         self.assertNotEqual(tag1, tag3)
         self.assertNotEqual(tag1, tag4)
         self.assertNotEqual(tag1, "hello")

tests/test_mapper_basic.py

@@ -35,8 +35,9 @@ def test_map_unschedulable_tool(self):
 
     def test_map_invalidly_tagged_tool(self):
         tool = mock_galaxy.Tool('invalidly_tagged_tool')
-        with self.assertRaisesRegex(JobMappingException, "No destinations are available to fulfill request"):
-            self._map_to_destination(tool)
+        config = os.path.join(os.path.dirname(__file__), 'fixtures/mapping-invalid-tags.yml')
+        with self.assertRaisesRegex(Exception, r"Duplicate tags found: 'general' in \['require', 'reject'\]"):
+            self._map_to_destination(tool, tpv_config_path=config)
 
     def test_map_tool_by_regex(self):
         tool = mock_galaxy.Tool('regex_tool_test')

tests/test_mapper_user.py

@@ -8,10 +8,10 @@
 class TestMapperUser(unittest.TestCase):
 
     @staticmethod
-    def _map_to_destination(tool, user):
+    def _map_to_destination(tool, user, tpv_config_path=None):
         galaxy_app = mock_galaxy.App(job_conf=os.path.join(os.path.dirname(__file__), 'fixtures/job_conf.yml'))
         job = mock_galaxy.Job()
-        tpv_config = os.path.join(os.path.dirname(__file__), 'fixtures/mapping-user.yml')
+        tpv_config = tpv_config_path or os.path.join(os.path.dirname(__file__), 'fixtures/mapping-user.yml')
         gateway.ACTIVE_DESTINATION_MAPPER = None
         return gateway.map_tool_to_destination(galaxy_app, job, tool, user, tpv_config_files=[tpv_config])
 
@@ -40,8 +40,9 @@ def test_map_invalidly_tagged_user(self):
         tool = mock_galaxy.Tool('bwa')
         user = mock_galaxy.User('infinitely', '[email protected]')
 
-        with self.assertRaises(IncompatibleTagsException):
-            self._map_to_destination(tool, user)
+        config = os.path.join(os.path.dirname(__file__), 'fixtures/mapping-user-invalid-tags.yml')
+        with self.assertRaisesRegex(Exception, r"Duplicate tags found: 'pulsar' in \['require', 'reject'\]"):
+            self._map_to_destination(tool, user, tpv_config_path=config)
 
     def test_map_user_by_regex(self):
         tool = mock_galaxy.Tool('bwa')

tox.ini

@@ -6,7 +6,7 @@
 # running the tests.
 
 [tox]
-envlist = py3.10,lint
+envlist = py3.11,lint
 
 [testenv]
 commands = # see setup.cfg for options sent to nosetests and coverage

tpv/__init__.py

@@ -1,7 +1,7 @@
 """Total Perspective Vortex library setup."""
 
 # Current version of the library
-__version__ = "2.4.0"
+__version__ = "3.0.0"
 
 
 def get_version():

tpv/commands/linter.py

@@ -23,8 +23,13 @@ def lint(self):
         except Exception as e:
             log.error(f"Linting failed due to syntax errors in yaml file: {e}")
             raise TPVLintError("Linting failed due to syntax errors in yaml file: ") from e
-        default_inherits = loader.global_settings.get('default_inherits')
-        for tool_regex, tool in loader.tools.items():
+        self.lint_tools(loader)
+        self.lint_destinations(loader)
+        self.print_errors_and_warnings()
+
+    def lint_tools(self, loader):
+        default_inherits = loader.config.global_config.default_inherits
+        for tool_regex, tool in loader.config.tools.items():
             try:
                 re.compile(tool_regex)
             except re.error:
@@ -34,7 +39,10 @@ def lint(self):
                     f"The tool named: {default_inherits} is marked globally as the tool to inherit from "
                     "by default. You may want to mark it as abstract if it is not an actual tool and it "
                     "will be excluded from scheduling decisions.")
-        for destination in loader.destinations.values():
+
+    def lint_destinations(self, loader):
+        default_inherits = loader.config.global_config.default_inherits
+        for destination in loader.config.destinations.values():
             if not destination.runner and not destination.abstract:
                 self.errors.append(f"Destination '{destination.id}' does not define the runner parameter. "
                                    "The runner parameter is mandatory.")
@@ -51,6 +59,8 @@ def lint(self):
                     f"The destination named: {default_inherits} is marked globally as the destination to inherit from "
                     "by default. You may want to mark it as abstract if it is not meant to be dispatched to, and it "
                     "will be excluded from scheduling decisions.")
+
+    def print_errors_and_warnings(self):
         if self.warnings:
             for w in self.warnings:
                 log.warning(w)