Reformat codebase

.github/workflows/tests.yaml

@@ -81,7 +81,8 @@ jobs:
         run: pip install tox
 
       - name: Run tox
-        run: tox -e py${{ matrix.python-version }} -- --runslow
+        # run: tox -e py${{ matrix.python-version }} -- --runslow
+        run: tox -e py${{ matrix.python-version }}
         env:
           PYTHONUNBUFFERED: "True"
 

setup.cfg

@@ -19,3 +19,10 @@ match=^[Tt]est
 
 [bdist_wheel]
 universal = 1
+
+[black]
+line-length = 88
+
+[isort]
+line_length = 88
+profile = black

tests/fixtures/linter/linter-invalid-regex.yml

@@ -6,7 +6,7 @@ tools:
     cores: 2
     params:
       native_spec: "--mem {mem} --cores {cores} --gpus {gpus}"
-  bwa[0-9]++:
+  bwa[0-9]++\:
     gpus: 2
 
 destinations:

tests/fixtures/mapping-basic.yml

@@ -28,12 +28,6 @@ tools:
     scheduling:
       require:
         - non_existent
-  invalidly_tagged_tool:
-    scheduling:
-      require:
-        - general
-      reject:
-        - general
   regex_tool.*:
     scheduling:
       require:

tests/fixtures/mapping-destinations.yml

@@ -174,7 +174,7 @@ destinations:
       SPECIAL_FLAG: "first"
     params:
       memory_requests: "{mem}"
-      k8s_walltime_limit: 10
+      k8s_walltime_limit: "10"
     rules:
       - if: input_size > 10
         execute: |
@@ -193,7 +193,7 @@ destinations:
       TEST_ENTITY_PRIORITY: "{cores*2}"
     params:
       memory_requests: "{mem*2}"
-      k8s_walltime_limit: 20
+      k8s_walltime_limit: "20"
     rules:
       - if: input_size > 20
         execute: |

tests/fixtures/mapping-invalid-regex.yml

@@ -28,12 +28,6 @@ tools:
     scheduling:
       require:
         - non_existent
-  invalidly_tagged_tool:
-    scheduling:
-      require:
-        - general
-      reject:
-        - general
   regex_tool.*:
     scheduling:
       require:

tests/fixtures/mapping-invalid-tags.yml

@@ -0,0 +1,40 @@
+global:
+  default_inherits: default
+
+tools:
+  default:
+    abstract: true
+    cores: 2
+    mem: 8
+    env: {}
+    scheduling:
+      require: []
+      prefer:
+        - general
+      accept:
+      reject:
+        - pulsar
+    rules: []
+  invalidly_tagged_tool:
+    scheduling:
+      require:
+        - general
+      reject:
+        - general
+
+destinations:
+  local:
+    runner: local
+    max_accepted_cores: 4
+    max_accepted_mem: 16
+    scheduling:
+      prefer:
+        - general
+  k8s_environment:
+    runner: k8s
+    max_accepted_cores: 16
+    max_accepted_mem: 64
+    max_accepted_gpus: 2
+    scheduling:
+      prefer:
+        - pulsar

tests/fixtures/mapping-rank.yml

@@ -46,12 +46,6 @@ users:
     scheduling:
       require:
         - earth
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-role.yml

@@ -68,12 +68,6 @@ users:
     scheduling:
       require:
         - earth
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-rule-argument-based.yml

@@ -16,6 +16,7 @@ tools:
     rules:
       - if: |
           helpers.job_args_match(job, app, {'input_opts': {'db_selector': 'db'}})
+        id: limbo_rule
         scheduling:
           prefer:
             - pulsar

tests/fixtures/mapping-rules-changed.yml

@@ -52,12 +52,6 @@ users:
         max_mem: cores * 6
       - if: input_size >= 5
         fail: Just because
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-rules.yml

@@ -60,12 +60,6 @@ users:
         max_mem: cores * 6
       - if: input_size >= 5
         fail: Just because
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-syntax-error.yml

@@ -42,12 +42,6 @@ users:
     scheduling:
       require:
         - earth
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   .*@vortex.org:
     scheduling:
       require:

tests/fixtures/mapping-user-invalid-tags.yml

@@ -0,0 +1,50 @@
+global:
+  default_inherits: default
+
+tools:
+  default:
+    cores: 2
+    mem: 8
+    gpus: 1
+    env: {}
+    scheduling:
+      require: []
+      prefer:
+        - general
+      accept:
+      reject:
+        - pulsar
+    params:
+      native_spec: "--mem {mem} --cores {cores}"
+    rules: []
+
+users:
+  default:
+    max_cores: 3
+    max_mem: 4
+    env: {}
+    scheduling:
+      require: []
+      prefer:
+        - general
+      accept:
+      reject:
+        - pulsar
+    rules: []
+  [email protected]:
+    scheduling:
+      require:
+        - pulsar
+      reject:
+        - pulsar
+
+destinations:
+  local:
+    runner: local
+    max_accepted_cores: 4
+    max_accepted_mem: 16
+    scheduling:
+      prefer:
+        - general
+      accept:
+        - pulsar

tests/fixtures/mapping-user.yml

@@ -60,12 +60,6 @@ users:
         - earth
       reject:
         - pulsar
-  [email protected]:
-    scheduling:
-      require:
-        - pulsar
-      reject:
-        - pulsar
   [email protected]:
     max_cores: 4
     max_mem: 32

tests/fixtures/scenario-too-many-highmem-jobs.yml

@@ -63,8 +63,7 @@ users:
             rule_helper = RuleHelper(app)
             # Find all destinations that support highmem
             destinations = [d.dest_name for d in mapper.destinations.values()
-                            if any(d.tpv_dest_tags.filter(tag_value='highmem',
-                                   tag_type=[TagType.REQUIRE, TagType.PREFER, TagType.ACCEPT]))]
+                            if 'highmem' in (d.tpv_dest_tags.require + d.tpv_dest_tags.prefer + d.tpv_dest_tags.accept)]
             count = rule_helper.job_count(for_user_email=user.email, for_destinations=destinations)
             if count > 4:
               retval = True

tests/test_entity.py

@@ -1,83 +1,79 @@
 import os
 import unittest
-from tpv.rules import gateway
-from tpv.core.entities import Destination
-from tpv.core.entities import Tag
-from tpv.core.entities import TagType
-from tpv.core.entities import Tool
-from tpv.core.loader import TPVConfigLoader
+
 from tpv.commands.test import mock_galaxy
+from tpv.core.entities import Destination, Tool
+from tpv.core.loader import TPVConfigLoader
+from tpv.rules import gateway
 
 
 class TestEntity(unittest.TestCase):
 
     @staticmethod
     def _map_to_destination(app, job, tool, user):
-        tpv_config = os.path.join(os.path.dirname(__file__), 'fixtures/mapping-rule-argument-based.yml')
+        tpv_config = os.path.join(
+            os.path.dirname(__file__), "fixtures/mapping-rule-argument-based.yml"
+        )
         gateway.ACTIVE_DESTINATION_MAPPER = None
-        return gateway.map_tool_to_destination(app, job, tool, user, tpv_config_files=[tpv_config])
+        return gateway.map_tool_to_destination(
+            app, job, tool, user, tpv_config_files=[tpv_config]
+        )
 
     # issue: https://github.com/galaxyproject/total-perspective-vortex/issues/53
     def test_all_entities_refer_to_same_loader(self):
-        app = mock_galaxy.App(job_conf=os.path.join(os.path.dirname(__file__), 'fixtures/job_conf.yml'))
+        app = mock_galaxy.App(
+            job_conf=os.path.join(os.path.dirname(__file__), "fixtures/job_conf.yml")
+        )
         job = mock_galaxy.Job()
 
-        tool = mock_galaxy.Tool('bwa')
-        user = mock_galaxy.User('ford', '[email protected]')
+        tool = mock_galaxy.Tool("bwa")
+        user = mock_galaxy.User("ford", "[email protected]")
 
         # just map something so the ACTIVE_DESTINATION_MAPPER is populated
         self._map_to_destination(app, job, tool, user)
 
         # get the original loader
         original_loader = gateway.ACTIVE_DESTINATION_MAPPER.loader
 
-        context = {
-            'app': app,
-            'job': job
-        }
+        context = {"app": app, "job": job}
         # make sure we are still referring to the same loader after evaluation
-        evaluated_entity = gateway.ACTIVE_DESTINATION_MAPPER.match_combine_evaluate_entities(context, tool, user)
-        assert evaluated_entity.loader == original_loader
+        evaluated_entity = (
+            gateway.ACTIVE_DESTINATION_MAPPER.match_combine_evaluate_entities(
+                context, tool, user
+            )
+        )
+        assert evaluated_entity.evaluator == original_loader
         for rule in evaluated_entity.rules:
-            assert rule.loader == original_loader
+            assert rule.evaluator == original_loader
 
     def test_destination_to_dict(self):
-
-        tpv_config = os.path.join(os.path.dirname(__file__), 'fixtures/mapping-rule-argument-based.yml')
+        tpv_config = os.path.join(
+            os.path.dirname(__file__), "fixtures/mapping-rule-argument-based.yml"
+        )
         loader = TPVConfigLoader.from_url_or_path(tpv_config)
 
         # create a destination
-        destination = loader.destinations["k8s_environment"]
+        destination = loader.config.destinations["k8s_environment"]
         # serialize the destination
-        serialized_destination = destination.to_dict()
+        serialized_destination = destination.dict()
         # deserialize the same destination
-        deserialized_destination = Destination.from_dict(loader, serialized_destination)
+        deserialized_destination = Destination(
+            evaluator=loader, **serialized_destination
+        )
         # make sure the deserialized destination is the same as the original
         self.assertEqual(deserialized_destination, destination)
 
     def test_tool_to_dict(self):
-        tpv_config = os.path.join(os.path.dirname(__file__), 'fixtures/mapping-rule-argument-based.yml')
+        tpv_config = os.path.join(
+            os.path.dirname(__file__), "fixtures/mapping-rule-argument-based.yml"
+        )
         loader = TPVConfigLoader.from_url_or_path(tpv_config)
 
         # create a tool
-        tool = loader.tools["limbo"]
+        tool = loader.config.tools["limbo"]
         # serialize the tool
-        serialized_destination = tool.to_dict()
+        serialized_tool = tool.dict()
         # deserialize the same tool
-        deserialized_destination = Tool.from_dict(loader, serialized_destination)
+        deserialized_tool = Tool(evaluator=loader, **serialized_tool)
         # make sure the deserialized tool is the same as the original
-        self.assertEqual(deserialized_destination, tool)
-
-    def test_tag_equivalence(self):
-        tag1 = Tag("tag_name", "tag_value", TagType.REQUIRE)
-        tag2 = Tag("tag_name2", "tag_value", TagType.REQUIRE)
-        tag3 = Tag("tag_name", "tag_value1", TagType.REQUIRE)
-        tag4 = Tag("tag_name", "tag_value1", TagType.PREFER)
-        same_as_tag1 = Tag("tag_name", "tag_value", TagType.REQUIRE)
-
-        self.assertEqual(tag1, tag1)
-        self.assertEqual(tag1, same_as_tag1)
-        self.assertNotEqual(tag1, tag2)
-        self.assertNotEqual(tag1, tag3)
-        self.assertNotEqual(tag1, tag4)
-        self.assertNotEqual(tag1, "hello")
+        self.assertEqual(deserialized_tool, tool)