Merge pull request #172 from PDXCapstoneF/bs-compliant-run

Do (limited) compliant run validation. Closes #167.

mainCLI.py

@@ -3,6 +3,7 @@
 Usage:
     mainCLI.py run [options] <config> [--props <props>]
     mainCLI.py validate [options] <config>
+    mainCLI.py compliant [options] <config>
     mainCLI.py dialogue [options]
     mainCLI.py script [options] <script> [ARG ...]
     mainCLI.py scripts [options]
@@ -119,13 +120,25 @@ def do_scripts(arguments):
     log.info("These are the scripts available to run using 'script':")
     log.info([script for script in os.listdir(join(dirname(__file__), "scripts")) if script.endswith("pl")])
 
+def do_compliant(arguments):
+    with open(arguments['<config>'], 'r') as f:
+        args = json.loads(f.read())
+    rs = run_generator.RunGenerator(**args)
+    for r in rs.runs:
+        s = benchmark_run.SpecJBBRun(**r)
+
+        log.info("validating the following run:")
+        s.dump(logging.INFO)
+        log.info("compliant? {}".format(s.compliant()))
+
 
 # dictionary of runnables
 # these are functions that take arguments from the
 # command line and do something with them.
 do = {
     'run': do_run,
     'validate': do_validate,
+    'compliant': do_compliant,
     'dialogue': do_dialogue,
     'script': do_script,
     'scripts': do_scripts,

src/benchmark_run.py

@@ -13,6 +13,7 @@
 
 from src.task_runner import TaskRunner
 from src.validate import random_run_id
+from src.compliant import compliant
 
 log = logging.getLogger(__name__)
 
@@ -47,7 +48,7 @@ def do_dry(task):
     """
     log.info("DRY: {}".format(task))
 
-class JvmRunOptions:
+class JvmRunOptions(dict):
     """
     A helper class for SpecJBBRun, to provide defaults and a way
     for lists, dict etc to be coerced into something that SpecJBBRun can work with.
@@ -61,15 +62,15 @@ def __init__(self, val=None):
         If dict: validate that the dict has the required keys, and set the internal dict to val.
         """
         if isinstance(val, str):
-            self.__dict__ = {
+            self.update({
                 "path": val,
                 "options": [],
-            }
+            })
         elif isinstance(val, list):
-            self.__dict__ = {
+            self.update({
                 "path": val[0],
                 "options": val[1:],
-            }
+            })
         elif isinstance(val, dict):
             if "path" not in val:
                 raise Exception("'path' not specified for JvmRunOptions")
@@ -80,31 +81,16 @@ def __init__(self, val=None):
             elif not isinstance(val["options"], list):
                 raise Exception("'path' must be a string")
 
-            self.__dict__ = val
+            self.update(val)
         elif val is None:
-            self.__dict__ = {
+            self.update({
                 "path": "java",
                 "options": []
-            }
+            })
         else:
             raise Exception(
                 "unrecognized type given to JvmRunOptions: {}".format(type(val)))
 
-    def __getitem__(self, name):
-        """
-        Defined so that JvmRunOptions is subscriptable.
-        """
-        return self.__dict__.__getitem__(name)
-
-    def __getattr__(self, name):
-        """
-        Defined so that JvmRunOptions can be accessed via attr names.
-        """
-        return self.__dict__.__getitem__(name)
-
-    def __repr__(self):
-        return "{}".format(self.__dict__)
-
 
 """
 These are valid SpecJBB components (what you'd pass into the '-m' flag to specjbb2015.jar).
@@ -148,40 +134,25 @@ def __init__(self, component_type, rest=None):
 
             rest["type"] = component_type
 
-            self.__dict__ = rest
+            self.update(rest)
         elif isinstance(rest, int):
-            self.__dict__ = {
+            self.update({
                 "type": component_type,
                 "count": rest,
                 "options": [],
                 "jvm_opts": []
-            }
+            })
         elif rest is None:
-            self.__dict__ = {
+            self.update({
                 "type": component_type,
                 "count": 1,
                 "options": [],
                 "jvm_opts": []
-            }
+            })
         else:
             raise Exception(
                 "Unrecognized 'rest' given to SpecJBBComponentOptions: {}".format(rest))
 
-    def __getitem__(self, name):
-        """
-        Defined so that SpecJBBComponentOptions is subscriptable.
-        """
-        return self.__dict__.__getitem__(name)
-
-    def __getattr__(self, name):
-        """
-        Defined so that SpecJBBComponentOptions can be accessed via attr names.
-        """
-        return self.__dict__.__getitem__(name)
-
-    def __repr__(self):
-        return "{}".format(self.__dict__)
-
 
 class SpecJBBRun:
     """
@@ -247,10 +218,10 @@ def __set_topology__(self, controller, backends, injectors):
             self.controller = SpecJBBComponentOptions("composite")
         else:
             self.controller = SpecJBBComponentOptions(
-                controller["type"], controller)
+                controller["type"], rest=controller)
 
-        self.backends = SpecJBBComponentOptions("backend", backends)
-        self.injectors = SpecJBBComponentOptions("txinjector", injectors)
+        self.backends = SpecJBBComponentOptions("backend", rest=backends)
+        self.injectors = SpecJBBComponentOptions("txinjector", rest=injectors)
 
     def _generate_tasks(self):
         """
@@ -411,3 +382,35 @@ def backend_run_args(self):
     def injector_run_args(self):
         """See self._full_options"""
         return self._full_options(self.injectors)
+
+    def compliant(self):
+        if not compliant(self.props):
+            self.log.error("prop file would have been NON-COMPLIANT")
+            return False
+
+        def contains_forbidden_flag(l):
+            """Returns if a list contains '-ikv'"""
+            return [match for match in l if match == "-ikv"]
+
+        if contains_forbidden_flag(self.controller["options"]):
+            self.log.error("controller arguments would have been NON-COMPLIANT")
+            return False
+
+        tasks = [task for task in self._generate_tasks()]
+
+        for task in tasks:
+            options = task.argument_list()
+
+            try:
+                jar_index = options.index("-jar")
+            except Exception:
+                continue
+
+            specjbb_options = options[jar_index+2:-1]
+
+            if contains_forbidden_flag(specjbb_options):
+                self.log.error("component arguments would have been NON-COMPLIANT")
+                return False
+
+        return True
+

src/compliant.py

@@ -0,0 +1,26 @@
+"""
+This module defines ways to validate that a run is compliant
+or not according to https://www.spec.org/jbb2015/docs/runrules.pdf.
+"""
+from schema import Schema, And, Or, Optional
+from src.validate import is_stringy
+
+def compliant(props=None):
+    """
+    Returns a boolean representing whether or not a run is compliant.
+    Props is a dictionary of key => value pairs representing SPECjbb properties.
+    """
+    if props is None:
+        return True
+
+    try:
+        return CompliantRunSchema.validate(props) is not None
+    except Exception:
+        return False
+
+CompliantRunSchema = Schema({
+    Optional("specjbb.group.count"): And(int, lambda group_count: group_count >= 1),
+    Optional("specjbb.txi.pergroup.count"): And(int, lambda injector_count: injector_count >= 1),
+    Optional("specjbb.mapreducer.pool.size"): And(int, lambda pool_size: pool_size >= 2),
+    Optional(is_stringy): object,
+    })

tests/test_benchmark_run.py

@@ -150,8 +150,58 @@ def test_run_multijvm(self):
                 self.assertTrue(filter(lambda line: "BACKEND" in line, component_invocations))
                 self.assertTrue(filter(lambda line: "TXINJECTOR" in line, component_invocations))
 
+    def test_compliant_runs_need_kit_validation(self):
+        ignore_kit_arguments = [{
+                "controller": {
+                    "type": "composite",
+                    "options": ["-ikv", "arg1", "arg2"],
+                },
+                "java": "java",
+                "jar": "env/Main.jar",
+            },
+            {
+                "controller": {
+                    "type": "multi",
+                    "options": ["-ikv", "arg1", "arg2"],
+                },
+                "java": "java",
+                "jar": "env/Main.jar",
+                },
+            { # on injector options
+                "controller": {
+                    "type": "multi",
+                    "options": ["arg1", "arg2"],
+                },
+                "backends": {
+                    "options": ["-ikv"],
+                    },
+                "java": "java",
+                "jar": "env/Main.jar",
+                },
+            { # on backend options
+                "controller": {
+                    "type": "multi",
+                    "options": ["arg1", "arg2"],
+                },
+                "injectors": {
+                    "options": ["-ikv"],
+                    },
+                "java": "java",
+                "jar": "env/Main.jar",
+
+                },
+            ]
 
 
+        for invalid_props in ignore_kit_arguments:
+            r = SpecJBBRun(**invalid_props)
+            self.assertFalse(r.compliant())
+
+    def test_valid_props_compliant(self):
+        for valid in self.valid_props:
+            r = SpecJBBRun(**valid)
+            self.assertTrue(r.compliant())
+
 class TestJvmRunOptions(unittest.TestCase):
     def test_given_none_will_fill_defaults(self):
         self.assertTrue(JvmRunOptions())
@@ -160,18 +210,14 @@ def test_given_str(self):
         java_path = "java"
         j = JvmRunOptions(java_path)
 
-        self.assertEqual(j.path, java_path)
         self.assertEqual(j["path"], java_path)
-        self.assertEqual(j.options, [])
         self.assertEqual(j["options"], [])
 
     def test_given_list(self):
         java_list = ["java", "-jar", "example_jar"]
         j = JvmRunOptions(java_list)
 
-        self.assertEqual(j.path, java_list[0])
         self.assertEqual(j["path"], java_list[0])
-        self.assertEqual(j.options, java_list[1:])
         self.assertEqual(j["options"], java_list[1:])
 
     def test_given_dict(self):
@@ -182,9 +228,7 @@ def test_given_dict(self):
 
         j = JvmRunOptions(valid)
 
-        self.assertEqual(j.path, valid["path"])
         self.assertEqual(j["path"], valid["path"])
-        self.assertEqual(j.options, valid["options"])
         self.assertEqual(j["options"], valid["options"])
 
     def test_with_dict_missing_options(self):
@@ -194,9 +238,7 @@ def test_with_dict_missing_options(self):
 
         j = JvmRunOptions(valid)
 
-        self.assertEqual(j.path, valid["path"])
         self.assertEqual(j["path"], valid["path"])
-        self.assertEqual(j.options, [])
         self.assertEqual(j["options"], [])
 
     def test_validates_dictionaries(self):

tests/test_compliant.py

@@ -0,0 +1,22 @@
+import unittest
+from src import compliant
+
+class TestComplianceProps(unittest.TestCase):
+    def test_defaults_are_compliant(self):
+        self.assertTrue(compliant.compliant())
+        self.assertTrue(compliant.compliant({}))
+
+    def test_group_count_must_be_compliant(self):
+        self.assertFalse(compliant.compliant({
+            "specjbb.group.count": -1,
+            }))
+
+    def test_transaction_injector_count_must_be_compliant(self):
+        self.assertFalse(compliant.compliant({
+            "specjbb.txi.pergroup.count": -1,
+            }))
+
+    def test_mapreducer_must_be_compliant(self):
+        self.assertFalse(compliant.compliant({
+            "specjbb.mapreducer.pool.size": 1,
+            }))