From 22e1acab6c5c643143690179414acbb90d35bf1b Mon Sep 17 00:00:00 2001 From: Hamid Musavi Date: Wed, 3 Apr 2019 11:03:42 -0400 Subject: [PATCH 1/5] Some re-orgs to make the package work. --- .gitignore | 2 +- install.sh | 2 ++ setup.py | 2 +- test.sh | 6 ++++++ 4 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 install.sh create mode 100644 test.sh diff --git a/.gitignore b/.gitignore index 7641cf3..ad0efa1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ /.vs/ode-output-validator-library/v15/*.suo /*.pyproj /*.sln -/__pycache__/*.pyc +*.pyc diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..20c778e --- /dev/null +++ b/install.sh @@ -0,0 +1,2 @@ +#!/bin/bash +pip install --user --upgrade ./ diff --git a/setup.py b/setup.py index ad7c5b6..de3fbbc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="odevalidator", - version="0.01", + version="0.0.1", author_email="fake@email.com", description="ODE Data Validation Library", packages=['odevalidator'] diff --git a/test.sh b/test.sh new file mode 100644 index 0000000..f9341c7 --- /dev/null +++ b/test.sh @@ -0,0 +1,6 @@ +#!/bin/sh +echo "Starting all tests..." +echo "=====" +python odevalidator/__init__.py +echo "=====" +echo "Testing complete." From aee58b2902a9e156a2dfe831d468582ba20627d3 Mon Sep 17 00:00:00 2001 From: Hamid Musavi Date: Wed, 3 Apr 2019 16:12:10 -0400 Subject: [PATCH 2/5] ODE-1239 Reorged so the library can be discovered correctly --- odevalidator/__init__.py | 170 ------------------------------------- odevalidator/sequential.py | 2 +- odevalidator/validator.py | 170 +++++++++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 171 deletions(-) create mode 100644 odevalidator/validator.py diff --git a/odevalidator/__init__.py b/odevalidator/__init__.py index c6115cd..e69de29 100644 --- a/odevalidator/__init__.py +++ b/odevalidator/__init__.py @@ -1,170 +0,0 @@ -import configparser -import dateutil.parser -import json -import logging -from decimal import Decimal -import queue -from result import ValidationResult, ValidatorException -from sequential import Sequential - -TYPE_DECIMAL = 'decimal' -TYPE_ENUM = 'enum' -TYPE_TIMESTAMP = 'timestamp' -TYPE_STRING = 'string' - -class Field: - def __init__(self, field): - # extract required settings - self.path = field.get('Path') - if self.path is None: - raise ValidatorException("Missing required configuration property 'Path' for field '%s'" % field) - self.type = field.get('Type') - if self.type is None: - raise ValidatorException("Missing required configuration property 'Type' for field '%s'" % field) - - # extract constraints - upper_limit = field.get('UpperLimit') - if upper_limit is not None: - self.upper_limit = Decimal(upper_limit) - lower_limit = field.get('LowerLimit') - if lower_limit is not None: - self.lower_limit = Decimal(lower_limit) - values = field.get('Values') - if values is not None: - self.values = json.loads(values) - increment = field.get('Increment') - if increment is not None: - self.increment = int(increment) - equals_value = field.get('EqualsValue') - if equals_value is not None: - self.equals_value = str(equals_value) - - def _get_field_value(self, data): - try: - path_keys = self.path.split(".") - value = data - for key in path_keys: - value = value.get(key) - return value - except AttributeError as e: - raise ValidatorException("Could not find field with path '%s' in message: '%s'" % (self.path, data)) - - def validate(self, data): - field_value = self._get_field_value(data) - if field_value is None: - return ValidationResult(False, "Field '%s' missing" % self.path) - if field_value == "": - return ValidationResult(False, "Field '%s' empty" % self.path) - if hasattr(self, 'upper_limit') and Decimal(field_value) > self.upper_limit: - return ValidationResult(False, "Field '%s' value '%d' is greater than upper limit '%d'" % (self.path, Decimal(field_value), self.upper_limit)) - if hasattr(self, 'lower_limit') and Decimal(field_value) < self.lower_limit: - return ValidationResult(False, "Field '%s' value '%d' is less than lower limit '%d'" % (self.path, Decimal(field_value), self.lower_limit)) - if hasattr(self, 'values') and str(field_value) not in self.values: - return ValidationResult(False, "Field '%s' value '%s' not in list of known values: [%s]" % (self.path, str(field_value), ', '.join(map(str, self.values)))) - if hasattr(self, 'equals_value') and str(field_value) != str(self.equals_value): - return ValidationResult(False, "Field '%s' value '%s' did not equal expected value '%s'" % (self.path, field_value, self.equals_value)) - if hasattr(self, 'increment'): - if not hasattr(self, 'previous_value'): - self.previous_value = field_value - else: - if field_value != (self.previous_value + self.increment): - result = ValidationResult(False, "Field '%s' successor value '%d' did not match expected value '%d', increment '%d'" % (self.path, field_value, self.previous_value+self.increment, self.increment)) - self.previous_value = field_value - return result - if self.type == TYPE_TIMESTAMP: - try: - dateutil.parser.parse(field_value) - except Exception as e: - return ValidationResult(False, "Field '%s' value could not be parsed as a timestamp, error: %s" % (self.path, str(e))) - return ValidationResult(True, "") - -class TestCase: - def __init__(self, filepath): - self.config = configparser.ConfigParser() - self.config.read(filepath) - self.field_list = [] - for key in self.config.sections(): - if key == "_settings": - continue - else: - self.field_list.append(Field(self.config[key])) - - def _validate(self, data): - validations = [] - for field in self.field_list: - result = field.validate(data) - validations.append({ - 'Field': field.path, - 'Valid': result.valid, - 'Details': result.error - }) - return validations - - def validate_queue(self, msg_queue): - results = [] - msg_list = [] - while not msg_queue.empty(): - current_msg = json.loads(msg_queue.get()) - msg_list.append(current_msg) - record_id = str(current_msg['metadata']['serialId']['recordId']) - field_validations = self._validate(current_msg) - results.append({ - 'RecordID': record_id, - 'Validations': field_validations - }) - - seq = Sequential() - sorted_list = sorted(msg_list, key=lambda msg: (msg['metadata']['logFileName'], msg['metadata']['serialId']['recordId'])) - - sequential_validations = seq.perform_sequential_validations(sorted_list) - serialized = [] - for x in sequential_validations: - serialized.append({ - 'Valid': x.valid, - 'Details': x.error - }) - - results.append({ - 'RecordID': -1, - 'Validations': serialized - }) - - return {'Results': results} - -# main function using old functionality -def test(): - config_file = "samples/bsmTx.ini" - # Parse test config and create test case - validator = TestCase(config_file) - - print("[START] Beginning test routine referencing configuration file '%s'." % config_file) - - data_file = "samples/bsmTxGood.json" - results = test_file(validator, data_file) - print(results) - - data_file = "samples/bsmTxBad.json" - results = test_file(validator, data_file) - print(results) - -# main function using old functionality -def test_file(validator, data_file): - print("Testing '%s'." % data_file) - - with open(data_file) as f: - content = f.readlines() - - # remove whitespace characters like `\n` at the end of each line - content = [x.strip() for x in content] - #msgs = [json.loads(line) for line in content] - - q = queue.Queue() - for msg in content: - q.put(msg) - - results = validator.validate_queue(q) - - return results - -if __name__ == '__main__': - test() diff --git a/odevalidator/sequential.py b/odevalidator/sequential.py index 600cc50..a33ea82 100644 --- a/odevalidator/sequential.py +++ b/odevalidator/sequential.py @@ -1,7 +1,7 @@ import json import dateutil.parser import copy -from result import ValidationResult +from odevalidator.result import ValidationResult class Sequential: def __init__(self): diff --git a/odevalidator/validator.py b/odevalidator/validator.py new file mode 100644 index 0000000..e1403f5 --- /dev/null +++ b/odevalidator/validator.py @@ -0,0 +1,170 @@ +import configparser +import dateutil.parser +import json +import logging +from decimal import Decimal +import queue +from odevalidator.result import ValidationResult, ValidatorException +from odevalidator.sequential import Sequential + +TYPE_DECIMAL = 'decimal' +TYPE_ENUM = 'enum' +TYPE_TIMESTAMP = 'timestamp' +TYPE_STRING = 'string' + +class Field: + def __init__(self, field): + # extract required settings + self.path = field.get('Path') + if self.path is None: + raise ValidatorException("Missing required configuration property 'Path' for field '%s'" % field) + self.type = field.get('Type') + if self.type is None: + raise ValidatorException("Missing required configuration property 'Type' for field '%s'" % field) + + # extract constraints + upper_limit = field.get('UpperLimit') + if upper_limit is not None: + self.upper_limit = Decimal(upper_limit) + lower_limit = field.get('LowerLimit') + if lower_limit is not None: + self.lower_limit = Decimal(lower_limit) + values = field.get('Values') + if values is not None: + self.values = json.loads(values) + increment = field.get('Increment') + if increment is not None: + self.increment = int(increment) + equals_value = field.get('EqualsValue') + if equals_value is not None: + self.equals_value = str(equals_value) + + def _get_field_value(self, data): + try: + path_keys = self.path.split(".") + value = data + for key in path_keys: + value = value.get(key) + return value + except AttributeError as e: + raise ValidatorException("Could not find field with path '%s' in message: '%s'" % (self.path, data)) + + def validate(self, data): + field_value = self._get_field_value(data) + if field_value is None: + return ValidationResult(False, "Field '%s' missing" % self.path) + if field_value == "": + return ValidationResult(False, "Field '%s' empty" % self.path) + if hasattr(self, 'upper_limit') and Decimal(field_value) > self.upper_limit: + return ValidationResult(False, "Field '%s' value '%d' is greater than upper limit '%d'" % (self.path, Decimal(field_value), self.upper_limit)) + if hasattr(self, 'lower_limit') and Decimal(field_value) < self.lower_limit: + return ValidationResult(False, "Field '%s' value '%d' is less than lower limit '%d'" % (self.path, Decimal(field_value), self.lower_limit)) + if hasattr(self, 'values') and str(field_value) not in self.values: + return ValidationResult(False, "Field '%s' value '%s' not in list of known values: [%s]" % (self.path, str(field_value), ', '.join(map(str, self.values)))) + if hasattr(self, 'equals_value') and str(field_value) != str(self.equals_value): + return ValidationResult(False, "Field '%s' value '%s' did not equal expected value '%s'" % (self.path, field_value, self.equals_value)) + if hasattr(self, 'increment'): + if not hasattr(self, 'previous_value'): + self.previous_value = field_value + else: + if field_value != (self.previous_value + self.increment): + result = ValidationResult(False, "Field '%s' successor value '%d' did not match expected value '%d', increment '%d'" % (self.path, field_value, self.previous_value+self.increment, self.increment)) + self.previous_value = field_value + return result + if self.type == TYPE_TIMESTAMP: + try: + dateutil.parser.parse(field_value) + except Exception as e: + return ValidationResult(False, "Field '%s' value could not be parsed as a timestamp, error: %s" % (self.path, str(e))) + return ValidationResult(True, "") + +class TestCase: + def __init__(self, filepath): + self.config = configparser.ConfigParser() + self.config.read(filepath) + self.field_list = [] + for key in self.config.sections(): + if key == "_settings": + continue + else: + self.field_list.append(Field(self.config[key])) + + def _validate(self, data): + validations = [] + for field in self.field_list: + result = field.validate(data) + validations.append({ + 'Field': field.path, + 'Valid': result.valid, + 'Details': result.error + }) + return validations + + def validate_queue(self, msg_queue): + results = [] + msg_list = [] + while not msg_queue.empty(): + current_msg = json.loads(msg_queue.get()) + msg_list.append(current_msg) + record_id = str(current_msg['metadata']['serialId']['recordId']) + field_validations = self._validate(current_msg) + results.append({ + 'RecordID': record_id, + 'Validations': field_validations + }) + + seq = Sequential() + sorted_list = sorted(msg_list, key=lambda msg: (msg['metadata']['logFileName'], msg['metadata']['serialId']['recordId'])) + + sequential_validations = seq.perform_sequential_validations(sorted_list) + serialized = [] + for x in sequential_validations: + serialized.append({ + 'Valid': x.valid, + 'Details': x.error + }) + + results.append({ + 'RecordID': -1, + 'Validations': serialized + }) + + return {'Results': results} + +# main function using old functionality +def test(): + config_file = "samples/bsmTx.ini" + # Parse test config and create test case + validator = TestCase(config_file) + + print("[START] Beginning test routine referencing configuration file '%s'." % config_file) + + data_file = "samples/bsmTxGood.json" + results = test_file(validator, data_file) + print(results) + + data_file = "samples/bsmTxBad.json" + results = test_file(validator, data_file) + print(results) + +# main function using old functionality +def test_file(validator, data_file): + print("Testing '%s'." % data_file) + + with open(data_file) as f: + content = f.readlines() + + # remove whitespace characters like `\n` at the end of each line + content = [x.strip() for x in content] + #msgs = [json.loads(line) for line in content] + + q = queue.Queue() + for msg in content: + q.put(msg) + + results = validator.validate_queue(q) + + return results + +if __name__ == '__main__': + test() From 8e8a45004d007b8573f68fb293ac5eee042243ef Mon Sep 17 00:00:00 2001 From: Hamid Musavi Date: Wed, 3 Apr 2019 16:15:18 -0400 Subject: [PATCH 3/5] ODE-1239 fixed test script --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index f9341c7..fd662d0 100644 --- a/test.sh +++ b/test.sh @@ -1,6 +1,6 @@ #!/bin/sh echo "Starting all tests..." echo "=====" -python odevalidator/__init__.py +python odevalidator/validator.py echo "=====" echo "Testing complete." From a5930e73d5d1a344fc6f95515aac698bb1e22651 Mon Sep 17 00:00:00 2001 From: Matthew Schwartz Date: Thu, 4 Apr 2019 08:41:36 -0400 Subject: [PATCH 4/5] Initialize module imports using init script to remove namespace requirement --- odevalidator/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/odevalidator/__init__.py b/odevalidator/__init__.py index e69de29..eaf30e5 100644 --- a/odevalidator/__init__.py +++ b/odevalidator/__init__.py @@ -0,0 +1,3 @@ +from .validator import * +from .result import * +from .sequential import * From 06245a9605ff3ddf6d28adce66b457658f28ae58 Mon Sep 17 00:00:00 2001 From: Matthew Schwartz Date: Thu, 4 Apr 2019 08:45:23 -0400 Subject: [PATCH 5/5] Change validator and sequential imports to relative imports --- odevalidator/sequential.py | 2 +- odevalidator/validator.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/odevalidator/sequential.py b/odevalidator/sequential.py index a33ea82..fa7cca6 100644 --- a/odevalidator/sequential.py +++ b/odevalidator/sequential.py @@ -1,7 +1,7 @@ import json import dateutil.parser import copy -from odevalidator.result import ValidationResult +from .result import ValidationResult class Sequential: def __init__(self): diff --git a/odevalidator/validator.py b/odevalidator/validator.py index 4ef1600..925932c 100644 --- a/odevalidator/validator.py +++ b/odevalidator/validator.py @@ -5,8 +5,8 @@ from decimal import Decimal from pathlib import Path import queue -from odevalidator.result import ValidationResult, ValidatorException -from odevalidator.sequential import Sequential +from .result import ValidationResult, ValidatorException +from .sequential import Sequential TYPE_DECIMAL = 'decimal' TYPE_ENUM = 'enum' @@ -157,14 +157,14 @@ def test_file(validator, data_file): content = f.readlines() # remove whitespace characters like `\n` at the end of each line - content = [x.strip() for x in content] + content = [x.strip() for x in content] #msgs = [json.loads(line) for line in content] q = queue.Queue() for msg in content: q.put(msg) - results = validator.validate_queue(q) + results = validator.validate_queue(q) return results