diff --git a/.gitignore b/.gitignore index 8cad575..9b8b083 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,5 @@ coverage.xml # Sphinx documentation docs/_build/ - +# IDE +.idea diff --git a/bin/csv2json b/bin/csv2json deleted file mode 100755 index 2551e7f..0000000 --- a/bin/csv2json +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python -m dsconfig.callcsv $* diff --git a/bin/xls2json b/bin/xls2json deleted file mode 100755 index 3d32bf8..0000000 --- a/bin/xls2json +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python - -from dsconfig.excel import main - -main() diff --git a/dsconfig/appending_dict/__init__.py b/dsconfig/appending_dict/__init__.py index 0ccddb8..1d17a8d 100644 --- a/dsconfig/appending_dict/__init__.py +++ b/dsconfig/appending_dict/__init__.py @@ -1,4 +1,5 @@ from collections import defaultdict, Mapping + from .caseless import CaselessDictionary @@ -32,7 +33,7 @@ def __init__(self, value={}, factory=None): self.__dict__["_factory"] = factory CaselessDictionary.__init__(self) defaultdict.__init__(self, factory) - for k, v in value.items(): + for k, v in list(value.items()): self[k] = v def __getitem__(self, key): @@ -56,9 +57,11 @@ def __setattr__(self, key, value): return self.__setitem__(key, value) def to_dict(self): - """Returns a ordinary dict version of itself""" + """ + Returns a ordinary dict version of itself + """ result = {} - for key, value in self.items(): + for key, value in list(self.items()): if isinstance(value, SetterDict): result[key] = value.to_dict() else: @@ -67,8 +70,10 @@ def to_dict(self): def merge(d, u): - "Recursively 'merge' a Mapping into another" - for k, v in u.iteritems(): + """ + Recursively 'merge' a Mapping into another + """ + for k, v in list(u.items()): if isinstance(v, Mapping): if k in d: merge(d[k], v) @@ -86,8 +91,8 @@ def list_of_strings(value): class AppendingDict(SetterDict): - - """An extra weird SetterDict where assignment adds items instead of + """ + An extra weird SetterDict where assignment adds items instead of overwriting. It also allows setting nested values using dicts (or any Mapping). Scalar values are converted into lists of strings. diff --git a/dsconfig/appending_dict/caseless.py b/dsconfig/appending_dict/caseless.py index 9aaa5ad..579e55e 100644 --- a/dsconfig/appending_dict/caseless.py +++ b/dsconfig/appending_dict/caseless.py @@ -1,10 +1,11 @@ -"""A caseless dictionary implementation.""" +""" +A caseless dictionary implementation. +""" from collections import MutableMapping class CaselessDictionary(MutableMapping): - """ A dictionary-like object which ignores but preserves the case of strings. @@ -44,8 +45,8 @@ class CaselessDictionary(MutableMapping): def __init__(self, *args, **kwargs): self.__dict__["_dict"] = {} temp_dict = dict(*args, **kwargs) - for key, value in temp_dict.iteritems(): - if isinstance(key, basestring): + for key, value in list(temp_dict.items()): + if isinstance(key, str): key = CaselessString.make_caseless(key) self._dict[key] = value @@ -72,12 +73,13 @@ def keys(self): return [str(k) for k in self._dict] def items(self): - return zip(self.keys(), self.values()) + return list(zip(list(self.keys()), list(self.values()))) class CaselessString(object): - - """A mixin to make a string subclass case-insensitive in dict lookups.""" + """ + A mixin to make a string subclass case-insensitive in dict lookups. + """ def __hash__(self): return hash(self.lower()) @@ -90,7 +92,7 @@ def __cmp__(self, other): @classmethod def make_caseless(cls, string): - if isinstance(string, unicode): + if isinstance(string, str): return CaselessUnicode(string) return CaselessStr(string) @@ -99,5 +101,5 @@ class CaselessStr(CaselessString, str): pass -class CaselessUnicode(CaselessString, unicode): +class CaselessUnicode(CaselessString, str): pass diff --git a/dsconfig/appending_dict/test_appendingdict.py b/dsconfig/appending_dict/test_appendingdict.py index 2be0067..3d8671b 100644 --- a/dsconfig/appending_dict/test_appendingdict.py +++ b/dsconfig/appending_dict/test_appendingdict.py @@ -31,52 +31,52 @@ def test_init_tiny(self): def test_init_flat(self): FLAT_DICT = {"a": 1, "b": 2, "c": 3} sd = SetterDict(FLAT_DICT) - self.assertDictEqual(sd, FLAT_DICT) + self.assertDictEqual(sd.to_dict(), FLAT_DICT) def test_init_nested(self): NESTED_DICT = {"a": {"b": 2}} sd = SetterDict(NESTED_DICT) - self.assertDictEqual(sd, NESTED_DICT) + self.assertDictEqual(sd.to_dict(), NESTED_DICT) def test_init_blended(self): BLENDED_DICT = {"a": 1, "b": {"c": 3}} sd = SetterDict(BLENDED_DICT) - self.assertDictEqual(sd, BLENDED_DICT) + self.assertDictEqual(sd.to_dict(), BLENDED_DICT) def test_init_deep(self): DEEP_DICT = {"a": {"b": {"c": {"d": 4}}}} sd = SetterDict(DEEP_DICT) - self.assertDictEqual(sd, DEEP_DICT) + self.assertDictEqual(sd.to_dict(), DEEP_DICT) def test_init_deep(self): COMPLEX_DICT = {"a": {"b": {"c": {"d": 4}}}, "e": 5} sd = SetterDict(COMPLEX_DICT) - self.assertDictEqual(sd, COMPLEX_DICT) + self.assertDictEqual(sd.to_dict(), COMPLEX_DICT) def test_setting(self): sd = SetterDict() sd["foo"] = 1 - self.assertDictEqual(sd, {"foo": 1}) + self.assertDictEqual(sd.to_dict(), {"foo": 1}) def test_setting_nested(self): sd = SetterDict() sd["foo"]["bar"] = 2 - self.assertDictEqual(sd, {"foo": {"bar": 2}}) + self.assertDictEqual(sd.to_dict(), {"foo": {"bar": 2}}) def test_setting_nested_nonempty(self): sd = SetterDict({"a": 1}) sd["foo"]["bar"] = 2 - self.assertDictEqual(sd, {"a": 1, "foo": {"bar": 2}}) + self.assertDictEqual(sd.to_dict(), {"a": 1, "foo": {"bar": 2}}) def test_setting_attr(self): sd = SetterDict({"a": 1}) sd.a = 2 - self.assertDictEqual(sd, {"a": 2}) + self.assertDictEqual(sd.to_dict(), {"a": 2}) def test_setting_attr_deep(self): sd = SetterDict() sd.a.b.c = 4 - self.assertDictEqual(sd, {"a": {"b": {"c": 4}}}) + self.assertDictEqual(sd.to_dict(), {"a": {"b": {"c": 4}}}) def test_to_dict(self): orig = {"a": {"b": ["3"], "c": {"d": ["4"]}, "e": ["1"]}} @@ -95,7 +95,7 @@ def test_keeps_original_key_case(self): sd.foo = 2 sd.baR = 3 sd.BAR = 4 - self.assertListEqual(sd.keys(), ["FoO", "baR"]) + self.assertListEqual(list(sd.keys()), ["FoO", "baR"]) class AppendingDictTestCase(unittest.TestCase): @@ -103,38 +103,38 @@ class AppendingDictTestCase(unittest.TestCase): def test_basic_appending(self): ad = AppendingDict() ad["a"] = 1 - self.assertDictEqual(ad, {"a": ['1']}) + self.assertDictEqual(ad.to_dict(), {"a": ['1']}) ad["a"] = 2 - self.assertDictEqual(ad, {"a": ['1', '2']}) + self.assertDictEqual(ad.to_dict(), {"a": ['1', '2']}) def test_deep_appending(self): ad = AppendingDict() ad["a"]["b"]["c"] = 1 ad["a"]["b"]["c"] = 2 - print type(ad["a"]["b"]) - self.assertDictEqual(ad, {"a": {"b": {"c": ['1', '2']}}}) + print((type(ad["a"]["b"]))) + self.assertDictEqual(ad.to_dict(), {"a": {"b": {"c": ['1', '2']}}}) def test_initial_setting_with_dict(self): ad = AppendingDict() ad.a = {"b": {"c": 1}} - self.assertDictEqual(ad, {"a": {"b": {"c": ["1"]}}}) + self.assertDictEqual(ad.to_dict(), {"a": {"b": {"c": ["1"]}}}) def test_deep_setting_with_dict(self): ad = AppendingDict() ad.a.b.c = 1 ad.a = {"b": {"d": 2}} - self.assertDictEqual(ad, {"a": {"b": {"c": ['1'], "d": ['2']}}}) + self.assertDictEqual(ad.to_dict(), {"a": {"b": {"c": ['1'], "d": ['2']}}}) def test_setting_with_sequence(self): ad = AppendingDict() ad.a = [1, "b"] - self.assertDictEqual(ad, {"a": ['1', 'b']}) + self.assertDictEqual(ad.to_dict(), {"a": ['1', 'b']}) def test_setting_existing_key_with_sequence(self): ad = AppendingDict() ad.a = [1, "b"] ad.a = [2, "c"] - self.assertDictEqual(ad, {"a": ['1', 'b', '2', 'c']}) + self.assertDictEqual(ad.to_dict(), {"a": ['1', 'b', '2', 'c']}) def test_error_setting_existing_subtree_with_scalar(self): ad = AppendingDict() @@ -142,6 +142,7 @@ def test_error_setting_existing_subtree_with_scalar(self): def set_subtree(): ad.a = 2 + self.assertRaises(ValueError, set_subtree) def test_setting_with_appendingdict(self): @@ -150,10 +151,12 @@ def test_setting_with_appendingdict(self): ad.a.e = 1 ad.a = ad2 self.assertDictEqual( - ad, {"a": {"b": ["3"], "c": {"d": ["4"]}, "e": ["1"]}}) + ad.to_dict(), {"a": {"b": ["3"], "c": {"d": ["4"]}, "e": ["1"]}}) def test_updating_does_not_work(self): - "Have not yet implemented this" + """ + Have not yet implemented this + """ ad = AppendingDict() d = {"a": 1, "b": {"c": 3}} self.assertRaises(NotImplementedError, ad.update(d)) @@ -161,7 +164,7 @@ def test_updating_does_not_work(self): def test_set_string_value(self): ad = AppendingDict() ad.a = "abc" - self.assertDictEqual(ad, {"a": ["abc"]}) + self.assertDictEqual(ad.to_dict(), {"a": ["abc"]}) def test_to_dict(self): orig = {"a": {"b": ["3"], "c": {"d": ["4"]}, "e": ["1"]}} diff --git a/dsconfig/callcsv.py b/dsconfig/callcsv.py deleted file mode 100644 index 606198c..0000000 --- a/dsconfig/callcsv.py +++ /dev/null @@ -1,289 +0,0 @@ -"""Provide functions to parse a callable csv file.""" - -# Imports - -import os -import sys -import csv -import json -from collections import Mapping -from importlib import import_module -from optparse import OptionParser - - -# Utils - -def special_update(d, u): - """Update nested dictionnaries while prioritizing the first argument.""" - if not (isinstance(d, Mapping) and isinstance(u, Mapping)): - return d if d is not None else u - for k, v in u.iteritems(): - d[k] = special_update(d.get(k), v) - return d - - -def max_or_none(*args): - """Maximum function considering None as a maximum value.""" - return None if None in args else max(*args) - - -def cast_list(lst): - """Convert a list of a string to the corresponding value.""" - result = [] - for value in lst: - # Integer conversion - try: - value = int(value) - except (ValueError, TypeError): - # Float conversion - try: - value = float(value) - except (ValueError, TypeError): - # Hexa conversion - try: - value = int(value, 16) - except (ValueError, TypeError): - # Ignore - pass - # Append - result.append(value) - # Return - if not result: - return "" - if len(result) == 1: - return result[0] - return tuple(result) - - -# Csv functions - -def get_column(matrix, col, start=None, stop=None, step=None): - """Get the column of a matrix, with optional range arguments.""" - return [row[col] for row in matrix][slice(start, stop, step)] - - -def get_markup_index(matrix, col, markup): - """Find a markup in a given column and return the following index.""" - for i, key in enumerate(get_column(matrix, col)): - if key == markup: - return i+1 - return None - - -def get_range_lst(matrix, col, start=0, stop=None, markup=None): - """Get a value->range dictionnary from a given column.""" - if markup: - start = max_or_none(start, get_markup_index(matrix, col, markup)) - if start is None: - return {} - result = [] - previous_key, previous_start = None, None - for i, key in enumerate(get_column(matrix, col, start, stop), start): - if key != "": - if previous_key is not None: - result.append((previous_key, previous_start, i)) - previous_key, previous_start = key, i - if previous_key is not None: - result.append((previous_key, previous_start, i+1)) - return result - - -# Callable csv functions - -def get_kwargs(matrix, start, stop): - """Get the keywords arguments between two indexes.""" - kwargs = {} - keyword_lst = get_range_lst(matrix, 2, start, stop) - for keyword, start, stop in keyword_lst: - lst = get_range_lst(matrix, 3, start, stop) - values = [key for key, _, _ in lst] - kwargs[str(keyword)] = cast_list(values) - return kwargs - - -def get_call_list(filename): - """Get the call list from a callable cls file.""" - result = [] - with open(filename) as csvfile: - reader = csv.reader(csvfile, delimiter=',') - matrix = [[value.strip() for value in row] for row in reader] - package_lst = get_range_lst(matrix, 0, markup="Package") - # Process - result = [] - for package, start, stop in package_lst: - func_lst = get_range_lst(matrix, 1, start, stop) - for func, start, stop in func_lst: - kwargs = get_kwargs(matrix, start, stop) - result.append((package, func, kwargs)) - return result - - -# Data functions - -def process_call_list(lst, skip=False, verbose=True): - """Process a given call list and return the results.""" - result = [] - errors = ImportError, ValueError, TypeError, AttributeError - for module_name, func_name, kwargs in lst: - # Build prototype - prototype = "{0}.{1}(".format(module_name, func_name) - for key, value in kwargs.items(): - prototype += '{0}={1}, '.format(key, value) - if prototype.endswith(' '): - prototype = prototype[:-2] - prototype += ')' - # Print prototype - if verbose: - print "Executing: " + prototype - # Execute - try: - module = import_module(module_name) - func = getattr(module, func_name) - value = func(**kwargs) - # Fail - except errors as exc: - if not skip: - raise exc - else: - print(exc) - # Success - else: - result.append(value) - return result - - -def join_data(lst, source=None): - """Join a list of json strings or dictionnaries into a single dict.""" - data = {} - for mapping in lst: - if isinstance(mapping, basestring): - mapping = json.loads(mapping) - special_update(data, mapping) - if source: - data['_source'] = source - return data - - -# CSV to Dict function - -def callable_csv_to_dict(filename, skip=False, verbose=True, to_json=False): - """Convert a callable csv file to a data dictionnary.""" - calls = get_call_list(filename) - if not calls: - return - strings = process_call_list(calls, skip, verbose) - data = join_data(strings, filename) - if to_json: - return json.dumps(data, indent=4, sort_keys=True) - return data - - -# Command lines arguments for configuration script - -def parse_command_line_args(desc): - """Parse arguments given in command line""" - usage = "%prog [-i INPUT] [-o OUTPUT] [-v] [-w]" - parser = OptionParser(usage=usage, description=desc, version='%prog v1.0') - - msg = "The input callable csv file" - parser.add_option('-i', '--input', metavar='IN', - type='str', help=msg, default='') - - msg = "The output tango database json file" - parser.add_option('-o', '--output', metavar='OUT', - type='str', help=msg, default='') - - msg = "Display informations" - parser.add_option('-v', '--verbose', - action="store_true", help=msg, default=False) - - msg = "Write the Tango Database" - parser.add_option('-w', '--write', - action="store_true", help=msg, default=False) - - options, args = parser.parse_args() - - if args: - msg = "No argument expected, options only.\n" - msg += "Use --help for further information." - parser.error(msg) - - return options.input, options.output, options.write, options.verbose - - -# Main function for configuration scripts - -def main(desc=None, module_name=None, function=None): - """Run the script.""" - kwargs = {} - remove = False - desc = desc or "Generate a Tango json file for a given callable csv file." - # Parse command line args - input_file, output_file, write, verbose = parse_command_line_args(desc) - # Process input file - if input_file and module_name and function: - prototype = ".".join((module_name, function.__name__)) - for module, func, keywords in get_call_list(input_file): - if module == module_name and func == function.__name__: - kwargs = keywords - if verbose: - print("'{0}' found".format(prototype)) - print("kwargs = " + str(kwargs)) - break - else: - msg = "'{0}' not found in {1}" - print msg.format(prototype, get_call_list(input_file)) - return - # Generate json file - if module_name and function: - if not input_file and verbose: - msg = 'No input file given. ' - msg += 'Default configuration will be used' - print(msg) - data = function(**kwargs) - if isinstance(data, Mapping): - string = json.dumps(data, indent=4, sort_keys=True) - elif isinstance(data, basestring): - string = data - else: - msg = "The function didn't return a valid data format.\n" - msg += "The type is {0} instead.".format(type(data)) - print(msg) - print(data) - return - elif input_file: - string = callable_csv_to_dict(input_file, True, verbose, True) - else: - print('An input file is required.') - return - # Display json file - if verbose: - print('Json string generated:') - print(string) - # Write temporary file - if output_file == "" and write: - remove = True - output_file = "temp.json" - # Write output file - if output_file: - with open(output_file, mode='w') as f: - f.write(string) - if verbose and output_file: - print('Exported to: ' + output_file) - # Write tango database - if write: - from dsconfig import configure - sys.argv = [__name__, output_file, "-w"] - configure.main() - # Remove temporary file - if remove: - os.remove(output_file) - if verbose: - print('Removed: ' + output_file) - print('OK!') - - -# Main execution - -if __name__ == "__main__": - main() diff --git a/dsconfig/configure.py b/dsconfig/configure.py index ba7d406..75e362f 100644 --- a/dsconfig/configure.py +++ b/dsconfig/configure.py @@ -1,19 +1,21 @@ -"""Functionality for configuring a Tango DB from a dsconfig file""" +""" +Functionality for configuring a Tango DB from a dsconfig file +""" from collections import defaultdict from functools import partial -import PyTango -from appending_dict.caseless import CaselessDictionary +import tango -from utils import ObjectWrapper -from tangodb import SPECIAL_ATTRIBUTE_PROPERTIES, is_protected +from .appending_dict.caseless import CaselessDictionary +from .tangodb import SPECIAL_ATTRIBUTE_PROPERTIES, is_protected +from .utils import ObjectWrapper def check_attribute_property(propname): # Is this too strict? Do we ever need non-standard attr props? if (not propname.startswith("_") - and propname not in SPECIAL_ATTRIBUTE_PROPERTIES): + and propname not in SPECIAL_ATTRIBUTE_PROPERTIES): raise KeyError("Bad attribute property name: %s" % propname) return True @@ -39,8 +41,8 @@ def update_properties(db, parent, db_props, new_props, # For attribute properties we need to go one step deeper into # the dict, since each attribute can have several properties. # A little messy, but at least it's consistent. - for attr, props in new_props.items(): - for prop, value in props.items(): + for attr, props in list(new_props.items()): + for prop, value in list(props.items()): if ignore_case: orig = CaselessDictionary(caseless_db_props.get(attr, {})).get(prop) else: @@ -49,7 +51,7 @@ def update_properties(db, parent, db_props, new_props, or check_attribute_property(prop)): added_props[attr][prop] = value removed_props = defaultdict(dict) - for attr, props in db_props.items(): + for attr, props in list(db_props.items()): for prop in props: if ignore_case: new = CaselessDictionary(new_props.get(attr, {})).get(prop) @@ -60,12 +62,12 @@ def update_properties(db, parent, db_props, new_props, removed_props[attr][prop] = value else: added_props = {} - for prop, value in new_props.items(): + for prop, value in list(new_props.items()): old_value = caseless_db_props.get(prop, []) if value and value != old_value: added_props[prop] = value removed_props = {} - for prop, value in db_props.items(): + for prop, value in list(db_props.items()): new_value = caseless_new_props.get(prop) if (new_value is None and not is_protected(prop)) or new_value == []: # empty list forces removal of "protected" properties @@ -89,22 +91,23 @@ def update_properties(db, parent, db_props, new_props, def update_server(db, server_name, server_dict, db_dict, update=False, ignore_case=False, - difactory=PyTango.DbDevInfo, strict_attr_props=True): - - """Creates/removes devices for a given server. Optionally - ignores removed devices, only adding new and updating old ones.""" + difactory=tango.DbDevInfo, strict_attr_props=True): + """ + Creates/removes devices for a given server. Optionally + ignores removed devices, only adding new and updating old ones. + """ if ignore_case: db_dict = CaselessDictionary(db_dict) - for class_name, cls in server_dict.items(): # classes + for class_name, cls in list(server_dict.items()): # classes if ignore_case: cls = CaselessDictionary(cls) removed_devices = [dev for dev in db_dict.get(class_name, {}) if dev not in cls # never remove dservers and not class_name.lower() == "dserver"] - added_devices = cls.items() + added_devices = list(cls.items()) if not update: for device_name in removed_devices: db.delete_device(device_name) @@ -131,8 +134,9 @@ def update_server(db, server_name, server_dict, db_dict, def update_device_or_class(db, name, db_dict, new_dict, cls=False, update=False, ignore_case=False, strict_attr_props=True): - - "Configure a device or a class" + """ + Configure a device or a class + """ # Note: if the "properties" key is missing, we'll just ignore any # existing properties in the DB. Ditto for attribute_properties. @@ -167,8 +171,8 @@ def update_device_or_class(db, name, db_dict, new_dict, def configure(data, dbdata, update=False, ignore_case=False, strict_attr_props=True): - - """Takes an input data dict and the relevant current DB data. Returns + """ + Takes an input data dict and the relevant current DB data. Returns the DB calls needed to bring the Tango DB to the state described by 'data'. The 'update' flag means that servers/devices are not removed, only added or changed. If the 'ignore_case' flag is True, @@ -182,8 +186,8 @@ def configure(data, dbdata, update=False, ignore_case=False, db = ObjectWrapper() - for servername, serverdata in data.get("servers", {}).items(): - for instname, instdata in serverdata.items(): + for servername, serverdata in list(data.get("servers", {}).items()): + for instname, instdata in list(serverdata.items()): dbinstdata = (dbdata.get("servers", {}) .get(servername, {}) .get(instname, {})) @@ -192,7 +196,7 @@ def configure(data, dbdata, update=False, ignore_case=False, instdata, dbinstdata, update, ignore_case, strict_attr_props=strict_attr_props) - for classname, classdata in data.get("classes", {}).items(): + for classname, classdata in list(data.get("classes", {}).items()): dbclassdata = dbdata.get("classes", {}).get(classname, {}) update_class(db, classname, dbclassdata, classdata, update=update) diff --git a/dsconfig/diff.py b/dsconfig/diff.py index abfca14..dc30ea8 100644 --- a/dsconfig/diff.py +++ b/dsconfig/diff.py @@ -1,19 +1,23 @@ -from collections import Mapping import json import sys +from collections import Mapping -from utils import green, yellow, red +from .utils import green, yellow, red def decode_pointer(ptr): - """Take a string representing a JSON pointer and return a - sequence of parts, decoded.""" + """ + Take a string representing a JSON pointer and return a + sequence of parts, decoded. + """ return [p.replace("~1", "/").replace("~0", "~") for p in ptr.split("/")] def dump_value(value): - "Make a string out of a value, for printing" + """ + Make a string out of a value, for printing + """ if value is not None: if isinstance(value, Mapping): dump = json.dumps(value, indent=4) @@ -24,8 +28,9 @@ def dump_value(value): def print_diff(dbdict, data, removes=True): - - "Print a (hopefully) human readable list of changes." + """ + Print a (hopefully) human readable list of changes. + """ # TODO: needs work, especially on multiline properties, # empty properties (should probably never be allowed but still) @@ -44,33 +49,31 @@ def print_diff(dbdict, data, removes=True): try: ptr = " > ".join(decode_pointer(d["path"])) if d["op"] == "replace": - print yellow("REPLACE:") - print yellow(ptr) + print(yellow("REPLACE:")) + print(yellow(ptr)) db_value = resolve_pointer(dbdict, d["path"]) - print red(dump_value(db_value)) - print green(dump_value(d["value"])) + print(red(dump_value(db_value))) + print(green(dump_value(d["value"]))) ops["replace"] += 1 if d["op"] == "add": - print green("ADD:") - print green(ptr) + print(green("ADD:")) + print(green(ptr)) if d["value"]: - print green(dump_value(d["value"])) + print(green(dump_value(d["value"]))) ops["add"] += 1 if removes and d["op"] == "remove": - print red("REMOVE:") - print red(ptr) + print(red("REMOVE:")) + print(red(ptr)) value = resolve_pointer(dbdict, d["path"]) if value: - print red(dump_value(value)) + print(red(dump_value(value))) ops["remove"] += 1 except JsonPointerException as e: - print " - Error parsing diff - report this!: %s" % e + print(" - Error parsing diff - report this!: %s" % e) # # The following output is a bit misleading, removing for now # print "Total: %d operations (%d replace, %d add, %d remove)" % ( # sum(ops.values()), ops["replace"], ops["add"], ops["remove"]) return diff except ImportError: - print >>sys.stderr, ("'jsonpatch' module not available - " - "no diff printouts for you! (Try -d instead.)") - - + print(("'jsonpatch' module not available - " + "no diff printouts for you! (Try -d instead.)"), file=sys.stderr) diff --git a/dsconfig/dump.py b/dsconfig/dump.py index 34614a0..8db4589 100755 --- a/dsconfig/dump.py +++ b/dsconfig/dump.py @@ -8,19 +8,19 @@ """ -from tangodb import get_servers_with_filters, get_classes_properties -from appending_dict import SetterDict -import PyTango +import tango +from .appending_dict import SetterDict +from .tangodb import get_servers_with_filters, get_classes_properties -def get_db_data(db, patterns=None, class_properties=False, **options): +def get_db_data(db, patterns=None, class_properties=False, **options): # dump TANGO database into JSON. Optionally filter which things to include # (currently only "positive" filters are possible; you can say which # servers/classes/devices to include, but you can't exclude selectively) # By default, dserver devices aren't included! - dbproxy = PyTango.DeviceProxy(db.dev_name()) + dbproxy = tango.DeviceProxy(db.dev_name()) data = SetterDict() if not patterns: @@ -40,13 +40,12 @@ def get_db_data(db, patterns=None, class_properties=False, **options): servers = get_servers_with_filters(dbproxy, **kwargs) data.servers.update(servers) if class_properties: - classes = get_classes_properties(dbproxy, server=pattern) + classes = get_classes_properties(dbproxy, server=pattern) data.classes.update(classes) return data.to_dict() def main(): - import json from optparse import OptionParser @@ -74,14 +73,14 @@ def main(): options, args = parser.parse_args() - db = PyTango.Database() + db = tango.Database() dbdata = get_db_data(db, args, properties=options.properties, class_properties=options.class_properties, attribute_properties=options.attribute_properties, aliases=options.aliases, dservers=options.dservers, subdevices=options.subdevices) - print json.dumps(dbdata, ensure_ascii=False, indent=4, sort_keys=True) + print((json.dumps(dbdata, ensure_ascii=False, indent=4, sort_keys=True))) if __name__ == "__main__": diff --git a/dsconfig/excel.py b/dsconfig/excel.py index 80a688c..6c7b584 100644 --- a/dsconfig/excel.py +++ b/dsconfig/excel.py @@ -3,17 +3,18 @@ producing a file in the TangoDB JSON format. """ -from datetime import datetime import json import os import re import sys -#from traceback import format_exc +from datetime import datetime -from utils import find_device -from appending_dict import AppendingDict -from utils import CaselessDict -from tangodb import SPECIAL_ATTRIBUTE_PROPERTIES +from .appending_dict import AppendingDict +from .tangodb import SPECIAL_ATTRIBUTE_PROPERTIES +from .utils import CaselessDict +from .utils import find_device + +# from traceback import format_exc MODE_MAPPING = CaselessDict({"ATTR": "DynamicAttributes", "CMD": "DynamicCommands", @@ -23,9 +24,11 @@ TYPE_MAPPING = CaselessDict({"INT": int, "FLOAT": float}) -def get_properties(row): - "Find property definitions on a row" +def get_properties(row): + """ + Find property definitions on a row + """ prop_dict = AppendingDict() @@ -42,7 +45,7 @@ def get_properties(row): name, value = prop.split("=") # need to decode the string, otherwise any linebreaks # will be escaped. - value = value.decode("string-escape") + value = str(value) # .decode("string-escape") # Support inline multiline properties using "\n" prop_dict[name.strip()] = [v.strip() for v in value.split("\n")] @@ -54,7 +57,7 @@ def get_properties(row): # as floats. If the number must be inserterd as an int, use the "(INT)" # modifier. There does not seem to be a way to force a numeric cell to # be interpreted as a string. - for col_name, value in row.items(): + for col_name, value in list(row.items()): match = re.match("property(?:\((.*)\))?:(.*)", col_name, re.IGNORECASE) if match and (value is not None): # protect against zero, false... type_, name = match.groups() @@ -62,7 +65,7 @@ def get_properties(row): convert = TYPE_MAPPING[type_] values = [convert(value)] else: - value = str(value).decode("string-escape") + value = str(value) values = [v.strip() for v in value.split("\n")] prop_dict[name] = values @@ -70,7 +73,6 @@ def get_properties(row): def get_attribute_properties(row): - if "attribute" in row: attribute = row["attribute"] prop_dict = AppendingDict() @@ -83,13 +85,12 @@ def get_attribute_properties(row): if name not in SPECIAL_ATTRIBUTE_PROPERTIES: raise ValueError( "'%s' is not a valid attribute property" % name) - value = value.decode("string-escape") # for linebreaks prop_dict[name.strip()] = [v.strip() for v in value.split("\n")] except ValueError: raise ValueError("could not parse AttributeProperties") - for col_name, value in row.items(): + for col_name, value in list(row.items()): match = re.match("attrprop:(.*)", col_name, re.IGNORECASE) if match and value: name, = match.groups() @@ -97,7 +98,7 @@ def get_attribute_properties(row): if name not in SPECIAL_ATTRIBUTE_PROPERTIES: raise ValueError("'%s' it not a valid attribute property" % name) - value = str(value).decode("string-escape") + value = str(value) values = [v.strip() for v in value.split("\n")] prop_dict[name] = values @@ -105,7 +106,9 @@ def get_attribute_properties(row): def get_dynamic(row): - "Find dynamic definitions on a row" + """ + Find dynamic definitions on a row + """ prop_dict = AppendingDict() try: @@ -127,17 +130,22 @@ def get_dynamic(row): def make_db_name(name): - "convert a Space Separated Name into a lowercase, underscore_separated_name" + """ + Convert a Space Separated Name into a lowercase, underscore_separated_name + """ return name.strip().lower().replace(" ", "_") def check_formula(formula): - "Syntax check a dynamic formula." + """ + Syntax check a dynamic formula. + """ compile(formula, "", "single") def check_device_format(devname): - """Verify that a device name is of the correct form (three parts + """ + Verify that a device name is of the correct form (three parts separated by slashes, only letters, numbers, dashes and underscores allowed.) Note: We could put more logic here to make device names conform to a standard. @@ -148,14 +156,17 @@ def check_device_format(devname): def format_server_instance(row): - "Format a server/instance string" + """ + Format a server/instance string + """ # TODO: handle numeric instance names? They tend to turn up as floats... return "%s/%s" % (row["server"], row["instance"]) def convert(rows, definitions, skip=True, dynamic=False, config=False): - - "Update a dict of definitions from data" + """ + Update a dict of definitions from data + """ errors = [] column_names = rows[0] @@ -188,8 +199,8 @@ def handle_error(i, msg): # full device definition # target is "lazily" evaluated, so that we don't create # an empty dict if it turns out there are no members - target = lambda: definitions.servers[row["server"]][row["instance"]]\ - [row["class"]][row["device"]] + target = lambda: definitions.servers[row["server"]][row["instance"]] \ + [row["class"]][row["device"]] else: # don't know if the device is already defined target = lambda: find_device(definitions, row["device"])[0] @@ -214,7 +225,7 @@ def handle_error(i, msg): target().properties = props except KeyError as ke: - #handle_error(i, "insufficient data (%s)" % ke) + # handle_error(i, "insufficient data (%s)" % ke) pass except ValueError as ve: handle_error(i, "Error: %s" % ve) @@ -229,15 +240,16 @@ def handle_error(i, msg): def print_errors(errors): if errors: - print >> sys.stderr, "%d lines skipped" % len(errors) + print("%d lines skipped" % len(errors), file=sys.stderr) for err in errors: line, msg = err - print >> sys.stderr, "%d: %s" % (line + 1, msg) + print("%d: %s" % (line + 1, msg), file=sys.stderr) def xls_to_dict(xls_filename, pages=None, skip=False): - - """Make JSON out of an XLS sheet of device definitions.""" + """ + Make JSON out of an XLS sheet of device definitions. + """ import xlrd @@ -256,10 +268,10 @@ def xls_to_dict(xls_filename, pages=None, skip=False): for page in pages: - print >>sys.stderr, "\nPage: %s" % page + print("\nPage: %s" % page, file=sys.stderr) sheet = xls.sheet_by_name(page) rows = [sheet.row_values(i) - for i in xrange(sheet.nrows)] + for i in range(sheet.nrows)] if not rows: continue # ignore empty pages errors = convert(rows, definitions, skip=skip, @@ -271,19 +283,21 @@ def xls_to_dict(xls_filename, pages=None, skip=False): def get_stats(defs): - "Calculate some numbers" + """ + Calculate some numbers + """ servers = set() instances = set() classes = set() devices = set() - for server, instances in defs.servers.items(): + for server, instances in list(defs.servers.items()): servers.add(server) instances.update(instances) - for clsname, devs in instances.items(): + for clsname, devs in list(instances.items()): classes.add(clsname) - for devname, dev in devs.items(): + for devname, dev in list(devs.items()): devices.add(devname) return {"servers": len(servers), "instances": len(instances), @@ -320,15 +334,15 @@ def main(): data.update(metadata) if not options.test: - print json.dumps(data, indent=4) + print(json.dumps(data, indent=4)) outfile = open('config.json', 'w') json.dump(data, outfile, indent=4) stats = get_stats(data) - print >>sys.stderr, ("\n" - "Total: %(servers)d servers, %(instances)d instances, " - "%(classes)d classes and %(devices)d devices defined.") % stats + print(("\n" + "Total: %(servers)d servers, %(instances)d instances, " + "%(classes)d classes and %(devices)d devices defined.") % stats, file=sys.stderr) if __name__ == "__main__": diff --git a/dsconfig/excel_alarms_userdef.py b/dsconfig/excel_alarms_userdef.py deleted file mode 100644 index 1c0b475..0000000 --- a/dsconfig/excel_alarms_userdef.py +++ /dev/null @@ -1,60 +0,0 @@ -from collections import defaultdict -import json - -import xlrd - - -class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev, al_rec): - print inst, dev, al_name, al_cond, al_desc, al_sev, al_rec - devdict = sdict.servers["PyAlarm/"+inst]["PyAlarm"][dev] - if "AlarmList" not in devdict.properties: - devdict.properties["AlarmList"] = [] - devdict.properties["AlarmList"].append(al_name+":"+al_cond) - if "AlarmDescriptions" not in devdict.properties: - devdict.properties["AlarmDescriptions"] = [] - devdict.properties["AlarmDescriptions"].append(al_name+":"+al_desc) - if "AlarmSeverities" not in devdict.properties: - devdict.properties["AlarmSeverities"] = [] - devdict.properties["AlarmSeverities"].append(al_name+":"+al_sev) - if "AlarmReceivers" not in devdict.properties: - devdict.properties["AlarmReceivers"] = [] - devdict.properties["AlarmReceivers"].append(al_name+":"+al_rec) - -def xls_to_dict(xls_filename): - json_dict = SuperDict() - xls = xlrd.open_workbook(xls_filename) - sheet = xls.sheet_by_name("Alarms") - for line in xrange(1, sheet.nrows): - # above skips row 0 (col headers) - # look at all rows but only read those with entry in first col - if sheet.row_values(line)[0] is not "": - print "IN LINE ", line, sheet.row_values(line)[0] - dev_config = sheet.row_values(line) - add_device(json_dict, *dev_config[:7]) - return json_dict - -def main(): - import sys - data = xls_to_dict(sys.argv[1]) - print json.dumps(data, indent=4) - outfile = open('pyalarm_config.json', 'w') - json.dump(data, outfile, indent=4) - -if __name__ == "__main__": - main() - - - diff --git a/dsconfig/excel_alarms_vac.py b/dsconfig/excel_alarms_vac.py deleted file mode 100644 index e044153..0000000 --- a/dsconfig/excel_alarms_vac.py +++ /dev/null @@ -1,118 +0,0 @@ -from collections import defaultdict -import json - -import xlrd - - -class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev): - print inst, dev, al_name, al_cond, al_desc, al_sev - # devdict = sdict.servers["PyAlarm"]["PyAlarm/"+inst]["PyAlarm"][dev] - devdict = sdict.servers["PyAlarm/"+inst]["PyAlarm"][dev] - if "AlarmList" not in devdict.properties: - devdict.properties["AlarmList"] = [] - devdict.properties["AlarmList"].append(al_name+":"+al_cond) - if "AlarmDescriptions" not in devdict.properties: - devdict.properties["AlarmDescriptions"] = [] - devdict.properties["AlarmDescriptions"].append(al_name+":"+al_desc) - #hard code severity and some other things - only one per instance - if "AlarmSeverities" not in devdict.properties: - devdict.properties["AlarmSeverities"] = [] - devdict.properties["AlarmSeverities"].append(al_name+":"+al_sev) - if "AlarmReceivers" not in devdict.properties: - devdict.properties["AlarmReceivers"] = [] - devdict.properties["AlarmReceivers"].append(al_name+":HTML") - #hard code severity and some other things - only one per instance - if "AlarmThreshold" not in devdict.properties: - devdict.properties["AlarmThreshold"] = [] - devdict.properties["AlarmThreshold"] = [1] - if "LogFile" not in devdict.properties: - devdict.properties["LogFile"] = [] - devdict.properties["LogFile"]= ["/tmp/pjb/log"] - if "HtmlFolder" not in devdict.properties: - devdict.properties["HtmlFolder"] = [] - devdict.properties["HtmlFolder"] = ["/tmp/pjb"] - if "PollingPeriod" not in devdict.properties: - devdict.properties["PollingPeriod"] = [] - devdict.properties["PollingPeriod"] = [5] - if "MaxMessagesPerAlarm" not in devdict.properties: - devdict.properties["MaxMessagesPerAlarm"] = [] - devdict.properties["MaxMessagesPerAlarm"]= [1] - if "AutoReset" not in devdict.properties: - devdict.properties["AutoReset"] = [] - devdict.properties["AutoReset"]= [0] - if "StartupDelay" not in devdict.properties: - devdict.properties["StartupDelay"] = [] - devdict.properties["StartupDelay"]= [0] - -def xls_to_dict(xls_filename): - json_dict = SuperDict() - xls = xlrd.open_workbook(xls_filename) - - for i in range (0,2): - - if i==1: - sheet = xls.sheet_by_name("Alarms") - nature="interlock" - else: - sheet = xls.sheet_by_name("Warnings") - nature="bypass" - - last_server="" - last_device="" - last_name="" - last_sev="" - summary_condition="" - for line in xrange(1, sheet.nrows): - # above skips row 0 (col headers) - # look at all rows but only read those with entry in first col - if sheet.row_values(line)[0] is not "": - print "IN LINE ", line, sheet.row_values(line)[0] - #assume that if you get to a new device, it means a new section of vacuum - #in this case, need to make a final alarm which is or of all others - dev_config = sheet.row_values(line) - print dev_config, dev_config[3].rsplit("/",1)[0] - if dev_config[1] != last_device or dev_config[0]=="end": - print "START NEW SECTION", dev_config[1] - print "---- ADDING TO JSON summary ", summary_condition, last_name - if summary_condition!="": - add_device(json_dict,last_server,last_device,last_name.rsplit("_",1)[0],summary_condition,"at least one vac. %s in section %s" %(nature,last_name.rsplit("__",2)[0]),last_sev) - last_server = dev_config[0] - last_device = dev_config[1] - last_name = dev_config[2] - last_sev = dev_config[5] - summary_condition="" - if summary_condition == "": - summary_condition = summary_condition + dev_config[3] - else: - summary_condition = summary_condition + " or " + dev_config[3] - - if dev_config[0]!="end": - add_device(json_dict, *dev_config[:6]) - - return json_dict - -def main(): - import sys - data = xls_to_dict(sys.argv[1]) - #print json.dumps(data, indent=4) - outfile = open('alarms_vac.json', 'w') - json.dump(data, outfile, indent=4) - -if __name__ == "__main__": - main() - - - diff --git a/dsconfig/filtering.py b/dsconfig/filtering.py index cc18697..18e7b91 100644 --- a/dsconfig/filtering.py +++ b/dsconfig/filtering.py @@ -1,6 +1,6 @@ import re -from appending_dict import merge +from .appending_dict import merge def filter_nested_dict(node, pattern, depth, level=0, invert=False): @@ -9,13 +9,13 @@ def filter_nested_dict(node, pattern, depth, level=0, invert=False): at the given depth. """ if level == depth: - return dict((key, value) for key, value in node.iteritems() + return dict((key, value) for key, value in list(node.items()) if (not invert and pattern.match(key)) or (invert and not pattern.match(key))) else: dupe_node = {} - for key, val in node.iteritems(): - cur_node = filter_nested_dict(val, pattern, depth, level+1, + for key, val in list(node.items()): + cur_node = filter_nested_dict(val, pattern, depth, level + 1, invert) if cur_node: dupe_node[key] = cur_node @@ -23,8 +23,8 @@ def filter_nested_dict(node, pattern, depth, level=0, invert=False): def filter_config(data, filters, levels, invert=False): - - """Filter the given config data according to a list of filters. + """ + Filter the given config data according to a list of filters. May be a positive filter (i.e. includes only matching things) or inverted (i.e. includes everything that does not match). The _levels_ argument is used to find at what depth in the data @@ -41,7 +41,7 @@ def filter_config(data, filters, levels, invert=False): srv, inst = [re.compile(r, flags=re.IGNORECASE) for r in regex.split("/")] servers = filter_nested_dict(data, srv, 0) - for k, v in servers.items(): + for k, v in list(servers.items()): tmp = filter_nested_dict(v, inst, 0) if tmp: filtered[k] = tmp @@ -53,7 +53,7 @@ def filter_config(data, filters, levels, invert=False): "Bad filter '%s'; should be ':'" % fltr) except KeyError: raise ValueError("Bad filter '%s'; term should be one of: %s" - % (fltr, ", ".join(levels.keys()))) + % (fltr, ", ".join(list(levels.keys())))) except re.error as e: raise ValueError("Bad regular expression '%s': %s" % (fltr, e)) if invert: diff --git a/dsconfig/formatting.py b/dsconfig/formatting.py index 75772fe..515aa76 100644 --- a/dsconfig/formatting.py +++ b/dsconfig/formatting.py @@ -1,20 +1,19 @@ "This module concerns the dsconfig JSON file format" -import sys import json +import sys from copy import deepcopy, copy from os import path -from appending_dict import AppendingDict, SetterDict - -import PyTango +import tango +from .appending_dict import SetterDict SERVERS_LEVELS = {"server": 0, "instance": 1, "class": 2, "device": 3, "property": 5} CLASSES_LEVELS = {"class": 1, "property": 2} module_path = path.dirname(path.realpath(__file__)) -SCHEMA_FILENAME = path.join(module_path, "schema/schema2.json") #rename +SCHEMA_FILENAME = path.join(module_path, "schema/schema2.json") # rename # functions to decode unicode JSON (PyTango does not like unicode strings) @@ -22,7 +21,7 @@ def decode_list(data): rv = [] for item in data: - if isinstance(item, unicode): + if isinstance(item, str): item = str(item.encode('utf-8')) elif isinstance(item, list): item = decode_list(item) @@ -34,10 +33,10 @@ def decode_list(data): def decode_dict(data): rv = {} - for key, value in data.iteritems(): - if isinstance(key, unicode): + for key, value in data.items(): + if isinstance(key, str): key = str(key.encode('utf-8')) - if isinstance(value, unicode): + if isinstance(value, str): value = str(value.encode('utf-8')) elif isinstance(value, list): value = decode_list(value) @@ -48,29 +47,32 @@ def decode_dict(data): def validate_json(data): - """Validate that a given dict is of the right form""" + """ + Validate that a given dict is of the right form + """ try: from jsonschema import validate, exceptions with open(SCHEMA_FILENAME) as schema_json: schema = json.load(schema_json) validate(data, schema) except ImportError: - print >>sys.stderr, ("WARNING: 'jsonschema' not installed, could not " - "validate json file. You're on your own.") + print(("WARNING: 'jsonschema' not installed, could not " + "validate json file. You're on your own."), file=sys.stderr) except exceptions.ValidationError as e: - print >>sys.stderr, "ERROR: JSON data does not match schema: %s" % e + print("ERROR: JSON data does not match schema: %s" % e, file=sys.stderr) sys.exit(1) def load_json(f): - return json.load(f, object_hook=decode_dict) + return json.load(f) def expand_config(config): - - """Takes a configuration dict and expands it into the canonical + """ + Takes a configuration dict and expands it into the canonical format. This currently means that the server instance level is - split into a server and an instance level.""" + split into a server and an instance level. + """ expanded = deepcopy(config) if "servers" in config: @@ -86,17 +88,19 @@ def expand_config(config): def clean_metadata(data): - "Removes any keys in the data that begin with '_'" + """ + Removes any keys in the data that begin with '_' + """ tmp = copy(data) - for key in tmp.keys(): + for key in list(tmp.keys()): if key.startswith("_"): tmp.pop(key, None) return tmp def normalize_config(config): - - """Take a 'loose' config and return a new config that conforms to the + """ + Take a 'loose' config and return a new config that conforms to the DSConfig format. Current transforms: @@ -118,11 +122,11 @@ def normalize_config(config): if "classes" in old_config: new_config.classes = old_config["classes"] if "devices" in old_config: - db = PyTango.Database() - for device, props in old_config["devices"].items(): + db = tango.Database() + for device, props in list(old_config["devices"].items()): try: info = db.get_device_info(device) - except PyTango.DevFailed as e: + except tango.DevFailed as e: sys.exit("Can't reconfigure device %s: %s" % (device, str(e[0].desc))) srv, inst = info.ds_full_name.split("/") new_config.servers[srv][inst][info.class_name][device] = props diff --git a/dsconfig/json2tango.py b/dsconfig/json2tango.py index 1abcd20..241580f 100755 --- a/dsconfig/json2tango.py +++ b/dsconfig/json2tango.py @@ -5,78 +5,27 @@ optionally be run. """ +import json import sys import time -import json from optparse import OptionParser from tempfile import NamedTemporaryFile -import PyTango +import tango +from dsconfig.appending_dict.caseless import CaselessDictionary from dsconfig.configure import configure +from dsconfig.dump import get_db_data from dsconfig.filtering import filter_config from dsconfig.formatting import (CLASSES_LEVELS, SERVERS_LEVELS, load_json, normalize_config, validate_json, clean_metadata) +from dsconfig.output import show_actions from dsconfig.tangodb import summarise_calls, get_devices_from_dict -from dsconfig.dump import get_db_data -from dsconfig.utils import green, red, yellow, progressbar, no_colors from dsconfig.utils import SUCCESS, ERROR, CONFIG_APPLIED, CONFIG_NOT_APPLIED -from dsconfig.output import show_actions -from dsconfig.appending_dict.caseless import CaselessDictionary - - -def main(): - - usage = "Usage: %prog [options] JSONFILE" - parser = OptionParser(usage=usage) - - parser.add_option("-w", "--write", dest="write", action="store_true", - help="write to the Tango DB", metavar="WRITE") - parser.add_option("-u", "--update", dest="update", action="store_true", - help="don't remove things, only add/update", - metavar="UPDATE") - parser.add_option("-c", "--case-sensitive", dest="case_sensitive", - action="store_true", - help=("Don't ignore the case of server, device, " - "attribute and property names"), - metavar="CASESENSITIVE") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose", default=True, - help="don't print actions to stderr") - parser.add_option("-o", "--output", dest="output", action="store_true", - help="Output the relevant DB state as JSON.") - parser.add_option("-p", "--input", dest="input", action="store_true", - help="Output the input JSON (after filtering).") - parser.add_option("-d", "--dbcalls", dest="dbcalls", action="store_true", - help="print out all db calls.") - parser.add_option("-v", "--no-validation", dest="validate", default=True, - action="store_false", help=("Skip JSON validation")) - parser.add_option("-s", "--sleep", dest="sleep", default=0.01, - type="float", - help=("Number of seconds to sleep between DB calls")) - parser.add_option("-n", "--no-colors", - action="store_true", dest="no_colors", default=False, - help="Don't print colored output") - parser.add_option("-i", "--include", dest="include", action="append", - help=("Inclusive filter on server configutation")) - parser.add_option("-x", "--exclude", dest="exclude", action="append", - help=("Exclusive filter on server configutation")) - parser.add_option("-a", "--no-strict-check", dest="nostrictcheck", - default=False, action="store_true", - help="Disable strick attribute property checking") - parser.add_option("-I", "--include-classes", dest="include_classes", - action="append", - help=("Inclusive filter on class configuration")) - parser.add_option("-X", "--exclude-classes", dest="exclude_classes", - action="append", - help=("Exclusive filter on class configuration")) +from dsconfig.utils import green, red, yellow, progressbar, no_colors - parser.add_option( - "-D", "--dbdata", - help="Read the given file as DB data instead of using the actual DB", - dest="dbdata") - options, args = parser.parse_args() +def json_to_tango(options, args): if options.no_colors: no_colors() @@ -114,19 +63,18 @@ def main(): data.get("classes", {}), options.exclude_classes, CLASSES_LEVELS, invert=True) except ValueError as e: - print >>sys.stderr, red("Filter error:\n%s" % e) + print(red("Filter error:\n%s" % e), file=sys.stderr) sys.exit(ERROR) if not any(k in data for k in ("devices", "servers", "classes")): sys.exit(ERROR) - if options.input: - print json.dumps(data, indent=4) + print(json.dumps(data, indent=4)) return # check if there is anything in the DB that will be changed or removed - db = PyTango.Database() + db = tango.Database() if options.dbdata: with open(options.dbdata) as f: original = json.loads(f.read()) @@ -147,7 +95,7 @@ def main(): in get_devices_from_dict(original["servers"]) }) collisions = {} - for dev, (srv, inst, cls) in devices.items(): + for dev, (srv, inst, cls) in list(devices.items()): if dev in orig_devices: server = "{}/{}".format(srv, inst) osrv, oinst, ocls = orig_devices[dev] @@ -173,25 +121,25 @@ def main(): if options.verbose: progressbar(i, len(dbcalls), 20) getattr(db, method)(*args, **kwargs) - print + print() # optionally dump some information to stdout if options.output: - print json.dumps(original, indent=4) + print(json.dumps(original, indent=4)) if options.dbcalls: - print >>sys.stderr, "Tango database calls:" + print("Tango database calls:", file=sys.stderr) for method, args, kwargs in dbcalls: - print >>sys.stderr, method, args + print(method, args, file=sys.stderr) # Check for moved devices and remove empty servers empty = set() - for srvname, devs in collisions.items(): + for srvname, devs in list(collisions.items()): if options.verbose: srv, inst = srvname.split("/") for cls, dev in devs: - print >>sys.stderr, red("MOVED (because of collision):"), dev - print >>sys.stderr, " Server: ", "{}/{}".format(srv, inst) - print >>sys.stderr, " Class: ", cls + print(red("MOVED (because of collision):"), dev, file=sys.stderr) + print(" Server: ", "{}/{}".format(srv, inst), file=sys.stderr) + print(" Class: ", cls, file=sys.stderr) if len(db.get_device_class_list(srvname)) == 2: # just dserver empty.add(srvname) if options.write: @@ -199,34 +147,90 @@ def main(): # finally print out a brief summary of what was done if dbcalls: - print - print >>sys.stderr, "Summary:" - print >>sys.stderr, "\n".join(summarise_calls(dbcalls, original)) + print() + print("Summary:", file=sys.stderr) + print("\n".join(summarise_calls(dbcalls, original)), file=sys.stderr) if collisions: servers = len(collisions) - devices = sum(len(devs) for devs in collisions.values()) - print >>sys.stderr, red("Move %d devices from %d servers." % - (devices, servers)) + devices = sum(len(devs) for devs in list(collisions.values())) + print(red("Move %d devices from %d servers." % + (devices, servers)), file=sys.stderr) if empty and options.verbose: - print >>sys.stderr, red("Removed %d empty servers." % len(empty)) + print(red("Removed %d empty servers." % len(empty)), file=sys.stderr) if options.write: - print >>sys.stderr, red("\n*** Data was written to the Tango DB ***") + print(red("\n*** Data was written to the Tango DB ***"), file=sys.stderr) with NamedTemporaryFile(prefix="dsconfig-", suffix=".json", delete=False) as f: - f.write(json.dumps(original, indent=4)) - print >>sys.stderr, ("The previous DB data was saved to %s" % - f.name) + f.write(json.dumps(original, indent=4).encode()) + print(("The previous DB data was saved to %s" % + f.name), file=sys.stderr) sys.exit(CONFIG_APPLIED) else: - print >>sys.stderr, yellow( - "\n*** Nothing was written to the Tango DB (use -w) ***") + print(yellow( + "\n*** Nothing was written to the Tango DB (use -w) ***"), file=sys.stderr) sys.exit(CONFIG_NOT_APPLIED) else: - print >>sys.stderr, green("\n*** No changes needed in Tango DB ***") + print(green("\n*** No changes needed in Tango DB ***"), file=sys.stderr) sys.exit(SUCCESS) +def main(): + + usage = "Usage: %prog [options] JSONFILE" + parser = OptionParser(usage=usage) + + parser.add_option("-w", "--write", dest="write", action="store_true", + help="write to the Tango DB", metavar="WRITE") + parser.add_option("-u", "--update", dest="update", action="store_true", + help="don't remove things, only add/update", + metavar="UPDATE") + parser.add_option("-c", "--case-sensitive", dest="case_sensitive", + action="store_true", + help=("Don't ignore the case of server, device, " + "attribute and property names"), + metavar="CASESENSITIVE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print actions to stderr") + parser.add_option("-o", "--output", dest="output", action="store_true", + help="Output the relevant DB state as JSON.") + parser.add_option("-p", "--input", dest="input", action="store_true", + help="Output the input JSON (after filtering).") + parser.add_option("-d", "--dbcalls", dest="dbcalls", action="store_true", + help="print out all db calls.") + parser.add_option("-v", "--no-validation", dest="validate", default=True, + action="store_false", help=("Skip JSON validation")) + parser.add_option("-s", "--sleep", dest="sleep", default=0.01, + type="float", + help=("Number of seconds to sleep between DB calls")) + parser.add_option("-n", "--no-colors", + action="store_true", dest="no_colors", default=False, + help="Don't print colored output") + parser.add_option("-i", "--include", dest="include", action="append", + help=("Inclusive filter on server configutation")) + parser.add_option("-x", "--exclude", dest="exclude", action="append", + help=("Exclusive filter on server configutation")) + parser.add_option("-a", "--no-strict-check", dest="nostrictcheck", + default=False, action="store_true", + help="Disable strick attribute property checking") + parser.add_option("-I", "--include-classes", dest="include_classes", + action="append", + help=("Inclusive filter on class configuration")) + parser.add_option("-X", "--exclude-classes", dest="exclude_classes", + action="append", + help=("Exclusive filter on class configuration")) + + parser.add_option( + "-D", "--dbdata", + help="Read the given file as DB data instead of using the actual DB", + dest="dbdata") + + options, args = parser.parse_args() + + json_to_tango(options, args) + + if __name__ == "__main__": main() diff --git a/dsconfig/magnets2json.py b/dsconfig/magnets2json.py deleted file mode 100644 index d417294..0000000 --- a/dsconfig/magnets2json.py +++ /dev/null @@ -1,883 +0,0 @@ -#!/usr/bin/env python -# "$Name: $"; -# "$Header: $"; -#============================================================================= -# -# file : lattice2json.py -# -# description : Python source for the lattice2json that is a tool to generate -# a json file from an elegant lattice file -# The json file can then be uploaded to a Tango database -# -# project : Virtual Accelerator -# -# $Author: $ -# -# $Revision: $ -# -# $Log: $ -# -# copyleft : Solaris/MAX IV -# Krakow,PL/Lund,SE# -# - -from collections import defaultdict -import json -import os -import io -import sys -import re -from TangoProperties import TANGO_PROPERTIES -from PowerSupplyMap import POWER_SUPPLY_MAP -import copy -import numpy as np - -cirlist = [] - - -class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -class LatticeFileItem: - ''' ''' - itemName = "" - itemType = '' - parameters = {} - properties = {} - alpars= {} - - def __init__(self, _line=''): - ''' - Construct an object parsing a _line from a lattice file - ''' - self.psparameters= {} - self.parameters= {} - self.properties= {} - - - # find a name - colon_pos = _line.find(':') - self.itemName = _line[:colon_pos].lstrip().rstrip().upper() - - # what left to be parsed - line_left = _line[colon_pos + 1:].lstrip() - - # find a type - param_name = '' # the first item after a colon could be also a parameter name, like for a line element - eq_pos = line_left.find('=') - comma_pos = line_left.find(',') - # let it work even there are no parameters defined - only element type - if eq_pos < 0: eq_pos = len(line_left) - if comma_pos < 0: comma_pos = len(line_left) - # so, we could read an element type - self.itemType = line_left[:min(comma_pos, eq_pos)].rstrip().lower() - - # this is waiting to be processed: - line_left = line_left[comma_pos + 1:].lstrip() - - # if the element type is also parameter name state this - if eq_pos < comma_pos: param_name = self.itemType - - # parse the rest for parameters - while line_left != '': - if param_name != '': - # searching for a value - param_value = '' - if line_left[0] == '(': - # value in brackets (may contain commas) - bracket_pos = line_left.index(')', 1) # will rise an exception in case of badly formated line - # so, the value is (including brackets): - param_value = line_left[:bracket_pos + 1] - # this is what left to be parsed - line_left = line_left[bracket_pos + 1:].lstrip() - - elif line_left[0] == '\"': - # value in quotes (could contain commas) - quote_pos = line_left.index('\"', 1) # will rise an exception in case of badly formated line - # so, the value is (including quote): - param_value = line_left[:quote_pos + 1] - - # this is what left to be parsed - line_left = line_left[quote_pos + 1:].lstrip() - else: - # typical case - the value between an equal and a comma characters - comma_pos = line_left.find(',') - if comma_pos < 0: comma_pos = len(line_left) - # a value, here you are - param_value = line_left[:comma_pos].rstrip() - # the following left to be parsed - line_left = line_left[comma_pos + 1:].lstrip() - # store the parameter with the corresponding value - self.parameters[param_name] = param_value - # PJB reset name back to empty here to find next parameter!(to enter else below) - param_name='' - else: - # searching for a parameter - eq_pos = line_left.find('=') - if eq_pos < 0: eq_pos = len(line_left) - # allow value-less parameters - comma_pos = line_left.find(',') - if comma_pos < 0: comma_pos = len(line_left) - # so we know where to find parameter name - param_name = line_left[:min(eq_pos, comma_pos)].rstrip().lower() - # if the parameter has no value add it directly to the dictionary - if comma_pos <= eq_pos: - self.parameters[param_name] = '' - param_name = '' - # this is what left to be parsed - line_left = line_left[min(eq_pos, comma_pos) + 1:].lstrip() - - - def handle_circuit_name(self,itemName,endnum): - - endname="" - #hack for bc1 - if "QD" in itemName and "BC1" in itemName: - endname = "CRQM-" + endnum - # - #hack for bc2 - elif "QF" in itemName and "BC2" in itemName and "3" not in itemName and "4" not in itemName and "5" not in itemName: - endname = "CRQM-" + endnum - elif "QF" in itemName and "BC2" in itemName and ("3" in itemName or "5" in itemName): - endname = "CRQ1-01" - elif "QF" in itemName and "BC2" in itemName and "4" in itemName: - endname = "CRQ2-01" - # - elif "Q" in itemName: - endname = "CRQ-" + endnum - elif "CO" in itemName and "X" in itemName: - endname = "CRCOX-" + endnum - elif "CO" in itemName and "Y" in itemName: - endname = "CRCOY-" + endnum - elif "DI" in itemName: - endname = "CRDI-" + endnum - print "dealing with endname ", endname - elif "SX" in itemName: - endname = "CRSX-" + endnum - elif "SOL" in itemName: - endname = "CRSOL-" + endnum - elif "SM" in itemName: - endname = "CRSM-" + endnum - else: - sys.exit("Cannot convert circuit name" + itemName) - - if "/" in endname: #in case did not end with number, endname will be some */*/ by mistake - endname=endname.split("-")[0]+"-01" - - return endname - - - def config_alarms(self,pyalarm,alname,alsev,aldesc,pyattname,key): - - alrec = alname+":"+"HTML" - - if "AlarmList" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmList"] = [] - self.alpars[pyalarm]['AlarmList'].append(alname+":"+pyattname+"/"+key) - - if "AlarmSeverities" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmSeverities"] = [] - self.alpars[pyalarm]['AlarmSeverities'].append(alsev) - - if "AlarmDescriptions" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmDescriptions"] = [] - self.alpars[pyalarm]['AlarmDescriptions'].append(aldesc) - - if "AlarmReceivers" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmReceivers"] = [] - self.alpars[pyalarm]['AlarmReceivers'].append(alrec) - - if "StartupDelay" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["StartupDelay"] = ["0"] - - if "AutoReset" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AutoReset"] = ["60"] - - if "MaxMessagesPerAlarm" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["MaxMessagesPerAlarm"] = ["1"] - - if "PollingPeriod" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["PollingPeriod"] = ["5"] - - if "LogFile" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["LogFile"] = ["/tmp/pjb/log"] - - if "HtmlFolder" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["HtmlFolder"] = ["/tmp/pjb"] - - if "AlarmThreshold" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmThreshold"] = ["1"] - - - def match_properties(self): - - - devclass = types_classes[self.itemType] - - if "CIR" in self.itemName: - print "dealing with magnet circuit" - - # for given item type, look up required attributes and properties of tango - elif devclass in TANGO_PROPERTIES: - - fixed_properties_l = list(TANGO_PROPERTIES[devclass][0].keys()) - #print "fixed tango properties are ", fixed_properties_l - - # add the fixed properties - self.parameters.update(TANGO_PROPERTIES[devclass][0]) - - lattice_properties_l = list(TANGO_PROPERTIES[devclass][1].keys()) - #print "possible lattice tango properties are ", lattice_properties_l - for k in self.parameters.keys(): - #print "pjb zzz 1", self.parameters["Tilt"], self.parameters - #print "key", k - #if not a required property or attribue then pop it - if k.lower() not in lattice_properties_l and k not in fixed_properties_l: - #print "popping ", k - self.parameters.pop(k) - - #otherwise rename key if an attribute - if k.lower() in lattice_properties_l: - #print "KEY ", k.lower(), TANGO_PROPERTIES[devclass][1][k.lower()], self.parameters[k] - self.parameters[TANGO_PROPERTIES[devclass][1][k.lower()]] = [self.parameters.pop(k)] - #print "pjb zzz", self.parameters["Tilt"], self.parameters - - - else: - for k in self.parameters.keys(): - self.parameters.pop(k) - - if "MAG" in self.itemName and not "CIR" in self.itemName: - self.parameters["Type"] = [self.itemType] - - # - if "Tilt" in self.parameters: - if "0.5 pi" in str(self.parameters["Tilt"]): - self.parameters["Tilt"] = ["90"] - else: - self.parameters["Tilt"] = ["0"] - print "pjbyyy", self.parameters - - def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[0-9]+)'): - ''' - Updates json file - ''' - # prepare pattern for parsing name - pattern = re.compile(name_parsing_string) - - print "In add device for item: " + self.itemName + " as " + self.itemType, self.alpars - # only when we know class for certain element - - print "adict is ", adict - circuit_alarm_params = [] - - #for case with no final number like I.TR1.MAG.DIE (no .1 etc at end) - alt_name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)' - alt_pattern = re.compile(alt_name_parsing_string) - - #self.alpars = {} - devdictalarm = None - - if types_classes.has_key(self.itemType): - - print "" - print "" - - # split name - name_items = pattern.search(self.itemName) - parsed=False - tryagain=False - if name_items == None: - print "Warning: Item name in lattice file doesn't match the naming convention.",self.itemName - tryagain=True - else: - parsed=True - if tryagain: - name_items = alt_pattern.search(self.itemName) - if name_items == None: - print "Warning: Item name in lattice file STILL doesn't match the naming convention.",self.itemName - else: - parsed=True - if parsed: - system = name_items.group('system') - subsystem = name_items.group('subsystem') - location = name_items.group('location') - device = name_items.group('device') - - if tryagain==False: - num = name_items.group('num') - else: - num="" - - if num == None: num = '' - - if num != "": - num2 = "%02d" % int(num) - else: - num2 = "01" - - - self.match_properties() - print "pjbxxx ", self.parameters - - # create device for json output - name = (system+"-"+location + '/' + subsystem + '/' + device + "-" + num2).encode('ascii', 'ignore') - devclass = types_classes[self.itemType].encode('ascii', 'ignore') - server = devclass + '/' + system+"-"+location - - #hack for circuits - if "CIR" in self.itemName: - print "orig name",self.itemName, name - - #quad circuits named like CRQ - #correctors like CRCOX and CRCOY - #dipoles like CRDI - #solenoid like CRSOL - #sextu like CRX - endname = name.rsplit("/",1)[1] - - #fix circuit names - endname = self.handle_circuit_name(self.itemName,num2) - name = name.rsplit("/",1)[0] + "/" + endname - - #e.g in G00 have COIX 1, 2, 3 and COHX 1 and 2, which would make COX 1, 2, 3 with COIX 1 and 2 being lost! - #increment number till unique - while name in cirlist: - print "danger! already named this circuit!", self.itemName, name - suffix = int(name.split("-")[2]) + 1 - newnum2 = "%02d" % suffix - print newnum2 - name = (name.rsplit("-",1)[0]) + "-" + newnum2 - print name - print "new name ", name - cirlist.append(name) - print cirlist - devclass = "MagnetCircuit" - - - #compact name is to find tag in plc alarms - name_l_cir = self.itemName.split(".") - section_cir = name_l_cir[1] - mid=name_l_cir[3] - if "_" in mid: - mid = mid.split("_")[0] - #hack for SM1A and SM1B in TR1, also DIE, DIF on circuit DI (DIC, DID on TR3) - compactnamecir = "_"+name_l_cir[1]+mid+"_" - compactnamecir = compactnamecir.replace("1A","1") - compactnamecir = compactnamecir.replace("1B","1") - if "DIE" in compactnamecir: - compactnamecir = compactnamecir.replace("DIE","DI") - if "DIF" in compactnamecir: - compactnamecir = compactnamecir.replace("DIF","DI") - if "DIC" in compactnamecir: - compactnamecir = compactnamecir.replace("DIC","DI") - if "DID" in compactnamecir: - compactnamecir = compactnamecir.replace("DID","DI") - - print "circuit compact name is ", compactnamecir, name_l_cir - - #fill alarm info for circuits - # - pyalarm = system+"-"+location + '/MAG/ALARM' - if adict is not None: - devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] - - print "init devdictalarm circuit" - - already_added = ["B_I_SP02DIPBD_DIA_TSW1_A","B_I_SP02DIPBD_DIA_TSW2_A"] #can use this to ignore to. e.g. SP02DIPDB doesnt end in a number but isn't a circuit! - for key in alarm_dict: - #compact name is like _TR1QF_ but some tags are like _TR1QF2_5_ - #if compactnamecir in key: - if (key not in already_added and compactnamecir in key) or (compactnamecir[:-1] in key and key.count("_")>5 and key not in already_added and "F"+num+"_" in key): - - print "key is", num, key - - already_added.append(key) - - print "FOUND ALARM INFO FOR CIR", compactnamecir, key, alarm_dict[key], section_cir - pyattname = "I-" + section_cir + "/DIA/COOLING" - - #for the magnets json file - circuit_alarm_params = [pyattname, key, alarm_dict[key]] - - #for the alarms json file - alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+endname.replace("-","_") - alsev = alname+":"+"ALARM" - alrec = alname+":"+"HTML" - aldesc = alname+":One magnet in circuit "+ alarm_dict[key] - - if pyalarm not in self.alpars: - self.alpars[pyalarm] = {} - self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars - - - - print "+++++++++++++++++ Creating device server : " + server + " for " + devclass + " (name= " + name + ")" - print "+++++++++ With properties : ", self.parameters - - # Dont actually write attributes to DB, only properties - # see if this class exists and append if so, or create - devdict = sdict.servers["%s/%s" % (devclass, system+"-"+location)][devclass][name] - - #for circuit json only - if "CR" in name: - psdevdict = psdict.Circuits[name] - - - if "MAG" in self.itemName and "CIR" not in self.itemName: - - #compact name is to find tag in plc alarms - name_l = self.itemName.split(".") - section = name_l[1] - del name_l[0] - del name_l[1] - #del name_l[2] - print "pjb kkk", name_l - compactfullname = "".join(name_l) - compactname = compactfullname.split("_")[0] - compactname_nonum = compactfullname.split("_")[0][:-1]+"_" - - print "-------------------- magnet not circuit", self.itemName, compactname - - #see what is the ps of the magnet - if name in POWER_SUPPLY_MAP: - powersupplyname = POWER_SUPPLY_MAP[name] - else: - print "magnet not in PS map, skipping", name - return - - #!!!!!!!!!!! *********** create circuit device for each new ps ************!!!!!!!!!!!! - # copy the magnet and call recursively add device! - magnetcircuit = copy.deepcopy(self) - magnetcircuit.itemName = self.itemName + ".CIR" - - magnetcircuit.parameters = {} - magnetcircuit.parameters['PowerSupplyProxy'] = [powersupplyname] - magnetcircuit.parameters['MagnetProxies'] = [name] - magnetcircuit.parameters['RiseTime'] = ["0.0"] - magnetcircuit.parameters['ResistanceReference'] = ["0.0"] - magnetcircuit.parameters['CoilNames'] = [""] - - #for the ps json file only - magnetcircuit.psparameters['PowerSupplyProxy'] = [powersupplyname] - magnetcircuit.psparameters['MagnetProxies'] = [name] - - #get alarm info from excel - pyalarm = system+"-"+location + '/MAG/ALARM' - - print "adict is again", adict - if adict is not None: - devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] - print "init devdictalarm" - - #set alarms in magnet.json file and in alarms json file - for key in alarm_dict: - if compactname in key and key.count("_")<6: - #if compactname in key: - - pyattname = "I-" + section + "/DIA/COOLING" - - print "FOUND ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict - - #for the magnets json file - if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] - else: - self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) - - #for the alarms json file - alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+'MAG'+'__'+ device + "_" +num2 - alsev = alname+":"+"ALARM" - aldesc = alname+":Magnet "+alarm_dict[key] - - if pyalarm not in self.alpars: - self.alpars[pyalarm] = {} - self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars - - devdictalarm.properties = self.alpars[pyalarm] - - #set alarms in magnet.json file for all magnets in circuit - for key in alarm_dict: - if compactname_nonum in key or (compactname[:-1] in key and key.count("_")>5 and ("F"+num+"_" in key or "_"+num+"_" in key)): - #if compactname_nonum in key: - print "mag key ", key, compactname_nonum, compactname, "F"+num+"_", "_"+num+"_" - pyattname = "I-" + section + "/DIA/COOLING" - - print "FOUND MORE ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict - - #for the magnets json file - if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] - else: - self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) - - polarity = 1 - orientation = 1 - - #get calibration info from the excel - if self.itemName.split("_")[0] in calib_dict: - print "FOUND CALIB INFO", self.itemName - #find max multipole expansions - dim = max(calib_dict[self.itemName.split("_")[0]].keys(), key=int) - - print "--- max order is", dim - - #create arrays of this dimensions, other dimension is 11 - - fieldsmatrix = [[0 for x in xrange(19)] for x in xrange(dim)] - #print fieldsmatrix - currentsmatrix = [[0 for x in xrange(19)] for x in xrange(dim)] - - #iterate over keys and add to the array - for key in calib_dict[self.itemName.split("_")[0]]: - print '--- ', key, 'corresponds to', calib_dict[self.itemName.split("_")[0]][key] - currents = calib_dict[self.itemName.split("_")[0]][key][5:25] - fields = calib_dict[self.itemName.split("_")[0]][key][25:45] - print '--- ',currents, fields - - fieldsmatrix[key-1]=fields - currentsmatrix[key-1]=currents - #key here is the multipole order. any one should have same polarity - polarity = calib_dict[self.itemName.split("_")[0]][key][3] - orientation = calib_dict[self.itemName.split("_")[0]][key][2] - #print "P, O", polarity, orientation - - print '--- ',fieldsmatrix - print '--- ',currentsmatrix - - #now trim the matrices (lists) - maxlength = 20 - for i,val in enumerate(fieldsmatrix[dim-1]): - if val=='': - print i , val - maxlength = i - break - print maxlength - for i in xrange(dim): - print i - del fieldsmatrix[i][maxlength:] - del currentsmatrix[i][maxlength:] - - print 'Now--- ',fieldsmatrix - print 'Now--- ',currentsmatrix - - magnetcircuit.parameters['ExcitationCurveCurrents']= currentsmatrix - magnetcircuit.parameters['ExcitationCurveFields']= fieldsmatrix - - self.parameters['Orientation'] = [str(int(orientation))] - self.parameters['Polarity'] = [str(int(polarity))] - - #assign circuit name as property of magnet device - #no regex to fix name here so do by hand - #e.g. I.BC1.MAG.COEX.4.CIR -> I-BC1/MAG/COEX-CIR-04 - - cname = name.rsplit("/CIR",1)[0] - endname = cname.rsplit("/",1)[1] - endnum = cname.rsplit("-",1)[1] - endname = self.handle_circuit_name(self.itemName,endnum) - cname = cname.rsplit("/",1)[0] + "/" + endname - - print "cname is ", cname, name, powersupplyname, circuit_ps_list - - while cname in cirlist: - print "danger2! already named this circuit!", cname - suffix = int(cname.split("-")[2]) + 1 - newnum2 = "%02d" % suffix - cname = (cname.rsplit("-",1)[0]) + "-" + newnum2 - print "new name ", cname - - #only add one circuit device per ps - if powersupplyname not in circuit_ps_list: - - magnetcircuit.add_device(sdict,adict,psdict) - circuit_ps_list[powersupplyname] = cname - - print "adding circuit name ", magnetcircuit.itemName, cname, circuit_ps_list - self.parameters['CircuitProxies'] = [cname] - - else: - #if we aleady made this circuit device, add it to this magnet properties - print "!!!ALART!!! already added a circuit device for ", self.itemName, name, system, location - - if system == "R3": - system= "I" - if location == "301L": - location = "TR3" - - self.parameters['CircuitProxies'] = [circuit_ps_list[powersupplyname]] - - #need to get the name of the circuit device from the ps dict though - print "exiting circuit device is", circuit_ps_list[powersupplyname] - - #print "current mags ", system+"-"+location - #print "current mags 2", sdict.servers - - current_mags = sdict.servers["%s/%s" % ("MagnetCircuit", system+"-"+location)]["MagnetCircuit"][circuit_ps_list[powersupplyname]].properties - #for circuits json - print "cir name from ps ", circuit_ps_list[powersupplyname], psdict - ps_current_mags = psdict.Circuits[circuit_ps_list[powersupplyname]].Properties - print "current mags ", current_mags['MagnetProxies'] - if name in current_mags['MagnetProxies']: - print "circuit already has magnet ", name - else: - ps_current_mags['MagnetProxies'].append(name) - current_mags['MagnetProxies'].append(name) - - print "magnets on cir ", current_mags['ExcitationCurveFields'], current_mags['MagnetProxies'], len(current_mags['MagnetProxies']) - #need to average the currents, even if already done so in excel (depends on field order) - if 'ExcitationCurveFields' in current_mags: - - assoc_field_m = current_mags['ExcitationCurveFields'] - this_field_m = fieldsmatrix - - assoc_curr_m = current_mags['ExcitationCurveCurrents'] - this_curr_m = currentsmatrix - - print "field matrix assoc is ", assoc_field_m - print "field matrix current is ", this_field_m - - print "current matrix assoc is ", assoc_curr_m - print "current matrix current is ", this_curr_m - - for i in xrange(dim): - print i - - #fix for CRSM take abs field values since opp sign - if circuit_ps_list[powersupplyname] in ["I-TR3/MAG/CRSM-01","I-TR3/MAG/CRDI-01"]: - newFields = [ ( abs(x)*(len(current_mags['MagnetProxies']) -1) + abs(y) ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] - else: - newFields = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] - - newCurrents = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_curr_m[i],assoc_curr_m[i])] - - print "new fields ", newFields - print "new currents ", newCurrents - current_mags['ExcitationCurveFields'][i] = newFields - print "updated: ", current_mags['ExcitationCurveFields'] - current_mags['ExcitationCurveCurrents'][i] = newCurrents - print "updated: ", current_mags['ExcitationCurveCurrents'] - - - else: - print "NOT A MAGNET" - - devdict.properties = self.parameters - - #for circuits json - if "CR" in name: - psdevdict.Properties = self.psparameters - - -class ElegantLatticeParser: - ''' Class for parsing an elegant lattice file. ''' - fileName = "" - file = None - items = [] - - def __init__(self, _fileName): - '''Constructs a parser object. - - Keyword arguments: - _fileName -- the name of file to be parsed - ''' - self.fileName = _fileName - self.file = io.open(_fileName) - - - - def parseLatticeFile(self): - ''' ''' - line = "" # this will be a line combined from lines to be connected - for ll in self.file: - l = ll.lstrip().rstrip() - if len(l) > 0: - if l[0] == '!': - pass # do not process comments - elif l[0] == '%': - pass # processing RPNs are not implemented - elif l[-1] == '&': - # Combine lines to be concated - line = line + ' ' + l[:-1] - else: - # So this is the last line to be combined - line = line + l - # Create an object and add it to list - self.items.append(LatticeFileItem(line.lstrip().rstrip())) - line = "" - - - -if __name__ == '__main__': - - - - inFileName = '' - doCalib=False - doAlarms=False - excelName = 'MagnetCalibrationData.xls' - alarmName = 'IMAG_ALARM_140923_Magnets.xls' - - # define classes for lattice elements - types_classes = {} - types_classes["dip"] = "Magnet" - types_classes["sbend"] = "Magnet" - types_classes["sben"] = "Magnet" - types_classes["rben"] = "Magnet" - types_classes["csrcsbend"] = "Magnet" - types_classes["sole"] = "Magnet" - types_classes["kquad"] = "Magnet" - types_classes["ksext"] = "Magnet" - types_classes["hkick"] = "Magnet" - types_classes["vkick"] = "Magnet" - #types_classes["hkick"] = "VACorrector" - #types_classes["vkick"] = "VACorrector" - #types_classes["monitor"] = "VABPM" - #types_classes["watch"] = "VAYAGScreen" - #types_classes["rfcw"] = "VARfCavity" - # - circuit_ps_list = {} - #circuit_alarm_params = None - - #read arguments - for par in sys.argv[1:]: - if par[0:2] == '--': - if par == '--calibration-data': - doCalib = True - elif par == '--alarm-data': - doAlarms = True - # elif par == '--test-mode': - # test_mode = True - elif inFileName == '': - inFileName = par - - print inFileName, doCalib - - - if doAlarms or doCalib: - import xlrd - - alarm_dict = {} - if doAlarms: - print "opening alarms xls" - xls = xlrd.open_workbook(alarmName) - sheet = xls.sheet_by_name('Sheet1') - rows = [sheet.row_values(i) for i in xrange(sheet.nrows)] - column_names = rows[0] - print "cols ", column_names - for row in enumerate(rows[1:]): - if row[1][0]=="": - continue - print row[1][0],row[1][1] - alarm_dict[row[1][0]] = row[1][1] - print "DICT IS ", alarm_dict - - #make dict just for alarms! for pyalarm - json_dict_alarms = SuperDict() - else: - json_dict_alarms = None - - calib_dict = {} - - if doCalib: - #open excel sheet - xls = xlrd.open_workbook(excelName) - - for name in ["linac", "transfer 1,5 GeV", "transfer 3 GeV", "thermionic gun"]: - - #sheet = xls.sheet_by_name('Linac') - sheet = xls.sheet_by_name(name) - rows = [sheet.row_values(i) for i in xrange(sheet.nrows)] - column_names = rows[7] - print "cols ", column_names - - for row in enumerate(rows[9:]): - print row[1] - if row[1][2]=="": - continue - #this is like - #[5.0, u'I.S01A', u'I.S01A.MAG.QE.1', 202005.0, u'#1168-10030-0001', 1.0, -1.0, 2.0, 6.3167, 5.6757, 5.0307500000000003, 4.4208999999999996, 3.8452999999999999, 3.1463999999999999, 2.5179624999999999, 1.8892374999999999, 1.2808725000000001, 0.63988750000000016, 0.0, 0.70470485548532313, 0.63908274382966312, 0.56946571499960408, 0.50203927491440703, 0.43686121069898298, 0.35966476443894108, 0.288993167760146, 0.21848942173091002, 0.14957521795596601, 0.077488874695939805, 0.0052044472873010797, u'T', u'Rotating coil-C1168, #0001.xls', u'https://alfresco.maxlab.lu.se/share/page/site/maxiv/document-details?nodeRef=workspace://SpacesStore/23cdc9d1-a01e-443e-b578-1538637a1472', u'Scanditronix Magnet', 40690.0, ''] - if row[1][2].strip() not in calib_dict: - if row[1][7] is not "": - data_key = int(row[1][7]) - data_list = row[1][3:48] - data_dict = {data_key : data_list} - calib_dict[row[1][2].strip()]=data_dict - #calib_dict[row[1][2]]=row[1][3:33] - else: - if row[1][7] is not "": - #we found more curves for the same magnet - print "found another entry", row[1][2], row[1][7] - data_key = int(row[1][7]) - data_list = row[1][3:48] - data_dict = {data_key : data_list} - calib_dict[row[1][2].strip()][data_key]=data_list - - print "DICT IS ", calib_dict - - - - # create a parser for the file - parser = ElegantLatticeParser(inFileName) - - # parse the file - parser.parseLatticeFile() - parser.file.close() - - - # make a json file - json_dict = SuperDict() - json_ps = SuperDict() - for item in parser.items: - item.add_device(json_dict,json_dict_alarms, json_ps) - #print json.dumps(json_dict, indent=4) - - #now we have the dict, loop over again and sort out magnets, power supplies and circuits - - print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ " - print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ " - print json_dict.servers - - outfile = open('magnets.json', 'w') - - #have final json dict here - print "THE FINAL DICT" - topl = json_dict['servers'].keys() - for item in topl: - if "Circuit" in item: - #print json_dict['servers'][item] - for cir in json_dict['servers'][item]: - #print json_dict['servers'][item][cir]['ExcitationCurveCurrents'] - #print json_dict['servers'][item]["MagnetCircuit"] - for c in json_dict['servers'][item]["MagnetCircuit"]: - for key in json_dict['servers'][item]["MagnetCircuit"][c]["properties"]: - if key == "ExcitationCurveCurrents": - ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print key, [str(x) for x in ls] - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] - if key == "ExcitationCurveFields": - ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print key, [str(x) for x in ls] - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] - - json.dump(json_dict, outfile, indent=4) - - if doAlarms: - outfile2 = open('magnets_alarms.json', 'w') - json.dump(json_dict_alarms, outfile2, indent=4) - - #!! note that item has parameters, but only want to extract those needed for tango! - - #dump ps circuit info - outfile3 = open('circuits.json', 'w') - json.dump(json_ps, outfile3, indent=4) - diff --git a/dsconfig/output.py b/dsconfig/output.py index 6504ec8..80cf01f 100644 --- a/dsconfig/output.py +++ b/dsconfig/output.py @@ -1,10 +1,10 @@ from difflib import ndiff -from PyTango.utils import CaselessDict - -from dsconfig.utils import green, red, yellow +from tango.utils import CaselessDict from dsconfig.tangodb import get_devices_from_dict -from appending_dict import SetterDict +from dsconfig.utils import green, red, yellow + +from .appending_dict import SetterDict def get_device_data(device, mapping, data): @@ -37,9 +37,10 @@ def format_property(value, indentation="", max_lines=10): def get_changes(data, calls): - - """Combine a list of database calls into "changes" that can - be more easily turned into a readable representation""" + """ + Combine a list of database calls into "changes" that can + be more easily turned into a readable representation + """ devices = get_devices_from_dict(data["servers"]) device_mapping = CaselessDict( @@ -125,7 +126,7 @@ def get_changes(data, calls): caseless_props = CaselessDict(old_data.get("properties", {})) if "properties" not in changes["devices"][device]: changes["devices"][device]["properties"] = {} - for name, value in properties.items(): + for name, value in list(properties.items()): old_value = caseless_props.get(name) if value != old_value: changes["devices"][device]["properties"].update({ @@ -152,9 +153,9 @@ def get_changes(data, calls): old_data = get_device_data(device, device_mapping, data) caseless_attrs = CaselessDict(old_data.get( "attribute_properties", {})) - for attr, props in properties.items(): + for attr, props in list(properties.items()): caseless_props = CaselessDict(caseless_attrs.get(attr, {})) - for name, value in props.items(): + for name, value in list(props.items()): old_value = caseless_props.get(name) if value != old_value: attr_props[attr] = { @@ -168,7 +169,7 @@ def get_changes(data, calls): "attribute_properties", {}) old_data = get_device_data(device, device_mapping, data) caseless_attrs = CaselessDict(old_data.get("attribute_properties", {})) - for attr, props in attributes.items(): + for attr, props in list(attributes.items()): caseless_props = CaselessDict(caseless_attrs[attr]) for prop in props: old_value = caseless_props.get(prop) @@ -179,7 +180,7 @@ def get_changes(data, calls): old_data = classes.get(clss, {}) caseless_props = CaselessDict(old_data.get("properties", {})) prop_changes = changes["classes"][clss].setdefault("properties", {}) - for name, value in properties.items(): + for name, value in list(properties.items()): old_value = caseless_props.get(name) if value != old_value: prop_changes.update({ @@ -204,9 +205,9 @@ def get_changes(data, calls): old_data = classes.get(clss, {}) caseless_attrs = CaselessDict( old_data.get("attribute_properties", {})) - for attr, props in properties.items(): + for attr, props in list(properties.items()): caseless_props = CaselessDict(caseless_attrs.get(attr, {})) - for name, value in props.items(): + for name, value in list(props.items()): old_value = caseless_props.get(name) if value != old_value: attr_props[attr] = { @@ -220,7 +221,7 @@ def get_changes(data, calls): "attribute_properties", {}) old_data = classes.get(clss, {}) caseless_attrs = CaselessDict(old_data.get("properties", {})) - for attr, props in attributes.items(): + for attr, props in list(attributes.items()): caseless_props = CaselessDict(caseless_attrs.get(attr, {})) for prop in props: old_value = caseless_props.get(prop) @@ -233,8 +234,9 @@ def get_changes(data, calls): def show_actions(data, calls): - - "Print out a human readable representation of changes" + """ + Print out a human readable representation of changes + """ changes = get_changes(data, calls) @@ -246,40 +248,40 @@ def show_actions(data, calls): info = changes["devices"][device] if info.get("added"): if info.get("old_server"): - print("{} Device: {}".format(yellow("="), device)) + print(("{} Device: {}".format(yellow("="), device))) else: - print("{} Device: {}".format(green("+"), device)) + print(("{} Device: {}".format(green("+"), device))) elif info.get("deleted"): - print("{} Device: {}".format(red("-"), device)) + print(("{} Device: {}".format(red("-"), device))) else: - print("{} Device: {}".format(yellow("="), device)) + print(("{} Device: {}".format(yellow("="), device))) if info.get("server"): if info.get("old_server"): - print("{}Server: {} -> {}".format(indent, - red(info["old_server"]), - green(info["server"]))) + print(("{}Server: {} -> {}".format(indent, + red(info["old_server"]), + green(info["server"])))) else: - print("{}Server: {}".format(indent, info["server"])) + print(("{}Server: {}".format(indent, info["server"]))) if info.get("device_class"): if info.get("old_class"): if info["old_class"] != info["device_class"]: - print("{}Class: {} -> {}".format(indent, - info["old_class"], - info["device_class"])) + print(("{}Class: {} -> {}".format(indent, + info["old_class"], + info["device_class"]))) else: - print("{}Class: {}".format(indent, info["device_class"])) + print(("{}Class: {}".format(indent, info["device_class"]))) if info.get("alias"): alias = info.get("alias").get("value") old_alias = info.get("alias").get("old_value") if old_alias: if old_alias != alias: - print("{}Alias: {} -> {}".format(indent, red(old_alias), - green(alias))) + print(("{}Alias: {} -> {}".format(indent, red(old_alias), + green(alias)))) else: - print("{}Alias: {}".format(indent, alias)) + print(("{}Alias: {}".format(indent, alias))) if info.get("properties"): lines = [] @@ -290,21 +292,21 @@ def show_actions(data, calls): if old_value is not None: if old_value != value: # change property - lines.append(yellow("{}= {}".format(indent*2, prop))) + lines.append(yellow("{}= {}".format(indent * 2, prop))) lines.append(property_diff( - change["old_value"], change["value"], indent*3)) + change["old_value"], change["value"], indent * 3)) else: # new property - lines.append(green("{}+ {}".format(indent*2, prop))) + lines.append(green("{}+ {}".format(indent * 2, prop))) lines.append(green(format_property(change["value"], - indent*3))) + indent * 3))) else: # delete property - lines.append(red("{}- {}".format(indent*2, prop))) - lines.append(red(format_property(change["old_value"], indent*3))) + lines.append(red("{}- {}".format(indent * 2, prop))) + lines.append(red(format_property(change["old_value"], indent * 3))) if lines: - print("{}Properties:".format(indent*1)) - print("\n".join(lines)) + print(("{}Properties:".format(indent * 1))) + print(("\n".join(lines))) if info.get("attribute_properties"): lines = [] @@ -318,26 +320,26 @@ def show_actions(data, calls): if old_value is not None: # change if value != old_value: - attr_lines.append(yellow("{}= {}".format(indent*3, prop))) + attr_lines.append(yellow("{}= {}".format(indent * 3, prop))) attr_lines.append(property_diff( - change["old_value"], change["value"], indent*4)) + change["old_value"], change["value"], indent * 4)) else: # addition - attr_lines.append(green("{}+ {}".format(indent*3, prop))) + attr_lines.append(green("{}+ {}".format(indent * 3, prop))) attr_lines.append(green(format_property(change["value"], - indent*4))) + indent * 4))) else: # removal - attr_lines.append(red("{}- {}".format(indent*3, prop))) + attr_lines.append(red("{}- {}".format(indent * 3, prop))) attr_lines.append(red(format_property(change["old_value"], - indent*4))) + indent * 4))) if attr_lines: - lines.append("{}{}".format(indent*2, attr)) + lines.append("{}{}".format(indent * 2, attr)) lines.extend(attr_lines) if lines: - print("{}Attribute properties:".format(indent)) - print("\n".join(lines)) - + print(("{}Attribute properties:".format(indent))) + print(("\n".join(lines))) + for clss in sorted(changes["classes"]): info = changes["classes"][clss] @@ -350,19 +352,19 @@ def show_actions(data, calls): if old_value is not None: if old_value != value: # change property - lines.append(yellow("{}= {}".format(indent*2, prop))) + lines.append(yellow("{}= {}".format(indent * 2, prop))) lines.append(property_diff( - change["old_value"], change["value"], indent*3)) + change["old_value"], change["value"], indent * 3)) else: # new property - lines.append(green("{}+ {}".format(indent*2, prop))) + lines.append(green("{}+ {}".format(indent * 2, prop))) lines.append(green(format_property(change["value"], - indent*3))) + indent * 3))) else: # delete property - lines.append(red("{}- {}".format(indent*2, prop))) - lines.append(red(format_property(change["old_value"], indent*3))) + lines.append(red("{}- {}".format(indent * 2, prop))) + lines.append(red(format_property(change["old_value"], indent * 3))) if lines: - print("{} Class Properties:".format(indent*1)) - print("\n".join(lines)) - print + print(("{} Class Properties:".format(indent * 1))) + print(("\n".join(lines))) + print() diff --git a/dsconfig/remove.py b/dsconfig/remove.py index 16fc93c..1ac0ede 100644 --- a/dsconfig/remove.py +++ b/dsconfig/remove.py @@ -1,10 +1,10 @@ import json import sys -import PyTango +import tango -from utils import (ObjectWrapper, decode_dict, get_dict_from_db, - RED, GREEN, YELLOW) +from .utils import (ObjectWrapper, decode_dict, get_dict_from_db, + RED, GREEN, YELLOW) def delete_devices(db, dbdict, server, cls, devices): @@ -12,74 +12,73 @@ def delete_devices(db, dbdict, server, cls, devices): if dbdict.servers[server][cls][devname]: db.delete_device(devname) else: - print "no device", devname + print("no device", devname) def delete_server(db, dbdict, servername, serverdata): - for clsname, devices in serverdata.items(): - delete_devices(db, dbdict, servername, clsname, devices.keys()) + for clsname, devices in list(serverdata.items()): + delete_devices(db, dbdict, servername, clsname, list(devices.keys())) try: db.delete_server_info(servername) # what does this do? db.delete_server(servername) - except PyTango.DevFailed: + except tango.DevFailed: # I'm not sure about this; sometimes deleting servers works # and sometimes not; should they be running? What's going on? # Anyway, the worst case scenario is that the servers are # still there but contain no devices... - print "Removing server '%s' may have failed." % servername + print("Removing server '%s' may have failed." % servername) def delete_class(db, dbdict, classname): - props = dbdict.classes[classname].properties.keys() + props = list(dbdict.classes[classname].properties.keys()) if props: db.delete_class_property(classname, props) - attr_props = dbdict.classes[classname].attribute_properties.keys() + attr_props = list(dbdict.classes[classname].attribute_properties.keys()) if attr_props: db.delete_class_attribute_property(classname, attr_props) def main(json_file, write=False, db_calls=False): - - """Remove the devices and servers defined in the given file - from the DB.""" + """ + Remove the devices and servers defined in the given file from the DB. + """ with open(json_file) as f: data = json.load(f, object_hook=decode_dict) - db = PyTango.Database() + db = tango.Database() dbdict = get_dict_from_db(db, data) # check the current DB state # wrap the database (and fake it if we're not going to write) db = ObjectWrapper(db if write else None) # delete servers and devices - for servername, serverdata in data.get("servers", {}).items(): + for servername, serverdata in list(data.get("servers", {}).items()): if servername in dbdict.servers: delete_server(db, dbdict, servername, serverdata) # delete classes - for classname, classdata in data.get("classes", {}).items(): + for classname, classdata in list(data.get("classes", {}).items()): if classname in dbdict.classes: delete_class(db, dbdict, classname) if db_calls: - print "\nTango database calls:" + print("\nTango database calls:") for method, args, kwargs in db.calls: - print method, args + print(method, args) if db.calls: if write: - print >>sys.stderr, ( - RED + "\n*** Data was written to the Tango DB ***") + print(( + RED + "\n*** Data was written to the Tango DB ***"), file=sys.stderr) else: - print >>sys.stderr, YELLOW +\ - "\n*** Nothing was written to the Tango DB (use -w) ***" + print(YELLOW + \ + "\n*** Nothing was written to the Tango DB (use -w) ***", file=sys.stderr) else: - print >>sys.stderr, GREEN + "\n*** No changes needed in Tango DB ***" + print(GREEN + "\n*** No changes needed in Tango DB ***", file=sys.stderr) if __name__ == "__main__": - from optparse import OptionParser usage = "Usage: %prog [options] JSONFILE" diff --git a/dsconfig/schema/dsconfig.json b/dsconfig/schema/dsconfig.json index 42f057a..59c3dec 100644 --- a/dsconfig/schema/dsconfig.json +++ b/dsconfig/schema/dsconfig.json @@ -1,112 +1,113 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Tango device generator JSON canonical format", - "type": "object", - "additionalProperties": false, - "properties": { - "_title": { - "type": "string" - }, - "_date": { - "type": "string" - }, - "_source": { - "type": "string" - }, - "_version": { - "enum": [2] - }, - "servers": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/server" - } - } - }, - "classes": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/device" - }, - - "properties": { - "properties": { - "$ref": "#definitions/property" - } - } + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Tango device generator JSON canonical format", + "type": "object", + "additionalProperties": false, + "properties": { + "_title": { + "type": "string" + }, + "_date": { + "type": "string" + }, + "_source": { + "type": "string" + }, + "_version": { + "enum": [ + 2 + ] + }, + "servers": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/server" } + } }, - "definitions": { - "server": { - "type": "object", - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/instance" - } - } - }, - "instance": { - "type": "object", - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/class" - } - } - }, - "class": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[\\-\\w]+/[\\-\\w]+/[\\-\\w]+$": { - "$ref": "#definitions/device" - } - } - }, - "device": { - "type": "object", - "additionalProperties": false, - "properties": { - "properties": { - "$ref": "#definitions/properties" - }, - "attribute_properties": { - "$ref": "#definitions/attribute_properties" - } - } - }, + "classes": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/device" + }, + "properties": { "properties": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/property" - } - }, - "property": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } + "$ref": "#definitions/property" + } + } + } + }, + "definitions": { + "server": { + "type": "object", + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/instance" + } + } + }, + "instance": { + "type": "object", + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/class" + } + } + }, + "class": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[\\-\\w]+/[\\-\\w]+/[\\-\\w]+$": { + "$ref": "#definitions/device" + } + } + }, + "device": { + "type": "object", + "additionalProperties": false, + "properties": { + "properties": { + "$ref": "#definitions/properties" }, "attribute_properties": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/attribute_property" - } - }, - "attribute_property": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/property" - } - }, - "class_property": { - "type": "object", - "additionalProperties": { - "type": "object", - "$ref": "#definitions/property" - } + "$ref": "#definitions/attribute_properties" } + } + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/property" + } + }, + "property": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "attribute_properties": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/attribute_property" + } + }, + "attribute_property": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/property" + } + }, + "class_property": { + "type": "object", + "additionalProperties": { + "type": "object", + "$ref": "#definitions/property" + } } + } } diff --git a/dsconfig/schema/schema2.json b/dsconfig/schema/schema2.json index 9dc9aaf..e29d3dd 100644 --- a/dsconfig/schema/schema2.json +++ b/dsconfig/schema/schema2.json @@ -1,114 +1,115 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Tango device generator JSON canonical format", - "type": "object", - "additionalProperties": false, - "properties": { - "_title": { - "type": "string" - }, - "_date": { - "type": "string" - }, - "_source": { - "type": "string" - }, - "_version": { - "enum": [2] - }, - "servers": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/server" - } - } - }, - "classes": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/device" - }, - - "properties": { - "properties": { - "$ref": "#definitions/property" - } - } + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Tango device generator JSON canonical format", + "type": "object", + "additionalProperties": false, + "properties": { + "_title": { + "type": "string" + }, + "_date": { + "type": "string" + }, + "_source": { + "type": "string" + }, + "_version": { + "enum": [ + 2 + ] + }, + "servers": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/server" } + } }, - "definitions": { - "server": { - "type": "object", - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/instance" - } - } - }, - "instance": { - "type": "object", - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/class" - } - } - }, - "class": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[\\-\\w.@]+/[\\-\\w.@]+/[\\-\\w.@]+$": { - "$ref": "#definitions/device" - } - } - }, - "device": { - "type": "object", - "additionalProperties": false, - "properties": { - "properties": { - "$ref": "#definitions/properties" - }, - "attribute_properties": { - "$ref": "#definitions/attribute_properties" - }, - "alias": { - "type": "string" - } - } - }, + "classes": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/device" + }, + "properties": { "properties": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/property" - } - }, - "property": { - "type": "array", - "items": { - "type": "string" - } + "$ref": "#definitions/property" + } + } + } + }, + "definitions": { + "server": { + "type": "object", + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/instance" + } + } + }, + "instance": { + "type": "object", + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/class" + } + } + }, + "class": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[\\-\\w.@]+/[\\-\\w.@]+/[\\-\\w.@]+$": { + "$ref": "#definitions/device" + } + } + }, + "device": { + "type": "object", + "additionalProperties": false, + "properties": { + "properties": { + "$ref": "#definitions/properties" }, "attribute_properties": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/attribute_property" - } + "$ref": "#definitions/attribute_properties" }, - "attribute_property": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/property" - } - }, - "class_property": { - "type": "object", - "additionalProperties": { - "type": "object", - "$ref": "#definitions/property" - } + "alias": { + "type": "string" } + } + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/property" + } + }, + "property": { + "type": "array", + "items": { + "type": "string" + } + }, + "attribute_properties": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/attribute_property" + } + }, + "attribute_property": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/property" + } + }, + "class_property": { + "type": "object", + "additionalProperties": { + "type": "object", + "$ref": "#definitions/property" + } } + } } diff --git a/dsconfig/tangodb.py b/dsconfig/tangodb.py index b71481c..c7bfa14 100755 --- a/dsconfig/tangodb.py +++ b/dsconfig/tangodb.py @@ -1,21 +1,18 @@ "Various functionality for dealing with the TANGO database" from collections import defaultdict -from difflib import unified_diff -from itertools import izip, islice +from itertools import islice -import PyTango - -from appending_dict import AppendingDict, SetterDict, CaselessDictionary +import tango from dsconfig.utils import green, red, yellow +from .appending_dict import AppendingDict, SetterDict, CaselessDictionary # These are special properties that we'll ignore for now PROTECTED_PROPERTIES = [ "polled_attr", "logging_level", "logging_target" ] - SPECIAL_ATTRIBUTE_PROPERTIES = [ "label", "format", "unit", "standard_unit", "display_unit", "min_value", "min_alarm", "min_warning", @@ -30,16 +27,16 @@ def get_devices_from_dict(dbdict, name=None): return [(server_name, instance_name, class_name, device_name) - for server_name, server in dbdict.items() - for instance_name, instance in server.items() - for class_name, clss in instance.items() + for server_name, server in list(dbdict.items()) + for instance_name, instance in list(server.items()) + for class_name, clss in list(instance.items()) for device_name in clss if name is None or device_name.lower() == name.lower()] def get_servers_from_dict(dbdict): servers = set() - for server, children in dbdict.get("servers", {}).items(): + for server, children in list(dbdict.get("servers", {}).items()): if "/" in server: servers.add(server) else: @@ -49,9 +46,11 @@ def get_servers_from_dict(dbdict): def is_protected(prop, attr=False): - """There are certain properties that need special treatment as they + """ + There are certain properties that need special treatment as they are handled in particular ways by Tango. In general, we don't want to - remove these if they exist, but we do allow overwriting them.""" + remove these if they exist, but we do allow overwriting them. + """ if attr: # Attribute config properties return prop.startswith("_") or prop in SPECIAL_ATTRIBUTE_PROPERTIES @@ -60,7 +59,6 @@ def is_protected(prop, attr=False): def present_calls(indata, dbdata, dbcalls): - # WIP! def add_device(devname, devinfo): @@ -74,8 +72,9 @@ def delete_device(devname): def summarise_calls(dbcalls, dbdata): - - "A brief summary of the operations performed by a list of DB calls" + """ + A brief summary of the operations performed by a list of DB calls + """ methods = [ "add_device", @@ -109,7 +108,7 @@ def summarise_calls(dbcalls, dbdata): n = len(args[1]) devices[method].add(args[0].upper()) elif "attribute_property" in method: - n = sum(len(ps) for attr, ps in args[1].items()) + n = sum(len(ps) for attr, ps in list(args[1].items())) devices[method].add(args[0].upper()) elif "property" in method: n = len(args[1]) @@ -155,16 +154,17 @@ def summarise_calls(dbcalls, dbdata): def get_device(db, devname, data, skip_protected=True): - - """Returns all relevant DB information about a given device: - alias (if any), properties, attribute properties""" + """ + Returns all relevant DB information about a given device: + alias (if any), properties, attribute properties + """ dev = {} try: alias = db.get_alias_from_device(devname) dev["alias"] = alias - except PyTango.DevFailed: + except tango.DevFailed: pass # Properties @@ -172,7 +172,7 @@ def get_device(db, devname, data, skip_protected=True): db_props = db.get_device_property_list(devname, "*") if db_props: props = db.get_device_property(devname, list(db_props)) - for prop, value in props.items(): + for prop, value in list(props.items()): # We'll ignore "protected" properties unless they are present # in the input data (in that case we want to show that # they are changed) @@ -195,11 +195,11 @@ def get_device(db, devname, data, skip_protected=True): if attr_props: attribute_properties = {} dbprops = db.get_device_attribute_property(devname, - attr_props.keys()) - for attr, props in dbprops.items(): + list(attr_props.keys())) + for attr, props in list(dbprops.items()): attr_props = dict( (prop, [str(v) for v in values]) - for prop, values in props.items() + for prop, values in list(props.items()) # Again, ignore attr_props that are not present in the # input data, as we most likely don't want to remove those if (not (skip_protected and is_protected(prop, True)) @@ -214,8 +214,8 @@ def get_device(db, devname, data, skip_protected=True): def get_dict_from_db(db, data, narrow=False, skip_protected=True): - - """Takes a data dict, checks if any if the definitions are already + """ + Takes a data dict, checks if any if the definitions are already in the DB and returns a dict describing them. By default it includes all devices for each server+class, use the @@ -235,16 +235,16 @@ def get_dict_from_db(db, data, narrow=False, skip_protected=True): if devinfo.ds_full_name.lower() != srvname.lower(): moved_devices[devinfo.ds_full_name].append((clss, device)) - except PyTango.DevFailed: + except tango.DevFailed: pass # Servers - for srvr, insts in data.get("servers", {}).items(): - for inst, classes in insts.items(): - for clss, devs in classes.items(): + for srvr, insts in list(data.get("servers", {}).items()): + for inst, classes in list(insts.items()): + for clss, devs in list(classes.items()): devs = CaselessDictionary(devs) if narrow: - devices = devs.keys() + devices = list(devs.keys()) else: srv_full_name = "%s/%s" % (srvr, inst) devices = db.get_device_name(srv_full_name, clss) @@ -254,9 +254,9 @@ def get_dict_from_db(db, data, narrow=False, skip_protected=True): dbdict.servers[srvr][inst][clss][device] = db_props # Classes - for class_name, cls in data.get("classes", {}).items(): - props = cls.get("properties", {}).keys() - for prop, value in db.get_class_property(class_name, props).items(): + for class_name, cls in list(data.get("classes", {}).items()): + props = list(cls.get("properties", {}).keys()) + for prop, value in list(db.get_class_property(class_name, props).items()): if value: value = [str(v) for v in value] dbdict.classes[class_name].properties[prop] = value @@ -264,20 +264,22 @@ def get_dict_from_db(db, data, narrow=False, skip_protected=True): attr_props = cls.get("attribute_properties") if attr_props: dbprops = db.get_class_attribute_property(class_name, - attr_props.keys()) - for attr, props in dbprops.items(): + list(attr_props.keys())) + for attr, props in list(dbprops.items()): props = dict((prop, [str(v) for v in values]) - for prop, values in props.items()) + for prop, values in list(props.items())) dbdict.classes[class_name].attribute_properties[attr] = props return dbdict.to_dict(), moved_devices def find_empty_servers(db, data): - "Find any servers in the data that contain no devices, and remove them" + """ + Find any servers in the data that contain no devices, and remove them + """ servers = ["%s/%s" % (srv, inst) - for srv, insts in data["servers"].items() - for inst in insts.keys()] + for srv, insts in list(data["servers"].items()) + for inst in list(insts.keys())] return [server for server in servers if all(d.lower().startswith('dserver/') for d in db.get_device_class_list(server))] @@ -290,7 +292,7 @@ def get_device_property_values(dbproxy, device, name="*", _, result = dbproxy.command_inout("DbMySqlSelect", query % (device, name.replace("*", "%"))) data = defaultdict(list) - for prop, row in izip(result[::2], result[1::2]): + for prop, row in zip(result[::2], result[1::2]): if prop != "__SubDevices" or include_subdevices: data[prop].append(row) return data @@ -300,9 +302,9 @@ def get_device_attribute_property_values(dbproxy, device, name="*"): query = ("SELECT attribute, name, value FROM property_attribute_device " "WHERE device = '%s' AND name LIKE '%s'") _, result = dbproxy.command_inout("DbMySqlSelect", - query % (device, name.replace("*", "%"))) + query % (device, name.replace("*", "%"))) data = AppendingDict() - for attr, prop, row in izip(result[::3], result[1::3], result[2::3]): + for attr, prop, row in zip(result[::3], result[1::3], result[2::3]): data[attr][prop] = row return data @@ -317,13 +319,13 @@ def get_devices_by_name_and_class(dbproxy, name, clss="*"): query = ("SELECT name FROM device WHERE name LIKE '%s' " "AND class LIKE '%s'") _, result = dbproxy.command_inout("DbMySqlSelect", - query % (name.replace("*", "%"), clss.replace("*", "%"))) + query % (name.replace("*", "%"), clss.replace("*", "%"))) return result def nwise(it, n): - "[s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...]" - return izip(*[islice(it, i, None, n) for i in xrange(n)]) + # [s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...] + return list(zip(*[islice(it, i, None, n) for i in range(n)])) def maybe_upper(s, upper=False): @@ -356,7 +358,7 @@ def get_servers_with_filters(dbproxy, server="*", clss="*", device="*", # good to increase the timeout a bit. # TODO: maybe instead use automatic retry and increase timeout # each time? - dbproxy.set_timeout_millis(timeout*1000) + dbproxy.set_timeout_millis(timeout * 1000) if properties: # Get all relevant device properties @@ -372,9 +374,7 @@ def get_servers_with_filters(dbproxy, server="*", clss="*", device="*", _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device)) for d, p, v in nwise(result, 3): - # the properties are encoded in latin-1; we want utf-8 - decoded_value = v.decode('iso-8859-1').encode('utf8') - devices[maybe_upper(d, uppercase_devices)].properties[p] = decoded_value + devices[maybe_upper(d, uppercase_devices)].properties[p] = v if attribute_properties: # Get all relevant attribute properties @@ -390,9 +390,7 @@ def get_servers_with_filters(dbproxy, server="*", clss="*", device="*", _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device)) for d, a, p, v in nwise(result, 4): dev = devices[maybe_upper(d, uppercase_devices)] - # the properties are encoded in latin-1; we want utf-8 - decoded_value = v.decode('iso-8859-1').encode('utf8') - dev.attribute_properties[a][p] = decoded_value + dev.attribute_properties[a][p] = v devices = devices.to_dict() @@ -430,7 +428,7 @@ def get_classes_properties(dbproxy, server='*', cls_properties=True, # Mysql wildcards server = server.replace("*", "%") # Change device proxy timeout - dbproxy.set_timeout_millis(timeout*1000) + dbproxy.set_timeout_millis(timeout * 1000) # Classes output dict classes = AppendingDict() # Get class properties @@ -448,9 +446,7 @@ def get_classes_properties(dbproxy, server='*', cls_properties=True, _, result = dbproxy.command_inout("DbMySqlSelect", querry % (server)) # Build the output based on: class, property: value for c, p, v in nwise(result, 3): - # the properties are encoded in latin-1; we want utf-8 - decoded_value = v.decode('iso-8859-1').encode('utf8') - classes[c].properties[p] = decoded_value + classes[c].properties[p] = v # Get class attribute properties if cls_attribute_properties: querry = ( diff --git a/dsconfig/utils.py b/dsconfig/utils.py index 109fd7f..844d4df 100644 --- a/dsconfig/utils.py +++ b/dsconfig/utils.py @@ -1,22 +1,19 @@ -from functools import partial import sys +from functools import partial -import PyTango - -from appending_dict import AppendingDict - -#exit codes -SUCCESS = 0 # NO DB CHANGES +# exit codes +SUCCESS = 0 # NO DB CHANGES ERROR = 1 CONFIG_APPLIED = 2 CONFIG_NOT_APPLIED = 3 -#colors +# colors ADD = GREEN = '\033[92m' REMOVE = RED = FAIL = '\033[91m' REPLACE = YELLOW = WARN = '\033[93m' ENDC = '\033[0m' + def no_colors(): global ADD, GREEN, REMOVE, RED, FILE, REPLACE, YELLOW, WARN, ENDC ADD = '' @@ -29,12 +26,15 @@ def no_colors(): WARN = '' ENDC = '' + def green(text): return GREEN + text + ENDC + def red(text): return RED + text + ENDC + def yellow(text): return YELLOW + text + ENDC @@ -53,14 +53,16 @@ def progressbar(i, n, width): def find_device(definitions, devname, caseless=False): - "Find a given device in a server dict" - for srvname, srv in definitions["servers"].items(): + """ + Find a given device in a server dict + """ + for srvname, srv in list(definitions["servers"].items()): if caseless: srv = CaselessDict(srv) - for instname, inst in srv.items(): + for instname, inst in list(srv.items()): if caseless: inst = CaselessDict(inst) - for classname, cls in inst.items(): + for classname, cls in list(inst.items()): if caseless: cls = CaselessDict(cls) if devname in cls: @@ -69,8 +71,10 @@ def find_device(definitions, devname, caseless=False): def find_class(definitions, clsname): - "Find a given device in a server dict" - for instname, inst in definitions["servers"].items(): + """ + Find a given device in a server dict + """ + for instname, inst in list(definitions["servers"].items()): if clsname in inst: return inst[clsname] raise ValueError("class '%s' not defined" % clsname) @@ -78,23 +82,23 @@ def find_class(definitions, clsname): def get_devices_from_dict(dbdict): return [(server_name, inst_name, class_name, device_name) - for server_name, server in dbdict.items() - for inst_name, inst in server.items() - for class_name, clss in inst.items() + for server_name, server in list(dbdict.items()) + for inst_name, inst in list(server.items()) + for class_name, clss in list(inst.items()) for device_name in clss] class ObjectWrapper(object): - - """An object that allows all method calls and records them, - then passes them on to a target object (if any).""" + """ + An object that allows all method calls and records them, + then passes them on to a target object (if any). + """ def __init__(self, target=None): self.target = target self.calls = [] def __getattr__(self, attr): - def method(attr, *args, **kwargs): self.calls.append((attr, args, kwargs)) if self.target: @@ -131,17 +135,26 @@ def method(attr, *args, **kwargs): class CaselessDict(dict): - """A case insensitive dictionary that only permits strings as keys.""" + """ + A case insensitive dictionary that only permits strings as keys. + """ + def __init__(self, indict={}): dict.__init__(self) - self._keydict = {} # not self.__keydict because I want it to be easily accessible by subclasses + # not self.__keydict because I want it to be easily accessible by subclasses + self._keydict = {} for entry in indict: - self[entry] = indict[entry] # not dict.__setitem__(self, entry, indict[entry]) becasue this causes errors (phantom entries) where indict has overlapping keys... + # not dict.__setitem__(self, entry, indict[entry]) becasue this causes errors + # (phantom entries) where indict has overlapping keys... + self[entry] = indict[entry] def findkey(self, item): - """A caseless way of checking if a key exists or not. - It returns None or the correct key.""" - if not isinstance(item, str): raise TypeError('Keywords for this object must be strings. You supplied %s' % type(item)) + """ + A caseless way of checking if a key exists or not. + It returns None or the correct key. + """ + if not isinstance(item, str): raise TypeError( + 'Keywords for this object must be strings. You supplied %s' % type(item)) key = item.lower() try: return self._keydict[key] @@ -149,12 +162,14 @@ def findkey(self, item): return None def changekey(self, item): - """For changing the casing of a key. + """ + For changing the casing of a key. If a key exists that is a caseless match for 'item' it will be changed to 'item'. - This is useful when initially setting up default keys - but later might want to preserve an alternative casing. - (e.g. if later read from a config file - and you might want to write back out with the user's casing preserved). + This is useful when initially setting up default keys - but later might want to + preserve an alternative casing. (e.g. if later read from a config file - and you + might want to write back out with the user's casing preserved). """ - key = self.findkey(item) # does the key exist + key = self.findkey(item) # does the key exist if key == None: raise KeyError(item) temp = self[key] del self[key] @@ -162,32 +177,40 @@ def changekey(self, item): self._keydict[item.lower()] = item def lowerkeys(self): - """Returns a lowercase list of all member keywords.""" - return self._keydict.keys() + """ + Returns a lowercase list of all member keywords. + """ + return list(self._keydict.keys()) - def __setitem__(self, item, value): # setting a keyword - """To implement lowercase keys.""" - key = self.findkey(item) # if the key already exists + def __setitem__(self, item, value): # setting a keyword + """ + To implement lowercase keys. + """ + key = self.findkey(item) # if the key already exists if key != None: - dict.__delitem__(self,key) + dict.__delitem__(self, key) self._keydict[item.lower()] = item dict.__setitem__(self, item, value) def __getitem__(self, item): - """To implement lowercase keys.""" - key = self.findkey(item) # does the key exist + """ + To implement lowercase keys. + """ + key = self.findkey(item) # does the key exist if key == None: raise KeyError(item) return dict.__getitem__(self, key) - def __delitem__(self, item): # deleting a keyword - key = self.findkey(item) # does the key exist + def __delitem__(self, item): # deleting a keyword + key = self.findkey(item) # does the key exist if key == None: raise KeyError(item) dict.__delitem__(self, key) del self._keydict[item.lower()] def pop(self, item, default=None): - """Correctly emulates the pop method.""" - key = self.findkey(item) # does the key exist + """ + Correctly emulates the pop method. + """ + key = self.findkey(item) # does the key exist if key == None: if default == None: raise KeyError(item) @@ -197,25 +220,33 @@ def pop(self, item, default=None): return dict.pop(self, key) def popitem(self): - """Correctly emulates the popitem method.""" + """ + Correctly emulates the popitem method. + """ popped = dict.popitem(self) del self._keydict[popped[0].lower()] return popped def has_key(self, item): - """A case insensitive test for keys.""" - if not isinstance(item, str): return False # should never have a non-string key - return self._keydict.has_key(item.lower()) # does the key exist + """ + A case insensitive test for keys. + """ + if not isinstance(item, str): return False # should never have a non-string key + return item.lower() in self._keydict # does the key exist def __contains__(self, item): - """A case insensitive __contains__.""" - if not isinstance(item, str): return False # should never have a non-string key - return self._keydict.has_key(item.lower()) # does the key exist + """ + A case insensitive __contains__. + """ + if not isinstance(item, str): return False # should never have a non-string key + return item.lower() in self._keydict # does the key exist def setdefault(self, item, default=None): - """A case insensitive setdefault. - If no default is supplied it sets the item to None""" - key = self.findkey(item) # does the key exist + """ + A case insensitive setdefault. + If no default is supplied it sets the item to None + """ + key = self.findkey(item) # does the key exist if key != None: return self[key] self.__setitem__(item, default) self._keydict[item.lower()] = item @@ -223,27 +254,34 @@ def setdefault(self, item, default=None): def get(self, item, default=None): """A case insensitive get.""" - key = self.findkey(item) # does the key exist + key = self.findkey(item) # does the key exist if key != None: return self[key] return default def update(self, indict): - """A case insensitive update. - If your dictionary has overlapping keys (e.g. 'FISH' and 'fish') then one will overwrite the other. - The one that is kept is arbitrary.""" + """ + A case insensitive update. If your dictionary has overlapping keys (e.g. 'FISH' + and 'fish') then one will overwrite the other. The one that is kept is arbitrary. + """ for entry in indict: - self[entry] = indict[entry] # this uses the new __setitem__ method + self[entry] = indict[entry] # this uses the new __setitem__ method def copy(self): - """Create a new caselessDict object that is a copy of this one.""" + """ + Create a new caselessDict object that is a copy of this one. + """ return CaselessDict(self) def dict(self): - """Create a dictionary version of this caselessDict.""" + """ + Create a dictionary version of this caselessDict. + """ return dict.copy(self) def clear(self): - """Clear this caselessDict.""" + """ + Clear this caselessDict. + """ self._keydict = {} dict.clear(self) @@ -252,41 +290,43 @@ def __repr__(self): return 'caselessDict(' + dict.__repr__(self) + ')' -"""A tuctionary, or tuct, is the combination of a tuple with +""" +A tuctionary, or tuct, is the combination of a tuple with a dictionary. A tuct has named items, but they cannot be deleted or rebound, nor new can be added. """ + class ImmutableDict(object): - """The tuct class. An immutable dictionary. + """ + The tuct class. An immutable dictionary. """ def __init__(self, dict=None, **kwds): - self.__data = {} - if dict is not None: - self.__data.update(dict) - if len(kwds): - self.__data.update(kwds) + self.__data = {} + if dict is not None: + self.__data.update(dict) + if len(kwds): + self.__data.update(kwds) - #del __init__ + # del __init__ def __repr__(self): - return repr(self.__data) + return repr(self.__data) def __cmp__(self, dict): - if isinstance(dict, ImmutableDict): - return cmp(self.__data, dict.__data) - else: - return cmp(self.__data, dict) + if isinstance(dict, ImmutableDict): + return cmp(self.__data, dict.__data) + else: + return cmp(self.__data, dict) def __len__(self): - return len(self.__data) + return len(self.__data) def __getitem__(self, key): - return self.__data[key] + return self.__data[key] def copy(self): - if self.__class__ is ImmutableDict: - return ImmutableDict(self.__data.copy()) - import copy - __data = self.__data + if self.__class__ is ImmutableDict: + return ImmutableDict(self.__data.copy()) + __data = self.__data diff --git a/dsconfig/validate.py b/dsconfig/validate.py index 28acd63..0ae6b6c 100644 --- a/dsconfig/validate.py +++ b/dsconfig/validate.py @@ -1,15 +1,14 @@ import json import sys -from jsonschema import Draft4Validator, validate, exceptions - -from utils import decode_dict +from jsonschema import validate, exceptions +from .utils import decode_dict if __name__ == "__main__": data_filename, schema_filename = sys.argv[1], sys.argv[2] - print "Validating '%s' against schema '%s'..." % (data_filename, schema_filename), + print("Validating '%s' against schema '%s'..." % (data_filename, schema_filename), end=' ') with open(data_filename) as data_json: data = json.load(data_json, object_hook=decode_dict) @@ -20,8 +19,8 @@ try: validate(data, schema) except exceptions.ValidationError as e: - print "data does not match schema:" - print e + print("data does not match schema:") + print(e) sys.exit(1) else: - print "success!" + print("success!") diff --git a/dsconfig/viewer.py b/dsconfig/viewer.py index 7add5c4..f47d16d 100644 --- a/dsconfig/viewer.py +++ b/dsconfig/viewer.py @@ -8,9 +8,9 @@ """ import urwid +from urwidtrees.decoration import CollapsibleIndentedTree from urwidtrees.tree import Tree from urwidtrees.widgets import TreeBox -from urwidtrees.decoration import CollapsibleIndentedTree class FocusableText(urwid.WidgetWrap): @@ -20,7 +20,7 @@ def __init__(self, path, data): child = get_path(path, data) if isinstance(txt, int): # we're in a list # TODO: support containers inside lists - a = urwid.Text(str(txt)+":") + a = urwid.Text(str(txt) + ":") if isinstance(child, dict): b = urwid.Text("") else: @@ -68,10 +68,10 @@ def _get_children(self, path): node = get_path(path, self.data) if node: if isinstance(node, dict): - children = node.keys() + children = list(node.keys()) return [path + (child,) for child in sorted(children)] elif isinstance(node, list): - children = range(len(node)) + children = list(range(len(node))) return [path + (child,) for child in children] return [] @@ -121,7 +121,9 @@ def prev_sibling_position(self, path): class MyTreeBox(TreeBox): def keypress(self, size, key): - "Spicing up the keybindings!" + """ + Spicing up the keybindings! + """ # if key == "delete": # _, path = self.get_focus() # self.set_focus(path[:-1]) @@ -149,7 +151,6 @@ def keypress(self, size, key): ('connectors', 'light red', 'light gray', ''), ] - if __name__ == "__main__": import json @@ -159,6 +160,8 @@ def keypress(self, size, key): # make ctrl+C exit properly, without breaking the terminal def exit_handler(signum, frame): raise urwid.ExitMainLoop() + + signal.signal(signal.SIGINT, exit_handler) # redirecting stdin breaks things. This was supposed to help diff --git a/setup.cfg b/setup.cfg index 319a641..d53fcea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,4 +4,5 @@ test = pytest [tool:pytest] addopts= --junit-xml=tests.xml --cov=dsconfig + --cov-report=term --cov-report=html diff --git a/setup.py b/setup.py index 1db8887..b730aec 100755 --- a/setup.py +++ b/setup.py @@ -5,20 +5,23 @@ setup( # Package name="python-dsconfig", - version="1.4.0", - packages=['dsconfig', 'dsconfig.appending_dict'], + version="1.5.0", + packages=["dsconfig", "dsconfig.appending_dict"], description="Library and utilities for Tango device configuration.", # Requirements - setup_requires=['setuptools', 'pytest-runner'], - install_requires=['jsonpatch>=1.13', 'jsonschema', 'xlrd', 'PyTango'], + python_requires='~=3.6', + setup_requires=["setuptools", "pytest-runner"], + install_requires=["jsonpatch>=1.13", "jsonschema", "xlrd", "PyTango"], tests_require=["pytest", "pytest-cov", "Faker", "mock"], # Resources package_data={ - 'dsconfig': ['schema/dsconfig.json', - 'schema/schema2.json']}, + "dsconfig": ["schema/dsconfig.json", + "schema/schema2.json"] + }, # Scripts entry_points={ - 'console_scripts': ['xls2json = dsconfig.excel:main', - 'csv2json = dsconfig.callcsv:main', - 'json2tango = dsconfig.json2tango:main']} + "console_scripts": ["xls2json = dsconfig.excel:main", + "csv2json = dsconfig.callcsv:main", + "json2tango = dsconfig.json2tango:main"] + } ) diff --git a/test/files/sample_db.json b/test/files/sample_db.json new file mode 100644 index 0000000..6a90534 --- /dev/null +++ b/test/files/sample_db.json @@ -0,0 +1,1146 @@ +{ + "classes": { + "ScienceYetChallenge": { + "properties": { + "RecordHead": ["wparks"], + "HotItself": ["tracy32"], + "DetailFear": ["haynesashlee"] + } + }, + "Site": { + "attribute_properties": { + "Approach": { + "delta_t": ["16"], + "__value": ["-22"], + "max_warning": ["-56"], + "unit": ["79"] + } + }, + "properties": { "ListenLess": ["melindahanson"] } + }, + "TreatHerselfPublic": { + "attribute_properties": { + "ResearchHeadOff": { "archive_period": ["-99"] } + }, + "properties": { "HowShouldWorldLong": ["joshuaholmes"] } + }, + "RecognizeRequire": { + "attribute_properties": { + "RequireGirlReflect": { "max_warning": ["-31"] } + }, + "properties": { "Teacher": ["chelsea64"] } + } + }, + "servers": { + "AttentionSea": { + "1JY-48-XAK": { + "HugeClassBook": { + "WEST/WILL/PROFESSIONAL-7": { + "attribute_properties": { + "MagazineLaugh": { "max_warning": ["46"], "format": ["16"] } + }, + "properties": { + "Car": ["evansjessica"], + "ShareLastNone": ["lindsey30"], + "ViewDreamShouldWhileOpen": ["cbanks"] + } + } + }, + "Life": { + "CONCERN/ENVIRONMENTAL/THEMSELVES-1": { + "properties": { + "DemocraticLocal": ["coledaniel", "juliecook", "eacosta"] + } + }, + "SAME/PROGRAM/SAFE-10": { + "properties": { + "Trouble": ["ovargas"], + "EnergyOneOff": ["carolinehendrix"], + "KeySkinItWalkReach": [ + "mobrien", + "davisalexandra", + "fburnett", + "qroberts" + ] + } + } + } + }, + "AXQ-AP-M8O7": { + "Rich": { + "WEST/OFFICER/COMPUTER-1": { + "attribute_properties": { + "DropCaseHere": { "abs_change": ["77"] } + }, + "properties": { + "SeemChangeHusbandMeetLot": ["alecfreeman"], + "TaskFocusGroupDecade": ["thomasmarie", "burtonjennifer"] + } + }, + "SITUATION/SIZE/WAY-4": { + "properties": { + "TeacherThereKitchenCapital": ["wayne35", "amberward"], + "NothingRelateAgainShoulderLaugh": ["yhart"], + "UseTurn": ["dustinjones", "joseph12", "esparzaandrew"] + } + }, + "ABOVE/AWAY/TEAM-9": { + "attribute_properties": { + "TreatBetterToward": { "max_warning": ["-92"] } + }, + "properties": { "CallThem": ["marissa86", "lisa32"] } + }, + "REVEAL/BAD/BODY-6": { "properties": { "Tax": ["jacksonpatricia"] } }, + "ROLE/HERSELF/SKIN-1": { + "attribute_properties": { + "Recent": { "rel_change": ["-32"], "archive_rel_change": ["-15"] } + }, + "properties": { "SeekSideMean": ["fyoung"] } + }, + "SO/ROOM/SECTION-7": { + "properties": { "LeaderInOpportunity": ["qrodriguez"] } + }, + "FRONT/LOT/THERE-2": { + "properties": { "Base": ["xbradley", "psmith", "william19"] } + } + } + }, + "WA-6": { + "IncludeParticipantMonth": { + "WEAR/LINE/QUESTION-7": { + "properties": { + "ParentHealthPositiveMinuteAccording": ["scottthompson"], + "OpportunityPassTwo": [ + "smithjennifer", + "michele44", + "debbieguzman", + "teresaallen" + ] + } + } + }, + "EnergyHeavyBuy": { + "THINK/DAY/WIN-4": { + "properties": { + "HeSendCitizen": [ + "mary84", + "wilcoxdaniel", + "audreybrown", + "cookjohn", + "lewisjeffrey" + ] + } + } + }, + "Campaign": { + "THANK/MODEL/POPULATION-10": { + "attribute_properties": { + "MeetingSitThat": { + "rel_change": ["-19"], + "archive_abs_change": ["98"] + }, + "IssueDevelop": { "archive_period": ["71"] }, + "IndeedCoverFact": { "__value": ["36"] }, + "Tell": { "mode": ["35"] } + }, + "properties": { "Suddenly": ["edwardsjill", "xbarajas"] } + } + }, + "Factor": { + "TRY/DIFFICULT/ON-1": { + "attribute_properties": { + "MeasureOld": { "min_warning": ["84"], "mode": ["79"] } + }, + "properties": { "AssumeTooGuess": ["nmartinez"] } + } + } + }, + "2N-UL7": { + "WithinCouldThem": { + "SHORT/WONDER/NATURAL-8": { + "properties": { + "Expect": ["rojassandra", "leemelissa", "sherri83"] + } + }, + "PROVE/CONTROL/MAGAZINE-0": { + "attribute_properties": { + "Line": { "__value": ["-82"], "unit": ["-16"] }, + "Side": { "unit": ["-32"] }, + "ListenHappen": { "__value": ["82"] } + }, + "properties": { "Place": ["peterboone", "hbarron", "rlynch"] } + } + } + }, + "X": { + "Media": { + "PREVENT/WHAT/SHARE-6": { "properties": { "Trouble": ["joe60"] } } + } + }, + "CV3-L": { + "MaybeHisTeacher": { + "ANYONE/WE/SET-10": { + "properties": { + "ManCollectionConsiderSomeonePolitics": ["arnoldkyle"], + "GunMediaPresentFather": ["denise75"] + } + }, + "PROTECT/NICE/LAW-4": { + "properties": { + "DogPowerWhom": ["wmanning"], + "GetAwayParticularly": ["jennifercastillo"] + } + } + }, + "NextDecisionOver": { + "THOUGHT/OCCUR/NEXT-3": { + "properties": { + "StockVariousFeelProgramBag": ["miahoward", "ywright"] + } + } + }, + "Article": { + "INSTEAD/ISSUE/FOR-0": { + "properties": { + "SubjectWorldUsuallyFeelDebate": ["davidterry"], + "RedSurfaceThemBlack": ["matthew12"] + } + }, + "SKILL/COLD/EVERYTHING-4": { + "attribute_properties": { + "Central": { "min_alarm": ["-71"] }, + "ResearchKitchen": { + "max_alarm": ["48"], + "min_warning": ["28"], + "mode": ["-65"] + }, + "RestFast": { "delta_val": ["-94"] } + }, + "properties": { + "ShortPriceSongOk": [ + "vbrown", + "billygarcia", + "vgrimes", + "wsmith", + "rarcher", + "daniel61" + ] + } + }, + "GET/NOR/FEW-0": { + "attribute_properties": { + "NorSuffer": { "archive_rel_change": ["-9"], "mode": ["-7"] }, + "EatRaise": { "abs_change": ["85"] }, + "OpenWaterFire": { "archive_rel_change": ["72"] } + }, + "properties": { "Treat": ["urodriguez"] } + } + } + } + }, + "RecentlyOnceCheck": { + "XS0-OJR5": { + "DetermineExplainNearly": { + "TEN/ABILITY/TAX-4": { + "attribute_properties": { + "ExactlyLater": { "__value": ["-86"] }, + "WestMostMoment": { "standard_unit": ["59"] }, + "Fact": { "max_value": ["-5"] } + }, + "properties": { + "AlwaysSixDetermine": ["brian58"], + "LikelyColorKeepOpen": ["apatton", "adam35"] + } + } + } + }, + "S-V89-2PUQP": { + "West": { + "STAFF/APPLY/ENOUGH-4": { + "properties": { "ReturnMeasureProjectAccount": ["jocelynleblanc"] } + } + } + }, + "3FSE": { + "HundredRequireCup": { + "PICK/APPLY/MATERIAL-5": { + "properties": { + "MoveJustWay": ["hramirez"], + "ConcernBoard": ["xgaines"] + } + }, + "CHOOSE/NOR/COLLECTION-0": { + "properties": { "ClassPrettyThanContainResource": ["jeffrey70"] } + }, + "DAUGHTER/ACROSS/REACH-3": { + "attribute_properties": { "Smile": { "max_value": ["-23"] } }, + "properties": { "DirectionExpect": ["qlopez"] } + }, + "TASK/POSITIVE/OUTSIDE-7": { + "properties": { + "FieldPutTodayGreatTotal": ["kylecollins"], + "FoodLawyer": ["denisesharp"] + } + }, + "PROGRAM/ME/IDEA-2": { + "properties": { "MainTreatmentImprove": ["amber81"] } + } + } + }, + "F5-7T-R": { + "StandardThrough": { + "HEALTH/EVIDENCE/ANY-7": { + "properties": { + "AtPutA": ["kathleen53"], + "AdmitGoSell": ["dudleymariah"], + "TodayFieldPlantServe": ["bkim"], + "BackArtist": ["diane28"] + } + }, + "HER/ITEM/RADIO-7": { + "attribute_properties": { + "NightPresidentVery": { "max_value": ["60"] } + }, + "properties": { + "Father": ["kwoods", "michaelsimon"], + "MoveRealizeEnterEatCentury": ["warnercaitlin", "eolsen"] + } + }, + "HUSBAND/ARGUE/EXECUTIVE-5": { + "properties": { + "SeriousMovementFollowMatter": ["jaredwilliams", "marissasalinas"] + } + }, + "PROVIDE/AUTHORITY/HOME-3": { + "attribute_properties": { + "LandStructureStop": { "unit": ["-23"] }, + "BallDog": { "label": ["12"] } + }, + "properties": { + "WhyParentFaceYourself": ["alexanderallen"], + "SubjectFallData": ["wgregory"] + } + } + } + } + }, + "Of": { + "MZ9N3-NO-FL": { + "SonAlongChange": { + "CONDITION/GUN/ARGUE-2": { + "properties": { + "LastIt": ["jacksonrichard", "xfisher", "fosterjason"], + "SendTraditional": ["troy76"] + } + } + } + } + }, + "AddExpect": { + "6JCZ-2D": { + "MeanBeautiful": { + "ADD/PAST/FALL-0": { + "attribute_properties": { + "OnlyItsLawyer": { "archive_abs_change": ["-22"] } + }, + "properties": { "MajorityThreeThese": ["perkinssandra"] } + }, + "THROUGHOUT/FUND/TONIGHT-1": { + "properties": { "Stop": ["ricecassie"] } + }, + "ACT/CHILD/LEARN-4": { + "properties": { + "QuicklyAlreadyApplyAction": [ + "tamaramiller", + "tiffanywilliams", + "kathleenfitzpatrick" + ] + } + } + } + }, + "3MY": { + "MediaYour": { + "GAS/SECOND/DECIDE-0": { + "attribute_properties": { + "AnythingEach": { "min_alarm": ["-88"], "mode": ["-81"] } + }, + "properties": { "LateTripMissionIndicate": ["ocooper"] } + } + } + }, + "CGXW-3Z6J-K": { + "SeasonMagazine": { + "INTEREST/LEAVE/FIRE-2": { + "attribute_properties": { + "Increase": { "__value_ts": ["-35"] }, + "EffortSouth": { "standard_unit": ["-81"] } + }, + "properties": { + "Always": ["sharoncole", "jmathews"], + "GameTendOfferShow": ["castromaria", "pricekaren"] + } + } + }, + "FaceCandidate": { + "GIVE/ALREADY/QUICKLY-9": { + "properties": { "GirlCongressBoyMore": ["mphelps"] } + }, + "PULL/AFFECT/FIGHT-3": { + "properties": { "OfficeLot": ["mikayla99", "davisjason"] } + } + } + }, + "WAAP-SS00-WNE": { + "ImportantIncludingBelieve": { + "HIMSELF/SUPPORT/DO-8": { + "attribute_properties": { "Early": { "rel_change": ["-5"] } }, + "properties": { + "Remain": [ + "ymcguire", + "roberttorres", + "brooke25", + "brandonsmith" + ], + "Son": ["deannamccullough"] + } + } + } + }, + "MUNYU-0VSAF": { + "Model": { + "A/ISSUE/ANOTHER-0": { + "attribute_properties": { "Ten": { "min_alarm": ["-51"] } }, + "properties": { "RatherSuch": ["parkersarah"] } + }, + "SUFFER/TOP/DEFENSE-10": { "properties": { "Quality": ["joseph55"] } } + }, + "Room": { + "THEY/STOCK/FOR-8": { + "properties": { "HitBarNotice": ["kevin21", "stephen46"] } + } + }, + "Your": { + "SPEAK/TURN/MAIN-6": { + "attribute_properties": { + "BornEat": { + "max_value": ["-54"], + "rel_change": ["39"], + "min_alarm": ["16"], + "standard_unit": ["-84"] + } + }, + "properties": { "EconomicInvestmentIndeedAlways": ["evanprice"] } + }, + "INVOLVE/WIN/AUTHOR-10": { + "attribute_properties": { "Cultural": { "rel_change": ["67"] } }, + "properties": { "InsteadOrder": ["jacksonaaron"] } + }, + "SERVE/YOU/QUESTION-1": { + "properties": { "FromToughStudentOffice": ["harriskelli"] } + } + } + }, + "G4NI-KTR2-OO": { + "VoiceRemember": { + "SEA/EASY/SURE-9": { + "attribute_properties": { + "PersonalInterestFirm": { "delta_t": ["-45"] } + }, + "properties": { + "Chance": ["garciajames", "patricia39", "phamjacqueline"], + "Long": ["carmen58", "mmunoz"] + } + } + }, + "AndHomeTest": { + "THREAT/EARLY/HUSBAND-9": { + "properties": { "To": ["qcole", "dbenton", "benjaminhayes"] } + }, + "GREEN/MOST/BEGIN-9": { + "properties": { + "Hot": ["carsonglenn"], + "WhoReceiveSurfacePolicy": ["vhowell", "jared29", "phillip84"] + } + } + } + }, + "9-8V9V": { + "ImproveRelationship": { + "FEEL/CHILD/CALL-9": { + "properties": { + "PersonBetterDifferenceSpeech": [ + "katherine80", + "kennethjohnson", + "robert45" + ] + } + } + } + }, + "GE-FEELL-K0EJV": { + "CareerDetermineAgency": { + "WAIT/PERFORM/STAR-7": { + "attribute_properties": { + "ScoreSuddenlyForeign": { "max_alarm": ["-92"] } + }, + "properties": { + "DayMorningSomethingDaughterLook": ["ibennett"], + "PlayerKitchenPersonAskOwn": ["dbrewer", "john62", "ewatson"], + "Stay": ["ramseycrystal"] + } + } + }, + "ListMotherPhone": { + "WANT/BECAUSE/AUTHOR-2": { + "properties": { "KeyRise": ["erikaevans", "jessica79"] } + } + } + }, + "X-S": { + "National": { + "OR/EVER/HUMAN-1": { + "properties": { + "CustomerAction": ["williamssergio"], + "AgoHeartPutSoundMrs": ["jimmy19", "oclark", "ericacarter"], + "PeaceVeryDetailBuyFinal": ["cfischer"], + "Blood": ["lisamahoney", "johnsonmichael"] + } + }, + "REQUIRE/ATTORNEY/HEAD-0": { + "properties": { + "WriterStopAgain": [ + "patrickmartin", + "kelly86", + "aaron78", + "myerskari", + "stephen09", + "lopezalice" + ] + } + }, + "HIMSELF/EXPLAIN/BEHIND-8": { + "properties": { + "EvidenceDownAddress": ["taylor46"], + "Why": ["brownrobert", "sabrinatodd"], + "AlwaysWord": ["sherrianderson", "tgonzalez"] + } + }, + "ESPECIALLY/ADDRESS/EFFORT-10": { + "properties": { "LittleTask": ["hornethan"] } + } + }, + "ThusDirection": { + "LOT/NEAR/UPON-3": { + "attribute_properties": { "Seven": { "format": ["-81"] } }, + "properties": { "DoctorTeacherRich": ["edwinlee"] } + } + } + } + }, + "PriceAlready": { + "Y": { + "ItselfNothingBudget": { + "DECISION/OTHER/GOVERNMENT-9": { + "properties": { + "InterestingTeacherSenior": ["harristheresa", "mark64"], + "ThroughRoad": [ + "josephlee", + "umclaughlin", + "josephmartinez", + "ashley14" + ] + } + } + }, + "JustJoin": { + "ISSUE/MISSION/THAT-4": { + "attribute_properties": { + "AgencyReasonCentury": { + "delta_t": ["-91"], + "min_alarm": ["-46"] + } + }, + "properties": { + "CommunityGoalAcross": ["bgordon", "mariapope"], + "GreatEstablishActPlan": ["clarkcollin", "mromero"], + "Final": ["joshua50", "amberthompson"], + "ResourceEverYear": ["morgantina"] + } + } + } + }, + "HOV-GYJMM-6F": { + "RespondBy": { + "INDICATE/ON/SOUTH-4": { + "properties": { + "KindAEarlyHow": ["philipfox"], + "BrotherQuality": ["apriljefferson"] + } + } + } + } + }, + "Clear": { + "1-U8K-3DION": { + "Else": { + "MOVEMENT/PART/LIVE-1": { + "properties": { "SoldierPhone": ["jennifer17"] } + }, + "LEAVE/COURT/TOWARD-4": { + "properties": { "Song": ["amanda82", "deannaporter"] } + } + } + }, + "85-A": { + "By": { + "THEIR/RECENT/INDEED-4": { + "attribute_properties": { + "MedicalTraditional": { "rel_change": ["7"] } + }, + "properties": { "DaughterAboveTestDraw": ["omeyer"] } + }, + "QUITE/OVER/OWN-1": { + "properties": { + "SortAcceptCareer": ["crawfordchristopher"], + "NationalTotalIntoThusAttack": ["ggibbs", "samanthapatton"], + "RoleKidDuring": ["karenyoung"] + } + }, + "FUTURE/MAN/STEP-7": { + "attribute_properties": { "End": { "max_alarm": ["46"] } }, + "properties": { "BillionDescribeBehavior": ["pamelanewman"] } + } + }, + "ColdHigh": { + "SERIES/MODERN/THOSE-8": { + "properties": { + "EvidenceAgencyTake": ["moorevictoria", "lisavasquez"], + "FineAppearTripOffice": ["joseph06", "beth75"] + } + }, + "BUILDING/STOCK/COLLEGE-2": { + "properties": { "EitherCurrentShouldStand": ["curtis25"] } + } + } + }, + "B-ONQCR-RJY": { + "Near": { + "ALLOW/FATHER/YET-4": { + "properties": { + "BadRespondCertainEveryone": ["richard14"], + "SeriesArgueWithin": ["patrickmahoney", "ychristensen"], + "BeforeIndicateGroup": ["cynthiaberry"] + } + } + } + }, + "VWVM": { + "Need": { + "TOTAL/ROCK/TRIAL-6": { + "properties": { + "YardPowerCommercialStyle": ["laurieperry"], + "LessRelationship": ["kaylagibbs"] + } + } + }, + "ThemSomething": { + "ITSELF/AMONG/DESCRIBE-7": { + "properties": { + "WrongStock": ["lrivera", "richleah"], + "WhyIntoHappenWestern": ["wmoore", "andrea05", "jennifer35"], + "Form": ["uking", "carrie27", "aandrews"], + "GovernmentAreaElse": ["sandraparsons"] + } + }, + "SUBJECT/CHARGE/SIMILAR-9": { + "attribute_properties": { + "WhyWhileChallenge": { "event_period": ["-80"] } + }, + "properties": { + "DataContainEvidenceTown": ["ashley89"], + "Phone": ["jillhoover"], + "Require": ["davisdwayne"], + "DifficultNewFive": ["madison05"] + } + } + }, + "GivePressureThreat": { + "ESTABLISH/YEAR/MY-10": { + "properties": { "DarkBackTreatmentI": ["timothy43"] } + } + } + }, + "5": { + "BloodThreatAny": { + "MIND/PRODUCT/FORM-9": { + "properties": { + "Whom": ["rhonda18"], + "TermTradeOffice": ["josephsmith"] + } + }, + "APPROACH/WAY/TRIAL-10": { + "attribute_properties": { + "PeopleAmongPut": { "delta_val": ["9"] } + }, + "properties": { + "StillAgencyFight": ["amoses"], + "MaterialCommunity": ["longcharles", "amandamartinez"] + } + } + }, + "MovementFederal": { + "CLASS/METHOD/STRATEGY-2": { + "properties": { + "QuicklyCulturalTreeAgree": ["gscott"], + "FineExperienceSinceInstitution": ["morarachel", "kara82"], + "Collection": ["priceelizabeth"], + "ForceItNetworkItself": [ + "michael14", + "bwerner", + "stephanie71", + "sheppardalicia" + ] + } + } + }, + "Education": { + "AUTHOR/PARTICULARLY/IMPACT-9": { + "attribute_properties": { + "NeverPurposePrice": { "event_period": ["56"] } + }, + "properties": { "Current": ["bbartlett", "kyle39"] } + } + }, + "BackDogAt": { + "KNOW/HEART/INVESTMENT-10": { + "attribute_properties": { + "DirectionProfessional": { + "rel_change": ["-14"], + "archive_rel_change": ["76"], + "__value_ts": ["79"] + } + }, + "properties": { + "FindCollectionFewStuffList": [ + "malonechristopher", + "xbradley", + "jose80" + ], + "TrainingPm": ["zbaxter"] + } + } + } + }, + "4ZA-5H": { + "HoweverTryOnce": { + "ACCORDING/HOW/DIFFERENT-1": { + "properties": { "Point": ["frenchjames"] } + } + }, + "WhatActShare": { + "ADDRESS/AGENCY/WATCH-7": { + "properties": { + "OldManagementShow": ["bennettteresa"], + "PageYourselfNew": ["jwalls", "andrew96"], + "YearBoard": ["sean49"] + } + }, + "GROW/FILL/BRING-6": { + "attribute_properties": { + "FormCampaign": { "format": ["-10"] }, + "AroundLargeHit": { "__value_ts": ["78"] }, + "ScienceConcernJob": { "event_period": ["2"] }, + "MuchNearCity": { "max_warning": ["51"] }, + "BedCarryAffect": { "archive_abs_change": ["59"] } + }, + "properties": { + "OccurFivePositiveWind": ["daykelly"], + "FaceThankSmileResult": ["kalexander"], + "FullKitchenInteresting": ["collinsdaniel"], + "CoachInterest": ["christopherhall", "qallen"], + "EarlyManage": ["underwoodmarissa", "ksnyder"], + "OilAgencyWorryWhether": ["christopher59"], + "Plan": ["mcdonaldjessica", "lewischristopher"] + } + } + } + }, + "D-LGWS6-Y104": { + "Sea": { + "ANALYSIS/MODEL/MONTH-2": { + "attribute_properties": { + "AlongSurfaceAssume": { "mode": ["64"] }, + "SomethingDifferenceDescribe": { + "delta_t": ["63"], + "standard_unit": ["94"], + "__value_ts": ["93"], + "abs_change": ["28"] + } + }, + "properties": { "CountrySide": ["erika52"] } + } + } + } + }, + "JobOfficer": { + "O9-8D-1Q": { + "SeniorAndMillion": { + "TOWARD/REMEMBER/SOURCE-5": { + "attribute_properties": { + "Born": { "event_period": ["-76"], "max_value": ["-48"] } + }, + "properties": { "RealizeSoldierCourse": ["mthompson", "eugene28"] } + } + }, + "SummerStandard": { + "CERTAIN/EAST/HISTORY-6": { + "properties": { + "AssumeReflectPrepareLargeGeneral": ["gonzalezgregory"], + "DevelopmentTogether": [ + "rileymichael", + "weberrichard", + "kimberlyibarra" + ], + "StarCheck": ["ibyrd", "umorse"] + } + } + }, + "NecessaryUntil": { + "FAST/THREAT/ONCE-5": { "properties": { "Idea": ["vbyrd"] } } + } + }, + "W8Q2-0XN-5": { + "UponRich": { + "LAND/WHERE/TRAINING-6": { + "attribute_properties": { + "Majority": { + "archive_period": ["-8"], + "event_period": ["-1"], + "max_value": ["-34"], + "label": ["-74"], + "__value": ["17"], + "archive_rel_change": ["-19"], + "max_warning": ["71"] + } + }, + "properties": { + "SeeIfAnimal": ["vkoch", "lwebster"], + "CompanyInteresting": ["xsparks", "jturner"], + "SpecialChallengeTeachFinishEmployee": ["snyderjacob"], + "GasProtectSport": ["jessica34", "hollyfigueroa", "tinaphillips"] + } + } + } + }, + "EWWMD-L": { + "Present": { + "GAS/GOOD/SEA-4": { + "properties": { "DefenseFoot": ["ronaldhester"] } + } + } + }, + "3RIM-ECO": { + "Just": { + "EFFECT/SERVE/BOY-6": { + "properties": { + "TellCertainlyYesRecently": ["obernard", "ofranklin"] + } + } + } + }, + "B5B0-G0EG-PXSSY": { + "FootQuickly": { + "GUY/FRIEND/WITHIN-6": { "properties": { "Give": ["asanders"] } } + } + }, + "X": { + "Individual": { + "EITHER/TV/FINE-0": { + "attribute_properties": { + "WellGasCompare": { "max_alarm": ["-86"] } + }, + "properties": { + "SimpleStructureWithRule": ["dawn97", "jenkinsgregory"] + } + }, + "GROW/RELATE/RICH-8": { + "properties": { + "PartnerTell": ["pharris"], + "ProductOptionOurPhone": ["masonjohn"] + } + } + } + }, + "R68": { + "Success": { + "WRITE/HIM/SPORT-6": { + "properties": { + "EstablishMessage": ["fisherchristopher"], + "ActivityRestBeforeThanToward": ["pamela80"] + } + }, + "GO/PARTICIPANT/NATURE-4": { + "properties": { + "ReflectManagerSituationBankOrder": [ + "molly55", + "david90", + "oscarpatel", + "omartinez" + ] + } + }, + "FORGET/OFFICE/OFFICER-2": { + "properties": { + "EnvironmentAccountStudentOldMother": ["volson", "swood"] + } + }, + "DECADE/STATION/SING-6": { + "properties": { + "FinePut": ["williamramos"], + "ChildEitherRepublicanDeep": ["briannorman"], + "AnythingRoad": ["ufaulkner"], + "NoEffect": ["melanie27"] + } + }, + "DURING/SERVE/FINALLY-10": { + "attribute_properties": { "WantMajor": { "min_warning": ["-13"] } }, + "properties": { "Wind": ["tony11"] } + }, + "BACK/COLOR/CALL-2": { + "properties": { + "MessageAppearNewFact": ["hayesjoseph"], + "LearnLeaveBecomeActuallyMorning": ["dominguezderek"] + } + } + } + }, + "97-R9PP-4": { + "Away": { + "PHYSICAL/NOTE/MIDDLE-1": { + "attribute_properties": { + "NoticeIdentify": { + "archive_period": ["-3"], + "max_alarm": ["-16"] + } + }, + "properties": { "NumberAhead": ["cunninghamandrew", "kendra68"] } + } + } + }, + "DD-YTR-2": { + "HospitalJustWell": { + "COMPANY/PULL/RESULT-3": { + "attribute_properties": { + "ExistLeft": { "__value": ["-26"], "delta_val": ["-3"] } + }, + "properties": { + "ReligiousQualityCreateVisit": ["debra64"], + "SortGive": ["shellytapia"], + "EndSubject": ["lanceruiz"] + } + } + } + }, + "L69J-MWDS": { + "RespondFactor": { + "NEAR/SOURCE/HER-10": { + "attribute_properties": { + "FarOfficerBorn": { "unit": ["-100"] }, + "WifeBut": { "archive_abs_change": ["14"] } + }, + "properties": { "SeasonAgainst": ["danny77"] } + } + } + } + }, + "Range": { + "E": { + "BankDoctorSon": { + "STEP/PARENT/FINISH-8": { + "attribute_properties": { + "LookNeverHimself": { "max_warning": ["74"] } + }, + "properties": { + "SingleNation": ["francisco76"], + "FallColorRange": ["emilynielsen"] + } + } + } + }, + "S-D3SJ-5E": { + "Join": { + "EVIDENCE/BETWEEN/ABILITY-6": { + "properties": { + "EffectSoonTurnKitchenClass": ["juangibson"], + "PartManagerGoFeel": ["pearsonisaiah"] + } + } + } + }, + "AI9HD-2R38-9SN": { + "Ready": { + "REAL/AWAY/DESIGN-7": { + "properties": { + "Card": ["williamsteresa", "sandra33"], + "NiceGuyBegin": ["hannahmelendez", "diane66"] + } + } + } + }, + "APEE-MM-PZV2": { + "ManagementAddressBall": { + "HISTORY/BENEFIT/FUND-2": { + "properties": { "DuringReasonSocial": ["rodriguezregina"] } + }, + "WHEN/AUTHORITY/SCHOOL-3": { + "attribute_properties": { + "WouldFoodBed": { "max_value": ["88"] }, + "HimSendFinish": { "event_period": ["-45"], "__value": ["-56"] }, + "HeavyTheBest": { + "archive_period": ["30"], + "__value": ["-33"], + "min_value": ["23"] + }, + "Road": { "min_alarm": ["37"] }, + "FootHistory": { "max_alarm": ["-67"] } + }, + "properties": { "EnoughPerhaps": ["raustin"] } + } + } + }, + "9Q3-TYI": { + "North": { + "WHITE/BUILD/VARIOUS-9": { + "attribute_properties": { + "EveningMeetingAnyone": { "max_alarm": ["-46"] }, + "SeaSchoolSet": { "rel_change": ["2"] } + }, + "properties": { + "HerDetermineEdge": ["jonesmegan"], + "MonthFilmNeedUnderstandPlant": ["smartinez", "johnsonmichael"] + } + }, + "DESIGN/DISCUSSION/ADD-9": { + "properties": { "TellTell": ["pamelafisher"] } + }, + "ANALYSIS/AWAY/PLAN-7": { + "attribute_properties": { + "Be": { "archive_rel_change": ["-63"], "description": ["-29"] } + }, + "properties": { + "While": ["holmeswilliam", "johnsonjason", "brandon15"], + "FutureDropFewFoot": [ + "ggarcia", + "warren87", + "lindavalentine", + "leroy71" + ] + } + } + }, + "DevelopFilm": { + "DRUG/NEWSPAPER/WE-7": { + "properties": { "HundredGroundYetPossible": ["victorharris"] } + } + } + }, + "IPCM": { + "Born": { + "SON/COULD/NATION-7": { + "attribute_properties": { "Remember": { "max_warning": ["-76"] } }, + "properties": { + "DemocratTypeSiteOwnInstitution": ["mlopez", "reyesann"] + } + }, + "MAJOR/CONTINUE/FAR-6": { + "attribute_properties": { + "SpecialTrainingThen": { + "max_warning": ["-67"], + "description": ["78"] + }, + "OrderFarFederal": { "max_warning": ["-34"] } + }, + "properties": { + "WriterRiseSaveLeavePerform": ["tbrown", "jenny34"], + "DogFoodEight": ["katherine56"] + } + } + }, + "Too": { + "KEY/RUN/CITY-0": { + "attribute_properties": { "TooLeaveMaybe": { "mode": ["-83"] } }, + "properties": { "AmongMyTheOutStructure": ["matthewkennedy"] } + }, + "PURPOSE/REFLECT/MAJORITY-8": { + "properties": { + "ThroughoutIDoPerhapsNecessary": ["dodsonthomas", "milesgregory"] + } + } + } + }, + "UC81Z-T7-MHHD7": { + "American": { + "BE/IMPORTANT/DURING-7": { + "attribute_properties": { + "NewspaperLong": { "display_unit": ["8"] } + }, + "properties": { "RecentlyLanguage": ["gregorydiane"] } + }, + "MOVE/SIZE/MISSION-7": { + "attribute_properties": { + "SimplyAnother": { "archive_abs_change": ["-37"] } + }, + "properties": { "WriteBar": ["wdunlap", "vcraig"] } + } + } + }, + "Y26G": { + "Leave": { + "AIR/DAY/GREAT-7": { + "attribute_properties": { + "WordSkinBuilding": { "max_warning": ["-47"] } + }, + "properties": { "HotPassDiscoverBed": ["wallen"] } + } + } + }, + "UK": { + "Include": { + "WORK/ENOUGH/ONCE-7": { + "attribute_properties": { + "ShareItselfHospital": { "delta_t": ["-57"] } + }, + "properties": { + "EverybodyEffectPositionMilitaryCrime": [ + "rthompson", + "christy61" + ], + "CommercialAlthoughUnderInstitutionSit": ["sierracabrera"], + "HitCenterSignificantChoiceIt": ["wmartin", "wallen"] + } + } + } + }, + "Z-S": { + "AgainListFamily": { + "BORN/HERSELF/BACK-5": { + "attribute_properties": { "LookInto": { "max_value": ["-94"] } }, + "properties": { + "Box": ["stevenperkins"], + "StandMorningEnoughOfficial": ["david90"], + "StreetThemselvesPassLet": ["padillasharon"], + "ButHighStarBaby": ["mercadochristine"] + } + }, + "DEVELOPMENT/NOR/SPORT-0": { + "properties": { "HearDemocratic": ["benjaminphillips", "lmurphy"] } + } + } + } + } + }, + "_title": "MAX-IV Tango JSON intermediate format", + "_date": "2006-10-25 02:25:12", + "_source": "education.xls" +} diff --git a/test/providers.py b/test/providers.py index 8e2784e..5daef3c 100644 --- a/test/providers.py +++ b/test/providers.py @@ -32,10 +32,10 @@ class TangoProvider(BaseProvider): def tango_property(self): n = randint(1, 5) - name = str("".join(_fake.word().capitalize() for i in xrange(n))) + name = str("".join(_fake.word().capitalize() for i in range(n))) n = int(ceil(expovariate(1))) # usually 1, sometimes larger - value = [str(_fake.user_name()) for i in xrange(n)] + value = [str(_fake.user_name()) for i in range(n)] return name, value @@ -46,10 +46,10 @@ def tango_attribute_property(self): def tango_attribute_config(self): n = randint(1, 3) - attr_name = str("".join(_fake.word().capitalize() for i in xrange(n))) + attr_name = str("".join(_fake.word().capitalize() for i in range(n))) n_props = int(ceil(expovariate(1))) # usually 1, sometimes larger attr_props = dict(_fake.tango_attribute_property() - for i in xrange(n_props)) + for i in range(n_props)) return attr_name, attr_props def tango_device(self): @@ -58,11 +58,11 @@ def tango_device(self): n_devprops = int(ceil(expovariate(1))) # usually 1, sometimes larger devprops = dict(_fake.tango_property() - for i in xrange(n_devprops)) + for i in range(n_devprops)) n_attrcfg = int(ceil(expovariate(1))) - 1 attrprops = dict(_fake.tango_attribute_config() - for i in xrange(n_attrcfg)) + for i in range(n_attrcfg)) value = {"properties": devprops} if attrprops: @@ -71,23 +71,23 @@ def tango_device(self): def tango_device_class(self): n = randint(1, 3) - name = str("".join(_fake.word().capitalize() for i in xrange(n))) + name = str("".join(_fake.word().capitalize() for i in range(n))) n_devs = int(ceil(expovariate(1))) - devices = dict(_fake.tango_device() for i in xrange(n_devs)) + devices = dict(_fake.tango_device() for i in range(n_devs)) return name, devices def tango_class(self): n = randint(1, 3) - name = str("".join(_fake.word().capitalize() for i in xrange(n))) + name = str("".join(_fake.word().capitalize() for i in range(n))) n_devprops = int(ceil(expovariate(1))) # usually 1, sometimes larger devprops = dict(_fake.tango_property() - for i in xrange(n_devprops)) + for i in range(n_devprops)) n_attrcfg = int(ceil(expovariate(1))) - 1 attrprops = dict(_fake.tango_attribute_config() - for i in xrange(n_attrcfg)) + for i in range(n_attrcfg)) value = {"properties": devprops} if attrprops: @@ -99,27 +99,27 @@ def tango_instance(self): n = randint(1, 3) chars = ascii_uppercase + digits name = "-".join("".join(choice(chars) - for i in xrange(randint(1, 5))) - for j in xrange(n)) + for i in range(randint(1, 5))) + for j in range(n)) n_classes = int(ceil(expovariate(1))) - value = dict(_fake.tango_device_class() for i in xrange(n_classes)) + value = dict(_fake.tango_device_class() for i in range(n_classes)) return name, value def tango_server(self): n = randint(1, 3) - name = "".join(_fake.word().capitalize() for i in xrange(n)) + name = "".join(_fake.word().capitalize() for i in range(n)) n_instances = randint(1, 10) - value = dict(_fake.tango_instance() for i in xrange(n_instances)) + value = dict(_fake.tango_instance() for i in range(n_instances)) return name, value def tango_database(self, servers=(5, 20), classes=(1, 5)): n_servers = randint(*servers) - servers = dict(_fake.tango_server() for i in xrange(n_servers)) + servers = dict(_fake.tango_server() for i in range(n_servers)) n_classes = randint(*classes) - classes = dict(_fake.tango_class() for i in xrange(n_classes)) + classes = dict(_fake.tango_class() for i in range(n_classes)) value = {"_title": "MAX-IV Tango JSON intermediate format", "_source": str(_fake.file_name(extension="xls")), "_date": str(_fake.date_time()), diff --git a/test/test_configure2.py b/test/test_configure2.py index 5bd4a2c..4f69297 100644 --- a/test/test_configure2.py +++ b/test/test_configure2.py @@ -13,7 +13,7 @@ import PyTango from dsconfig.configure import configure -from providers import TangoProvider +from .providers import TangoProvider fake = Faker() @@ -23,39 +23,39 @@ # # # # HELPERS # # # # def pick_random_server(config): - server = choice(config["servers"].keys()) + server = choice(list(config["servers"].keys())) return server, config["servers"][server] def pick_random_instance(config): servername, instances = pick_random_server(config) - instance = choice(instances.keys()) + instance = choice(list(instances.keys())) return servername, instance, instances[instance] def pick_random_class(config): servername, instname, classes = pick_random_instance(config) - classname = choice(classes.keys()) + classname = choice(list(classes.keys())) return servername, instname, classname, classes[classname] def pick_random_device(config): srv, inst, clss, devices = pick_random_class(config) - devname = choice(devices.keys()) + devname = choice(list(devices.keys())) return srv, inst, clss, devname, devices[devname] def pick_random_property(config): srv, inst, clss, dev, properties = pick_random_device(config) - propname = choice(properties["properties"].keys()) + propname = choice(list(properties["properties"].keys())) return srv, inst, clss, dev, propname, properties["properties"][propname] def pick_random_class_property(config): classes = config["classes"] - classname = choice(classes.keys()) + classname = choice(list(classes.keys())) properties = classes[classname] - propname = choice(properties["properties"].keys()) + propname = choice(list(properties["properties"].keys())) return classname, propname, properties["properties"][propname] @@ -191,7 +191,7 @@ def test_add_class_property(): orig_config = deepcopy(config) name, value = fake.tango_property() - clss = choice(config["classes"].keys()) + clss = choice(list(config["classes"].keys())) props = config["classes"][clss] props["properties"][name] = value calls = configure(config, orig_config) @@ -274,7 +274,7 @@ def test_add_class_attribute_property(): config = fake.tango_database(classes=(3, 5)) orig_config = deepcopy(config) - clss = choice(config["classes"].keys()) + clss = choice(list(config["classes"].keys())) props = config["classes"][clss] attr = "test_attribute" propname, value = fake.tango_attribute_property() @@ -296,7 +296,7 @@ def test_modify_class_attribute_property(): config = fake.tango_database() if "classes" in config: - clss = choice(config["classes"].keys()) + clss = choice(list(config["classes"].keys())) props = config["classes"][clss] attr = "test_attribute" propname, value = fake.tango_attribute_property() @@ -319,7 +319,7 @@ def test_cant_remove_protected_class_attribute_property(): config = fake.tango_database() - clss = choice(config["classes"].keys()) + clss = choice(list(config["classes"].keys())) props = config["classes"][clss] attr = "test_attribute" propname, value = fake.tango_attribute_property() @@ -401,7 +401,7 @@ def test_remove_device(): orig_config = deepcopy(config) _, _, classname, devices = pick_random_class(config) - device = choice(devices.keys()) + device = choice(list(devices.keys())) del devices[device] calls = configure(config, orig_config) diff --git a/test/test_dump.py b/test/test_dump.py new file mode 100644 index 0000000..e6d1371 --- /dev/null +++ b/test/test_dump.py @@ -0,0 +1,74 @@ +import json + +from mock import MagicMock, patch +from os.path import dirname, abspath, join + +from .test_tangodb import make_db +from dsconfig.dump import get_db_data + + +query1 = ("SELECT device, property_device.name, property_device.value FROM " + "property_device INNER JOIN device ON property_device.device = device.name " + "WHERE server LIKE '%' AND class LIKE '%' AND device LIKE '%' AND " + "class != 'DServer' AND property_device.name != '__SubDevices'") +query2 = ("SELECT device, attribute, property_attribute_device.name, " + "property_attribute_device.value FROM property_attribute_device INNER JOIN " + "device ON property_attribute_device.device = device.name WHERE server " + "LIKE '%' AND class LIKE '%' AND device LIKE '%' AND class != 'DServer'") +query3 = ("SELECT server, class, name, alias FROM device WHERE server LIKE '%' AND " + "class LIKE '%' AND name LIKE '%' AND class != 'DServer'") +query4 = ("select DISTINCT property_class.class, property_class.name, " + "property_class.value FROM property_class INNER JOIN device ON " + "property_class.class = device.class WHERE server like '%' AND " + "device.class != 'DServer' AND device.class != 'TangoAccessControl'") +query5 = ("select DISTINCT property_attribute_class.class, " + "property_attribute_class.attribute, property_attribute_class.name, " + "property_attribute_class.value FROM property_attribute_class INNER JOIN " + "device ON property_attribute_class.class = device.class WHERE server " + "like '%' AND device.class != 'DServer' AND " + "device.class != 'TangoAccessControl'") +query6 = ("select DISTINCT property_attribute_class.class, " + "property_attribute_class.attribute, property_attribute_class.name, " + "property_attribute_class.value FROM property_attribute_class INNER JOIN " + "device ON property_attribute_class.class = device.class WHERE server " + "like 'SOMEDEVICE' AND device.class != 'DServer' AND " + "device.class != 'TangoAccessControl'") +query7 = ("select DISTINCT property_class.class, property_class.name, " + "property_class.value FROM property_class INNER JOIN device ON " + "property_class.class = device.class WHERE server like 'SOMEDEVICE' AND " + "device.class != 'DServer' AND device.class != 'TangoAccessControl'") +query8 = ("SELECT device, attribute, property_attribute_device.name, " + "property_attribute_device.value FROM property_attribute_device INNER JOIN " + "device ON property_attribute_device.device = device.name WHERE server " + "LIKE 'SOMESERVER' AND class LIKE '%' AND device LIKE '%' AND " + "class != 'DServer'") + + +def test_db_dump(): + json_data_file = join(dirname(abspath(__file__)), 'files', 'sample_db.json') + with open(json_data_file, 'r') as json_file: + db_data = json.load(json_file) + db = make_db(db_data) + + with patch('dsconfig.dump.tango') as mocked_pytango: + + in_out_mock = MagicMock(name='in_out_mock', return_value = ("A", "B")) + device_proxy_mock = MagicMock(name='device_proxy_mock') + device_proxy_mock.command_inout = in_out_mock + mocked_pytango.DeviceProxy.return_value = device_proxy_mock + + get_db_data(db, class_properties=True) + assert in_out_mock.call_count == 5 + in_out_mock.assert_any_call('DbMySqlSelect', query1) + in_out_mock.assert_any_call('DbMySqlSelect', query2) + in_out_mock.assert_any_call('DbMySqlSelect', query3) + in_out_mock.assert_any_call('DbMySqlSelect', query4) + in_out_mock.assert_any_call('DbMySqlSelect', query5) + + in_out_mock.reset_mock() + get_db_data(db, patterns=["server:SOMESERVER", 'clss:SOMECLASS', + 'device:SOMEDEVICE'], class_properties=True) + assert in_out_mock.call_count == 15 + in_out_mock.assert_any_call('DbMySqlSelect', query6) + in_out_mock.assert_any_call('DbMySqlSelect', query7) + in_out_mock.assert_any_call('DbMySqlSelect', query8) diff --git a/test/test_excel.py b/test/test_excel.py deleted file mode 100644 index 241f545..0000000 --- a/test/test_excel.py +++ /dev/null @@ -1,135 +0,0 @@ -try: - from unittest2 import TestCase -except ImportError: - from unittest import TestCase - -from dsconfig import excel -from dsconfig.utils import CaselessDict - - -class TestExcel(TestCase): - - def test_get_properties_combined(self): - row = CaselessDict({"Properties": "a=1; b=2; c=3"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1"], "b": ["2"], "c": ["3"]}) - - def test_get_properties_separate(self): - row = CaselessDict({"Property:a": "1", "Property:b": "2"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1"], "b": ["2"]}) - - def test_get_properties_both(self): - row = CaselessDict({"Properties": "c=3", - "Property:a": "1", "Property:b": "2"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1"], "b": ["2"], "c": ["3"]}) - - def test_get_properties_splits_combined(self): - row = CaselessDict({"Properties": "a=1\n2\n3"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1", "2", "3"]}) - - def test_get_properties_splits_separate(self): - row = CaselessDict({"Property:a": "1\n2\n3"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1", "2", "3"]}) - - def test_get_properties_with_zero_value(self): - row = CaselessDict({"Property:a": 0}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["0"]}) - - def test_get_properties_with_types(self): - row = CaselessDict({"Property(int):a": 1.0, - "Property(float):b": 1.0}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1"], "b": ["1.0"]}) - - def test_get_dynamic_attribute(self): - row = CaselessDict({"name": "TestAttribute", "formula": "a + b", - "type": "int", "mode": "attr"}) - result = excel.get_dynamic(row) - self.assertDictEqual( - result.to_dict(), {"DynamicAttributes": ["TestAttribute=int(a + b)"]}) - - def test_get_dynamic_command(self): - row = CaselessDict({"name": "TestCommand", "formula": "1 + 2", - "type": "bool", "mode": "cmd"}) - result = excel.get_dynamic(row) - self.assertDictEqual( - result, {"DynamicCommands": ["TestCommand=bool(1 + 2)"]}) - - def test_get_dynamic_state(self): - row = CaselessDict({"name": "WEIRD", "formula": "1 == 2", - "type": "bool", "mode": "state"}) - result = excel.get_dynamic(row) - self.assertDictEqual( - result, {"DynamicStates": ["WEIRD=bool(1 == 2)"]}) - - def test_get_dynamic_status(self): - row = CaselessDict({"name": "Status", "formula": "'Something witty here'", - "type": "str", "mode": "status"}) - result = excel.get_dynamic(row) - self.assertDictEqual( - result, {"DynamicStatus": ["str('Something witty here')"]}) - - def test_dynamic_attribute_barfs_on_bad_syntax(self): - row = CaselessDict({"name": "TestAttribute", "formula": "a ? b", - "type": "int", "mode": "attr"}) - self.assertRaises(SyntaxError, excel.get_dynamic, row) - - def test_dynamic_attribute_errors_if_missing_stuff(self): - row = CaselessDict({"name": "TestAttribute", "formula": "a + b", - "type": "int"}) - self.assertRaises(ValueError, excel.get_dynamic, row) - - def test_get_attribute_properties(self): - row = CaselessDict({"Attribute": "myAttribute", - "AttributeProperties": "min_value=1;max_value=2"}) - result = excel.get_attribute_properties(row) - self.assertDictEqual(result, - {"myAttribute": {"min_value": ["1"], - "max_value": ["2"]}}) - - def test_get_attribute_properties_multiple(self): - row = CaselessDict({"Attribute": "myAttribute", - "AttrProp:Label": "Something", - "AttrProp:Min value": "45"}) - result = excel.get_attribute_properties(row) - self.assertDictEqual(result, - {"myAttribute": {"min_value": ["45"], - "label": ["Something"]}}) - - def test_get_attribute_properties_errors_on_invalid_name(self): - row = CaselessDict({"Attribute": "myAttribute", - "AttrProp:Label": "Something", - "AttrProp:Min value": "45", - "AttrProp:Not valid": "foo"}) - self.assertRaises(ValueError, excel.get_attribute_properties, row) - - def test_check_device_format_lets_valid_names_pass(self): - excel.check_device_format("i-a/test-test/device-0") - - def test_check_device_format_ignores_case(self): - excel.check_device_format("I-A/TEST-TEST/DEVICE-0") - - def test_check_device_format_catches_bad_names(self): - self.assertRaises(ValueError, excel.check_device_format, - "not/a/device/name") - - def test_check_device_format_catches_non_device_names(self): - self.assertRaises(ValueError, excel.check_device_format, "just a string") - - def test_check_device_format_catches_names_with_invalid_characters(self): - self.assertRaises(ValueError, excel.check_device_format, "I/can.has/dots") - - def test_format_server_instance(self): - row = {"server": "TestServer", "instance": 1} - result = excel.format_server_instance(row) - self.assertEqual(result, "TestServer/1") - - # def test_format_server_instance_handles_floats(self): - # row = {"server": "TestServer", "instance": 1.0} - # result = excel.format_server_instance(row) - # self.assertEqual(result, "TestServer/1") diff --git a/test/test_filtering.py b/test/test_filtering.py index 76fb07e..a2eeb38 100644 --- a/test/test_filtering.py +++ b/test/test_filtering.py @@ -168,6 +168,6 @@ def test_filter_json_include_several(self): filtered = filter_config(data, ["device:.*test/3", "device:^a/"], SERVERS_LEVELS) import json - print(json.dumps(filtered, indent=4)) - print(json.dumps(expected, indent=4)) + print((json.dumps(filtered, indent=4))) + print((json.dumps(expected, indent=4))) self.assertEqual(filtered, expected) diff --git a/test/test_json_to_tango.py b/test/test_json_to_tango.py new file mode 100644 index 0000000..9e8cee0 --- /dev/null +++ b/test/test_json_to_tango.py @@ -0,0 +1,52 @@ +from os.path import dirname, abspath, join +from mock import MagicMock, patch + +from dsconfig.json2tango import json_to_tango + + +def test_json_to_tango(capsys): + json_data_file = join(dirname(abspath(__file__)), 'files', 'sample_db.json') + + args = [json_data_file] + + options = MagicMock() + options.write = False + options.update = False + options.case_sensitive = True + options.verbose = True + options.output = False + options.input = False + options.dbcalls = True + options.validate = True + options.sleep = 0.0 + options.no_colors = True + options.include = [] + options.exclude = ['server:SOMESERVER'] + options.nostrictcheck = False + options.include_classes = [] + options.exclude_classes = ['class:SOMECLASS'] + options.dbdata = False + + with patch('dsconfig.json2tango.tango'): + with patch('dsconfig.json2tango.get_db_data') as mocked_get_db_data: + try: + json_to_tango(options, args) + except SystemExit: # The script exits with SystemExit even when successful + assert mocked_get_db_data.call_count == 1 + captured = capsys.readouterr() + + # stderr spot checks + assert 'Summary:' in captured.err + assert 'Add 49 servers.' in captured.err + assert 'Add 121 devices to 49 servers.' in captured.err + assert 'Add/change 207 device properties in 121 devices.' in captured.err + assert '108 device attribute properties in 51 devices.' in captured.err + assert 'Add/change 6 class properties.' in captured.err + assert 'Add/change 6 class attribute properties' in captured.err + + # stdout spot checks + assert "+ Device: GROW/RELATE/RICH-8" in captured.out + assert "Server: JobOfficer/X" in captured.out + assert " Class: Individual" in captured.out + assert " Properties:" in captured.out + assert " + PartnerTell" in captured.out diff --git a/test/test_tangodb.py b/test/test_tangodb.py index 7662b7b..fb591c2 100644 --- a/test/test_tangodb.py +++ b/test/test_tangodb.py @@ -16,12 +16,12 @@ def get_device_info(dev): def get_device_name(server, clss): srv, inst = server.split("/") - return dbdata["servers"][srv][inst][clss].keys() + return list(dbdata["servers"][srv][inst][clss].keys()) db.get_device_name.side_effect = get_device_name def get_device_property_list(dev, pattern): data, _ = find_device(dbdata, dev) - return data["properties"].keys() + return list(data["properties"].keys()) db.get_device_property_list.side_effect = get_device_property_list def get_device_property(dev, props): @@ -53,4 +53,4 @@ def test_get_dict_from_db(): }}}}}}} db = make_db(dbdata) - print get_dict_from_db(db, indata) + print(get_dict_from_db(db, indata)) diff --git a/test/test_utils.py b/test/test_utils.py index 54d06e3..79e95ad 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,8 +1,10 @@ import pytest +import unittest from mock import Mock from dsconfig.tangodb import is_protected -from dsconfig.utils import progressbar +from dsconfig.utils import progressbar, CaselessDict, ImmutableDict +from dsconfig.diff import print_diff @pytest.fixture @@ -29,3 +31,39 @@ def test_get_dict_from_db_skips_protected(db, monkeypatch): def test_progressbar_when_only_one_item(): progressbar(0, 1, 100) progressbar(0, 1, 100) + + +def test_caseless_dict(): + test_dict = CaselessDict({}) + test_dict['Key1'] = 'Value1' + assert 'KEY1' in test_dict + test_dict['keY1'] = 'Value1a' + assert test_dict['Key1'] != 'Value1' + assert 'Value1a' == test_dict.pop('key1') + + test_dict['Key2'] = 'Value2' + del(test_dict['kEy2']) + + test_dict['Key3'] = 'Value3' + test_dict.changekey('keY3') + assert 'key3' in test_dict + assert 'KEY3' in test_dict + + +class TestImmutableDict(unittest.TestCase): + def test_immutable(self): + test_dict = ImmutableDict({'key1': 'value1'}) + with self.assertRaises(TypeError): + test_dict['key2'] = 'value2' + + +def test_print_diff(capsys): + test_str = ('[{"op": "remove", "path": "/a"}, {"op": "add", "path": "/c", "value": 4}' + ', {"op": "replace", "path": "/b", "value": 3}]') + print(test_str) + print(str(print_diff({'a': 1, 'b': 2}, {'b': 3, 'c': 4}))) + assert test_str == str(print_diff({'a': 1, 'b': 2}, {'b': 3, 'c': 4})) + captured = capsys.readouterr() + assert "REMOVE:\n > a" in captured.out + assert "ADD:\n > c" in captured.out + assert "REPLACE:\n > b" in captured.out