From 3371908dcd11fe68c08f70b8fcce50f8bd938920 Mon Sep 17 00:00:00 2001 From: Sett Wai Date: Wed, 11 Sep 2019 16:25:09 +0200 Subject: [PATCH 01/22] Create basic pipeline file --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..11213c7 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,6 @@ +image: "nexus.engageska-portugal.pt/ska-docker/tango-dsconfig" + +test: + script: + - python setup.py test + From 490c605a3640dd2bd11d38b7d4fb835f3462d03f Mon Sep 17 00:00:00 2001 From: Sett Wai Date: Thu, 12 Sep 2019 09:58:20 +0200 Subject: [PATCH 02/22] Invoke `pytest` instead of plain `test` --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11213c7..adbc496 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,5 +2,5 @@ image: "nexus.engageska-portugal.pt/ska-docker/tango-dsconfig" test: script: - - python setup.py test + - python setup.py pytest From 3b58c0f2857c6307bb1c6985f71e34b0d2698af3 Mon Sep 17 00:00:00 2001 From: Sett Wai Date: Thu, 12 Sep 2019 15:32:34 +0200 Subject: [PATCH 03/22] Workaround: install pytest globally Issue related to https://github.com/pytest-dev/pluggy/issues/205 This should be resolved once we have ported to Python3 completely. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index adbc496..7e801e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,8 @@ image: "nexus.engageska-portugal.pt/ska-docker/tango-dsconfig" test: + before_script: + - pip install pytest script: - python setup.py pytest From a2dc8ef8e074cd5e5b40df27c92f4da1d5e4404d Mon Sep 17 00:00:00 2001 From: Sett Wai Date: Fri, 13 Sep 2019 11:17:33 +0200 Subject: [PATCH 04/22] Save coverage report as artifacts --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e801e7..a6bef21 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,4 +5,7 @@ test: - pip install pytest script: - python setup.py pytest + artifacts: + paths: + - htmlcov/ From e3aed300d384ec9b139f13193b7b8dfb73930385 Mon Sep 17 00:00:00 2001 From: Sett Wai Date: Fri, 13 Sep 2019 12:02:22 +0200 Subject: [PATCH 05/22] Enable coverage report in terminal too So that Gitlab CI can parse it. --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 319a641..d53fcea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,4 +4,5 @@ test = pytest [tool:pytest] addopts= --junit-xml=tests.xml --cov=dsconfig + --cov-report=term --cov-report=html From a8bc716104d69759dfb9b69608ebde215462c9e8 Mon Sep 17 00:00:00 2001 From: Sett Wai Date: Fri, 13 Sep 2019 13:47:50 +0200 Subject: [PATCH 06/22] Rebase master into automated build branch We want to always integrate with master from the mirror. --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6bef21..1613fd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,9 @@ image: "nexus.engageska-portugal.pt/ska-docker/tango-dsconfig" +default: + before_script: + - git rebase master + test: before_script: - pip install pytest From 2a8c135588e21e57b569458b1e3f89d2b473fa2c Mon Sep 17 00:00:00 2001 From: Sett Wai Date: Fri, 13 Sep 2019 14:14:42 +0200 Subject: [PATCH 07/22] Add coverage and junit report --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1613fd9..6f47530 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,8 @@ test: - pip install pytest script: - python setup.py pytest + coverage: '/^TOTAL.+?(\d+\%)$/' artifacts: - paths: - - htmlcov/ + reports: + junit: tests.xml From b5cd728341d91b8df6badb912977796f4140beda Mon Sep 17 00:00:00 2001 From: SKAJohanVenter Date: Mon, 30 Sep 2019 14:08:41 +0200 Subject: [PATCH 08/22] SAR-2 expanded unit tests for dsconfig. --- dsconfig/json2tango.py | 109 +-- test/files/json_sample_db.json | 1146 +++++++++++++++++++++++++++++ test/test_dump.py | 74 ++ test/test_excel.py | 8 + test/test_excel_alarms_userdef.py | 12 + test/test_json_to_tango.py | 52 ++ test/test_utils.py | 38 +- 7 files changed, 1386 insertions(+), 53 deletions(-) create mode 100644 test/files/json_sample_db.json create mode 100644 test/test_dump.py create mode 100644 test/test_excel_alarms_userdef.py create mode 100644 test/test_json_to_tango.py diff --git a/dsconfig/json2tango.py b/dsconfig/json2tango.py index 1abcd20..5c007a8 100755 --- a/dsconfig/json2tango.py +++ b/dsconfig/json2tango.py @@ -25,58 +25,7 @@ from dsconfig.appending_dict.caseless import CaselessDictionary -def main(): - - usage = "Usage: %prog [options] JSONFILE" - parser = OptionParser(usage=usage) - - parser.add_option("-w", "--write", dest="write", action="store_true", - help="write to the Tango DB", metavar="WRITE") - parser.add_option("-u", "--update", dest="update", action="store_true", - help="don't remove things, only add/update", - metavar="UPDATE") - parser.add_option("-c", "--case-sensitive", dest="case_sensitive", - action="store_true", - help=("Don't ignore the case of server, device, " - "attribute and property names"), - metavar="CASESENSITIVE") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose", default=True, - help="don't print actions to stderr") - parser.add_option("-o", "--output", dest="output", action="store_true", - help="Output the relevant DB state as JSON.") - parser.add_option("-p", "--input", dest="input", action="store_true", - help="Output the input JSON (after filtering).") - parser.add_option("-d", "--dbcalls", dest="dbcalls", action="store_true", - help="print out all db calls.") - parser.add_option("-v", "--no-validation", dest="validate", default=True, - action="store_false", help=("Skip JSON validation")) - parser.add_option("-s", "--sleep", dest="sleep", default=0.01, - type="float", - help=("Number of seconds to sleep between DB calls")) - parser.add_option("-n", "--no-colors", - action="store_true", dest="no_colors", default=False, - help="Don't print colored output") - parser.add_option("-i", "--include", dest="include", action="append", - help=("Inclusive filter on server configutation")) - parser.add_option("-x", "--exclude", dest="exclude", action="append", - help=("Exclusive filter on server configutation")) - parser.add_option("-a", "--no-strict-check", dest="nostrictcheck", - default=False, action="store_true", - help="Disable strick attribute property checking") - parser.add_option("-I", "--include-classes", dest="include_classes", - action="append", - help=("Inclusive filter on class configuration")) - parser.add_option("-X", "--exclude-classes", dest="exclude_classes", - action="append", - help=("Exclusive filter on class configuration")) - - parser.add_option( - "-D", "--dbdata", - help="Read the given file as DB data instead of using the actual DB", - dest="dbdata") - - options, args = parser.parse_args() +def json_to_tango(options, args): if options.no_colors: no_colors() @@ -228,5 +177,61 @@ def main(): sys.exit(SUCCESS) +def main(): + + usage = "Usage: %prog [options] JSONFILE" + parser = OptionParser(usage=usage) + + parser.add_option("-w", "--write", dest="write", action="store_true", + help="write to the Tango DB", metavar="WRITE") + parser.add_option("-u", "--update", dest="update", action="store_true", + help="don't remove things, only add/update", + metavar="UPDATE") + parser.add_option("-c", "--case-sensitive", dest="case_sensitive", + action="store_true", + help=("Don't ignore the case of server, device, " + "attribute and property names"), + metavar="CASESENSITIVE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print actions to stderr") + parser.add_option("-o", "--output", dest="output", action="store_true", + help="Output the relevant DB state as JSON.") + parser.add_option("-p", "--input", dest="input", action="store_true", + help="Output the input JSON (after filtering).") + parser.add_option("-d", "--dbcalls", dest="dbcalls", action="store_true", + help="print out all db calls.") + parser.add_option("-v", "--no-validation", dest="validate", default=True, + action="store_false", help=("Skip JSON validation")) + parser.add_option("-s", "--sleep", dest="sleep", default=0.01, + type="float", + help=("Number of seconds to sleep between DB calls")) + parser.add_option("-n", "--no-colors", + action="store_true", dest="no_colors", default=False, + help="Don't print colored output") + parser.add_option("-i", "--include", dest="include", action="append", + help=("Inclusive filter on server configutation")) + parser.add_option("-x", "--exclude", dest="exclude", action="append", + help=("Exclusive filter on server configutation")) + parser.add_option("-a", "--no-strict-check", dest="nostrictcheck", + default=False, action="store_true", + help="Disable strick attribute property checking") + parser.add_option("-I", "--include-classes", dest="include_classes", + action="append", + help=("Inclusive filter on class configuration")) + parser.add_option("-X", "--exclude-classes", dest="exclude_classes", + action="append", + help=("Exclusive filter on class configuration")) + + parser.add_option( + "-D", "--dbdata", + help="Read the given file as DB data instead of using the actual DB", + dest="dbdata") + + options, args = parser.parse_args() + + json_to_tango(options, args) + + if __name__ == "__main__": main() diff --git a/test/files/json_sample_db.json b/test/files/json_sample_db.json new file mode 100644 index 0000000..6a90534 --- /dev/null +++ b/test/files/json_sample_db.json @@ -0,0 +1,1146 @@ +{ + "classes": { + "ScienceYetChallenge": { + "properties": { + "RecordHead": ["wparks"], + "HotItself": ["tracy32"], + "DetailFear": ["haynesashlee"] + } + }, + "Site": { + "attribute_properties": { + "Approach": { + "delta_t": ["16"], + "__value": ["-22"], + "max_warning": ["-56"], + "unit": ["79"] + } + }, + "properties": { "ListenLess": ["melindahanson"] } + }, + "TreatHerselfPublic": { + "attribute_properties": { + "ResearchHeadOff": { "archive_period": ["-99"] } + }, + "properties": { "HowShouldWorldLong": ["joshuaholmes"] } + }, + "RecognizeRequire": { + "attribute_properties": { + "RequireGirlReflect": { "max_warning": ["-31"] } + }, + "properties": { "Teacher": ["chelsea64"] } + } + }, + "servers": { + "AttentionSea": { + "1JY-48-XAK": { + "HugeClassBook": { + "WEST/WILL/PROFESSIONAL-7": { + "attribute_properties": { + "MagazineLaugh": { "max_warning": ["46"], "format": ["16"] } + }, + "properties": { + "Car": ["evansjessica"], + "ShareLastNone": ["lindsey30"], + "ViewDreamShouldWhileOpen": ["cbanks"] + } + } + }, + "Life": { + "CONCERN/ENVIRONMENTAL/THEMSELVES-1": { + "properties": { + "DemocraticLocal": ["coledaniel", "juliecook", "eacosta"] + } + }, + "SAME/PROGRAM/SAFE-10": { + "properties": { + "Trouble": ["ovargas"], + "EnergyOneOff": ["carolinehendrix"], + "KeySkinItWalkReach": [ + "mobrien", + "davisalexandra", + "fburnett", + "qroberts" + ] + } + } + } + }, + "AXQ-AP-M8O7": { + "Rich": { + "WEST/OFFICER/COMPUTER-1": { + "attribute_properties": { + "DropCaseHere": { "abs_change": ["77"] } + }, + "properties": { + "SeemChangeHusbandMeetLot": ["alecfreeman"], + "TaskFocusGroupDecade": ["thomasmarie", "burtonjennifer"] + } + }, + "SITUATION/SIZE/WAY-4": { + "properties": { + "TeacherThereKitchenCapital": ["wayne35", "amberward"], + "NothingRelateAgainShoulderLaugh": ["yhart"], + "UseTurn": ["dustinjones", "joseph12", "esparzaandrew"] + } + }, + "ABOVE/AWAY/TEAM-9": { + "attribute_properties": { + "TreatBetterToward": { "max_warning": ["-92"] } + }, + "properties": { "CallThem": ["marissa86", "lisa32"] } + }, + "REVEAL/BAD/BODY-6": { "properties": { "Tax": ["jacksonpatricia"] } }, + "ROLE/HERSELF/SKIN-1": { + "attribute_properties": { + "Recent": { "rel_change": ["-32"], "archive_rel_change": ["-15"] } + }, + "properties": { "SeekSideMean": ["fyoung"] } + }, + "SO/ROOM/SECTION-7": { + "properties": { "LeaderInOpportunity": ["qrodriguez"] } + }, + "FRONT/LOT/THERE-2": { + "properties": { "Base": ["xbradley", "psmith", "william19"] } + } + } + }, + "WA-6": { + "IncludeParticipantMonth": { + "WEAR/LINE/QUESTION-7": { + "properties": { + "ParentHealthPositiveMinuteAccording": ["scottthompson"], + "OpportunityPassTwo": [ + "smithjennifer", + "michele44", + "debbieguzman", + "teresaallen" + ] + } + } + }, + "EnergyHeavyBuy": { + "THINK/DAY/WIN-4": { + "properties": { + "HeSendCitizen": [ + "mary84", + "wilcoxdaniel", + "audreybrown", + "cookjohn", + "lewisjeffrey" + ] + } + } + }, + "Campaign": { + "THANK/MODEL/POPULATION-10": { + "attribute_properties": { + "MeetingSitThat": { + "rel_change": ["-19"], + "archive_abs_change": ["98"] + }, + "IssueDevelop": { "archive_period": ["71"] }, + "IndeedCoverFact": { "__value": ["36"] }, + "Tell": { "mode": ["35"] } + }, + "properties": { "Suddenly": ["edwardsjill", "xbarajas"] } + } + }, + "Factor": { + "TRY/DIFFICULT/ON-1": { + "attribute_properties": { + "MeasureOld": { "min_warning": ["84"], "mode": ["79"] } + }, + "properties": { "AssumeTooGuess": ["nmartinez"] } + } + } + }, + "2N-UL7": { + "WithinCouldThem": { + "SHORT/WONDER/NATURAL-8": { + "properties": { + "Expect": ["rojassandra", "leemelissa", "sherri83"] + } + }, + "PROVE/CONTROL/MAGAZINE-0": { + "attribute_properties": { + "Line": { "__value": ["-82"], "unit": ["-16"] }, + "Side": { "unit": ["-32"] }, + "ListenHappen": { "__value": ["82"] } + }, + "properties": { "Place": ["peterboone", "hbarron", "rlynch"] } + } + } + }, + "X": { + "Media": { + "PREVENT/WHAT/SHARE-6": { "properties": { "Trouble": ["joe60"] } } + } + }, + "CV3-L": { + "MaybeHisTeacher": { + "ANYONE/WE/SET-10": { + "properties": { + "ManCollectionConsiderSomeonePolitics": ["arnoldkyle"], + "GunMediaPresentFather": ["denise75"] + } + }, + "PROTECT/NICE/LAW-4": { + "properties": { + "DogPowerWhom": ["wmanning"], + "GetAwayParticularly": ["jennifercastillo"] + } + } + }, + "NextDecisionOver": { + "THOUGHT/OCCUR/NEXT-3": { + "properties": { + "StockVariousFeelProgramBag": ["miahoward", "ywright"] + } + } + }, + "Article": { + "INSTEAD/ISSUE/FOR-0": { + "properties": { + "SubjectWorldUsuallyFeelDebate": ["davidterry"], + "RedSurfaceThemBlack": ["matthew12"] + } + }, + "SKILL/COLD/EVERYTHING-4": { + "attribute_properties": { + "Central": { "min_alarm": ["-71"] }, + "ResearchKitchen": { + "max_alarm": ["48"], + "min_warning": ["28"], + "mode": ["-65"] + }, + "RestFast": { "delta_val": ["-94"] } + }, + "properties": { + "ShortPriceSongOk": [ + "vbrown", + "billygarcia", + "vgrimes", + "wsmith", + "rarcher", + "daniel61" + ] + } + }, + "GET/NOR/FEW-0": { + "attribute_properties": { + "NorSuffer": { "archive_rel_change": ["-9"], "mode": ["-7"] }, + "EatRaise": { "abs_change": ["85"] }, + "OpenWaterFire": { "archive_rel_change": ["72"] } + }, + "properties": { "Treat": ["urodriguez"] } + } + } + } + }, + "RecentlyOnceCheck": { + "XS0-OJR5": { + "DetermineExplainNearly": { + "TEN/ABILITY/TAX-4": { + "attribute_properties": { + "ExactlyLater": { "__value": ["-86"] }, + "WestMostMoment": { "standard_unit": ["59"] }, + "Fact": { "max_value": ["-5"] } + }, + "properties": { + "AlwaysSixDetermine": ["brian58"], + "LikelyColorKeepOpen": ["apatton", "adam35"] + } + } + } + }, + "S-V89-2PUQP": { + "West": { + "STAFF/APPLY/ENOUGH-4": { + "properties": { "ReturnMeasureProjectAccount": ["jocelynleblanc"] } + } + } + }, + "3FSE": { + "HundredRequireCup": { + "PICK/APPLY/MATERIAL-5": { + "properties": { + "MoveJustWay": ["hramirez"], + "ConcernBoard": ["xgaines"] + } + }, + "CHOOSE/NOR/COLLECTION-0": { + "properties": { "ClassPrettyThanContainResource": ["jeffrey70"] } + }, + "DAUGHTER/ACROSS/REACH-3": { + "attribute_properties": { "Smile": { "max_value": ["-23"] } }, + "properties": { "DirectionExpect": ["qlopez"] } + }, + "TASK/POSITIVE/OUTSIDE-7": { + "properties": { + "FieldPutTodayGreatTotal": ["kylecollins"], + "FoodLawyer": ["denisesharp"] + } + }, + "PROGRAM/ME/IDEA-2": { + "properties": { "MainTreatmentImprove": ["amber81"] } + } + } + }, + "F5-7T-R": { + "StandardThrough": { + "HEALTH/EVIDENCE/ANY-7": { + "properties": { + "AtPutA": ["kathleen53"], + "AdmitGoSell": ["dudleymariah"], + "TodayFieldPlantServe": ["bkim"], + "BackArtist": ["diane28"] + } + }, + "HER/ITEM/RADIO-7": { + "attribute_properties": { + "NightPresidentVery": { "max_value": ["60"] } + }, + "properties": { + "Father": ["kwoods", "michaelsimon"], + "MoveRealizeEnterEatCentury": ["warnercaitlin", "eolsen"] + } + }, + "HUSBAND/ARGUE/EXECUTIVE-5": { + "properties": { + "SeriousMovementFollowMatter": ["jaredwilliams", "marissasalinas"] + } + }, + "PROVIDE/AUTHORITY/HOME-3": { + "attribute_properties": { + "LandStructureStop": { "unit": ["-23"] }, + "BallDog": { "label": ["12"] } + }, + "properties": { + "WhyParentFaceYourself": ["alexanderallen"], + "SubjectFallData": ["wgregory"] + } + } + } + } + }, + "Of": { + "MZ9N3-NO-FL": { + "SonAlongChange": { + "CONDITION/GUN/ARGUE-2": { + "properties": { + "LastIt": ["jacksonrichard", "xfisher", "fosterjason"], + "SendTraditional": ["troy76"] + } + } + } + } + }, + "AddExpect": { + "6JCZ-2D": { + "MeanBeautiful": { + "ADD/PAST/FALL-0": { + "attribute_properties": { + "OnlyItsLawyer": { "archive_abs_change": ["-22"] } + }, + "properties": { "MajorityThreeThese": ["perkinssandra"] } + }, + "THROUGHOUT/FUND/TONIGHT-1": { + "properties": { "Stop": ["ricecassie"] } + }, + "ACT/CHILD/LEARN-4": { + "properties": { + "QuicklyAlreadyApplyAction": [ + "tamaramiller", + "tiffanywilliams", + "kathleenfitzpatrick" + ] + } + } + } + }, + "3MY": { + "MediaYour": { + "GAS/SECOND/DECIDE-0": { + "attribute_properties": { + "AnythingEach": { "min_alarm": ["-88"], "mode": ["-81"] } + }, + "properties": { "LateTripMissionIndicate": ["ocooper"] } + } + } + }, + "CGXW-3Z6J-K": { + "SeasonMagazine": { + "INTEREST/LEAVE/FIRE-2": { + "attribute_properties": { + "Increase": { "__value_ts": ["-35"] }, + "EffortSouth": { "standard_unit": ["-81"] } + }, + "properties": { + "Always": ["sharoncole", "jmathews"], + "GameTendOfferShow": ["castromaria", "pricekaren"] + } + } + }, + "FaceCandidate": { + "GIVE/ALREADY/QUICKLY-9": { + "properties": { "GirlCongressBoyMore": ["mphelps"] } + }, + "PULL/AFFECT/FIGHT-3": { + "properties": { "OfficeLot": ["mikayla99", "davisjason"] } + } + } + }, + "WAAP-SS00-WNE": { + "ImportantIncludingBelieve": { + "HIMSELF/SUPPORT/DO-8": { + "attribute_properties": { "Early": { "rel_change": ["-5"] } }, + "properties": { + "Remain": [ + "ymcguire", + "roberttorres", + "brooke25", + "brandonsmith" + ], + "Son": ["deannamccullough"] + } + } + } + }, + "MUNYU-0VSAF": { + "Model": { + "A/ISSUE/ANOTHER-0": { + "attribute_properties": { "Ten": { "min_alarm": ["-51"] } }, + "properties": { "RatherSuch": ["parkersarah"] } + }, + "SUFFER/TOP/DEFENSE-10": { "properties": { "Quality": ["joseph55"] } } + }, + "Room": { + "THEY/STOCK/FOR-8": { + "properties": { "HitBarNotice": ["kevin21", "stephen46"] } + } + }, + "Your": { + "SPEAK/TURN/MAIN-6": { + "attribute_properties": { + "BornEat": { + "max_value": ["-54"], + "rel_change": ["39"], + "min_alarm": ["16"], + "standard_unit": ["-84"] + } + }, + "properties": { "EconomicInvestmentIndeedAlways": ["evanprice"] } + }, + "INVOLVE/WIN/AUTHOR-10": { + "attribute_properties": { "Cultural": { "rel_change": ["67"] } }, + "properties": { "InsteadOrder": ["jacksonaaron"] } + }, + "SERVE/YOU/QUESTION-1": { + "properties": { "FromToughStudentOffice": ["harriskelli"] } + } + } + }, + "G4NI-KTR2-OO": { + "VoiceRemember": { + "SEA/EASY/SURE-9": { + "attribute_properties": { + "PersonalInterestFirm": { "delta_t": ["-45"] } + }, + "properties": { + "Chance": ["garciajames", "patricia39", "phamjacqueline"], + "Long": ["carmen58", "mmunoz"] + } + } + }, + "AndHomeTest": { + "THREAT/EARLY/HUSBAND-9": { + "properties": { "To": ["qcole", "dbenton", "benjaminhayes"] } + }, + "GREEN/MOST/BEGIN-9": { + "properties": { + "Hot": ["carsonglenn"], + "WhoReceiveSurfacePolicy": ["vhowell", "jared29", "phillip84"] + } + } + } + }, + "9-8V9V": { + "ImproveRelationship": { + "FEEL/CHILD/CALL-9": { + "properties": { + "PersonBetterDifferenceSpeech": [ + "katherine80", + "kennethjohnson", + "robert45" + ] + } + } + } + }, + "GE-FEELL-K0EJV": { + "CareerDetermineAgency": { + "WAIT/PERFORM/STAR-7": { + "attribute_properties": { + "ScoreSuddenlyForeign": { "max_alarm": ["-92"] } + }, + "properties": { + "DayMorningSomethingDaughterLook": ["ibennett"], + "PlayerKitchenPersonAskOwn": ["dbrewer", "john62", "ewatson"], + "Stay": ["ramseycrystal"] + } + } + }, + "ListMotherPhone": { + "WANT/BECAUSE/AUTHOR-2": { + "properties": { "KeyRise": ["erikaevans", "jessica79"] } + } + } + }, + "X-S": { + "National": { + "OR/EVER/HUMAN-1": { + "properties": { + "CustomerAction": ["williamssergio"], + "AgoHeartPutSoundMrs": ["jimmy19", "oclark", "ericacarter"], + "PeaceVeryDetailBuyFinal": ["cfischer"], + "Blood": ["lisamahoney", "johnsonmichael"] + } + }, + "REQUIRE/ATTORNEY/HEAD-0": { + "properties": { + "WriterStopAgain": [ + "patrickmartin", + "kelly86", + "aaron78", + "myerskari", + "stephen09", + "lopezalice" + ] + } + }, + "HIMSELF/EXPLAIN/BEHIND-8": { + "properties": { + "EvidenceDownAddress": ["taylor46"], + "Why": ["brownrobert", "sabrinatodd"], + "AlwaysWord": ["sherrianderson", "tgonzalez"] + } + }, + "ESPECIALLY/ADDRESS/EFFORT-10": { + "properties": { "LittleTask": ["hornethan"] } + } + }, + "ThusDirection": { + "LOT/NEAR/UPON-3": { + "attribute_properties": { "Seven": { "format": ["-81"] } }, + "properties": { "DoctorTeacherRich": ["edwinlee"] } + } + } + } + }, + "PriceAlready": { + "Y": { + "ItselfNothingBudget": { + "DECISION/OTHER/GOVERNMENT-9": { + "properties": { + "InterestingTeacherSenior": ["harristheresa", "mark64"], + "ThroughRoad": [ + "josephlee", + "umclaughlin", + "josephmartinez", + "ashley14" + ] + } + } + }, + "JustJoin": { + "ISSUE/MISSION/THAT-4": { + "attribute_properties": { + "AgencyReasonCentury": { + "delta_t": ["-91"], + "min_alarm": ["-46"] + } + }, + "properties": { + "CommunityGoalAcross": ["bgordon", "mariapope"], + "GreatEstablishActPlan": ["clarkcollin", "mromero"], + "Final": ["joshua50", "amberthompson"], + "ResourceEverYear": ["morgantina"] + } + } + } + }, + "HOV-GYJMM-6F": { + "RespondBy": { + "INDICATE/ON/SOUTH-4": { + "properties": { + "KindAEarlyHow": ["philipfox"], + "BrotherQuality": ["apriljefferson"] + } + } + } + } + }, + "Clear": { + "1-U8K-3DION": { + "Else": { + "MOVEMENT/PART/LIVE-1": { + "properties": { "SoldierPhone": ["jennifer17"] } + }, + "LEAVE/COURT/TOWARD-4": { + "properties": { "Song": ["amanda82", "deannaporter"] } + } + } + }, + "85-A": { + "By": { + "THEIR/RECENT/INDEED-4": { + "attribute_properties": { + "MedicalTraditional": { "rel_change": ["7"] } + }, + "properties": { "DaughterAboveTestDraw": ["omeyer"] } + }, + "QUITE/OVER/OWN-1": { + "properties": { + "SortAcceptCareer": ["crawfordchristopher"], + "NationalTotalIntoThusAttack": ["ggibbs", "samanthapatton"], + "RoleKidDuring": ["karenyoung"] + } + }, + "FUTURE/MAN/STEP-7": { + "attribute_properties": { "End": { "max_alarm": ["46"] } }, + "properties": { "BillionDescribeBehavior": ["pamelanewman"] } + } + }, + "ColdHigh": { + "SERIES/MODERN/THOSE-8": { + "properties": { + "EvidenceAgencyTake": ["moorevictoria", "lisavasquez"], + "FineAppearTripOffice": ["joseph06", "beth75"] + } + }, + "BUILDING/STOCK/COLLEGE-2": { + "properties": { "EitherCurrentShouldStand": ["curtis25"] } + } + } + }, + "B-ONQCR-RJY": { + "Near": { + "ALLOW/FATHER/YET-4": { + "properties": { + "BadRespondCertainEveryone": ["richard14"], + "SeriesArgueWithin": ["patrickmahoney", "ychristensen"], + "BeforeIndicateGroup": ["cynthiaberry"] + } + } + } + }, + "VWVM": { + "Need": { + "TOTAL/ROCK/TRIAL-6": { + "properties": { + "YardPowerCommercialStyle": ["laurieperry"], + "LessRelationship": ["kaylagibbs"] + } + } + }, + "ThemSomething": { + "ITSELF/AMONG/DESCRIBE-7": { + "properties": { + "WrongStock": ["lrivera", "richleah"], + "WhyIntoHappenWestern": ["wmoore", "andrea05", "jennifer35"], + "Form": ["uking", "carrie27", "aandrews"], + "GovernmentAreaElse": ["sandraparsons"] + } + }, + "SUBJECT/CHARGE/SIMILAR-9": { + "attribute_properties": { + "WhyWhileChallenge": { "event_period": ["-80"] } + }, + "properties": { + "DataContainEvidenceTown": ["ashley89"], + "Phone": ["jillhoover"], + "Require": ["davisdwayne"], + "DifficultNewFive": ["madison05"] + } + } + }, + "GivePressureThreat": { + "ESTABLISH/YEAR/MY-10": { + "properties": { "DarkBackTreatmentI": ["timothy43"] } + } + } + }, + "5": { + "BloodThreatAny": { + "MIND/PRODUCT/FORM-9": { + "properties": { + "Whom": ["rhonda18"], + "TermTradeOffice": ["josephsmith"] + } + }, + "APPROACH/WAY/TRIAL-10": { + "attribute_properties": { + "PeopleAmongPut": { "delta_val": ["9"] } + }, + "properties": { + "StillAgencyFight": ["amoses"], + "MaterialCommunity": ["longcharles", "amandamartinez"] + } + } + }, + "MovementFederal": { + "CLASS/METHOD/STRATEGY-2": { + "properties": { + "QuicklyCulturalTreeAgree": ["gscott"], + "FineExperienceSinceInstitution": ["morarachel", "kara82"], + "Collection": ["priceelizabeth"], + "ForceItNetworkItself": [ + "michael14", + "bwerner", + "stephanie71", + "sheppardalicia" + ] + } + } + }, + "Education": { + "AUTHOR/PARTICULARLY/IMPACT-9": { + "attribute_properties": { + "NeverPurposePrice": { "event_period": ["56"] } + }, + "properties": { "Current": ["bbartlett", "kyle39"] } + } + }, + "BackDogAt": { + "KNOW/HEART/INVESTMENT-10": { + "attribute_properties": { + "DirectionProfessional": { + "rel_change": ["-14"], + "archive_rel_change": ["76"], + "__value_ts": ["79"] + } + }, + "properties": { + "FindCollectionFewStuffList": [ + "malonechristopher", + "xbradley", + "jose80" + ], + "TrainingPm": ["zbaxter"] + } + } + } + }, + "4ZA-5H": { + "HoweverTryOnce": { + "ACCORDING/HOW/DIFFERENT-1": { + "properties": { "Point": ["frenchjames"] } + } + }, + "WhatActShare": { + "ADDRESS/AGENCY/WATCH-7": { + "properties": { + "OldManagementShow": ["bennettteresa"], + "PageYourselfNew": ["jwalls", "andrew96"], + "YearBoard": ["sean49"] + } + }, + "GROW/FILL/BRING-6": { + "attribute_properties": { + "FormCampaign": { "format": ["-10"] }, + "AroundLargeHit": { "__value_ts": ["78"] }, + "ScienceConcernJob": { "event_period": ["2"] }, + "MuchNearCity": { "max_warning": ["51"] }, + "BedCarryAffect": { "archive_abs_change": ["59"] } + }, + "properties": { + "OccurFivePositiveWind": ["daykelly"], + "FaceThankSmileResult": ["kalexander"], + "FullKitchenInteresting": ["collinsdaniel"], + "CoachInterest": ["christopherhall", "qallen"], + "EarlyManage": ["underwoodmarissa", "ksnyder"], + "OilAgencyWorryWhether": ["christopher59"], + "Plan": ["mcdonaldjessica", "lewischristopher"] + } + } + } + }, + "D-LGWS6-Y104": { + "Sea": { + "ANALYSIS/MODEL/MONTH-2": { + "attribute_properties": { + "AlongSurfaceAssume": { "mode": ["64"] }, + "SomethingDifferenceDescribe": { + "delta_t": ["63"], + "standard_unit": ["94"], + "__value_ts": ["93"], + "abs_change": ["28"] + } + }, + "properties": { "CountrySide": ["erika52"] } + } + } + } + }, + "JobOfficer": { + "O9-8D-1Q": { + "SeniorAndMillion": { + "TOWARD/REMEMBER/SOURCE-5": { + "attribute_properties": { + "Born": { "event_period": ["-76"], "max_value": ["-48"] } + }, + "properties": { "RealizeSoldierCourse": ["mthompson", "eugene28"] } + } + }, + "SummerStandard": { + "CERTAIN/EAST/HISTORY-6": { + "properties": { + "AssumeReflectPrepareLargeGeneral": ["gonzalezgregory"], + "DevelopmentTogether": [ + "rileymichael", + "weberrichard", + "kimberlyibarra" + ], + "StarCheck": ["ibyrd", "umorse"] + } + } + }, + "NecessaryUntil": { + "FAST/THREAT/ONCE-5": { "properties": { "Idea": ["vbyrd"] } } + } + }, + "W8Q2-0XN-5": { + "UponRich": { + "LAND/WHERE/TRAINING-6": { + "attribute_properties": { + "Majority": { + "archive_period": ["-8"], + "event_period": ["-1"], + "max_value": ["-34"], + "label": ["-74"], + "__value": ["17"], + "archive_rel_change": ["-19"], + "max_warning": ["71"] + } + }, + "properties": { + "SeeIfAnimal": ["vkoch", "lwebster"], + "CompanyInteresting": ["xsparks", "jturner"], + "SpecialChallengeTeachFinishEmployee": ["snyderjacob"], + "GasProtectSport": ["jessica34", "hollyfigueroa", "tinaphillips"] + } + } + } + }, + "EWWMD-L": { + "Present": { + "GAS/GOOD/SEA-4": { + "properties": { "DefenseFoot": ["ronaldhester"] } + } + } + }, + "3RIM-ECO": { + "Just": { + "EFFECT/SERVE/BOY-6": { + "properties": { + "TellCertainlyYesRecently": ["obernard", "ofranklin"] + } + } + } + }, + "B5B0-G0EG-PXSSY": { + "FootQuickly": { + "GUY/FRIEND/WITHIN-6": { "properties": { "Give": ["asanders"] } } + } + }, + "X": { + "Individual": { + "EITHER/TV/FINE-0": { + "attribute_properties": { + "WellGasCompare": { "max_alarm": ["-86"] } + }, + "properties": { + "SimpleStructureWithRule": ["dawn97", "jenkinsgregory"] + } + }, + "GROW/RELATE/RICH-8": { + "properties": { + "PartnerTell": ["pharris"], + "ProductOptionOurPhone": ["masonjohn"] + } + } + } + }, + "R68": { + "Success": { + "WRITE/HIM/SPORT-6": { + "properties": { + "EstablishMessage": ["fisherchristopher"], + "ActivityRestBeforeThanToward": ["pamela80"] + } + }, + "GO/PARTICIPANT/NATURE-4": { + "properties": { + "ReflectManagerSituationBankOrder": [ + "molly55", + "david90", + "oscarpatel", + "omartinez" + ] + } + }, + "FORGET/OFFICE/OFFICER-2": { + "properties": { + "EnvironmentAccountStudentOldMother": ["volson", "swood"] + } + }, + "DECADE/STATION/SING-6": { + "properties": { + "FinePut": ["williamramos"], + "ChildEitherRepublicanDeep": ["briannorman"], + "AnythingRoad": ["ufaulkner"], + "NoEffect": ["melanie27"] + } + }, + "DURING/SERVE/FINALLY-10": { + "attribute_properties": { "WantMajor": { "min_warning": ["-13"] } }, + "properties": { "Wind": ["tony11"] } + }, + "BACK/COLOR/CALL-2": { + "properties": { + "MessageAppearNewFact": ["hayesjoseph"], + "LearnLeaveBecomeActuallyMorning": ["dominguezderek"] + } + } + } + }, + "97-R9PP-4": { + "Away": { + "PHYSICAL/NOTE/MIDDLE-1": { + "attribute_properties": { + "NoticeIdentify": { + "archive_period": ["-3"], + "max_alarm": ["-16"] + } + }, + "properties": { "NumberAhead": ["cunninghamandrew", "kendra68"] } + } + } + }, + "DD-YTR-2": { + "HospitalJustWell": { + "COMPANY/PULL/RESULT-3": { + "attribute_properties": { + "ExistLeft": { "__value": ["-26"], "delta_val": ["-3"] } + }, + "properties": { + "ReligiousQualityCreateVisit": ["debra64"], + "SortGive": ["shellytapia"], + "EndSubject": ["lanceruiz"] + } + } + } + }, + "L69J-MWDS": { + "RespondFactor": { + "NEAR/SOURCE/HER-10": { + "attribute_properties": { + "FarOfficerBorn": { "unit": ["-100"] }, + "WifeBut": { "archive_abs_change": ["14"] } + }, + "properties": { "SeasonAgainst": ["danny77"] } + } + } + } + }, + "Range": { + "E": { + "BankDoctorSon": { + "STEP/PARENT/FINISH-8": { + "attribute_properties": { + "LookNeverHimself": { "max_warning": ["74"] } + }, + "properties": { + "SingleNation": ["francisco76"], + "FallColorRange": ["emilynielsen"] + } + } + } + }, + "S-D3SJ-5E": { + "Join": { + "EVIDENCE/BETWEEN/ABILITY-6": { + "properties": { + "EffectSoonTurnKitchenClass": ["juangibson"], + "PartManagerGoFeel": ["pearsonisaiah"] + } + } + } + }, + "AI9HD-2R38-9SN": { + "Ready": { + "REAL/AWAY/DESIGN-7": { + "properties": { + "Card": ["williamsteresa", "sandra33"], + "NiceGuyBegin": ["hannahmelendez", "diane66"] + } + } + } + }, + "APEE-MM-PZV2": { + "ManagementAddressBall": { + "HISTORY/BENEFIT/FUND-2": { + "properties": { "DuringReasonSocial": ["rodriguezregina"] } + }, + "WHEN/AUTHORITY/SCHOOL-3": { + "attribute_properties": { + "WouldFoodBed": { "max_value": ["88"] }, + "HimSendFinish": { "event_period": ["-45"], "__value": ["-56"] }, + "HeavyTheBest": { + "archive_period": ["30"], + "__value": ["-33"], + "min_value": ["23"] + }, + "Road": { "min_alarm": ["37"] }, + "FootHistory": { "max_alarm": ["-67"] } + }, + "properties": { "EnoughPerhaps": ["raustin"] } + } + } + }, + "9Q3-TYI": { + "North": { + "WHITE/BUILD/VARIOUS-9": { + "attribute_properties": { + "EveningMeetingAnyone": { "max_alarm": ["-46"] }, + "SeaSchoolSet": { "rel_change": ["2"] } + }, + "properties": { + "HerDetermineEdge": ["jonesmegan"], + "MonthFilmNeedUnderstandPlant": ["smartinez", "johnsonmichael"] + } + }, + "DESIGN/DISCUSSION/ADD-9": { + "properties": { "TellTell": ["pamelafisher"] } + }, + "ANALYSIS/AWAY/PLAN-7": { + "attribute_properties": { + "Be": { "archive_rel_change": ["-63"], "description": ["-29"] } + }, + "properties": { + "While": ["holmeswilliam", "johnsonjason", "brandon15"], + "FutureDropFewFoot": [ + "ggarcia", + "warren87", + "lindavalentine", + "leroy71" + ] + } + } + }, + "DevelopFilm": { + "DRUG/NEWSPAPER/WE-7": { + "properties": { "HundredGroundYetPossible": ["victorharris"] } + } + } + }, + "IPCM": { + "Born": { + "SON/COULD/NATION-7": { + "attribute_properties": { "Remember": { "max_warning": ["-76"] } }, + "properties": { + "DemocratTypeSiteOwnInstitution": ["mlopez", "reyesann"] + } + }, + "MAJOR/CONTINUE/FAR-6": { + "attribute_properties": { + "SpecialTrainingThen": { + "max_warning": ["-67"], + "description": ["78"] + }, + "OrderFarFederal": { "max_warning": ["-34"] } + }, + "properties": { + "WriterRiseSaveLeavePerform": ["tbrown", "jenny34"], + "DogFoodEight": ["katherine56"] + } + } + }, + "Too": { + "KEY/RUN/CITY-0": { + "attribute_properties": { "TooLeaveMaybe": { "mode": ["-83"] } }, + "properties": { "AmongMyTheOutStructure": ["matthewkennedy"] } + }, + "PURPOSE/REFLECT/MAJORITY-8": { + "properties": { + "ThroughoutIDoPerhapsNecessary": ["dodsonthomas", "milesgregory"] + } + } + } + }, + "UC81Z-T7-MHHD7": { + "American": { + "BE/IMPORTANT/DURING-7": { + "attribute_properties": { + "NewspaperLong": { "display_unit": ["8"] } + }, + "properties": { "RecentlyLanguage": ["gregorydiane"] } + }, + "MOVE/SIZE/MISSION-7": { + "attribute_properties": { + "SimplyAnother": { "archive_abs_change": ["-37"] } + }, + "properties": { "WriteBar": ["wdunlap", "vcraig"] } + } + } + }, + "Y26G": { + "Leave": { + "AIR/DAY/GREAT-7": { + "attribute_properties": { + "WordSkinBuilding": { "max_warning": ["-47"] } + }, + "properties": { "HotPassDiscoverBed": ["wallen"] } + } + } + }, + "UK": { + "Include": { + "WORK/ENOUGH/ONCE-7": { + "attribute_properties": { + "ShareItselfHospital": { "delta_t": ["-57"] } + }, + "properties": { + "EverybodyEffectPositionMilitaryCrime": [ + "rthompson", + "christy61" + ], + "CommercialAlthoughUnderInstitutionSit": ["sierracabrera"], + "HitCenterSignificantChoiceIt": ["wmartin", "wallen"] + } + } + } + }, + "Z-S": { + "AgainListFamily": { + "BORN/HERSELF/BACK-5": { + "attribute_properties": { "LookInto": { "max_value": ["-94"] } }, + "properties": { + "Box": ["stevenperkins"], + "StandMorningEnoughOfficial": ["david90"], + "StreetThemselvesPassLet": ["padillasharon"], + "ButHighStarBaby": ["mercadochristine"] + } + }, + "DEVELOPMENT/NOR/SPORT-0": { + "properties": { "HearDemocratic": ["benjaminphillips", "lmurphy"] } + } + } + } + } + }, + "_title": "MAX-IV Tango JSON intermediate format", + "_date": "2006-10-25 02:25:12", + "_source": "education.xls" +} diff --git a/test/test_dump.py b/test/test_dump.py new file mode 100644 index 0000000..5713551 --- /dev/null +++ b/test/test_dump.py @@ -0,0 +1,74 @@ +import json + +from mock import MagicMock, patch +from os.path import dirname, abspath, join + +from test_tangodb import make_db +from dsconfig.dump import get_db_data + + +query1 = ("SELECT device, property_device.name, property_device.value FROM " + "property_device INNER JOIN device ON property_device.device = device.name " + "WHERE server LIKE '%' AND class LIKE '%' AND device LIKE '%' AND " + "class != 'DServer' AND property_device.name != '__SubDevices'") +query2 = ("SELECT device, attribute, property_attribute_device.name, " + "property_attribute_device.value FROM property_attribute_device INNER JOIN " + "device ON property_attribute_device.device = device.name WHERE server " + "LIKE '%' AND class LIKE '%' AND device LIKE '%' AND class != 'DServer'") +query3 = ("SELECT server, class, name, alias FROM device WHERE server LIKE '%' AND " + "class LIKE '%' AND name LIKE '%' AND class != 'DServer'") +query4 = ("select DISTINCT property_class.class, property_class.name, " + "property_class.value FROM property_class INNER JOIN device ON " + "property_class.class = device.class WHERE server like '%' AND " + "device.class != 'DServer' AND device.class != 'TangoAccessControl'") +query5 = ("select DISTINCT property_attribute_class.class, " + "property_attribute_class.attribute, property_attribute_class.name, " + "property_attribute_class.value FROM property_attribute_class INNER JOIN " + "device ON property_attribute_class.class = device.class WHERE server " + "like '%' AND device.class != 'DServer' AND " + "device.class != 'TangoAccessControl'") +query6 = ("select DISTINCT property_attribute_class.class, " + "property_attribute_class.attribute, property_attribute_class.name, " + "property_attribute_class.value FROM property_attribute_class INNER JOIN " + "device ON property_attribute_class.class = device.class WHERE server " + "like 'SOMEDEVICE' AND device.class != 'DServer' AND " + "device.class != 'TangoAccessControl'") +query7 = ("select DISTINCT property_class.class, property_class.name, " + "property_class.value FROM property_class INNER JOIN device ON " + "property_class.class = device.class WHERE server like 'SOMEDEVICE' AND " + "device.class != 'DServer' AND device.class != 'TangoAccessControl'") +query8 = ("SELECT device, attribute, property_attribute_device.name, " + "property_attribute_device.value FROM property_attribute_device INNER JOIN " + "device ON property_attribute_device.device = device.name WHERE server " + "LIKE 'SOMESERVER' AND class LIKE '%' AND device LIKE '%' AND " + "class != 'DServer'") + + +def test_db_dump(): + json_data_file = join(dirname(abspath(__file__)), 'files', 'json_sample_db.json') + with open(json_data_file, 'r') as json_file: + db_data = json.load(json_file) + db = make_db(db_data) + + with patch('dsconfig.dump.PyTango') as mocked_pytango: + + in_out_mock = MagicMock(name='in_out_mock', return_value = ("A", "B")) + device_proxy_mock = MagicMock(name='device_proxy_mock') + device_proxy_mock.command_inout = in_out_mock + mocked_pytango.DeviceProxy.return_value = device_proxy_mock + + get_db_data(db, class_properties=True) + assert in_out_mock.call_count == 5 + in_out_mock.assert_any_call('DbMySqlSelect', query1) + in_out_mock.assert_any_call('DbMySqlSelect', query2) + in_out_mock.assert_any_call('DbMySqlSelect', query3) + in_out_mock.assert_any_call('DbMySqlSelect', query4) + in_out_mock.assert_any_call('DbMySqlSelect', query5) + + in_out_mock.reset_mock() + get_db_data(db, patterns=["server:SOMESERVER", 'clss:SOMECLASS', + 'device:SOMEDEVICE'], class_properties=True) + assert in_out_mock.call_count == 15 + in_out_mock.assert_any_call('DbMySqlSelect', query6) + in_out_mock.assert_any_call('DbMySqlSelect', query7) + in_out_mock.assert_any_call('DbMySqlSelect', query8) diff --git a/test/test_excel.py b/test/test_excel.py index 241f545..a4b930a 100644 --- a/test/test_excel.py +++ b/test/test_excel.py @@ -1,3 +1,5 @@ +from os.path import dirname, abspath, join + try: from unittest2 import TestCase except ImportError: @@ -133,3 +135,9 @@ def test_format_server_instance(self): # row = {"server": "TestServer", "instance": 1.0} # result = excel.format_server_instance(row) # self.assertEqual(result, "TestServer/1") + + def test_xls_to_dict(self): + xls_file = join(dirname(abspath(__file__)), 'files', 'AlarmDemoV6.xls') + data = excel.xls_to_dict(xls_file) + stats = excel.get_stats(data) + assert stats == {'instances': 0, 'classes': 0, 'devices': 0, 'servers': 0} diff --git a/test/test_excel_alarms_userdef.py b/test/test_excel_alarms_userdef.py new file mode 100644 index 0000000..1aca424 --- /dev/null +++ b/test/test_excel_alarms_userdef.py @@ -0,0 +1,12 @@ +from os.path import dirname, abspath, join + +from dsconfig.excel_alarms_userdef import xls_to_dict + + +def test_excel_alarms_user_def(): + xls_file = join(dirname(abspath(__file__)), 'files', 'AlarmDemoV6.xls') + result = xls_to_dict(xls_file) + assert result['servers']['PyAlarm/']['PyAlarm'] + alarm_prop = result['servers']['PyAlarm/mypyalarm']['PyAlarm']['Alarms/test/1'] + assert alarm_prop['properties']['AlarmSeverities'] == ['VacPressure:WARNING', + 'MagCurrent:ALARM'] diff --git a/test/test_json_to_tango.py b/test/test_json_to_tango.py new file mode 100644 index 0000000..21ed6b3 --- /dev/null +++ b/test/test_json_to_tango.py @@ -0,0 +1,52 @@ +from os.path import dirname, abspath, join +from mock import MagicMock, patch + +from dsconfig.json2tango import json_to_tango + + +def test_json_to_tango(capsys): + json_data_file = join(dirname(abspath(__file__)), 'files', 'json_sample_db.json') + + args = [json_data_file] + + options = MagicMock() + options.write = False + options.update = False + options.case_sensitive = True + options.verbose = True + options.output = False + options.input = False + options.dbcalls = True + options.validate = True + options.sleep = 0.0 + options.no_colors = True + options.include = [] + options.exclude = ['server:SOMESERVER'] + options.nostrictcheck = False + options.include_classes = [] + options.exclude_classes = ['class:SOMECLASS'] + options.dbdata = False + + with patch('dsconfig.json2tango.PyTango'): + with patch('dsconfig.json2tango.get_db_data') as mocked_get_db_data: + try: + json_to_tango(options, args) + except SystemExit: # The script exits with SystemExit even when successful + assert mocked_get_db_data.call_count == 1 + captured = capsys.readouterr() + + # stderr spot checks + assert 'Summary:' in captured.err + assert 'Add 49 servers.' in captured.err + assert 'Add 121 devices to 49 servers.' in captured.err + assert 'Add/change 207 device properties in 121 devices.' in captured.err + assert '108 device attribute properties in 51 devices.' in captured.err + assert 'Add/change 6 class properties.' in captured.err + assert 'Add/change 6 class attribute properties' in captured.err + + # stdout spot checks + assert "+ Device: GROW/RELATE/RICH-8" in captured.out + assert "Server: JobOfficer/X" in captured.out + assert " Class: Individual" in captured.out + assert " Properties:" in captured.out + assert " + PartnerTell" in captured.out diff --git a/test/test_utils.py b/test/test_utils.py index 54d06e3..ad367c2 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,8 +1,10 @@ import pytest +import unittest from mock import Mock from dsconfig.tangodb import is_protected -from dsconfig.utils import progressbar +from dsconfig.utils import progressbar, CaselessDict, ImmutableDict +from dsconfig.diff import print_diff @pytest.fixture @@ -29,3 +31,37 @@ def test_get_dict_from_db_skips_protected(db, monkeypatch): def test_progressbar_when_only_one_item(): progressbar(0, 1, 100) progressbar(0, 1, 100) + + +def test_caseless_dict(): + test_dict = CaselessDict({}) + test_dict['Key1'] = 'Value1' + assert 'KEY1' in test_dict + test_dict['keY1'] = 'Value1a' + assert test_dict['Key1'] != 'Value1' + assert 'Value1a' == test_dict.pop('key1') + + test_dict['Key2'] = 'Value2' + del(test_dict['kEy2']) + + test_dict['Key3'] = 'Value3' + test_dict.changekey('keY3') + assert 'key3' in test_dict + assert 'KEY3' in test_dict + + +class TestImmutableDict(unittest.TestCase): + def test_immutable(self): + test_dict = ImmutableDict({'key1': 'value1'}) + with self.assertRaises(TypeError): + test_dict['key2'] = 'value2' + + +def test_print_diff(capsys): + test_str = ('[{"path": "/a", "op": "remove"}, {"path": "/c", "value": 4, "op": "add"}' + ', {"path": "/b", "value": 3, "op": "replace"}]') + assert test_str == str(print_diff({'a': 1, 'b': 2}, {'b': 3, 'c': 4})) + captured = capsys.readouterr() + assert "REMOVE:\n > a" in captured.out + assert "ADD:\n > c" in captured.out + assert "REPLACE:\n > b" in captured.out From f4db645b92e4216ca195dbb7bab79f3d5869f60d Mon Sep 17 00:00:00 2001 From: jventer Date: Tue, 1 Oct 2019 10:14:35 +0200 Subject: [PATCH 09/22] Improved sample DB file name --- test/files/{json_sample_db.json => sample_db.json} | 0 test/test_dump.py | 2 +- test/test_json_to_tango.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename test/files/{json_sample_db.json => sample_db.json} (100%) diff --git a/test/files/json_sample_db.json b/test/files/sample_db.json similarity index 100% rename from test/files/json_sample_db.json rename to test/files/sample_db.json diff --git a/test/test_dump.py b/test/test_dump.py index 5713551..6068709 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -45,7 +45,7 @@ def test_db_dump(): - json_data_file = join(dirname(abspath(__file__)), 'files', 'json_sample_db.json') + json_data_file = join(dirname(abspath(__file__)), 'files', 'sample_db.json') with open(json_data_file, 'r') as json_file: db_data = json.load(json_file) db = make_db(db_data) diff --git a/test/test_json_to_tango.py b/test/test_json_to_tango.py index 21ed6b3..1616a98 100644 --- a/test/test_json_to_tango.py +++ b/test/test_json_to_tango.py @@ -5,7 +5,7 @@ def test_json_to_tango(capsys): - json_data_file = join(dirname(abspath(__file__)), 'files', 'json_sample_db.json') + json_data_file = join(dirname(abspath(__file__)), 'files', 'sample_db.json') args = [json_data_file] From 6c0bc871d84e425d417fac493ad3ba1e7ac2b104 Mon Sep 17 00:00:00 2001 From: jventer Date: Tue, 8 Oct 2019 08:22:46 -0400 Subject: [PATCH 10/22] Deleted files to match PR 12 in the gitlab lib-maxiv-dsconfig project --- bin/csv2json | 2 - bin/xls2json | 5 - dsconfig/callcsv.py | 289 ---------- dsconfig/excel.py | 335 ------------ dsconfig/excel_alarms_userdef.py | 60 -- dsconfig/excel_alarms_vac.py | 118 ---- dsconfig/magnets2json.py | 883 ------------------------------ setup.py | 5 - test/test_excel.py | 143 ----- test/test_excel_alarms_userdef.py | 12 - 10 files changed, 1852 deletions(-) delete mode 100755 bin/csv2json delete mode 100755 bin/xls2json delete mode 100644 dsconfig/callcsv.py delete mode 100644 dsconfig/excel.py delete mode 100644 dsconfig/excel_alarms_userdef.py delete mode 100644 dsconfig/excel_alarms_vac.py delete mode 100644 dsconfig/magnets2json.py delete mode 100644 test/test_excel.py delete mode 100644 test/test_excel_alarms_userdef.py diff --git a/bin/csv2json b/bin/csv2json deleted file mode 100755 index 2551e7f..0000000 --- a/bin/csv2json +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python -m dsconfig.callcsv $* diff --git a/bin/xls2json b/bin/xls2json deleted file mode 100755 index 3d32bf8..0000000 --- a/bin/xls2json +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python - -from dsconfig.excel import main - -main() diff --git a/dsconfig/callcsv.py b/dsconfig/callcsv.py deleted file mode 100644 index 606198c..0000000 --- a/dsconfig/callcsv.py +++ /dev/null @@ -1,289 +0,0 @@ -"""Provide functions to parse a callable csv file.""" - -# Imports - -import os -import sys -import csv -import json -from collections import Mapping -from importlib import import_module -from optparse import OptionParser - - -# Utils - -def special_update(d, u): - """Update nested dictionnaries while prioritizing the first argument.""" - if not (isinstance(d, Mapping) and isinstance(u, Mapping)): - return d if d is not None else u - for k, v in u.iteritems(): - d[k] = special_update(d.get(k), v) - return d - - -def max_or_none(*args): - """Maximum function considering None as a maximum value.""" - return None if None in args else max(*args) - - -def cast_list(lst): - """Convert a list of a string to the corresponding value.""" - result = [] - for value in lst: - # Integer conversion - try: - value = int(value) - except (ValueError, TypeError): - # Float conversion - try: - value = float(value) - except (ValueError, TypeError): - # Hexa conversion - try: - value = int(value, 16) - except (ValueError, TypeError): - # Ignore - pass - # Append - result.append(value) - # Return - if not result: - return "" - if len(result) == 1: - return result[0] - return tuple(result) - - -# Csv functions - -def get_column(matrix, col, start=None, stop=None, step=None): - """Get the column of a matrix, with optional range arguments.""" - return [row[col] for row in matrix][slice(start, stop, step)] - - -def get_markup_index(matrix, col, markup): - """Find a markup in a given column and return the following index.""" - for i, key in enumerate(get_column(matrix, col)): - if key == markup: - return i+1 - return None - - -def get_range_lst(matrix, col, start=0, stop=None, markup=None): - """Get a value->range dictionnary from a given column.""" - if markup: - start = max_or_none(start, get_markup_index(matrix, col, markup)) - if start is None: - return {} - result = [] - previous_key, previous_start = None, None - for i, key in enumerate(get_column(matrix, col, start, stop), start): - if key != "": - if previous_key is not None: - result.append((previous_key, previous_start, i)) - previous_key, previous_start = key, i - if previous_key is not None: - result.append((previous_key, previous_start, i+1)) - return result - - -# Callable csv functions - -def get_kwargs(matrix, start, stop): - """Get the keywords arguments between two indexes.""" - kwargs = {} - keyword_lst = get_range_lst(matrix, 2, start, stop) - for keyword, start, stop in keyword_lst: - lst = get_range_lst(matrix, 3, start, stop) - values = [key for key, _, _ in lst] - kwargs[str(keyword)] = cast_list(values) - return kwargs - - -def get_call_list(filename): - """Get the call list from a callable cls file.""" - result = [] - with open(filename) as csvfile: - reader = csv.reader(csvfile, delimiter=',') - matrix = [[value.strip() for value in row] for row in reader] - package_lst = get_range_lst(matrix, 0, markup="Package") - # Process - result = [] - for package, start, stop in package_lst: - func_lst = get_range_lst(matrix, 1, start, stop) - for func, start, stop in func_lst: - kwargs = get_kwargs(matrix, start, stop) - result.append((package, func, kwargs)) - return result - - -# Data functions - -def process_call_list(lst, skip=False, verbose=True): - """Process a given call list and return the results.""" - result = [] - errors = ImportError, ValueError, TypeError, AttributeError - for module_name, func_name, kwargs in lst: - # Build prototype - prototype = "{0}.{1}(".format(module_name, func_name) - for key, value in kwargs.items(): - prototype += '{0}={1}, '.format(key, value) - if prototype.endswith(' '): - prototype = prototype[:-2] - prototype += ')' - # Print prototype - if verbose: - print "Executing: " + prototype - # Execute - try: - module = import_module(module_name) - func = getattr(module, func_name) - value = func(**kwargs) - # Fail - except errors as exc: - if not skip: - raise exc - else: - print(exc) - # Success - else: - result.append(value) - return result - - -def join_data(lst, source=None): - """Join a list of json strings or dictionnaries into a single dict.""" - data = {} - for mapping in lst: - if isinstance(mapping, basestring): - mapping = json.loads(mapping) - special_update(data, mapping) - if source: - data['_source'] = source - return data - - -# CSV to Dict function - -def callable_csv_to_dict(filename, skip=False, verbose=True, to_json=False): - """Convert a callable csv file to a data dictionnary.""" - calls = get_call_list(filename) - if not calls: - return - strings = process_call_list(calls, skip, verbose) - data = join_data(strings, filename) - if to_json: - return json.dumps(data, indent=4, sort_keys=True) - return data - - -# Command lines arguments for configuration script - -def parse_command_line_args(desc): - """Parse arguments given in command line""" - usage = "%prog [-i INPUT] [-o OUTPUT] [-v] [-w]" - parser = OptionParser(usage=usage, description=desc, version='%prog v1.0') - - msg = "The input callable csv file" - parser.add_option('-i', '--input', metavar='IN', - type='str', help=msg, default='') - - msg = "The output tango database json file" - parser.add_option('-o', '--output', metavar='OUT', - type='str', help=msg, default='') - - msg = "Display informations" - parser.add_option('-v', '--verbose', - action="store_true", help=msg, default=False) - - msg = "Write the Tango Database" - parser.add_option('-w', '--write', - action="store_true", help=msg, default=False) - - options, args = parser.parse_args() - - if args: - msg = "No argument expected, options only.\n" - msg += "Use --help for further information." - parser.error(msg) - - return options.input, options.output, options.write, options.verbose - - -# Main function for configuration scripts - -def main(desc=None, module_name=None, function=None): - """Run the script.""" - kwargs = {} - remove = False - desc = desc or "Generate a Tango json file for a given callable csv file." - # Parse command line args - input_file, output_file, write, verbose = parse_command_line_args(desc) - # Process input file - if input_file and module_name and function: - prototype = ".".join((module_name, function.__name__)) - for module, func, keywords in get_call_list(input_file): - if module == module_name and func == function.__name__: - kwargs = keywords - if verbose: - print("'{0}' found".format(prototype)) - print("kwargs = " + str(kwargs)) - break - else: - msg = "'{0}' not found in {1}" - print msg.format(prototype, get_call_list(input_file)) - return - # Generate json file - if module_name and function: - if not input_file and verbose: - msg = 'No input file given. ' - msg += 'Default configuration will be used' - print(msg) - data = function(**kwargs) - if isinstance(data, Mapping): - string = json.dumps(data, indent=4, sort_keys=True) - elif isinstance(data, basestring): - string = data - else: - msg = "The function didn't return a valid data format.\n" - msg += "The type is {0} instead.".format(type(data)) - print(msg) - print(data) - return - elif input_file: - string = callable_csv_to_dict(input_file, True, verbose, True) - else: - print('An input file is required.') - return - # Display json file - if verbose: - print('Json string generated:') - print(string) - # Write temporary file - if output_file == "" and write: - remove = True - output_file = "temp.json" - # Write output file - if output_file: - with open(output_file, mode='w') as f: - f.write(string) - if verbose and output_file: - print('Exported to: ' + output_file) - # Write tango database - if write: - from dsconfig import configure - sys.argv = [__name__, output_file, "-w"] - configure.main() - # Remove temporary file - if remove: - os.remove(output_file) - if verbose: - print('Removed: ' + output_file) - print('OK!') - - -# Main execution - -if __name__ == "__main__": - main() diff --git a/dsconfig/excel.py b/dsconfig/excel.py deleted file mode 100644 index 80a688c..0000000 --- a/dsconfig/excel.py +++ /dev/null @@ -1,335 +0,0 @@ -""" -Routines for reading an Excel file containing server, class and device definitions, -producing a file in the TangoDB JSON format. -""" - -from datetime import datetime -import json -import os -import re -import sys -#from traceback import format_exc - -from utils import find_device -from appending_dict import AppendingDict -from utils import CaselessDict -from tangodb import SPECIAL_ATTRIBUTE_PROPERTIES - -MODE_MAPPING = CaselessDict({"ATTR": "DynamicAttributes", - "CMD": "DynamicCommands", - "STATE": "DynamicStates", - "STATUS": "DynamicStatus"}) - -TYPE_MAPPING = CaselessDict({"INT": int, - "FLOAT": float}) - -def get_properties(row): - - "Find property definitions on a row" - - prop_dict = AppendingDict() - - # "Properties" column - # The cell value is expected to be on the form - # "property1=value1;property2=value2" etc - # Note: In this case we cannot know the type of the value so we will - # use the string as it is. This should be safe for e.g. numbers, - # as long as the format of the string is correct - if "properties" in row: - properties = row["properties"] - try: - for prop in properties.split(";"): - name, value = prop.split("=") - # need to decode the string, otherwise any linebreaks - # will be escaped. - value = value.decode("string-escape") - # Support inline multiline properties using "\n" - prop_dict[name.strip()] = [v.strip() - for v in value.split("\n")] - except ValueError: - raise ValueError("could not parse Properties") - - # "Property:xyz" and "Property(type):xyz columns - # The main issue here is that spreadsheet programs treat numeric cells - # as floats. If the number must be inserterd as an int, use the "(INT)" - # modifier. There does not seem to be a way to force a numeric cell to - # be interpreted as a string. - for col_name, value in row.items(): - match = re.match("property(?:\((.*)\))?:(.*)", col_name, re.IGNORECASE) - if match and (value is not None): # protect against zero, false... - type_, name = match.groups() - if type_: - convert = TYPE_MAPPING[type_] - values = [convert(value)] - else: - value = str(value).decode("string-escape") - values = [v.strip() for v in value.split("\n")] - prop_dict[name] = values - - return prop_dict - - -def get_attribute_properties(row): - - if "attribute" in row: - attribute = row["attribute"] - prop_dict = AppendingDict() - if "attributeproperties" in row: - properties = row["attributeproperties"] - try: - for prop in properties.split(";"): - name, value = prop.split("=") - name = name.strip() - if name not in SPECIAL_ATTRIBUTE_PROPERTIES: - raise ValueError( - "'%s' is not a valid attribute property" % name) - value = value.decode("string-escape") # for linebreaks - prop_dict[name.strip()] = [v.strip() - for v in value.split("\n")] - except ValueError: - raise ValueError("could not parse AttributeProperties") - - for col_name, value in row.items(): - match = re.match("attrprop:(.*)", col_name, re.IGNORECASE) - if match and value: - name, = match.groups() - name = make_db_name(name.strip()) - if name not in SPECIAL_ATTRIBUTE_PROPERTIES: - raise ValueError("'%s' it not a valid attribute property" - % name) - value = str(value).decode("string-escape") - values = [v.strip() for v in value.split("\n")] - prop_dict[name] = values - - return {attribute: prop_dict} - - -def get_dynamic(row): - "Find dynamic definitions on a row" - - prop_dict = AppendingDict() - try: - formula = row["formula"].strip() - if "type" in row: - # TODO: Sanity check type? - formula = "%s(%s)" % (row["type"], formula) - check_formula(formula) - mode = str(row["mode"]) - if mode.lower() == "status": - dyn = formula - else: - dyn = "%s=%s" % (row["name"], formula) - prop_dict[MODE_MAPPING[mode]] = dyn - except KeyError as e: - raise ValueError("Problem with formula: %s" % e) - - return prop_dict - - -def make_db_name(name): - "convert a Space Separated Name into a lowercase, underscore_separated_name" - return name.strip().lower().replace(" ", "_") - - -def check_formula(formula): - "Syntax check a dynamic formula." - compile(formula, "", "single") - - -def check_device_format(devname): - """Verify that a device name is of the correct form (three parts - separated by slashes, only letters, numbers, dashes and - underscores allowed.) Note: We could put more logic here to make - device names conform to a standard. - """ - device_pattern = "^[\w-]+/[\w-]+/[\w-]+$" - if not re.match(device_pattern, devname): - raise ValueError("device name '%s' not valid" % devname) - - -def format_server_instance(row): - "Format a server/instance string" - # TODO: handle numeric instance names? They tend to turn up as floats... - return "%s/%s" % (row["server"], row["instance"]) - - -def convert(rows, definitions, skip=True, dynamic=False, config=False): - - "Update a dict of definitions from data" - - errors = [] - column_names = rows[0] - - def handle_error(i, msg): - if skip: - errors.append((i, msg)) - else: - raise - - for i, row_ in enumerate(rows[1:]): - try: - # The plan is to try to find all information on the - # line, raising exceptions if there are unrecoverable - # problems. Those are caught and reported. - - # Filter out empty columns - row = CaselessDict(dict((str(name), col) - for name, col in zip(column_names, row_) - if col not in ("", None))) - - # Skip empty lines - if not row: - continue - - # Target of the properties; device or class? - if "device" in row: - check_device_format(row["device"]) - if "server" in row: - # full device definition - # target is "lazily" evaluated, so that we don't create - # an empty dict if it turns out there are no members - target = lambda: definitions.servers[row["server"]][row["instance"]]\ - [row["class"]][row["device"]] - else: - # don't know if the device is already defined - target = lambda: find_device(definitions, row["device"])[0] - elif "class" in row: - # class definition - target = lambda: definitions.classes[row["class"]] - else: - continue - - if dynamic: - props = get_dynamic(row) - if props: - target().properties = props - - elif config: - attr_props = get_attribute_properties(row) - if attr_props: - target().attribute_properties = attr_props - else: - props = get_properties(row) - if props: - target().properties = props - - except KeyError as ke: - #handle_error(i, "insufficient data (%s)" % ke) - pass - except ValueError as ve: - handle_error(i, "Error: %s" % ve) - except SyntaxError as se: - # TODO: do something here to show more info about the error - # ex_type, ex, tb = sys.exc_info() - # "\n".join(format_exc(ex).splitlines()[-3:] - handle_error(i, "SyntaxError: %s" % se) - - return errors - - -def print_errors(errors): - if errors: - print >> sys.stderr, "%d lines skipped" % len(errors) - for err in errors: - line, msg = err - print >> sys.stderr, "%d: %s" % (line + 1, msg) - - -def xls_to_dict(xls_filename, pages=None, skip=False): - - """Make JSON out of an XLS sheet of device definitions.""" - - import xlrd - - xls = xlrd.open_workbook(xls_filename) - definitions = AppendingDict() - - if not pages: # if no pages given, assume all pages are wanted - pages = xls.sheet_names() - else: - # Always include Dynamics and ParamConfig as they only add stuff - # to devices already configured anyway. - if "Dynamics" not in pages and "Dynamics" in xls.sheet_names(): - pages.append("Dynamics") - if "ParamConfig" not in pages and "ParamConfig" in xls.sheet_names(): - pages.append("ParamConfig") - - for page in pages: - - print >>sys.stderr, "\nPage: %s" % page - sheet = xls.sheet_by_name(page) - rows = [sheet.row_values(i) - for i in xrange(sheet.nrows)] - if not rows: - continue # ignore empty pages - errors = convert(rows, definitions, skip=skip, - dynamic=(page == "Dynamics"), - config=(page == "ParamConfig")) - print_errors(errors) - - return definitions - - -def get_stats(defs): - "Calculate some numbers" - - servers = set() - instances = set() - classes = set() - devices = set() - - for server, instances in defs.servers.items(): - servers.add(server) - instances.update(instances) - for clsname, devs in instances.items(): - classes.add(clsname) - for devname, dev in devs.items(): - devices.add(devname) - - return {"servers": len(servers), "instances": len(instances), - "classes": len(classes), "devices": len(devices)} - - -def main(): - from optparse import OptionParser - - usage = "usage: %prog [options] XLS [PAGE1, PAGE2, ...]" - parser = OptionParser(usage=usage) - parser.add_option("-t", "--test", action="store_true", - dest="test", default=False, - help="just test, produce no JSON") - parser.add_option("-q", "--quiet", action="store_false", - dest="verbose", default=True, - help="don't print errors to stdout") - parser.add_option("-f", "--fatal", action="store_false", - dest="skip", default=True, - help="don't skip, treat any parsing error as fatal") - - options, args = parser.parse_args() - if len(args) < 1: - sys.exit("You need to give an XLS file as argument.") - filename = args[0] - pages = args[1:] - - data = xls_to_dict(filename, pages, skip=options.skip) - metadata = dict( - _title="MAX-IV Tango JSON intermediate format", - _source=os.path.split(sys.argv[1])[-1], - _version=1, - _date=str(datetime.now())) - data.update(metadata) - - if not options.test: - print json.dumps(data, indent=4) - outfile = open('config.json', 'w') - json.dump(data, outfile, indent=4) - - stats = get_stats(data) - - print >>sys.stderr, ("\n" - "Total: %(servers)d servers, %(instances)d instances, " - "%(classes)d classes and %(devices)d devices defined.") % stats - - -if __name__ == "__main__": - main() diff --git a/dsconfig/excel_alarms_userdef.py b/dsconfig/excel_alarms_userdef.py deleted file mode 100644 index 1c0b475..0000000 --- a/dsconfig/excel_alarms_userdef.py +++ /dev/null @@ -1,60 +0,0 @@ -from collections import defaultdict -import json - -import xlrd - - -class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev, al_rec): - print inst, dev, al_name, al_cond, al_desc, al_sev, al_rec - devdict = sdict.servers["PyAlarm/"+inst]["PyAlarm"][dev] - if "AlarmList" not in devdict.properties: - devdict.properties["AlarmList"] = [] - devdict.properties["AlarmList"].append(al_name+":"+al_cond) - if "AlarmDescriptions" not in devdict.properties: - devdict.properties["AlarmDescriptions"] = [] - devdict.properties["AlarmDescriptions"].append(al_name+":"+al_desc) - if "AlarmSeverities" not in devdict.properties: - devdict.properties["AlarmSeverities"] = [] - devdict.properties["AlarmSeverities"].append(al_name+":"+al_sev) - if "AlarmReceivers" not in devdict.properties: - devdict.properties["AlarmReceivers"] = [] - devdict.properties["AlarmReceivers"].append(al_name+":"+al_rec) - -def xls_to_dict(xls_filename): - json_dict = SuperDict() - xls = xlrd.open_workbook(xls_filename) - sheet = xls.sheet_by_name("Alarms") - for line in xrange(1, sheet.nrows): - # above skips row 0 (col headers) - # look at all rows but only read those with entry in first col - if sheet.row_values(line)[0] is not "": - print "IN LINE ", line, sheet.row_values(line)[0] - dev_config = sheet.row_values(line) - add_device(json_dict, *dev_config[:7]) - return json_dict - -def main(): - import sys - data = xls_to_dict(sys.argv[1]) - print json.dumps(data, indent=4) - outfile = open('pyalarm_config.json', 'w') - json.dump(data, outfile, indent=4) - -if __name__ == "__main__": - main() - - - diff --git a/dsconfig/excel_alarms_vac.py b/dsconfig/excel_alarms_vac.py deleted file mode 100644 index e044153..0000000 --- a/dsconfig/excel_alarms_vac.py +++ /dev/null @@ -1,118 +0,0 @@ -from collections import defaultdict -import json - -import xlrd - - -class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev): - print inst, dev, al_name, al_cond, al_desc, al_sev - # devdict = sdict.servers["PyAlarm"]["PyAlarm/"+inst]["PyAlarm"][dev] - devdict = sdict.servers["PyAlarm/"+inst]["PyAlarm"][dev] - if "AlarmList" not in devdict.properties: - devdict.properties["AlarmList"] = [] - devdict.properties["AlarmList"].append(al_name+":"+al_cond) - if "AlarmDescriptions" not in devdict.properties: - devdict.properties["AlarmDescriptions"] = [] - devdict.properties["AlarmDescriptions"].append(al_name+":"+al_desc) - #hard code severity and some other things - only one per instance - if "AlarmSeverities" not in devdict.properties: - devdict.properties["AlarmSeverities"] = [] - devdict.properties["AlarmSeverities"].append(al_name+":"+al_sev) - if "AlarmReceivers" not in devdict.properties: - devdict.properties["AlarmReceivers"] = [] - devdict.properties["AlarmReceivers"].append(al_name+":HTML") - #hard code severity and some other things - only one per instance - if "AlarmThreshold" not in devdict.properties: - devdict.properties["AlarmThreshold"] = [] - devdict.properties["AlarmThreshold"] = [1] - if "LogFile" not in devdict.properties: - devdict.properties["LogFile"] = [] - devdict.properties["LogFile"]= ["/tmp/pjb/log"] - if "HtmlFolder" not in devdict.properties: - devdict.properties["HtmlFolder"] = [] - devdict.properties["HtmlFolder"] = ["/tmp/pjb"] - if "PollingPeriod" not in devdict.properties: - devdict.properties["PollingPeriod"] = [] - devdict.properties["PollingPeriod"] = [5] - if "MaxMessagesPerAlarm" not in devdict.properties: - devdict.properties["MaxMessagesPerAlarm"] = [] - devdict.properties["MaxMessagesPerAlarm"]= [1] - if "AutoReset" not in devdict.properties: - devdict.properties["AutoReset"] = [] - devdict.properties["AutoReset"]= [0] - if "StartupDelay" not in devdict.properties: - devdict.properties["StartupDelay"] = [] - devdict.properties["StartupDelay"]= [0] - -def xls_to_dict(xls_filename): - json_dict = SuperDict() - xls = xlrd.open_workbook(xls_filename) - - for i in range (0,2): - - if i==1: - sheet = xls.sheet_by_name("Alarms") - nature="interlock" - else: - sheet = xls.sheet_by_name("Warnings") - nature="bypass" - - last_server="" - last_device="" - last_name="" - last_sev="" - summary_condition="" - for line in xrange(1, sheet.nrows): - # above skips row 0 (col headers) - # look at all rows but only read those with entry in first col - if sheet.row_values(line)[0] is not "": - print "IN LINE ", line, sheet.row_values(line)[0] - #assume that if you get to a new device, it means a new section of vacuum - #in this case, need to make a final alarm which is or of all others - dev_config = sheet.row_values(line) - print dev_config, dev_config[3].rsplit("/",1)[0] - if dev_config[1] != last_device or dev_config[0]=="end": - print "START NEW SECTION", dev_config[1] - print "---- ADDING TO JSON summary ", summary_condition, last_name - if summary_condition!="": - add_device(json_dict,last_server,last_device,last_name.rsplit("_",1)[0],summary_condition,"at least one vac. %s in section %s" %(nature,last_name.rsplit("__",2)[0]),last_sev) - last_server = dev_config[0] - last_device = dev_config[1] - last_name = dev_config[2] - last_sev = dev_config[5] - summary_condition="" - if summary_condition == "": - summary_condition = summary_condition + dev_config[3] - else: - summary_condition = summary_condition + " or " + dev_config[3] - - if dev_config[0]!="end": - add_device(json_dict, *dev_config[:6]) - - return json_dict - -def main(): - import sys - data = xls_to_dict(sys.argv[1]) - #print json.dumps(data, indent=4) - outfile = open('alarms_vac.json', 'w') - json.dump(data, outfile, indent=4) - -if __name__ == "__main__": - main() - - - diff --git a/dsconfig/magnets2json.py b/dsconfig/magnets2json.py deleted file mode 100644 index d417294..0000000 --- a/dsconfig/magnets2json.py +++ /dev/null @@ -1,883 +0,0 @@ -#!/usr/bin/env python -# "$Name: $"; -# "$Header: $"; -#============================================================================= -# -# file : lattice2json.py -# -# description : Python source for the lattice2json that is a tool to generate -# a json file from an elegant lattice file -# The json file can then be uploaded to a Tango database -# -# project : Virtual Accelerator -# -# $Author: $ -# -# $Revision: $ -# -# $Log: $ -# -# copyleft : Solaris/MAX IV -# Krakow,PL/Lund,SE# -# - -from collections import defaultdict -import json -import os -import io -import sys -import re -from TangoProperties import TANGO_PROPERTIES -from PowerSupplyMap import POWER_SUPPLY_MAP -import copy -import numpy as np - -cirlist = [] - - -class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -class LatticeFileItem: - ''' ''' - itemName = "" - itemType = '' - parameters = {} - properties = {} - alpars= {} - - def __init__(self, _line=''): - ''' - Construct an object parsing a _line from a lattice file - ''' - self.psparameters= {} - self.parameters= {} - self.properties= {} - - - # find a name - colon_pos = _line.find(':') - self.itemName = _line[:colon_pos].lstrip().rstrip().upper() - - # what left to be parsed - line_left = _line[colon_pos + 1:].lstrip() - - # find a type - param_name = '' # the first item after a colon could be also a parameter name, like for a line element - eq_pos = line_left.find('=') - comma_pos = line_left.find(',') - # let it work even there are no parameters defined - only element type - if eq_pos < 0: eq_pos = len(line_left) - if comma_pos < 0: comma_pos = len(line_left) - # so, we could read an element type - self.itemType = line_left[:min(comma_pos, eq_pos)].rstrip().lower() - - # this is waiting to be processed: - line_left = line_left[comma_pos + 1:].lstrip() - - # if the element type is also parameter name state this - if eq_pos < comma_pos: param_name = self.itemType - - # parse the rest for parameters - while line_left != '': - if param_name != '': - # searching for a value - param_value = '' - if line_left[0] == '(': - # value in brackets (may contain commas) - bracket_pos = line_left.index(')', 1) # will rise an exception in case of badly formated line - # so, the value is (including brackets): - param_value = line_left[:bracket_pos + 1] - # this is what left to be parsed - line_left = line_left[bracket_pos + 1:].lstrip() - - elif line_left[0] == '\"': - # value in quotes (could contain commas) - quote_pos = line_left.index('\"', 1) # will rise an exception in case of badly formated line - # so, the value is (including quote): - param_value = line_left[:quote_pos + 1] - - # this is what left to be parsed - line_left = line_left[quote_pos + 1:].lstrip() - else: - # typical case - the value between an equal and a comma characters - comma_pos = line_left.find(',') - if comma_pos < 0: comma_pos = len(line_left) - # a value, here you are - param_value = line_left[:comma_pos].rstrip() - # the following left to be parsed - line_left = line_left[comma_pos + 1:].lstrip() - # store the parameter with the corresponding value - self.parameters[param_name] = param_value - # PJB reset name back to empty here to find next parameter!(to enter else below) - param_name='' - else: - # searching for a parameter - eq_pos = line_left.find('=') - if eq_pos < 0: eq_pos = len(line_left) - # allow value-less parameters - comma_pos = line_left.find(',') - if comma_pos < 0: comma_pos = len(line_left) - # so we know where to find parameter name - param_name = line_left[:min(eq_pos, comma_pos)].rstrip().lower() - # if the parameter has no value add it directly to the dictionary - if comma_pos <= eq_pos: - self.parameters[param_name] = '' - param_name = '' - # this is what left to be parsed - line_left = line_left[min(eq_pos, comma_pos) + 1:].lstrip() - - - def handle_circuit_name(self,itemName,endnum): - - endname="" - #hack for bc1 - if "QD" in itemName and "BC1" in itemName: - endname = "CRQM-" + endnum - # - #hack for bc2 - elif "QF" in itemName and "BC2" in itemName and "3" not in itemName and "4" not in itemName and "5" not in itemName: - endname = "CRQM-" + endnum - elif "QF" in itemName and "BC2" in itemName and ("3" in itemName or "5" in itemName): - endname = "CRQ1-01" - elif "QF" in itemName and "BC2" in itemName and "4" in itemName: - endname = "CRQ2-01" - # - elif "Q" in itemName: - endname = "CRQ-" + endnum - elif "CO" in itemName and "X" in itemName: - endname = "CRCOX-" + endnum - elif "CO" in itemName and "Y" in itemName: - endname = "CRCOY-" + endnum - elif "DI" in itemName: - endname = "CRDI-" + endnum - print "dealing with endname ", endname - elif "SX" in itemName: - endname = "CRSX-" + endnum - elif "SOL" in itemName: - endname = "CRSOL-" + endnum - elif "SM" in itemName: - endname = "CRSM-" + endnum - else: - sys.exit("Cannot convert circuit name" + itemName) - - if "/" in endname: #in case did not end with number, endname will be some */*/ by mistake - endname=endname.split("-")[0]+"-01" - - return endname - - - def config_alarms(self,pyalarm,alname,alsev,aldesc,pyattname,key): - - alrec = alname+":"+"HTML" - - if "AlarmList" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmList"] = [] - self.alpars[pyalarm]['AlarmList'].append(alname+":"+pyattname+"/"+key) - - if "AlarmSeverities" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmSeverities"] = [] - self.alpars[pyalarm]['AlarmSeverities'].append(alsev) - - if "AlarmDescriptions" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmDescriptions"] = [] - self.alpars[pyalarm]['AlarmDescriptions'].append(aldesc) - - if "AlarmReceivers" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmReceivers"] = [] - self.alpars[pyalarm]['AlarmReceivers'].append(alrec) - - if "StartupDelay" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["StartupDelay"] = ["0"] - - if "AutoReset" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AutoReset"] = ["60"] - - if "MaxMessagesPerAlarm" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["MaxMessagesPerAlarm"] = ["1"] - - if "PollingPeriod" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["PollingPeriod"] = ["5"] - - if "LogFile" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["LogFile"] = ["/tmp/pjb/log"] - - if "HtmlFolder" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["HtmlFolder"] = ["/tmp/pjb"] - - if "AlarmThreshold" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmThreshold"] = ["1"] - - - def match_properties(self): - - - devclass = types_classes[self.itemType] - - if "CIR" in self.itemName: - print "dealing with magnet circuit" - - # for given item type, look up required attributes and properties of tango - elif devclass in TANGO_PROPERTIES: - - fixed_properties_l = list(TANGO_PROPERTIES[devclass][0].keys()) - #print "fixed tango properties are ", fixed_properties_l - - # add the fixed properties - self.parameters.update(TANGO_PROPERTIES[devclass][0]) - - lattice_properties_l = list(TANGO_PROPERTIES[devclass][1].keys()) - #print "possible lattice tango properties are ", lattice_properties_l - for k in self.parameters.keys(): - #print "pjb zzz 1", self.parameters["Tilt"], self.parameters - #print "key", k - #if not a required property or attribue then pop it - if k.lower() not in lattice_properties_l and k not in fixed_properties_l: - #print "popping ", k - self.parameters.pop(k) - - #otherwise rename key if an attribute - if k.lower() in lattice_properties_l: - #print "KEY ", k.lower(), TANGO_PROPERTIES[devclass][1][k.lower()], self.parameters[k] - self.parameters[TANGO_PROPERTIES[devclass][1][k.lower()]] = [self.parameters.pop(k)] - #print "pjb zzz", self.parameters["Tilt"], self.parameters - - - else: - for k in self.parameters.keys(): - self.parameters.pop(k) - - if "MAG" in self.itemName and not "CIR" in self.itemName: - self.parameters["Type"] = [self.itemType] - - # - if "Tilt" in self.parameters: - if "0.5 pi" in str(self.parameters["Tilt"]): - self.parameters["Tilt"] = ["90"] - else: - self.parameters["Tilt"] = ["0"] - print "pjbyyy", self.parameters - - def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[0-9]+)'): - ''' - Updates json file - ''' - # prepare pattern for parsing name - pattern = re.compile(name_parsing_string) - - print "In add device for item: " + self.itemName + " as " + self.itemType, self.alpars - # only when we know class for certain element - - print "adict is ", adict - circuit_alarm_params = [] - - #for case with no final number like I.TR1.MAG.DIE (no .1 etc at end) - alt_name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)' - alt_pattern = re.compile(alt_name_parsing_string) - - #self.alpars = {} - devdictalarm = None - - if types_classes.has_key(self.itemType): - - print "" - print "" - - # split name - name_items = pattern.search(self.itemName) - parsed=False - tryagain=False - if name_items == None: - print "Warning: Item name in lattice file doesn't match the naming convention.",self.itemName - tryagain=True - else: - parsed=True - if tryagain: - name_items = alt_pattern.search(self.itemName) - if name_items == None: - print "Warning: Item name in lattice file STILL doesn't match the naming convention.",self.itemName - else: - parsed=True - if parsed: - system = name_items.group('system') - subsystem = name_items.group('subsystem') - location = name_items.group('location') - device = name_items.group('device') - - if tryagain==False: - num = name_items.group('num') - else: - num="" - - if num == None: num = '' - - if num != "": - num2 = "%02d" % int(num) - else: - num2 = "01" - - - self.match_properties() - print "pjbxxx ", self.parameters - - # create device for json output - name = (system+"-"+location + '/' + subsystem + '/' + device + "-" + num2).encode('ascii', 'ignore') - devclass = types_classes[self.itemType].encode('ascii', 'ignore') - server = devclass + '/' + system+"-"+location - - #hack for circuits - if "CIR" in self.itemName: - print "orig name",self.itemName, name - - #quad circuits named like CRQ - #correctors like CRCOX and CRCOY - #dipoles like CRDI - #solenoid like CRSOL - #sextu like CRX - endname = name.rsplit("/",1)[1] - - #fix circuit names - endname = self.handle_circuit_name(self.itemName,num2) - name = name.rsplit("/",1)[0] + "/" + endname - - #e.g in G00 have COIX 1, 2, 3 and COHX 1 and 2, which would make COX 1, 2, 3 with COIX 1 and 2 being lost! - #increment number till unique - while name in cirlist: - print "danger! already named this circuit!", self.itemName, name - suffix = int(name.split("-")[2]) + 1 - newnum2 = "%02d" % suffix - print newnum2 - name = (name.rsplit("-",1)[0]) + "-" + newnum2 - print name - print "new name ", name - cirlist.append(name) - print cirlist - devclass = "MagnetCircuit" - - - #compact name is to find tag in plc alarms - name_l_cir = self.itemName.split(".") - section_cir = name_l_cir[1] - mid=name_l_cir[3] - if "_" in mid: - mid = mid.split("_")[0] - #hack for SM1A and SM1B in TR1, also DIE, DIF on circuit DI (DIC, DID on TR3) - compactnamecir = "_"+name_l_cir[1]+mid+"_" - compactnamecir = compactnamecir.replace("1A","1") - compactnamecir = compactnamecir.replace("1B","1") - if "DIE" in compactnamecir: - compactnamecir = compactnamecir.replace("DIE","DI") - if "DIF" in compactnamecir: - compactnamecir = compactnamecir.replace("DIF","DI") - if "DIC" in compactnamecir: - compactnamecir = compactnamecir.replace("DIC","DI") - if "DID" in compactnamecir: - compactnamecir = compactnamecir.replace("DID","DI") - - print "circuit compact name is ", compactnamecir, name_l_cir - - #fill alarm info for circuits - # - pyalarm = system+"-"+location + '/MAG/ALARM' - if adict is not None: - devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] - - print "init devdictalarm circuit" - - already_added = ["B_I_SP02DIPBD_DIA_TSW1_A","B_I_SP02DIPBD_DIA_TSW2_A"] #can use this to ignore to. e.g. SP02DIPDB doesnt end in a number but isn't a circuit! - for key in alarm_dict: - #compact name is like _TR1QF_ but some tags are like _TR1QF2_5_ - #if compactnamecir in key: - if (key not in already_added and compactnamecir in key) or (compactnamecir[:-1] in key and key.count("_")>5 and key not in already_added and "F"+num+"_" in key): - - print "key is", num, key - - already_added.append(key) - - print "FOUND ALARM INFO FOR CIR", compactnamecir, key, alarm_dict[key], section_cir - pyattname = "I-" + section_cir + "/DIA/COOLING" - - #for the magnets json file - circuit_alarm_params = [pyattname, key, alarm_dict[key]] - - #for the alarms json file - alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+endname.replace("-","_") - alsev = alname+":"+"ALARM" - alrec = alname+":"+"HTML" - aldesc = alname+":One magnet in circuit "+ alarm_dict[key] - - if pyalarm not in self.alpars: - self.alpars[pyalarm] = {} - self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars - - - - print "+++++++++++++++++ Creating device server : " + server + " for " + devclass + " (name= " + name + ")" - print "+++++++++ With properties : ", self.parameters - - # Dont actually write attributes to DB, only properties - # see if this class exists and append if so, or create - devdict = sdict.servers["%s/%s" % (devclass, system+"-"+location)][devclass][name] - - #for circuit json only - if "CR" in name: - psdevdict = psdict.Circuits[name] - - - if "MAG" in self.itemName and "CIR" not in self.itemName: - - #compact name is to find tag in plc alarms - name_l = self.itemName.split(".") - section = name_l[1] - del name_l[0] - del name_l[1] - #del name_l[2] - print "pjb kkk", name_l - compactfullname = "".join(name_l) - compactname = compactfullname.split("_")[0] - compactname_nonum = compactfullname.split("_")[0][:-1]+"_" - - print "-------------------- magnet not circuit", self.itemName, compactname - - #see what is the ps of the magnet - if name in POWER_SUPPLY_MAP: - powersupplyname = POWER_SUPPLY_MAP[name] - else: - print "magnet not in PS map, skipping", name - return - - #!!!!!!!!!!! *********** create circuit device for each new ps ************!!!!!!!!!!!! - # copy the magnet and call recursively add device! - magnetcircuit = copy.deepcopy(self) - magnetcircuit.itemName = self.itemName + ".CIR" - - magnetcircuit.parameters = {} - magnetcircuit.parameters['PowerSupplyProxy'] = [powersupplyname] - magnetcircuit.parameters['MagnetProxies'] = [name] - magnetcircuit.parameters['RiseTime'] = ["0.0"] - magnetcircuit.parameters['ResistanceReference'] = ["0.0"] - magnetcircuit.parameters['CoilNames'] = [""] - - #for the ps json file only - magnetcircuit.psparameters['PowerSupplyProxy'] = [powersupplyname] - magnetcircuit.psparameters['MagnetProxies'] = [name] - - #get alarm info from excel - pyalarm = system+"-"+location + '/MAG/ALARM' - - print "adict is again", adict - if adict is not None: - devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] - print "init devdictalarm" - - #set alarms in magnet.json file and in alarms json file - for key in alarm_dict: - if compactname in key and key.count("_")<6: - #if compactname in key: - - pyattname = "I-" + section + "/DIA/COOLING" - - print "FOUND ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict - - #for the magnets json file - if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] - else: - self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) - - #for the alarms json file - alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+'MAG'+'__'+ device + "_" +num2 - alsev = alname+":"+"ALARM" - aldesc = alname+":Magnet "+alarm_dict[key] - - if pyalarm not in self.alpars: - self.alpars[pyalarm] = {} - self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars - - devdictalarm.properties = self.alpars[pyalarm] - - #set alarms in magnet.json file for all magnets in circuit - for key in alarm_dict: - if compactname_nonum in key or (compactname[:-1] in key and key.count("_")>5 and ("F"+num+"_" in key or "_"+num+"_" in key)): - #if compactname_nonum in key: - print "mag key ", key, compactname_nonum, compactname, "F"+num+"_", "_"+num+"_" - pyattname = "I-" + section + "/DIA/COOLING" - - print "FOUND MORE ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict - - #for the magnets json file - if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] - else: - self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) - - polarity = 1 - orientation = 1 - - #get calibration info from the excel - if self.itemName.split("_")[0] in calib_dict: - print "FOUND CALIB INFO", self.itemName - #find max multipole expansions - dim = max(calib_dict[self.itemName.split("_")[0]].keys(), key=int) - - print "--- max order is", dim - - #create arrays of this dimensions, other dimension is 11 - - fieldsmatrix = [[0 for x in xrange(19)] for x in xrange(dim)] - #print fieldsmatrix - currentsmatrix = [[0 for x in xrange(19)] for x in xrange(dim)] - - #iterate over keys and add to the array - for key in calib_dict[self.itemName.split("_")[0]]: - print '--- ', key, 'corresponds to', calib_dict[self.itemName.split("_")[0]][key] - currents = calib_dict[self.itemName.split("_")[0]][key][5:25] - fields = calib_dict[self.itemName.split("_")[0]][key][25:45] - print '--- ',currents, fields - - fieldsmatrix[key-1]=fields - currentsmatrix[key-1]=currents - #key here is the multipole order. any one should have same polarity - polarity = calib_dict[self.itemName.split("_")[0]][key][3] - orientation = calib_dict[self.itemName.split("_")[0]][key][2] - #print "P, O", polarity, orientation - - print '--- ',fieldsmatrix - print '--- ',currentsmatrix - - #now trim the matrices (lists) - maxlength = 20 - for i,val in enumerate(fieldsmatrix[dim-1]): - if val=='': - print i , val - maxlength = i - break - print maxlength - for i in xrange(dim): - print i - del fieldsmatrix[i][maxlength:] - del currentsmatrix[i][maxlength:] - - print 'Now--- ',fieldsmatrix - print 'Now--- ',currentsmatrix - - magnetcircuit.parameters['ExcitationCurveCurrents']= currentsmatrix - magnetcircuit.parameters['ExcitationCurveFields']= fieldsmatrix - - self.parameters['Orientation'] = [str(int(orientation))] - self.parameters['Polarity'] = [str(int(polarity))] - - #assign circuit name as property of magnet device - #no regex to fix name here so do by hand - #e.g. I.BC1.MAG.COEX.4.CIR -> I-BC1/MAG/COEX-CIR-04 - - cname = name.rsplit("/CIR",1)[0] - endname = cname.rsplit("/",1)[1] - endnum = cname.rsplit("-",1)[1] - endname = self.handle_circuit_name(self.itemName,endnum) - cname = cname.rsplit("/",1)[0] + "/" + endname - - print "cname is ", cname, name, powersupplyname, circuit_ps_list - - while cname in cirlist: - print "danger2! already named this circuit!", cname - suffix = int(cname.split("-")[2]) + 1 - newnum2 = "%02d" % suffix - cname = (cname.rsplit("-",1)[0]) + "-" + newnum2 - print "new name ", cname - - #only add one circuit device per ps - if powersupplyname not in circuit_ps_list: - - magnetcircuit.add_device(sdict,adict,psdict) - circuit_ps_list[powersupplyname] = cname - - print "adding circuit name ", magnetcircuit.itemName, cname, circuit_ps_list - self.parameters['CircuitProxies'] = [cname] - - else: - #if we aleady made this circuit device, add it to this magnet properties - print "!!!ALART!!! already added a circuit device for ", self.itemName, name, system, location - - if system == "R3": - system= "I" - if location == "301L": - location = "TR3" - - self.parameters['CircuitProxies'] = [circuit_ps_list[powersupplyname]] - - #need to get the name of the circuit device from the ps dict though - print "exiting circuit device is", circuit_ps_list[powersupplyname] - - #print "current mags ", system+"-"+location - #print "current mags 2", sdict.servers - - current_mags = sdict.servers["%s/%s" % ("MagnetCircuit", system+"-"+location)]["MagnetCircuit"][circuit_ps_list[powersupplyname]].properties - #for circuits json - print "cir name from ps ", circuit_ps_list[powersupplyname], psdict - ps_current_mags = psdict.Circuits[circuit_ps_list[powersupplyname]].Properties - print "current mags ", current_mags['MagnetProxies'] - if name in current_mags['MagnetProxies']: - print "circuit already has magnet ", name - else: - ps_current_mags['MagnetProxies'].append(name) - current_mags['MagnetProxies'].append(name) - - print "magnets on cir ", current_mags['ExcitationCurveFields'], current_mags['MagnetProxies'], len(current_mags['MagnetProxies']) - #need to average the currents, even if already done so in excel (depends on field order) - if 'ExcitationCurveFields' in current_mags: - - assoc_field_m = current_mags['ExcitationCurveFields'] - this_field_m = fieldsmatrix - - assoc_curr_m = current_mags['ExcitationCurveCurrents'] - this_curr_m = currentsmatrix - - print "field matrix assoc is ", assoc_field_m - print "field matrix current is ", this_field_m - - print "current matrix assoc is ", assoc_curr_m - print "current matrix current is ", this_curr_m - - for i in xrange(dim): - print i - - #fix for CRSM take abs field values since opp sign - if circuit_ps_list[powersupplyname] in ["I-TR3/MAG/CRSM-01","I-TR3/MAG/CRDI-01"]: - newFields = [ ( abs(x)*(len(current_mags['MagnetProxies']) -1) + abs(y) ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] - else: - newFields = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] - - newCurrents = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_curr_m[i],assoc_curr_m[i])] - - print "new fields ", newFields - print "new currents ", newCurrents - current_mags['ExcitationCurveFields'][i] = newFields - print "updated: ", current_mags['ExcitationCurveFields'] - current_mags['ExcitationCurveCurrents'][i] = newCurrents - print "updated: ", current_mags['ExcitationCurveCurrents'] - - - else: - print "NOT A MAGNET" - - devdict.properties = self.parameters - - #for circuits json - if "CR" in name: - psdevdict.Properties = self.psparameters - - -class ElegantLatticeParser: - ''' Class for parsing an elegant lattice file. ''' - fileName = "" - file = None - items = [] - - def __init__(self, _fileName): - '''Constructs a parser object. - - Keyword arguments: - _fileName -- the name of file to be parsed - ''' - self.fileName = _fileName - self.file = io.open(_fileName) - - - - def parseLatticeFile(self): - ''' ''' - line = "" # this will be a line combined from lines to be connected - for ll in self.file: - l = ll.lstrip().rstrip() - if len(l) > 0: - if l[0] == '!': - pass # do not process comments - elif l[0] == '%': - pass # processing RPNs are not implemented - elif l[-1] == '&': - # Combine lines to be concated - line = line + ' ' + l[:-1] - else: - # So this is the last line to be combined - line = line + l - # Create an object and add it to list - self.items.append(LatticeFileItem(line.lstrip().rstrip())) - line = "" - - - -if __name__ == '__main__': - - - - inFileName = '' - doCalib=False - doAlarms=False - excelName = 'MagnetCalibrationData.xls' - alarmName = 'IMAG_ALARM_140923_Magnets.xls' - - # define classes for lattice elements - types_classes = {} - types_classes["dip"] = "Magnet" - types_classes["sbend"] = "Magnet" - types_classes["sben"] = "Magnet" - types_classes["rben"] = "Magnet" - types_classes["csrcsbend"] = "Magnet" - types_classes["sole"] = "Magnet" - types_classes["kquad"] = "Magnet" - types_classes["ksext"] = "Magnet" - types_classes["hkick"] = "Magnet" - types_classes["vkick"] = "Magnet" - #types_classes["hkick"] = "VACorrector" - #types_classes["vkick"] = "VACorrector" - #types_classes["monitor"] = "VABPM" - #types_classes["watch"] = "VAYAGScreen" - #types_classes["rfcw"] = "VARfCavity" - # - circuit_ps_list = {} - #circuit_alarm_params = None - - #read arguments - for par in sys.argv[1:]: - if par[0:2] == '--': - if par == '--calibration-data': - doCalib = True - elif par == '--alarm-data': - doAlarms = True - # elif par == '--test-mode': - # test_mode = True - elif inFileName == '': - inFileName = par - - print inFileName, doCalib - - - if doAlarms or doCalib: - import xlrd - - alarm_dict = {} - if doAlarms: - print "opening alarms xls" - xls = xlrd.open_workbook(alarmName) - sheet = xls.sheet_by_name('Sheet1') - rows = [sheet.row_values(i) for i in xrange(sheet.nrows)] - column_names = rows[0] - print "cols ", column_names - for row in enumerate(rows[1:]): - if row[1][0]=="": - continue - print row[1][0],row[1][1] - alarm_dict[row[1][0]] = row[1][1] - print "DICT IS ", alarm_dict - - #make dict just for alarms! for pyalarm - json_dict_alarms = SuperDict() - else: - json_dict_alarms = None - - calib_dict = {} - - if doCalib: - #open excel sheet - xls = xlrd.open_workbook(excelName) - - for name in ["linac", "transfer 1,5 GeV", "transfer 3 GeV", "thermionic gun"]: - - #sheet = xls.sheet_by_name('Linac') - sheet = xls.sheet_by_name(name) - rows = [sheet.row_values(i) for i in xrange(sheet.nrows)] - column_names = rows[7] - print "cols ", column_names - - for row in enumerate(rows[9:]): - print row[1] - if row[1][2]=="": - continue - #this is like - #[5.0, u'I.S01A', u'I.S01A.MAG.QE.1', 202005.0, u'#1168-10030-0001', 1.0, -1.0, 2.0, 6.3167, 5.6757, 5.0307500000000003, 4.4208999999999996, 3.8452999999999999, 3.1463999999999999, 2.5179624999999999, 1.8892374999999999, 1.2808725000000001, 0.63988750000000016, 0.0, 0.70470485548532313, 0.63908274382966312, 0.56946571499960408, 0.50203927491440703, 0.43686121069898298, 0.35966476443894108, 0.288993167760146, 0.21848942173091002, 0.14957521795596601, 0.077488874695939805, 0.0052044472873010797, u'T', u'Rotating coil-C1168, #0001.xls', u'https://alfresco.maxlab.lu.se/share/page/site/maxiv/document-details?nodeRef=workspace://SpacesStore/23cdc9d1-a01e-443e-b578-1538637a1472', u'Scanditronix Magnet', 40690.0, ''] - if row[1][2].strip() not in calib_dict: - if row[1][7] is not "": - data_key = int(row[1][7]) - data_list = row[1][3:48] - data_dict = {data_key : data_list} - calib_dict[row[1][2].strip()]=data_dict - #calib_dict[row[1][2]]=row[1][3:33] - else: - if row[1][7] is not "": - #we found more curves for the same magnet - print "found another entry", row[1][2], row[1][7] - data_key = int(row[1][7]) - data_list = row[1][3:48] - data_dict = {data_key : data_list} - calib_dict[row[1][2].strip()][data_key]=data_list - - print "DICT IS ", calib_dict - - - - # create a parser for the file - parser = ElegantLatticeParser(inFileName) - - # parse the file - parser.parseLatticeFile() - parser.file.close() - - - # make a json file - json_dict = SuperDict() - json_ps = SuperDict() - for item in parser.items: - item.add_device(json_dict,json_dict_alarms, json_ps) - #print json.dumps(json_dict, indent=4) - - #now we have the dict, loop over again and sort out magnets, power supplies and circuits - - print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ " - print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ " - print json_dict.servers - - outfile = open('magnets.json', 'w') - - #have final json dict here - print "THE FINAL DICT" - topl = json_dict['servers'].keys() - for item in topl: - if "Circuit" in item: - #print json_dict['servers'][item] - for cir in json_dict['servers'][item]: - #print json_dict['servers'][item][cir]['ExcitationCurveCurrents'] - #print json_dict['servers'][item]["MagnetCircuit"] - for c in json_dict['servers'][item]["MagnetCircuit"]: - for key in json_dict['servers'][item]["MagnetCircuit"][c]["properties"]: - if key == "ExcitationCurveCurrents": - ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print key, [str(x) for x in ls] - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] - if key == "ExcitationCurveFields": - ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print key, [str(x) for x in ls] - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] - - json.dump(json_dict, outfile, indent=4) - - if doAlarms: - outfile2 = open('magnets_alarms.json', 'w') - json.dump(json_dict_alarms, outfile2, indent=4) - - #!! note that item has parameters, but only want to extract those needed for tango! - - #dump ps circuit info - outfile3 = open('circuits.json', 'w') - json.dump(json_ps, outfile3, indent=4) - diff --git a/setup.py b/setup.py index 0934e01..c404047 100755 --- a/setup.py +++ b/setup.py @@ -16,9 +16,4 @@ package_data={ 'dsconfig': ['schema/dsconfig.json', 'schema/schema2.json']}, - # Scripts - entry_points={ - 'console_scripts': ['xls2json = dsconfig.excel:main', - 'csv2json = dsconfig.callcsv:main', - 'json2tango = dsconfig.json2tango:main']} ) diff --git a/test/test_excel.py b/test/test_excel.py deleted file mode 100644 index a4b930a..0000000 --- a/test/test_excel.py +++ /dev/null @@ -1,143 +0,0 @@ -from os.path import dirname, abspath, join - -try: - from unittest2 import TestCase -except ImportError: - from unittest import TestCase - -from dsconfig import excel -from dsconfig.utils import CaselessDict - - -class TestExcel(TestCase): - - def test_get_properties_combined(self): - row = CaselessDict({"Properties": "a=1; b=2; c=3"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1"], "b": ["2"], "c": ["3"]}) - - def test_get_properties_separate(self): - row = CaselessDict({"Property:a": "1", "Property:b": "2"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1"], "b": ["2"]}) - - def test_get_properties_both(self): - row = CaselessDict({"Properties": "c=3", - "Property:a": "1", "Property:b": "2"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1"], "b": ["2"], "c": ["3"]}) - - def test_get_properties_splits_combined(self): - row = CaselessDict({"Properties": "a=1\n2\n3"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1", "2", "3"]}) - - def test_get_properties_splits_separate(self): - row = CaselessDict({"Property:a": "1\n2\n3"}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1", "2", "3"]}) - - def test_get_properties_with_zero_value(self): - row = CaselessDict({"Property:a": 0}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["0"]}) - - def test_get_properties_with_types(self): - row = CaselessDict({"Property(int):a": 1.0, - "Property(float):b": 1.0}) - result = excel.get_properties(row) - self.assertDictEqual(result, {"a": ["1"], "b": ["1.0"]}) - - def test_get_dynamic_attribute(self): - row = CaselessDict({"name": "TestAttribute", "formula": "a + b", - "type": "int", "mode": "attr"}) - result = excel.get_dynamic(row) - self.assertDictEqual( - result.to_dict(), {"DynamicAttributes": ["TestAttribute=int(a + b)"]}) - - def test_get_dynamic_command(self): - row = CaselessDict({"name": "TestCommand", "formula": "1 + 2", - "type": "bool", "mode": "cmd"}) - result = excel.get_dynamic(row) - self.assertDictEqual( - result, {"DynamicCommands": ["TestCommand=bool(1 + 2)"]}) - - def test_get_dynamic_state(self): - row = CaselessDict({"name": "WEIRD", "formula": "1 == 2", - "type": "bool", "mode": "state"}) - result = excel.get_dynamic(row) - self.assertDictEqual( - result, {"DynamicStates": ["WEIRD=bool(1 == 2)"]}) - - def test_get_dynamic_status(self): - row = CaselessDict({"name": "Status", "formula": "'Something witty here'", - "type": "str", "mode": "status"}) - result = excel.get_dynamic(row) - self.assertDictEqual( - result, {"DynamicStatus": ["str('Something witty here')"]}) - - def test_dynamic_attribute_barfs_on_bad_syntax(self): - row = CaselessDict({"name": "TestAttribute", "formula": "a ? b", - "type": "int", "mode": "attr"}) - self.assertRaises(SyntaxError, excel.get_dynamic, row) - - def test_dynamic_attribute_errors_if_missing_stuff(self): - row = CaselessDict({"name": "TestAttribute", "formula": "a + b", - "type": "int"}) - self.assertRaises(ValueError, excel.get_dynamic, row) - - def test_get_attribute_properties(self): - row = CaselessDict({"Attribute": "myAttribute", - "AttributeProperties": "min_value=1;max_value=2"}) - result = excel.get_attribute_properties(row) - self.assertDictEqual(result, - {"myAttribute": {"min_value": ["1"], - "max_value": ["2"]}}) - - def test_get_attribute_properties_multiple(self): - row = CaselessDict({"Attribute": "myAttribute", - "AttrProp:Label": "Something", - "AttrProp:Min value": "45"}) - result = excel.get_attribute_properties(row) - self.assertDictEqual(result, - {"myAttribute": {"min_value": ["45"], - "label": ["Something"]}}) - - def test_get_attribute_properties_errors_on_invalid_name(self): - row = CaselessDict({"Attribute": "myAttribute", - "AttrProp:Label": "Something", - "AttrProp:Min value": "45", - "AttrProp:Not valid": "foo"}) - self.assertRaises(ValueError, excel.get_attribute_properties, row) - - def test_check_device_format_lets_valid_names_pass(self): - excel.check_device_format("i-a/test-test/device-0") - - def test_check_device_format_ignores_case(self): - excel.check_device_format("I-A/TEST-TEST/DEVICE-0") - - def test_check_device_format_catches_bad_names(self): - self.assertRaises(ValueError, excel.check_device_format, - "not/a/device/name") - - def test_check_device_format_catches_non_device_names(self): - self.assertRaises(ValueError, excel.check_device_format, "just a string") - - def test_check_device_format_catches_names_with_invalid_characters(self): - self.assertRaises(ValueError, excel.check_device_format, "I/can.has/dots") - - def test_format_server_instance(self): - row = {"server": "TestServer", "instance": 1} - result = excel.format_server_instance(row) - self.assertEqual(result, "TestServer/1") - - # def test_format_server_instance_handles_floats(self): - # row = {"server": "TestServer", "instance": 1.0} - # result = excel.format_server_instance(row) - # self.assertEqual(result, "TestServer/1") - - def test_xls_to_dict(self): - xls_file = join(dirname(abspath(__file__)), 'files', 'AlarmDemoV6.xls') - data = excel.xls_to_dict(xls_file) - stats = excel.get_stats(data) - assert stats == {'instances': 0, 'classes': 0, 'devices': 0, 'servers': 0} diff --git a/test/test_excel_alarms_userdef.py b/test/test_excel_alarms_userdef.py deleted file mode 100644 index 1aca424..0000000 --- a/test/test_excel_alarms_userdef.py +++ /dev/null @@ -1,12 +0,0 @@ -from os.path import dirname, abspath, join - -from dsconfig.excel_alarms_userdef import xls_to_dict - - -def test_excel_alarms_user_def(): - xls_file = join(dirname(abspath(__file__)), 'files', 'AlarmDemoV6.xls') - result = xls_to_dict(xls_file) - assert result['servers']['PyAlarm/']['PyAlarm'] - alarm_prop = result['servers']['PyAlarm/mypyalarm']['PyAlarm']['Alarms/test/1'] - assert alarm_prop['properties']['AlarmSeverities'] == ['VacPressure:WARNING', - 'MagCurrent:ALARM'] From c58f1a9ca290bd9e6b4a3072e906a7577ddeb780 Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Wed, 3 Jun 2020 12:54:11 +0200 Subject: [PATCH 11/22] Add .idea to .gitignore. --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8cad575..9b8b083 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,5 @@ coverage.xml # Sphinx documentation docs/_build/ - +# IDE +.idea From a18b88284c2a338b505ead3cfd905372972f027d Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Wed, 3 Jun 2020 13:04:14 +0200 Subject: [PATCH 12/22] Use 2to3 tool. --- dsconfig/appending_dict/__init__.py | 6 +- dsconfig/appending_dict/caseless.py | 10 +- dsconfig/appending_dict/test_appendingdict.py | 4 +- dsconfig/callcsv.py | 20 +- dsconfig/configure.py | 26 +- dsconfig/diff.py | 28 +- dsconfig/dump.py | 6 +- dsconfig/excel.py | 32 +- dsconfig/excel_alarms_userdef.py | 8 +- dsconfig/excel_alarms_vac.py | 12 +- dsconfig/filtering.py | 10 +- dsconfig/formatting.py | 20 +- dsconfig/json2tango.py | 48 +- dsconfig/magnets2json.py | 1766 ++++++++--------- dsconfig/output.py | 58 +- dsconfig/remove.py | 32 +- dsconfig/tangodb.py | 52 +- dsconfig/utils.py | 22 +- dsconfig/validate.py | 10 +- dsconfig/viewer.py | 4 +- test/providers.py | 36 +- test/test_configure2.py | 26 +- test/test_filtering.py | 4 +- test/test_tangodb.py | 6 +- 24 files changed, 1123 insertions(+), 1123 deletions(-) diff --git a/dsconfig/appending_dict/__init__.py b/dsconfig/appending_dict/__init__.py index 0ccddb8..ec36c9e 100644 --- a/dsconfig/appending_dict/__init__.py +++ b/dsconfig/appending_dict/__init__.py @@ -32,7 +32,7 @@ def __init__(self, value={}, factory=None): self.__dict__["_factory"] = factory CaselessDictionary.__init__(self) defaultdict.__init__(self, factory) - for k, v in value.items(): + for k, v in list(value.items()): self[k] = v def __getitem__(self, key): @@ -58,7 +58,7 @@ def __setattr__(self, key, value): def to_dict(self): """Returns a ordinary dict version of itself""" result = {} - for key, value in self.items(): + for key, value in list(self.items()): if isinstance(value, SetterDict): result[key] = value.to_dict() else: @@ -68,7 +68,7 @@ def to_dict(self): def merge(d, u): "Recursively 'merge' a Mapping into another" - for k, v in u.iteritems(): + for k, v in u.items(): if isinstance(v, Mapping): if k in d: merge(d[k], v) diff --git a/dsconfig/appending_dict/caseless.py b/dsconfig/appending_dict/caseless.py index 9aaa5ad..eaa82ac 100644 --- a/dsconfig/appending_dict/caseless.py +++ b/dsconfig/appending_dict/caseless.py @@ -44,8 +44,8 @@ class CaselessDictionary(MutableMapping): def __init__(self, *args, **kwargs): self.__dict__["_dict"] = {} temp_dict = dict(*args, **kwargs) - for key, value in temp_dict.iteritems(): - if isinstance(key, basestring): + for key, value in temp_dict.items(): + if isinstance(key, str): key = CaselessString.make_caseless(key) self._dict[key] = value @@ -72,7 +72,7 @@ def keys(self): return [str(k) for k in self._dict] def items(self): - return zip(self.keys(), self.values()) + return list(zip(list(self.keys()), list(self.values()))) class CaselessString(object): @@ -90,7 +90,7 @@ def __cmp__(self, other): @classmethod def make_caseless(cls, string): - if isinstance(string, unicode): + if isinstance(string, str): return CaselessUnicode(string) return CaselessStr(string) @@ -99,5 +99,5 @@ class CaselessStr(CaselessString, str): pass -class CaselessUnicode(CaselessString, unicode): +class CaselessUnicode(CaselessString, str): pass diff --git a/dsconfig/appending_dict/test_appendingdict.py b/dsconfig/appending_dict/test_appendingdict.py index 2be0067..6ef1a34 100644 --- a/dsconfig/appending_dict/test_appendingdict.py +++ b/dsconfig/appending_dict/test_appendingdict.py @@ -95,7 +95,7 @@ def test_keeps_original_key_case(self): sd.foo = 2 sd.baR = 3 sd.BAR = 4 - self.assertListEqual(sd.keys(), ["FoO", "baR"]) + self.assertListEqual(list(sd.keys()), ["FoO", "baR"]) class AppendingDictTestCase(unittest.TestCase): @@ -111,7 +111,7 @@ def test_deep_appending(self): ad = AppendingDict() ad["a"]["b"]["c"] = 1 ad["a"]["b"]["c"] = 2 - print type(ad["a"]["b"]) + print(type(ad["a"]["b"])) self.assertDictEqual(ad, {"a": {"b": {"c": ['1', '2']}}}) def test_initial_setting_with_dict(self): diff --git a/dsconfig/callcsv.py b/dsconfig/callcsv.py index 606198c..4a603bf 100644 --- a/dsconfig/callcsv.py +++ b/dsconfig/callcsv.py @@ -17,7 +17,7 @@ def special_update(d, u): """Update nested dictionnaries while prioritizing the first argument.""" if not (isinstance(d, Mapping) and isinstance(u, Mapping)): return d if d is not None else u - for k, v in u.iteritems(): + for k, v in u.items(): d[k] = special_update(d.get(k), v) return d @@ -127,14 +127,14 @@ def process_call_list(lst, skip=False, verbose=True): for module_name, func_name, kwargs in lst: # Build prototype prototype = "{0}.{1}(".format(module_name, func_name) - for key, value in kwargs.items(): + for key, value in list(kwargs.items()): prototype += '{0}={1}, '.format(key, value) if prototype.endswith(' '): prototype = prototype[:-2] prototype += ')' # Print prototype if verbose: - print "Executing: " + prototype + print("Executing: " + prototype) # Execute try: module = import_module(module_name) @@ -156,7 +156,7 @@ def join_data(lst, source=None): """Join a list of json strings or dictionnaries into a single dict.""" data = {} for mapping in lst: - if isinstance(mapping, basestring): + if isinstance(mapping, str): mapping = json.loads(mapping) special_update(data, mapping) if source: @@ -227,12 +227,12 @@ def main(desc=None, module_name=None, function=None): if module == module_name and func == function.__name__: kwargs = keywords if verbose: - print("'{0}' found".format(prototype)) - print("kwargs = " + str(kwargs)) + print(("'{0}' found".format(prototype))) + print(("kwargs = " + str(kwargs))) break else: msg = "'{0}' not found in {1}" - print msg.format(prototype, get_call_list(input_file)) + print(msg.format(prototype, get_call_list(input_file))) return # Generate json file if module_name and function: @@ -243,7 +243,7 @@ def main(desc=None, module_name=None, function=None): data = function(**kwargs) if isinstance(data, Mapping): string = json.dumps(data, indent=4, sort_keys=True) - elif isinstance(data, basestring): + elif isinstance(data, str): string = data else: msg = "The function didn't return a valid data format.\n" @@ -269,7 +269,7 @@ def main(desc=None, module_name=None, function=None): with open(output_file, mode='w') as f: f.write(string) if verbose and output_file: - print('Exported to: ' + output_file) + print(('Exported to: ' + output_file)) # Write tango database if write: from dsconfig import configure @@ -279,7 +279,7 @@ def main(desc=None, module_name=None, function=None): if remove: os.remove(output_file) if verbose: - print('Removed: ' + output_file) + print(('Removed: ' + output_file)) print('OK!') diff --git a/dsconfig/configure.py b/dsconfig/configure.py index ba7d406..6344acf 100644 --- a/dsconfig/configure.py +++ b/dsconfig/configure.py @@ -4,10 +4,10 @@ from functools import partial import PyTango -from appending_dict.caseless import CaselessDictionary +from .appending_dict.caseless import CaselessDictionary -from utils import ObjectWrapper -from tangodb import SPECIAL_ATTRIBUTE_PROPERTIES, is_protected +from .utils import ObjectWrapper +from .tangodb import SPECIAL_ATTRIBUTE_PROPERTIES, is_protected def check_attribute_property(propname): @@ -39,8 +39,8 @@ def update_properties(db, parent, db_props, new_props, # For attribute properties we need to go one step deeper into # the dict, since each attribute can have several properties. # A little messy, but at least it's consistent. - for attr, props in new_props.items(): - for prop, value in props.items(): + for attr, props in list(new_props.items()): + for prop, value in list(props.items()): if ignore_case: orig = CaselessDictionary(caseless_db_props.get(attr, {})).get(prop) else: @@ -49,7 +49,7 @@ def update_properties(db, parent, db_props, new_props, or check_attribute_property(prop)): added_props[attr][prop] = value removed_props = defaultdict(dict) - for attr, props in db_props.items(): + for attr, props in list(db_props.items()): for prop in props: if ignore_case: new = CaselessDictionary(new_props.get(attr, {})).get(prop) @@ -60,12 +60,12 @@ def update_properties(db, parent, db_props, new_props, removed_props[attr][prop] = value else: added_props = {} - for prop, value in new_props.items(): + for prop, value in list(new_props.items()): old_value = caseless_db_props.get(prop, []) if value and value != old_value: added_props[prop] = value removed_props = {} - for prop, value in db_props.items(): + for prop, value in list(db_props.items()): new_value = caseless_new_props.get(prop) if (new_value is None and not is_protected(prop)) or new_value == []: # empty list forces removal of "protected" properties @@ -97,14 +97,14 @@ def update_server(db, server_name, server_dict, db_dict, if ignore_case: db_dict = CaselessDictionary(db_dict) - for class_name, cls in server_dict.items(): # classes + for class_name, cls in list(server_dict.items()): # classes if ignore_case: cls = CaselessDictionary(cls) removed_devices = [dev for dev in db_dict.get(class_name, {}) if dev not in cls # never remove dservers and not class_name.lower() == "dserver"] - added_devices = cls.items() + added_devices = list(cls.items()) if not update: for device_name in removed_devices: db.delete_device(device_name) @@ -182,8 +182,8 @@ def configure(data, dbdata, update=False, ignore_case=False, db = ObjectWrapper() - for servername, serverdata in data.get("servers", {}).items(): - for instname, instdata in serverdata.items(): + for servername, serverdata in list(data.get("servers", {}).items()): + for instname, instdata in list(serverdata.items()): dbinstdata = (dbdata.get("servers", {}) .get(servername, {}) .get(instname, {})) @@ -192,7 +192,7 @@ def configure(data, dbdata, update=False, ignore_case=False, instdata, dbinstdata, update, ignore_case, strict_attr_props=strict_attr_props) - for classname, classdata in data.get("classes", {}).items(): + for classname, classdata in list(data.get("classes", {}).items()): dbclassdata = dbdata.get("classes", {}).get(classname, {}) update_class(db, classname, dbclassdata, classdata, update=update) diff --git a/dsconfig/diff.py b/dsconfig/diff.py index abfca14..ba8bbfc 100644 --- a/dsconfig/diff.py +++ b/dsconfig/diff.py @@ -2,7 +2,7 @@ import json import sys -from utils import green, yellow, red +from .utils import green, yellow, red def decode_pointer(ptr): @@ -44,33 +44,33 @@ def print_diff(dbdict, data, removes=True): try: ptr = " > ".join(decode_pointer(d["path"])) if d["op"] == "replace": - print yellow("REPLACE:") - print yellow(ptr) + print(yellow("REPLACE:")) + print(yellow(ptr)) db_value = resolve_pointer(dbdict, d["path"]) - print red(dump_value(db_value)) - print green(dump_value(d["value"])) + print(red(dump_value(db_value))) + print(green(dump_value(d["value"]))) ops["replace"] += 1 if d["op"] == "add": - print green("ADD:") - print green(ptr) + print(green("ADD:")) + print(green(ptr)) if d["value"]: - print green(dump_value(d["value"])) + print(green(dump_value(d["value"]))) ops["add"] += 1 if removes and d["op"] == "remove": - print red("REMOVE:") - print red(ptr) + print(red("REMOVE:")) + print(red(ptr)) value = resolve_pointer(dbdict, d["path"]) if value: - print red(dump_value(value)) + print(red(dump_value(value))) ops["remove"] += 1 except JsonPointerException as e: - print " - Error parsing diff - report this!: %s" % e + print(" - Error parsing diff - report this!: %s" % e) # # The following output is a bit misleading, removing for now # print "Total: %d operations (%d replace, %d add, %d remove)" % ( # sum(ops.values()), ops["replace"], ops["add"], ops["remove"]) return diff except ImportError: - print >>sys.stderr, ("'jsonpatch' module not available - " - "no diff printouts for you! (Try -d instead.)") + print(("'jsonpatch' module not available - " + "no diff printouts for you! (Try -d instead.)"), file=sys.stderr) diff --git a/dsconfig/dump.py b/dsconfig/dump.py index 34614a0..9cbc491 100755 --- a/dsconfig/dump.py +++ b/dsconfig/dump.py @@ -8,8 +8,8 @@ """ -from tangodb import get_servers_with_filters, get_classes_properties -from appending_dict import SetterDict +from .tangodb import get_servers_with_filters, get_classes_properties +from .appending_dict import SetterDict import PyTango @@ -81,7 +81,7 @@ def main(): attribute_properties=options.attribute_properties, aliases=options.aliases, dservers=options.dservers, subdevices=options.subdevices) - print json.dumps(dbdata, ensure_ascii=False, indent=4, sort_keys=True) + print(json.dumps(dbdata, ensure_ascii=False, indent=4, sort_keys=True)) if __name__ == "__main__": diff --git a/dsconfig/excel.py b/dsconfig/excel.py index 80a688c..6c123fa 100644 --- a/dsconfig/excel.py +++ b/dsconfig/excel.py @@ -10,10 +10,10 @@ import sys #from traceback import format_exc -from utils import find_device -from appending_dict import AppendingDict -from utils import CaselessDict -from tangodb import SPECIAL_ATTRIBUTE_PROPERTIES +from .utils import find_device +from .appending_dict import AppendingDict +from .utils import CaselessDict +from .tangodb import SPECIAL_ATTRIBUTE_PROPERTIES MODE_MAPPING = CaselessDict({"ATTR": "DynamicAttributes", "CMD": "DynamicCommands", @@ -54,7 +54,7 @@ def get_properties(row): # as floats. If the number must be inserterd as an int, use the "(INT)" # modifier. There does not seem to be a way to force a numeric cell to # be interpreted as a string. - for col_name, value in row.items(): + for col_name, value in list(row.items()): match = re.match("property(?:\((.*)\))?:(.*)", col_name, re.IGNORECASE) if match and (value is not None): # protect against zero, false... type_, name = match.groups() @@ -89,7 +89,7 @@ def get_attribute_properties(row): except ValueError: raise ValueError("could not parse AttributeProperties") - for col_name, value in row.items(): + for col_name, value in list(row.items()): match = re.match("attrprop:(.*)", col_name, re.IGNORECASE) if match and value: name, = match.groups() @@ -229,10 +229,10 @@ def handle_error(i, msg): def print_errors(errors): if errors: - print >> sys.stderr, "%d lines skipped" % len(errors) + print("%d lines skipped" % len(errors), file=sys.stderr) for err in errors: line, msg = err - print >> sys.stderr, "%d: %s" % (line + 1, msg) + print("%d: %s" % (line + 1, msg), file=sys.stderr) def xls_to_dict(xls_filename, pages=None, skip=False): @@ -256,10 +256,10 @@ def xls_to_dict(xls_filename, pages=None, skip=False): for page in pages: - print >>sys.stderr, "\nPage: %s" % page + print("\nPage: %s" % page, file=sys.stderr) sheet = xls.sheet_by_name(page) rows = [sheet.row_values(i) - for i in xrange(sheet.nrows)] + for i in range(sheet.nrows)] if not rows: continue # ignore empty pages errors = convert(rows, definitions, skip=skip, @@ -278,12 +278,12 @@ def get_stats(defs): classes = set() devices = set() - for server, instances in defs.servers.items(): + for server, instances in list(defs.servers.items()): servers.add(server) instances.update(instances) - for clsname, devs in instances.items(): + for clsname, devs in list(instances.items()): classes.add(clsname) - for devname, dev in devs.items(): + for devname, dev in list(devs.items()): devices.add(devname) return {"servers": len(servers), "instances": len(instances), @@ -320,15 +320,15 @@ def main(): data.update(metadata) if not options.test: - print json.dumps(data, indent=4) + print(json.dumps(data, indent=4)) outfile = open('config.json', 'w') json.dump(data, outfile, indent=4) stats = get_stats(data) - print >>sys.stderr, ("\n" + print(("\n" "Total: %(servers)d servers, %(instances)d instances, " - "%(classes)d classes and %(devices)d devices defined.") % stats + "%(classes)d classes and %(devices)d devices defined.") % stats, file=sys.stderr) if __name__ == "__main__": diff --git a/dsconfig/excel_alarms_userdef.py b/dsconfig/excel_alarms_userdef.py index 1c0b475..e258646 100644 --- a/dsconfig/excel_alarms_userdef.py +++ b/dsconfig/excel_alarms_userdef.py @@ -18,7 +18,7 @@ def __getattr__(self, attr): def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev, al_rec): - print inst, dev, al_name, al_cond, al_desc, al_sev, al_rec + print(inst, dev, al_name, al_cond, al_desc, al_sev, al_rec) devdict = sdict.servers["PyAlarm/"+inst]["PyAlarm"][dev] if "AlarmList" not in devdict.properties: devdict.properties["AlarmList"] = [] @@ -37,11 +37,11 @@ def xls_to_dict(xls_filename): json_dict = SuperDict() xls = xlrd.open_workbook(xls_filename) sheet = xls.sheet_by_name("Alarms") - for line in xrange(1, sheet.nrows): + for line in range(1, sheet.nrows): # above skips row 0 (col headers) # look at all rows but only read those with entry in first col if sheet.row_values(line)[0] is not "": - print "IN LINE ", line, sheet.row_values(line)[0] + print("IN LINE ", line, sheet.row_values(line)[0]) dev_config = sheet.row_values(line) add_device(json_dict, *dev_config[:7]) return json_dict @@ -49,7 +49,7 @@ def xls_to_dict(xls_filename): def main(): import sys data = xls_to_dict(sys.argv[1]) - print json.dumps(data, indent=4) + print(json.dumps(data, indent=4)) outfile = open('pyalarm_config.json', 'w') json.dump(data, outfile, indent=4) diff --git a/dsconfig/excel_alarms_vac.py b/dsconfig/excel_alarms_vac.py index e044153..00f9c40 100644 --- a/dsconfig/excel_alarms_vac.py +++ b/dsconfig/excel_alarms_vac.py @@ -18,7 +18,7 @@ def __getattr__(self, attr): def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev): - print inst, dev, al_name, al_cond, al_desc, al_sev + print(inst, dev, al_name, al_cond, al_desc, al_sev) # devdict = sdict.servers["PyAlarm"]["PyAlarm/"+inst]["PyAlarm"][dev] devdict = sdict.servers["PyAlarm/"+inst]["PyAlarm"][dev] if "AlarmList" not in devdict.properties: @@ -75,18 +75,18 @@ def xls_to_dict(xls_filename): last_name="" last_sev="" summary_condition="" - for line in xrange(1, sheet.nrows): + for line in range(1, sheet.nrows): # above skips row 0 (col headers) # look at all rows but only read those with entry in first col if sheet.row_values(line)[0] is not "": - print "IN LINE ", line, sheet.row_values(line)[0] + print("IN LINE ", line, sheet.row_values(line)[0]) #assume that if you get to a new device, it means a new section of vacuum #in this case, need to make a final alarm which is or of all others dev_config = sheet.row_values(line) - print dev_config, dev_config[3].rsplit("/",1)[0] + print(dev_config, dev_config[3].rsplit("/",1)[0]) if dev_config[1] != last_device or dev_config[0]=="end": - print "START NEW SECTION", dev_config[1] - print "---- ADDING TO JSON summary ", summary_condition, last_name + print("START NEW SECTION", dev_config[1]) + print("---- ADDING TO JSON summary ", summary_condition, last_name) if summary_condition!="": add_device(json_dict,last_server,last_device,last_name.rsplit("_",1)[0],summary_condition,"at least one vac. %s in section %s" %(nature,last_name.rsplit("__",2)[0]),last_sev) last_server = dev_config[0] diff --git a/dsconfig/filtering.py b/dsconfig/filtering.py index cc18697..c4645f9 100644 --- a/dsconfig/filtering.py +++ b/dsconfig/filtering.py @@ -1,6 +1,6 @@ import re -from appending_dict import merge +from .appending_dict import merge def filter_nested_dict(node, pattern, depth, level=0, invert=False): @@ -9,12 +9,12 @@ def filter_nested_dict(node, pattern, depth, level=0, invert=False): at the given depth. """ if level == depth: - return dict((key, value) for key, value in node.iteritems() + return dict((key, value) for key, value in node.items() if (not invert and pattern.match(key)) or (invert and not pattern.match(key))) else: dupe_node = {} - for key, val in node.iteritems(): + for key, val in node.items(): cur_node = filter_nested_dict(val, pattern, depth, level+1, invert) if cur_node: @@ -41,7 +41,7 @@ def filter_config(data, filters, levels, invert=False): srv, inst = [re.compile(r, flags=re.IGNORECASE) for r in regex.split("/")] servers = filter_nested_dict(data, srv, 0) - for k, v in servers.items(): + for k, v in list(servers.items()): tmp = filter_nested_dict(v, inst, 0) if tmp: filtered[k] = tmp @@ -53,7 +53,7 @@ def filter_config(data, filters, levels, invert=False): "Bad filter '%s'; should be ':'" % fltr) except KeyError: raise ValueError("Bad filter '%s'; term should be one of: %s" - % (fltr, ", ".join(levels.keys()))) + % (fltr, ", ".join(list(levels.keys())))) except re.error as e: raise ValueError("Bad regular expression '%s': %s" % (fltr, e)) if invert: diff --git a/dsconfig/formatting.py b/dsconfig/formatting.py index 75772fe..eb3017d 100644 --- a/dsconfig/formatting.py +++ b/dsconfig/formatting.py @@ -5,7 +5,7 @@ from copy import deepcopy, copy from os import path -from appending_dict import AppendingDict, SetterDict +from .appending_dict import AppendingDict, SetterDict import PyTango @@ -22,7 +22,7 @@ def decode_list(data): rv = [] for item in data: - if isinstance(item, unicode): + if isinstance(item, str): item = str(item.encode('utf-8')) elif isinstance(item, list): item = decode_list(item) @@ -34,10 +34,10 @@ def decode_list(data): def decode_dict(data): rv = {} - for key, value in data.iteritems(): - if isinstance(key, unicode): + for key, value in data.items(): + if isinstance(key, str): key = str(key.encode('utf-8')) - if isinstance(value, unicode): + if isinstance(value, str): value = str(value.encode('utf-8')) elif isinstance(value, list): value = decode_list(value) @@ -55,10 +55,10 @@ def validate_json(data): schema = json.load(schema_json) validate(data, schema) except ImportError: - print >>sys.stderr, ("WARNING: 'jsonschema' not installed, could not " - "validate json file. You're on your own.") + print(("WARNING: 'jsonschema' not installed, could not " + "validate json file. You're on your own."), file=sys.stderr) except exceptions.ValidationError as e: - print >>sys.stderr, "ERROR: JSON data does not match schema: %s" % e + print("ERROR: JSON data does not match schema: %s" % e, file=sys.stderr) sys.exit(1) @@ -88,7 +88,7 @@ def expand_config(config): def clean_metadata(data): "Removes any keys in the data that begin with '_'" tmp = copy(data) - for key in tmp.keys(): + for key in list(tmp.keys()): if key.startswith("_"): tmp.pop(key, None) return tmp @@ -119,7 +119,7 @@ def normalize_config(config): new_config.classes = old_config["classes"] if "devices" in old_config: db = PyTango.Database() - for device, props in old_config["devices"].items(): + for device, props in list(old_config["devices"].items()): try: info = db.get_device_info(device) except PyTango.DevFailed as e: diff --git a/dsconfig/json2tango.py b/dsconfig/json2tango.py index 1abcd20..1b11a10 100755 --- a/dsconfig/json2tango.py +++ b/dsconfig/json2tango.py @@ -114,7 +114,7 @@ def main(): data.get("classes", {}), options.exclude_classes, CLASSES_LEVELS, invert=True) except ValueError as e: - print >>sys.stderr, red("Filter error:\n%s" % e) + print(red("Filter error:\n%s" % e), file=sys.stderr) sys.exit(ERROR) if not any(k in data for k in ("devices", "servers", "classes")): @@ -122,7 +122,7 @@ def main(): if options.input: - print json.dumps(data, indent=4) + print(json.dumps(data, indent=4)) return # check if there is anything in the DB that will be changed or removed @@ -147,7 +147,7 @@ def main(): in get_devices_from_dict(original["servers"]) }) collisions = {} - for dev, (srv, inst, cls) in devices.items(): + for dev, (srv, inst, cls) in list(devices.items()): if dev in orig_devices: server = "{}/{}".format(srv, inst) osrv, oinst, ocls = orig_devices[dev] @@ -173,25 +173,25 @@ def main(): if options.verbose: progressbar(i, len(dbcalls), 20) getattr(db, method)(*args, **kwargs) - print + print() # optionally dump some information to stdout if options.output: - print json.dumps(original, indent=4) + print(json.dumps(original, indent=4)) if options.dbcalls: - print >>sys.stderr, "Tango database calls:" + print("Tango database calls:", file=sys.stderr) for method, args, kwargs in dbcalls: - print >>sys.stderr, method, args + print(method, args, file=sys.stderr) # Check for moved devices and remove empty servers empty = set() - for srvname, devs in collisions.items(): + for srvname, devs in list(collisions.items()): if options.verbose: srv, inst = srvname.split("/") for cls, dev in devs: - print >>sys.stderr, red("MOVED (because of collision):"), dev - print >>sys.stderr, " Server: ", "{}/{}".format(srv, inst) - print >>sys.stderr, " Class: ", cls + print(red("MOVED (because of collision):"), dev, file=sys.stderr) + print(" Server: ", "{}/{}".format(srv, inst), file=sys.stderr) + print(" Class: ", cls, file=sys.stderr) if len(db.get_device_class_list(srvname)) == 2: # just dserver empty.add(srvname) if options.write: @@ -199,32 +199,32 @@ def main(): # finally print out a brief summary of what was done if dbcalls: - print - print >>sys.stderr, "Summary:" - print >>sys.stderr, "\n".join(summarise_calls(dbcalls, original)) + print() + print("Summary:", file=sys.stderr) + print("\n".join(summarise_calls(dbcalls, original)), file=sys.stderr) if collisions: servers = len(collisions) - devices = sum(len(devs) for devs in collisions.values()) - print >>sys.stderr, red("Move %d devices from %d servers." % - (devices, servers)) + devices = sum(len(devs) for devs in list(collisions.values())) + print(red("Move %d devices from %d servers." % + (devices, servers)), file=sys.stderr) if empty and options.verbose: - print >>sys.stderr, red("Removed %d empty servers." % len(empty)) + print(red("Removed %d empty servers." % len(empty)), file=sys.stderr) if options.write: - print >>sys.stderr, red("\n*** Data was written to the Tango DB ***") + print(red("\n*** Data was written to the Tango DB ***"), file=sys.stderr) with NamedTemporaryFile(prefix="dsconfig-", suffix=".json", delete=False) as f: f.write(json.dumps(original, indent=4)) - print >>sys.stderr, ("The previous DB data was saved to %s" % - f.name) + print(("The previous DB data was saved to %s" % + f.name), file=sys.stderr) sys.exit(CONFIG_APPLIED) else: - print >>sys.stderr, yellow( - "\n*** Nothing was written to the Tango DB (use -w) ***") + print(yellow( + "\n*** Nothing was written to the Tango DB (use -w) ***"), file=sys.stderr) sys.exit(CONFIG_NOT_APPLIED) else: - print >>sys.stderr, green("\n*** No changes needed in Tango DB ***") + print(green("\n*** No changes needed in Tango DB ***"), file=sys.stderr) sys.exit(SUCCESS) diff --git a/dsconfig/magnets2json.py b/dsconfig/magnets2json.py index d417294..5f00862 100644 --- a/dsconfig/magnets2json.py +++ b/dsconfig/magnets2json.py @@ -1,883 +1,883 @@ -#!/usr/bin/env python -# "$Name: $"; -# "$Header: $"; -#============================================================================= -# -# file : lattice2json.py -# -# description : Python source for the lattice2json that is a tool to generate -# a json file from an elegant lattice file -# The json file can then be uploaded to a Tango database -# -# project : Virtual Accelerator -# -# $Author: $ -# -# $Revision: $ -# -# $Log: $ -# -# copyleft : Solaris/MAX IV -# Krakow,PL/Lund,SE# -# - -from collections import defaultdict -import json -import os -import io -import sys -import re -from TangoProperties import TANGO_PROPERTIES -from PowerSupplyMap import POWER_SUPPLY_MAP -import copy -import numpy as np - -cirlist = [] - - -class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -class LatticeFileItem: - ''' ''' - itemName = "" - itemType = '' - parameters = {} - properties = {} - alpars= {} - - def __init__(self, _line=''): - ''' - Construct an object parsing a _line from a lattice file - ''' - self.psparameters= {} - self.parameters= {} - self.properties= {} - - - # find a name - colon_pos = _line.find(':') - self.itemName = _line[:colon_pos].lstrip().rstrip().upper() - - # what left to be parsed - line_left = _line[colon_pos + 1:].lstrip() - - # find a type - param_name = '' # the first item after a colon could be also a parameter name, like for a line element - eq_pos = line_left.find('=') - comma_pos = line_left.find(',') - # let it work even there are no parameters defined - only element type - if eq_pos < 0: eq_pos = len(line_left) - if comma_pos < 0: comma_pos = len(line_left) - # so, we could read an element type - self.itemType = line_left[:min(comma_pos, eq_pos)].rstrip().lower() - - # this is waiting to be processed: - line_left = line_left[comma_pos + 1:].lstrip() - - # if the element type is also parameter name state this - if eq_pos < comma_pos: param_name = self.itemType - - # parse the rest for parameters - while line_left != '': - if param_name != '': - # searching for a value - param_value = '' - if line_left[0] == '(': - # value in brackets (may contain commas) - bracket_pos = line_left.index(')', 1) # will rise an exception in case of badly formated line - # so, the value is (including brackets): - param_value = line_left[:bracket_pos + 1] - # this is what left to be parsed - line_left = line_left[bracket_pos + 1:].lstrip() - - elif line_left[0] == '\"': - # value in quotes (could contain commas) - quote_pos = line_left.index('\"', 1) # will rise an exception in case of badly formated line - # so, the value is (including quote): - param_value = line_left[:quote_pos + 1] - - # this is what left to be parsed - line_left = line_left[quote_pos + 1:].lstrip() - else: - # typical case - the value between an equal and a comma characters - comma_pos = line_left.find(',') - if comma_pos < 0: comma_pos = len(line_left) - # a value, here you are - param_value = line_left[:comma_pos].rstrip() - # the following left to be parsed - line_left = line_left[comma_pos + 1:].lstrip() - # store the parameter with the corresponding value - self.parameters[param_name] = param_value - # PJB reset name back to empty here to find next parameter!(to enter else below) - param_name='' - else: - # searching for a parameter - eq_pos = line_left.find('=') - if eq_pos < 0: eq_pos = len(line_left) - # allow value-less parameters - comma_pos = line_left.find(',') - if comma_pos < 0: comma_pos = len(line_left) - # so we know where to find parameter name - param_name = line_left[:min(eq_pos, comma_pos)].rstrip().lower() - # if the parameter has no value add it directly to the dictionary - if comma_pos <= eq_pos: - self.parameters[param_name] = '' - param_name = '' - # this is what left to be parsed - line_left = line_left[min(eq_pos, comma_pos) + 1:].lstrip() - - - def handle_circuit_name(self,itemName,endnum): - - endname="" - #hack for bc1 - if "QD" in itemName and "BC1" in itemName: - endname = "CRQM-" + endnum - # - #hack for bc2 - elif "QF" in itemName and "BC2" in itemName and "3" not in itemName and "4" not in itemName and "5" not in itemName: - endname = "CRQM-" + endnum - elif "QF" in itemName and "BC2" in itemName and ("3" in itemName or "5" in itemName): - endname = "CRQ1-01" - elif "QF" in itemName and "BC2" in itemName and "4" in itemName: - endname = "CRQ2-01" - # - elif "Q" in itemName: - endname = "CRQ-" + endnum - elif "CO" in itemName and "X" in itemName: - endname = "CRCOX-" + endnum - elif "CO" in itemName and "Y" in itemName: - endname = "CRCOY-" + endnum - elif "DI" in itemName: - endname = "CRDI-" + endnum - print "dealing with endname ", endname - elif "SX" in itemName: - endname = "CRSX-" + endnum - elif "SOL" in itemName: - endname = "CRSOL-" + endnum - elif "SM" in itemName: - endname = "CRSM-" + endnum - else: - sys.exit("Cannot convert circuit name" + itemName) - - if "/" in endname: #in case did not end with number, endname will be some */*/ by mistake - endname=endname.split("-")[0]+"-01" - - return endname - - - def config_alarms(self,pyalarm,alname,alsev,aldesc,pyattname,key): - - alrec = alname+":"+"HTML" - - if "AlarmList" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmList"] = [] - self.alpars[pyalarm]['AlarmList'].append(alname+":"+pyattname+"/"+key) - - if "AlarmSeverities" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmSeverities"] = [] - self.alpars[pyalarm]['AlarmSeverities'].append(alsev) - - if "AlarmDescriptions" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmDescriptions"] = [] - self.alpars[pyalarm]['AlarmDescriptions'].append(aldesc) - - if "AlarmReceivers" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmReceivers"] = [] - self.alpars[pyalarm]['AlarmReceivers'].append(alrec) - - if "StartupDelay" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["StartupDelay"] = ["0"] - - if "AutoReset" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AutoReset"] = ["60"] - - if "MaxMessagesPerAlarm" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["MaxMessagesPerAlarm"] = ["1"] - - if "PollingPeriod" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["PollingPeriod"] = ["5"] - - if "LogFile" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["LogFile"] = ["/tmp/pjb/log"] - - if "HtmlFolder" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["HtmlFolder"] = ["/tmp/pjb"] - - if "AlarmThreshold" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmThreshold"] = ["1"] - - - def match_properties(self): - - - devclass = types_classes[self.itemType] - - if "CIR" in self.itemName: - print "dealing with magnet circuit" - - # for given item type, look up required attributes and properties of tango - elif devclass in TANGO_PROPERTIES: - - fixed_properties_l = list(TANGO_PROPERTIES[devclass][0].keys()) - #print "fixed tango properties are ", fixed_properties_l - - # add the fixed properties - self.parameters.update(TANGO_PROPERTIES[devclass][0]) - - lattice_properties_l = list(TANGO_PROPERTIES[devclass][1].keys()) - #print "possible lattice tango properties are ", lattice_properties_l - for k in self.parameters.keys(): - #print "pjb zzz 1", self.parameters["Tilt"], self.parameters - #print "key", k - #if not a required property or attribue then pop it - if k.lower() not in lattice_properties_l and k not in fixed_properties_l: - #print "popping ", k - self.parameters.pop(k) - - #otherwise rename key if an attribute - if k.lower() in lattice_properties_l: - #print "KEY ", k.lower(), TANGO_PROPERTIES[devclass][1][k.lower()], self.parameters[k] - self.parameters[TANGO_PROPERTIES[devclass][1][k.lower()]] = [self.parameters.pop(k)] - #print "pjb zzz", self.parameters["Tilt"], self.parameters - - - else: - for k in self.parameters.keys(): - self.parameters.pop(k) - - if "MAG" in self.itemName and not "CIR" in self.itemName: - self.parameters["Type"] = [self.itemType] - - # - if "Tilt" in self.parameters: - if "0.5 pi" in str(self.parameters["Tilt"]): - self.parameters["Tilt"] = ["90"] - else: - self.parameters["Tilt"] = ["0"] - print "pjbyyy", self.parameters - - def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[0-9]+)'): - ''' - Updates json file - ''' - # prepare pattern for parsing name - pattern = re.compile(name_parsing_string) - - print "In add device for item: " + self.itemName + " as " + self.itemType, self.alpars - # only when we know class for certain element - - print "adict is ", adict - circuit_alarm_params = [] - - #for case with no final number like I.TR1.MAG.DIE (no .1 etc at end) - alt_name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)' - alt_pattern = re.compile(alt_name_parsing_string) - - #self.alpars = {} - devdictalarm = None - - if types_classes.has_key(self.itemType): - - print "" - print "" - - # split name - name_items = pattern.search(self.itemName) - parsed=False - tryagain=False - if name_items == None: - print "Warning: Item name in lattice file doesn't match the naming convention.",self.itemName - tryagain=True - else: - parsed=True - if tryagain: - name_items = alt_pattern.search(self.itemName) - if name_items == None: - print "Warning: Item name in lattice file STILL doesn't match the naming convention.",self.itemName - else: - parsed=True - if parsed: - system = name_items.group('system') - subsystem = name_items.group('subsystem') - location = name_items.group('location') - device = name_items.group('device') - - if tryagain==False: - num = name_items.group('num') - else: - num="" - - if num == None: num = '' - - if num != "": - num2 = "%02d" % int(num) - else: - num2 = "01" - - - self.match_properties() - print "pjbxxx ", self.parameters - - # create device for json output - name = (system+"-"+location + '/' + subsystem + '/' + device + "-" + num2).encode('ascii', 'ignore') - devclass = types_classes[self.itemType].encode('ascii', 'ignore') - server = devclass + '/' + system+"-"+location - - #hack for circuits - if "CIR" in self.itemName: - print "orig name",self.itemName, name - - #quad circuits named like CRQ - #correctors like CRCOX and CRCOY - #dipoles like CRDI - #solenoid like CRSOL - #sextu like CRX - endname = name.rsplit("/",1)[1] - - #fix circuit names - endname = self.handle_circuit_name(self.itemName,num2) - name = name.rsplit("/",1)[0] + "/" + endname - - #e.g in G00 have COIX 1, 2, 3 and COHX 1 and 2, which would make COX 1, 2, 3 with COIX 1 and 2 being lost! - #increment number till unique - while name in cirlist: - print "danger! already named this circuit!", self.itemName, name - suffix = int(name.split("-")[2]) + 1 - newnum2 = "%02d" % suffix - print newnum2 - name = (name.rsplit("-",1)[0]) + "-" + newnum2 - print name - print "new name ", name - cirlist.append(name) - print cirlist - devclass = "MagnetCircuit" - - - #compact name is to find tag in plc alarms - name_l_cir = self.itemName.split(".") - section_cir = name_l_cir[1] - mid=name_l_cir[3] - if "_" in mid: - mid = mid.split("_")[0] - #hack for SM1A and SM1B in TR1, also DIE, DIF on circuit DI (DIC, DID on TR3) - compactnamecir = "_"+name_l_cir[1]+mid+"_" - compactnamecir = compactnamecir.replace("1A","1") - compactnamecir = compactnamecir.replace("1B","1") - if "DIE" in compactnamecir: - compactnamecir = compactnamecir.replace("DIE","DI") - if "DIF" in compactnamecir: - compactnamecir = compactnamecir.replace("DIF","DI") - if "DIC" in compactnamecir: - compactnamecir = compactnamecir.replace("DIC","DI") - if "DID" in compactnamecir: - compactnamecir = compactnamecir.replace("DID","DI") - - print "circuit compact name is ", compactnamecir, name_l_cir - - #fill alarm info for circuits - # - pyalarm = system+"-"+location + '/MAG/ALARM' - if adict is not None: - devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] - - print "init devdictalarm circuit" - - already_added = ["B_I_SP02DIPBD_DIA_TSW1_A","B_I_SP02DIPBD_DIA_TSW2_A"] #can use this to ignore to. e.g. SP02DIPDB doesnt end in a number but isn't a circuit! - for key in alarm_dict: - #compact name is like _TR1QF_ but some tags are like _TR1QF2_5_ - #if compactnamecir in key: - if (key not in already_added and compactnamecir in key) or (compactnamecir[:-1] in key and key.count("_")>5 and key not in already_added and "F"+num+"_" in key): - - print "key is", num, key - - already_added.append(key) - - print "FOUND ALARM INFO FOR CIR", compactnamecir, key, alarm_dict[key], section_cir - pyattname = "I-" + section_cir + "/DIA/COOLING" - - #for the magnets json file - circuit_alarm_params = [pyattname, key, alarm_dict[key]] - - #for the alarms json file - alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+endname.replace("-","_") - alsev = alname+":"+"ALARM" - alrec = alname+":"+"HTML" - aldesc = alname+":One magnet in circuit "+ alarm_dict[key] - - if pyalarm not in self.alpars: - self.alpars[pyalarm] = {} - self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars - - - - print "+++++++++++++++++ Creating device server : " + server + " for " + devclass + " (name= " + name + ")" - print "+++++++++ With properties : ", self.parameters - - # Dont actually write attributes to DB, only properties - # see if this class exists and append if so, or create - devdict = sdict.servers["%s/%s" % (devclass, system+"-"+location)][devclass][name] - - #for circuit json only - if "CR" in name: - psdevdict = psdict.Circuits[name] - - - if "MAG" in self.itemName and "CIR" not in self.itemName: - - #compact name is to find tag in plc alarms - name_l = self.itemName.split(".") - section = name_l[1] - del name_l[0] - del name_l[1] - #del name_l[2] - print "pjb kkk", name_l - compactfullname = "".join(name_l) - compactname = compactfullname.split("_")[0] - compactname_nonum = compactfullname.split("_")[0][:-1]+"_" - - print "-------------------- magnet not circuit", self.itemName, compactname - - #see what is the ps of the magnet - if name in POWER_SUPPLY_MAP: - powersupplyname = POWER_SUPPLY_MAP[name] - else: - print "magnet not in PS map, skipping", name - return - - #!!!!!!!!!!! *********** create circuit device for each new ps ************!!!!!!!!!!!! - # copy the magnet and call recursively add device! - magnetcircuit = copy.deepcopy(self) - magnetcircuit.itemName = self.itemName + ".CIR" - - magnetcircuit.parameters = {} - magnetcircuit.parameters['PowerSupplyProxy'] = [powersupplyname] - magnetcircuit.parameters['MagnetProxies'] = [name] - magnetcircuit.parameters['RiseTime'] = ["0.0"] - magnetcircuit.parameters['ResistanceReference'] = ["0.0"] - magnetcircuit.parameters['CoilNames'] = [""] - - #for the ps json file only - magnetcircuit.psparameters['PowerSupplyProxy'] = [powersupplyname] - magnetcircuit.psparameters['MagnetProxies'] = [name] - - #get alarm info from excel - pyalarm = system+"-"+location + '/MAG/ALARM' - - print "adict is again", adict - if adict is not None: - devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] - print "init devdictalarm" - - #set alarms in magnet.json file and in alarms json file - for key in alarm_dict: - if compactname in key and key.count("_")<6: - #if compactname in key: - - pyattname = "I-" + section + "/DIA/COOLING" - - print "FOUND ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict - - #for the magnets json file - if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] - else: - self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) - - #for the alarms json file - alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+'MAG'+'__'+ device + "_" +num2 - alsev = alname+":"+"ALARM" - aldesc = alname+":Magnet "+alarm_dict[key] - - if pyalarm not in self.alpars: - self.alpars[pyalarm] = {} - self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars - - devdictalarm.properties = self.alpars[pyalarm] - - #set alarms in magnet.json file for all magnets in circuit - for key in alarm_dict: - if compactname_nonum in key or (compactname[:-1] in key and key.count("_")>5 and ("F"+num+"_" in key or "_"+num+"_" in key)): - #if compactname_nonum in key: - print "mag key ", key, compactname_nonum, compactname, "F"+num+"_", "_"+num+"_" - pyattname = "I-" + section + "/DIA/COOLING" - - print "FOUND MORE ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict - - #for the magnets json file - if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] - else: - self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) - - polarity = 1 - orientation = 1 - - #get calibration info from the excel - if self.itemName.split("_")[0] in calib_dict: - print "FOUND CALIB INFO", self.itemName - #find max multipole expansions - dim = max(calib_dict[self.itemName.split("_")[0]].keys(), key=int) - - print "--- max order is", dim - - #create arrays of this dimensions, other dimension is 11 - - fieldsmatrix = [[0 for x in xrange(19)] for x in xrange(dim)] - #print fieldsmatrix - currentsmatrix = [[0 for x in xrange(19)] for x in xrange(dim)] - - #iterate over keys and add to the array - for key in calib_dict[self.itemName.split("_")[0]]: - print '--- ', key, 'corresponds to', calib_dict[self.itemName.split("_")[0]][key] - currents = calib_dict[self.itemName.split("_")[0]][key][5:25] - fields = calib_dict[self.itemName.split("_")[0]][key][25:45] - print '--- ',currents, fields - - fieldsmatrix[key-1]=fields - currentsmatrix[key-1]=currents - #key here is the multipole order. any one should have same polarity - polarity = calib_dict[self.itemName.split("_")[0]][key][3] - orientation = calib_dict[self.itemName.split("_")[0]][key][2] - #print "P, O", polarity, orientation - - print '--- ',fieldsmatrix - print '--- ',currentsmatrix - - #now trim the matrices (lists) - maxlength = 20 - for i,val in enumerate(fieldsmatrix[dim-1]): - if val=='': - print i , val - maxlength = i - break - print maxlength - for i in xrange(dim): - print i - del fieldsmatrix[i][maxlength:] - del currentsmatrix[i][maxlength:] - - print 'Now--- ',fieldsmatrix - print 'Now--- ',currentsmatrix - - magnetcircuit.parameters['ExcitationCurveCurrents']= currentsmatrix - magnetcircuit.parameters['ExcitationCurveFields']= fieldsmatrix - - self.parameters['Orientation'] = [str(int(orientation))] - self.parameters['Polarity'] = [str(int(polarity))] - - #assign circuit name as property of magnet device - #no regex to fix name here so do by hand - #e.g. I.BC1.MAG.COEX.4.CIR -> I-BC1/MAG/COEX-CIR-04 - - cname = name.rsplit("/CIR",1)[0] - endname = cname.rsplit("/",1)[1] - endnum = cname.rsplit("-",1)[1] - endname = self.handle_circuit_name(self.itemName,endnum) - cname = cname.rsplit("/",1)[0] + "/" + endname - - print "cname is ", cname, name, powersupplyname, circuit_ps_list - - while cname in cirlist: - print "danger2! already named this circuit!", cname - suffix = int(cname.split("-")[2]) + 1 - newnum2 = "%02d" % suffix - cname = (cname.rsplit("-",1)[0]) + "-" + newnum2 - print "new name ", cname - - #only add one circuit device per ps - if powersupplyname not in circuit_ps_list: - - magnetcircuit.add_device(sdict,adict,psdict) - circuit_ps_list[powersupplyname] = cname - - print "adding circuit name ", magnetcircuit.itemName, cname, circuit_ps_list - self.parameters['CircuitProxies'] = [cname] - - else: - #if we aleady made this circuit device, add it to this magnet properties - print "!!!ALART!!! already added a circuit device for ", self.itemName, name, system, location - - if system == "R3": - system= "I" - if location == "301L": - location = "TR3" - - self.parameters['CircuitProxies'] = [circuit_ps_list[powersupplyname]] - - #need to get the name of the circuit device from the ps dict though - print "exiting circuit device is", circuit_ps_list[powersupplyname] - - #print "current mags ", system+"-"+location - #print "current mags 2", sdict.servers - - current_mags = sdict.servers["%s/%s" % ("MagnetCircuit", system+"-"+location)]["MagnetCircuit"][circuit_ps_list[powersupplyname]].properties - #for circuits json - print "cir name from ps ", circuit_ps_list[powersupplyname], psdict - ps_current_mags = psdict.Circuits[circuit_ps_list[powersupplyname]].Properties - print "current mags ", current_mags['MagnetProxies'] - if name in current_mags['MagnetProxies']: - print "circuit already has magnet ", name - else: - ps_current_mags['MagnetProxies'].append(name) - current_mags['MagnetProxies'].append(name) - - print "magnets on cir ", current_mags['ExcitationCurveFields'], current_mags['MagnetProxies'], len(current_mags['MagnetProxies']) - #need to average the currents, even if already done so in excel (depends on field order) - if 'ExcitationCurveFields' in current_mags: - - assoc_field_m = current_mags['ExcitationCurveFields'] - this_field_m = fieldsmatrix - - assoc_curr_m = current_mags['ExcitationCurveCurrents'] - this_curr_m = currentsmatrix - - print "field matrix assoc is ", assoc_field_m - print "field matrix current is ", this_field_m - - print "current matrix assoc is ", assoc_curr_m - print "current matrix current is ", this_curr_m - - for i in xrange(dim): - print i - - #fix for CRSM take abs field values since opp sign - if circuit_ps_list[powersupplyname] in ["I-TR3/MAG/CRSM-01","I-TR3/MAG/CRDI-01"]: - newFields = [ ( abs(x)*(len(current_mags['MagnetProxies']) -1) + abs(y) ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] - else: - newFields = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] - - newCurrents = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_curr_m[i],assoc_curr_m[i])] - - print "new fields ", newFields - print "new currents ", newCurrents - current_mags['ExcitationCurveFields'][i] = newFields - print "updated: ", current_mags['ExcitationCurveFields'] - current_mags['ExcitationCurveCurrents'][i] = newCurrents - print "updated: ", current_mags['ExcitationCurveCurrents'] - - - else: - print "NOT A MAGNET" - - devdict.properties = self.parameters - - #for circuits json - if "CR" in name: - psdevdict.Properties = self.psparameters - - -class ElegantLatticeParser: - ''' Class for parsing an elegant lattice file. ''' - fileName = "" - file = None - items = [] - - def __init__(self, _fileName): - '''Constructs a parser object. - - Keyword arguments: - _fileName -- the name of file to be parsed - ''' - self.fileName = _fileName - self.file = io.open(_fileName) - - - - def parseLatticeFile(self): - ''' ''' - line = "" # this will be a line combined from lines to be connected - for ll in self.file: - l = ll.lstrip().rstrip() - if len(l) > 0: - if l[0] == '!': - pass # do not process comments - elif l[0] == '%': - pass # processing RPNs are not implemented - elif l[-1] == '&': - # Combine lines to be concated - line = line + ' ' + l[:-1] - else: - # So this is the last line to be combined - line = line + l - # Create an object and add it to list - self.items.append(LatticeFileItem(line.lstrip().rstrip())) - line = "" - - - -if __name__ == '__main__': - - - - inFileName = '' - doCalib=False - doAlarms=False - excelName = 'MagnetCalibrationData.xls' - alarmName = 'IMAG_ALARM_140923_Magnets.xls' - - # define classes for lattice elements - types_classes = {} - types_classes["dip"] = "Magnet" - types_classes["sbend"] = "Magnet" - types_classes["sben"] = "Magnet" - types_classes["rben"] = "Magnet" - types_classes["csrcsbend"] = "Magnet" - types_classes["sole"] = "Magnet" - types_classes["kquad"] = "Magnet" - types_classes["ksext"] = "Magnet" - types_classes["hkick"] = "Magnet" - types_classes["vkick"] = "Magnet" - #types_classes["hkick"] = "VACorrector" - #types_classes["vkick"] = "VACorrector" - #types_classes["monitor"] = "VABPM" - #types_classes["watch"] = "VAYAGScreen" - #types_classes["rfcw"] = "VARfCavity" - # - circuit_ps_list = {} - #circuit_alarm_params = None - - #read arguments - for par in sys.argv[1:]: - if par[0:2] == '--': - if par == '--calibration-data': - doCalib = True - elif par == '--alarm-data': - doAlarms = True - # elif par == '--test-mode': - # test_mode = True - elif inFileName == '': - inFileName = par - - print inFileName, doCalib - - - if doAlarms or doCalib: - import xlrd - - alarm_dict = {} - if doAlarms: - print "opening alarms xls" - xls = xlrd.open_workbook(alarmName) - sheet = xls.sheet_by_name('Sheet1') - rows = [sheet.row_values(i) for i in xrange(sheet.nrows)] - column_names = rows[0] - print "cols ", column_names - for row in enumerate(rows[1:]): - if row[1][0]=="": - continue - print row[1][0],row[1][1] - alarm_dict[row[1][0]] = row[1][1] - print "DICT IS ", alarm_dict - - #make dict just for alarms! for pyalarm - json_dict_alarms = SuperDict() - else: - json_dict_alarms = None - - calib_dict = {} - - if doCalib: - #open excel sheet - xls = xlrd.open_workbook(excelName) - - for name in ["linac", "transfer 1,5 GeV", "transfer 3 GeV", "thermionic gun"]: - - #sheet = xls.sheet_by_name('Linac') - sheet = xls.sheet_by_name(name) - rows = [sheet.row_values(i) for i in xrange(sheet.nrows)] - column_names = rows[7] - print "cols ", column_names - - for row in enumerate(rows[9:]): - print row[1] - if row[1][2]=="": - continue - #this is like - #[5.0, u'I.S01A', u'I.S01A.MAG.QE.1', 202005.0, u'#1168-10030-0001', 1.0, -1.0, 2.0, 6.3167, 5.6757, 5.0307500000000003, 4.4208999999999996, 3.8452999999999999, 3.1463999999999999, 2.5179624999999999, 1.8892374999999999, 1.2808725000000001, 0.63988750000000016, 0.0, 0.70470485548532313, 0.63908274382966312, 0.56946571499960408, 0.50203927491440703, 0.43686121069898298, 0.35966476443894108, 0.288993167760146, 0.21848942173091002, 0.14957521795596601, 0.077488874695939805, 0.0052044472873010797, u'T', u'Rotating coil-C1168, #0001.xls', u'https://alfresco.maxlab.lu.se/share/page/site/maxiv/document-details?nodeRef=workspace://SpacesStore/23cdc9d1-a01e-443e-b578-1538637a1472', u'Scanditronix Magnet', 40690.0, ''] - if row[1][2].strip() not in calib_dict: - if row[1][7] is not "": - data_key = int(row[1][7]) - data_list = row[1][3:48] - data_dict = {data_key : data_list} - calib_dict[row[1][2].strip()]=data_dict - #calib_dict[row[1][2]]=row[1][3:33] - else: - if row[1][7] is not "": - #we found more curves for the same magnet - print "found another entry", row[1][2], row[1][7] - data_key = int(row[1][7]) - data_list = row[1][3:48] - data_dict = {data_key : data_list} - calib_dict[row[1][2].strip()][data_key]=data_list - - print "DICT IS ", calib_dict - - - - # create a parser for the file - parser = ElegantLatticeParser(inFileName) - - # parse the file - parser.parseLatticeFile() - parser.file.close() - - - # make a json file - json_dict = SuperDict() - json_ps = SuperDict() - for item in parser.items: - item.add_device(json_dict,json_dict_alarms, json_ps) - #print json.dumps(json_dict, indent=4) - - #now we have the dict, loop over again and sort out magnets, power supplies and circuits - - print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ " - print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ " - print json_dict.servers - - outfile = open('magnets.json', 'w') - - #have final json dict here - print "THE FINAL DICT" - topl = json_dict['servers'].keys() - for item in topl: - if "Circuit" in item: - #print json_dict['servers'][item] - for cir in json_dict['servers'][item]: - #print json_dict['servers'][item][cir]['ExcitationCurveCurrents'] - #print json_dict['servers'][item]["MagnetCircuit"] - for c in json_dict['servers'][item]["MagnetCircuit"]: - for key in json_dict['servers'][item]["MagnetCircuit"][c]["properties"]: - if key == "ExcitationCurveCurrents": - ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print key, [str(x) for x in ls] - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] - if key == "ExcitationCurveFields": - ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print key, [str(x) for x in ls] - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] - - json.dump(json_dict, outfile, indent=4) - - if doAlarms: - outfile2 = open('magnets_alarms.json', 'w') - json.dump(json_dict_alarms, outfile2, indent=4) - - #!! note that item has parameters, but only want to extract those needed for tango! - - #dump ps circuit info - outfile3 = open('circuits.json', 'w') - json.dump(json_ps, outfile3, indent=4) - +#!/usr/bin/env python +# "$Name: $"; +# "$Header: $"; +#============================================================================= +# +# file : lattice2json.py +# +# description : Python source for the lattice2json that is a tool to generate +# a json file from an elegant lattice file +# The json file can then be uploaded to a Tango database +# +# project : Virtual Accelerator +# +# $Author: $ +# +# $Revision: $ +# +# $Log: $ +# +# copyleft : Solaris/MAX IV +# Krakow,PL/Lund,SE# +# + +from collections import defaultdict +import json +import os +import io +import sys +import re +from TangoProperties import TANGO_PROPERTIES +from PowerSupplyMap import POWER_SUPPLY_MAP +import copy +import numpy as np + +cirlist = [] + + +class SuperDict(defaultdict): + "A recursive defaultdict with extra bells & whistles" + + def __init__(self): + defaultdict.__init__(self, SuperDict) + + def __setattr__(self, attr, value): + self[attr] = value + + def __getattr__(self, attr): + return self[attr] + + +class LatticeFileItem: + ''' ''' + itemName = "" + itemType = '' + parameters = {} + properties = {} + alpars= {} + + def __init__(self, _line=''): + ''' + Construct an object parsing a _line from a lattice file + ''' + self.psparameters= {} + self.parameters= {} + self.properties= {} + + + # find a name + colon_pos = _line.find(':') + self.itemName = _line[:colon_pos].lstrip().rstrip().upper() + + # what left to be parsed + line_left = _line[colon_pos + 1:].lstrip() + + # find a type + param_name = '' # the first item after a colon could be also a parameter name, like for a line element + eq_pos = line_left.find('=') + comma_pos = line_left.find(',') + # let it work even there are no parameters defined - only element type + if eq_pos < 0: eq_pos = len(line_left) + if comma_pos < 0: comma_pos = len(line_left) + # so, we could read an element type + self.itemType = line_left[:min(comma_pos, eq_pos)].rstrip().lower() + + # this is waiting to be processed: + line_left = line_left[comma_pos + 1:].lstrip() + + # if the element type is also parameter name state this + if eq_pos < comma_pos: param_name = self.itemType + + # parse the rest for parameters + while line_left != '': + if param_name != '': + # searching for a value + param_value = '' + if line_left[0] == '(': + # value in brackets (may contain commas) + bracket_pos = line_left.index(')', 1) # will rise an exception in case of badly formated line + # so, the value is (including brackets): + param_value = line_left[:bracket_pos + 1] + # this is what left to be parsed + line_left = line_left[bracket_pos + 1:].lstrip() + + elif line_left[0] == '\"': + # value in quotes (could contain commas) + quote_pos = line_left.index('\"', 1) # will rise an exception in case of badly formated line + # so, the value is (including quote): + param_value = line_left[:quote_pos + 1] + + # this is what left to be parsed + line_left = line_left[quote_pos + 1:].lstrip() + else: + # typical case - the value between an equal and a comma characters + comma_pos = line_left.find(',') + if comma_pos < 0: comma_pos = len(line_left) + # a value, here you are + param_value = line_left[:comma_pos].rstrip() + # the following left to be parsed + line_left = line_left[comma_pos + 1:].lstrip() + # store the parameter with the corresponding value + self.parameters[param_name] = param_value + # PJB reset name back to empty here to find next parameter!(to enter else below) + param_name='' + else: + # searching for a parameter + eq_pos = line_left.find('=') + if eq_pos < 0: eq_pos = len(line_left) + # allow value-less parameters + comma_pos = line_left.find(',') + if comma_pos < 0: comma_pos = len(line_left) + # so we know where to find parameter name + param_name = line_left[:min(eq_pos, comma_pos)].rstrip().lower() + # if the parameter has no value add it directly to the dictionary + if comma_pos <= eq_pos: + self.parameters[param_name] = '' + param_name = '' + # this is what left to be parsed + line_left = line_left[min(eq_pos, comma_pos) + 1:].lstrip() + + + def handle_circuit_name(self,itemName,endnum): + + endname="" + #hack for bc1 + if "QD" in itemName and "BC1" in itemName: + endname = "CRQM-" + endnum + # + #hack for bc2 + elif "QF" in itemName and "BC2" in itemName and "3" not in itemName and "4" not in itemName and "5" not in itemName: + endname = "CRQM-" + endnum + elif "QF" in itemName and "BC2" in itemName and ("3" in itemName or "5" in itemName): + endname = "CRQ1-01" + elif "QF" in itemName and "BC2" in itemName and "4" in itemName: + endname = "CRQ2-01" + # + elif "Q" in itemName: + endname = "CRQ-" + endnum + elif "CO" in itemName and "X" in itemName: + endname = "CRCOX-" + endnum + elif "CO" in itemName and "Y" in itemName: + endname = "CRCOY-" + endnum + elif "DI" in itemName: + endname = "CRDI-" + endnum + print("dealing with endname ", endname) + elif "SX" in itemName: + endname = "CRSX-" + endnum + elif "SOL" in itemName: + endname = "CRSOL-" + endnum + elif "SM" in itemName: + endname = "CRSM-" + endnum + else: + sys.exit("Cannot convert circuit name" + itemName) + + if "/" in endname: #in case did not end with number, endname will be some */*/ by mistake + endname=endname.split("-")[0]+"-01" + + return endname + + + def config_alarms(self,pyalarm,alname,alsev,aldesc,pyattname,key): + + alrec = alname+":"+"HTML" + + if "AlarmList" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["AlarmList"] = [] + self.alpars[pyalarm]['AlarmList'].append(alname+":"+pyattname+"/"+key) + + if "AlarmSeverities" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["AlarmSeverities"] = [] + self.alpars[pyalarm]['AlarmSeverities'].append(alsev) + + if "AlarmDescriptions" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["AlarmDescriptions"] = [] + self.alpars[pyalarm]['AlarmDescriptions'].append(aldesc) + + if "AlarmReceivers" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["AlarmReceivers"] = [] + self.alpars[pyalarm]['AlarmReceivers'].append(alrec) + + if "StartupDelay" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["StartupDelay"] = ["0"] + + if "AutoReset" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["AutoReset"] = ["60"] + + if "MaxMessagesPerAlarm" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["MaxMessagesPerAlarm"] = ["1"] + + if "PollingPeriod" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["PollingPeriod"] = ["5"] + + if "LogFile" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["LogFile"] = ["/tmp/pjb/log"] + + if "HtmlFolder" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["HtmlFolder"] = ["/tmp/pjb"] + + if "AlarmThreshold" not in self.alpars[pyalarm]: + self.alpars[pyalarm]["AlarmThreshold"] = ["1"] + + + def match_properties(self): + + + devclass = types_classes[self.itemType] + + if "CIR" in self.itemName: + print("dealing with magnet circuit") + + # for given item type, look up required attributes and properties of tango + elif devclass in TANGO_PROPERTIES: + + fixed_properties_l = list(TANGO_PROPERTIES[devclass][0].keys()) + #print "fixed tango properties are ", fixed_properties_l + + # add the fixed properties + self.parameters.update(TANGO_PROPERTIES[devclass][0]) + + lattice_properties_l = list(TANGO_PROPERTIES[devclass][1].keys()) + #print "possible lattice tango properties are ", lattice_properties_l + for k in list(self.parameters.keys()): + #print "pjb zzz 1", self.parameters["Tilt"], self.parameters + #print "key", k + #if not a required property or attribue then pop it + if k.lower() not in lattice_properties_l and k not in fixed_properties_l: + #print "popping ", k + self.parameters.pop(k) + + #otherwise rename key if an attribute + if k.lower() in lattice_properties_l: + #print "KEY ", k.lower(), TANGO_PROPERTIES[devclass][1][k.lower()], self.parameters[k] + self.parameters[TANGO_PROPERTIES[devclass][1][k.lower()]] = [self.parameters.pop(k)] + #print "pjb zzz", self.parameters["Tilt"], self.parameters + + + else: + for k in list(self.parameters.keys()): + self.parameters.pop(k) + + if "MAG" in self.itemName and not "CIR" in self.itemName: + self.parameters["Type"] = [self.itemType] + + # + if "Tilt" in self.parameters: + if "0.5 pi" in str(self.parameters["Tilt"]): + self.parameters["Tilt"] = ["90"] + else: + self.parameters["Tilt"] = ["0"] + print("pjbyyy", self.parameters) + + def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[0-9]+)'): + ''' + Updates json file + ''' + # prepare pattern for parsing name + pattern = re.compile(name_parsing_string) + + print("In add device for item: " + self.itemName + " as " + self.itemType, self.alpars) + # only when we know class for certain element + + print("adict is ", adict) + circuit_alarm_params = [] + + #for case with no final number like I.TR1.MAG.DIE (no .1 etc at end) + alt_name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)' + alt_pattern = re.compile(alt_name_parsing_string) + + #self.alpars = {} + devdictalarm = None + + if self.itemType in types_classes: + + print("") + print("") + + # split name + name_items = pattern.search(self.itemName) + parsed=False + tryagain=False + if name_items == None: + print("Warning: Item name in lattice file doesn't match the naming convention.",self.itemName) + tryagain=True + else: + parsed=True + if tryagain: + name_items = alt_pattern.search(self.itemName) + if name_items == None: + print("Warning: Item name in lattice file STILL doesn't match the naming convention.",self.itemName) + else: + parsed=True + if parsed: + system = name_items.group('system') + subsystem = name_items.group('subsystem') + location = name_items.group('location') + device = name_items.group('device') + + if tryagain==False: + num = name_items.group('num') + else: + num="" + + if num == None: num = '' + + if num != "": + num2 = "%02d" % int(num) + else: + num2 = "01" + + + self.match_properties() + print("pjbxxx ", self.parameters) + + # create device for json output + name = (system+"-"+location + '/' + subsystem + '/' + device + "-" + num2).encode('ascii', 'ignore') + devclass = types_classes[self.itemType].encode('ascii', 'ignore') + server = devclass + '/' + system+"-"+location + + #hack for circuits + if "CIR" in self.itemName: + print("orig name",self.itemName, name) + + #quad circuits named like CRQ + #correctors like CRCOX and CRCOY + #dipoles like CRDI + #solenoid like CRSOL + #sextu like CRX + endname = name.rsplit("/",1)[1] + + #fix circuit names + endname = self.handle_circuit_name(self.itemName,num2) + name = name.rsplit("/",1)[0] + "/" + endname + + #e.g in G00 have COIX 1, 2, 3 and COHX 1 and 2, which would make COX 1, 2, 3 with COIX 1 and 2 being lost! + #increment number till unique + while name in cirlist: + print("danger! already named this circuit!", self.itemName, name) + suffix = int(name.split("-")[2]) + 1 + newnum2 = "%02d" % suffix + print(newnum2) + name = (name.rsplit("-",1)[0]) + "-" + newnum2 + print(name) + print("new name ", name) + cirlist.append(name) + print(cirlist) + devclass = "MagnetCircuit" + + + #compact name is to find tag in plc alarms + name_l_cir = self.itemName.split(".") + section_cir = name_l_cir[1] + mid=name_l_cir[3] + if "_" in mid: + mid = mid.split("_")[0] + #hack for SM1A and SM1B in TR1, also DIE, DIF on circuit DI (DIC, DID on TR3) + compactnamecir = "_"+name_l_cir[1]+mid+"_" + compactnamecir = compactnamecir.replace("1A","1") + compactnamecir = compactnamecir.replace("1B","1") + if "DIE" in compactnamecir: + compactnamecir = compactnamecir.replace("DIE","DI") + if "DIF" in compactnamecir: + compactnamecir = compactnamecir.replace("DIF","DI") + if "DIC" in compactnamecir: + compactnamecir = compactnamecir.replace("DIC","DI") + if "DID" in compactnamecir: + compactnamecir = compactnamecir.replace("DID","DI") + + print("circuit compact name is ", compactnamecir, name_l_cir) + + #fill alarm info for circuits + # + pyalarm = system+"-"+location + '/MAG/ALARM' + if adict is not None: + devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] + + print("init devdictalarm circuit") + + already_added = ["B_I_SP02DIPBD_DIA_TSW1_A","B_I_SP02DIPBD_DIA_TSW2_A"] #can use this to ignore to. e.g. SP02DIPDB doesnt end in a number but isn't a circuit! + for key in alarm_dict: + #compact name is like _TR1QF_ but some tags are like _TR1QF2_5_ + #if compactnamecir in key: + if (key not in already_added and compactnamecir in key) or (compactnamecir[:-1] in key and key.count("_")>5 and key not in already_added and "F"+num+"_" in key): + + print("key is", num, key) + + already_added.append(key) + + print("FOUND ALARM INFO FOR CIR", compactnamecir, key, alarm_dict[key], section_cir) + pyattname = "I-" + section_cir + "/DIA/COOLING" + + #for the magnets json file + circuit_alarm_params = [pyattname, key, alarm_dict[key]] + + #for the alarms json file + alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+endname.replace("-","_") + alsev = alname+":"+"ALARM" + alrec = alname+":"+"HTML" + aldesc = alname+":One magnet in circuit "+ alarm_dict[key] + + if pyalarm not in self.alpars: + self.alpars[pyalarm] = {} + self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars + + + + print("+++++++++++++++++ Creating device server : " + server + " for " + devclass + " (name= " + name + ")") + print("+++++++++ With properties : ", self.parameters) + + # Dont actually write attributes to DB, only properties + # see if this class exists and append if so, or create + devdict = sdict.servers["%s/%s" % (devclass, system+"-"+location)][devclass][name] + + #for circuit json only + if "CR" in name: + psdevdict = psdict.Circuits[name] + + + if "MAG" in self.itemName and "CIR" not in self.itemName: + + #compact name is to find tag in plc alarms + name_l = self.itemName.split(".") + section = name_l[1] + del name_l[0] + del name_l[1] + #del name_l[2] + print("pjb kkk", name_l) + compactfullname = "".join(name_l) + compactname = compactfullname.split("_")[0] + compactname_nonum = compactfullname.split("_")[0][:-1]+"_" + + print("-------------------- magnet not circuit", self.itemName, compactname) + + #see what is the ps of the magnet + if name in POWER_SUPPLY_MAP: + powersupplyname = POWER_SUPPLY_MAP[name] + else: + print("magnet not in PS map, skipping", name) + return + + #!!!!!!!!!!! *********** create circuit device for each new ps ************!!!!!!!!!!!! + # copy the magnet and call recursively add device! + magnetcircuit = copy.deepcopy(self) + magnetcircuit.itemName = self.itemName + ".CIR" + + magnetcircuit.parameters = {} + magnetcircuit.parameters['PowerSupplyProxy'] = [powersupplyname] + magnetcircuit.parameters['MagnetProxies'] = [name] + magnetcircuit.parameters['RiseTime'] = ["0.0"] + magnetcircuit.parameters['ResistanceReference'] = ["0.0"] + magnetcircuit.parameters['CoilNames'] = [""] + + #for the ps json file only + magnetcircuit.psparameters['PowerSupplyProxy'] = [powersupplyname] + magnetcircuit.psparameters['MagnetProxies'] = [name] + + #get alarm info from excel + pyalarm = system+"-"+location + '/MAG/ALARM' + + print("adict is again", adict) + if adict is not None: + devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] + print("init devdictalarm") + + #set alarms in magnet.json file and in alarms json file + for key in alarm_dict: + if compactname in key and key.count("_")<6: + #if compactname in key: + + pyattname = "I-" + section + "/DIA/COOLING" + + print("FOUND ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict) + + #for the magnets json file + if 'TemperatureInterlock' not in self.parameters: + self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] + else: + self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) + + #for the alarms json file + alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+'MAG'+'__'+ device + "_" +num2 + alsev = alname+":"+"ALARM" + aldesc = alname+":Magnet "+alarm_dict[key] + + if pyalarm not in self.alpars: + self.alpars[pyalarm] = {} + self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars + + devdictalarm.properties = self.alpars[pyalarm] + + #set alarms in magnet.json file for all magnets in circuit + for key in alarm_dict: + if compactname_nonum in key or (compactname[:-1] in key and key.count("_")>5 and ("F"+num+"_" in key or "_"+num+"_" in key)): + #if compactname_nonum in key: + print("mag key ", key, compactname_nonum, compactname, "F"+num+"_", "_"+num+"_") + pyattname = "I-" + section + "/DIA/COOLING" + + print("FOUND MORE ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict) + + #for the magnets json file + if 'TemperatureInterlock' not in self.parameters: + self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] + else: + self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) + + polarity = 1 + orientation = 1 + + #get calibration info from the excel + if self.itemName.split("_")[0] in calib_dict: + print("FOUND CALIB INFO", self.itemName) + #find max multipole expansions + dim = max(list(calib_dict[self.itemName.split("_")[0]].keys()), key=int) + + print("--- max order is", dim) + + #create arrays of this dimensions, other dimension is 11 + + fieldsmatrix = [[0 for x in range(19)] for x in range(dim)] + #print fieldsmatrix + currentsmatrix = [[0 for x in range(19)] for x in range(dim)] + + #iterate over keys and add to the array + for key in calib_dict[self.itemName.split("_")[0]]: + print('--- ', key, 'corresponds to', calib_dict[self.itemName.split("_")[0]][key]) + currents = calib_dict[self.itemName.split("_")[0]][key][5:25] + fields = calib_dict[self.itemName.split("_")[0]][key][25:45] + print('--- ',currents, fields) + + fieldsmatrix[key-1]=fields + currentsmatrix[key-1]=currents + #key here is the multipole order. any one should have same polarity + polarity = calib_dict[self.itemName.split("_")[0]][key][3] + orientation = calib_dict[self.itemName.split("_")[0]][key][2] + #print "P, O", polarity, orientation + + print('--- ',fieldsmatrix) + print('--- ',currentsmatrix) + + #now trim the matrices (lists) + maxlength = 20 + for i,val in enumerate(fieldsmatrix[dim-1]): + if val=='': + print(i , val) + maxlength = i + break + print(maxlength) + for i in range(dim): + print(i) + del fieldsmatrix[i][maxlength:] + del currentsmatrix[i][maxlength:] + + print('Now--- ',fieldsmatrix) + print('Now--- ',currentsmatrix) + + magnetcircuit.parameters['ExcitationCurveCurrents']= currentsmatrix + magnetcircuit.parameters['ExcitationCurveFields']= fieldsmatrix + + self.parameters['Orientation'] = [str(int(orientation))] + self.parameters['Polarity'] = [str(int(polarity))] + + #assign circuit name as property of magnet device + #no regex to fix name here so do by hand + #e.g. I.BC1.MAG.COEX.4.CIR -> I-BC1/MAG/COEX-CIR-04 + + cname = name.rsplit("/CIR",1)[0] + endname = cname.rsplit("/",1)[1] + endnum = cname.rsplit("-",1)[1] + endname = self.handle_circuit_name(self.itemName,endnum) + cname = cname.rsplit("/",1)[0] + "/" + endname + + print("cname is ", cname, name, powersupplyname, circuit_ps_list) + + while cname in cirlist: + print("danger2! already named this circuit!", cname) + suffix = int(cname.split("-")[2]) + 1 + newnum2 = "%02d" % suffix + cname = (cname.rsplit("-",1)[0]) + "-" + newnum2 + print("new name ", cname) + + #only add one circuit device per ps + if powersupplyname not in circuit_ps_list: + + magnetcircuit.add_device(sdict,adict,psdict) + circuit_ps_list[powersupplyname] = cname + + print("adding circuit name ", magnetcircuit.itemName, cname, circuit_ps_list) + self.parameters['CircuitProxies'] = [cname] + + else: + #if we aleady made this circuit device, add it to this magnet properties + print("!!!ALART!!! already added a circuit device for ", self.itemName, name, system, location) + + if system == "R3": + system= "I" + if location == "301L": + location = "TR3" + + self.parameters['CircuitProxies'] = [circuit_ps_list[powersupplyname]] + + #need to get the name of the circuit device from the ps dict though + print("exiting circuit device is", circuit_ps_list[powersupplyname]) + + #print "current mags ", system+"-"+location + #print "current mags 2", sdict.servers + + current_mags = sdict.servers["%s/%s" % ("MagnetCircuit", system+"-"+location)]["MagnetCircuit"][circuit_ps_list[powersupplyname]].properties + #for circuits json + print("cir name from ps ", circuit_ps_list[powersupplyname], psdict) + ps_current_mags = psdict.Circuits[circuit_ps_list[powersupplyname]].Properties + print("current mags ", current_mags['MagnetProxies']) + if name in current_mags['MagnetProxies']: + print("circuit already has magnet ", name) + else: + ps_current_mags['MagnetProxies'].append(name) + current_mags['MagnetProxies'].append(name) + + print("magnets on cir ", current_mags['ExcitationCurveFields'], current_mags['MagnetProxies'], len(current_mags['MagnetProxies'])) + #need to average the currents, even if already done so in excel (depends on field order) + if 'ExcitationCurveFields' in current_mags: + + assoc_field_m = current_mags['ExcitationCurveFields'] + this_field_m = fieldsmatrix + + assoc_curr_m = current_mags['ExcitationCurveCurrents'] + this_curr_m = currentsmatrix + + print("field matrix assoc is ", assoc_field_m) + print("field matrix current is ", this_field_m) + + print("current matrix assoc is ", assoc_curr_m) + print("current matrix current is ", this_curr_m) + + for i in range(dim): + print(i) + + #fix for CRSM take abs field values since opp sign + if circuit_ps_list[powersupplyname] in ["I-TR3/MAG/CRSM-01","I-TR3/MAG/CRDI-01"]: + newFields = [ ( abs(x)*(len(current_mags['MagnetProxies']) -1) + abs(y) ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] + else: + newFields = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] + + newCurrents = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_curr_m[i],assoc_curr_m[i])] + + print("new fields ", newFields) + print("new currents ", newCurrents) + current_mags['ExcitationCurveFields'][i] = newFields + print("updated: ", current_mags['ExcitationCurveFields']) + current_mags['ExcitationCurveCurrents'][i] = newCurrents + print("updated: ", current_mags['ExcitationCurveCurrents']) + + + else: + print("NOT A MAGNET") + + devdict.properties = self.parameters + + #for circuits json + if "CR" in name: + psdevdict.Properties = self.psparameters + + +class ElegantLatticeParser: + ''' Class for parsing an elegant lattice file. ''' + fileName = "" + file = None + items = [] + + def __init__(self, _fileName): + '''Constructs a parser object. + + Keyword arguments: + _fileName -- the name of file to be parsed + ''' + self.fileName = _fileName + self.file = io.open(_fileName) + + + + def parseLatticeFile(self): + ''' ''' + line = "" # this will be a line combined from lines to be connected + for ll in self.file: + l = ll.lstrip().rstrip() + if len(l) > 0: + if l[0] == '!': + pass # do not process comments + elif l[0] == '%': + pass # processing RPNs are not implemented + elif l[-1] == '&': + # Combine lines to be concated + line = line + ' ' + l[:-1] + else: + # So this is the last line to be combined + line = line + l + # Create an object and add it to list + self.items.append(LatticeFileItem(line.lstrip().rstrip())) + line = "" + + + +if __name__ == '__main__': + + + + inFileName = '' + doCalib=False + doAlarms=False + excelName = 'MagnetCalibrationData.xls' + alarmName = 'IMAG_ALARM_140923_Magnets.xls' + + # define classes for lattice elements + types_classes = {} + types_classes["dip"] = "Magnet" + types_classes["sbend"] = "Magnet" + types_classes["sben"] = "Magnet" + types_classes["rben"] = "Magnet" + types_classes["csrcsbend"] = "Magnet" + types_classes["sole"] = "Magnet" + types_classes["kquad"] = "Magnet" + types_classes["ksext"] = "Magnet" + types_classes["hkick"] = "Magnet" + types_classes["vkick"] = "Magnet" + #types_classes["hkick"] = "VACorrector" + #types_classes["vkick"] = "VACorrector" + #types_classes["monitor"] = "VABPM" + #types_classes["watch"] = "VAYAGScreen" + #types_classes["rfcw"] = "VARfCavity" + # + circuit_ps_list = {} + #circuit_alarm_params = None + + #read arguments + for par in sys.argv[1:]: + if par[0:2] == '--': + if par == '--calibration-data': + doCalib = True + elif par == '--alarm-data': + doAlarms = True + # elif par == '--test-mode': + # test_mode = True + elif inFileName == '': + inFileName = par + + print(inFileName, doCalib) + + + if doAlarms or doCalib: + import xlrd + + alarm_dict = {} + if doAlarms: + print("opening alarms xls") + xls = xlrd.open_workbook(alarmName) + sheet = xls.sheet_by_name('Sheet1') + rows = [sheet.row_values(i) for i in range(sheet.nrows)] + column_names = rows[0] + print("cols ", column_names) + for row in enumerate(rows[1:]): + if row[1][0]=="": + continue + print(row[1][0],row[1][1]) + alarm_dict[row[1][0]] = row[1][1] + print("DICT IS ", alarm_dict) + + #make dict just for alarms! for pyalarm + json_dict_alarms = SuperDict() + else: + json_dict_alarms = None + + calib_dict = {} + + if doCalib: + #open excel sheet + xls = xlrd.open_workbook(excelName) + + for name in ["linac", "transfer 1,5 GeV", "transfer 3 GeV", "thermionic gun"]: + + #sheet = xls.sheet_by_name('Linac') + sheet = xls.sheet_by_name(name) + rows = [sheet.row_values(i) for i in range(sheet.nrows)] + column_names = rows[7] + print("cols ", column_names) + + for row in enumerate(rows[9:]): + print(row[1]) + if row[1][2]=="": + continue + #this is like + #[5.0, u'I.S01A', u'I.S01A.MAG.QE.1', 202005.0, u'#1168-10030-0001', 1.0, -1.0, 2.0, 6.3167, 5.6757, 5.0307500000000003, 4.4208999999999996, 3.8452999999999999, 3.1463999999999999, 2.5179624999999999, 1.8892374999999999, 1.2808725000000001, 0.63988750000000016, 0.0, 0.70470485548532313, 0.63908274382966312, 0.56946571499960408, 0.50203927491440703, 0.43686121069898298, 0.35966476443894108, 0.288993167760146, 0.21848942173091002, 0.14957521795596601, 0.077488874695939805, 0.0052044472873010797, u'T', u'Rotating coil-C1168, #0001.xls', u'https://alfresco.maxlab.lu.se/share/page/site/maxiv/document-details?nodeRef=workspace://SpacesStore/23cdc9d1-a01e-443e-b578-1538637a1472', u'Scanditronix Magnet', 40690.0, ''] + if row[1][2].strip() not in calib_dict: + if row[1][7] is not "": + data_key = int(row[1][7]) + data_list = row[1][3:48] + data_dict = {data_key : data_list} + calib_dict[row[1][2].strip()]=data_dict + #calib_dict[row[1][2]]=row[1][3:33] + else: + if row[1][7] is not "": + #we found more curves for the same magnet + print("found another entry", row[1][2], row[1][7]) + data_key = int(row[1][7]) + data_list = row[1][3:48] + data_dict = {data_key : data_list} + calib_dict[row[1][2].strip()][data_key]=data_list + + print("DICT IS ", calib_dict) + + + + # create a parser for the file + parser = ElegantLatticeParser(inFileName) + + # parse the file + parser.parseLatticeFile() + parser.file.close() + + + # make a json file + json_dict = SuperDict() + json_ps = SuperDict() + for item in parser.items: + item.add_device(json_dict,json_dict_alarms, json_ps) + #print json.dumps(json_dict, indent=4) + + #now we have the dict, loop over again and sort out magnets, power supplies and circuits + + print("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ") + print("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ") + print(json_dict.servers) + + outfile = open('magnets.json', 'w') + + #have final json dict here + print("THE FINAL DICT") + topl = list(json_dict['servers'].keys()) + for item in topl: + if "Circuit" in item: + #print json_dict['servers'][item] + for cir in json_dict['servers'][item]: + #print json_dict['servers'][item][cir]['ExcitationCurveCurrents'] + #print json_dict['servers'][item]["MagnetCircuit"] + for c in json_dict['servers'][item]["MagnetCircuit"]: + for key in json_dict['servers'][item]["MagnetCircuit"][c]["properties"]: + if key == "ExcitationCurveCurrents": + ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] + print(key, [str(x) for x in ls]) + json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] + if key == "ExcitationCurveFields": + ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] + print(key, [str(x) for x in ls]) + json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] + + json.dump(json_dict, outfile, indent=4) + + if doAlarms: + outfile2 = open('magnets_alarms.json', 'w') + json.dump(json_dict_alarms, outfile2, indent=4) + + #!! note that item has parameters, but only want to extract those needed for tango! + + #dump ps circuit info + outfile3 = open('circuits.json', 'w') + json.dump(json_ps, outfile3, indent=4) + diff --git a/dsconfig/output.py b/dsconfig/output.py index 6504ec8..ff627c7 100644 --- a/dsconfig/output.py +++ b/dsconfig/output.py @@ -4,7 +4,7 @@ from dsconfig.utils import green, red, yellow from dsconfig.tangodb import get_devices_from_dict -from appending_dict import SetterDict +from .appending_dict import SetterDict def get_device_data(device, mapping, data): @@ -125,7 +125,7 @@ def get_changes(data, calls): caseless_props = CaselessDict(old_data.get("properties", {})) if "properties" not in changes["devices"][device]: changes["devices"][device]["properties"] = {} - for name, value in properties.items(): + for name, value in list(properties.items()): old_value = caseless_props.get(name) if value != old_value: changes["devices"][device]["properties"].update({ @@ -152,9 +152,9 @@ def get_changes(data, calls): old_data = get_device_data(device, device_mapping, data) caseless_attrs = CaselessDict(old_data.get( "attribute_properties", {})) - for attr, props in properties.items(): + for attr, props in list(properties.items()): caseless_props = CaselessDict(caseless_attrs.get(attr, {})) - for name, value in props.items(): + for name, value in list(props.items()): old_value = caseless_props.get(name) if value != old_value: attr_props[attr] = { @@ -168,7 +168,7 @@ def get_changes(data, calls): "attribute_properties", {}) old_data = get_device_data(device, device_mapping, data) caseless_attrs = CaselessDict(old_data.get("attribute_properties", {})) - for attr, props in attributes.items(): + for attr, props in list(attributes.items()): caseless_props = CaselessDict(caseless_attrs[attr]) for prop in props: old_value = caseless_props.get(prop) @@ -179,7 +179,7 @@ def get_changes(data, calls): old_data = classes.get(clss, {}) caseless_props = CaselessDict(old_data.get("properties", {})) prop_changes = changes["classes"][clss].setdefault("properties", {}) - for name, value in properties.items(): + for name, value in list(properties.items()): old_value = caseless_props.get(name) if value != old_value: prop_changes.update({ @@ -204,9 +204,9 @@ def get_changes(data, calls): old_data = classes.get(clss, {}) caseless_attrs = CaselessDict( old_data.get("attribute_properties", {})) - for attr, props in properties.items(): + for attr, props in list(properties.items()): caseless_props = CaselessDict(caseless_attrs.get(attr, {})) - for name, value in props.items(): + for name, value in list(props.items()): old_value = caseless_props.get(name) if value != old_value: attr_props[attr] = { @@ -220,7 +220,7 @@ def get_changes(data, calls): "attribute_properties", {}) old_data = classes.get(clss, {}) caseless_attrs = CaselessDict(old_data.get("properties", {})) - for attr, props in attributes.items(): + for attr, props in list(attributes.items()): caseless_props = CaselessDict(caseless_attrs.get(attr, {})) for prop in props: old_value = caseless_props.get(prop) @@ -246,40 +246,40 @@ def show_actions(data, calls): info = changes["devices"][device] if info.get("added"): if info.get("old_server"): - print("{} Device: {}".format(yellow("="), device)) + print(("{} Device: {}".format(yellow("="), device))) else: - print("{} Device: {}".format(green("+"), device)) + print(("{} Device: {}".format(green("+"), device))) elif info.get("deleted"): - print("{} Device: {}".format(red("-"), device)) + print(("{} Device: {}".format(red("-"), device))) else: - print("{} Device: {}".format(yellow("="), device)) + print(("{} Device: {}".format(yellow("="), device))) if info.get("server"): if info.get("old_server"): - print("{}Server: {} -> {}".format(indent, + print(("{}Server: {} -> {}".format(indent, red(info["old_server"]), - green(info["server"]))) + green(info["server"])))) else: - print("{}Server: {}".format(indent, info["server"])) + print(("{}Server: {}".format(indent, info["server"]))) if info.get("device_class"): if info.get("old_class"): if info["old_class"] != info["device_class"]: - print("{}Class: {} -> {}".format(indent, + print(("{}Class: {} -> {}".format(indent, info["old_class"], - info["device_class"])) + info["device_class"]))) else: - print("{}Class: {}".format(indent, info["device_class"])) + print(("{}Class: {}".format(indent, info["device_class"]))) if info.get("alias"): alias = info.get("alias").get("value") old_alias = info.get("alias").get("old_value") if old_alias: if old_alias != alias: - print("{}Alias: {} -> {}".format(indent, red(old_alias), - green(alias))) + print(("{}Alias: {} -> {}".format(indent, red(old_alias), + green(alias)))) else: - print("{}Alias: {}".format(indent, alias)) + print(("{}Alias: {}".format(indent, alias))) if info.get("properties"): lines = [] @@ -303,8 +303,8 @@ def show_actions(data, calls): lines.append(red("{}- {}".format(indent*2, prop))) lines.append(red(format_property(change["old_value"], indent*3))) if lines: - print("{}Properties:".format(indent*1)) - print("\n".join(lines)) + print(("{}Properties:".format(indent*1))) + print(("\n".join(lines))) if info.get("attribute_properties"): lines = [] @@ -335,8 +335,8 @@ def show_actions(data, calls): lines.append("{}{}".format(indent*2, attr)) lines.extend(attr_lines) if lines: - print("{}Attribute properties:".format(indent)) - print("\n".join(lines)) + print(("{}Attribute properties:".format(indent))) + print(("\n".join(lines))) for clss in sorted(changes["classes"]): info = changes["classes"][clss] @@ -363,6 +363,6 @@ def show_actions(data, calls): lines.append(red("{}- {}".format(indent*2, prop))) lines.append(red(format_property(change["old_value"], indent*3))) if lines: - print("{} Class Properties:".format(indent*1)) - print("\n".join(lines)) - print + print(("{} Class Properties:".format(indent*1))) + print(("\n".join(lines))) + print() diff --git a/dsconfig/remove.py b/dsconfig/remove.py index 16fc93c..7862f5c 100644 --- a/dsconfig/remove.py +++ b/dsconfig/remove.py @@ -3,7 +3,7 @@ import PyTango -from utils import (ObjectWrapper, decode_dict, get_dict_from_db, +from .utils import (ObjectWrapper, decode_dict, get_dict_from_db, RED, GREEN, YELLOW) @@ -12,12 +12,12 @@ def delete_devices(db, dbdict, server, cls, devices): if dbdict.servers[server][cls][devname]: db.delete_device(devname) else: - print "no device", devname + print("no device", devname) def delete_server(db, dbdict, servername, serverdata): - for clsname, devices in serverdata.items(): - delete_devices(db, dbdict, servername, clsname, devices.keys()) + for clsname, devices in list(serverdata.items()): + delete_devices(db, dbdict, servername, clsname, list(devices.keys())) try: db.delete_server_info(servername) # what does this do? db.delete_server(servername) @@ -26,14 +26,14 @@ def delete_server(db, dbdict, servername, serverdata): # and sometimes not; should they be running? What's going on? # Anyway, the worst case scenario is that the servers are # still there but contain no devices... - print "Removing server '%s' may have failed." % servername + print("Removing server '%s' may have failed." % servername) def delete_class(db, dbdict, classname): - props = dbdict.classes[classname].properties.keys() + props = list(dbdict.classes[classname].properties.keys()) if props: db.delete_class_property(classname, props) - attr_props = dbdict.classes[classname].attribute_properties.keys() + attr_props = list(dbdict.classes[classname].attribute_properties.keys()) if attr_props: db.delete_class_attribute_property(classname, attr_props) @@ -53,29 +53,29 @@ def main(json_file, write=False, db_calls=False): db = ObjectWrapper(db if write else None) # delete servers and devices - for servername, serverdata in data.get("servers", {}).items(): + for servername, serverdata in list(data.get("servers", {}).items()): if servername in dbdict.servers: delete_server(db, dbdict, servername, serverdata) # delete classes - for classname, classdata in data.get("classes", {}).items(): + for classname, classdata in list(data.get("classes", {}).items()): if classname in dbdict.classes: delete_class(db, dbdict, classname) if db_calls: - print "\nTango database calls:" + print("\nTango database calls:") for method, args, kwargs in db.calls: - print method, args + print(method, args) if db.calls: if write: - print >>sys.stderr, ( - RED + "\n*** Data was written to the Tango DB ***") + print(( + RED + "\n*** Data was written to the Tango DB ***"), file=sys.stderr) else: - print >>sys.stderr, YELLOW +\ - "\n*** Nothing was written to the Tango DB (use -w) ***" + print(YELLOW +\ + "\n*** Nothing was written to the Tango DB (use -w) ***", file=sys.stderr) else: - print >>sys.stderr, GREEN + "\n*** No changes needed in Tango DB ***" + print(GREEN + "\n*** No changes needed in Tango DB ***", file=sys.stderr) if __name__ == "__main__": diff --git a/dsconfig/tangodb.py b/dsconfig/tangodb.py index b71481c..7ef7b84 100755 --- a/dsconfig/tangodb.py +++ b/dsconfig/tangodb.py @@ -2,11 +2,11 @@ from collections import defaultdict from difflib import unified_diff -from itertools import izip, islice +from itertools import islice import PyTango -from appending_dict import AppendingDict, SetterDict, CaselessDictionary +from .appending_dict import AppendingDict, SetterDict, CaselessDictionary from dsconfig.utils import green, red, yellow @@ -30,16 +30,16 @@ def get_devices_from_dict(dbdict, name=None): return [(server_name, instance_name, class_name, device_name) - for server_name, server in dbdict.items() - for instance_name, instance in server.items() - for class_name, clss in instance.items() + for server_name, server in list(dbdict.items()) + for instance_name, instance in list(server.items()) + for class_name, clss in list(instance.items()) for device_name in clss if name is None or device_name.lower() == name.lower()] def get_servers_from_dict(dbdict): servers = set() - for server, children in dbdict.get("servers", {}).items(): + for server, children in list(dbdict.get("servers", {}).items()): if "/" in server: servers.add(server) else: @@ -109,7 +109,7 @@ def summarise_calls(dbcalls, dbdata): n = len(args[1]) devices[method].add(args[0].upper()) elif "attribute_property" in method: - n = sum(len(ps) for attr, ps in args[1].items()) + n = sum(len(ps) for attr, ps in list(args[1].items())) devices[method].add(args[0].upper()) elif "property" in method: n = len(args[1]) @@ -172,7 +172,7 @@ def get_device(db, devname, data, skip_protected=True): db_props = db.get_device_property_list(devname, "*") if db_props: props = db.get_device_property(devname, list(db_props)) - for prop, value in props.items(): + for prop, value in list(props.items()): # We'll ignore "protected" properties unless they are present # in the input data (in that case we want to show that # they are changed) @@ -195,11 +195,11 @@ def get_device(db, devname, data, skip_protected=True): if attr_props: attribute_properties = {} dbprops = db.get_device_attribute_property(devname, - attr_props.keys()) - for attr, props in dbprops.items(): + list(attr_props.keys())) + for attr, props in list(dbprops.items()): attr_props = dict( (prop, [str(v) for v in values]) - for prop, values in props.items() + for prop, values in list(props.items()) # Again, ignore attr_props that are not present in the # input data, as we most likely don't want to remove those if (not (skip_protected and is_protected(prop, True)) @@ -239,12 +239,12 @@ def get_dict_from_db(db, data, narrow=False, skip_protected=True): pass # Servers - for srvr, insts in data.get("servers", {}).items(): - for inst, classes in insts.items(): - for clss, devs in classes.items(): + for srvr, insts in list(data.get("servers", {}).items()): + for inst, classes in list(insts.items()): + for clss, devs in list(classes.items()): devs = CaselessDictionary(devs) if narrow: - devices = devs.keys() + devices = list(devs.keys()) else: srv_full_name = "%s/%s" % (srvr, inst) devices = db.get_device_name(srv_full_name, clss) @@ -254,9 +254,9 @@ def get_dict_from_db(db, data, narrow=False, skip_protected=True): dbdict.servers[srvr][inst][clss][device] = db_props # Classes - for class_name, cls in data.get("classes", {}).items(): - props = cls.get("properties", {}).keys() - for prop, value in db.get_class_property(class_name, props).items(): + for class_name, cls in list(data.get("classes", {}).items()): + props = list(cls.get("properties", {}).keys()) + for prop, value in list(db.get_class_property(class_name, props).items()): if value: value = [str(v) for v in value] dbdict.classes[class_name].properties[prop] = value @@ -264,10 +264,10 @@ def get_dict_from_db(db, data, narrow=False, skip_protected=True): attr_props = cls.get("attribute_properties") if attr_props: dbprops = db.get_class_attribute_property(class_name, - attr_props.keys()) - for attr, props in dbprops.items(): + list(attr_props.keys())) + for attr, props in list(dbprops.items()): props = dict((prop, [str(v) for v in values]) - for prop, values in props.items()) + for prop, values in list(props.items())) dbdict.classes[class_name].attribute_properties[attr] = props return dbdict.to_dict(), moved_devices @@ -276,8 +276,8 @@ def get_dict_from_db(db, data, narrow=False, skip_protected=True): def find_empty_servers(db, data): "Find any servers in the data that contain no devices, and remove them" servers = ["%s/%s" % (srv, inst) - for srv, insts in data["servers"].items() - for inst in insts.keys()] + for srv, insts in list(data["servers"].items()) + for inst in list(insts.keys())] return [server for server in servers if all(d.lower().startswith('dserver/') for d in db.get_device_class_list(server))] @@ -290,7 +290,7 @@ def get_device_property_values(dbproxy, device, name="*", _, result = dbproxy.command_inout("DbMySqlSelect", query % (device, name.replace("*", "%"))) data = defaultdict(list) - for prop, row in izip(result[::2], result[1::2]): + for prop, row in zip(result[::2], result[1::2]): if prop != "__SubDevices" or include_subdevices: data[prop].append(row) return data @@ -302,7 +302,7 @@ def get_device_attribute_property_values(dbproxy, device, name="*"): _, result = dbproxy.command_inout("DbMySqlSelect", query % (device, name.replace("*", "%"))) data = AppendingDict() - for attr, prop, row in izip(result[::3], result[1::3], result[2::3]): + for attr, prop, row in zip(result[::3], result[1::3], result[2::3]): data[attr][prop] = row return data @@ -323,7 +323,7 @@ def get_devices_by_name_and_class(dbproxy, name, clss="*"): def nwise(it, n): "[s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...]" - return izip(*[islice(it, i, None, n) for i in xrange(n)]) + return zip(*[islice(it, i, None, n) for i in range(n)]) def maybe_upper(s, upper=False): diff --git a/dsconfig/utils.py b/dsconfig/utils.py index 109fd7f..8e60cbb 100644 --- a/dsconfig/utils.py +++ b/dsconfig/utils.py @@ -3,7 +3,7 @@ import PyTango -from appending_dict import AppendingDict +from .appending_dict import AppendingDict #exit codes SUCCESS = 0 # NO DB CHANGES @@ -54,13 +54,13 @@ def progressbar(i, n, width): def find_device(definitions, devname, caseless=False): "Find a given device in a server dict" - for srvname, srv in definitions["servers"].items(): + for srvname, srv in list(definitions["servers"].items()): if caseless: srv = CaselessDict(srv) - for instname, inst in srv.items(): + for instname, inst in list(srv.items()): if caseless: inst = CaselessDict(inst) - for classname, cls in inst.items(): + for classname, cls in list(inst.items()): if caseless: cls = CaselessDict(cls) if devname in cls: @@ -70,7 +70,7 @@ def find_device(definitions, devname, caseless=False): def find_class(definitions, clsname): "Find a given device in a server dict" - for instname, inst in definitions["servers"].items(): + for instname, inst in list(definitions["servers"].items()): if clsname in inst: return inst[clsname] raise ValueError("class '%s' not defined" % clsname) @@ -78,9 +78,9 @@ def find_class(definitions, clsname): def get_devices_from_dict(dbdict): return [(server_name, inst_name, class_name, device_name) - for server_name, server in dbdict.items() - for inst_name, inst in server.items() - for class_name, clss in inst.items() + for server_name, server in list(dbdict.items()) + for inst_name, inst in list(server.items()) + for class_name, clss in list(inst.items()) for device_name in clss] @@ -163,7 +163,7 @@ def changekey(self, item): def lowerkeys(self): """Returns a lowercase list of all member keywords.""" - return self._keydict.keys() + return list(self._keydict.keys()) def __setitem__(self, item, value): # setting a keyword """To implement lowercase keys.""" @@ -205,12 +205,12 @@ def popitem(self): def has_key(self, item): """A case insensitive test for keys.""" if not isinstance(item, str): return False # should never have a non-string key - return self._keydict.has_key(item.lower()) # does the key exist + return item.lower() in self._keydict # does the key exist def __contains__(self, item): """A case insensitive __contains__.""" if not isinstance(item, str): return False # should never have a non-string key - return self._keydict.has_key(item.lower()) # does the key exist + return item.lower() in self._keydict # does the key exist def setdefault(self, item, default=None): """A case insensitive setdefault. diff --git a/dsconfig/validate.py b/dsconfig/validate.py index 28acd63..65eb5b5 100644 --- a/dsconfig/validate.py +++ b/dsconfig/validate.py @@ -3,13 +3,13 @@ from jsonschema import Draft4Validator, validate, exceptions -from utils import decode_dict +from .utils import decode_dict if __name__ == "__main__": data_filename, schema_filename = sys.argv[1], sys.argv[2] - print "Validating '%s' against schema '%s'..." % (data_filename, schema_filename), + print("Validating '%s' against schema '%s'..." % (data_filename, schema_filename), end=' ') with open(data_filename) as data_json: data = json.load(data_json, object_hook=decode_dict) @@ -20,8 +20,8 @@ try: validate(data, schema) except exceptions.ValidationError as e: - print "data does not match schema:" - print e + print("data does not match schema:") + print(e) sys.exit(1) else: - print "success!" + print("success!") diff --git a/dsconfig/viewer.py b/dsconfig/viewer.py index 7add5c4..88c6c9c 100644 --- a/dsconfig/viewer.py +++ b/dsconfig/viewer.py @@ -68,10 +68,10 @@ def _get_children(self, path): node = get_path(path, self.data) if node: if isinstance(node, dict): - children = node.keys() + children = list(node.keys()) return [path + (child,) for child in sorted(children)] elif isinstance(node, list): - children = range(len(node)) + children = list(range(len(node))) return [path + (child,) for child in children] return [] diff --git a/test/providers.py b/test/providers.py index 8e2784e..5daef3c 100644 --- a/test/providers.py +++ b/test/providers.py @@ -32,10 +32,10 @@ class TangoProvider(BaseProvider): def tango_property(self): n = randint(1, 5) - name = str("".join(_fake.word().capitalize() for i in xrange(n))) + name = str("".join(_fake.word().capitalize() for i in range(n))) n = int(ceil(expovariate(1))) # usually 1, sometimes larger - value = [str(_fake.user_name()) for i in xrange(n)] + value = [str(_fake.user_name()) for i in range(n)] return name, value @@ -46,10 +46,10 @@ def tango_attribute_property(self): def tango_attribute_config(self): n = randint(1, 3) - attr_name = str("".join(_fake.word().capitalize() for i in xrange(n))) + attr_name = str("".join(_fake.word().capitalize() for i in range(n))) n_props = int(ceil(expovariate(1))) # usually 1, sometimes larger attr_props = dict(_fake.tango_attribute_property() - for i in xrange(n_props)) + for i in range(n_props)) return attr_name, attr_props def tango_device(self): @@ -58,11 +58,11 @@ def tango_device(self): n_devprops = int(ceil(expovariate(1))) # usually 1, sometimes larger devprops = dict(_fake.tango_property() - for i in xrange(n_devprops)) + for i in range(n_devprops)) n_attrcfg = int(ceil(expovariate(1))) - 1 attrprops = dict(_fake.tango_attribute_config() - for i in xrange(n_attrcfg)) + for i in range(n_attrcfg)) value = {"properties": devprops} if attrprops: @@ -71,23 +71,23 @@ def tango_device(self): def tango_device_class(self): n = randint(1, 3) - name = str("".join(_fake.word().capitalize() for i in xrange(n))) + name = str("".join(_fake.word().capitalize() for i in range(n))) n_devs = int(ceil(expovariate(1))) - devices = dict(_fake.tango_device() for i in xrange(n_devs)) + devices = dict(_fake.tango_device() for i in range(n_devs)) return name, devices def tango_class(self): n = randint(1, 3) - name = str("".join(_fake.word().capitalize() for i in xrange(n))) + name = str("".join(_fake.word().capitalize() for i in range(n))) n_devprops = int(ceil(expovariate(1))) # usually 1, sometimes larger devprops = dict(_fake.tango_property() - for i in xrange(n_devprops)) + for i in range(n_devprops)) n_attrcfg = int(ceil(expovariate(1))) - 1 attrprops = dict(_fake.tango_attribute_config() - for i in xrange(n_attrcfg)) + for i in range(n_attrcfg)) value = {"properties": devprops} if attrprops: @@ -99,27 +99,27 @@ def tango_instance(self): n = randint(1, 3) chars = ascii_uppercase + digits name = "-".join("".join(choice(chars) - for i in xrange(randint(1, 5))) - for j in xrange(n)) + for i in range(randint(1, 5))) + for j in range(n)) n_classes = int(ceil(expovariate(1))) - value = dict(_fake.tango_device_class() for i in xrange(n_classes)) + value = dict(_fake.tango_device_class() for i in range(n_classes)) return name, value def tango_server(self): n = randint(1, 3) - name = "".join(_fake.word().capitalize() for i in xrange(n)) + name = "".join(_fake.word().capitalize() for i in range(n)) n_instances = randint(1, 10) - value = dict(_fake.tango_instance() for i in xrange(n_instances)) + value = dict(_fake.tango_instance() for i in range(n_instances)) return name, value def tango_database(self, servers=(5, 20), classes=(1, 5)): n_servers = randint(*servers) - servers = dict(_fake.tango_server() for i in xrange(n_servers)) + servers = dict(_fake.tango_server() for i in range(n_servers)) n_classes = randint(*classes) - classes = dict(_fake.tango_class() for i in xrange(n_classes)) + classes = dict(_fake.tango_class() for i in range(n_classes)) value = {"_title": "MAX-IV Tango JSON intermediate format", "_source": str(_fake.file_name(extension="xls")), "_date": str(_fake.date_time()), diff --git a/test/test_configure2.py b/test/test_configure2.py index 5bd4a2c..4f69297 100644 --- a/test/test_configure2.py +++ b/test/test_configure2.py @@ -13,7 +13,7 @@ import PyTango from dsconfig.configure import configure -from providers import TangoProvider +from .providers import TangoProvider fake = Faker() @@ -23,39 +23,39 @@ # # # # HELPERS # # # # def pick_random_server(config): - server = choice(config["servers"].keys()) + server = choice(list(config["servers"].keys())) return server, config["servers"][server] def pick_random_instance(config): servername, instances = pick_random_server(config) - instance = choice(instances.keys()) + instance = choice(list(instances.keys())) return servername, instance, instances[instance] def pick_random_class(config): servername, instname, classes = pick_random_instance(config) - classname = choice(classes.keys()) + classname = choice(list(classes.keys())) return servername, instname, classname, classes[classname] def pick_random_device(config): srv, inst, clss, devices = pick_random_class(config) - devname = choice(devices.keys()) + devname = choice(list(devices.keys())) return srv, inst, clss, devname, devices[devname] def pick_random_property(config): srv, inst, clss, dev, properties = pick_random_device(config) - propname = choice(properties["properties"].keys()) + propname = choice(list(properties["properties"].keys())) return srv, inst, clss, dev, propname, properties["properties"][propname] def pick_random_class_property(config): classes = config["classes"] - classname = choice(classes.keys()) + classname = choice(list(classes.keys())) properties = classes[classname] - propname = choice(properties["properties"].keys()) + propname = choice(list(properties["properties"].keys())) return classname, propname, properties["properties"][propname] @@ -191,7 +191,7 @@ def test_add_class_property(): orig_config = deepcopy(config) name, value = fake.tango_property() - clss = choice(config["classes"].keys()) + clss = choice(list(config["classes"].keys())) props = config["classes"][clss] props["properties"][name] = value calls = configure(config, orig_config) @@ -274,7 +274,7 @@ def test_add_class_attribute_property(): config = fake.tango_database(classes=(3, 5)) orig_config = deepcopy(config) - clss = choice(config["classes"].keys()) + clss = choice(list(config["classes"].keys())) props = config["classes"][clss] attr = "test_attribute" propname, value = fake.tango_attribute_property() @@ -296,7 +296,7 @@ def test_modify_class_attribute_property(): config = fake.tango_database() if "classes" in config: - clss = choice(config["classes"].keys()) + clss = choice(list(config["classes"].keys())) props = config["classes"][clss] attr = "test_attribute" propname, value = fake.tango_attribute_property() @@ -319,7 +319,7 @@ def test_cant_remove_protected_class_attribute_property(): config = fake.tango_database() - clss = choice(config["classes"].keys()) + clss = choice(list(config["classes"].keys())) props = config["classes"][clss] attr = "test_attribute" propname, value = fake.tango_attribute_property() @@ -401,7 +401,7 @@ def test_remove_device(): orig_config = deepcopy(config) _, _, classname, devices = pick_random_class(config) - device = choice(devices.keys()) + device = choice(list(devices.keys())) del devices[device] calls = configure(config, orig_config) diff --git a/test/test_filtering.py b/test/test_filtering.py index 76fb07e..a2eeb38 100644 --- a/test/test_filtering.py +++ b/test/test_filtering.py @@ -168,6 +168,6 @@ def test_filter_json_include_several(self): filtered = filter_config(data, ["device:.*test/3", "device:^a/"], SERVERS_LEVELS) import json - print(json.dumps(filtered, indent=4)) - print(json.dumps(expected, indent=4)) + print((json.dumps(filtered, indent=4))) + print((json.dumps(expected, indent=4))) self.assertEqual(filtered, expected) diff --git a/test/test_tangodb.py b/test/test_tangodb.py index 7662b7b..fb591c2 100644 --- a/test/test_tangodb.py +++ b/test/test_tangodb.py @@ -16,12 +16,12 @@ def get_device_info(dev): def get_device_name(server, clss): srv, inst = server.split("/") - return dbdata["servers"][srv][inst][clss].keys() + return list(dbdata["servers"][srv][inst][clss].keys()) db.get_device_name.side_effect = get_device_name def get_device_property_list(dev, pattern): data, _ = find_device(dbdata, dev) - return data["properties"].keys() + return list(data["properties"].keys()) db.get_device_property_list.side_effect = get_device_property_list def get_device_property(dev, props): @@ -53,4 +53,4 @@ def test_get_dict_from_db(): }}}}}}} db = make_db(dbdata) - print get_dict_from_db(db, indata) + print(get_dict_from_db(db, indata)) From 1a3111822a22bf65bcd6bbe4d28d2181cbfacfc1 Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Wed, 3 Jun 2020 15:12:44 +0200 Subject: [PATCH 13/22] Reformat magnets2json.py --- dsconfig/magnets2json.py | 607 ++++++++++++++++++++------------------- 1 file changed, 304 insertions(+), 303 deletions(-) diff --git a/dsconfig/magnets2json.py b/dsconfig/magnets2json.py index 5f00862..ccb72f1 100644 --- a/dsconfig/magnets2json.py +++ b/dsconfig/magnets2json.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # "$Name: $"; # "$Header: $"; -#============================================================================= +# ============================================================================= # # file : lattice2json.py # @@ -37,57 +37,56 @@ class SuperDict(defaultdict): "A recursive defaultdict with extra bells & whistles" - + def __init__(self): defaultdict.__init__(self, SuperDict) - + def __setattr__(self, attr, value): self[attr] = value - + def __getattr__(self, attr): return self[attr] class LatticeFileItem: - ''' ''' + """ """ itemName = "" itemType = '' parameters = {} properties = {} - alpars= {} - + alpars = {} + def __init__(self, _line=''): - ''' + """ Construct an object parsing a _line from a lattice file - ''' - self.psparameters= {} - self.parameters= {} - self.properties= {} + """ + self.psparameters = {} + self.parameters = {} + self.properties = {} - # find a name colon_pos = _line.find(':') self.itemName = _line[:colon_pos].lstrip().rstrip().upper() # what left to be parsed - line_left = _line[colon_pos + 1:].lstrip() - + line_left = _line[colon_pos + 1:].lstrip() + # find a type param_name = '' # the first item after a colon could be also a parameter name, like for a line element - eq_pos = line_left.find('=') + eq_pos = line_left.find('=') comma_pos = line_left.find(',') # let it work even there are no parameters defined - only element type if eq_pos < 0: eq_pos = len(line_left) - if comma_pos < 0: comma_pos = len(line_left) + if comma_pos < 0: comma_pos = len(line_left) # so, we could read an element type self.itemType = line_left[:min(comma_pos, eq_pos)].rstrip().lower() - + # this is waiting to be processed: line_left = line_left[comma_pos + 1:].lstrip() - + # if the element type is also parameter name state this - if eq_pos < comma_pos: param_name = self.itemType - + if eq_pos < comma_pos: param_name = self.itemType + # parse the rest for parameters while line_left != '': if param_name != '': @@ -100,7 +99,7 @@ def __init__(self, _line=''): param_value = line_left[:bracket_pos + 1] # this is what left to be parsed line_left = line_left[bracket_pos + 1:].lstrip() - + elif line_left[0] == '\"': # value in quotes (could contain commas) quote_pos = line_left.index('\"', 1) # will rise an exception in case of badly formated line @@ -120,75 +119,73 @@ def __init__(self, _line=''): # store the parameter with the corresponding value self.parameters[param_name] = param_value # PJB reset name back to empty here to find next parameter!(to enter else below) - param_name='' + param_name = '' else: # searching for a parameter - eq_pos = line_left.find('=') + eq_pos = line_left.find('=') if eq_pos < 0: eq_pos = len(line_left) # allow value-less parameters - comma_pos = line_left.find(',') + comma_pos = line_left.find(',') if comma_pos < 0: comma_pos = len(line_left) # so we know where to find parameter name param_name = line_left[:min(eq_pos, comma_pos)].rstrip().lower() # if the parameter has no value add it directly to the dictionary if comma_pos <= eq_pos: self.parameters[param_name] = '' - param_name = '' - # this is what left to be parsed - line_left = line_left[min(eq_pos, comma_pos) + 1:].lstrip() - + param_name = '' + # this is what left to be parsed + line_left = line_left[min(eq_pos, comma_pos) + 1:].lstrip() - def handle_circuit_name(self,itemName,endnum): + def handle_circuit_name(self, itemName, endnum): - endname="" - #hack for bc1 + endname = "" + # hack for bc1 if "QD" in itemName and "BC1" in itemName: - endname = "CRQM-" + endnum + endname = "CRQM-" + endnum # - #hack for bc2 + # hack for bc2 elif "QF" in itemName and "BC2" in itemName and "3" not in itemName and "4" not in itemName and "5" not in itemName: - endname = "CRQM-" + endnum + endname = "CRQM-" + endnum elif "QF" in itemName and "BC2" in itemName and ("3" in itemName or "5" in itemName): endname = "CRQ1-01" elif "QF" in itemName and "BC2" in itemName and "4" in itemName: endname = "CRQ2-01" # elif "Q" in itemName: - endname = "CRQ-" + endnum + endname = "CRQ-" + endnum elif "CO" in itemName and "X" in itemName: - endname = "CRCOX-" + endnum + endname = "CRCOX-" + endnum elif "CO" in itemName and "Y" in itemName: - endname = "CRCOY-" + endnum + endname = "CRCOY-" + endnum elif "DI" in itemName: - endname = "CRDI-" + endnum + endname = "CRDI-" + endnum print("dealing with endname ", endname) elif "SX" in itemName: - endname = "CRSX-" + endnum + endname = "CRSX-" + endnum elif "SOL" in itemName: endname = "CRSOL-" + endnum elif "SM" in itemName: - endname = "CRSM-" + endnum + endname = "CRSM-" + endnum else: sys.exit("Cannot convert circuit name" + itemName) - - if "/" in endname: #in case did not end with number, endname will be some */*/ by mistake - endname=endname.split("-")[0]+"-01" - return endname + if "/" in endname: # in case did not end with number, endname will be some */*/ by mistake + endname = endname.split("-")[0] + "-01" + return endname - def config_alarms(self,pyalarm,alname,alsev,aldesc,pyattname,key): + def config_alarms(self, pyalarm, alname, alsev, aldesc, pyattname, key): - alrec = alname+":"+"HTML" + alrec = alname + ":" + "HTML" if "AlarmList" not in self.alpars[pyalarm]: self.alpars[pyalarm]["AlarmList"] = [] - self.alpars[pyalarm]['AlarmList'].append(alname+":"+pyattname+"/"+key) + self.alpars[pyalarm]['AlarmList'].append(alname + ":" + pyattname + "/" + key) if "AlarmSeverities" not in self.alpars[pyalarm]: self.alpars[pyalarm]["AlarmSeverities"] = [] self.alpars[pyalarm]['AlarmSeverities'].append(alsev) - + if "AlarmDescriptions" not in self.alpars[pyalarm]: self.alpars[pyalarm]["AlarmDescriptions"] = [] self.alpars[pyalarm]['AlarmDescriptions'].append(aldesc) @@ -199,58 +196,56 @@ def config_alarms(self,pyalarm,alname,alsev,aldesc,pyattname,key): if "StartupDelay" not in self.alpars[pyalarm]: self.alpars[pyalarm]["StartupDelay"] = ["0"] - + if "AutoReset" not in self.alpars[pyalarm]: self.alpars[pyalarm]["AutoReset"] = ["60"] - + if "MaxMessagesPerAlarm" not in self.alpars[pyalarm]: self.alpars[pyalarm]["MaxMessagesPerAlarm"] = ["1"] - + if "PollingPeriod" not in self.alpars[pyalarm]: self.alpars[pyalarm]["PollingPeriod"] = ["5"] - + if "LogFile" not in self.alpars[pyalarm]: self.alpars[pyalarm]["LogFile"] = ["/tmp/pjb/log"] - + if "HtmlFolder" not in self.alpars[pyalarm]: self.alpars[pyalarm]["HtmlFolder"] = ["/tmp/pjb"] - + if "AlarmThreshold" not in self.alpars[pyalarm]: self.alpars[pyalarm]["AlarmThreshold"] = ["1"] - def match_properties(self): - devclass = types_classes[self.itemType] if "CIR" in self.itemName: - print("dealing with magnet circuit") + print("dealing with magnet circuit") # for given item type, look up required attributes and properties of tango elif devclass in TANGO_PROPERTIES: - fixed_properties_l = list(TANGO_PROPERTIES[devclass][0].keys()) - #print "fixed tango properties are ", fixed_properties_l + fixed_properties_l = list(TANGO_PROPERTIES[devclass][0].keys()) + # print "fixed tango properties are ", fixed_properties_l # add the fixed properties self.parameters.update(TANGO_PROPERTIES[devclass][0]) lattice_properties_l = list(TANGO_PROPERTIES[devclass][1].keys()) - #print "possible lattice tango properties are ", lattice_properties_l + # print "possible lattice tango properties are ", lattice_properties_l for k in list(self.parameters.keys()): - #print "pjb zzz 1", self.parameters["Tilt"], self.parameters - #print "key", k - #if not a required property or attribue then pop it - if k.lower() not in lattice_properties_l and k not in fixed_properties_l: - #print "popping ", k + # print "pjb zzz 1", self.parameters["Tilt"], self.parameters + # print "key", k + # if not a required property or attribue then pop it + if k.lower() not in lattice_properties_l and k not in fixed_properties_l: + # print "popping ", k self.parameters.pop(k) - #otherwise rename key if an attribute + # otherwise rename key if an attribute if k.lower() in lattice_properties_l: - #print "KEY ", k.lower(), TANGO_PROPERTIES[devclass][1][k.lower()], self.parameters[k] + # print "KEY ", k.lower(), TANGO_PROPERTIES[devclass][1][k.lower()], self.parameters[k] self.parameters[TANGO_PROPERTIES[devclass][1][k.lower()]] = [self.parameters.pop(k)] - #print "pjb zzz", self.parameters["Tilt"], self.parameters + # print "pjb zzz", self.parameters["Tilt"], self.parameters else: @@ -268,137 +263,140 @@ def match_properties(self): self.parameters["Tilt"] = ["0"] print("pjbyyy", self.parameters) - def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[0-9]+)'): - ''' + def add_device(self, sdict, adict, psdict, + name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[0-9]+)'): + """ Updates json file - ''' + """ # prepare pattern for parsing name - pattern = re.compile(name_parsing_string) + pattern = re.compile(name_parsing_string) print("In add device for item: " + self.itemName + " as " + self.itemType, self.alpars) # only when we know class for certain element print("adict is ", adict) circuit_alarm_params = [] - - #for case with no final number like I.TR1.MAG.DIE (no .1 etc at end) - alt_name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)' - alt_pattern = re.compile(alt_name_parsing_string) - #self.alpars = {} + # for case with no final number like I.TR1.MAG.DIE (no .1 etc at end) + alt_name_parsing_string = '(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)' + alt_pattern = re.compile(alt_name_parsing_string) + + # self.alpars = {} devdictalarm = None if self.itemType in types_classes: - print("") - print("") + print("") + print("") # split name - name_items = pattern.search(self.itemName) - parsed=False - tryagain=False + name_items = pattern.search(self.itemName) + parsed = False + tryagain = False if name_items == None: - print("Warning: Item name in lattice file doesn't match the naming convention.",self.itemName) - tryagain=True + print("Warning: Item name in lattice file doesn't match the naming convention.", self.itemName) + tryagain = True else: - parsed=True + parsed = True if tryagain: - name_items = alt_pattern.search(self.itemName) + name_items = alt_pattern.search(self.itemName) if name_items == None: - print("Warning: Item name in lattice file STILL doesn't match the naming convention.",self.itemName) + print("Warning: Item name in lattice file STILL doesn't match the naming convention.", + self.itemName) else: - parsed=True + parsed = True if parsed: system = name_items.group('system') subsystem = name_items.group('subsystem') location = name_items.group('location') device = name_items.group('device') - - if tryagain==False: - num = name_items.group('num') + + if tryagain == False: + num = name_items.group('num') else: - num="" + num = "" if num == None: num = '' if num != "": - num2 = "%02d" % int(num) + num2 = "%02d" % int(num) else: num2 = "01" - self.match_properties() print("pjbxxx ", self.parameters) # create device for json output - name = (system+"-"+location + '/' + subsystem + '/' + device + "-" + num2).encode('ascii', 'ignore') + name = (system + "-" + location + '/' + subsystem + '/' + device + "-" + num2).encode('ascii', 'ignore') devclass = types_classes[self.itemType].encode('ascii', 'ignore') - server = devclass + '/' + system+"-"+location + server = devclass + '/' + system + "-" + location - #hack for circuits + # hack for circuits if "CIR" in self.itemName: - print("orig name",self.itemName, name) + print("orig name", self.itemName, name) - #quad circuits named like CRQ - #correctors like CRCOX and CRCOY - #dipoles like CRDI - #solenoid like CRSOL - #sextu like CRX - endname = name.rsplit("/",1)[1] + # quad circuits named like CRQ + # correctors like CRCOX and CRCOY + # dipoles like CRDI + # solenoid like CRSOL + # sextu like CRX + endname = name.rsplit("/", 1)[1] - #fix circuit names - endname = self.handle_circuit_name(self.itemName,num2) - name = name.rsplit("/",1)[0] + "/" + endname + # fix circuit names + endname = self.handle_circuit_name(self.itemName, num2) + name = name.rsplit("/", 1)[0] + "/" + endname - #e.g in G00 have COIX 1, 2, 3 and COHX 1 and 2, which would make COX 1, 2, 3 with COIX 1 and 2 being lost! - #increment number till unique + # e.g in G00 have COIX 1, 2, 3 and COHX 1 and 2, which would make COX 1, 2, 3 with COIX 1 and 2 being lost! + # increment number till unique while name in cirlist: print("danger! already named this circuit!", self.itemName, name) suffix = int(name.split("-")[2]) + 1 - newnum2 = "%02d" % suffix + newnum2 = "%02d" % suffix print(newnum2) - name = (name.rsplit("-",1)[0]) + "-" + newnum2 + name = (name.rsplit("-", 1)[0]) + "-" + newnum2 print(name) print("new name ", name) cirlist.append(name) print(cirlist) devclass = "MagnetCircuit" - - #compact name is to find tag in plc alarms + # compact name is to find tag in plc alarms name_l_cir = self.itemName.split(".") section_cir = name_l_cir[1] - mid=name_l_cir[3] + mid = name_l_cir[3] if "_" in mid: mid = mid.split("_")[0] - #hack for SM1A and SM1B in TR1, also DIE, DIF on circuit DI (DIC, DID on TR3) - compactnamecir = "_"+name_l_cir[1]+mid+"_" - compactnamecir = compactnamecir.replace("1A","1") - compactnamecir = compactnamecir.replace("1B","1") + # hack for SM1A and SM1B in TR1, also DIE, DIF on circuit DI (DIC, DID on TR3) + compactnamecir = "_" + name_l_cir[1] + mid + "_" + compactnamecir = compactnamecir.replace("1A", "1") + compactnamecir = compactnamecir.replace("1B", "1") if "DIE" in compactnamecir: - compactnamecir = compactnamecir.replace("DIE","DI") + compactnamecir = compactnamecir.replace("DIE", "DI") if "DIF" in compactnamecir: - compactnamecir = compactnamecir.replace("DIF","DI") + compactnamecir = compactnamecir.replace("DIF", "DI") if "DIC" in compactnamecir: - compactnamecir = compactnamecir.replace("DIC","DI") + compactnamecir = compactnamecir.replace("DIC", "DI") if "DID" in compactnamecir: - compactnamecir = compactnamecir.replace("DID","DI") - + compactnamecir = compactnamecir.replace("DID", "DI") + print("circuit compact name is ", compactnamecir, name_l_cir) - #fill alarm info for circuits + # fill alarm info for circuits # - pyalarm = system+"-"+location + '/MAG/ALARM' + pyalarm = system + "-" + location + '/MAG/ALARM' if adict is not None: devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] print("init devdictalarm circuit") - already_added = ["B_I_SP02DIPBD_DIA_TSW1_A","B_I_SP02DIPBD_DIA_TSW2_A"] #can use this to ignore to. e.g. SP02DIPDB doesnt end in a number but isn't a circuit! + already_added = ["B_I_SP02DIPBD_DIA_TSW1_A", + "B_I_SP02DIPBD_DIA_TSW2_A"] # can use this to ignore to. e.g. SP02DIPDB doesnt end in a number but isn't a circuit! for key in alarm_dict: - #compact name is like _TR1QF_ but some tags are like _TR1QF2_5_ - #if compactnamecir in key: - if (key not in already_added and compactnamecir in key) or (compactnamecir[:-1] in key and key.count("_")>5 and key not in already_added and "F"+num+"_" in key): + # compact name is like _TR1QF_ but some tags are like _TR1QF2_5_ + # if compactnamecir in key: + if (key not in already_added and compactnamecir in key) or ( + compactnamecir[:-1] in key and key.count( + "_") > 5 and key not in already_added and "F" + num + "_" in key): print("key is", num, key) @@ -406,57 +404,56 @@ def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA print("FOUND ALARM INFO FOR CIR", compactnamecir, key, alarm_dict[key], section_cir) pyattname = "I-" + section_cir + "/DIA/COOLING" - - #for the magnets json file + + # for the magnets json file circuit_alarm_params = [pyattname, key, alarm_dict[key]] - #for the alarms json file - alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+endname.replace("-","_") - alsev = alname+":"+"ALARM" - alrec = alname+":"+"HTML" - aldesc = alname+":One magnet in circuit "+ alarm_dict[key] + # for the alarms json file + alname = 'TemperatureInterlock' + key.split('_')[ + -2] + '_' + system + "_" + location + '__' + endname.replace("-", "_") + alsev = alname + ":" + "ALARM" + alrec = alname + ":" + "HTML" + aldesc = alname + ":One magnet in circuit " + alarm_dict[key] if pyalarm not in self.alpars: self.alpars[pyalarm] = {} - self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars + self.config_alarms(pyalarm, alname, alsev, aldesc, pyattname, key) # will fill self.alpars - - - print("+++++++++++++++++ Creating device server : " + server + " for " + devclass + " (name= " + name + ")") + print( + "+++++++++++++++++ Creating device server : " + server + " for " + devclass + " (name= " + name + ")") print("+++++++++ With properties : ", self.parameters) # Dont actually write attributes to DB, only properties # see if this class exists and append if so, or create - devdict = sdict.servers["%s/%s" % (devclass, system+"-"+location)][devclass][name] + devdict = sdict.servers["%s/%s" % (devclass, system + "-" + location)][devclass][name] - #for circuit json only + # for circuit json only if "CR" in name: psdevdict = psdict.Circuits[name] - if "MAG" in self.itemName and "CIR" not in self.itemName: - #compact name is to find tag in plc alarms + # compact name is to find tag in plc alarms name_l = self.itemName.split(".") section = name_l[1] del name_l[0] del name_l[1] - #del name_l[2] + # del name_l[2] print("pjb kkk", name_l) compactfullname = "".join(name_l) compactname = compactfullname.split("_")[0] - compactname_nonum = compactfullname.split("_")[0][:-1]+"_" - - print("-------------------- magnet not circuit", self.itemName, compactname) + compactname_nonum = compactfullname.split("_")[0][:-1] + "_" + + print("-------------------- magnet not circuit", self.itemName, compactname) - #see what is the ps of the magnet + # see what is the ps of the magnet if name in POWER_SUPPLY_MAP: powersupplyname = POWER_SUPPLY_MAP[name] else: print("magnet not in PS map, skipping", name) - return + return - #!!!!!!!!!!! *********** create circuit device for each new ps ************!!!!!!!!!!!! + # !!!!!!!!!!! *********** create circuit device for each new ps ************!!!!!!!!!!!! # copy the magnet and call recursively add device! magnetcircuit = copy.deepcopy(self) magnetcircuit.itemName = self.itemName + ".CIR" @@ -468,98 +465,104 @@ def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA magnetcircuit.parameters['ResistanceReference'] = ["0.0"] magnetcircuit.parameters['CoilNames'] = [""] - #for the ps json file only + # for the ps json file only magnetcircuit.psparameters['PowerSupplyProxy'] = [powersupplyname] magnetcircuit.psparameters['MagnetProxies'] = [name] - #get alarm info from excel - pyalarm = system+"-"+location + '/MAG/ALARM' + # get alarm info from excel + pyalarm = system + "-" + location + '/MAG/ALARM' print("adict is again", adict) if adict is not None: devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] print("init devdictalarm") - #set alarms in magnet.json file and in alarms json file + # set alarms in magnet.json file and in alarms json file for key in alarm_dict: - if compactname in key and key.count("_")<6: - #if compactname in key: - + if compactname in key and key.count("_") < 6: + # if compactname in key: + pyattname = "I-" + section + "/DIA/COOLING" print("FOUND ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict) - #for the magnets json file + # for the magnets json file if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] + self.parameters['TemperatureInterlock'] = [ + pyattname + "," + key + "," + alarm_dict[key]] else: - self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) + self.parameters['TemperatureInterlock'].append( + pyattname + "," + key + "," + alarm_dict[key]) - #for the alarms json file - alname = 'TemperatureInterlock'+key.split('_')[-2]+'_'+system+"_"+location+'__'+'MAG'+'__'+ device + "_" +num2 - alsev = alname+":"+"ALARM" - aldesc = alname+":Magnet "+alarm_dict[key] + # for the alarms json file + alname = 'TemperatureInterlock' + key.split('_')[ + -2] + '_' + system + "_" + location + '__' + 'MAG' + '__' + device + "_" + num2 + alsev = alname + ":" + "ALARM" + aldesc = alname + ":Magnet " + alarm_dict[key] if pyalarm not in self.alpars: self.alpars[pyalarm] = {} - self.config_alarms(pyalarm,alname,alsev,aldesc,pyattname,key) #will fill self.alpars + self.config_alarms(pyalarm, alname, alsev, aldesc, pyattname, key) # will fill self.alpars devdictalarm.properties = self.alpars[pyalarm] - #set alarms in magnet.json file for all magnets in circuit + # set alarms in magnet.json file for all magnets in circuit for key in alarm_dict: - if compactname_nonum in key or (compactname[:-1] in key and key.count("_")>5 and ("F"+num+"_" in key or "_"+num+"_" in key)): - #if compactname_nonum in key: - print("mag key ", key, compactname_nonum, compactname, "F"+num+"_", "_"+num+"_") + if compactname_nonum in key or (compactname[:-1] in key and key.count("_") > 5 and ( + "F" + num + "_" in key or "_" + num + "_" in key)): + # if compactname_nonum in key: + print("mag key ", key, compactname_nonum, compactname, "F" + num + "_", "_" + num + "_") pyattname = "I-" + section + "/DIA/COOLING" print("FOUND MORE ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict) - #for the magnets json file + # for the magnets json file if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [pyattname+","+key+","+alarm_dict[key]] + self.parameters['TemperatureInterlock'] = [ + pyattname + "," + key + "," + alarm_dict[key]] else: - self.parameters['TemperatureInterlock'].append(pyattname+","+key+","+alarm_dict[key]) + self.parameters['TemperatureInterlock'].append( + pyattname + "," + key + "," + alarm_dict[key]) polarity = 1 orientation = 1 - #get calibration info from the excel + # get calibration info from the excel if self.itemName.split("_")[0] in calib_dict: print("FOUND CALIB INFO", self.itemName) - #find max multipole expansions + # find max multipole expansions dim = max(list(calib_dict[self.itemName.split("_")[0]].keys()), key=int) print("--- max order is", dim) - #create arrays of this dimensions, other dimension is 11 + # create arrays of this dimensions, other dimension is 11 - fieldsmatrix = [[0 for x in range(19)] for x in range(dim)] - #print fieldsmatrix - currentsmatrix = [[0 for x in range(19)] for x in range(dim)] + fieldsmatrix = [[0 for x in range(19)] for x in range(dim)] + # print fieldsmatrix + currentsmatrix = [[0 for x in range(19)] for x in range(dim)] - #iterate over keys and add to the array + # iterate over keys and add to the array for key in calib_dict[self.itemName.split("_")[0]]: print('--- ', key, 'corresponds to', calib_dict[self.itemName.split("_")[0]][key]) currents = calib_dict[self.itemName.split("_")[0]][key][5:25] - fields = calib_dict[self.itemName.split("_")[0]][key][25:45] - print('--- ',currents, fields) + fields = calib_dict[self.itemName.split("_")[0]][key][25:45] + print('--- ', currents, fields) - fieldsmatrix[key-1]=fields - currentsmatrix[key-1]=currents - #key here is the multipole order. any one should have same polarity + fieldsmatrix[key - 1] = fields + currentsmatrix[key - 1] = currents + # key here is the multipole order. any one should have same polarity polarity = calib_dict[self.itemName.split("_")[0]][key][3] orientation = calib_dict[self.itemName.split("_")[0]][key][2] - #print "P, O", polarity, orientation + # print "P, O", polarity, orientation - print('--- ',fieldsmatrix) - print('--- ',currentsmatrix) + print('--- ', fieldsmatrix) + print('--- ', currentsmatrix) - #now trim the matrices (lists) + # now trim the matrices (lists) maxlength = 20 - for i,val in enumerate(fieldsmatrix[dim-1]): - if val=='': - print(i , val) + for i, val in enumerate(fieldsmatrix[dim - 1]): + if val == '': + print(i, val) maxlength = i break print(maxlength) @@ -568,62 +571,64 @@ def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA del fieldsmatrix[i][maxlength:] del currentsmatrix[i][maxlength:] - print('Now--- ',fieldsmatrix) - print('Now--- ',currentsmatrix) + print('Now--- ', fieldsmatrix) + print('Now--- ', currentsmatrix) - magnetcircuit.parameters['ExcitationCurveCurrents']= currentsmatrix - magnetcircuit.parameters['ExcitationCurveFields']= fieldsmatrix + magnetcircuit.parameters['ExcitationCurveCurrents'] = currentsmatrix + magnetcircuit.parameters['ExcitationCurveFields'] = fieldsmatrix self.parameters['Orientation'] = [str(int(orientation))] - self.parameters['Polarity'] = [str(int(polarity))] + self.parameters['Polarity'] = [str(int(polarity))] - #assign circuit name as property of magnet device - #no regex to fix name here so do by hand - #e.g. I.BC1.MAG.COEX.4.CIR -> I-BC1/MAG/COEX-CIR-04 + # assign circuit name as property of magnet device + # no regex to fix name here so do by hand + # e.g. I.BC1.MAG.COEX.4.CIR -> I-BC1/MAG/COEX-CIR-04 - cname = name.rsplit("/CIR",1)[0] - endname = cname.rsplit("/",1)[1] - endnum = cname.rsplit("-",1)[1] - endname = self.handle_circuit_name(self.itemName,endnum) - cname = cname.rsplit("/",1)[0] + "/" + endname + cname = name.rsplit("/CIR", 1)[0] + endname = cname.rsplit("/", 1)[1] + endnum = cname.rsplit("-", 1)[1] + endname = self.handle_circuit_name(self.itemName, endnum) + cname = cname.rsplit("/", 1)[0] + "/" + endname print("cname is ", cname, name, powersupplyname, circuit_ps_list) while cname in cirlist: print("danger2! already named this circuit!", cname) suffix = int(cname.split("-")[2]) + 1 - newnum2 = "%02d" % suffix - cname = (cname.rsplit("-",1)[0]) + "-" + newnum2 + newnum2 = "%02d" % suffix + cname = (cname.rsplit("-", 1)[0]) + "-" + newnum2 print("new name ", cname) - #only add one circuit device per ps + # only add one circuit device per ps if powersupplyname not in circuit_ps_list: - - magnetcircuit.add_device(sdict,adict,psdict) - circuit_ps_list[powersupplyname] = cname + + magnetcircuit.add_device(sdict, adict, psdict) + circuit_ps_list[powersupplyname] = cname print("adding circuit name ", magnetcircuit.itemName, cname, circuit_ps_list) self.parameters['CircuitProxies'] = [cname] - + else: - #if we aleady made this circuit device, add it to this magnet properties + # if we aleady made this circuit device, add it to this magnet properties print("!!!ALART!!! already added a circuit device for ", self.itemName, name, system, location) if system == "R3": - system= "I" + system = "I" if location == "301L": location = "TR3" self.parameters['CircuitProxies'] = [circuit_ps_list[powersupplyname]] - #need to get the name of the circuit device from the ps dict though + # need to get the name of the circuit device from the ps dict though print("exiting circuit device is", circuit_ps_list[powersupplyname]) - - #print "current mags ", system+"-"+location - #print "current mags 2", sdict.servers - current_mags = sdict.servers["%s/%s" % ("MagnetCircuit", system+"-"+location)]["MagnetCircuit"][circuit_ps_list[powersupplyname]].properties - #for circuits json + # print "current mags ", system+"-"+location + # print "current mags 2", sdict.servers + + current_mags = \ + sdict.servers["%s/%s" % ("MagnetCircuit", system + "-" + location)]["MagnetCircuit"][ + circuit_ps_list[powersupplyname]].properties + # for circuits json print("cir name from ps ", circuit_ps_list[powersupplyname], psdict) ps_current_mags = psdict.Circuits[circuit_ps_list[powersupplyname]].Properties print("current mags ", current_mags['MagnetProxies']) @@ -633,32 +638,38 @@ def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA ps_current_mags['MagnetProxies'].append(name) current_mags['MagnetProxies'].append(name) - print("magnets on cir ", current_mags['ExcitationCurveFields'], current_mags['MagnetProxies'], len(current_mags['MagnetProxies'])) - #need to average the currents, even if already done so in excel (depends on field order) + print("magnets on cir ", current_mags['ExcitationCurveFields'], current_mags['MagnetProxies'], + len(current_mags['MagnetProxies'])) + # need to average the currents, even if already done so in excel (depends on field order) if 'ExcitationCurveFields' in current_mags: - assoc_field_m = current_mags['ExcitationCurveFields'] - this_field_m = fieldsmatrix + assoc_field_m = current_mags['ExcitationCurveFields'] + this_field_m = fieldsmatrix + + assoc_curr_m = current_mags['ExcitationCurveCurrents'] + this_curr_m = currentsmatrix - assoc_curr_m = current_mags['ExcitationCurveCurrents'] - this_curr_m = currentsmatrix - print("field matrix assoc is ", assoc_field_m) print("field matrix current is ", this_field_m) print("current matrix assoc is ", assoc_curr_m) print("current matrix current is ", this_curr_m) - + for i in range(dim): print(i) - #fix for CRSM take abs field values since opp sign - if circuit_ps_list[powersupplyname] in ["I-TR3/MAG/CRSM-01","I-TR3/MAG/CRDI-01"]: - newFields = [ ( abs(x)*(len(current_mags['MagnetProxies']) -1) + abs(y) ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] + # fix for CRSM take abs field values since opp sign + if circuit_ps_list[powersupplyname] in ["I-TR3/MAG/CRSM-01", "I-TR3/MAG/CRDI-01"]: + newFields = [(abs(x) * (len(current_mags['MagnetProxies']) - 1) + abs(y)) / len( + current_mags['MagnetProxies']) for y, x in + zip(this_field_m[i], assoc_field_m[i])] else: - newFields = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_field_m[i],assoc_field_m[i])] + newFields = [(x * (len(current_mags['MagnetProxies']) - 1) + y) / len( + current_mags['MagnetProxies']) for y, x in + zip(this_field_m[i], assoc_field_m[i])] - newCurrents = [ ( x*(len(current_mags['MagnetProxies']) -1) + y ) / len(current_mags['MagnetProxies']) for y,x in zip(this_curr_m[i],assoc_curr_m[i])] + newCurrents = [(x * (len(current_mags['MagnetProxies']) - 1) + y) / len( + current_mags['MagnetProxies']) for y, x in zip(this_curr_m[i], assoc_curr_m[i])] print("new fields ", newFields) print("new currents ", newCurrents) @@ -667,36 +678,34 @@ def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA current_mags['ExcitationCurveCurrents'][i] = newCurrents print("updated: ", current_mags['ExcitationCurveCurrents']) - + else: print("NOT A MAGNET") devdict.properties = self.parameters - #for circuits json + # for circuits json if "CR" in name: psdevdict.Properties = self.psparameters - + class ElegantLatticeParser: - ''' Class for parsing an elegant lattice file. ''' + """ Class for parsing an elegant lattice file. """ fileName = "" file = None - items = [] + items = [] def __init__(self, _fileName): - '''Constructs a parser object. + """Constructs a parser object. Keyword arguments: _fileName -- the name of file to be parsed - ''' + """ self.fileName = _fileName self.file = io.open(_fileName) - - def parseLatticeFile(self): - ''' ''' + """ """ line = "" # this will be a line combined from lines to be connected for ll in self.file: l = ll.lstrip().rstrip() @@ -707,26 +716,23 @@ def parseLatticeFile(self): pass # processing RPNs are not implemented elif l[-1] == '&': # Combine lines to be concated - line = line + ' ' + l[:-1] + line = line + ' ' + l[:-1] else: # So this is the last line to be combined - line = line + l + line = line + l # Create an object and add it to list - self.items.append(LatticeFileItem(line.lstrip().rstrip())) + self.items.append(LatticeFileItem(line.lstrip().rstrip())) line = "" - if __name__ == '__main__': - - inFileName = '' - doCalib=False - doAlarms=False + doCalib = False + doAlarms = False excelName = 'MagnetCalibrationData.xls' alarmName = 'IMAG_ALARM_140923_Magnets.xls' - + # define classes for lattice elements types_classes = {} types_classes["dip"] = "Magnet" @@ -737,18 +743,18 @@ def parseLatticeFile(self): types_classes["sole"] = "Magnet" types_classes["kquad"] = "Magnet" types_classes["ksext"] = "Magnet" - types_classes["hkick"] = "Magnet" + types_classes["hkick"] = "Magnet" types_classes["vkick"] = "Magnet" - #types_classes["hkick"] = "VACorrector" - #types_classes["vkick"] = "VACorrector" - #types_classes["monitor"] = "VABPM" - #types_classes["watch"] = "VAYAGScreen" - #types_classes["rfcw"] = "VARfCavity" + # types_classes["hkick"] = "VACorrector" + # types_classes["vkick"] = "VACorrector" + # types_classes["monitor"] = "VABPM" + # types_classes["watch"] = "VAYAGScreen" + # types_classes["rfcw"] = "VARfCavity" # circuit_ps_list = {} - #circuit_alarm_params = None + # circuit_alarm_params = None - #read arguments + # read arguments for par in sys.argv[1:]: if par[0:2] == '--': if par == '--calibration-data': @@ -759,9 +765,8 @@ def parseLatticeFile(self): # test_mode = True elif inFileName == '': inFileName = par - - print(inFileName, doCalib) + print(inFileName, doCalib) if doAlarms or doCalib: import xlrd @@ -771,77 +776,74 @@ def parseLatticeFile(self): print("opening alarms xls") xls = xlrd.open_workbook(alarmName) sheet = xls.sheet_by_name('Sheet1') - rows = [sheet.row_values(i) for i in range(sheet.nrows)] + rows = [sheet.row_values(i) for i in range(sheet.nrows)] column_names = rows[0] print("cols ", column_names) - for row in enumerate(rows[1:]): - if row[1][0]=="": - continue - print(row[1][0],row[1][1]) + for row in enumerate(rows[1:]): + if row[1][0] == "": + continue + print(row[1][0], row[1][1]) alarm_dict[row[1][0]] = row[1][1] print("DICT IS ", alarm_dict) - #make dict just for alarms! for pyalarm + # make dict just for alarms! for pyalarm json_dict_alarms = SuperDict() else: json_dict_alarms = None - + calib_dict = {} - + if doCalib: - #open excel sheet + # open excel sheet xls = xlrd.open_workbook(excelName) for name in ["linac", "transfer 1,5 GeV", "transfer 3 GeV", "thermionic gun"]: - #sheet = xls.sheet_by_name('Linac') + # sheet = xls.sheet_by_name('Linac') sheet = xls.sheet_by_name(name) - rows = [sheet.row_values(i) for i in range(sheet.nrows)] + rows = [sheet.row_values(i) for i in range(sheet.nrows)] column_names = rows[7] print("cols ", column_names) - for row in enumerate(rows[9:]): + for row in enumerate(rows[9:]): print(row[1]) - if row[1][2]=="": + if row[1][2] == "": continue - #this is like - #[5.0, u'I.S01A', u'I.S01A.MAG.QE.1', 202005.0, u'#1168-10030-0001', 1.0, -1.0, 2.0, 6.3167, 5.6757, 5.0307500000000003, 4.4208999999999996, 3.8452999999999999, 3.1463999999999999, 2.5179624999999999, 1.8892374999999999, 1.2808725000000001, 0.63988750000000016, 0.0, 0.70470485548532313, 0.63908274382966312, 0.56946571499960408, 0.50203927491440703, 0.43686121069898298, 0.35966476443894108, 0.288993167760146, 0.21848942173091002, 0.14957521795596601, 0.077488874695939805, 0.0052044472873010797, u'T', u'Rotating coil-C1168, #0001.xls', u'https://alfresco.maxlab.lu.se/share/page/site/maxiv/document-details?nodeRef=workspace://SpacesStore/23cdc9d1-a01e-443e-b578-1538637a1472', u'Scanditronix Magnet', 40690.0, ''] + # this is like + # [5.0, u'I.S01A', u'I.S01A.MAG.QE.1', 202005.0, u'#1168-10030-0001', 1.0, -1.0, 2.0, 6.3167, 5.6757, 5.0307500000000003, 4.4208999999999996, 3.8452999999999999, 3.1463999999999999, 2.5179624999999999, 1.8892374999999999, 1.2808725000000001, 0.63988750000000016, 0.0, 0.70470485548532313, 0.63908274382966312, 0.56946571499960408, 0.50203927491440703, 0.43686121069898298, 0.35966476443894108, 0.288993167760146, 0.21848942173091002, 0.14957521795596601, 0.077488874695939805, 0.0052044472873010797, u'T', u'Rotating coil-C1168, #0001.xls', u'https://alfresco.maxlab.lu.se/share/page/site/maxiv/document-details?nodeRef=workspace://SpacesStore/23cdc9d1-a01e-443e-b578-1538637a1472', u'Scanditronix Magnet', 40690.0, ''] if row[1][2].strip() not in calib_dict: if row[1][7] is not "": data_key = int(row[1][7]) data_list = row[1][3:48] - data_dict = {data_key : data_list} - calib_dict[row[1][2].strip()]=data_dict - #calib_dict[row[1][2]]=row[1][3:33] + data_dict = {data_key: data_list} + calib_dict[row[1][2].strip()] = data_dict + # calib_dict[row[1][2]]=row[1][3:33] else: if row[1][7] is not "": - #we found more curves for the same magnet + # we found more curves for the same magnet print("found another entry", row[1][2], row[1][7]) data_key = int(row[1][7]) data_list = row[1][3:48] - data_dict = {data_key : data_list} - calib_dict[row[1][2].strip()][data_key]=data_list + data_dict = {data_key: data_list} + calib_dict[row[1][2].strip()][data_key] = data_list print("DICT IS ", calib_dict) - - # create a parser for the file - parser = ElegantLatticeParser(inFileName) - + parser = ElegantLatticeParser(inFileName) + # parse the file parser.parseLatticeFile() parser.file.close() - - + # make a json file json_dict = SuperDict() - json_ps = SuperDict() + json_ps = SuperDict() for item in parser.items: - item.add_device(json_dict,json_dict_alarms, json_ps) - #print json.dumps(json_dict, indent=4) + item.add_device(json_dict, json_dict_alarms, json_ps) + # print json.dumps(json_dict, indent=4) - #now we have the dict, loop over again and sort out magnets, power supplies and circuits + # now we have the dict, loop over again and sort out magnets, power supplies and circuits print("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ") print("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ") @@ -849,25 +851,25 @@ def parseLatticeFile(self): outfile = open('magnets.json', 'w') - #have final json dict here + # have final json dict here print("THE FINAL DICT") topl = list(json_dict['servers'].keys()) for item in topl: if "Circuit" in item: - #print json_dict['servers'][item] + # print json_dict['servers'][item] for cir in json_dict['servers'][item]: - #print json_dict['servers'][item][cir]['ExcitationCurveCurrents'] - #print json_dict['servers'][item]["MagnetCircuit"] + # print json_dict['servers'][item][cir]['ExcitationCurveCurrents'] + # print json_dict['servers'][item]["MagnetCircuit"] for c in json_dict['servers'][item]["MagnetCircuit"]: for key in json_dict['servers'][item]["MagnetCircuit"][c]["properties"]: if key == "ExcitationCurveCurrents": ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] print(key, [str(x) for x in ls]) - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] + json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] if key == "ExcitationCurveFields": ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] print(key, [str(x) for x in ls]) - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] + json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] json.dump(json_dict, outfile, indent=4) @@ -875,9 +877,8 @@ def parseLatticeFile(self): outfile2 = open('magnets_alarms.json', 'w') json.dump(json_dict_alarms, outfile2, indent=4) - #!! note that item has parameters, but only want to extract those needed for tango! + # !! note that item has parameters, but only want to extract those needed for tango! - #dump ps circuit info + # dump ps circuit info outfile3 = open('circuits.json', 'w') json.dump(json_ps, outfile3, indent=4) - From 07119a305976e2cf3b5d5d97b81c24265954806e Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Wed, 3 Jun 2020 15:16:00 +0200 Subject: [PATCH 14/22] Test basic usage of json2tango and dump. --- dsconfig/formatting.py | 7 ++++++- dsconfig/json2tango.py | 1 - dsconfig/tangodb.py | 12 +++--------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dsconfig/formatting.py b/dsconfig/formatting.py index eb3017d..53c3043 100644 --- a/dsconfig/formatting.py +++ b/dsconfig/formatting.py @@ -63,7 +63,7 @@ def validate_json(data): def load_json(f): - return json.load(f, object_hook=decode_dict) + return json.load(f) def expand_config(config): @@ -111,13 +111,18 @@ def normalize_config(config): information can be gotten from the DB.) """ + print("config") old_config = expand_config(config) + print("old_config") new_config = SetterDict() if "servers" in old_config: + print("servers") new_config.servers = old_config["servers"] if "classes" in old_config: + print("classes") new_config.classes = old_config["classes"] if "devices" in old_config: + print("devices") db = PyTango.Database() for device, props in list(old_config["devices"].items()): try: diff --git a/dsconfig/json2tango.py b/dsconfig/json2tango.py index 1b11a10..fa50004 100755 --- a/dsconfig/json2tango.py +++ b/dsconfig/json2tango.py @@ -120,7 +120,6 @@ def main(): if not any(k in data for k in ("devices", "servers", "classes")): sys.exit(ERROR) - if options.input: print(json.dumps(data, indent=4)) return diff --git a/dsconfig/tangodb.py b/dsconfig/tangodb.py index 7ef7b84..a6995f1 100755 --- a/dsconfig/tangodb.py +++ b/dsconfig/tangodb.py @@ -372,9 +372,7 @@ def get_servers_with_filters(dbproxy, server="*", clss="*", device="*", _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device)) for d, p, v in nwise(result, 3): - # the properties are encoded in latin-1; we want utf-8 - decoded_value = v.decode('iso-8859-1').encode('utf8') - devices[maybe_upper(d, uppercase_devices)].properties[p] = decoded_value + devices[maybe_upper(d, uppercase_devices)].properties[p] = v if attribute_properties: # Get all relevant attribute properties @@ -390,9 +388,7 @@ def get_servers_with_filters(dbproxy, server="*", clss="*", device="*", _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device)) for d, a, p, v in nwise(result, 4): dev = devices[maybe_upper(d, uppercase_devices)] - # the properties are encoded in latin-1; we want utf-8 - decoded_value = v.decode('iso-8859-1').encode('utf8') - dev.attribute_properties[a][p] = decoded_value + dev.attribute_properties[a][p] = v devices = devices.to_dict() @@ -448,9 +444,7 @@ def get_classes_properties(dbproxy, server='*', cls_properties=True, _, result = dbproxy.command_inout("DbMySqlSelect", querry % (server)) # Build the output based on: class, property: value for c, p, v in nwise(result, 3): - # the properties are encoded in latin-1; we want utf-8 - decoded_value = v.decode('iso-8859-1').encode('utf8') - classes[c].properties[p] = decoded_value + classes[c].properties[p] = v # Get class attribute properties if cls_attribute_properties: querry = ( From b944879c1d6044041f02ae3c90a83337eb97f410 Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Wed, 3 Jun 2020 16:22:52 +0200 Subject: [PATCH 15/22] Tidy up code a little. --- dsconfig/appending_dict/__init__.py | 13 +- dsconfig/appending_dict/caseless.py | 10 +- dsconfig/appending_dict/test_appendingdict.py | 5 +- dsconfig/callcsv.py | 66 ++++-- dsconfig/configure.py | 30 +-- dsconfig/diff.py | 21 +- dsconfig/dump.py | 13 +- dsconfig/excel.py | 59 +++-- dsconfig/excel_alarms_userdef.py | 24 +- dsconfig/excel_alarms_vac.py | 82 +++---- dsconfig/filtering.py | 6 +- dsconfig/formatting.py | 39 ++-- dsconfig/json2tango.py | 21 +- dsconfig/magnets2json.py | 27 ++- dsconfig/output.py | 72 +++--- dsconfig/remove.py | 21 +- dsconfig/schema/dsconfig.json | 207 ++++++++--------- dsconfig/schema/schema2.json | 209 +++++++++--------- dsconfig/tangodb.py | 48 ++-- dsconfig/utils.py | 184 +++++++++------ dsconfig/validate.py | 3 +- dsconfig/viewer.py | 11 +- 22 files changed, 642 insertions(+), 529 deletions(-) diff --git a/dsconfig/appending_dict/__init__.py b/dsconfig/appending_dict/__init__.py index ec36c9e..efbaf39 100644 --- a/dsconfig/appending_dict/__init__.py +++ b/dsconfig/appending_dict/__init__.py @@ -1,4 +1,5 @@ from collections import defaultdict, Mapping + from .caseless import CaselessDictionary @@ -56,7 +57,9 @@ def __setattr__(self, key, value): return self.__setitem__(key, value) def to_dict(self): - """Returns a ordinary dict version of itself""" + """ + Returns a ordinary dict version of itself + """ result = {} for key, value in list(self.items()): if isinstance(value, SetterDict): @@ -67,7 +70,9 @@ def to_dict(self): def merge(d, u): - "Recursively 'merge' a Mapping into another" + """ + Recursively 'merge' a Mapping into another + """ for k, v in u.items(): if isinstance(v, Mapping): if k in d: @@ -86,8 +91,8 @@ def list_of_strings(value): class AppendingDict(SetterDict): - - """An extra weird SetterDict where assignment adds items instead of + """ + An extra weird SetterDict where assignment adds items instead of overwriting. It also allows setting nested values using dicts (or any Mapping). Scalar values are converted into lists of strings. diff --git a/dsconfig/appending_dict/caseless.py b/dsconfig/appending_dict/caseless.py index eaa82ac..bd3b97b 100644 --- a/dsconfig/appending_dict/caseless.py +++ b/dsconfig/appending_dict/caseless.py @@ -1,10 +1,11 @@ -"""A caseless dictionary implementation.""" +""" +A caseless dictionary implementation. +""" from collections import MutableMapping class CaselessDictionary(MutableMapping): - """ A dictionary-like object which ignores but preserves the case of strings. @@ -76,8 +77,9 @@ def items(self): class CaselessString(object): - - """A mixin to make a string subclass case-insensitive in dict lookups.""" + """ + A mixin to make a string subclass case-insensitive in dict lookups. + """ def __hash__(self): return hash(self.lower()) diff --git a/dsconfig/appending_dict/test_appendingdict.py b/dsconfig/appending_dict/test_appendingdict.py index 6ef1a34..f7dbda0 100644 --- a/dsconfig/appending_dict/test_appendingdict.py +++ b/dsconfig/appending_dict/test_appendingdict.py @@ -142,6 +142,7 @@ def test_error_setting_existing_subtree_with_scalar(self): def set_subtree(): ad.a = 2 + self.assertRaises(ValueError, set_subtree) def test_setting_with_appendingdict(self): @@ -153,7 +154,9 @@ def test_setting_with_appendingdict(self): ad, {"a": {"b": ["3"], "c": {"d": ["4"]}, "e": ["1"]}}) def test_updating_does_not_work(self): - "Have not yet implemented this" + """ + Have not yet implemented this + """ ad = AppendingDict() d = {"a": 1, "b": {"c": 3}} self.assertRaises(NotImplementedError, ad.update(d)) diff --git a/dsconfig/callcsv.py b/dsconfig/callcsv.py index 4a603bf..3e46430 100644 --- a/dsconfig/callcsv.py +++ b/dsconfig/callcsv.py @@ -1,11 +1,13 @@ -"""Provide functions to parse a callable csv file.""" +""" +Provide functions to parse a callable csv file. +""" # Imports -import os -import sys import csv import json +import os +import sys from collections import Mapping from importlib import import_module from optparse import OptionParser @@ -14,7 +16,9 @@ # Utils def special_update(d, u): - """Update nested dictionnaries while prioritizing the first argument.""" + """ + Update nested dictionnaries while prioritizing the first argument. + """ if not (isinstance(d, Mapping) and isinstance(u, Mapping)): return d if d is not None else u for k, v in u.items(): @@ -23,12 +27,16 @@ def special_update(d, u): def max_or_none(*args): - """Maximum function considering None as a maximum value.""" + """ + Maximum function considering None as a maximum value. + """ return None if None in args else max(*args) def cast_list(lst): - """Convert a list of a string to the corresponding value.""" + """ + Convert a list of a string to the corresponding value. + """ result = [] for value in lst: # Integer conversion @@ -58,20 +66,26 @@ def cast_list(lst): # Csv functions def get_column(matrix, col, start=None, stop=None, step=None): - """Get the column of a matrix, with optional range arguments.""" + """ + Get the column of a matrix, with optional range arguments. + """ return [row[col] for row in matrix][slice(start, stop, step)] def get_markup_index(matrix, col, markup): - """Find a markup in a given column and return the following index.""" + """ + Find a markup in a given column and return the following index. + """ for i, key in enumerate(get_column(matrix, col)): - if key == markup: - return i+1 + if key == markup: + return i + 1 return None def get_range_lst(matrix, col, start=0, stop=None, markup=None): - """Get a value->range dictionnary from a given column.""" + """ + Get a value->range dictionnary from a given column. + """ if markup: start = max_or_none(start, get_markup_index(matrix, col, markup)) if start is None: @@ -84,14 +98,16 @@ def get_range_lst(matrix, col, start=0, stop=None, markup=None): result.append((previous_key, previous_start, i)) previous_key, previous_start = key, i if previous_key is not None: - result.append((previous_key, previous_start, i+1)) + result.append((previous_key, previous_start, i + 1)) return result # Callable csv functions def get_kwargs(matrix, start, stop): - """Get the keywords arguments between two indexes.""" + """ + Get the keywords arguments between two indexes. + """ kwargs = {} keyword_lst = get_range_lst(matrix, 2, start, stop) for keyword, start, stop in keyword_lst: @@ -102,7 +118,9 @@ def get_kwargs(matrix, start, stop): def get_call_list(filename): - """Get the call list from a callable cls file.""" + """ + Get the call list from a callable cls file. + """ result = [] with open(filename) as csvfile: reader = csv.reader(csvfile, delimiter=',') @@ -121,7 +139,9 @@ def get_call_list(filename): # Data functions def process_call_list(lst, skip=False, verbose=True): - """Process a given call list and return the results.""" + """ + Process a given call list and return the results. + """ result = [] errors = ImportError, ValueError, TypeError, AttributeError for module_name, func_name, kwargs in lst: @@ -153,7 +173,9 @@ def process_call_list(lst, skip=False, verbose=True): def join_data(lst, source=None): - """Join a list of json strings or dictionnaries into a single dict.""" + """ + Join a list of json strings or dictionnaries into a single dict. + """ data = {} for mapping in lst: if isinstance(mapping, str): @@ -167,7 +189,9 @@ def join_data(lst, source=None): # CSV to Dict function def callable_csv_to_dict(filename, skip=False, verbose=True, to_json=False): - """Convert a callable csv file to a data dictionnary.""" + """ + Convert a callable csv file to a data dictionnary. + """ calls = get_call_list(filename) if not calls: return @@ -181,7 +205,9 @@ def callable_csv_to_dict(filename, skip=False, verbose=True, to_json=False): # Command lines arguments for configuration script def parse_command_line_args(desc): - """Parse arguments given in command line""" + """ + Parse arguments given in command line + """ usage = "%prog [-i INPUT] [-o OUTPUT] [-v] [-w]" parser = OptionParser(usage=usage, description=desc, version='%prog v1.0') @@ -214,7 +240,9 @@ def parse_command_line_args(desc): # Main function for configuration scripts def main(desc=None, module_name=None, function=None): - """Run the script.""" + """ + Run the script. + """ kwargs = {} remove = False desc = desc or "Generate a Tango json file for a given callable csv file." diff --git a/dsconfig/configure.py b/dsconfig/configure.py index 6344acf..75e362f 100644 --- a/dsconfig/configure.py +++ b/dsconfig/configure.py @@ -1,19 +1,21 @@ -"""Functionality for configuring a Tango DB from a dsconfig file""" +""" +Functionality for configuring a Tango DB from a dsconfig file +""" from collections import defaultdict from functools import partial -import PyTango -from .appending_dict.caseless import CaselessDictionary +import tango -from .utils import ObjectWrapper +from .appending_dict.caseless import CaselessDictionary from .tangodb import SPECIAL_ATTRIBUTE_PROPERTIES, is_protected +from .utils import ObjectWrapper def check_attribute_property(propname): # Is this too strict? Do we ever need non-standard attr props? if (not propname.startswith("_") - and propname not in SPECIAL_ATTRIBUTE_PROPERTIES): + and propname not in SPECIAL_ATTRIBUTE_PROPERTIES): raise KeyError("Bad attribute property name: %s" % propname) return True @@ -89,10 +91,11 @@ def update_properties(db, parent, db_props, new_props, def update_server(db, server_name, server_dict, db_dict, update=False, ignore_case=False, - difactory=PyTango.DbDevInfo, strict_attr_props=True): - - """Creates/removes devices for a given server. Optionally - ignores removed devices, only adding new and updating old ones.""" + difactory=tango.DbDevInfo, strict_attr_props=True): + """ + Creates/removes devices for a given server. Optionally + ignores removed devices, only adding new and updating old ones. + """ if ignore_case: db_dict = CaselessDictionary(db_dict) @@ -131,8 +134,9 @@ def update_server(db, server_name, server_dict, db_dict, def update_device_or_class(db, name, db_dict, new_dict, cls=False, update=False, ignore_case=False, strict_attr_props=True): - - "Configure a device or a class" + """ + Configure a device or a class + """ # Note: if the "properties" key is missing, we'll just ignore any # existing properties in the DB. Ditto for attribute_properties. @@ -167,8 +171,8 @@ def update_device_or_class(db, name, db_dict, new_dict, def configure(data, dbdata, update=False, ignore_case=False, strict_attr_props=True): - - """Takes an input data dict and the relevant current DB data. Returns + """ + Takes an input data dict and the relevant current DB data. Returns the DB calls needed to bring the Tango DB to the state described by 'data'. The 'update' flag means that servers/devices are not removed, only added or changed. If the 'ignore_case' flag is True, diff --git a/dsconfig/diff.py b/dsconfig/diff.py index ba8bbfc..dc30ea8 100644 --- a/dsconfig/diff.py +++ b/dsconfig/diff.py @@ -1,19 +1,23 @@ -from collections import Mapping import json import sys +from collections import Mapping from .utils import green, yellow, red def decode_pointer(ptr): - """Take a string representing a JSON pointer and return a - sequence of parts, decoded.""" + """ + Take a string representing a JSON pointer and return a + sequence of parts, decoded. + """ return [p.replace("~1", "/").replace("~0", "~") for p in ptr.split("/")] def dump_value(value): - "Make a string out of a value, for printing" + """ + Make a string out of a value, for printing + """ if value is not None: if isinstance(value, Mapping): dump = json.dumps(value, indent=4) @@ -24,8 +28,9 @@ def dump_value(value): def print_diff(dbdict, data, removes=True): - - "Print a (hopefully) human readable list of changes." + """ + Print a (hopefully) human readable list of changes. + """ # TODO: needs work, especially on multiline properties, # empty properties (should probably never be allowed but still) @@ -71,6 +76,4 @@ def print_diff(dbdict, data, removes=True): return diff except ImportError: print(("'jsonpatch' module not available - " - "no diff printouts for you! (Try -d instead.)"), file=sys.stderr) - - + "no diff printouts for you! (Try -d instead.)"), file=sys.stderr) diff --git a/dsconfig/dump.py b/dsconfig/dump.py index 9cbc491..c7458d9 100755 --- a/dsconfig/dump.py +++ b/dsconfig/dump.py @@ -8,19 +8,19 @@ """ -from .tangodb import get_servers_with_filters, get_classes_properties +import tango + from .appending_dict import SetterDict -import PyTango +from .tangodb import get_servers_with_filters, get_classes_properties def get_db_data(db, patterns=None, class_properties=False, **options): - # dump TANGO database into JSON. Optionally filter which things to include # (currently only "positive" filters are possible; you can say which # servers/classes/devices to include, but you can't exclude selectively) # By default, dserver devices aren't included! - dbproxy = PyTango.DeviceProxy(db.dev_name()) + dbproxy = tango.DeviceProxy(db.dev_name()) data = SetterDict() if not patterns: @@ -40,13 +40,12 @@ def get_db_data(db, patterns=None, class_properties=False, **options): servers = get_servers_with_filters(dbproxy, **kwargs) data.servers.update(servers) if class_properties: - classes = get_classes_properties(dbproxy, server=pattern) + classes = get_classes_properties(dbproxy, server=pattern) data.classes.update(classes) return data.to_dict() def main(): - import json from optparse import OptionParser @@ -74,7 +73,7 @@ def main(): options, args = parser.parse_args() - db = PyTango.Database() + db = tango.Database() dbdata = get_db_data(db, args, properties=options.properties, class_properties=options.class_properties, diff --git a/dsconfig/excel.py b/dsconfig/excel.py index 6c123fa..8bf6374 100644 --- a/dsconfig/excel.py +++ b/dsconfig/excel.py @@ -3,17 +3,18 @@ producing a file in the TangoDB JSON format. """ -from datetime import datetime import json import os import re import sys -#from traceback import format_exc +from datetime import datetime -from .utils import find_device from .appending_dict import AppendingDict -from .utils import CaselessDict from .tangodb import SPECIAL_ATTRIBUTE_PROPERTIES +from .utils import CaselessDict +from .utils import find_device + +# from traceback import format_exc MODE_MAPPING = CaselessDict({"ATTR": "DynamicAttributes", "CMD": "DynamicCommands", @@ -23,9 +24,11 @@ TYPE_MAPPING = CaselessDict({"INT": int, "FLOAT": float}) -def get_properties(row): - "Find property definitions on a row" +def get_properties(row): + """ + Find property definitions on a row + """ prop_dict = AppendingDict() @@ -70,7 +73,6 @@ def get_properties(row): def get_attribute_properties(row): - if "attribute" in row: attribute = row["attribute"] prop_dict = AppendingDict() @@ -105,7 +107,9 @@ def get_attribute_properties(row): def get_dynamic(row): - "Find dynamic definitions on a row" + """ + Find dynamic definitions on a row + """ prop_dict = AppendingDict() try: @@ -127,17 +131,22 @@ def get_dynamic(row): def make_db_name(name): - "convert a Space Separated Name into a lowercase, underscore_separated_name" + """ + Convert a Space Separated Name into a lowercase, underscore_separated_name + """ return name.strip().lower().replace(" ", "_") def check_formula(formula): - "Syntax check a dynamic formula." + """ + Syntax check a dynamic formula. + """ compile(formula, "", "single") def check_device_format(devname): - """Verify that a device name is of the correct form (three parts + """ + Verify that a device name is of the correct form (three parts separated by slashes, only letters, numbers, dashes and underscores allowed.) Note: We could put more logic here to make device names conform to a standard. @@ -148,14 +157,17 @@ def check_device_format(devname): def format_server_instance(row): - "Format a server/instance string" + """ + Format a server/instance string + """ # TODO: handle numeric instance names? They tend to turn up as floats... return "%s/%s" % (row["server"], row["instance"]) def convert(rows, definitions, skip=True, dynamic=False, config=False): - - "Update a dict of definitions from data" + """ + Update a dict of definitions from data + """ errors = [] column_names = rows[0] @@ -188,8 +200,8 @@ def handle_error(i, msg): # full device definition # target is "lazily" evaluated, so that we don't create # an empty dict if it turns out there are no members - target = lambda: definitions.servers[row["server"]][row["instance"]]\ - [row["class"]][row["device"]] + target = lambda: definitions.servers[row["server"]][row["instance"]] \ + [row["class"]][row["device"]] else: # don't know if the device is already defined target = lambda: find_device(definitions, row["device"])[0] @@ -214,7 +226,7 @@ def handle_error(i, msg): target().properties = props except KeyError as ke: - #handle_error(i, "insufficient data (%s)" % ke) + # handle_error(i, "insufficient data (%s)" % ke) pass except ValueError as ve: handle_error(i, "Error: %s" % ve) @@ -236,8 +248,9 @@ def print_errors(errors): def xls_to_dict(xls_filename, pages=None, skip=False): - - """Make JSON out of an XLS sheet of device definitions.""" + """ + Make JSON out of an XLS sheet of device definitions. + """ import xlrd @@ -271,7 +284,9 @@ def xls_to_dict(xls_filename, pages=None, skip=False): def get_stats(defs): - "Calculate some numbers" + """ + Calculate some numbers + """ servers = set() instances = set() @@ -327,8 +342,8 @@ def main(): stats = get_stats(data) print(("\n" - "Total: %(servers)d servers, %(instances)d instances, " - "%(classes)d classes and %(devices)d devices defined.") % stats, file=sys.stderr) + "Total: %(servers)d servers, %(instances)d instances, " + "%(classes)d classes and %(devices)d devices defined.") % stats, file=sys.stderr) if __name__ == "__main__": diff --git a/dsconfig/excel_alarms_userdef.py b/dsconfig/excel_alarms_userdef.py index e258646..729b707 100644 --- a/dsconfig/excel_alarms_userdef.py +++ b/dsconfig/excel_alarms_userdef.py @@ -1,11 +1,13 @@ -from collections import defaultdict import json +from collections import defaultdict import xlrd class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" + """ + A recursive defaultdict with extra bells & whistles + """ def __init__(self): defaultdict.__init__(self, SuperDict) @@ -19,19 +21,20 @@ def __getattr__(self, attr): def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev, al_rec): print(inst, dev, al_name, al_cond, al_desc, al_sev, al_rec) - devdict = sdict.servers["PyAlarm/"+inst]["PyAlarm"][dev] + devdict = sdict.servers["PyAlarm/" + inst]["PyAlarm"][dev] if "AlarmList" not in devdict.properties: devdict.properties["AlarmList"] = [] - devdict.properties["AlarmList"].append(al_name+":"+al_cond) + devdict.properties["AlarmList"].append(al_name + ":" + al_cond) if "AlarmDescriptions" not in devdict.properties: devdict.properties["AlarmDescriptions"] = [] - devdict.properties["AlarmDescriptions"].append(al_name+":"+al_desc) + devdict.properties["AlarmDescriptions"].append(al_name + ":" + al_desc) if "AlarmSeverities" not in devdict.properties: devdict.properties["AlarmSeverities"] = [] - devdict.properties["AlarmSeverities"].append(al_name+":"+al_sev) + devdict.properties["AlarmSeverities"].append(al_name + ":" + al_sev) if "AlarmReceivers" not in devdict.properties: devdict.properties["AlarmReceivers"] = [] - devdict.properties["AlarmReceivers"].append(al_name+":"+al_rec) + devdict.properties["AlarmReceivers"].append(al_name + ":" + al_rec) + def xls_to_dict(xls_filename): json_dict = SuperDict() @@ -41,11 +44,12 @@ def xls_to_dict(xls_filename): # above skips row 0 (col headers) # look at all rows but only read those with entry in first col if sheet.row_values(line)[0] is not "": - print("IN LINE ", line, sheet.row_values(line)[0]) + print("IN LINE ", line, sheet.row_values(line)[0]) dev_config = sheet.row_values(line) add_device(json_dict, *dev_config[:7]) return json_dict + def main(): import sys data = xls_to_dict(sys.argv[1]) @@ -53,8 +57,6 @@ def main(): outfile = open('pyalarm_config.json', 'w') json.dump(data, outfile, indent=4) + if __name__ == "__main__": main() - - - diff --git a/dsconfig/excel_alarms_vac.py b/dsconfig/excel_alarms_vac.py index 00f9c40..61b3290 100644 --- a/dsconfig/excel_alarms_vac.py +++ b/dsconfig/excel_alarms_vac.py @@ -1,11 +1,13 @@ -from collections import defaultdict import json +from collections import defaultdict import xlrd class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" + """ + A recursive defaultdict with extra bells & whistles + """ def __init__(self): defaultdict.__init__(self, SuperDict) @@ -20,27 +22,27 @@ def __getattr__(self, attr): def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev): print(inst, dev, al_name, al_cond, al_desc, al_sev) # devdict = sdict.servers["PyAlarm"]["PyAlarm/"+inst]["PyAlarm"][dev] - devdict = sdict.servers["PyAlarm/"+inst]["PyAlarm"][dev] + devdict = sdict.servers["PyAlarm/" + inst]["PyAlarm"][dev] if "AlarmList" not in devdict.properties: devdict.properties["AlarmList"] = [] - devdict.properties["AlarmList"].append(al_name+":"+al_cond) + devdict.properties["AlarmList"].append(al_name + ":" + al_cond) if "AlarmDescriptions" not in devdict.properties: devdict.properties["AlarmDescriptions"] = [] - devdict.properties["AlarmDescriptions"].append(al_name+":"+al_desc) - #hard code severity and some other things - only one per instance + devdict.properties["AlarmDescriptions"].append(al_name + ":" + al_desc) + # hard code severity and some other things - only one per instance if "AlarmSeverities" not in devdict.properties: devdict.properties["AlarmSeverities"] = [] - devdict.properties["AlarmSeverities"].append(al_name+":"+al_sev) + devdict.properties["AlarmSeverities"].append(al_name + ":" + al_sev) if "AlarmReceivers" not in devdict.properties: devdict.properties["AlarmReceivers"] = [] - devdict.properties["AlarmReceivers"].append(al_name+":HTML") - #hard code severity and some other things - only one per instance + devdict.properties["AlarmReceivers"].append(al_name + ":HTML") + # hard code severity and some other things - only one per instance if "AlarmThreshold" not in devdict.properties: devdict.properties["AlarmThreshold"] = [] - devdict.properties["AlarmThreshold"] = [1] + devdict.properties["AlarmThreshold"] = [1] if "LogFile" not in devdict.properties: devdict.properties["LogFile"] = [] - devdict.properties["LogFile"]= ["/tmp/pjb/log"] + devdict.properties["LogFile"] = ["/tmp/pjb/log"] if "HtmlFolder" not in devdict.properties: devdict.properties["HtmlFolder"] = [] devdict.properties["HtmlFolder"] = ["/tmp/pjb"] @@ -49,70 +51,72 @@ def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev): devdict.properties["PollingPeriod"] = [5] if "MaxMessagesPerAlarm" not in devdict.properties: devdict.properties["MaxMessagesPerAlarm"] = [] - devdict.properties["MaxMessagesPerAlarm"]= [1] + devdict.properties["MaxMessagesPerAlarm"] = [1] if "AutoReset" not in devdict.properties: devdict.properties["AutoReset"] = [] - devdict.properties["AutoReset"]= [0] + devdict.properties["AutoReset"] = [0] if "StartupDelay" not in devdict.properties: devdict.properties["StartupDelay"] = [] - devdict.properties["StartupDelay"]= [0] + devdict.properties["StartupDelay"] = [0] + def xls_to_dict(xls_filename): json_dict = SuperDict() xls = xlrd.open_workbook(xls_filename) - for i in range (0,2): + for i in range(0, 2): - if i==1: - sheet = xls.sheet_by_name("Alarms") - nature="interlock" + if i == 1: + sheet = xls.sheet_by_name("Alarms") + nature = "interlock" else: - sheet = xls.sheet_by_name("Warnings") - nature="bypass" - - last_server="" - last_device="" - last_name="" - last_sev="" - summary_condition="" + sheet = xls.sheet_by_name("Warnings") + nature = "bypass" + + last_server = "" + last_device = "" + last_name = "" + last_sev = "" + summary_condition = "" for line in range(1, sheet.nrows): # above skips row 0 (col headers) # look at all rows but only read those with entry in first col if sheet.row_values(line)[0] is not "": - print("IN LINE ", line, sheet.row_values(line)[0]) - #assume that if you get to a new device, it means a new section of vacuum - #in this case, need to make a final alarm which is or of all others + print("IN LINE ", line, sheet.row_values(line)[0]) + # assume that if you get to a new device, it means a new section of vacuum + # in this case, need to make a final alarm which is or of all others dev_config = sheet.row_values(line) - print(dev_config, dev_config[3].rsplit("/",1)[0]) - if dev_config[1] != last_device or dev_config[0]=="end": + print(dev_config, dev_config[3].rsplit("/", 1)[0]) + if dev_config[1] != last_device or dev_config[0] == "end": print("START NEW SECTION", dev_config[1]) print("---- ADDING TO JSON summary ", summary_condition, last_name) - if summary_condition!="": - add_device(json_dict,last_server,last_device,last_name.rsplit("_",1)[0],summary_condition,"at least one vac. %s in section %s" %(nature,last_name.rsplit("__",2)[0]),last_sev) + if summary_condition != "": + add_device(json_dict, last_server, last_device, last_name.rsplit("_", 1)[0], summary_condition, + "at least one vac. %s in section %s" % (nature, last_name.rsplit("__", 2)[0]), + last_sev) last_server = dev_config[0] last_device = dev_config[1] last_name = dev_config[2] last_sev = dev_config[5] - summary_condition="" + summary_condition = "" if summary_condition == "": summary_condition = summary_condition + dev_config[3] else: summary_condition = summary_condition + " or " + dev_config[3] - if dev_config[0]!="end": + if dev_config[0] != "end": add_device(json_dict, *dev_config[:6]) return json_dict + def main(): import sys data = xls_to_dict(sys.argv[1]) - #print json.dumps(data, indent=4) + # print json.dumps(data, indent=4) outfile = open('alarms_vac.json', 'w') json.dump(data, outfile, indent=4) + if __name__ == "__main__": main() - - - diff --git a/dsconfig/filtering.py b/dsconfig/filtering.py index c4645f9..2f4c149 100644 --- a/dsconfig/filtering.py +++ b/dsconfig/filtering.py @@ -15,7 +15,7 @@ def filter_nested_dict(node, pattern, depth, level=0, invert=False): else: dupe_node = {} for key, val in node.items(): - cur_node = filter_nested_dict(val, pattern, depth, level+1, + cur_node = filter_nested_dict(val, pattern, depth, level + 1, invert) if cur_node: dupe_node[key] = cur_node @@ -23,8 +23,8 @@ def filter_nested_dict(node, pattern, depth, level=0, invert=False): def filter_config(data, filters, levels, invert=False): - - """Filter the given config data according to a list of filters. + """ + Filter the given config data according to a list of filters. May be a positive filter (i.e. includes only matching things) or inverted (i.e. includes everything that does not match). The _levels_ argument is used to find at what depth in the data diff --git a/dsconfig/formatting.py b/dsconfig/formatting.py index 53c3043..515aa76 100644 --- a/dsconfig/formatting.py +++ b/dsconfig/formatting.py @@ -1,20 +1,19 @@ "This module concerns the dsconfig JSON file format" -import sys import json +import sys from copy import deepcopy, copy from os import path -from .appending_dict import AppendingDict, SetterDict - -import PyTango +import tango +from .appending_dict import SetterDict SERVERS_LEVELS = {"server": 0, "instance": 1, "class": 2, "device": 3, "property": 5} CLASSES_LEVELS = {"class": 1, "property": 2} module_path = path.dirname(path.realpath(__file__)) -SCHEMA_FILENAME = path.join(module_path, "schema/schema2.json") #rename +SCHEMA_FILENAME = path.join(module_path, "schema/schema2.json") # rename # functions to decode unicode JSON (PyTango does not like unicode strings) @@ -48,7 +47,9 @@ def decode_dict(data): def validate_json(data): - """Validate that a given dict is of the right form""" + """ + Validate that a given dict is of the right form + """ try: from jsonschema import validate, exceptions with open(SCHEMA_FILENAME) as schema_json: @@ -56,7 +57,7 @@ def validate_json(data): validate(data, schema) except ImportError: print(("WARNING: 'jsonschema' not installed, could not " - "validate json file. You're on your own."), file=sys.stderr) + "validate json file. You're on your own."), file=sys.stderr) except exceptions.ValidationError as e: print("ERROR: JSON data does not match schema: %s" % e, file=sys.stderr) sys.exit(1) @@ -67,10 +68,11 @@ def load_json(f): def expand_config(config): - - """Takes a configuration dict and expands it into the canonical + """ + Takes a configuration dict and expands it into the canonical format. This currently means that the server instance level is - split into a server and an instance level.""" + split into a server and an instance level. + """ expanded = deepcopy(config) if "servers" in config: @@ -86,7 +88,9 @@ def expand_config(config): def clean_metadata(data): - "Removes any keys in the data that begin with '_'" + """ + Removes any keys in the data that begin with '_' + """ tmp = copy(data) for key in list(tmp.keys()): if key.startswith("_"): @@ -95,8 +99,8 @@ def clean_metadata(data): def normalize_config(config): - - """Take a 'loose' config and return a new config that conforms to the + """ + Take a 'loose' config and return a new config that conforms to the DSConfig format. Current transforms: @@ -111,23 +115,18 @@ def normalize_config(config): information can be gotten from the DB.) """ - print("config") old_config = expand_config(config) - print("old_config") new_config = SetterDict() if "servers" in old_config: - print("servers") new_config.servers = old_config["servers"] if "classes" in old_config: - print("classes") new_config.classes = old_config["classes"] if "devices" in old_config: - print("devices") - db = PyTango.Database() + db = tango.Database() for device, props in list(old_config["devices"].items()): try: info = db.get_device_info(device) - except PyTango.DevFailed as e: + except tango.DevFailed as e: sys.exit("Can't reconfigure device %s: %s" % (device, str(e[0].desc))) srv, inst = info.ds_full_name.split("/") new_config.servers[srv][inst][info.class_name][device] = props diff --git a/dsconfig/json2tango.py b/dsconfig/json2tango.py index fa50004..224af88 100755 --- a/dsconfig/json2tango.py +++ b/dsconfig/json2tango.py @@ -5,28 +5,27 @@ optionally be run. """ +import json import sys import time -import json from optparse import OptionParser from tempfile import NamedTemporaryFile -import PyTango +import tango +from dsconfig.appending_dict.caseless import CaselessDictionary from dsconfig.configure import configure +from dsconfig.dump import get_db_data from dsconfig.filtering import filter_config from dsconfig.formatting import (CLASSES_LEVELS, SERVERS_LEVELS, load_json, normalize_config, validate_json, clean_metadata) +from dsconfig.output import show_actions from dsconfig.tangodb import summarise_calls, get_devices_from_dict -from dsconfig.dump import get_db_data -from dsconfig.utils import green, red, yellow, progressbar, no_colors from dsconfig.utils import SUCCESS, ERROR, CONFIG_APPLIED, CONFIG_NOT_APPLIED -from dsconfig.output import show_actions -from dsconfig.appending_dict.caseless import CaselessDictionary +from dsconfig.utils import green, red, yellow, progressbar, no_colors def main(): - usage = "Usage: %prog [options] JSONFILE" parser = OptionParser(usage=usage) @@ -125,7 +124,7 @@ def main(): return # check if there is anything in the DB that will be changed or removed - db = PyTango.Database() + db = tango.Database() if options.dbdata: with open(options.dbdata) as f: original = json.loads(f.read()) @@ -205,7 +204,7 @@ def main(): servers = len(collisions) devices = sum(len(devs) for devs in list(collisions.values())) print(red("Move %d devices from %d servers." % - (devices, servers)), file=sys.stderr) + (devices, servers)), file=sys.stderr) if empty and options.verbose: print(red("Removed %d empty servers." % len(empty)), file=sys.stderr) @@ -213,9 +212,9 @@ def main(): print(red("\n*** Data was written to the Tango DB ***"), file=sys.stderr) with NamedTemporaryFile(prefix="dsconfig-", suffix=".json", delete=False) as f: - f.write(json.dumps(original, indent=4)) + f.write(json.dumps(original, indent=4).encode()) print(("The previous DB data was saved to %s" % - f.name), file=sys.stderr) + f.name), file=sys.stderr) sys.exit(CONFIG_APPLIED) else: print(yellow( diff --git a/dsconfig/magnets2json.py b/dsconfig/magnets2json.py index ccb72f1..09fdde1 100644 --- a/dsconfig/magnets2json.py +++ b/dsconfig/magnets2json.py @@ -21,16 +21,15 @@ # Krakow,PL/Lund,SE# # -from collections import defaultdict -import json -import os +import copy import io -import sys +import json import re -from TangoProperties import TANGO_PROPERTIES +import sys +from collections import defaultdict + from PowerSupplyMap import POWER_SUPPLY_MAP -import copy -import numpy as np +from TangoProperties import TANGO_PROPERTIES cirlist = [] @@ -395,8 +394,8 @@ def add_device(self, sdict, adict, psdict, # compact name is like _TR1QF_ but some tags are like _TR1QF2_5_ # if compactnamecir in key: if (key not in already_added and compactnamecir in key) or ( - compactnamecir[:-1] in key and key.count( - "_") > 5 and key not in already_added and "F" + num + "_" in key): + compactnamecir[:-1] in key and key.count("_") > 5 and key not in already_added and + "F" + num + "_" in key): print("key is", num, key) @@ -489,7 +488,8 @@ def add_device(self, sdict, adict, psdict, # for the magnets json file if 'TemperatureInterlock' not in self.parameters: self.parameters['TemperatureInterlock'] = [ - pyattname + "," + key + "," + alarm_dict[key]] + pyattname + "," + key + "," + alarm_dict[key] + ] else: self.parameters['TemperatureInterlock'].append( pyattname + "," + key + "," + alarm_dict[key]) @@ -690,13 +690,16 @@ def add_device(self, sdict, adict, psdict, class ElegantLatticeParser: - """ Class for parsing an elegant lattice file. """ + """ + Class for parsing an elegant lattice file. + """ fileName = "" file = None items = [] def __init__(self, _fileName): - """Constructs a parser object. + """ + Constructs a parser object. Keyword arguments: _fileName -- the name of file to be parsed diff --git a/dsconfig/output.py b/dsconfig/output.py index ff627c7..80cf01f 100644 --- a/dsconfig/output.py +++ b/dsconfig/output.py @@ -1,9 +1,9 @@ from difflib import ndiff -from PyTango.utils import CaselessDict - -from dsconfig.utils import green, red, yellow +from tango.utils import CaselessDict from dsconfig.tangodb import get_devices_from_dict +from dsconfig.utils import green, red, yellow + from .appending_dict import SetterDict @@ -37,9 +37,10 @@ def format_property(value, indentation="", max_lines=10): def get_changes(data, calls): - - """Combine a list of database calls into "changes" that can - be more easily turned into a readable representation""" + """ + Combine a list of database calls into "changes" that can + be more easily turned into a readable representation + """ devices = get_devices_from_dict(data["servers"]) device_mapping = CaselessDict( @@ -233,8 +234,9 @@ def get_changes(data, calls): def show_actions(data, calls): - - "Print out a human readable representation of changes" + """ + Print out a human readable representation of changes + """ changes = get_changes(data, calls) @@ -257,8 +259,8 @@ def show_actions(data, calls): if info.get("server"): if info.get("old_server"): print(("{}Server: {} -> {}".format(indent, - red(info["old_server"]), - green(info["server"])))) + red(info["old_server"]), + green(info["server"])))) else: print(("{}Server: {}".format(indent, info["server"]))) @@ -266,8 +268,8 @@ def show_actions(data, calls): if info.get("old_class"): if info["old_class"] != info["device_class"]: print(("{}Class: {} -> {}".format(indent, - info["old_class"], - info["device_class"]))) + info["old_class"], + info["device_class"]))) else: print(("{}Class: {}".format(indent, info["device_class"]))) @@ -277,7 +279,7 @@ def show_actions(data, calls): if old_alias: if old_alias != alias: print(("{}Alias: {} -> {}".format(indent, red(old_alias), - green(alias)))) + green(alias)))) else: print(("{}Alias: {}".format(indent, alias))) @@ -290,20 +292,20 @@ def show_actions(data, calls): if old_value is not None: if old_value != value: # change property - lines.append(yellow("{}= {}".format(indent*2, prop))) + lines.append(yellow("{}= {}".format(indent * 2, prop))) lines.append(property_diff( - change["old_value"], change["value"], indent*3)) + change["old_value"], change["value"], indent * 3)) else: # new property - lines.append(green("{}+ {}".format(indent*2, prop))) + lines.append(green("{}+ {}".format(indent * 2, prop))) lines.append(green(format_property(change["value"], - indent*3))) + indent * 3))) else: # delete property - lines.append(red("{}- {}".format(indent*2, prop))) - lines.append(red(format_property(change["old_value"], indent*3))) + lines.append(red("{}- {}".format(indent * 2, prop))) + lines.append(red(format_property(change["old_value"], indent * 3))) if lines: - print(("{}Properties:".format(indent*1))) + print(("{}Properties:".format(indent * 1))) print(("\n".join(lines))) if info.get("attribute_properties"): @@ -318,26 +320,26 @@ def show_actions(data, calls): if old_value is not None: # change if value != old_value: - attr_lines.append(yellow("{}= {}".format(indent*3, prop))) + attr_lines.append(yellow("{}= {}".format(indent * 3, prop))) attr_lines.append(property_diff( - change["old_value"], change["value"], indent*4)) + change["old_value"], change["value"], indent * 4)) else: # addition - attr_lines.append(green("{}+ {}".format(indent*3, prop))) + attr_lines.append(green("{}+ {}".format(indent * 3, prop))) attr_lines.append(green(format_property(change["value"], - indent*4))) + indent * 4))) else: # removal - attr_lines.append(red("{}- {}".format(indent*3, prop))) + attr_lines.append(red("{}- {}".format(indent * 3, prop))) attr_lines.append(red(format_property(change["old_value"], - indent*4))) + indent * 4))) if attr_lines: - lines.append("{}{}".format(indent*2, attr)) + lines.append("{}{}".format(indent * 2, attr)) lines.extend(attr_lines) if lines: print(("{}Attribute properties:".format(indent))) print(("\n".join(lines))) - + for clss in sorted(changes["classes"]): info = changes["classes"][clss] @@ -350,19 +352,19 @@ def show_actions(data, calls): if old_value is not None: if old_value != value: # change property - lines.append(yellow("{}= {}".format(indent*2, prop))) + lines.append(yellow("{}= {}".format(indent * 2, prop))) lines.append(property_diff( - change["old_value"], change["value"], indent*3)) + change["old_value"], change["value"], indent * 3)) else: # new property - lines.append(green("{}+ {}".format(indent*2, prop))) + lines.append(green("{}+ {}".format(indent * 2, prop))) lines.append(green(format_property(change["value"], - indent*3))) + indent * 3))) else: # delete property - lines.append(red("{}- {}".format(indent*2, prop))) - lines.append(red(format_property(change["old_value"], indent*3))) + lines.append(red("{}- {}".format(indent * 2, prop))) + lines.append(red(format_property(change["old_value"], indent * 3))) if lines: - print(("{} Class Properties:".format(indent*1))) + print(("{} Class Properties:".format(indent * 1))) print(("\n".join(lines))) print() diff --git a/dsconfig/remove.py b/dsconfig/remove.py index 7862f5c..1ac0ede 100644 --- a/dsconfig/remove.py +++ b/dsconfig/remove.py @@ -1,10 +1,10 @@ import json import sys -import PyTango +import tango from .utils import (ObjectWrapper, decode_dict, get_dict_from_db, - RED, GREEN, YELLOW) + RED, GREEN, YELLOW) def delete_devices(db, dbdict, server, cls, devices): @@ -21,7 +21,7 @@ def delete_server(db, dbdict, servername, serverdata): try: db.delete_server_info(servername) # what does this do? db.delete_server(servername) - except PyTango.DevFailed: + except tango.DevFailed: # I'm not sure about this; sometimes deleting servers works # and sometimes not; should they be running? What's going on? # Anyway, the worst case scenario is that the servers are @@ -39,14 +39,14 @@ def delete_class(db, dbdict, classname): def main(json_file, write=False, db_calls=False): - - """Remove the devices and servers defined in the given file - from the DB.""" + """ + Remove the devices and servers defined in the given file from the DB. + """ with open(json_file) as f: data = json.load(f, object_hook=decode_dict) - db = PyTango.Database() + db = tango.Database() dbdict = get_dict_from_db(db, data) # check the current DB state # wrap the database (and fake it if we're not going to write) @@ -70,16 +70,15 @@ def main(json_file, write=False, db_calls=False): if db.calls: if write: print(( - RED + "\n*** Data was written to the Tango DB ***"), file=sys.stderr) + RED + "\n*** Data was written to the Tango DB ***"), file=sys.stderr) else: - print(YELLOW +\ - "\n*** Nothing was written to the Tango DB (use -w) ***", file=sys.stderr) + print(YELLOW + \ + "\n*** Nothing was written to the Tango DB (use -w) ***", file=sys.stderr) else: print(GREEN + "\n*** No changes needed in Tango DB ***", file=sys.stderr) if __name__ == "__main__": - from optparse import OptionParser usage = "Usage: %prog [options] JSONFILE" diff --git a/dsconfig/schema/dsconfig.json b/dsconfig/schema/dsconfig.json index 42f057a..59c3dec 100644 --- a/dsconfig/schema/dsconfig.json +++ b/dsconfig/schema/dsconfig.json @@ -1,112 +1,113 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Tango device generator JSON canonical format", - "type": "object", - "additionalProperties": false, - "properties": { - "_title": { - "type": "string" - }, - "_date": { - "type": "string" - }, - "_source": { - "type": "string" - }, - "_version": { - "enum": [2] - }, - "servers": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/server" - } - } - }, - "classes": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/device" - }, - - "properties": { - "properties": { - "$ref": "#definitions/property" - } - } + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Tango device generator JSON canonical format", + "type": "object", + "additionalProperties": false, + "properties": { + "_title": { + "type": "string" + }, + "_date": { + "type": "string" + }, + "_source": { + "type": "string" + }, + "_version": { + "enum": [ + 2 + ] + }, + "servers": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/server" } + } }, - "definitions": { - "server": { - "type": "object", - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/instance" - } - } - }, - "instance": { - "type": "object", - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/class" - } - } - }, - "class": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[\\-\\w]+/[\\-\\w]+/[\\-\\w]+$": { - "$ref": "#definitions/device" - } - } - }, - "device": { - "type": "object", - "additionalProperties": false, - "properties": { - "properties": { - "$ref": "#definitions/properties" - }, - "attribute_properties": { - "$ref": "#definitions/attribute_properties" - } - } - }, + "classes": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/device" + }, + "properties": { "properties": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/property" - } - }, - "property": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } + "$ref": "#definitions/property" + } + } + } + }, + "definitions": { + "server": { + "type": "object", + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/instance" + } + } + }, + "instance": { + "type": "object", + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/class" + } + } + }, + "class": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[\\-\\w]+/[\\-\\w]+/[\\-\\w]+$": { + "$ref": "#definitions/device" + } + } + }, + "device": { + "type": "object", + "additionalProperties": false, + "properties": { + "properties": { + "$ref": "#definitions/properties" }, "attribute_properties": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/attribute_property" - } - }, - "attribute_property": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/property" - } - }, - "class_property": { - "type": "object", - "additionalProperties": { - "type": "object", - "$ref": "#definitions/property" - } + "$ref": "#definitions/attribute_properties" } + } + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/property" + } + }, + "property": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "attribute_properties": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/attribute_property" + } + }, + "attribute_property": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/property" + } + }, + "class_property": { + "type": "object", + "additionalProperties": { + "type": "object", + "$ref": "#definitions/property" + } } + } } diff --git a/dsconfig/schema/schema2.json b/dsconfig/schema/schema2.json index 9dc9aaf..e29d3dd 100644 --- a/dsconfig/schema/schema2.json +++ b/dsconfig/schema/schema2.json @@ -1,114 +1,115 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Tango device generator JSON canonical format", - "type": "object", - "additionalProperties": false, - "properties": { - "_title": { - "type": "string" - }, - "_date": { - "type": "string" - }, - "_source": { - "type": "string" - }, - "_version": { - "enum": [2] - }, - "servers": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/server" - } - } - }, - "classes": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/device" - }, - - "properties": { - "properties": { - "$ref": "#definitions/property" - } - } + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Tango device generator JSON canonical format", + "type": "object", + "additionalProperties": false, + "properties": { + "_title": { + "type": "string" + }, + "_date": { + "type": "string" + }, + "_source": { + "type": "string" + }, + "_version": { + "enum": [ + 2 + ] + }, + "servers": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/server" } + } }, - "definitions": { - "server": { - "type": "object", - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/instance" - } - } - }, - "instance": { - "type": "object", - "patternProperties": { - "^[\\-\\w]+$": { - "$ref": "#definitions/class" - } - } - }, - "class": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[\\-\\w.@]+/[\\-\\w.@]+/[\\-\\w.@]+$": { - "$ref": "#definitions/device" - } - } - }, - "device": { - "type": "object", - "additionalProperties": false, - "properties": { - "properties": { - "$ref": "#definitions/properties" - }, - "attribute_properties": { - "$ref": "#definitions/attribute_properties" - }, - "alias": { - "type": "string" - } - } - }, + "classes": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/device" + }, + "properties": { "properties": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/property" - } - }, - "property": { - "type": "array", - "items": { - "type": "string" - } + "$ref": "#definitions/property" + } + } + } + }, + "definitions": { + "server": { + "type": "object", + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/instance" + } + } + }, + "instance": { + "type": "object", + "patternProperties": { + "^[\\-\\w]+$": { + "$ref": "#definitions/class" + } + } + }, + "class": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[\\-\\w.@]+/[\\-\\w.@]+/[\\-\\w.@]+$": { + "$ref": "#definitions/device" + } + } + }, + "device": { + "type": "object", + "additionalProperties": false, + "properties": { + "properties": { + "$ref": "#definitions/properties" }, "attribute_properties": { - "type": "object", - "additionalProperties": { - "$ref": "#definitions/attribute_property" - } + "$ref": "#definitions/attribute_properties" }, - "attribute_property": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/property" - } - }, - "class_property": { - "type": "object", - "additionalProperties": { - "type": "object", - "$ref": "#definitions/property" - } + "alias": { + "type": "string" } + } + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/property" + } + }, + "property": { + "type": "array", + "items": { + "type": "string" + } + }, + "attribute_properties": { + "type": "object", + "additionalProperties": { + "$ref": "#definitions/attribute_property" + } + }, + "attribute_property": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/property" + } + }, + "class_property": { + "type": "object", + "additionalProperties": { + "type": "object", + "$ref": "#definitions/property" + } } + } } diff --git a/dsconfig/tangodb.py b/dsconfig/tangodb.py index a6995f1..541c641 100755 --- a/dsconfig/tangodb.py +++ b/dsconfig/tangodb.py @@ -1,21 +1,18 @@ "Various functionality for dealing with the TANGO database" from collections import defaultdict -from difflib import unified_diff from itertools import islice -import PyTango - -from .appending_dict import AppendingDict, SetterDict, CaselessDictionary +import tango from dsconfig.utils import green, red, yellow +from .appending_dict import AppendingDict, SetterDict, CaselessDictionary # These are special properties that we'll ignore for now PROTECTED_PROPERTIES = [ "polled_attr", "logging_level", "logging_target" ] - SPECIAL_ATTRIBUTE_PROPERTIES = [ "label", "format", "unit", "standard_unit", "display_unit", "min_value", "min_alarm", "min_warning", @@ -49,9 +46,11 @@ def get_servers_from_dict(dbdict): def is_protected(prop, attr=False): - """There are certain properties that need special treatment as they + """ + There are certain properties that need special treatment as they are handled in particular ways by Tango. In general, we don't want to - remove these if they exist, but we do allow overwriting them.""" + remove these if they exist, but we do allow overwriting them. + """ if attr: # Attribute config properties return prop.startswith("_") or prop in SPECIAL_ATTRIBUTE_PROPERTIES @@ -60,7 +59,6 @@ def is_protected(prop, attr=False): def present_calls(indata, dbdata, dbcalls): - # WIP! def add_device(devname, devinfo): @@ -74,8 +72,9 @@ def delete_device(devname): def summarise_calls(dbcalls, dbdata): - - "A brief summary of the operations performed by a list of DB calls" + """ + A brief summary of the operations performed by a list of DB calls + """ methods = [ "add_device", @@ -155,16 +154,17 @@ def summarise_calls(dbcalls, dbdata): def get_device(db, devname, data, skip_protected=True): - - """Returns all relevant DB information about a given device: - alias (if any), properties, attribute properties""" + """ + Returns all relevant DB information about a given device: + alias (if any), properties, attribute properties + """ dev = {} try: alias = db.get_alias_from_device(devname) dev["alias"] = alias - except PyTango.DevFailed: + except tango.DevFailed: pass # Properties @@ -214,8 +214,8 @@ def get_device(db, devname, data, skip_protected=True): def get_dict_from_db(db, data, narrow=False, skip_protected=True): - - """Takes a data dict, checks if any if the definitions are already + """ + Takes a data dict, checks if any if the definitions are already in the DB and returns a dict describing them. By default it includes all devices for each server+class, use the @@ -235,7 +235,7 @@ def get_dict_from_db(db, data, narrow=False, skip_protected=True): if devinfo.ds_full_name.lower() != srvname.lower(): moved_devices[devinfo.ds_full_name].append((clss, device)) - except PyTango.DevFailed: + except tango.DevFailed: pass # Servers @@ -274,7 +274,9 @@ def get_dict_from_db(db, data, narrow=False, skip_protected=True): def find_empty_servers(db, data): - "Find any servers in the data that contain no devices, and remove them" + """ + Find any servers in the data that contain no devices, and remove them + """ servers = ["%s/%s" % (srv, inst) for srv, insts in list(data["servers"].items()) for inst in list(insts.keys())] @@ -300,7 +302,7 @@ def get_device_attribute_property_values(dbproxy, device, name="*"): query = ("SELECT attribute, name, value FROM property_attribute_device " "WHERE device = '%s' AND name LIKE '%s'") _, result = dbproxy.command_inout("DbMySqlSelect", - query % (device, name.replace("*", "%"))) + query % (device, name.replace("*", "%"))) data = AppendingDict() for attr, prop, row in zip(result[::3], result[1::3], result[2::3]): data[attr][prop] = row @@ -317,12 +319,12 @@ def get_devices_by_name_and_class(dbproxy, name, clss="*"): query = ("SELECT name FROM device WHERE name LIKE '%s' " "AND class LIKE '%s'") _, result = dbproxy.command_inout("DbMySqlSelect", - query % (name.replace("*", "%"), clss.replace("*", "%"))) + query % (name.replace("*", "%"), clss.replace("*", "%"))) return result def nwise(it, n): - "[s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...]" + # [s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...] return zip(*[islice(it, i, None, n) for i in range(n)]) @@ -356,7 +358,7 @@ def get_servers_with_filters(dbproxy, server="*", clss="*", device="*", # good to increase the timeout a bit. # TODO: maybe instead use automatic retry and increase timeout # each time? - dbproxy.set_timeout_millis(timeout*1000) + dbproxy.set_timeout_millis(timeout * 1000) if properties: # Get all relevant device properties @@ -426,7 +428,7 @@ def get_classes_properties(dbproxy, server='*', cls_properties=True, # Mysql wildcards server = server.replace("*", "%") # Change device proxy timeout - dbproxy.set_timeout_millis(timeout*1000) + dbproxy.set_timeout_millis(timeout * 1000) # Classes output dict classes = AppendingDict() # Get class properties diff --git a/dsconfig/utils.py b/dsconfig/utils.py index 8e60cbb..844d4df 100644 --- a/dsconfig/utils.py +++ b/dsconfig/utils.py @@ -1,22 +1,19 @@ -from functools import partial import sys +from functools import partial -import PyTango - -from .appending_dict import AppendingDict - -#exit codes -SUCCESS = 0 # NO DB CHANGES +# exit codes +SUCCESS = 0 # NO DB CHANGES ERROR = 1 CONFIG_APPLIED = 2 CONFIG_NOT_APPLIED = 3 -#colors +# colors ADD = GREEN = '\033[92m' REMOVE = RED = FAIL = '\033[91m' REPLACE = YELLOW = WARN = '\033[93m' ENDC = '\033[0m' + def no_colors(): global ADD, GREEN, REMOVE, RED, FILE, REPLACE, YELLOW, WARN, ENDC ADD = '' @@ -29,12 +26,15 @@ def no_colors(): WARN = '' ENDC = '' + def green(text): return GREEN + text + ENDC + def red(text): return RED + text + ENDC + def yellow(text): return YELLOW + text + ENDC @@ -53,7 +53,9 @@ def progressbar(i, n, width): def find_device(definitions, devname, caseless=False): - "Find a given device in a server dict" + """ + Find a given device in a server dict + """ for srvname, srv in list(definitions["servers"].items()): if caseless: srv = CaselessDict(srv) @@ -69,7 +71,9 @@ def find_device(definitions, devname, caseless=False): def find_class(definitions, clsname): - "Find a given device in a server dict" + """ + Find a given device in a server dict + """ for instname, inst in list(definitions["servers"].items()): if clsname in inst: return inst[clsname] @@ -85,16 +89,16 @@ def get_devices_from_dict(dbdict): class ObjectWrapper(object): - - """An object that allows all method calls and records them, - then passes them on to a target object (if any).""" + """ + An object that allows all method calls and records them, + then passes them on to a target object (if any). + """ def __init__(self, target=None): self.target = target self.calls = [] def __getattr__(self, attr): - def method(attr, *args, **kwargs): self.calls.append((attr, args, kwargs)) if self.target: @@ -131,17 +135,26 @@ def method(attr, *args, **kwargs): class CaselessDict(dict): - """A case insensitive dictionary that only permits strings as keys.""" + """ + A case insensitive dictionary that only permits strings as keys. + """ + def __init__(self, indict={}): dict.__init__(self) - self._keydict = {} # not self.__keydict because I want it to be easily accessible by subclasses + # not self.__keydict because I want it to be easily accessible by subclasses + self._keydict = {} for entry in indict: - self[entry] = indict[entry] # not dict.__setitem__(self, entry, indict[entry]) becasue this causes errors (phantom entries) where indict has overlapping keys... + # not dict.__setitem__(self, entry, indict[entry]) becasue this causes errors + # (phantom entries) where indict has overlapping keys... + self[entry] = indict[entry] def findkey(self, item): - """A caseless way of checking if a key exists or not. - It returns None or the correct key.""" - if not isinstance(item, str): raise TypeError('Keywords for this object must be strings. You supplied %s' % type(item)) + """ + A caseless way of checking if a key exists or not. + It returns None or the correct key. + """ + if not isinstance(item, str): raise TypeError( + 'Keywords for this object must be strings. You supplied %s' % type(item)) key = item.lower() try: return self._keydict[key] @@ -149,12 +162,14 @@ def findkey(self, item): return None def changekey(self, item): - """For changing the casing of a key. + """ + For changing the casing of a key. If a key exists that is a caseless match for 'item' it will be changed to 'item'. - This is useful when initially setting up default keys - but later might want to preserve an alternative casing. - (e.g. if later read from a config file - and you might want to write back out with the user's casing preserved). + This is useful when initially setting up default keys - but later might want to + preserve an alternative casing. (e.g. if later read from a config file - and you + might want to write back out with the user's casing preserved). """ - key = self.findkey(item) # does the key exist + key = self.findkey(item) # does the key exist if key == None: raise KeyError(item) temp = self[key] del self[key] @@ -162,32 +177,40 @@ def changekey(self, item): self._keydict[item.lower()] = item def lowerkeys(self): - """Returns a lowercase list of all member keywords.""" + """ + Returns a lowercase list of all member keywords. + """ return list(self._keydict.keys()) - def __setitem__(self, item, value): # setting a keyword - """To implement lowercase keys.""" - key = self.findkey(item) # if the key already exists + def __setitem__(self, item, value): # setting a keyword + """ + To implement lowercase keys. + """ + key = self.findkey(item) # if the key already exists if key != None: - dict.__delitem__(self,key) + dict.__delitem__(self, key) self._keydict[item.lower()] = item dict.__setitem__(self, item, value) def __getitem__(self, item): - """To implement lowercase keys.""" - key = self.findkey(item) # does the key exist + """ + To implement lowercase keys. + """ + key = self.findkey(item) # does the key exist if key == None: raise KeyError(item) return dict.__getitem__(self, key) - def __delitem__(self, item): # deleting a keyword - key = self.findkey(item) # does the key exist + def __delitem__(self, item): # deleting a keyword + key = self.findkey(item) # does the key exist if key == None: raise KeyError(item) dict.__delitem__(self, key) del self._keydict[item.lower()] def pop(self, item, default=None): - """Correctly emulates the pop method.""" - key = self.findkey(item) # does the key exist + """ + Correctly emulates the pop method. + """ + key = self.findkey(item) # does the key exist if key == None: if default == None: raise KeyError(item) @@ -197,25 +220,33 @@ def pop(self, item, default=None): return dict.pop(self, key) def popitem(self): - """Correctly emulates the popitem method.""" + """ + Correctly emulates the popitem method. + """ popped = dict.popitem(self) del self._keydict[popped[0].lower()] return popped def has_key(self, item): - """A case insensitive test for keys.""" - if not isinstance(item, str): return False # should never have a non-string key - return item.lower() in self._keydict # does the key exist + """ + A case insensitive test for keys. + """ + if not isinstance(item, str): return False # should never have a non-string key + return item.lower() in self._keydict # does the key exist def __contains__(self, item): - """A case insensitive __contains__.""" - if not isinstance(item, str): return False # should never have a non-string key - return item.lower() in self._keydict # does the key exist + """ + A case insensitive __contains__. + """ + if not isinstance(item, str): return False # should never have a non-string key + return item.lower() in self._keydict # does the key exist def setdefault(self, item, default=None): - """A case insensitive setdefault. - If no default is supplied it sets the item to None""" - key = self.findkey(item) # does the key exist + """ + A case insensitive setdefault. + If no default is supplied it sets the item to None + """ + key = self.findkey(item) # does the key exist if key != None: return self[key] self.__setitem__(item, default) self._keydict[item.lower()] = item @@ -223,27 +254,34 @@ def setdefault(self, item, default=None): def get(self, item, default=None): """A case insensitive get.""" - key = self.findkey(item) # does the key exist + key = self.findkey(item) # does the key exist if key != None: return self[key] return default def update(self, indict): - """A case insensitive update. - If your dictionary has overlapping keys (e.g. 'FISH' and 'fish') then one will overwrite the other. - The one that is kept is arbitrary.""" + """ + A case insensitive update. If your dictionary has overlapping keys (e.g. 'FISH' + and 'fish') then one will overwrite the other. The one that is kept is arbitrary. + """ for entry in indict: - self[entry] = indict[entry] # this uses the new __setitem__ method + self[entry] = indict[entry] # this uses the new __setitem__ method def copy(self): - """Create a new caselessDict object that is a copy of this one.""" + """ + Create a new caselessDict object that is a copy of this one. + """ return CaselessDict(self) def dict(self): - """Create a dictionary version of this caselessDict.""" + """ + Create a dictionary version of this caselessDict. + """ return dict.copy(self) def clear(self): - """Clear this caselessDict.""" + """ + Clear this caselessDict. + """ self._keydict = {} dict.clear(self) @@ -252,41 +290,43 @@ def __repr__(self): return 'caselessDict(' + dict.__repr__(self) + ')' -"""A tuctionary, or tuct, is the combination of a tuple with +""" +A tuctionary, or tuct, is the combination of a tuple with a dictionary. A tuct has named items, but they cannot be deleted or rebound, nor new can be added. """ + class ImmutableDict(object): - """The tuct class. An immutable dictionary. + """ + The tuct class. An immutable dictionary. """ def __init__(self, dict=None, **kwds): - self.__data = {} - if dict is not None: - self.__data.update(dict) - if len(kwds): - self.__data.update(kwds) + self.__data = {} + if dict is not None: + self.__data.update(dict) + if len(kwds): + self.__data.update(kwds) - #del __init__ + # del __init__ def __repr__(self): - return repr(self.__data) + return repr(self.__data) def __cmp__(self, dict): - if isinstance(dict, ImmutableDict): - return cmp(self.__data, dict.__data) - else: - return cmp(self.__data, dict) + if isinstance(dict, ImmutableDict): + return cmp(self.__data, dict.__data) + else: + return cmp(self.__data, dict) def __len__(self): - return len(self.__data) + return len(self.__data) def __getitem__(self, key): - return self.__data[key] + return self.__data[key] def copy(self): - if self.__class__ is ImmutableDict: - return ImmutableDict(self.__data.copy()) - import copy - __data = self.__data + if self.__class__ is ImmutableDict: + return ImmutableDict(self.__data.copy()) + __data = self.__data diff --git a/dsconfig/validate.py b/dsconfig/validate.py index 65eb5b5..0ae6b6c 100644 --- a/dsconfig/validate.py +++ b/dsconfig/validate.py @@ -1,11 +1,10 @@ import json import sys -from jsonschema import Draft4Validator, validate, exceptions +from jsonschema import validate, exceptions from .utils import decode_dict - if __name__ == "__main__": data_filename, schema_filename = sys.argv[1], sys.argv[2] diff --git a/dsconfig/viewer.py b/dsconfig/viewer.py index 88c6c9c..f47d16d 100644 --- a/dsconfig/viewer.py +++ b/dsconfig/viewer.py @@ -8,9 +8,9 @@ """ import urwid +from urwidtrees.decoration import CollapsibleIndentedTree from urwidtrees.tree import Tree from urwidtrees.widgets import TreeBox -from urwidtrees.decoration import CollapsibleIndentedTree class FocusableText(urwid.WidgetWrap): @@ -20,7 +20,7 @@ def __init__(self, path, data): child = get_path(path, data) if isinstance(txt, int): # we're in a list # TODO: support containers inside lists - a = urwid.Text(str(txt)+":") + a = urwid.Text(str(txt) + ":") if isinstance(child, dict): b = urwid.Text("") else: @@ -121,7 +121,9 @@ def prev_sibling_position(self, path): class MyTreeBox(TreeBox): def keypress(self, size, key): - "Spicing up the keybindings!" + """ + Spicing up the keybindings! + """ # if key == "delete": # _, path = self.get_focus() # self.set_focus(path[:-1]) @@ -149,7 +151,6 @@ def keypress(self, size, key): ('connectors', 'light red', 'light gray', ''), ] - if __name__ == "__main__": import json @@ -159,6 +160,8 @@ def keypress(self, size, key): # make ctrl+C exit properly, without breaking the terminal def exit_handler(signum, frame): raise urwid.ExitMainLoop() + + signal.signal(signal.SIGINT, exit_handler) # redirecting stdin breaks things. This was supposed to help From 3a9b2ad28b222f20a48d3c1fa1de700a2944693d Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Wed, 3 Jun 2020 16:39:04 +0200 Subject: [PATCH 16/22] Test xls2json. --- dsconfig/excel.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dsconfig/excel.py b/dsconfig/excel.py index 8bf6374..6c7b584 100644 --- a/dsconfig/excel.py +++ b/dsconfig/excel.py @@ -45,7 +45,7 @@ def get_properties(row): name, value = prop.split("=") # need to decode the string, otherwise any linebreaks # will be escaped. - value = value.decode("string-escape") + value = str(value) # .decode("string-escape") # Support inline multiline properties using "\n" prop_dict[name.strip()] = [v.strip() for v in value.split("\n")] @@ -65,7 +65,7 @@ def get_properties(row): convert = TYPE_MAPPING[type_] values = [convert(value)] else: - value = str(value).decode("string-escape") + value = str(value) values = [v.strip() for v in value.split("\n")] prop_dict[name] = values @@ -85,7 +85,6 @@ def get_attribute_properties(row): if name not in SPECIAL_ATTRIBUTE_PROPERTIES: raise ValueError( "'%s' is not a valid attribute property" % name) - value = value.decode("string-escape") # for linebreaks prop_dict[name.strip()] = [v.strip() for v in value.split("\n")] except ValueError: @@ -99,7 +98,7 @@ def get_attribute_properties(row): if name not in SPECIAL_ATTRIBUTE_PROPERTIES: raise ValueError("'%s' it not a valid attribute property" % name) - value = str(value).decode("string-escape") + value = str(value) values = [v.strip() for v in value.split("\n")] prop_dict[name] = values From e7d021e741d0a44c6aae1a011ca8243ec6f3f33d Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Wed, 3 Jun 2020 16:44:00 +0200 Subject: [PATCH 17/22] Refactor setup.py. --- setup.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 1db8887..767f3b8 100755 --- a/setup.py +++ b/setup.py @@ -6,19 +6,21 @@ # Package name="python-dsconfig", version="1.4.0", - packages=['dsconfig', 'dsconfig.appending_dict'], + packages=["dsconfig", "dsconfig.appending_dict"], description="Library and utilities for Tango device configuration.", # Requirements - setup_requires=['setuptools', 'pytest-runner'], - install_requires=['jsonpatch>=1.13', 'jsonschema', 'xlrd', 'PyTango'], + setup_requires=["setuptools", "pytest-runner"], + install_requires=["jsonpatch>=1.13", "jsonschema", "xlrd", "PyTango"], tests_require=["pytest", "pytest-cov", "Faker", "mock"], # Resources package_data={ - 'dsconfig': ['schema/dsconfig.json', - 'schema/schema2.json']}, + "dsconfig": ["schema/dsconfig.json", + "schema/schema2.json"] + }, # Scripts entry_points={ - 'console_scripts': ['xls2json = dsconfig.excel:main', - 'csv2json = dsconfig.callcsv:main', - 'json2tango = dsconfig.json2tango:main']} + "console_scripts": ["xls2json = dsconfig.excel:main", + "csv2json = dsconfig.callcsv:main", + "json2tango = dsconfig.json2tango:main"] + } ) From af234c77d2b20d1ef5ed6232ecacc8b86bf4c67d Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Thu, 4 Jun 2020 09:07:48 +0200 Subject: [PATCH 18/22] Use 2to3 second time. --- dsconfig/appending_dict/__init__.py | 2 +- dsconfig/appending_dict/caseless.py | 2 +- dsconfig/appending_dict/test_appendingdict.py | 2 +- dsconfig/callcsv.py | 6 +- dsconfig/dump.py | 2 +- dsconfig/excel_alarms_userdef.py | 6 +- dsconfig/excel_alarms_vac.py | 10 +- dsconfig/filtering.py | 4 +- dsconfig/magnets2json.py | 126 +++++++++--------- dsconfig/tangodb.py | 2 +- 10 files changed, 81 insertions(+), 81 deletions(-) diff --git a/dsconfig/appending_dict/__init__.py b/dsconfig/appending_dict/__init__.py index efbaf39..1d17a8d 100644 --- a/dsconfig/appending_dict/__init__.py +++ b/dsconfig/appending_dict/__init__.py @@ -73,7 +73,7 @@ def merge(d, u): """ Recursively 'merge' a Mapping into another """ - for k, v in u.items(): + for k, v in list(u.items()): if isinstance(v, Mapping): if k in d: merge(d[k], v) diff --git a/dsconfig/appending_dict/caseless.py b/dsconfig/appending_dict/caseless.py index bd3b97b..579e55e 100644 --- a/dsconfig/appending_dict/caseless.py +++ b/dsconfig/appending_dict/caseless.py @@ -45,7 +45,7 @@ class CaselessDictionary(MutableMapping): def __init__(self, *args, **kwargs): self.__dict__["_dict"] = {} temp_dict = dict(*args, **kwargs) - for key, value in temp_dict.items(): + for key, value in list(temp_dict.items()): if isinstance(key, str): key = CaselessString.make_caseless(key) self._dict[key] = value diff --git a/dsconfig/appending_dict/test_appendingdict.py b/dsconfig/appending_dict/test_appendingdict.py index f7dbda0..f64aa07 100644 --- a/dsconfig/appending_dict/test_appendingdict.py +++ b/dsconfig/appending_dict/test_appendingdict.py @@ -111,7 +111,7 @@ def test_deep_appending(self): ad = AppendingDict() ad["a"]["b"]["c"] = 1 ad["a"]["b"]["c"] = 2 - print(type(ad["a"]["b"])) + print((type(ad["a"]["b"]))) self.assertDictEqual(ad, {"a": {"b": {"c": ['1', '2']}}}) def test_initial_setting_with_dict(self): diff --git a/dsconfig/callcsv.py b/dsconfig/callcsv.py index 3e46430..0e2a8ea 100644 --- a/dsconfig/callcsv.py +++ b/dsconfig/callcsv.py @@ -21,7 +21,7 @@ def special_update(d, u): """ if not (isinstance(d, Mapping) and isinstance(u, Mapping)): return d if d is not None else u - for k, v in u.items(): + for k, v in list(u.items()): d[k] = special_update(d.get(k), v) return d @@ -154,7 +154,7 @@ def process_call_list(lst, skip=False, verbose=True): prototype += ')' # Print prototype if verbose: - print("Executing: " + prototype) + print(("Executing: " + prototype)) # Execute try: module = import_module(module_name) @@ -260,7 +260,7 @@ def main(desc=None, module_name=None, function=None): break else: msg = "'{0}' not found in {1}" - print(msg.format(prototype, get_call_list(input_file))) + print((msg.format(prototype, get_call_list(input_file)))) return # Generate json file if module_name and function: diff --git a/dsconfig/dump.py b/dsconfig/dump.py index c7458d9..8db4589 100755 --- a/dsconfig/dump.py +++ b/dsconfig/dump.py @@ -80,7 +80,7 @@ def main(): attribute_properties=options.attribute_properties, aliases=options.aliases, dservers=options.dservers, subdevices=options.subdevices) - print(json.dumps(dbdata, ensure_ascii=False, indent=4, sort_keys=True)) + print((json.dumps(dbdata, ensure_ascii=False, indent=4, sort_keys=True))) if __name__ == "__main__": diff --git a/dsconfig/excel_alarms_userdef.py b/dsconfig/excel_alarms_userdef.py index 729b707..1f477db 100644 --- a/dsconfig/excel_alarms_userdef.py +++ b/dsconfig/excel_alarms_userdef.py @@ -20,7 +20,7 @@ def __getattr__(self, attr): def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev, al_rec): - print(inst, dev, al_name, al_cond, al_desc, al_sev, al_rec) + print((inst, dev, al_name, al_cond, al_desc, al_sev, al_rec)) devdict = sdict.servers["PyAlarm/" + inst]["PyAlarm"][dev] if "AlarmList" not in devdict.properties: devdict.properties["AlarmList"] = [] @@ -44,7 +44,7 @@ def xls_to_dict(xls_filename): # above skips row 0 (col headers) # look at all rows but only read those with entry in first col if sheet.row_values(line)[0] is not "": - print("IN LINE ", line, sheet.row_values(line)[0]) + print(("IN LINE ", line, sheet.row_values(line)[0])) dev_config = sheet.row_values(line) add_device(json_dict, *dev_config[:7]) return json_dict @@ -53,7 +53,7 @@ def xls_to_dict(xls_filename): def main(): import sys data = xls_to_dict(sys.argv[1]) - print(json.dumps(data, indent=4)) + print((json.dumps(data, indent=4))) outfile = open('pyalarm_config.json', 'w') json.dump(data, outfile, indent=4) diff --git a/dsconfig/excel_alarms_vac.py b/dsconfig/excel_alarms_vac.py index 61b3290..f4b0685 100644 --- a/dsconfig/excel_alarms_vac.py +++ b/dsconfig/excel_alarms_vac.py @@ -20,7 +20,7 @@ def __getattr__(self, attr): def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev): - print(inst, dev, al_name, al_cond, al_desc, al_sev) + print((inst, dev, al_name, al_cond, al_desc, al_sev)) # devdict = sdict.servers["PyAlarm"]["PyAlarm/"+inst]["PyAlarm"][dev] devdict = sdict.servers["PyAlarm/" + inst]["PyAlarm"][dev] if "AlarmList" not in devdict.properties: @@ -82,14 +82,14 @@ def xls_to_dict(xls_filename): # above skips row 0 (col headers) # look at all rows but only read those with entry in first col if sheet.row_values(line)[0] is not "": - print("IN LINE ", line, sheet.row_values(line)[0]) + print(("IN LINE ", line, sheet.row_values(line)[0])) # assume that if you get to a new device, it means a new section of vacuum # in this case, need to make a final alarm which is or of all others dev_config = sheet.row_values(line) - print(dev_config, dev_config[3].rsplit("/", 1)[0]) + print((dev_config, dev_config[3].rsplit("/", 1)[0])) if dev_config[1] != last_device or dev_config[0] == "end": - print("START NEW SECTION", dev_config[1]) - print("---- ADDING TO JSON summary ", summary_condition, last_name) + print(("START NEW SECTION", dev_config[1])) + print(("---- ADDING TO JSON summary ", summary_condition, last_name)) if summary_condition != "": add_device(json_dict, last_server, last_device, last_name.rsplit("_", 1)[0], summary_condition, "at least one vac. %s in section %s" % (nature, last_name.rsplit("__", 2)[0]), diff --git a/dsconfig/filtering.py b/dsconfig/filtering.py index 2f4c149..18e7b91 100644 --- a/dsconfig/filtering.py +++ b/dsconfig/filtering.py @@ -9,12 +9,12 @@ def filter_nested_dict(node, pattern, depth, level=0, invert=False): at the given depth. """ if level == depth: - return dict((key, value) for key, value in node.items() + return dict((key, value) for key, value in list(node.items()) if (not invert and pattern.match(key)) or (invert and not pattern.match(key))) else: dupe_node = {} - for key, val in node.items(): + for key, val in list(node.items()): cur_node = filter_nested_dict(val, pattern, depth, level + 1, invert) if cur_node: diff --git a/dsconfig/magnets2json.py b/dsconfig/magnets2json.py index 09fdde1..308b641 100644 --- a/dsconfig/magnets2json.py +++ b/dsconfig/magnets2json.py @@ -158,7 +158,7 @@ def handle_circuit_name(self, itemName, endnum): endname = "CRCOY-" + endnum elif "DI" in itemName: endname = "CRDI-" + endnum - print("dealing with endname ", endname) + print(("dealing with endname ", endname)) elif "SX" in itemName: endname = "CRSX-" + endnum elif "SOL" in itemName: @@ -260,7 +260,7 @@ def match_properties(self): self.parameters["Tilt"] = ["90"] else: self.parameters["Tilt"] = ["0"] - print("pjbyyy", self.parameters) + print(("pjbyyy", self.parameters)) def add_device(self, sdict, adict, psdict, name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[0-9]+)'): @@ -270,10 +270,10 @@ def add_device(self, sdict, adict, psdict, # prepare pattern for parsing name pattern = re.compile(name_parsing_string) - print("In add device for item: " + self.itemName + " as " + self.itemType, self.alpars) + print(("In add device for item: " + self.itemName + " as " + self.itemType, self.alpars)) # only when we know class for certain element - print("adict is ", adict) + print(("adict is ", adict)) circuit_alarm_params = [] # for case with no final number like I.TR1.MAG.DIE (no .1 etc at end) @@ -293,15 +293,15 @@ def add_device(self, sdict, adict, psdict, parsed = False tryagain = False if name_items == None: - print("Warning: Item name in lattice file doesn't match the naming convention.", self.itemName) + print(("Warning: Item name in lattice file doesn't match the naming convention.", self.itemName)) tryagain = True else: parsed = True if tryagain: name_items = alt_pattern.search(self.itemName) if name_items == None: - print("Warning: Item name in lattice file STILL doesn't match the naming convention.", - self.itemName) + print(("Warning: Item name in lattice file STILL doesn't match the naming convention.", + self.itemName)) else: parsed = True if parsed: @@ -323,7 +323,7 @@ def add_device(self, sdict, adict, psdict, num2 = "01" self.match_properties() - print("pjbxxx ", self.parameters) + print(("pjbxxx ", self.parameters)) # create device for json output name = (system + "-" + location + '/' + subsystem + '/' + device + "-" + num2).encode('ascii', 'ignore') @@ -332,7 +332,7 @@ def add_device(self, sdict, adict, psdict, # hack for circuits if "CIR" in self.itemName: - print("orig name", self.itemName, name) + print(("orig name", self.itemName, name)) # quad circuits named like CRQ # correctors like CRCOX and CRCOY @@ -348,13 +348,13 @@ def add_device(self, sdict, adict, psdict, # e.g in G00 have COIX 1, 2, 3 and COHX 1 and 2, which would make COX 1, 2, 3 with COIX 1 and 2 being lost! # increment number till unique while name in cirlist: - print("danger! already named this circuit!", self.itemName, name) + print(("danger! already named this circuit!", self.itemName, name)) suffix = int(name.split("-")[2]) + 1 newnum2 = "%02d" % suffix print(newnum2) name = (name.rsplit("-", 1)[0]) + "-" + newnum2 print(name) - print("new name ", name) + print(("new name ", name)) cirlist.append(name) print(cirlist) devclass = "MagnetCircuit" @@ -378,7 +378,7 @@ def add_device(self, sdict, adict, psdict, if "DID" in compactnamecir: compactnamecir = compactnamecir.replace("DID", "DI") - print("circuit compact name is ", compactnamecir, name_l_cir) + print(("circuit compact name is ", compactnamecir, name_l_cir)) # fill alarm info for circuits # @@ -397,11 +397,11 @@ def add_device(self, sdict, adict, psdict, compactnamecir[:-1] in key and key.count("_") > 5 and key not in already_added and "F" + num + "_" in key): - print("key is", num, key) + print(("key is", num, key)) already_added.append(key) - print("FOUND ALARM INFO FOR CIR", compactnamecir, key, alarm_dict[key], section_cir) + print(("FOUND ALARM INFO FOR CIR", compactnamecir, key, alarm_dict[key], section_cir)) pyattname = "I-" + section_cir + "/DIA/COOLING" # for the magnets json file @@ -418,9 +418,9 @@ def add_device(self, sdict, adict, psdict, self.alpars[pyalarm] = {} self.config_alarms(pyalarm, alname, alsev, aldesc, pyattname, key) # will fill self.alpars - print( - "+++++++++++++++++ Creating device server : " + server + " for " + devclass + " (name= " + name + ")") - print("+++++++++ With properties : ", self.parameters) + print(( + "+++++++++++++++++ Creating device server : " + server + " for " + devclass + " (name= " + name + ")")) + print(("+++++++++ With properties : ", self.parameters)) # Dont actually write attributes to DB, only properties # see if this class exists and append if so, or create @@ -438,18 +438,18 @@ def add_device(self, sdict, adict, psdict, del name_l[0] del name_l[1] # del name_l[2] - print("pjb kkk", name_l) + print(("pjb kkk", name_l)) compactfullname = "".join(name_l) compactname = compactfullname.split("_")[0] compactname_nonum = compactfullname.split("_")[0][:-1] + "_" - print("-------------------- magnet not circuit", self.itemName, compactname) + print(("-------------------- magnet not circuit", self.itemName, compactname)) # see what is the ps of the magnet if name in POWER_SUPPLY_MAP: powersupplyname = POWER_SUPPLY_MAP[name] else: - print("magnet not in PS map, skipping", name) + print(("magnet not in PS map, skipping", name)) return # !!!!!!!!!!! *********** create circuit device for each new ps ************!!!!!!!!!!!! @@ -471,7 +471,7 @@ def add_device(self, sdict, adict, psdict, # get alarm info from excel pyalarm = system + "-" + location + '/MAG/ALARM' - print("adict is again", adict) + print(("adict is again", adict)) if adict is not None: devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] print("init devdictalarm") @@ -483,7 +483,7 @@ def add_device(self, sdict, adict, psdict, pyattname = "I-" + section + "/DIA/COOLING" - print("FOUND ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict) + print(("FOUND ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict)) # for the magnets json file if 'TemperatureInterlock' not in self.parameters: @@ -511,10 +511,10 @@ def add_device(self, sdict, adict, psdict, if compactname_nonum in key or (compactname[:-1] in key and key.count("_") > 5 and ( "F" + num + "_" in key or "_" + num + "_" in key)): # if compactname_nonum in key: - print("mag key ", key, compactname_nonum, compactname, "F" + num + "_", "_" + num + "_") + print(("mag key ", key, compactname_nonum, compactname, "F" + num + "_", "_" + num + "_")) pyattname = "I-" + section + "/DIA/COOLING" - print("FOUND MORE ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict) + print(("FOUND MORE ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict)) # for the magnets json file if 'TemperatureInterlock' not in self.parameters: @@ -529,11 +529,11 @@ def add_device(self, sdict, adict, psdict, # get calibration info from the excel if self.itemName.split("_")[0] in calib_dict: - print("FOUND CALIB INFO", self.itemName) + print(("FOUND CALIB INFO", self.itemName)) # find max multipole expansions dim = max(list(calib_dict[self.itemName.split("_")[0]].keys()), key=int) - print("--- max order is", dim) + print(("--- max order is", dim)) # create arrays of this dimensions, other dimension is 11 @@ -543,10 +543,10 @@ def add_device(self, sdict, adict, psdict, # iterate over keys and add to the array for key in calib_dict[self.itemName.split("_")[0]]: - print('--- ', key, 'corresponds to', calib_dict[self.itemName.split("_")[0]][key]) + print(('--- ', key, 'corresponds to', calib_dict[self.itemName.split("_")[0]][key])) currents = calib_dict[self.itemName.split("_")[0]][key][5:25] fields = calib_dict[self.itemName.split("_")[0]][key][25:45] - print('--- ', currents, fields) + print(('--- ', currents, fields)) fieldsmatrix[key - 1] = fields currentsmatrix[key - 1] = currents @@ -555,14 +555,14 @@ def add_device(self, sdict, adict, psdict, orientation = calib_dict[self.itemName.split("_")[0]][key][2] # print "P, O", polarity, orientation - print('--- ', fieldsmatrix) - print('--- ', currentsmatrix) + print(('--- ', fieldsmatrix)) + print(('--- ', currentsmatrix)) # now trim the matrices (lists) maxlength = 20 for i, val in enumerate(fieldsmatrix[dim - 1]): if val == '': - print(i, val) + print((i, val)) maxlength = i break print(maxlength) @@ -571,8 +571,8 @@ def add_device(self, sdict, adict, psdict, del fieldsmatrix[i][maxlength:] del currentsmatrix[i][maxlength:] - print('Now--- ', fieldsmatrix) - print('Now--- ', currentsmatrix) + print(('Now--- ', fieldsmatrix)) + print(('Now--- ', currentsmatrix)) magnetcircuit.parameters['ExcitationCurveCurrents'] = currentsmatrix magnetcircuit.parameters['ExcitationCurveFields'] = fieldsmatrix @@ -590,14 +590,14 @@ def add_device(self, sdict, adict, psdict, endname = self.handle_circuit_name(self.itemName, endnum) cname = cname.rsplit("/", 1)[0] + "/" + endname - print("cname is ", cname, name, powersupplyname, circuit_ps_list) + print(("cname is ", cname, name, powersupplyname, circuit_ps_list)) while cname in cirlist: - print("danger2! already named this circuit!", cname) + print(("danger2! already named this circuit!", cname)) suffix = int(cname.split("-")[2]) + 1 newnum2 = "%02d" % suffix cname = (cname.rsplit("-", 1)[0]) + "-" + newnum2 - print("new name ", cname) + print(("new name ", cname)) # only add one circuit device per ps if powersupplyname not in circuit_ps_list: @@ -605,12 +605,12 @@ def add_device(self, sdict, adict, psdict, magnetcircuit.add_device(sdict, adict, psdict) circuit_ps_list[powersupplyname] = cname - print("adding circuit name ", magnetcircuit.itemName, cname, circuit_ps_list) + print(("adding circuit name ", magnetcircuit.itemName, cname, circuit_ps_list)) self.parameters['CircuitProxies'] = [cname] else: # if we aleady made this circuit device, add it to this magnet properties - print("!!!ALART!!! already added a circuit device for ", self.itemName, name, system, location) + print(("!!!ALART!!! already added a circuit device for ", self.itemName, name, system, location)) if system == "R3": system = "I" @@ -620,7 +620,7 @@ def add_device(self, sdict, adict, psdict, self.parameters['CircuitProxies'] = [circuit_ps_list[powersupplyname]] # need to get the name of the circuit device from the ps dict though - print("exiting circuit device is", circuit_ps_list[powersupplyname]) + print(("exiting circuit device is", circuit_ps_list[powersupplyname])) # print "current mags ", system+"-"+location # print "current mags 2", sdict.servers @@ -629,17 +629,17 @@ def add_device(self, sdict, adict, psdict, sdict.servers["%s/%s" % ("MagnetCircuit", system + "-" + location)]["MagnetCircuit"][ circuit_ps_list[powersupplyname]].properties # for circuits json - print("cir name from ps ", circuit_ps_list[powersupplyname], psdict) + print(("cir name from ps ", circuit_ps_list[powersupplyname], psdict)) ps_current_mags = psdict.Circuits[circuit_ps_list[powersupplyname]].Properties - print("current mags ", current_mags['MagnetProxies']) + print(("current mags ", current_mags['MagnetProxies'])) if name in current_mags['MagnetProxies']: - print("circuit already has magnet ", name) + print(("circuit already has magnet ", name)) else: ps_current_mags['MagnetProxies'].append(name) current_mags['MagnetProxies'].append(name) - print("magnets on cir ", current_mags['ExcitationCurveFields'], current_mags['MagnetProxies'], - len(current_mags['MagnetProxies'])) + print(("magnets on cir ", current_mags['ExcitationCurveFields'], current_mags['MagnetProxies'], + len(current_mags['MagnetProxies']))) # need to average the currents, even if already done so in excel (depends on field order) if 'ExcitationCurveFields' in current_mags: @@ -649,11 +649,11 @@ def add_device(self, sdict, adict, psdict, assoc_curr_m = current_mags['ExcitationCurveCurrents'] this_curr_m = currentsmatrix - print("field matrix assoc is ", assoc_field_m) - print("field matrix current is ", this_field_m) + print(("field matrix assoc is ", assoc_field_m)) + print(("field matrix current is ", this_field_m)) - print("current matrix assoc is ", assoc_curr_m) - print("current matrix current is ", this_curr_m) + print(("current matrix assoc is ", assoc_curr_m)) + print(("current matrix current is ", this_curr_m)) for i in range(dim): print(i) @@ -671,12 +671,12 @@ def add_device(self, sdict, adict, psdict, newCurrents = [(x * (len(current_mags['MagnetProxies']) - 1) + y) / len( current_mags['MagnetProxies']) for y, x in zip(this_curr_m[i], assoc_curr_m[i])] - print("new fields ", newFields) - print("new currents ", newCurrents) + print(("new fields ", newFields)) + print(("new currents ", newCurrents)) current_mags['ExcitationCurveFields'][i] = newFields - print("updated: ", current_mags['ExcitationCurveFields']) + print(("updated: ", current_mags['ExcitationCurveFields'])) current_mags['ExcitationCurveCurrents'][i] = newCurrents - print("updated: ", current_mags['ExcitationCurveCurrents']) + print(("updated: ", current_mags['ExcitationCurveCurrents'])) else: @@ -769,7 +769,7 @@ def parseLatticeFile(self): elif inFileName == '': inFileName = par - print(inFileName, doCalib) + print((inFileName, doCalib)) if doAlarms or doCalib: import xlrd @@ -781,13 +781,13 @@ def parseLatticeFile(self): sheet = xls.sheet_by_name('Sheet1') rows = [sheet.row_values(i) for i in range(sheet.nrows)] column_names = rows[0] - print("cols ", column_names) + print(("cols ", column_names)) for row in enumerate(rows[1:]): if row[1][0] == "": continue - print(row[1][0], row[1][1]) + print((row[1][0], row[1][1])) alarm_dict[row[1][0]] = row[1][1] - print("DICT IS ", alarm_dict) + print(("DICT IS ", alarm_dict)) # make dict just for alarms! for pyalarm json_dict_alarms = SuperDict() @@ -806,10 +806,10 @@ def parseLatticeFile(self): sheet = xls.sheet_by_name(name) rows = [sheet.row_values(i) for i in range(sheet.nrows)] column_names = rows[7] - print("cols ", column_names) + print(("cols ", column_names)) for row in enumerate(rows[9:]): - print(row[1]) + print((row[1])) if row[1][2] == "": continue # this is like @@ -824,13 +824,13 @@ def parseLatticeFile(self): else: if row[1][7] is not "": # we found more curves for the same magnet - print("found another entry", row[1][2], row[1][7]) + print(("found another entry", row[1][2], row[1][7])) data_key = int(row[1][7]) data_list = row[1][3:48] data_dict = {data_key: data_list} calib_dict[row[1][2].strip()][data_key] = data_list - print("DICT IS ", calib_dict) + print(("DICT IS ", calib_dict)) # create a parser for the file parser = ElegantLatticeParser(inFileName) @@ -850,7 +850,7 @@ def parseLatticeFile(self): print("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ") print("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ") - print(json_dict.servers) + print((json_dict.servers)) outfile = open('magnets.json', 'w') @@ -867,11 +867,11 @@ def parseLatticeFile(self): for key in json_dict['servers'][item]["MagnetCircuit"][c]["properties"]: if key == "ExcitationCurveCurrents": ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print(key, [str(x) for x in ls]) + print((key, [str(x) for x in ls])) json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] if key == "ExcitationCurveFields": ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print(key, [str(x) for x in ls]) + print((key, [str(x) for x in ls])) json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] json.dump(json_dict, outfile, indent=4) diff --git a/dsconfig/tangodb.py b/dsconfig/tangodb.py index 541c641..c7bfa14 100755 --- a/dsconfig/tangodb.py +++ b/dsconfig/tangodb.py @@ -325,7 +325,7 @@ def get_devices_by_name_and_class(dbproxy, name, clss="*"): def nwise(it, n): # [s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...] - return zip(*[islice(it, i, None, n) for i in range(n)]) + return list(zip(*[islice(it, i, None, n) for i in range(n)])) def maybe_upper(s, upper=False): From 6245006a63163b8a6d556b242bb3d25c654502e7 Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Thu, 4 Jun 2020 09:12:13 +0200 Subject: [PATCH 19/22] Bump version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 767f3b8..290e172 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( # Package name="python-dsconfig", - version="1.4.0", + version="1.5.0", packages=["dsconfig", "dsconfig.appending_dict"], description="Library and utilities for Tango device configuration.", # Requirements From e663d27006d9825fb2eb9f988dbd682062d648ca Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Thu, 4 Jun 2020 14:17:54 +0200 Subject: [PATCH 20/22] Remove files from PR12. --- bin/csv2json | 2 - bin/xls2json | 5 - dsconfig/callcsv.py | 317 ----------- dsconfig/excel_alarms_userdef.py | 62 --- dsconfig/excel_alarms_vac.py | 122 ----- dsconfig/magnets2json.py | 887 ------------------------------- 6 files changed, 1395 deletions(-) delete mode 100755 bin/csv2json delete mode 100755 bin/xls2json delete mode 100644 dsconfig/callcsv.py delete mode 100644 dsconfig/excel_alarms_userdef.py delete mode 100644 dsconfig/excel_alarms_vac.py delete mode 100644 dsconfig/magnets2json.py diff --git a/bin/csv2json b/bin/csv2json deleted file mode 100755 index 2551e7f..0000000 --- a/bin/csv2json +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python -m dsconfig.callcsv $* diff --git a/bin/xls2json b/bin/xls2json deleted file mode 100755 index 3d32bf8..0000000 --- a/bin/xls2json +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python - -from dsconfig.excel import main - -main() diff --git a/dsconfig/callcsv.py b/dsconfig/callcsv.py deleted file mode 100644 index 0e2a8ea..0000000 --- a/dsconfig/callcsv.py +++ /dev/null @@ -1,317 +0,0 @@ -""" -Provide functions to parse a callable csv file. -""" - -# Imports - -import csv -import json -import os -import sys -from collections import Mapping -from importlib import import_module -from optparse import OptionParser - - -# Utils - -def special_update(d, u): - """ - Update nested dictionnaries while prioritizing the first argument. - """ - if not (isinstance(d, Mapping) and isinstance(u, Mapping)): - return d if d is not None else u - for k, v in list(u.items()): - d[k] = special_update(d.get(k), v) - return d - - -def max_or_none(*args): - """ - Maximum function considering None as a maximum value. - """ - return None if None in args else max(*args) - - -def cast_list(lst): - """ - Convert a list of a string to the corresponding value. - """ - result = [] - for value in lst: - # Integer conversion - try: - value = int(value) - except (ValueError, TypeError): - # Float conversion - try: - value = float(value) - except (ValueError, TypeError): - # Hexa conversion - try: - value = int(value, 16) - except (ValueError, TypeError): - # Ignore - pass - # Append - result.append(value) - # Return - if not result: - return "" - if len(result) == 1: - return result[0] - return tuple(result) - - -# Csv functions - -def get_column(matrix, col, start=None, stop=None, step=None): - """ - Get the column of a matrix, with optional range arguments. - """ - return [row[col] for row in matrix][slice(start, stop, step)] - - -def get_markup_index(matrix, col, markup): - """ - Find a markup in a given column and return the following index. - """ - for i, key in enumerate(get_column(matrix, col)): - if key == markup: - return i + 1 - return None - - -def get_range_lst(matrix, col, start=0, stop=None, markup=None): - """ - Get a value->range dictionnary from a given column. - """ - if markup: - start = max_or_none(start, get_markup_index(matrix, col, markup)) - if start is None: - return {} - result = [] - previous_key, previous_start = None, None - for i, key in enumerate(get_column(matrix, col, start, stop), start): - if key != "": - if previous_key is not None: - result.append((previous_key, previous_start, i)) - previous_key, previous_start = key, i - if previous_key is not None: - result.append((previous_key, previous_start, i + 1)) - return result - - -# Callable csv functions - -def get_kwargs(matrix, start, stop): - """ - Get the keywords arguments between two indexes. - """ - kwargs = {} - keyword_lst = get_range_lst(matrix, 2, start, stop) - for keyword, start, stop in keyword_lst: - lst = get_range_lst(matrix, 3, start, stop) - values = [key for key, _, _ in lst] - kwargs[str(keyword)] = cast_list(values) - return kwargs - - -def get_call_list(filename): - """ - Get the call list from a callable cls file. - """ - result = [] - with open(filename) as csvfile: - reader = csv.reader(csvfile, delimiter=',') - matrix = [[value.strip() for value in row] for row in reader] - package_lst = get_range_lst(matrix, 0, markup="Package") - # Process - result = [] - for package, start, stop in package_lst: - func_lst = get_range_lst(matrix, 1, start, stop) - for func, start, stop in func_lst: - kwargs = get_kwargs(matrix, start, stop) - result.append((package, func, kwargs)) - return result - - -# Data functions - -def process_call_list(lst, skip=False, verbose=True): - """ - Process a given call list and return the results. - """ - result = [] - errors = ImportError, ValueError, TypeError, AttributeError - for module_name, func_name, kwargs in lst: - # Build prototype - prototype = "{0}.{1}(".format(module_name, func_name) - for key, value in list(kwargs.items()): - prototype += '{0}={1}, '.format(key, value) - if prototype.endswith(' '): - prototype = prototype[:-2] - prototype += ')' - # Print prototype - if verbose: - print(("Executing: " + prototype)) - # Execute - try: - module = import_module(module_name) - func = getattr(module, func_name) - value = func(**kwargs) - # Fail - except errors as exc: - if not skip: - raise exc - else: - print(exc) - # Success - else: - result.append(value) - return result - - -def join_data(lst, source=None): - """ - Join a list of json strings or dictionnaries into a single dict. - """ - data = {} - for mapping in lst: - if isinstance(mapping, str): - mapping = json.loads(mapping) - special_update(data, mapping) - if source: - data['_source'] = source - return data - - -# CSV to Dict function - -def callable_csv_to_dict(filename, skip=False, verbose=True, to_json=False): - """ - Convert a callable csv file to a data dictionnary. - """ - calls = get_call_list(filename) - if not calls: - return - strings = process_call_list(calls, skip, verbose) - data = join_data(strings, filename) - if to_json: - return json.dumps(data, indent=4, sort_keys=True) - return data - - -# Command lines arguments for configuration script - -def parse_command_line_args(desc): - """ - Parse arguments given in command line - """ - usage = "%prog [-i INPUT] [-o OUTPUT] [-v] [-w]" - parser = OptionParser(usage=usage, description=desc, version='%prog v1.0') - - msg = "The input callable csv file" - parser.add_option('-i', '--input', metavar='IN', - type='str', help=msg, default='') - - msg = "The output tango database json file" - parser.add_option('-o', '--output', metavar='OUT', - type='str', help=msg, default='') - - msg = "Display informations" - parser.add_option('-v', '--verbose', - action="store_true", help=msg, default=False) - - msg = "Write the Tango Database" - parser.add_option('-w', '--write', - action="store_true", help=msg, default=False) - - options, args = parser.parse_args() - - if args: - msg = "No argument expected, options only.\n" - msg += "Use --help for further information." - parser.error(msg) - - return options.input, options.output, options.write, options.verbose - - -# Main function for configuration scripts - -def main(desc=None, module_name=None, function=None): - """ - Run the script. - """ - kwargs = {} - remove = False - desc = desc or "Generate a Tango json file for a given callable csv file." - # Parse command line args - input_file, output_file, write, verbose = parse_command_line_args(desc) - # Process input file - if input_file and module_name and function: - prototype = ".".join((module_name, function.__name__)) - for module, func, keywords in get_call_list(input_file): - if module == module_name and func == function.__name__: - kwargs = keywords - if verbose: - print(("'{0}' found".format(prototype))) - print(("kwargs = " + str(kwargs))) - break - else: - msg = "'{0}' not found in {1}" - print((msg.format(prototype, get_call_list(input_file)))) - return - # Generate json file - if module_name and function: - if not input_file and verbose: - msg = 'No input file given. ' - msg += 'Default configuration will be used' - print(msg) - data = function(**kwargs) - if isinstance(data, Mapping): - string = json.dumps(data, indent=4, sort_keys=True) - elif isinstance(data, str): - string = data - else: - msg = "The function didn't return a valid data format.\n" - msg += "The type is {0} instead.".format(type(data)) - print(msg) - print(data) - return - elif input_file: - string = callable_csv_to_dict(input_file, True, verbose, True) - else: - print('An input file is required.') - return - # Display json file - if verbose: - print('Json string generated:') - print(string) - # Write temporary file - if output_file == "" and write: - remove = True - output_file = "temp.json" - # Write output file - if output_file: - with open(output_file, mode='w') as f: - f.write(string) - if verbose and output_file: - print(('Exported to: ' + output_file)) - # Write tango database - if write: - from dsconfig import configure - sys.argv = [__name__, output_file, "-w"] - configure.main() - # Remove temporary file - if remove: - os.remove(output_file) - if verbose: - print(('Removed: ' + output_file)) - print('OK!') - - -# Main execution - -if __name__ == "__main__": - main() diff --git a/dsconfig/excel_alarms_userdef.py b/dsconfig/excel_alarms_userdef.py deleted file mode 100644 index 1f477db..0000000 --- a/dsconfig/excel_alarms_userdef.py +++ /dev/null @@ -1,62 +0,0 @@ -import json -from collections import defaultdict - -import xlrd - - -class SuperDict(defaultdict): - """ - A recursive defaultdict with extra bells & whistles - """ - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev, al_rec): - print((inst, dev, al_name, al_cond, al_desc, al_sev, al_rec)) - devdict = sdict.servers["PyAlarm/" + inst]["PyAlarm"][dev] - if "AlarmList" not in devdict.properties: - devdict.properties["AlarmList"] = [] - devdict.properties["AlarmList"].append(al_name + ":" + al_cond) - if "AlarmDescriptions" not in devdict.properties: - devdict.properties["AlarmDescriptions"] = [] - devdict.properties["AlarmDescriptions"].append(al_name + ":" + al_desc) - if "AlarmSeverities" not in devdict.properties: - devdict.properties["AlarmSeverities"] = [] - devdict.properties["AlarmSeverities"].append(al_name + ":" + al_sev) - if "AlarmReceivers" not in devdict.properties: - devdict.properties["AlarmReceivers"] = [] - devdict.properties["AlarmReceivers"].append(al_name + ":" + al_rec) - - -def xls_to_dict(xls_filename): - json_dict = SuperDict() - xls = xlrd.open_workbook(xls_filename) - sheet = xls.sheet_by_name("Alarms") - for line in range(1, sheet.nrows): - # above skips row 0 (col headers) - # look at all rows but only read those with entry in first col - if sheet.row_values(line)[0] is not "": - print(("IN LINE ", line, sheet.row_values(line)[0])) - dev_config = sheet.row_values(line) - add_device(json_dict, *dev_config[:7]) - return json_dict - - -def main(): - import sys - data = xls_to_dict(sys.argv[1]) - print((json.dumps(data, indent=4))) - outfile = open('pyalarm_config.json', 'w') - json.dump(data, outfile, indent=4) - - -if __name__ == "__main__": - main() diff --git a/dsconfig/excel_alarms_vac.py b/dsconfig/excel_alarms_vac.py deleted file mode 100644 index f4b0685..0000000 --- a/dsconfig/excel_alarms_vac.py +++ /dev/null @@ -1,122 +0,0 @@ -import json -from collections import defaultdict - -import xlrd - - -class SuperDict(defaultdict): - """ - A recursive defaultdict with extra bells & whistles - """ - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -def add_device(sdict, inst, dev, al_name, al_cond, al_desc, al_sev): - print((inst, dev, al_name, al_cond, al_desc, al_sev)) - # devdict = sdict.servers["PyAlarm"]["PyAlarm/"+inst]["PyAlarm"][dev] - devdict = sdict.servers["PyAlarm/" + inst]["PyAlarm"][dev] - if "AlarmList" not in devdict.properties: - devdict.properties["AlarmList"] = [] - devdict.properties["AlarmList"].append(al_name + ":" + al_cond) - if "AlarmDescriptions" not in devdict.properties: - devdict.properties["AlarmDescriptions"] = [] - devdict.properties["AlarmDescriptions"].append(al_name + ":" + al_desc) - # hard code severity and some other things - only one per instance - if "AlarmSeverities" not in devdict.properties: - devdict.properties["AlarmSeverities"] = [] - devdict.properties["AlarmSeverities"].append(al_name + ":" + al_sev) - if "AlarmReceivers" not in devdict.properties: - devdict.properties["AlarmReceivers"] = [] - devdict.properties["AlarmReceivers"].append(al_name + ":HTML") - # hard code severity and some other things - only one per instance - if "AlarmThreshold" not in devdict.properties: - devdict.properties["AlarmThreshold"] = [] - devdict.properties["AlarmThreshold"] = [1] - if "LogFile" not in devdict.properties: - devdict.properties["LogFile"] = [] - devdict.properties["LogFile"] = ["/tmp/pjb/log"] - if "HtmlFolder" not in devdict.properties: - devdict.properties["HtmlFolder"] = [] - devdict.properties["HtmlFolder"] = ["/tmp/pjb"] - if "PollingPeriod" not in devdict.properties: - devdict.properties["PollingPeriod"] = [] - devdict.properties["PollingPeriod"] = [5] - if "MaxMessagesPerAlarm" not in devdict.properties: - devdict.properties["MaxMessagesPerAlarm"] = [] - devdict.properties["MaxMessagesPerAlarm"] = [1] - if "AutoReset" not in devdict.properties: - devdict.properties["AutoReset"] = [] - devdict.properties["AutoReset"] = [0] - if "StartupDelay" not in devdict.properties: - devdict.properties["StartupDelay"] = [] - devdict.properties["StartupDelay"] = [0] - - -def xls_to_dict(xls_filename): - json_dict = SuperDict() - xls = xlrd.open_workbook(xls_filename) - - for i in range(0, 2): - - if i == 1: - sheet = xls.sheet_by_name("Alarms") - nature = "interlock" - else: - sheet = xls.sheet_by_name("Warnings") - nature = "bypass" - - last_server = "" - last_device = "" - last_name = "" - last_sev = "" - summary_condition = "" - for line in range(1, sheet.nrows): - # above skips row 0 (col headers) - # look at all rows but only read those with entry in first col - if sheet.row_values(line)[0] is not "": - print(("IN LINE ", line, sheet.row_values(line)[0])) - # assume that if you get to a new device, it means a new section of vacuum - # in this case, need to make a final alarm which is or of all others - dev_config = sheet.row_values(line) - print((dev_config, dev_config[3].rsplit("/", 1)[0])) - if dev_config[1] != last_device or dev_config[0] == "end": - print(("START NEW SECTION", dev_config[1])) - print(("---- ADDING TO JSON summary ", summary_condition, last_name)) - if summary_condition != "": - add_device(json_dict, last_server, last_device, last_name.rsplit("_", 1)[0], summary_condition, - "at least one vac. %s in section %s" % (nature, last_name.rsplit("__", 2)[0]), - last_sev) - last_server = dev_config[0] - last_device = dev_config[1] - last_name = dev_config[2] - last_sev = dev_config[5] - summary_condition = "" - if summary_condition == "": - summary_condition = summary_condition + dev_config[3] - else: - summary_condition = summary_condition + " or " + dev_config[3] - - if dev_config[0] != "end": - add_device(json_dict, *dev_config[:6]) - - return json_dict - - -def main(): - import sys - data = xls_to_dict(sys.argv[1]) - # print json.dumps(data, indent=4) - outfile = open('alarms_vac.json', 'w') - json.dump(data, outfile, indent=4) - - -if __name__ == "__main__": - main() diff --git a/dsconfig/magnets2json.py b/dsconfig/magnets2json.py deleted file mode 100644 index 308b641..0000000 --- a/dsconfig/magnets2json.py +++ /dev/null @@ -1,887 +0,0 @@ -#!/usr/bin/env python -# "$Name: $"; -# "$Header: $"; -# ============================================================================= -# -# file : lattice2json.py -# -# description : Python source for the lattice2json that is a tool to generate -# a json file from an elegant lattice file -# The json file can then be uploaded to a Tango database -# -# project : Virtual Accelerator -# -# $Author: $ -# -# $Revision: $ -# -# $Log: $ -# -# copyleft : Solaris/MAX IV -# Krakow,PL/Lund,SE# -# - -import copy -import io -import json -import re -import sys -from collections import defaultdict - -from PowerSupplyMap import POWER_SUPPLY_MAP -from TangoProperties import TANGO_PROPERTIES - -cirlist = [] - - -class SuperDict(defaultdict): - "A recursive defaultdict with extra bells & whistles" - - def __init__(self): - defaultdict.__init__(self, SuperDict) - - def __setattr__(self, attr, value): - self[attr] = value - - def __getattr__(self, attr): - return self[attr] - - -class LatticeFileItem: - """ """ - itemName = "" - itemType = '' - parameters = {} - properties = {} - alpars = {} - - def __init__(self, _line=''): - """ - Construct an object parsing a _line from a lattice file - """ - self.psparameters = {} - self.parameters = {} - self.properties = {} - - # find a name - colon_pos = _line.find(':') - self.itemName = _line[:colon_pos].lstrip().rstrip().upper() - - # what left to be parsed - line_left = _line[colon_pos + 1:].lstrip() - - # find a type - param_name = '' # the first item after a colon could be also a parameter name, like for a line element - eq_pos = line_left.find('=') - comma_pos = line_left.find(',') - # let it work even there are no parameters defined - only element type - if eq_pos < 0: eq_pos = len(line_left) - if comma_pos < 0: comma_pos = len(line_left) - # so, we could read an element type - self.itemType = line_left[:min(comma_pos, eq_pos)].rstrip().lower() - - # this is waiting to be processed: - line_left = line_left[comma_pos + 1:].lstrip() - - # if the element type is also parameter name state this - if eq_pos < comma_pos: param_name = self.itemType - - # parse the rest for parameters - while line_left != '': - if param_name != '': - # searching for a value - param_value = '' - if line_left[0] == '(': - # value in brackets (may contain commas) - bracket_pos = line_left.index(')', 1) # will rise an exception in case of badly formated line - # so, the value is (including brackets): - param_value = line_left[:bracket_pos + 1] - # this is what left to be parsed - line_left = line_left[bracket_pos + 1:].lstrip() - - elif line_left[0] == '\"': - # value in quotes (could contain commas) - quote_pos = line_left.index('\"', 1) # will rise an exception in case of badly formated line - # so, the value is (including quote): - param_value = line_left[:quote_pos + 1] - - # this is what left to be parsed - line_left = line_left[quote_pos + 1:].lstrip() - else: - # typical case - the value between an equal and a comma characters - comma_pos = line_left.find(',') - if comma_pos < 0: comma_pos = len(line_left) - # a value, here you are - param_value = line_left[:comma_pos].rstrip() - # the following left to be parsed - line_left = line_left[comma_pos + 1:].lstrip() - # store the parameter with the corresponding value - self.parameters[param_name] = param_value - # PJB reset name back to empty here to find next parameter!(to enter else below) - param_name = '' - else: - # searching for a parameter - eq_pos = line_left.find('=') - if eq_pos < 0: eq_pos = len(line_left) - # allow value-less parameters - comma_pos = line_left.find(',') - if comma_pos < 0: comma_pos = len(line_left) - # so we know where to find parameter name - param_name = line_left[:min(eq_pos, comma_pos)].rstrip().lower() - # if the parameter has no value add it directly to the dictionary - if comma_pos <= eq_pos: - self.parameters[param_name] = '' - param_name = '' - # this is what left to be parsed - line_left = line_left[min(eq_pos, comma_pos) + 1:].lstrip() - - def handle_circuit_name(self, itemName, endnum): - - endname = "" - # hack for bc1 - if "QD" in itemName and "BC1" in itemName: - endname = "CRQM-" + endnum - # - # hack for bc2 - elif "QF" in itemName and "BC2" in itemName and "3" not in itemName and "4" not in itemName and "5" not in itemName: - endname = "CRQM-" + endnum - elif "QF" in itemName and "BC2" in itemName and ("3" in itemName or "5" in itemName): - endname = "CRQ1-01" - elif "QF" in itemName and "BC2" in itemName and "4" in itemName: - endname = "CRQ2-01" - # - elif "Q" in itemName: - endname = "CRQ-" + endnum - elif "CO" in itemName and "X" in itemName: - endname = "CRCOX-" + endnum - elif "CO" in itemName and "Y" in itemName: - endname = "CRCOY-" + endnum - elif "DI" in itemName: - endname = "CRDI-" + endnum - print(("dealing with endname ", endname)) - elif "SX" in itemName: - endname = "CRSX-" + endnum - elif "SOL" in itemName: - endname = "CRSOL-" + endnum - elif "SM" in itemName: - endname = "CRSM-" + endnum - else: - sys.exit("Cannot convert circuit name" + itemName) - - if "/" in endname: # in case did not end with number, endname will be some */*/ by mistake - endname = endname.split("-")[0] + "-01" - - return endname - - def config_alarms(self, pyalarm, alname, alsev, aldesc, pyattname, key): - - alrec = alname + ":" + "HTML" - - if "AlarmList" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmList"] = [] - self.alpars[pyalarm]['AlarmList'].append(alname + ":" + pyattname + "/" + key) - - if "AlarmSeverities" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmSeverities"] = [] - self.alpars[pyalarm]['AlarmSeverities'].append(alsev) - - if "AlarmDescriptions" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmDescriptions"] = [] - self.alpars[pyalarm]['AlarmDescriptions'].append(aldesc) - - if "AlarmReceivers" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmReceivers"] = [] - self.alpars[pyalarm]['AlarmReceivers'].append(alrec) - - if "StartupDelay" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["StartupDelay"] = ["0"] - - if "AutoReset" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AutoReset"] = ["60"] - - if "MaxMessagesPerAlarm" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["MaxMessagesPerAlarm"] = ["1"] - - if "PollingPeriod" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["PollingPeriod"] = ["5"] - - if "LogFile" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["LogFile"] = ["/tmp/pjb/log"] - - if "HtmlFolder" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["HtmlFolder"] = ["/tmp/pjb"] - - if "AlarmThreshold" not in self.alpars[pyalarm]: - self.alpars[pyalarm]["AlarmThreshold"] = ["1"] - - def match_properties(self): - - devclass = types_classes[self.itemType] - - if "CIR" in self.itemName: - print("dealing with magnet circuit") - - # for given item type, look up required attributes and properties of tango - elif devclass in TANGO_PROPERTIES: - - fixed_properties_l = list(TANGO_PROPERTIES[devclass][0].keys()) - # print "fixed tango properties are ", fixed_properties_l - - # add the fixed properties - self.parameters.update(TANGO_PROPERTIES[devclass][0]) - - lattice_properties_l = list(TANGO_PROPERTIES[devclass][1].keys()) - # print "possible lattice tango properties are ", lattice_properties_l - for k in list(self.parameters.keys()): - # print "pjb zzz 1", self.parameters["Tilt"], self.parameters - # print "key", k - # if not a required property or attribue then pop it - if k.lower() not in lattice_properties_l and k not in fixed_properties_l: - # print "popping ", k - self.parameters.pop(k) - - # otherwise rename key if an attribute - if k.lower() in lattice_properties_l: - # print "KEY ", k.lower(), TANGO_PROPERTIES[devclass][1][k.lower()], self.parameters[k] - self.parameters[TANGO_PROPERTIES[devclass][1][k.lower()]] = [self.parameters.pop(k)] - # print "pjb zzz", self.parameters["Tilt"], self.parameters - - - else: - for k in list(self.parameters.keys()): - self.parameters.pop(k) - - if "MAG" in self.itemName and not "CIR" in self.itemName: - self.parameters["Type"] = [self.itemType] - - # - if "Tilt" in self.parameters: - if "0.5 pi" in str(self.parameters["Tilt"]): - self.parameters["Tilt"] = ["90"] - else: - self.parameters["Tilt"] = ["0"] - print(("pjbyyy", self.parameters)) - - def add_device(self, sdict, adict, psdict, - name_parsing_string='(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[0-9]+)'): - """ - Updates json file - """ - # prepare pattern for parsing name - pattern = re.compile(name_parsing_string) - - print(("In add device for item: " + self.itemName + " as " + self.itemType, self.alpars)) - # only when we know class for certain element - - print(("adict is ", adict)) - circuit_alarm_params = [] - - # for case with no final number like I.TR1.MAG.DIE (no .1 etc at end) - alt_name_parsing_string = '(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)\.(?P[a-zA-Z0-9]+)' - alt_pattern = re.compile(alt_name_parsing_string) - - # self.alpars = {} - devdictalarm = None - - if self.itemType in types_classes: - - print("") - print("") - - # split name - name_items = pattern.search(self.itemName) - parsed = False - tryagain = False - if name_items == None: - print(("Warning: Item name in lattice file doesn't match the naming convention.", self.itemName)) - tryagain = True - else: - parsed = True - if tryagain: - name_items = alt_pattern.search(self.itemName) - if name_items == None: - print(("Warning: Item name in lattice file STILL doesn't match the naming convention.", - self.itemName)) - else: - parsed = True - if parsed: - system = name_items.group('system') - subsystem = name_items.group('subsystem') - location = name_items.group('location') - device = name_items.group('device') - - if tryagain == False: - num = name_items.group('num') - else: - num = "" - - if num == None: num = '' - - if num != "": - num2 = "%02d" % int(num) - else: - num2 = "01" - - self.match_properties() - print(("pjbxxx ", self.parameters)) - - # create device for json output - name = (system + "-" + location + '/' + subsystem + '/' + device + "-" + num2).encode('ascii', 'ignore') - devclass = types_classes[self.itemType].encode('ascii', 'ignore') - server = devclass + '/' + system + "-" + location - - # hack for circuits - if "CIR" in self.itemName: - print(("orig name", self.itemName, name)) - - # quad circuits named like CRQ - # correctors like CRCOX and CRCOY - # dipoles like CRDI - # solenoid like CRSOL - # sextu like CRX - endname = name.rsplit("/", 1)[1] - - # fix circuit names - endname = self.handle_circuit_name(self.itemName, num2) - name = name.rsplit("/", 1)[0] + "/" + endname - - # e.g in G00 have COIX 1, 2, 3 and COHX 1 and 2, which would make COX 1, 2, 3 with COIX 1 and 2 being lost! - # increment number till unique - while name in cirlist: - print(("danger! already named this circuit!", self.itemName, name)) - suffix = int(name.split("-")[2]) + 1 - newnum2 = "%02d" % suffix - print(newnum2) - name = (name.rsplit("-", 1)[0]) + "-" + newnum2 - print(name) - print(("new name ", name)) - cirlist.append(name) - print(cirlist) - devclass = "MagnetCircuit" - - # compact name is to find tag in plc alarms - name_l_cir = self.itemName.split(".") - section_cir = name_l_cir[1] - mid = name_l_cir[3] - if "_" in mid: - mid = mid.split("_")[0] - # hack for SM1A and SM1B in TR1, also DIE, DIF on circuit DI (DIC, DID on TR3) - compactnamecir = "_" + name_l_cir[1] + mid + "_" - compactnamecir = compactnamecir.replace("1A", "1") - compactnamecir = compactnamecir.replace("1B", "1") - if "DIE" in compactnamecir: - compactnamecir = compactnamecir.replace("DIE", "DI") - if "DIF" in compactnamecir: - compactnamecir = compactnamecir.replace("DIF", "DI") - if "DIC" in compactnamecir: - compactnamecir = compactnamecir.replace("DIC", "DI") - if "DID" in compactnamecir: - compactnamecir = compactnamecir.replace("DID", "DI") - - print(("circuit compact name is ", compactnamecir, name_l_cir)) - - # fill alarm info for circuits - # - pyalarm = system + "-" + location + '/MAG/ALARM' - if adict is not None: - devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] - - print("init devdictalarm circuit") - - already_added = ["B_I_SP02DIPBD_DIA_TSW1_A", - "B_I_SP02DIPBD_DIA_TSW2_A"] # can use this to ignore to. e.g. SP02DIPDB doesnt end in a number but isn't a circuit! - for key in alarm_dict: - # compact name is like _TR1QF_ but some tags are like _TR1QF2_5_ - # if compactnamecir in key: - if (key not in already_added and compactnamecir in key) or ( - compactnamecir[:-1] in key and key.count("_") > 5 and key not in already_added and - "F" + num + "_" in key): - - print(("key is", num, key)) - - already_added.append(key) - - print(("FOUND ALARM INFO FOR CIR", compactnamecir, key, alarm_dict[key], section_cir)) - pyattname = "I-" + section_cir + "/DIA/COOLING" - - # for the magnets json file - circuit_alarm_params = [pyattname, key, alarm_dict[key]] - - # for the alarms json file - alname = 'TemperatureInterlock' + key.split('_')[ - -2] + '_' + system + "_" + location + '__' + endname.replace("-", "_") - alsev = alname + ":" + "ALARM" - alrec = alname + ":" + "HTML" - aldesc = alname + ":One magnet in circuit " + alarm_dict[key] - - if pyalarm not in self.alpars: - self.alpars[pyalarm] = {} - self.config_alarms(pyalarm, alname, alsev, aldesc, pyattname, key) # will fill self.alpars - - print(( - "+++++++++++++++++ Creating device server : " + server + " for " + devclass + " (name= " + name + ")")) - print(("+++++++++ With properties : ", self.parameters)) - - # Dont actually write attributes to DB, only properties - # see if this class exists and append if so, or create - devdict = sdict.servers["%s/%s" % (devclass, system + "-" + location)][devclass][name] - - # for circuit json only - if "CR" in name: - psdevdict = psdict.Circuits[name] - - if "MAG" in self.itemName and "CIR" not in self.itemName: - - # compact name is to find tag in plc alarms - name_l = self.itemName.split(".") - section = name_l[1] - del name_l[0] - del name_l[1] - # del name_l[2] - print(("pjb kkk", name_l)) - compactfullname = "".join(name_l) - compactname = compactfullname.split("_")[0] - compactname_nonum = compactfullname.split("_")[0][:-1] + "_" - - print(("-------------------- magnet not circuit", self.itemName, compactname)) - - # see what is the ps of the magnet - if name in POWER_SUPPLY_MAP: - powersupplyname = POWER_SUPPLY_MAP[name] - else: - print(("magnet not in PS map, skipping", name)) - return - - # !!!!!!!!!!! *********** create circuit device for each new ps ************!!!!!!!!!!!! - # copy the magnet and call recursively add device! - magnetcircuit = copy.deepcopy(self) - magnetcircuit.itemName = self.itemName + ".CIR" - - magnetcircuit.parameters = {} - magnetcircuit.parameters['PowerSupplyProxy'] = [powersupplyname] - magnetcircuit.parameters['MagnetProxies'] = [name] - magnetcircuit.parameters['RiseTime'] = ["0.0"] - magnetcircuit.parameters['ResistanceReference'] = ["0.0"] - magnetcircuit.parameters['CoilNames'] = [""] - - # for the ps json file only - magnetcircuit.psparameters['PowerSupplyProxy'] = [powersupplyname] - magnetcircuit.psparameters['MagnetProxies'] = [name] - - # get alarm info from excel - pyalarm = system + "-" + location + '/MAG/ALARM' - - print(("adict is again", adict)) - if adict is not None: - devdictalarm = adict.servers["%s/%s" % ("PyAlarm", "I-MAG")]["PyAlarm"][pyalarm] - print("init devdictalarm") - - # set alarms in magnet.json file and in alarms json file - for key in alarm_dict: - if compactname in key and key.count("_") < 6: - # if compactname in key: - - pyattname = "I-" + section + "/DIA/COOLING" - - print(("FOUND ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict)) - - # for the magnets json file - if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [ - pyattname + "," + key + "," + alarm_dict[key] - ] - else: - self.parameters['TemperatureInterlock'].append( - pyattname + "," + key + "," + alarm_dict[key]) - - # for the alarms json file - alname = 'TemperatureInterlock' + key.split('_')[ - -2] + '_' + system + "_" + location + '__' + 'MAG' + '__' + device + "_" + num2 - alsev = alname + ":" + "ALARM" - aldesc = alname + ":Magnet " + alarm_dict[key] - - if pyalarm not in self.alpars: - self.alpars[pyalarm] = {} - self.config_alarms(pyalarm, alname, alsev, aldesc, pyattname, key) # will fill self.alpars - - devdictalarm.properties = self.alpars[pyalarm] - - # set alarms in magnet.json file for all magnets in circuit - for key in alarm_dict: - if compactname_nonum in key or (compactname[:-1] in key and key.count("_") > 5 and ( - "F" + num + "_" in key or "_" + num + "_" in key)): - # if compactname_nonum in key: - print(("mag key ", key, compactname_nonum, compactname, "F" + num + "_", "_" + num + "_")) - pyattname = "I-" + section + "/DIA/COOLING" - - print(("FOUND MORE ALARM INFO FOR ", compactname, key, alarm_dict[key], pyattname, adict)) - - # for the magnets json file - if 'TemperatureInterlock' not in self.parameters: - self.parameters['TemperatureInterlock'] = [ - pyattname + "," + key + "," + alarm_dict[key]] - else: - self.parameters['TemperatureInterlock'].append( - pyattname + "," + key + "," + alarm_dict[key]) - - polarity = 1 - orientation = 1 - - # get calibration info from the excel - if self.itemName.split("_")[0] in calib_dict: - print(("FOUND CALIB INFO", self.itemName)) - # find max multipole expansions - dim = max(list(calib_dict[self.itemName.split("_")[0]].keys()), key=int) - - print(("--- max order is", dim)) - - # create arrays of this dimensions, other dimension is 11 - - fieldsmatrix = [[0 for x in range(19)] for x in range(dim)] - # print fieldsmatrix - currentsmatrix = [[0 for x in range(19)] for x in range(dim)] - - # iterate over keys and add to the array - for key in calib_dict[self.itemName.split("_")[0]]: - print(('--- ', key, 'corresponds to', calib_dict[self.itemName.split("_")[0]][key])) - currents = calib_dict[self.itemName.split("_")[0]][key][5:25] - fields = calib_dict[self.itemName.split("_")[0]][key][25:45] - print(('--- ', currents, fields)) - - fieldsmatrix[key - 1] = fields - currentsmatrix[key - 1] = currents - # key here is the multipole order. any one should have same polarity - polarity = calib_dict[self.itemName.split("_")[0]][key][3] - orientation = calib_dict[self.itemName.split("_")[0]][key][2] - # print "P, O", polarity, orientation - - print(('--- ', fieldsmatrix)) - print(('--- ', currentsmatrix)) - - # now trim the matrices (lists) - maxlength = 20 - for i, val in enumerate(fieldsmatrix[dim - 1]): - if val == '': - print((i, val)) - maxlength = i - break - print(maxlength) - for i in range(dim): - print(i) - del fieldsmatrix[i][maxlength:] - del currentsmatrix[i][maxlength:] - - print(('Now--- ', fieldsmatrix)) - print(('Now--- ', currentsmatrix)) - - magnetcircuit.parameters['ExcitationCurveCurrents'] = currentsmatrix - magnetcircuit.parameters['ExcitationCurveFields'] = fieldsmatrix - - self.parameters['Orientation'] = [str(int(orientation))] - self.parameters['Polarity'] = [str(int(polarity))] - - # assign circuit name as property of magnet device - # no regex to fix name here so do by hand - # e.g. I.BC1.MAG.COEX.4.CIR -> I-BC1/MAG/COEX-CIR-04 - - cname = name.rsplit("/CIR", 1)[0] - endname = cname.rsplit("/", 1)[1] - endnum = cname.rsplit("-", 1)[1] - endname = self.handle_circuit_name(self.itemName, endnum) - cname = cname.rsplit("/", 1)[0] + "/" + endname - - print(("cname is ", cname, name, powersupplyname, circuit_ps_list)) - - while cname in cirlist: - print(("danger2! already named this circuit!", cname)) - suffix = int(cname.split("-")[2]) + 1 - newnum2 = "%02d" % suffix - cname = (cname.rsplit("-", 1)[0]) + "-" + newnum2 - print(("new name ", cname)) - - # only add one circuit device per ps - if powersupplyname not in circuit_ps_list: - - magnetcircuit.add_device(sdict, adict, psdict) - circuit_ps_list[powersupplyname] = cname - - print(("adding circuit name ", magnetcircuit.itemName, cname, circuit_ps_list)) - self.parameters['CircuitProxies'] = [cname] - - else: - # if we aleady made this circuit device, add it to this magnet properties - print(("!!!ALART!!! already added a circuit device for ", self.itemName, name, system, location)) - - if system == "R3": - system = "I" - if location == "301L": - location = "TR3" - - self.parameters['CircuitProxies'] = [circuit_ps_list[powersupplyname]] - - # need to get the name of the circuit device from the ps dict though - print(("exiting circuit device is", circuit_ps_list[powersupplyname])) - - # print "current mags ", system+"-"+location - # print "current mags 2", sdict.servers - - current_mags = \ - sdict.servers["%s/%s" % ("MagnetCircuit", system + "-" + location)]["MagnetCircuit"][ - circuit_ps_list[powersupplyname]].properties - # for circuits json - print(("cir name from ps ", circuit_ps_list[powersupplyname], psdict)) - ps_current_mags = psdict.Circuits[circuit_ps_list[powersupplyname]].Properties - print(("current mags ", current_mags['MagnetProxies'])) - if name in current_mags['MagnetProxies']: - print(("circuit already has magnet ", name)) - else: - ps_current_mags['MagnetProxies'].append(name) - current_mags['MagnetProxies'].append(name) - - print(("magnets on cir ", current_mags['ExcitationCurveFields'], current_mags['MagnetProxies'], - len(current_mags['MagnetProxies']))) - # need to average the currents, even if already done so in excel (depends on field order) - if 'ExcitationCurveFields' in current_mags: - - assoc_field_m = current_mags['ExcitationCurveFields'] - this_field_m = fieldsmatrix - - assoc_curr_m = current_mags['ExcitationCurveCurrents'] - this_curr_m = currentsmatrix - - print(("field matrix assoc is ", assoc_field_m)) - print(("field matrix current is ", this_field_m)) - - print(("current matrix assoc is ", assoc_curr_m)) - print(("current matrix current is ", this_curr_m)) - - for i in range(dim): - print(i) - - # fix for CRSM take abs field values since opp sign - if circuit_ps_list[powersupplyname] in ["I-TR3/MAG/CRSM-01", "I-TR3/MAG/CRDI-01"]: - newFields = [(abs(x) * (len(current_mags['MagnetProxies']) - 1) + abs(y)) / len( - current_mags['MagnetProxies']) for y, x in - zip(this_field_m[i], assoc_field_m[i])] - else: - newFields = [(x * (len(current_mags['MagnetProxies']) - 1) + y) / len( - current_mags['MagnetProxies']) for y, x in - zip(this_field_m[i], assoc_field_m[i])] - - newCurrents = [(x * (len(current_mags['MagnetProxies']) - 1) + y) / len( - current_mags['MagnetProxies']) for y, x in zip(this_curr_m[i], assoc_curr_m[i])] - - print(("new fields ", newFields)) - print(("new currents ", newCurrents)) - current_mags['ExcitationCurveFields'][i] = newFields - print(("updated: ", current_mags['ExcitationCurveFields'])) - current_mags['ExcitationCurveCurrents'][i] = newCurrents - print(("updated: ", current_mags['ExcitationCurveCurrents'])) - - - else: - print("NOT A MAGNET") - - devdict.properties = self.parameters - - # for circuits json - if "CR" in name: - psdevdict.Properties = self.psparameters - - -class ElegantLatticeParser: - """ - Class for parsing an elegant lattice file. - """ - fileName = "" - file = None - items = [] - - def __init__(self, _fileName): - """ - Constructs a parser object. - - Keyword arguments: - _fileName -- the name of file to be parsed - """ - self.fileName = _fileName - self.file = io.open(_fileName) - - def parseLatticeFile(self): - """ """ - line = "" # this will be a line combined from lines to be connected - for ll in self.file: - l = ll.lstrip().rstrip() - if len(l) > 0: - if l[0] == '!': - pass # do not process comments - elif l[0] == '%': - pass # processing RPNs are not implemented - elif l[-1] == '&': - # Combine lines to be concated - line = line + ' ' + l[:-1] - else: - # So this is the last line to be combined - line = line + l - # Create an object and add it to list - self.items.append(LatticeFileItem(line.lstrip().rstrip())) - line = "" - - -if __name__ == '__main__': - - inFileName = '' - doCalib = False - doAlarms = False - excelName = 'MagnetCalibrationData.xls' - alarmName = 'IMAG_ALARM_140923_Magnets.xls' - - # define classes for lattice elements - types_classes = {} - types_classes["dip"] = "Magnet" - types_classes["sbend"] = "Magnet" - types_classes["sben"] = "Magnet" - types_classes["rben"] = "Magnet" - types_classes["csrcsbend"] = "Magnet" - types_classes["sole"] = "Magnet" - types_classes["kquad"] = "Magnet" - types_classes["ksext"] = "Magnet" - types_classes["hkick"] = "Magnet" - types_classes["vkick"] = "Magnet" - # types_classes["hkick"] = "VACorrector" - # types_classes["vkick"] = "VACorrector" - # types_classes["monitor"] = "VABPM" - # types_classes["watch"] = "VAYAGScreen" - # types_classes["rfcw"] = "VARfCavity" - # - circuit_ps_list = {} - # circuit_alarm_params = None - - # read arguments - for par in sys.argv[1:]: - if par[0:2] == '--': - if par == '--calibration-data': - doCalib = True - elif par == '--alarm-data': - doAlarms = True - # elif par == '--test-mode': - # test_mode = True - elif inFileName == '': - inFileName = par - - print((inFileName, doCalib)) - - if doAlarms or doCalib: - import xlrd - - alarm_dict = {} - if doAlarms: - print("opening alarms xls") - xls = xlrd.open_workbook(alarmName) - sheet = xls.sheet_by_name('Sheet1') - rows = [sheet.row_values(i) for i in range(sheet.nrows)] - column_names = rows[0] - print(("cols ", column_names)) - for row in enumerate(rows[1:]): - if row[1][0] == "": - continue - print((row[1][0], row[1][1])) - alarm_dict[row[1][0]] = row[1][1] - print(("DICT IS ", alarm_dict)) - - # make dict just for alarms! for pyalarm - json_dict_alarms = SuperDict() - else: - json_dict_alarms = None - - calib_dict = {} - - if doCalib: - # open excel sheet - xls = xlrd.open_workbook(excelName) - - for name in ["linac", "transfer 1,5 GeV", "transfer 3 GeV", "thermionic gun"]: - - # sheet = xls.sheet_by_name('Linac') - sheet = xls.sheet_by_name(name) - rows = [sheet.row_values(i) for i in range(sheet.nrows)] - column_names = rows[7] - print(("cols ", column_names)) - - for row in enumerate(rows[9:]): - print((row[1])) - if row[1][2] == "": - continue - # this is like - # [5.0, u'I.S01A', u'I.S01A.MAG.QE.1', 202005.0, u'#1168-10030-0001', 1.0, -1.0, 2.0, 6.3167, 5.6757, 5.0307500000000003, 4.4208999999999996, 3.8452999999999999, 3.1463999999999999, 2.5179624999999999, 1.8892374999999999, 1.2808725000000001, 0.63988750000000016, 0.0, 0.70470485548532313, 0.63908274382966312, 0.56946571499960408, 0.50203927491440703, 0.43686121069898298, 0.35966476443894108, 0.288993167760146, 0.21848942173091002, 0.14957521795596601, 0.077488874695939805, 0.0052044472873010797, u'T', u'Rotating coil-C1168, #0001.xls', u'https://alfresco.maxlab.lu.se/share/page/site/maxiv/document-details?nodeRef=workspace://SpacesStore/23cdc9d1-a01e-443e-b578-1538637a1472', u'Scanditronix Magnet', 40690.0, ''] - if row[1][2].strip() not in calib_dict: - if row[1][7] is not "": - data_key = int(row[1][7]) - data_list = row[1][3:48] - data_dict = {data_key: data_list} - calib_dict[row[1][2].strip()] = data_dict - # calib_dict[row[1][2]]=row[1][3:33] - else: - if row[1][7] is not "": - # we found more curves for the same magnet - print(("found another entry", row[1][2], row[1][7])) - data_key = int(row[1][7]) - data_list = row[1][3:48] - data_dict = {data_key: data_list} - calib_dict[row[1][2].strip()][data_key] = data_list - - print(("DICT IS ", calib_dict)) - - # create a parser for the file - parser = ElegantLatticeParser(inFileName) - - # parse the file - parser.parseLatticeFile() - parser.file.close() - - # make a json file - json_dict = SuperDict() - json_ps = SuperDict() - for item in parser.items: - item.add_device(json_dict, json_dict_alarms, json_ps) - # print json.dumps(json_dict, indent=4) - - # now we have the dict, loop over again and sort out magnets, power supplies and circuits - - print("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ") - print("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ") - print((json_dict.servers)) - - outfile = open('magnets.json', 'w') - - # have final json dict here - print("THE FINAL DICT") - topl = list(json_dict['servers'].keys()) - for item in topl: - if "Circuit" in item: - # print json_dict['servers'][item] - for cir in json_dict['servers'][item]: - # print json_dict['servers'][item][cir]['ExcitationCurveCurrents'] - # print json_dict['servers'][item]["MagnetCircuit"] - for c in json_dict['servers'][item]["MagnetCircuit"]: - for key in json_dict['servers'][item]["MagnetCircuit"][c]["properties"]: - if key == "ExcitationCurveCurrents": - ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print((key, [str(x) for x in ls])) - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] - if key == "ExcitationCurveFields": - ls = json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] - print((key, [str(x) for x in ls])) - json_dict['servers'][item]["MagnetCircuit"][c]["properties"][key] = [str(x) for x in ls] - - json.dump(json_dict, outfile, indent=4) - - if doAlarms: - outfile2 = open('magnets_alarms.json', 'w') - json.dump(json_dict_alarms, outfile2, indent=4) - - # !! note that item has parameters, but only want to extract those needed for tango! - - # dump ps circuit info - outfile3 = open('circuits.json', 'w') - json.dump(json_ps, outfile3, indent=4) From b308e95b5766e395f55c9125e4242426fc77e4a8 Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Thu, 4 Jun 2020 14:28:48 +0200 Subject: [PATCH 21/22] Add python_requires in setup.py. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 290e172..b730aec 100755 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ packages=["dsconfig", "dsconfig.appending_dict"], description="Library and utilities for Tango device configuration.", # Requirements + python_requires='~=3.6', setup_requires=["setuptools", "pytest-runner"], install_requires=["jsonpatch>=1.13", "jsonschema", "xlrd", "PyTango"], tests_require=["pytest", "pytest-cov", "Faker", "mock"], From cc2874fb1643bb6a34102f0ea04ca1e9cd0fcd9c Mon Sep 17 00:00:00 2001 From: Wojciech Kitka Date: Thu, 4 Jun 2020 16:18:59 +0200 Subject: [PATCH 22/22] Fix tests. --- dsconfig/appending_dict/test_appendingdict.py | 38 +++++++++---------- test/test_utils.py | 6 ++- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/dsconfig/appending_dict/test_appendingdict.py b/dsconfig/appending_dict/test_appendingdict.py index f64aa07..3d8671b 100644 --- a/dsconfig/appending_dict/test_appendingdict.py +++ b/dsconfig/appending_dict/test_appendingdict.py @@ -31,52 +31,52 @@ def test_init_tiny(self): def test_init_flat(self): FLAT_DICT = {"a": 1, "b": 2, "c": 3} sd = SetterDict(FLAT_DICT) - self.assertDictEqual(sd, FLAT_DICT) + self.assertDictEqual(sd.to_dict(), FLAT_DICT) def test_init_nested(self): NESTED_DICT = {"a": {"b": 2}} sd = SetterDict(NESTED_DICT) - self.assertDictEqual(sd, NESTED_DICT) + self.assertDictEqual(sd.to_dict(), NESTED_DICT) def test_init_blended(self): BLENDED_DICT = {"a": 1, "b": {"c": 3}} sd = SetterDict(BLENDED_DICT) - self.assertDictEqual(sd, BLENDED_DICT) + self.assertDictEqual(sd.to_dict(), BLENDED_DICT) def test_init_deep(self): DEEP_DICT = {"a": {"b": {"c": {"d": 4}}}} sd = SetterDict(DEEP_DICT) - self.assertDictEqual(sd, DEEP_DICT) + self.assertDictEqual(sd.to_dict(), DEEP_DICT) def test_init_deep(self): COMPLEX_DICT = {"a": {"b": {"c": {"d": 4}}}, "e": 5} sd = SetterDict(COMPLEX_DICT) - self.assertDictEqual(sd, COMPLEX_DICT) + self.assertDictEqual(sd.to_dict(), COMPLEX_DICT) def test_setting(self): sd = SetterDict() sd["foo"] = 1 - self.assertDictEqual(sd, {"foo": 1}) + self.assertDictEqual(sd.to_dict(), {"foo": 1}) def test_setting_nested(self): sd = SetterDict() sd["foo"]["bar"] = 2 - self.assertDictEqual(sd, {"foo": {"bar": 2}}) + self.assertDictEqual(sd.to_dict(), {"foo": {"bar": 2}}) def test_setting_nested_nonempty(self): sd = SetterDict({"a": 1}) sd["foo"]["bar"] = 2 - self.assertDictEqual(sd, {"a": 1, "foo": {"bar": 2}}) + self.assertDictEqual(sd.to_dict(), {"a": 1, "foo": {"bar": 2}}) def test_setting_attr(self): sd = SetterDict({"a": 1}) sd.a = 2 - self.assertDictEqual(sd, {"a": 2}) + self.assertDictEqual(sd.to_dict(), {"a": 2}) def test_setting_attr_deep(self): sd = SetterDict() sd.a.b.c = 4 - self.assertDictEqual(sd, {"a": {"b": {"c": 4}}}) + self.assertDictEqual(sd.to_dict(), {"a": {"b": {"c": 4}}}) def test_to_dict(self): orig = {"a": {"b": ["3"], "c": {"d": ["4"]}, "e": ["1"]}} @@ -103,38 +103,38 @@ class AppendingDictTestCase(unittest.TestCase): def test_basic_appending(self): ad = AppendingDict() ad["a"] = 1 - self.assertDictEqual(ad, {"a": ['1']}) + self.assertDictEqual(ad.to_dict(), {"a": ['1']}) ad["a"] = 2 - self.assertDictEqual(ad, {"a": ['1', '2']}) + self.assertDictEqual(ad.to_dict(), {"a": ['1', '2']}) def test_deep_appending(self): ad = AppendingDict() ad["a"]["b"]["c"] = 1 ad["a"]["b"]["c"] = 2 print((type(ad["a"]["b"]))) - self.assertDictEqual(ad, {"a": {"b": {"c": ['1', '2']}}}) + self.assertDictEqual(ad.to_dict(), {"a": {"b": {"c": ['1', '2']}}}) def test_initial_setting_with_dict(self): ad = AppendingDict() ad.a = {"b": {"c": 1}} - self.assertDictEqual(ad, {"a": {"b": {"c": ["1"]}}}) + self.assertDictEqual(ad.to_dict(), {"a": {"b": {"c": ["1"]}}}) def test_deep_setting_with_dict(self): ad = AppendingDict() ad.a.b.c = 1 ad.a = {"b": {"d": 2}} - self.assertDictEqual(ad, {"a": {"b": {"c": ['1'], "d": ['2']}}}) + self.assertDictEqual(ad.to_dict(), {"a": {"b": {"c": ['1'], "d": ['2']}}}) def test_setting_with_sequence(self): ad = AppendingDict() ad.a = [1, "b"] - self.assertDictEqual(ad, {"a": ['1', 'b']}) + self.assertDictEqual(ad.to_dict(), {"a": ['1', 'b']}) def test_setting_existing_key_with_sequence(self): ad = AppendingDict() ad.a = [1, "b"] ad.a = [2, "c"] - self.assertDictEqual(ad, {"a": ['1', 'b', '2', 'c']}) + self.assertDictEqual(ad.to_dict(), {"a": ['1', 'b', '2', 'c']}) def test_error_setting_existing_subtree_with_scalar(self): ad = AppendingDict() @@ -151,7 +151,7 @@ def test_setting_with_appendingdict(self): ad.a.e = 1 ad.a = ad2 self.assertDictEqual( - ad, {"a": {"b": ["3"], "c": {"d": ["4"]}, "e": ["1"]}}) + ad.to_dict(), {"a": {"b": ["3"], "c": {"d": ["4"]}, "e": ["1"]}}) def test_updating_does_not_work(self): """ @@ -164,7 +164,7 @@ def test_updating_does_not_work(self): def test_set_string_value(self): ad = AppendingDict() ad.a = "abc" - self.assertDictEqual(ad, {"a": ["abc"]}) + self.assertDictEqual(ad.to_dict(), {"a": ["abc"]}) def test_to_dict(self): orig = {"a": {"b": ["3"], "c": {"d": ["4"]}, "e": ["1"]}} diff --git a/test/test_utils.py b/test/test_utils.py index ad367c2..79e95ad 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -58,8 +58,10 @@ def test_immutable(self): def test_print_diff(capsys): - test_str = ('[{"path": "/a", "op": "remove"}, {"path": "/c", "value": 4, "op": "add"}' - ', {"path": "/b", "value": 3, "op": "replace"}]') + test_str = ('[{"op": "remove", "path": "/a"}, {"op": "add", "path": "/c", "value": 4}' + ', {"op": "replace", "path": "/b", "value": 3}]') + print(test_str) + print(str(print_diff({'a': 1, 'b': 2}, {'b': 3, 'c': 4}))) assert test_str == str(print_diff({'a': 1, 'b': 2}, {'b': 3, 'c': 4})) captured = capsys.readouterr() assert "REMOVE:\n > a" in captured.out