From 2527e0e2acd0f0f5080eae3eb62a49896da01c18 Mon Sep 17 00:00:00 2001
From: Kristofer Hallin <kristofer@sunet.se>
Date: Fri, 6 Mar 2020 13:40:15 +0100
Subject: [PATCH 001/102] When removing the device we now have the option to do
 a factory reset and reboot it as well. Somewhat dangerous, thereefor we can
 only do it on access switches and only on one device.

---
 docs/apiref/devices.rst         |  9 ++++
 src/cnaas_nms/api/device.py     | 28 +++++++---
 src/cnaas_nms/confpush/erase.py | 93 +++++++++++++++++++++++++++++++++
 3 files changed, 122 insertions(+), 8 deletions(-)
 create mode 100644 src/cnaas_nms/confpush/erase.py

diff --git a/docs/apiref/devices.rst b/docs/apiref/devices.rst
index e61533cb..dda137fb 100644
--- a/docs/apiref/devices.rst
+++ b/docs/apiref/devices.rst
@@ -160,6 +160,15 @@ To remove a device, pass the device ID in a DELTE call:
    curl -X DELETE https://hostname/api/v1.0/device/10
 
 
+There is also the option to factory default and reboott the device
+when removing it. This can be done like this:
+
+::
+
+   curl -H "Content-Type: application/json" -X DELETE -d
+   '{"factory_default": true}' https://hostname/api/v1.0/device/10
+
+
 Preview config
 --------------
 
diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 9bc0b283..dbcd547b 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -90,14 +90,26 @@ def get(self, device_id):
     @jwt_required
     def delete(self, device_id):
         """ Delete device from ID """
-        with sqla_session() as session:
-            dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none()
-            if dev:
-                session.delete(dev)
-                session.commit()
-                return empty_result(status="success", data={"deleted_device": dev.as_dict()}), 200
-            else:
-                return empty_result('error', "Device not found"), 404
+        json_data = request.get_json()
+
+        if 'factory_default' in json_data:
+            if isinstance(json_data['factory_default'], bool) and json_data['factory_default'] is True:
+                scheduler = Scheduler()
+                job_id = scheduler.add_onetime_job(
+                    'cnaas_nms.confpush.erase:device_erase',
+                    when=1,
+                    scheduled_by=get_jwt_identity(),
+                    kwargs={'device_id': device_id})
+                return empty_result(data='Scheduled job {} to factory default device'.format(job_id))
+        else:
+            with sqla_session() as session:
+                dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none()
+                if dev:
+                    session.delete(dev)
+                    session.commit()
+                    return empty_result(status="success", data={"deleted_device": dev.as_dict()}), 200
+                else:
+                    return empty_result('error', "Device not found"), 404
 
     @jwt_required
     @device_api.expect(device_model)
diff --git a/src/cnaas_nms/confpush/erase.py b/src/cnaas_nms/confpush/erase.py
new file mode 100644
index 00000000..04c8a2d6
--- /dev/null
+++ b/src/cnaas_nms/confpush/erase.py
@@ -0,0 +1,93 @@
+import cnaas_nms.confpush.nornir_helper
+
+from cnaas_nms.tools.log import get_logger
+from cnaas_nms.scheduler.scheduler import Scheduler
+from cnaas_nms.scheduler.wrapper import job_wrapper
+from cnaas_nms.confpush.nornir_helper import NornirJobResult
+from cnaas_nms.db.session import sqla_session
+from cnaas_nms.db.device import DeviceType, Device
+
+from nornir.plugins.functions.text import print_result
+from nornir.plugins.tasks.networking import netmiko_send_command
+
+
+logger = get_logger()
+
+
+def device_erase_task(task, hostname: str) -> str:
+    try:
+        res = task.run(netmiko_send_command, command_string='enable',
+                       expect_string='.*#',
+                       name='Enable')
+        print_result(res)
+
+        res = task.run(netmiko_send_command,
+                       command_string='write erase now',
+                       expect_string='.*#',
+                       name='Write rase')
+        print_result(res)
+    except Exception as e:
+        logger.info('Failed to factory default device {}, reason: {}'.format(
+            task.host.name, e))
+        raise Exception('Factory default device')
+
+    try:
+        res = task.run(netmiko_send_command, command_string='reload force',
+                       max_loops=2,
+                       expect_string='.*')
+        print_result(res)
+    except Exception as e:
+        pass
+
+    return "Device factory defaulted"
+
+
+@job_wrapper
+def device_erase(device_id: int = None, job_id: int = None) -> NornirJobResult:
+
+    with sqla_session() as session:
+        dev: Device = session.query(Device).filter(Device.id ==
+                                                   device_id).one_or_none()
+        if dev:
+            hostname = dev.hostname
+            device_type = dev.device_type
+        else:
+            raise Exception('Could not find a device with ID {}'.format(
+                device_id))
+
+    if device_type != DeviceType.ACCESS:
+        raise Exception('Can only do factory default on access')
+
+    nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
+    nr_filtered = nr.filter(name=hostname).filter(managed=True)
+
+    device_list = list(nr_filtered.inventory.hosts.keys())
+    logger.info("Device selected: {}".format(
+        device_list
+    ))
+
+    try:
+        nrresult = nr_filtered.run(task=device_erase_task,
+                                   hostname=hostname)
+        print_result(nrresult)
+    except Exception as e:
+        logger.exception('Exception while erasing device: {}'.format(
+            str(e)))
+        return NornirJobResult(nrresult=nrresult)
+
+    failed_hosts = list(nrresult.failed_hosts.keys())
+    for hostname in failed_hosts:
+        logger.error("Failed to factory default device '{}' failed".format(
+            hostname))
+
+    if nrresult.failed:
+        logger.error("Factory default failed")
+
+    if failed_hosts == []:
+        with sqla_session() as session:
+            dev: Device = session.query(Device).filter(Device.id ==
+                                                       device_id).one_or_none()
+            session.delete(dev)
+            session.commit()
+
+    return NornirJobResult(nrresult=nrresult)

From 9914145a52ead959bf64f4bd36c0ff2cdaeecc4c Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 18 Mar 2020 16:30:51 +0100
Subject: [PATCH 002/102] Add new interface type MLAG_PEER for the peer-link
 connecting two MLAG/MC-LAG devices

---
 src/cnaas_nms/db/interface.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cnaas_nms/db/interface.py b/src/cnaas_nms/db/interface.py
index eae0bccf..25143889 100644
--- a/src/cnaas_nms/db/interface.py
+++ b/src/cnaas_nms/db/interface.py
@@ -17,6 +17,7 @@ class InterfaceConfigType(enum.Enum):
     CONFIGFILE = 2
     CUSTOM = 3
     TEMPLATE = 4
+    MLAG_PEER = 5
     ACCESS_AUTO = 10
     ACCESS_UNTAGGED = 11
     ACCESS_TAGGED = 12

From cf826ea2582010556e60f01af9114d04a6c42ca8 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 18 Mar 2020 16:31:38 +0100
Subject: [PATCH 003/102] Start working on extending device_init API and Device
 helper functions for MLAG access switch pairs

---
 src/cnaas_nms/api/device.py            | 24 ++++++++++++++++++++++--
 src/cnaas_nms/confpush/sync_devices.py |  2 +-
 src/cnaas_nms/db/device.py             | 23 ++++++++++++++++++++++-
 3 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index c29fcce8..05f85a75 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -219,14 +219,34 @@ def post(self, device_id: int):
             if not DeviceType.has_name(device_type):
                 return empty_result(status='error', data="Invalid 'device_type' provided"), 400
 
+        job_kwargs = {
+            'device_id': device_id,
+            'new_hostname': new_hostname
+        }
+
+        if 'mlag_peer_id' in json_data or 'mlag_peer_hostname' in json_data:
+            if 'mlag_peer_id' not in json_data or 'mlag_peer_hostname' not in json_data:
+                return empty_result(
+                    status='error',
+                    data="Both 'mlag_peer_id' and 'mlag_peer_hostname' must be specified"), 400
+            if not isinstance(json_data['mlag_peer_id'], int):
+                return empty_result(status='error', data="'mlag_peer_id' must be an integer"), 400
+            if not Device.valid_hostname(json_data['mlag_peer_hostname']):
+                return empty_result(
+                    status='error',
+                    data="Provided 'mlag_peer_hostname' is not valid"), 400
+            job_kwargs['mlag_peer_id'] = json_data['mlag_peer_id']
+            job_kwargs['mlag_peer_hostname'] = json_data['mlag_peer_hostname']
+
         if device_type == DeviceType.ACCESS.name:
             scheduler = Scheduler()
             job_id = scheduler.add_onetime_job(
                 'cnaas_nms.confpush.init_device:init_access_device_step1',
                 when=1,
                 scheduled_by=get_jwt_identity(),
-                kwargs={'device_id': device_id,
-                        'new_hostname': new_hostname})
+                kwargs=job_kwargs)
+        else:
+            return empty_result(status='error', data="Unsupported 'device_type' provided"), 400
 
         res = empty_result(data=f"Scheduled job to initialize device_id { device_id }")
         res['job_id'] = job_id
diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 973b06f8..f935acf4 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -116,7 +116,7 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False,
         }
 
         if devtype == DeviceType.ACCESS:
-            neighbor_hostnames = dev.get_uplink_peers(session)
+            neighbor_hostnames = dev.get_uplink_peer_hostnames(session)
             if not neighbor_hostnames:
                 raise Exception("Could not find any uplink neighbors for device {}".format(
                     hostname))
diff --git a/src/cnaas_nms/db/device.py b/src/cnaas_nms/db/device.py
index 81cfd4ab..ccf8a73d 100644
--- a/src/cnaas_nms/db/device.py
+++ b/src/cnaas_nms/db/device.py
@@ -187,7 +187,7 @@ def get_neighbor_ip(self, session, peer_device: Device):
         elif linknet.device_b_id == self.id:
             return linknet.device_a_ip
 
-    def get_uplink_peers(self, session):
+    def get_uplink_peer_hostnames(self, session) -> List[str]:
         intfs = session.query(Interface).filter(Interface.device == self).\
             filter(Interface.configtype == InterfaceConfigType.ACCESS_UPLINK).all()
         peer_hostnames = []
@@ -197,6 +197,27 @@ def get_uplink_peers(self, session):
                 peer_hostnames.append(intf.data['neighbor'])
         return peer_hostnames
 
+    def get_mlag_peers(self, session) -> List[Device]:
+        intfs = session.query(Interface).filter(Interface.device == self). \
+            filter(Interface.configtype == InterfaceConfigType.MLAG_PEER).all()
+        peers: List[Device] = []
+        linknets = self.get_linknets(session)
+        intf: Interface = Interface()
+        for intf in intfs:
+            for linknet in linknets:
+                if linknet.device_a == self and linknet.device_a_port == intf.name:
+                    peers.append(linknet.device_b)
+                elif linknet.device_b == self and linknet.device_b_port == intf.name:
+                    peers.append(linknet.device_a)
+        if len(peers) > 1:
+            raise DeviceException("More than one MLAG peer found: {}".format(
+                [x.hostname for x in peers]
+            ))
+        elif len(peers) == 1:
+            if self.device_type != peers[0].device_type:
+                raise DeviceException("MLAG peers are not the same device type")
+        return peers
+
     @classmethod
     def valid_hostname(cls, hostname: str) -> bool:
         if not isinstance(hostname, str):

From 2b941e445eda8b717a8ba4633ac02539c614ea27 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 19 Mar 2020 11:00:17 +0100
Subject: [PATCH 004/102] Restructure init pre checks to prepare for access
 switch mlag init

---
 src/cnaas_nms/confpush/get.py            | 112 +++++++++++------------
 src/cnaas_nms/confpush/init_device.py    |  47 ++++++----
 src/cnaas_nms/confpush/tests/test_get.py |   6 +-
 src/cnaas_nms/version.py                 |   2 +-
 4 files changed, 89 insertions(+), 78 deletions(-)

diff --git a/src/cnaas_nms/confpush/get.py b/src/cnaas_nms/confpush/get.py
index e4c0c405..92712fd2 100644
--- a/src/cnaas_nms/confpush/get.py
+++ b/src/cnaas_nms/confpush/get.py
@@ -206,7 +206,7 @@ def update_inventory(hostname: str, site='default') -> dict:
         return diff
 
 
-def update_linknets(hostname):
+def update_linknets(session, hostname):
     """Update linknet data for specified device using LLDP neighbor data.
     """
     logger = get_logger()
@@ -217,60 +217,60 @@ def update_linknets(hostname):
 
     ret = []
 
-    with sqla_session() as session:
-        local_device_inst = session.query(Device).filter(Device.hostname == hostname).one()
-        logger.debug("Updating linknets for device {} ...".format(local_device_inst.id))
-
-        for local_if, data in neighbors.items():
-            logger.debug(f"Local: {local_if}, remote: {data[0]['hostname']} {data[0]['port']}")
-            remote_device_inst = session.query(Device).\
-                filter(Device.hostname == data[0]['hostname']).one_or_none()
-            if not remote_device_inst:
-                logger.info(f"Unknown connected device: {data[0]['hostname']}")
+    local_device_inst = session.query(Device).filter(Device.hostname == hostname).one()
+    logger.debug("Updating linknets for device {} ...".format(local_device_inst.id))
+
+    for local_if, data in neighbors.items():
+        logger.debug(f"Local: {local_if}, remote: {data[0]['hostname']} {data[0]['port']}")
+        remote_device_inst = session.query(Device).\
+            filter(Device.hostname == data[0]['hostname']).one_or_none()
+        if not remote_device_inst:
+            logger.info(f"Unknown connected device: {data[0]['hostname']}")
+            continue
+        logger.debug(f"Remote device found, device id: {remote_device_inst.id}")
+
+        # Check if linknet object already exists in database
+        local_devid = local_device_inst.id
+        check_linknet = session.query(Linknet).\
+            filter(
+                ((Linknet.device_a_id == local_devid) & (Linknet.device_a_port == local_if))
+                |
+                ((Linknet.device_b_id == local_devid) & (Linknet.device_b_port == local_if))
+                |
+                ((Linknet.device_a_id == remote_device_inst.id) &
+                 (Linknet.device_a_port == data[0]['port']))
+                |
+                ((Linknet.device_b_id == remote_device_inst.id) &
+                 (Linknet.device_b_port == data[0]['port']))
+            ).one_or_none()
+        if check_linknet:
+            logger.debug(f"Found entry: {check_linknet.id}")
+            if (
+                    (       check_linknet.device_a_id == local_devid
+                        and check_linknet.device_a_port == local_if
+                        and check_linknet.device_b_id == remote_device_inst.id
+                        and check_linknet.device_b_port == data[0]['port']
+                    )
+                    or
+                    (       check_linknet.device_a_id == local_devid
+                        and check_linknet.device_a_port == local_if
+                        and check_linknet.device_b_id == remote_device_inst.id
+                        and check_linknet.device_b_port == data[0]['port']
+                    )
+            ):
+                # All info is the same, no update required
                 continue
-            logger.debug(f"Remote device found, device id: {remote_device_inst.id}")
-
-            # Check if linknet object already exists in database
-            local_devid = local_device_inst.id
-            check_linknet = session.query(Linknet).\
-                filter(
-                    ((Linknet.device_a_id == local_devid) & (Linknet.device_a_port == local_if))
-                    |
-                    ((Linknet.device_b_id == local_devid) & (Linknet.device_b_port == local_if))
-                    |
-                    ((Linknet.device_a_id == remote_device_inst.id) &
-                     (Linknet.device_a_port == data[0]['port']))
-                    |
-                    ((Linknet.device_b_id == remote_device_inst.id) &
-                     (Linknet.device_b_port == data[0]['port']))
-                ).one_or_none()
-            if check_linknet:
-                logger.debug(f"Found entry: {check_linknet.id}")
-                if (
-                        (       check_linknet.device_a_id == local_devid
-                            and check_linknet.device_a_port == local_if
-                            and check_linknet.device_b_id == remote_device_inst.id
-                            and check_linknet.device_b_port == data[0]['port']
-                        )
-                        or
-                        (       check_linknet.device_a_id == local_devid
-                            and check_linknet.device_a_port == local_if
-                            and check_linknet.device_b_id == remote_device_inst.id
-                            and check_linknet.device_b_port == data[0]['port']
-                        )
-                ):
-                    # All info is the same, no update required
-                    continue
-                else:
-                    # TODO: update instead of delete+new insert?
-                    session.delete(check_linknet)
-                    session.commit()
-
-            new_link = Linknet()
-            new_link.device_a = local_device_inst
-            new_link.device_a_port = local_if
-            new_link.device_b = remote_device_inst
-            new_link.device_b_port = data[0]['port']
-            session.add(new_link)
-            ret.append(new_link.as_dict())
+            else:
+                # TODO: update instead of delete+new insert?
+                session.delete(check_linknet)
+                session.commit()
+
+        new_link = Linknet()
+        new_link.device_a = local_device_inst
+        new_link.device_a_port = local_if
+        new_link.device_b = remote_device_inst
+        new_link.device_b_port = data[0]['port']
+        session.add(new_link)
+        ret.append(new_link.as_dict())
+    session.commit()
     return ret
diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 1d828e03..343d52d5 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -83,8 +83,30 @@ def push_base_management_access(task, device_variables, job_id):
              )
 
 
+def pre_init_checks(session, device_id) -> Device:
+    # Check that we can find device and that it's in the correct state to start init
+    dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none()
+    if not dev:
+        raise ValueError(f"No device with id {device_id} found")
+    if dev.state != DeviceState.DISCOVERED:
+        raise DeviceStateException("Device must be in state DISCOVERED to begin init")
+    old_hostname = dev.hostname
+    # Perform connectivity check
+    nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
+    nr_old_filtered = nr.filter(name=old_hostname)
+    try:
+        nrresult_old = nr_old_filtered.run(task=networking.napalm_get, getters=["facts"])
+    except Exception as e:
+        raise ConnectionCheckError(f"Failed to connect to device_id {device_id}: {str(e)}")
+    if nrresult_old.failed:
+        print_result(nrresult_old)
+        raise ConnectionCheckError(f"Failed to connect to device_id {device_id}")
+
+
 @job_wrapper
 def init_access_device_step1(device_id: int, new_hostname: str,
+                             mlag_peer_id: Optional[int],
+                             mlag_peer_hostname: Optional[str],
                              job_id: Optional[str] = None,
                              scheduled_by: Optional[str] = None) -> NornirJobResult:
     """Initialize access device for management by CNaaS-NMS
@@ -102,29 +124,14 @@ def init_access_device_step1(device_id: int, new_hostname: str,
         DeviceStateException
     """
     logger = get_logger()
-    # Check that we can find device and that it's in the correct state to start init
     with sqla_session() as session:
-        dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none()
-        if not dev:
-            raise ValueError(f"No device with id {device_id} found")
-        if dev.state != DeviceState.DISCOVERED:
-            raise DeviceStateException("Device must be in state DISCOVERED to begin init")
+        dev = pre_init_checks(session, device_id)
         old_hostname = dev.hostname
-    # Perform connectivity check
-    nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
-    nr_old_filtered = nr.filter(name=old_hostname)
-    try:
-        nrresult_old = nr_old_filtered.run(task=networking.napalm_get, getters=["facts"])
-    except Exception as e:
-        raise ConnectionCheckError(f"Failed to connect to device_id {device_id}: {str(e)}")
-    if nrresult_old.failed:
-        print_result(nrresult_old)
-        raise ConnectionCheckError(f"Failed to connect to device_id {device_id}")
 
-    cnaas_nms.confpush.get.update_linknets(old_hostname)
-    uplinks = []
-    neighbor_hostnames = []
-    with sqla_session() as session:
+        cnaas_nms.confpush.get.update_linknets(old_hostname)
+        uplinks = []
+        neighbor_hostnames = []
+
         # Find management domain to use for this access switch
         dev = session.query(Device).filter(Device.hostname == old_hostname).one()
         for neighbor_d in dev.get_neighbors(session):
diff --git a/src/cnaas_nms/confpush/tests/test_get.py b/src/cnaas_nms/confpush/tests/test_get.py
index aec4d645..79851bd6 100644
--- a/src/cnaas_nms/confpush/tests/test_get.py
+++ b/src/cnaas_nms/confpush/tests/test_get.py
@@ -6,6 +6,8 @@
 import yaml
 import os
 
+from cnaas_nms.db.session import sqla_session
+
 class GetTests(unittest.TestCase):
     def setUp(self):
         data_dir = pkg_resources.resource_filename(__name__, 'data')
@@ -33,8 +35,10 @@ def test_update_inventory(self):
         pprint.pprint(diff)
 
     def test_update_links(self):
-        new_links = cnaas_nms.confpush.get.update_linknets(self.testdata['update_hostname'])
+        with sqla_session() as session:
+            new_links = cnaas_nms.confpush.get.update_linknets(session, self.testdata['update_hostname'])
         pprint.pprint(new_links)
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/src/cnaas_nms/version.py b/src/cnaas_nms/version.py
index 6b998fcf..02a84bff 100644
--- a/src/cnaas_nms/version.py
+++ b/src/cnaas_nms/version.py
@@ -1,3 +1,3 @@
-__version__ = '1.0.0'
+__version__ = '1.1.0dev0'
 __version_info__ = tuple([field for field in __version__.split('.')])
 __api_version__ = 'v1.0'

From 3947f096ac1077651085fb97269c5171c8fccd6a Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 19 Mar 2020 16:54:05 +0100
Subject: [PATCH 005/102] Restructure get_uplinks since we don't need old
 return data format anymore (because previous updates done in sync_device).
 Create a new get_mlag_ifs based on get_uplinks. Update interfacedb in first
 step of init with intention of using database info to populate template
 variables. Restructure update_interfacedb to account for mlag_peers. Start
 doing some consistency checks for mlag device pairs in access_init.

---
 src/cnaas_nms/api/device.py            |  2 +-
 src/cnaas_nms/confpush/get.py          | 33 ++++++---
 src/cnaas_nms/confpush/init_device.py  | 35 ++++++----
 src/cnaas_nms/confpush/sync_devices.py |  2 +-
 src/cnaas_nms/confpush/update.py       | 94 ++++++++++++++------------
 5 files changed, 99 insertions(+), 67 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 7b3b8bf3..6cdac53b 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -236,7 +236,7 @@ def post(self, device_id: int):
                     status='error',
                     data="Provided 'mlag_peer_hostname' is not valid"), 400
             job_kwargs['mlag_peer_id'] = json_data['mlag_peer_id']
-            job_kwargs['mlag_peer_hostname'] = json_data['mlag_peer_hostname']
+            job_kwargs['mlag_peer_new_hostname'] = json_data['mlag_peer_hostname']
 
         if device_type == DeviceType.ACCESS.name:
             scheduler = Scheduler()
diff --git a/src/cnaas_nms/confpush/get.py b/src/cnaas_nms/confpush/get.py
index 92712fd2..633814a8 100644
--- a/src/cnaas_nms/confpush/get.py
+++ b/src/cnaas_nms/confpush/get.py
@@ -93,23 +93,40 @@ def get_neighbors(hostname: Optional[str] = None, group: Optional[str] = None)\
     return result
 
 
-def get_uplinks(session, hostname: str) -> Tuple[List, List]:
+def get_uplinks(session, hostname: str) -> dict[str, str]:
+    """Returns dict with mapping of interface -> neighbor hostname"""
     logger = get_logger()
     # TODO: check if uplinks are already saved in database?
-    uplinks = []
-    neighbor_hostnames = []
+    uplinks = {}
 
     dev = session.query(Device).filter(Device.hostname == hostname).one()
     for neighbor_d in dev.get_neighbors(session):
         if neighbor_d.device_type == DeviceType.DIST:
             local_if = dev.get_neighbor_local_ifname(session, neighbor_d)
             if local_if:
-                uplinks.append({'ifname': local_if})
-                neighbor_hostnames.append(neighbor_d.hostname)
-    logger.debug("Uplinks for device {} detected: {} neighbor_hostnames: {}". \
-                 format(hostname, uplinks, neighbor_hostnames))
+                uplinks[local_if] = neighbor_d.hostname
+    logger.debug("Uplinks for device {} detected: {}".
+                 format(hostname, ', '.join(["{}: {}".format(ifname, hostname)
+                                             for ifname, hostname in uplinks.items()])))
 
-    return (uplinks, neighbor_hostnames)
+    return uplinks
+
+
+def get_mlag_ifs(session, hostname, mlag_peer_hostname) -> dict[str, str]:
+    """Returns dict with mapping of interface -> neighbor hostname"""
+    logger = get_logger()
+    mlag_ifs = {}
+
+    dev = session.query(Device).filter(Device.hostname == hostname).one()
+    for neighbor_d in dev.get_neighbors(session):
+        if neighbor_d.hostname == mlag_peer_hostname:
+            local_if = dev.get_neighbor_local_ifname(session, neighbor_d)
+            if local_if:
+                mlag_ifs[local_if] = neighbor_d.hostname
+    logger.debug("MLAG peer interfaces for device {} detected: {}".
+                 format(hostname, ', '.join(["{}: {}".format(ifname, hostname)
+                                             for ifname, hostname in mlag_ifs.items()])))
+    return mlag_ifs
 
 
 def get_interfaces(hostname: str) -> AggregatedResult:
diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 343d52d5..2f45b6c3 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -17,7 +17,7 @@
 from cnaas_nms.scheduler.scheduler import Scheduler
 from cnaas_nms.scheduler.wrapper import job_wrapper
 from cnaas_nms.confpush.nornir_helper import NornirJobResult
-from cnaas_nms.confpush.update import update_interfacedb
+from cnaas_nms.confpush.update import update_interfacedb_worker
 from cnaas_nms.db.git import RepoStructureException
 from cnaas_nms.db.settings import get_settings
 from cnaas_nms.plugins.pluginmanager import PluginManagerHandler
@@ -106,14 +106,18 @@ def pre_init_checks(session, device_id) -> Device:
 @job_wrapper
 def init_access_device_step1(device_id: int, new_hostname: str,
                              mlag_peer_id: Optional[int],
-                             mlag_peer_hostname: Optional[str],
+                             mlag_peer_new_hostname: Optional[str],
                              job_id: Optional[str] = None,
                              scheduled_by: Optional[str] = None) -> NornirJobResult:
-    """Initialize access device for management by CNaaS-NMS
+    """Initialize access device for management by CNaaS-NMS.
+    If a MLAG/MC-LAG pair is to be configured both mlag_peer_id and
+    mlag_peer_new_hostname must be set.
 
     Args:
         device_id: Device to select for initialization
-        new_hostname: Hostname to configure for the new device
+        new_hostname: Hostname to configure on this device
+        mlag_peer_id: Device ID of MLAG peer device (optional)
+        mlag_peer_new_hostname: Hostname to configure on peer device (optional)
         job_id: job_id provided by scheduler when adding job
         scheduled_by: Username from JWT.
 
@@ -122,18 +126,28 @@ def init_access_device_step1(device_id: int, new_hostname: str,
 
     Raises:
         DeviceStateException
+        ValueError
     """
     logger = get_logger()
     with sqla_session() as session:
         dev = pre_init_checks(session, device_id)
-        old_hostname = dev.hostname
 
-        cnaas_nms.confpush.get.update_linknets(old_hostname)
+        cnaas_nms.confpush.get.update_linknets(session, dev.hostname)  # update linknets using LLDP data
         uplinks = []
         neighbor_hostnames = []
 
+        if mlag_peer_id and mlag_peer_new_hostname:
+            mlag_peer_dev = pre_init_checks(session, mlag_peer_id)
+            cnaas_nms.confpush.get.update_linknets(session, mlag_peer_dev.hostname)
+            update_interfacedb_worker(session, dev, replace=True, delete=False,
+                                      mlag_peer_hostname=mlag_peer_dev.hostname)
+        elif mlag_peer_id or mlag_peer_new_hostname:
+            raise ValueError("mlag_peer_id and mlag_peer_new_hostname must be specified together")
+        else:
+            update_interfacedb_worker(session, dev, replace=True, delete=False)
+
+        # TODO: break out into separate function, don't populate uplink vars here
         # Find management domain to use for this access switch
-        dev = session.query(Device).filter(Device.hostname == old_hostname).one()
         for neighbor_d in dev.get_neighbors(session):
             if neighbor_d.device_type == DeviceType.DIST:
                 local_if = dev.get_neighbor_local_ifname(session, neighbor_d)
@@ -313,13 +327,6 @@ def init_access_device_step2(device_id: int, iteration: int = -1,
     except Exception as e:
         logger.exception("Error while running plugin hooks for new_managed_device: ".format(str(e)))
 
-    try:
-        update_interfacedb(hostname, replace=True)
-    except Exception as e:
-        logger.exception(
-            "Exception while updating interface database for device {}: {}".\
-            format(hostname, str(e)))
-
     return NornirJobResult(
         nrresult = nrresult
     )
diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 78d1fd57..e5406790 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -13,7 +13,7 @@
 import cnaas_nms.db.helper
 import cnaas_nms.confpush.nornir_helper
 from cnaas_nms.db.session import sqla_session, redis_session
-from cnaas_nms.confpush.get import get_uplinks, calc_config_hash
+from cnaas_nms.confpush.get import calc_config_hash
 from cnaas_nms.confpush.changescore import calculate_score
 from cnaas_nms.tools.log import get_logger
 from cnaas_nms.db.settings import get_settings
diff --git a/src/cnaas_nms/confpush/update.py b/src/cnaas_nms/confpush/update.py
index 08e7da2f..e944f75c 100644
--- a/src/cnaas_nms/confpush/update.py
+++ b/src/cnaas_nms/confpush/update.py
@@ -4,12 +4,58 @@
 from cnaas_nms.db.device import Device, DeviceType, DeviceState
 from cnaas_nms.db.interface import Interface, InterfaceConfigType
 from cnaas_nms.confpush.get import get_interfaces_names, get_uplinks, \
-    filter_interfaces, get_interfacedb_ifs
+    filter_interfaces, get_mlag_ifs
 from cnaas_nms.tools.log import get_logger
 
 
+def update_interfacedb_worker(session, dev: Device, replace: bool, delete: bool,
+                              mlag_peer_hostname: str) -> List[dict]:
+    """Perform actual work of updating database for update_interfacedb"""
+    logger = get_logger()
+    ret = []
+
+    iflist = get_interfaces_names(dev.hostname)
+    uplinks = get_uplinks(session, dev.hostname)
+    mlag_ifs = get_mlag_ifs(session, dev.hostname)
+    phy_interfaces = filter_interfaces(iflist, platform=dev.platform, include='physical')
+
+    for intf_name in phy_interfaces:
+        intf: Interface = session.query(Interface).filter(Interface.device == dev). \
+            filter(Interface.name == intf_name).one_or_none()
+        if intf:
+            new_intf = False
+        else:
+            new_intf = True
+            intf: Interface = Interface()
+        if not new_intf and delete:  # 'not new_intf' means interface exists in database
+            logger.debug("Deleting interface {} on device {} from interface DB".format(
+                intf_name, dev.hostname
+            ))
+            session.delete(intf)
+            continue
+        elif not new_intf and not replace:
+            continue
+        logger.debug("New/updated physical interface found on device {}: {}".format(
+            dev.hostname, intf_name
+        ))
+        if intf_name in uplinks.keys():
+            intf.configtype = InterfaceConfigType.ACCESS_UPLINK
+            intf.data = {'neighbor': uplinks[intf_name]}
+        elif intf_name in mlag_ifs.keys():
+            intf.configtype = InterfaceConfigType.MLAG_PEER
+            intf.data = {'neighbor': mlag_ifs[intf_name]}
+        else:
+            intf.configtype = InterfaceConfigType.ACCESS_AUTO
+        intf.name = intf_name
+        intf.device = dev
+        if new_intf:
+            session.add(intf)
+        ret.append(intf.as_dict())
+    return ret
+
+
 def update_interfacedb(hostname: str, replace: bool = False, delete: bool = False) \
-        -> Optional[List[dict]]:
+        -> List[dict]:
     """Update interface DB with any new physical interfaces for specified device.
     If replace is set, any existing records in the database will get overwritten.
     If delete is set, all entries in database for this device will be removed.
@@ -17,8 +63,6 @@ def update_interfacedb(hostname: str, replace: bool = False, delete: bool = Fals
     Returns:
         List of interfaces that was added to DB
     """
-    logger = get_logger()
-    ret = []
     with sqla_session() as session:
         dev: Device = session.query(Device).filter(Device.hostname == hostname).one_or_none()
         if not dev:
@@ -27,48 +71,12 @@ def update_interfacedb(hostname: str, replace: bool = False, delete: bool = Fals
             raise ValueError(f"Hostname {hostname} is not a managed device")
         if dev.device_type != DeviceType.ACCESS:
             raise ValueError("This function currently only supports access devices")
-        # TODO: add support for dist/core devices?
 
-        iflist = get_interfaces_names(hostname)
-        uplinks, neighbor_hostnames = get_uplinks(session, hostname)
-        uplinks_ifnames = [x['ifname'] for x in uplinks]
-        phy_interfaces = filter_interfaces(iflist, platform=dev.platform, include='physical')
-#        existing_ifs = get_interfacedb_ifs(session, hostname)
+        result = update_interfacedb_worker(session, dev, replace, delete)
 
-        updated = False
-        for intf_name in phy_interfaces:
-            intf: Interface = session.query(Interface).filter(Interface.device == dev).\
-                filter(Interface.name == intf_name).one_or_none()
-            if intf:
-                new_intf = False
-            else:
-                new_intf = True
-                intf: Interface = Interface()
-            if not new_intf and delete:
-                logger.debug("Deleting interface {} on device {} from interface DB".format(
-                    intf_name, dev.hostname
-                ))
-                session.delete(intf)
-                continue
-            elif not new_intf and not replace:
-                continue
-            updated = True
-            logger.debug("New/updated physical interface found on device {}: {}".format(
-                dev.hostname, intf_name
-            ))
-            if intf_name in uplinks_ifnames:
-                intf.configtype = InterfaceConfigType.ACCESS_UPLINK
-                intf.data = {'neighbor': neighbor_hostnames[uplinks_ifnames.index(intf_name)]}
-            else:
-                intf.configtype = InterfaceConfigType.ACCESS_AUTO
-            intf.name = intf_name
-            intf.device = dev
-            if new_intf:
-                session.add(intf)
-            ret.append(intf.as_dict())
-        if updated:
+        if result:
             dev.synchronized = False
-    return ret
+    return result
 
 
 def reset_interfacedb(hostname: str):

From 7894ddabfe30d15d03d9d3e87a6682a60636aca7 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 20 Mar 2020 09:36:01 +0100
Subject: [PATCH 006/102] Add interface config option for access switch to have
 downlink interfaces to other access switches

---
 src/cnaas_nms/db/interface.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cnaas_nms/db/interface.py b/src/cnaas_nms/db/interface.py
index 25143889..533c4d00 100644
--- a/src/cnaas_nms/db/interface.py
+++ b/src/cnaas_nms/db/interface.py
@@ -22,6 +22,7 @@ class InterfaceConfigType(enum.Enum):
     ACCESS_UNTAGGED = 11
     ACCESS_TAGGED = 12
     ACCESS_UPLINK = 13
+    ACCESS_DOWNLINK = 14
 
     @classmethod
     def has_value(cls, value):

From 578650aa162f55b3c08f6183c6542186d2816bdc Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 20 Mar 2020 09:36:57 +0100
Subject: [PATCH 007/102] Do consistency checks for mlag peers. Figure out
 uplinks to dist from both devices in mlag pair.

---
 src/cnaas_nms/confpush/get.py         |  7 ++++---
 src/cnaas_nms/confpush/init_device.py | 27 +++++++++++++++++++++++----
 src/cnaas_nms/confpush/update.py      |  3 ++-
 3 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/src/cnaas_nms/confpush/get.py b/src/cnaas_nms/confpush/get.py
index 633814a8..d4faf0dd 100644
--- a/src/cnaas_nms/confpush/get.py
+++ b/src/cnaas_nms/confpush/get.py
@@ -112,8 +112,9 @@ def get_uplinks(session, hostname: str) -> dict[str, str]:
     return uplinks
 
 
-def get_mlag_ifs(session, hostname, mlag_peer_hostname) -> dict[str, str]:
-    """Returns dict with mapping of interface -> neighbor hostname"""
+def get_mlag_ifs(session, hostname, mlag_peer_hostname) -> dict[str, int]:
+    """Returns dict with mapping of interface -> neighbor id
+    Return id instead of hostname since mlag peer will change hostname during init"""
     logger = get_logger()
     mlag_ifs = {}
 
@@ -122,7 +123,7 @@ def get_mlag_ifs(session, hostname, mlag_peer_hostname) -> dict[str, str]:
         if neighbor_d.hostname == mlag_peer_hostname:
             local_if = dev.get_neighbor_local_ifname(session, neighbor_d)
             if local_if:
-                mlag_ifs[local_if] = neighbor_d.hostname
+                mlag_ifs[local_if] = neighbor_d.id
     logger.debug("MLAG peer interfaces for device {} detected: {}".
                  format(hostname, ', '.join(["{}: {}".format(ifname, hostname)
                                              for ifname, hostname in mlag_ifs.items()])))
diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 2f45b6c3..e5843b0b 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -14,6 +14,7 @@
 import cnaas_nms.db.helper
 from cnaas_nms.db.session import sqla_session
 from cnaas_nms.db.device import Device, DeviceState, DeviceType, DeviceStateException
+from cnaas_nms.db.interface import Interface, InterfaceConfigType
 from cnaas_nms.scheduler.scheduler import Scheduler
 from cnaas_nms.scheduler.wrapper import job_wrapper
 from cnaas_nms.confpush.nornir_helper import NornirJobResult
@@ -103,6 +104,19 @@ def pre_init_checks(session, device_id) -> Device:
         raise ConnectionCheckError(f"Failed to connect to device_id {device_id}")
 
 
+def pre_init_check_mlag(session, dev, mlag_peer_dev):
+    intfs: Interface = session.query(Interface).filter(Interface.device == dev).\
+        filter(InterfaceConfigType == InterfaceConfigType.MLAG_PEER).all()
+    intf: Interface
+    for intf in intfs:
+        if intf.data['neighbor_id'] == mlag_peer_dev.id:
+            continue
+        else:
+            raise Exception("Inconsistent MLAG peer {} detected for device {}".format(
+                intf.data['neighbor'], dev.hostname
+            ))
+
+
 @job_wrapper
 def init_access_device_step1(device_id: int, new_hostname: str,
                              mlag_peer_id: Optional[int],
@@ -133,14 +147,17 @@ def init_access_device_step1(device_id: int, new_hostname: str,
         dev = pre_init_checks(session, device_id)
 
         cnaas_nms.confpush.get.update_linknets(session, dev.hostname)  # update linknets using LLDP data
-        uplinks = []
-        neighbor_hostnames = []
+        uplink_hostnames = dev.get_uplink_peer_hostnames(session)
 
         if mlag_peer_id and mlag_peer_new_hostname:
             mlag_peer_dev = pre_init_checks(session, mlag_peer_id)
             cnaas_nms.confpush.get.update_linknets(session, mlag_peer_dev.hostname)
             update_interfacedb_worker(session, dev, replace=True, delete=False,
                                       mlag_peer_hostname=mlag_peer_dev.hostname)
+            uplink_hostnames.append(mlag_peer_dev.get_uplink_peer_hostnames(session))
+            # check that both devices see the correct MLAG peer
+            pre_init_check_mlag(session, dev, mlag_peer_dev)
+            pre_init_check_mlag(session, mlag_peer_dev, dev)
         elif mlag_peer_id or mlag_peer_new_hostname:
             raise ValueError("mlag_peer_id and mlag_peer_new_hostname must be specified together")
         else:
@@ -157,11 +174,11 @@ def init_access_device_step1(device_id: int, new_hostname: str,
         logger.debug("Uplinks for device {} detected: {} neighbor_hostnames: {}".\
                      format(device_id, uplinks, neighbor_hostnames))
         # TODO: check compatability, same dist pair and same ports on dists
-        mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, neighbor_hostnames)
+        mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, uplink_hostnames)
         if not mgmtdomain:
             raise Exception(
                 "Could not find appropriate management domain for uplink peer devices: {}".format(
-                    neighbor_hostnames))
+                    uplink_hostnames))
         # Select a new management IP for the device
         ReservedIP.clean_reservations(session, device=dev)
         session.commit()
@@ -171,6 +188,8 @@ def init_access_device_step1(device_id: int, new_hostname: str,
                 mgmtdomain.id, mgmtdomain.description))
         reserved_ip = ReservedIP(device=dev, ip=mgmt_ip)
         session.add(reserved_ip)
+        # TODO: query interface db
+        uplinks = []
         # Populate variables for template rendering
         mgmt_gw_ipif = IPv4Interface(mgmtdomain.ipv4_gw)
         device_variables = {
diff --git a/src/cnaas_nms/confpush/update.py b/src/cnaas_nms/confpush/update.py
index e944f75c..7b2c57c2 100644
--- a/src/cnaas_nms/confpush/update.py
+++ b/src/cnaas_nms/confpush/update.py
@@ -43,7 +43,7 @@ def update_interfacedb_worker(session, dev: Device, replace: bool, delete: bool,
             intf.data = {'neighbor': uplinks[intf_name]}
         elif intf_name in mlag_ifs.keys():
             intf.configtype = InterfaceConfigType.MLAG_PEER
-            intf.data = {'neighbor': mlag_ifs[intf_name]}
+            intf.data = {'neighbor_id': mlag_ifs[intf_name]}
         else:
             intf.configtype = InterfaceConfigType.ACCESS_AUTO
         intf.name = intf_name
@@ -51,6 +51,7 @@ def update_interfacedb_worker(session, dev: Device, replace: bool, delete: bool,
         if new_intf:
             session.add(intf)
         ret.append(intf.as_dict())
+    session.commit()
     return ret
 
 

From c37c3c9f2a76f6dc7245f486e1c336d380a8a093 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 20 Mar 2020 13:08:57 +0100
Subject: [PATCH 008/102] Build template interface list for init in a similar
 way to syncto by using interfaces table

---
 src/cnaas_nms/confpush/get.py         |  6 +++---
 src/cnaas_nms/confpush/init_device.py | 24 +++++++++---------------
 2 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/src/cnaas_nms/confpush/get.py b/src/cnaas_nms/confpush/get.py
index d4faf0dd..29969b5e 100644
--- a/src/cnaas_nms/confpush/get.py
+++ b/src/cnaas_nms/confpush/get.py
@@ -2,7 +2,7 @@
 import re
 import hashlib
 
-from typing import Optional, Tuple, List
+from typing import Optional, Tuple, List, Dict
 
 from nornir.core.deserializer.inventory import Inventory
 from nornir.core.filter import F
@@ -93,7 +93,7 @@ def get_neighbors(hostname: Optional[str] = None, group: Optional[str] = None)\
     return result
 
 
-def get_uplinks(session, hostname: str) -> dict[str, str]:
+def get_uplinks(session, hostname: str) -> Dict[str, str]:
     """Returns dict with mapping of interface -> neighbor hostname"""
     logger = get_logger()
     # TODO: check if uplinks are already saved in database?
@@ -112,7 +112,7 @@ def get_uplinks(session, hostname: str) -> dict[str, str]:
     return uplinks
 
 
-def get_mlag_ifs(session, hostname, mlag_peer_hostname) -> dict[str, int]:
+def get_mlag_ifs(session, hostname, mlag_peer_hostname) -> Dict[str, int]:
     """Returns dict with mapping of interface -> neighbor id
     Return id instead of hostname since mlag peer will change hostname during init"""
     logger = get_logger()
diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index e5843b0b..bd3e5b30 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -163,16 +163,6 @@ def init_access_device_step1(device_id: int, new_hostname: str,
         else:
             update_interfacedb_worker(session, dev, replace=True, delete=False)
 
-        # TODO: break out into separate function, don't populate uplink vars here
-        # Find management domain to use for this access switch
-        for neighbor_d in dev.get_neighbors(session):
-            if neighbor_d.device_type == DeviceType.DIST:
-                local_if = dev.get_neighbor_local_ifname(session, neighbor_d)
-                if local_if:
-                    uplinks.append({'ifname': local_if})
-                    neighbor_hostnames.append(neighbor_d.hostname)
-        logger.debug("Uplinks for device {} detected: {} neighbor_hostnames: {}".\
-                     format(device_id, uplinks, neighbor_hostnames))
         # TODO: check compatability, same dist pair and same ports on dists
         mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, uplink_hostnames)
         if not mgmtdomain:
@@ -188,8 +178,6 @@ def init_access_device_step1(device_id: int, new_hostname: str,
                 mgmtdomain.id, mgmtdomain.description))
         reserved_ip = ReservedIP(device=dev, ip=mgmt_ip)
         session.add(reserved_ip)
-        # TODO: query interface db
-        uplinks = []
         # Populate variables for template rendering
         mgmt_gw_ipif = IPv4Interface(mgmtdomain.ipv4_gw)
         device_variables = {
@@ -200,10 +188,16 @@ def init_access_device_step1(device_id: int, new_hostname: str,
             'mgmt_vlan_id': mgmtdomain.vlan,
             'mgmt_gw': mgmt_gw_ipif.ip
         }
-        for uplink in uplinks:
+        intfs = session.query(Interface).filter(Interface.device == dev).all()
+        intf: Interface
+        for intf in intfs:
+            intfdata = None
+            if intf.data:
+                intfdata = dict(intf.data)
             device_variables['interfaces'].append({
-                'name': uplink['ifname'],
-                'ifclass': 'ACCESS_UPLINK',
+                'name': intf.name,
+                'ifclass': intf.configtype.name,
+                'data': intfdata
             })
         # Update device state
         dev = session.query(Device).filter(Device.id == device_id).one()

From f43cd2c5e81c34e82fd5509980c6cee201d6a3ff Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 20 Mar 2020 14:13:06 +0100
Subject: [PATCH 009/102] Bugfixes for init_device

---
 src/cnaas_nms/confpush/init_device.py | 5 +++--
 src/cnaas_nms/confpush/update.py      | 7 +++++--
 src/cnaas_nms/db/helper.py            | 5 ++++-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index bd3e5b30..af000f70 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -102,6 +102,7 @@ def pre_init_checks(session, device_id) -> Device:
     if nrresult_old.failed:
         print_result(nrresult_old)
         raise ConnectionCheckError(f"Failed to connect to device_id {device_id}")
+    return dev
 
 
 def pre_init_check_mlag(session, dev, mlag_peer_dev):
@@ -119,8 +120,8 @@ def pre_init_check_mlag(session, dev, mlag_peer_dev):
 
 @job_wrapper
 def init_access_device_step1(device_id: int, new_hostname: str,
-                             mlag_peer_id: Optional[int],
-                             mlag_peer_new_hostname: Optional[str],
+                             mlag_peer_id: Optional[int] = None,
+                             mlag_peer_new_hostname: Optional[str] = None,
                              job_id: Optional[str] = None,
                              scheduled_by: Optional[str] = None) -> NornirJobResult:
     """Initialize access device for management by CNaaS-NMS.
diff --git a/src/cnaas_nms/confpush/update.py b/src/cnaas_nms/confpush/update.py
index 7b2c57c2..2f2b3f33 100644
--- a/src/cnaas_nms/confpush/update.py
+++ b/src/cnaas_nms/confpush/update.py
@@ -9,14 +9,17 @@
 
 
 def update_interfacedb_worker(session, dev: Device, replace: bool, delete: bool,
-                              mlag_peer_hostname: str) -> List[dict]:
+                              mlag_peer_hostname: Optional[str] = None) -> List[dict]:
     """Perform actual work of updating database for update_interfacedb"""
     logger = get_logger()
     ret = []
 
     iflist = get_interfaces_names(dev.hostname)
     uplinks = get_uplinks(session, dev.hostname)
-    mlag_ifs = get_mlag_ifs(session, dev.hostname)
+    if mlag_peer_hostname:
+        mlag_ifs = get_mlag_ifs(session, dev.hostname, mlag_peer_hostname)
+    else:
+        mlag_ifs = {}
     phy_interfaces = filter_interfaces(iflist, platform=dev.platform, include='physical')
 
     for intf_name in phy_interfaces:
diff --git a/src/cnaas_nms/db/helper.py b/src/cnaas_nms/db/helper.py
index e70cbe83..e178cd2b 100644
--- a/src/cnaas_nms/db/helper.py
+++ b/src/cnaas_nms/db/helper.py
@@ -27,7 +27,10 @@ def find_mgmtdomain(session, hostnames: List[str]) -> Optional[Mgmtdomain]:
         ValueError: On invalid hostnames etc
     """
     if not isinstance(hostnames, list) or not len(hostnames) == 2:
-        raise ValueError("hostnames argument must be a list with two device hostnames")
+        raise ValueError(
+            "hostnames argument must be a list with two device hostnames, got: {}".format(
+                hostnames
+            ))
     for hostname in hostnames:
         if not Device.valid_hostname(hostname):
             raise ValueError(f"Argument {hostname} is not a valid hostname")

From 3ff06c8dc893048e2e9155ddf3752b599c540319 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 23 Mar 2020 13:34:18 +0100
Subject: [PATCH 010/102] Add possibility to detect uplink interfaces towards
 other access switches

---
 src/cnaas_nms/confpush/get.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/cnaas_nms/confpush/get.py b/src/cnaas_nms/confpush/get.py
index 29969b5e..5e848925 100644
--- a/src/cnaas_nms/confpush/get.py
+++ b/src/cnaas_nms/confpush/get.py
@@ -16,7 +16,7 @@
 from cnaas_nms.db.device import Device, DeviceType
 from cnaas_nms.db.linknet import Linknet
 from cnaas_nms.tools.log import get_logger
-from cnaas_nms.db.interface import Interface
+from cnaas_nms.db.interface import Interface, InterfaceConfigType
 
 
 def get_inventory():
@@ -100,11 +100,24 @@ def get_uplinks(session, hostname: str) -> Dict[str, str]:
     uplinks = {}
 
     dev = session.query(Device).filter(Device.hostname == hostname).one()
+    neighbor_d: Device
     for neighbor_d in dev.get_neighbors(session):
         if neighbor_d.device_type == DeviceType.DIST:
             local_if = dev.get_neighbor_local_ifname(session, neighbor_d)
+            # TODO: check that dist interface is configured as downlink
             if local_if:
                 uplinks[local_if] = neighbor_d.hostname
+        elif neighbor_d.device_type == DeviceType.ACCESS:
+            intfs: Interface = session.query(Interface).filter(Interface.device == neighbor_d). \
+                filter(InterfaceConfigType == InterfaceConfigType.ACCESS_DOWNLINK).all()
+            local_if = dev.get_neighbor_local_ifname(session, neighbor_d)
+            remote_if = neighbor_d.get_neighbor_local_ifname(session, dev)
+
+            intf: Interface
+            for intf in intfs:
+                if intf.name == remote_if:
+                    uplinks[local_if] = neighbor_d.hostname
+
     logger.debug("Uplinks for device {} detected: {}".
                  format(hostname, ', '.join(["{}: {}".format(ifname, hostname)
                                              for ifname, hostname in uplinks.items()])))

From 7fc69c544c6fe24059dd0c32c2a0152e3144af6e Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 23 Mar 2020 15:38:57 +0100
Subject: [PATCH 011/102] Return mgmtdomain based on management_ip of uplink
 device

---
 src/cnaas_nms/db/helper.py                | 47 ++++++++++++++++++-----
 src/cnaas_nms/db/tests/test_mgmtdomain.py | 20 +++++-----
 2 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/src/cnaas_nms/db/helper.py b/src/cnaas_nms/db/helper.py
index e178cd2b..c47def7c 100644
--- a/src/cnaas_nms/db/helper.py
+++ b/src/cnaas_nms/db/helper.py
@@ -1,10 +1,11 @@
 import datetime
 from typing import List, Optional
+from ipaddress import IPv4Interface, IPv4Address
 
 import netaddr
 from sqlalchemy.orm.exc import NoResultFound
 
-from cnaas_nms.db.device import Device
+from cnaas_nms.db.device import Device, DeviceType
 from cnaas_nms.db.mgmtdomain import Mgmtdomain
 
 
@@ -35,22 +36,50 @@ def find_mgmtdomain(session, hostnames: List[str]) -> Optional[Mgmtdomain]:
         if not Device.valid_hostname(hostname):
             raise ValueError(f"Argument {hostname} is not a valid hostname")
     try:
-        device0 = session.query(Device).filter(Device.hostname == hostnames[0]).one()
+        device0: Device = session.query(Device).filter(Device.hostname == hostnames[0]).one()
     except NoResultFound:
         raise ValueError(f"hostname {hostnames[0]} not found in device database")
     try:
-        device1 = session.query(Device).filter(Device.hostname == hostnames[1]).one()
+        device1: Device = session.query(Device).filter(Device.hostname == hostnames[1]).one()
     except NoResultFound:
         raise ValueError(f"hostname {hostnames[1]} not found in device database")
-    mgmtdomain = session.query(Mgmtdomain).\
-        filter(
-            ((Mgmtdomain.device_a == device0) & (Mgmtdomain.device_b == device1))
-            |
-            ((Mgmtdomain.device_a == device1) & (Mgmtdomain.device_b == device0))
-        ).one_or_none()
+
+    if device0.device_type == DeviceType.DIST or device1.device_type == DeviceType.DIST:
+        if device0.device_type != DeviceType.DIST or device1.device_type != DeviceType.DIST:
+            raise ValueError("Both uplink devices must be of same device type: {}, {}".format(
+                device0.hostname, device1.hostname
+            ))
+        mgmtdomain: Mgmtdomain = session.query(Mgmtdomain).\
+            filter(
+                ((Mgmtdomain.device_a == device0) & (Mgmtdomain.device_b == device1))
+                |
+                ((Mgmtdomain.device_a == device1) & (Mgmtdomain.device_b == device0))
+            ).one_or_none()
+    elif device0.device_type == DeviceType.ACCESS or device1.device_type == DeviceType.ACCESS:
+        if device0.device_type != DeviceType.ACCESS or device1.device_type != DeviceType.ACCESS:
+            raise ValueError("Both uplink devices must be of same device type: {}, {}".format(
+                device0.hostname, device1.hostname
+            ))
+        mgmtdomain: Mgmtdomain = find_mgmtdomain_by_ip(session, device0.management_ip)
+        if mgmtdomain.id != find_mgmtdomain_by_ip(session, device1.management_ip).id:
+            raise Exception("Uplink access devices have different mgmtdomains: {}, {}".format(
+                device0.hostname, device1.hostname
+            ))
+    else:
+        raise Exception("Unknown uplink device type")
     return mgmtdomain
 
 
+def find_mgmtdomain_by_ip(session, ipv4_address: IPv4Address) -> Optional[Mgmtdomain]:
+    mgmtdomains = session.query(Mgmtdomain).all()
+    mgmtdom: Mgmtdomain
+    for mgmtdom in mgmtdomains:
+        mgmtdom_ipv4_network = IPv4Interface(mgmtdom.ipv4_gw).network
+        if ipv4_address in mgmtdom_ipv4_network:
+            return mgmtdom
+    return None
+
+
 def get_all_mgmtdomains(session, hostname: str) -> List[Mgmtdomain]:
     """
     Get all mgmtdomains for a specific distribution switch.
diff --git a/src/cnaas_nms/db/tests/test_mgmtdomain.py b/src/cnaas_nms/db/tests/test_mgmtdomain.py
index 0c0fe112..339da5ed 100644
--- a/src/cnaas_nms/db/tests/test_mgmtdomain.py
+++ b/src/cnaas_nms/db/tests/test_mgmtdomain.py
@@ -5,26 +5,21 @@
 import yaml
 import os
 import pprint
+from ipaddress import IPv4Address, IPv4Network, IPv4Interface
 
 import cnaas_nms.db.helper
 from cnaas_nms.db.device import Device, DeviceState, DeviceType
 from cnaas_nms.db.session import sqla_session
 from cnaas_nms.db.mgmtdomain import Mgmtdomain
 
-from cnaas_nms.tools.testsetup import PostgresTemporaryInstance
-
 
 class MgmtdomainTests(unittest.TestCase):
     def setUp(self):
         data_dir = pkg_resources.resource_filename(__name__, 'data')
         with open(os.path.join(data_dir, 'testdata.yml'), 'r') as f_testdata:
             self.testdata = yaml.safe_load(f_testdata)
-        self.tmp_postgres = PostgresTemporaryInstance()
-
-    def tearDown(self):
-        self.tmp_postgres.shutdown()
 
-    def add_mgmt_domain(self):
+    def add_mgmtdomain(self):
         with sqla_session() as session:
             d_a = session.query(Device).filter(Device.hostname == 'eosdist1').one()
             d_b = session.query(Device).filter(Device.hostname == 'eosdist2').one()
@@ -46,7 +41,7 @@ def add_mgmt_domain(self):
         #    1,
         #    len(result['hosts'].items()))
 
-    def delete_mgmt_domain(self):
+    def delete_mgmtdomain(self):
         with sqla_session() as session:
             d_a = session.query(Device).filter(Device.hostname == 'eosdist1').one()
             instance = session.query(Mgmtdomain).filter(Mgmtdomain.device_a == d_a).first()
@@ -56,19 +51,24 @@ def delete_mgmt_domain(self):
             else:
                 print(f"Mgmtdomain for device {d_a.hostname} not found")
 
-    def test_find_mgmt_domain(self):
+    def test_find_mgmt_omain(self):
         with sqla_session() as session:
             mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, ['eosdist1', 'eosdist2'])
             if mgmtdomain:
                 pprint.pprint(mgmtdomain.as_dict())
 
     def test_find_free_mgmt_ip(self):
-        mgmtdomain_id = 2
+        mgmtdomain_id = 1
         with sqla_session() as session:
             mgmtdomain = session.query(Mgmtdomain).filter(Mgmtdomain.id == mgmtdomain_id).one()
             if mgmtdomain:
                 print(mgmtdomain.find_free_mgmt_ip(session))
 
+    def test_find_mgmtdomain_by_ip(self):
+        with sqla_session() as session:
+            mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain_by_ip(session, IPv4Address('10.0.6.6'))
+            self.assertEqual(IPv4Interface(mgmtdomain.ipv4_gw).network, IPv4Network('10.0.6.0/24'))
+
 
 if __name__ == '__main__':
     unittest.main()

From 488e4d78899eb3a5c61739fc3b3f827c0285aa7f Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 23 Mar 2020 15:55:21 +0100
Subject: [PATCH 012/102] Find mgmtdomain from management IP instead of uplink
 neighbors when doing sync, to account for access switches connecting to other
 access switches instead of directly to dist devices

---
 src/cnaas_nms/confpush/sync_devices.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index e5406790..15f6c033 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -116,15 +116,11 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False,
         }
 
         if devtype == DeviceType.ACCESS:
-            neighbor_hostnames = dev.get_uplink_peer_hostnames(session)
-            if not neighbor_hostnames:
-                raise Exception("Could not find any uplink neighbors for device {}".format(
-                    hostname))
-            mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, neighbor_hostnames)
+            mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain_by_ip(session, dev.management_ip)
             if not mgmtdomain:
                 raise Exception(
-                    "Could not find appropriate management domain for uplink peer devices: {}".
-                    format(neighbor_hostnames))
+                    "Could not find appropriate management domain for management_ip: {}".
+                    format(dev.management_ip))
 
             mgmt_gw_ipif = IPv4Interface(mgmtdomain.ipv4_gw)
             access_device_variables = {

From 425479b78a31e015fd1ada8447c9b3b0523d1702 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 24 Mar 2020 09:52:45 +0100
Subject: [PATCH 013/102] Populate mlag_peer variables for syncto

---
 src/cnaas_nms/confpush/sync_devices.py | 22 ++++++++++++++++++++--
 src/cnaas_nms/db/device.py             | 14 ++++++++------
 2 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 15f6c033..8da3df62 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -81,6 +81,24 @@ def resolve_vlanid_list(vlan_name_list: List[str], vxlans: dict) -> List[int]:
     return ret
 
 
+def get_mlag_vars(session, dev: Device) -> dict:
+    ret = {
+        'mlag_peer': False,
+        'mlag_peer_hostname': None,
+        'mlag_peer_low': None
+    }
+    mlag_peer: Device = dev.get_mlag_peer(session)
+    if not mlag_peer:
+        return ret
+    ret['mlag_peer'] = True
+    ret['mlag_peer_hostname'] = mlag_peer.hostname
+    if dev.id < mlag_peer.id:
+        ret['mlag_peer_low'] = True
+    else:
+        ret['mlag_peer_low'] = False
+    return ret
+
+
 def push_sync_device(task, dry_run: bool = True, generate_only: bool = False,
                      job_id: Optional[str] = None,
                      scheduled_by: Optional[str] = None):
@@ -152,8 +170,8 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False,
                     'tagged_vlan_list': tagged_vlan_list,
                     'data': intfdata
                 })
-
-            device_variables = {**access_device_variables, **device_variables}
+            mlag_vars = get_mlag_vars(session, dev)
+            device_variables = {**access_device_variables, **device_variables, **mlag_vars}
         elif devtype == DeviceType.DIST or devtype == DeviceType.CORE:
             asn = generate_asn(infra_ip)
             fabric_device_variables = {
diff --git a/src/cnaas_nms/db/device.py b/src/cnaas_nms/db/device.py
index ccf8a73d..5c71a232 100644
--- a/src/cnaas_nms/db/device.py
+++ b/src/cnaas_nms/db/device.py
@@ -4,7 +4,7 @@
 import datetime
 import enum
 import re
-from typing import Optional, List
+from typing import Optional, List, Set
 
 from sqlalchemy import Column, Integer, Unicode, String, UniqueConstraint
 from sqlalchemy import Enum, DateTime, Boolean
@@ -197,18 +197,18 @@ def get_uplink_peer_hostnames(self, session) -> List[str]:
                 peer_hostnames.append(intf.data['neighbor'])
         return peer_hostnames
 
-    def get_mlag_peers(self, session) -> List[Device]:
+    def get_mlag_peer(self, session) -> Optional[Device]:
         intfs = session.query(Interface).filter(Interface.device == self). \
             filter(Interface.configtype == InterfaceConfigType.MLAG_PEER).all()
-        peers: List[Device] = []
+        peers: Set[Device] = set()
         linknets = self.get_linknets(session)
         intf: Interface = Interface()
         for intf in intfs:
             for linknet in linknets:
                 if linknet.device_a == self and linknet.device_a_port == intf.name:
-                    peers.append(linknet.device_b)
+                    peers.add(linknet.device_b)
                 elif linknet.device_b == self and linknet.device_b_port == intf.name:
-                    peers.append(linknet.device_a)
+                    peers.add(linknet.device_a)
         if len(peers) > 1:
             raise DeviceException("More than one MLAG peer found: {}".format(
                 [x.hostname for x in peers]
@@ -216,7 +216,9 @@ def get_mlag_peers(self, session) -> List[Device]:
         elif len(peers) == 1:
             if self.device_type != peers[0].device_type:
                 raise DeviceException("MLAG peers are not the same device type")
-        return peers
+            return peers[0]
+        else:
+            return None
 
     @classmethod
     def valid_hostname(cls, hostname: str) -> bool:

From a20bb7780bc0ef79c2617943397099c2e842153e Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 24 Mar 2020 16:16:44 +0100
Subject: [PATCH 014/102] Add mlag_vars when doing init templates

---
 src/cnaas_nms/confpush/init_device.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index af000f70..3e51f0e0 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -19,6 +19,7 @@
 from cnaas_nms.scheduler.wrapper import job_wrapper
 from cnaas_nms.confpush.nornir_helper import NornirJobResult
 from cnaas_nms.confpush.update import update_interfacedb_worker
+from cnaas_nms.confpush.sync_devices import get_mlag_vars
 from cnaas_nms.db.git import RepoStructureException
 from cnaas_nms.db.settings import get_settings
 from cnaas_nms.plugins.pluginmanager import PluginManagerHandler
@@ -200,6 +201,8 @@ def init_access_device_step1(device_id: int, new_hostname: str,
                 'ifclass': intf.configtype.name,
                 'data': intfdata
             })
+        mlag_vars = get_mlag_vars(session, dev)
+        device_variables = {**device_variables, **mlag_vars}
         # Update device state
         dev = session.query(Device).filter(Device.id == device_id).one()
         dev.state = DeviceState.INIT

From f15644095eed6cd65722aad46f781a93d0a31ee2 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 24 Mar 2020 16:21:41 +0100
Subject: [PATCH 015/102] Add support for multiple mlag peer links

---
 src/cnaas_nms/confpush/get.py          |  3 +-
 src/cnaas_nms/confpush/sync_devices.py |  1 +
 src/cnaas_nms/db/device.py             | 42 ++++++++++++++++++++------
 3 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/src/cnaas_nms/confpush/get.py b/src/cnaas_nms/confpush/get.py
index 5e848925..b0aa4f18 100644
--- a/src/cnaas_nms/confpush/get.py
+++ b/src/cnaas_nms/confpush/get.py
@@ -134,8 +134,7 @@ def get_mlag_ifs(session, hostname, mlag_peer_hostname) -> Dict[str, int]:
     dev = session.query(Device).filter(Device.hostname == hostname).one()
     for neighbor_d in dev.get_neighbors(session):
         if neighbor_d.hostname == mlag_peer_hostname:
-            local_if = dev.get_neighbor_local_ifname(session, neighbor_d)
-            if local_if:
+            for local_if in dev.get_neighbor_local_ifnames(session, neighbor_d):
                 mlag_ifs[local_if] = neighbor_d.id
     logger.debug("MLAG peer interfaces for device {} detected: {}".
                  format(hostname, ', '.join(["{}: {}".format(ifname, hostname)
diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 8da3df62..79618009 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -222,6 +222,7 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False,
             fabric_links = []
             for neighbor_d in dev.get_neighbors(session):
                 if neighbor_d.device_type == DeviceType.DIST or neighbor_d.device_type == DeviceType.CORE:
+                    # TODO: support multiple links to the same neighbor?
                     local_if = dev.get_neighbor_local_ifname(session, neighbor_d)
                     local_ipif = dev.get_neighbor_local_ipif(session, neighbor_d)
                     neighbor_ip = dev.get_neighbor_ip(session, neighbor_d)
diff --git a/src/cnaas_nms/db/device.py b/src/cnaas_nms/db/device.py
index 5c71a232..032da38f 100644
--- a/src/cnaas_nms/db/device.py
+++ b/src/cnaas_nms/db/device.py
@@ -145,9 +145,8 @@ def get_linknet_localif_mapping(self, session) -> dict[str, str]:
                 ))
         return ret
 
-    def get_link_to(self, session, peer_device: Device) -> Optional[cnaas_nms.db.linknet.Linknet]:
+    def get_links_to(self, session, peer_device: Device) -> List[cnaas_nms.db.linknet.Linknet]:
         """Return linknet connecting to device peer_device."""
-        # TODO: support multiple links to the same neighbor?
         return session.query(cnaas_nms.db.linknet.Linknet).\
             filter(
                 ((cnaas_nms.db.linknet.Linknet.device_a_id == self.id) &
@@ -155,23 +154,44 @@ def get_link_to(self, session, peer_device: Device) -> Optional[cnaas_nms.db.lin
                 |
                 ((cnaas_nms.db.linknet.Linknet.device_b_id == self.id) &
                  (cnaas_nms.db.linknet.Linknet.device_a_id == peer_device.id))
-            ).one_or_none()
+            ).all()
 
     def get_neighbor_local_ifname(self, session, peer_device: Device) -> Optional[str]:
         """Get the local interface name on this device that links to peer_device."""
-        linknet = self.get_link_to(session, peer_device)
-        if not linknet:
+        linknets = self.get_links_to(session, peer_device)
+        if not linknets:
             return None
+        elif len(linknets) > 1:
+            raise ValueError("Multiple linknets between devices not supported")
+        else:
+            linknet = linknets[0]
         if linknet.device_a_id == self.id:
             return linknet.device_a_port
         elif linknet.device_b_id == self.id:
             return linknet.device_b_port
 
+    def get_neighbor_local_ifnames(self, session, peer_device: Device) -> List[str]:
+        """Get the local interface name on this device that links to peer_device."""
+        linknets = self.get_links_to(session, peer_device)
+        ifnames = []
+        if not linknets:
+            return ifnames
+        for linknet in linknets:
+            if linknet.device_a_id == self.id:
+                ifnames.append(linknet.device_a_port)
+            elif linknet.device_b_id == self.id:
+                ifnames.append(linknet.device_b_port)
+        return ifnames
+
     def get_neighbor_local_ipif(self, session, peer_device: Device) -> Optional[str]:
         """Get the local interface IP on this device that links to peer_device."""
-        linknet = self.get_link_to(session, peer_device)
-        if not linknet:
+        linknets = self.get_links_to(session, peer_device)
+        if not linknets:
             return None
+        elif len(linknets) > 1:
+            raise ValueError("Multiple linknets between devices not supported")
+        else:
+            linknet = linknets[0]
         if linknet.device_a_id == self.id:
             return "{}/{}".format(linknet.device_a_ip, ipaddress.IPv4Network(linknet.ipv4_network).prefixlen)
         elif linknet.device_b_id == self.id:
@@ -179,9 +199,13 @@ def get_neighbor_local_ipif(self, session, peer_device: Device) -> Optional[str]
 
     def get_neighbor_ip(self, session, peer_device: Device):
         """Get the remote peer IP address for the linknet going towards device."""
-        linknet = self.get_link_to(session, peer_device)
-        if not linknet:
+        linknets = self.get_links_to(session, peer_device)
+        if not linknets:
             return None
+        elif len(linknets) > 1:
+            raise ValueError("Multiple linknets between devices not supported")
+        else:
+            linknet = linknets[0]
         if linknet.device_a_id == self.id:
             return linknet.device_b_ip
         elif linknet.device_b_id == self.id:

From 5e57ddc9441ef65f11a636c1ecb4defdfb89e6b6 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 26 Mar 2020 09:20:28 +0100
Subject: [PATCH 016/102] Set hostname to mac instead of just localhost so we
 can find mlag neighbors during init

---
 src/cnaas_nms/confpush/init_device.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 3e51f0e0..0503721d 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -365,6 +365,22 @@ def schedule_discover_device(ztp_mac: str, dhcp_ip: str, iteration: int,
         return None
 
 
+def set_hostname_task(task, new_hostname: str):
+    with open('/etc/cnaas-nms/repository.yml', 'r') as db_file:
+        repo_config = yaml.safe_load(db_file)
+        local_repo_path = repo_config['templates_local']
+    template_vars = {
+        'hostname': new_hostname
+    }
+    task.run(
+        task=text.template_file,
+        name="Configure hostname",
+        template="hostname.j2",
+        path=f"{local_repo_path}/{task.host.platform}",
+        **template_vars
+    )
+
+
 @job_wrapper
 def discover_device(ztp_mac: str, dhcp_ip: str, iteration=-1,
                     job_id: Optional[str] = None,
@@ -409,6 +425,7 @@ def discover_device(ztp_mac: str, dhcp_ip: str, iteration=-1,
             dev.model = facts['model']
             dev.os_version = facts['os_version']
             dev.state = DeviceState.DISCOVERED
+            new_hostname = dev.hostname
             logger.info(f"Device with ztp_mac {ztp_mac} successfully scanned, " +
                         "moving to DISCOVERED state")
     except Exception as e:
@@ -418,4 +435,10 @@ def discover_device(ztp_mac: str, dhcp_ip: str, iteration=-1,
         logger.debug("nrresult for ztp_mac {}: {}".format(ztp_mac, nrresult))
         raise e
 
+    nrresult_hostname = nr_filtered.run(task=set_hostname_task, new_hostname=new_hostname)
+    if nrresult_hostname.failed:
+        logger.info("Could not set hostname for ztp_mac: {}".format(
+            ztp_mac
+        ))
+
     return NornirJobResult(nrresult=nrresult)

From 40bb02e33697c220e1d74a140098642b4121013d Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 26 Mar 2020 09:42:32 +0100
Subject: [PATCH 017/102] add new interfaceconfigtype options

---
 ...5012afa7_add_new_interface_config_types.py | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 alembic/versions/8a635012afa7_add_new_interface_config_types.py

diff --git a/alembic/versions/8a635012afa7_add_new_interface_config_types.py b/alembic/versions/8a635012afa7_add_new_interface_config_types.py
new file mode 100644
index 00000000..fe8bdad1
--- /dev/null
+++ b/alembic/versions/8a635012afa7_add_new_interface_config_types.py
@@ -0,0 +1,30 @@
+"""add new interface config types
+
+Revision ID: 8a635012afa7
+Revises: 395427a732d6
+Create Date: 2020-03-26 09:21:15.439761
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8a635012afa7'
+down_revision = '395427a732d6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # ### end Alembic commands ###
+    op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'TEMPLATE' AFTER 'CUSTOM'")
+    op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'MLAG_PEER' AFTER 'TEMPLATE'")
+    op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'ACCESS_DOWNLINK' AFTER 'ACCESS_UPLINK'")
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

From b81fe198a8088a756f7ba5cd09fc14f56eabdc3e Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 26 Mar 2020 09:43:58 +0100
Subject: [PATCH 018/102] fix previous alembic rev

---
 alembic/versions/8a635012afa7_add_new_interface_config_types.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/alembic/versions/8a635012afa7_add_new_interface_config_types.py b/alembic/versions/8a635012afa7_add_new_interface_config_types.py
index fe8bdad1..dfb4687c 100644
--- a/alembic/versions/8a635012afa7_add_new_interface_config_types.py
+++ b/alembic/versions/8a635012afa7_add_new_interface_config_types.py
@@ -19,6 +19,7 @@
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     # ### end Alembic commands ###
+    op.execute("COMMIT")
     op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'TEMPLATE' AFTER 'CUSTOM'")
     op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'MLAG_PEER' AFTER 'TEMPLATE'")
     op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'ACCESS_DOWNLINK' AFTER 'ACCESS_UPLINK'")

From ae73ab59404f6270879c45c2a30e05b10fc893b0 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 26 Mar 2020 15:39:29 +0100
Subject: [PATCH 019/102] Bugfix: update interfaces of mlag peer device as well
 during init

---
 src/cnaas_nms/confpush/init_device.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 0503721d..c3e5468a 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -156,7 +156,9 @@ def init_access_device_step1(device_id: int, new_hostname: str,
             cnaas_nms.confpush.get.update_linknets(session, mlag_peer_dev.hostname)
             update_interfacedb_worker(session, dev, replace=True, delete=False,
                                       mlag_peer_hostname=mlag_peer_dev.hostname)
-            uplink_hostnames.append(mlag_peer_dev.get_uplink_peer_hostnames(session))
+            update_interfacedb_worker(session, mlag_peer_dev, replace=True, delete=False,
+                                      mlag_peer_hostname=dev.hostname)
+            uplink_hostnames += mlag_peer_dev.get_uplink_peer_hostnames(session)
             # check that both devices see the correct MLAG peer
             pre_init_check_mlag(session, dev, mlag_peer_dev)
             pre_init_check_mlag(session, mlag_peer_dev, dev)

From 45324810a89eb5d99e2a7c7e6089b4278b2b0152 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 26 Mar 2020 15:43:29 +0100
Subject: [PATCH 020/102] Bugfix, you can only get uplinks after updating
 interfaces

---
 src/cnaas_nms/confpush/init_device.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index c3e5468a..b9679050 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -149,7 +149,6 @@ def init_access_device_step1(device_id: int, new_hostname: str,
         dev = pre_init_checks(session, device_id)
 
         cnaas_nms.confpush.get.update_linknets(session, dev.hostname)  # update linknets using LLDP data
-        uplink_hostnames = dev.get_uplink_peer_hostnames(session)
 
         if mlag_peer_id and mlag_peer_new_hostname:
             mlag_peer_dev = pre_init_checks(session, mlag_peer_id)
@@ -158,6 +157,7 @@ def init_access_device_step1(device_id: int, new_hostname: str,
                                       mlag_peer_hostname=mlag_peer_dev.hostname)
             update_interfacedb_worker(session, mlag_peer_dev, replace=True, delete=False,
                                       mlag_peer_hostname=dev.hostname)
+            uplink_hostnames = dev.get_uplink_peer_hostnames(session)
             uplink_hostnames += mlag_peer_dev.get_uplink_peer_hostnames(session)
             # check that both devices see the correct MLAG peer
             pre_init_check_mlag(session, dev, mlag_peer_dev)
@@ -166,6 +166,7 @@ def init_access_device_step1(device_id: int, new_hostname: str,
             raise ValueError("mlag_peer_id and mlag_peer_new_hostname must be specified together")
         else:
             update_interfacedb_worker(session, dev, replace=True, delete=False)
+            uplink_hostnames = dev.get_uplink_peer_hostnames(session)
 
         # TODO: check compatability, same dist pair and same ports on dists
         mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, uplink_hostnames)

From 415600d93c1a4bf7b2c3565d5b86aa51c15c1b5f Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 26 Mar 2020 15:48:09 +0100
Subject: [PATCH 021/102] Bugfix, update syntax for set instead of list

---
 src/cnaas_nms/db/device.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cnaas_nms/db/device.py b/src/cnaas_nms/db/device.py
index 032da38f..2a65c0a3 100644
--- a/src/cnaas_nms/db/device.py
+++ b/src/cnaas_nms/db/device.py
@@ -238,9 +238,9 @@ def get_mlag_peer(self, session) -> Optional[Device]:
                 [x.hostname for x in peers]
             ))
         elif len(peers) == 1:
-            if self.device_type != peers[0].device_type:
+            if self.device_type != next(iter(peers)).device_type:
                 raise DeviceException("MLAG peers are not the same device type")
-            return peers[0]
+            return next(iter(peers))
         else:
             return None
 

From 66fafbd96dd28dbdce51c65eb29873d9a2645f95 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 27 Mar 2020 10:52:05 +0100
Subject: [PATCH 022/102] Do not remove warning on empty subnet in dhcpd.conf

---
 docker/dhcpd/dhcpd.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/dhcpd/dhcpd.conf b/docker/dhcpd/dhcpd.conf
index 1c40e8a4..6422ca9e 100644
--- a/docker/dhcpd/dhcpd.conf
+++ b/docker/dhcpd/dhcpd.conf
@@ -52,6 +52,7 @@ class "JUNIPER" {
   option tftp-server-name "10.0.1.3";
 }
 
+# This empty subnet is required for dhcpd to start in the cnaas docker-compose network, do not remove
 subnet 10.0.1.0 netmask 255.255.255.0 {
 }
 

From 50db3955495478c32db2a54f7d596065f3a49f0e Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 27 Mar 2020 16:30:56 +0100
Subject: [PATCH 023/102] schedule job to do init step1 for mlag peer 60s after
 step1 for original device finished

---
 src/cnaas_nms/confpush/init_device.py | 29 +++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index b9679050..118433ac 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, List
 from ipaddress import IPv4Interface
 
 from nornir.plugins.tasks import networking, text
@@ -123,6 +123,7 @@ def pre_init_check_mlag(session, dev, mlag_peer_dev):
 def init_access_device_step1(device_id: int, new_hostname: str,
                              mlag_peer_id: Optional[int] = None,
                              mlag_peer_new_hostname: Optional[str] = None,
+                             uplink_hostnames_arg: Optional[List[str]] = [],
                              job_id: Optional[str] = None,
                              scheduled_by: Optional[str] = None) -> NornirJobResult:
     """Initialize access device for management by CNaaS-NMS.
@@ -134,6 +135,8 @@ def init_access_device_step1(device_id: int, new_hostname: str,
         new_hostname: Hostname to configure on this device
         mlag_peer_id: Device ID of MLAG peer device (optional)
         mlag_peer_new_hostname: Hostname to configure on peer device (optional)
+        uplink_hostnames_arg: List of hostnames of uplink peer devices (optional)
+                              Used when initializing MLAG peer device
         job_id: job_id provided by scheduler when adding job
         scheduled_by: Username from JWT.
 
@@ -150,6 +153,7 @@ def init_access_device_step1(device_id: int, new_hostname: str,
 
         cnaas_nms.confpush.get.update_linknets(session, dev.hostname)  # update linknets using LLDP data
 
+        # If this is the first device in an MLAG pair
         if mlag_peer_id and mlag_peer_new_hostname:
             mlag_peer_dev = pre_init_checks(session, mlag_peer_id)
             cnaas_nms.confpush.get.update_linknets(session, mlag_peer_dev.hostname)
@@ -162,8 +166,12 @@ def init_access_device_step1(device_id: int, new_hostname: str,
             # check that both devices see the correct MLAG peer
             pre_init_check_mlag(session, dev, mlag_peer_dev)
             pre_init_check_mlag(session, mlag_peer_dev, dev)
+        # If this is the second device in an MLAG pair
+        elif uplink_hostnames_arg:
+            uplink_hostnames = uplink_hostnames_arg
         elif mlag_peer_id or mlag_peer_new_hostname:
             raise ValueError("mlag_peer_id and mlag_peer_new_hostname must be specified together")
+        # If this device is not part of an MLAG pair
         else:
             update_interfacedb_worker(session, dev, replace=True, delete=False)
             uplink_hostnames = dev.get_uplink_peer_hostnames(session)
@@ -259,7 +267,24 @@ def init_access_device_step1(device_id: int, new_hostname: str,
         scheduled_by=scheduled_by,
         kwargs={'device_id': device_id, 'iteration': 1})
 
-    logger.debug(f"Step 2 scheduled as ID {next_job_id}")
+    logger.info("Init step 2 for {} scheduled as job # {}".format(
+        new_hostname, next_job_id
+    ))
+
+    if mlag_peer_id and mlag_peer_new_hostname:
+        mlag_peer_job_id = scheduler.add_onetime_job(
+            'cnaas_nms.confpush.init_device:init_access_device_step1',
+            when=60,
+            scheduled_by=scheduled_by,
+            kwargs={
+                'device_id': mlag_peer_id,
+                'new_hostname': mlag_peer_new_hostname,
+                'uplink_hostnames_arg': uplink_hostnames,
+                'scheduled_by': scheduled_by
+            })
+        logger.info("MLAG peer (id {}) init scheduled as job # {}".format(
+            mlag_peer_id, mlag_peer_job_id
+        ))
 
     return NornirJobResult(
         nrresult=nrresult,

From 27501e2aeb1820566ace9061ea45ea7e1f652760 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 31 Mar 2020 10:21:20 +0200
Subject: [PATCH 024/102] set hostname during init

---
 src/cnaas_nms/confpush/init_device.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 118433ac..b6ed6951 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -81,7 +81,7 @@ def push_base_management_access(task, device_variables, job_id):
              name="Push base management config",
              replace=True,
              configuration=task.host["config"],
-             dry_run=False # TODO: temp for testing
+             dry_run=False
              )
 
 
@@ -400,13 +400,20 @@ def set_hostname_task(task, new_hostname: str):
     template_vars = {
         'hostname': new_hostname
     }
-    task.run(
+    r = task.run(
         task=text.template_file,
-        name="Configure hostname",
+        name="Generate hostname config",
         template="hostname.j2",
         path=f"{local_repo_path}/{task.host.platform}",
         **template_vars
     )
+    task.host["config"] = r.result
+    task.run(
+        task=networking.napalm_configure,
+        name="Configure hostname",
+        replace=False,
+        configuration=task.host["config"],
+    )
 
 
 @job_wrapper

From bb6c6f6c0985f2b45c5d8d494785e6e0748d583d Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 31 Mar 2020 10:25:56 +0200
Subject: [PATCH 025/102] make discover set hostname match with variable name
 from napalm

---
 src/cnaas_nms/confpush/init_device.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index b6ed6951..1970947f 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -398,7 +398,7 @@ def set_hostname_task(task, new_hostname: str):
         repo_config = yaml.safe_load(db_file)
         local_repo_path = repo_config['templates_local']
     template_vars = {
-        'hostname': new_hostname
+        'host': new_hostname
     }
     r = task.run(
         task=text.template_file,

From 9d6d422aeb98ed86885fc14c51d121eee3fa44d4 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 31 Mar 2020 15:16:05 +0200
Subject: [PATCH 026/102] fix double setting of host var in set_hostname task

---
 src/cnaas_nms/confpush/init_device.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 1970947f..5e0bb502 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -397,9 +397,7 @@ def set_hostname_task(task, new_hostname: str):
     with open('/etc/cnaas-nms/repository.yml', 'r') as db_file:
         repo_config = yaml.safe_load(db_file)
         local_repo_path = repo_config['templates_local']
-    template_vars = {
-        'host': new_hostname
-    }
+    template_vars = {}  # host is already set by nornir
     r = task.run(
         task=text.template_file,
         name="Generate hostname config",

From a82530badb12da91fbf3f7ed5938262fedbaee34 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 1 Apr 2020 08:45:51 +0200
Subject: [PATCH 027/102] Debug read timeout when changing managament ip

---
 src/cnaas_nms/confpush/init_device.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 5e0bb502..106882dd 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -263,7 +263,7 @@ def init_access_device_step1(device_id: int, new_hostname: str,
     scheduler = Scheduler()
     next_job_id = scheduler.add_onetime_job(
         'cnaas_nms.confpush.init_device:init_access_device_step2',
-        when=0,
+        when=30,
         scheduled_by=scheduled_by,
         kwargs={'device_id': device_id, 'iteration': 1})
 
@@ -406,12 +406,14 @@ def set_hostname_task(task, new_hostname: str):
         **template_vars
     )
     task.host["config"] = r.result
+    task.host.open_connection("napalm", configuration=task.nornir.config)
     task.run(
         task=networking.napalm_configure,
         name="Configure hostname",
         replace=False,
         configuration=task.host["config"],
     )
+    task.host.close_connection("napalm")
 
 
 @job_wrapper

From bcae7186ba927a7ecb4475c9ec79645c3809ad80 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 1 Apr 2020 11:10:40 +0200
Subject: [PATCH 028/102] Increase connection timeout, seems to take longer to
 commit baseconfig on arista now. Change the way we handle connection loss
 after change of management ip.

---
 src/cnaas_nms/confpush/init_device.py | 40 ++++++++++++---------------
 1 file changed, 17 insertions(+), 23 deletions(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 106882dd..8c75e997 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -75,14 +75,21 @@ def push_base_management_access(task, device_variables, job_id):
 
     task.host["config"] = r.result
     # Use extra low timeout for this since we expect to loose connectivity after changing IP
-    task.host.connection_options["napalm"] = ConnectionOptions(extras={"timeout": 5})
+    task.host.connection_options["napalm"] = ConnectionOptions(extras={"timeout": 30})
 
-    task.run(task=networking.napalm_configure,
-             name="Push base management config",
-             replace=True,
-             configuration=task.host["config"],
-             dry_run=False
-             )
+    try:
+        task.run(task=networking.napalm_configure,
+                 name="Push base management config",
+                 replace=True,
+                 configuration=task.host["config"],
+                 dry_run=False
+                 )
+    except Exception:
+        task.run(task=networking.napalm_get, getters=["facts"])
+        if not task.results[-1].failed:
+            raise InitError("Device {} did not commit new base management config".format(
+                task.host.name
+            ))
 
 
 def pre_init_checks(session, device_id) -> Device:
@@ -225,21 +232,9 @@ def init_access_device_step1(device_id: int, new_hostname: str,
     nr_filtered = nr.filter(name=hostname)
 
     # step2. push management config
-    try:
-        nrresult = nr_filtered.run(task=push_base_management_access,
-                                   device_variables=device_variables,
-                                   job_id=job_id)
-    except SessionLockedException as e:
-        # TODO: Handle this somehow?
-        pass
-    except Exception as e:
-        # Ignore exception, we expect to loose connectivity.
-        # Sometimes we get no exception here, but it's saved in result
-        # other times we get socket.timeout, pyeapi.eapilib.ConnectionError or
-        # napalm.base.exceptions.ConnectionException to handle here?
-        pass
-    if not nrresult.failed:
-        raise Exception  # we don't expect success here
+    nrresult = nr_filtered.run(task=push_base_management_access,
+                               device_variables=device_variables,
+                               job_id=job_id)
 
     with sqla_session() as session:
         dev = session.query(Device).filter(Device.id == device_id).one()
@@ -406,7 +401,6 @@ def set_hostname_task(task, new_hostname: str):
         **template_vars
     )
     task.host["config"] = r.result
-    task.host.open_connection("napalm", configuration=task.nornir.config)
     task.run(
         task=networking.napalm_configure,
         name="Configure hostname",

From 2e59c977d9a63a0ad3879eef4b519958e33a5f53 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 3 Apr 2020 15:22:31 +0200
Subject: [PATCH 029/102] Fix for when dhcp-commit hook triggers two times for
 the same device

---
 src/cnaas_nms/scheduler_mule.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/cnaas_nms/scheduler_mule.py b/src/cnaas_nms/scheduler_mule.py
index d9e02902..8f247554 100644
--- a/src/cnaas_nms/scheduler_mule.py
+++ b/src/cnaas_nms/scheduler_mule.py
@@ -62,8 +62,20 @@ def main_loop():
         for k, v in data.items():
             if k not in ['func', 'trigger', 'id', 'run_date']:
                 kwargs[k] = v
+        # Perform pre-schedule job checks
+        try:
+            for job in scheduler.get_scheduler().get_jobs():
+                # Only allow scheduling of one discover_device job at the same time
+                if job.name == 'cnaas_nms.confpush.init_device:discover_device':
+                    if job.kwargs['kwargs']['ztp_mac'] == kwargs['kwargs']['ztp_mac']:
+                        logger.debug("There is already another scheduled job to discover device {}, skipping ".
+                                     format(kwargs['kwargs']['ztp_mac']))
+                        continue
+        except Exception as e:
+            logger.exception("Unable to perform pre-schedule job checks: {}".format(e))
+
         scheduler.add_job(data['func'], trigger=data['trigger'], kwargs=kwargs,
-                          id=data['id'], run_date=data['run_date'])
+                          id=data['id'], run_date=data['run_date'], name=data['func'])
 
 
 if __name__ == '__main__':

From ff07ed37bf6ebd40deb2239a21e2d748cad7fdfd Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 3 Apr 2020 15:37:57 +0200
Subject: [PATCH 030/102] Make pre-schedule checks for job update job database
 if job was aborted

---
 src/cnaas_nms/db/job.py         |  7 +++++++
 src/cnaas_nms/scheduler_mule.py | 30 +++++++++++++++++++++++-------
 2 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/src/cnaas_nms/db/job.py b/src/cnaas_nms/db/job.py
index efb1a7f9..4de89b15 100644
--- a/src/cnaas_nms/db/job.py
+++ b/src/cnaas_nms/db/job.py
@@ -120,6 +120,13 @@ def finish_exception(self, e: Exception, traceback: str):
             logger.exception(errmsg)
             self.exception = {"error": errmsg}
 
+    def finish_abort(self, message: str):
+        logger.debug("Job {} aborted: {}".format(self.id, message))
+        self.finish_time = datetime.datetime.utcnow()
+        self.status = JobStatus.ABORTED
+        self.result = {"message": message}
+
+
     @classmethod
     def clear_jobs(cls, session):
         """Clear/release all locks in the database."""
diff --git a/src/cnaas_nms/scheduler_mule.py b/src/cnaas_nms/scheduler_mule.py
index 8f247554..627b2b43 100644
--- a/src/cnaas_nms/scheduler_mule.py
+++ b/src/cnaas_nms/scheduler_mule.py
@@ -9,6 +9,7 @@
 from cnaas_nms.plugins.pluginmanager import PluginManagerHandler
 from cnaas_nms.db.session import sqla_session
 from cnaas_nms.db.joblock import Joblock
+from cnaas_nms.db.job import Job, JobStatus
 from cnaas_nms.tools.log import get_logger
 
 
@@ -31,6 +32,26 @@ def save_coverage():
     signal.signal(signal.SIGINT, save_coverage)
 
 
+def pre_schedule_checks(scheduler, kwargs):
+    check_ok = True
+    message = ""
+    for job in scheduler.get_scheduler().get_jobs():
+        # Only allow scheduling of one discover_device job at the same time
+        if job.name == 'cnaas_nms.confpush.init_device:discover_device':
+            if job.kwargs['kwargs']['ztp_mac'] == kwargs['kwargs']['ztp_mac']:
+                message = ("There is already another scheduled job to discover device {}, skipping ".
+                           format(kwargs['kwargs']['ztp_mac']))
+                check_ok = False
+
+    if not check_ok:
+        logger.debug(message)
+        with sqla_session() as session:
+            job_entry: Job = session.query(Job).filter(Job.id == kwargs['job_id']).one_or_none()
+            job_entry.finish_abort(message)
+
+    return check_ok
+
+
 def main_loop():
     try:
         import uwsgi
@@ -64,13 +85,8 @@ def main_loop():
                 kwargs[k] = v
         # Perform pre-schedule job checks
         try:
-            for job in scheduler.get_scheduler().get_jobs():
-                # Only allow scheduling of one discover_device job at the same time
-                if job.name == 'cnaas_nms.confpush.init_device:discover_device':
-                    if job.kwargs['kwargs']['ztp_mac'] == kwargs['kwargs']['ztp_mac']:
-                        logger.debug("There is already another scheduled job to discover device {}, skipping ".
-                                     format(kwargs['kwargs']['ztp_mac']))
-                        continue
+            if not pre_schedule_checks(scheduler, kwargs):
+                continue
         except Exception as e:
             logger.exception("Unable to perform pre-schedule job checks: {}".format(e))
 

From 6031122d185358c7a099e3d78c5e3dce151d8828 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 6 Apr 2020 08:52:37 +0200
Subject: [PATCH 031/102] spelling

---
 docs/apiref/devices.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/apiref/devices.rst b/docs/apiref/devices.rst
index dda137fb..d10947ff 100644
--- a/docs/apiref/devices.rst
+++ b/docs/apiref/devices.rst
@@ -160,7 +160,7 @@ To remove a device, pass the device ID in a DELTE call:
    curl -X DELETE https://hostname/api/v1.0/device/10
 
 
-There is also the option to factory default and reboott the device
+There is also the option to factory default and reboot the device
 when removing it. This can be done like this:
 
 ::

From 811f95ea85f6f91b5f1aa2f362676b5c7e5fc5a5 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 6 Apr 2020 15:34:22 +0200
Subject: [PATCH 032/102] Add docs for MLAG init

---
 docs/apiref/devices.rst | 21 +++++++++++++++++++++
 docs/howto/index.rst    |  2 ++
 2 files changed, 23 insertions(+)

diff --git a/docs/apiref/devices.rst b/docs/apiref/devices.rst
index 2c32363f..fae19084 100644
--- a/docs/apiref/devices.rst
+++ b/docs/apiref/devices.rst
@@ -173,3 +173,24 @@ touching the device use generate_config:
 This will return both the generated configuration based on the template for
 this device type, and also a list of available vaiables that could be used
 in the template.
+
+Initialize device
+-----------------
+
+For a more detailed explanation see documentation under Howto :ref:`ztp_intro`.
+
+To initialize a single ACCESS type device:
+
+::
+
+   curl https://localhost/api/v1.0/device_init/45 -d '{"hostname": "ex2300-top", "device_type": "ACCESS"}' -X POST -H "Content-Type: application/json"
+
+The device must be in state DISCOVERED to start initialization. The device must be able to detect compatible uplink devices via LLDP for initialization to finish.
+
+To initialize a pair of ACCESS devices as an MLAG pair:
+
+::
+
+   curl https://localhost/api/v1.0/device_init/45 -d '{"hostname": "a1", "device_type": "ACCESS", "mlag_peer_id": 46, "mlag_peer_hostname": "a2"}' -X POST -H "Content-Type: application/json"
+
+For MLAG pairs the devices must be able to dectect it's peer via LLDP neighbors and compatible uplink devices for initialization to finish.
\ No newline at end of file
diff --git a/docs/howto/index.rst b/docs/howto/index.rst
index f3b67af7..a0d30aa1 100644
--- a/docs/howto/index.rst
+++ b/docs/howto/index.rst
@@ -24,6 +24,8 @@ with dry_run to preview changes::
 The API call to device_syncto will start a job running in the background on the API server. To
 show the progress/output of the job run the last command (/job) until you get a finished result.
 
+.. _ztp_intro:
+
 Zero-touch provisioning of access switch
 ----------------------------------------
 

From a128389fb312d076d34cd20bf7da2cc06546266a Mon Sep 17 00:00:00 2001
From: Kristofer Hallin <kristofer@sunet.se>
Date: Mon, 6 Apr 2020 21:30:22 +0200
Subject: [PATCH 033/102] * Missing url variable. * Avoid getting including the
 full response from the HTTPD API.

---
 src/cnaas_nms/api/firmware.py      | 2 +-
 src/cnaas_nms/confpush/firmware.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cnaas_nms/api/firmware.py b/src/cnaas_nms/api/firmware.py
index 62214938..58d4af09 100644
--- a/src/cnaas_nms/api/firmware.py
+++ b/src/cnaas_nms/api/firmware.py
@@ -139,7 +139,7 @@ def get(self) -> tuple:
         try:
             res = requests.get(get_httpd_url(),
                                verify=verify_tls())
-            json_data = json.loads(res.content)
+            json_data = json.loads(res.content)['data']
         except Exception as e:
             logger.exception(f"Exception when getting images: {e}")
             return empty_result(status='error',
diff --git a/src/cnaas_nms/confpush/firmware.py b/src/cnaas_nms/confpush/firmware.py
index c5219dfc..98dee5bd 100644
--- a/src/cnaas_nms/confpush/firmware.py
+++ b/src/cnaas_nms/confpush/firmware.py
@@ -69,6 +69,8 @@ def arista_firmware_download(task, filename: str, httpd_url: str) -> None:
     """
     logger.info('Downloading firmware for {}'.format(task.host.name))
 
+    url = httpd_url + '/' + filename
+
     try:
         with sqla_session() as session:
             dev: Device = session.query(Device).\

From d542a10c3c351307240dcad226f97ab0a97bc03f Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 7 Apr 2020 13:42:40 +0200
Subject: [PATCH 034/102] VXLAN settings: make ipv4_gw optional, make vrf
 optional unless a ipv4_gw is specified

---
 src/cnaas_nms/db/settings_fields.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/cnaas_nms/db/settings_fields.py b/src/cnaas_nms/db/settings_fields.py
index 505b201b..a15ba264 100644
--- a/src/cnaas_nms/db/settings_fields.py
+++ b/src/cnaas_nms/db/settings_fields.py
@@ -1,6 +1,6 @@
 from typing import List, Optional, Dict
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, validator
 
 
 # HOSTNAME_REGEX = r'([a-z0-9-]{1,63}\.?)+'
@@ -29,7 +29,7 @@
 ipv4_schema = Field(..., regex=f"^{IPV4_REGEX}$",
                     description="IPv4 address")
 IPV4_IF_REGEX = f"{IPV4_REGEX}" + r"\/[0-9]{1,2}"
-ipv4_if_schema = Field(..., regex=f"^{IPV4_IF_REGEX}$",
+ipv4_if_schema = Field(None, regex=f"^{IPV4_IF_REGEX}$",
                        description="IPv4 address in CIDR/prefix notation (0.0.0.0/0)")
 ipv6_schema = Field(..., regex=f"^{IPV6_REGEX}$",
                     description="IPv6 address")
@@ -40,7 +40,7 @@
 # VLAN name is alphanumeric max 32 chars on Cisco
 # should not start with number according to some Juniper doc
 VLAN_NAME_REGEX = r'^[a-zA-Z][a-zA-Z0-9-_]{0,31}$'
-vlan_name_schema = Field(..., regex=VLAN_NAME_REGEX,
+vlan_name_schema = Field(None, regex=VLAN_NAME_REGEX,
                          description="Max 32 alphanumeric chars, " +
                                      "beginning with a non-numeric character")
 vlan_id_schema = Field(..., gt=0, lt=4096, description="Numeric 802.1Q VLAN ID, 1-4095")
@@ -176,15 +176,22 @@ class f_extroute_bgp(BaseModel):
 class f_vxlan(BaseModel):
     description: str = None
     vni: int = vxlan_vni_schema
-    vrf: str = vlan_name_schema
+    vrf: Optional[str] = vlan_name_schema
     vlan_id: int = vlan_id_schema
     vlan_name: str = vlan_name_schema
-    ipv4_gw: str = ipv4_if_schema
+    ipv4_gw: Optional[str] = ipv4_if_schema
     dhcp_relays: Optional[List[f_dhcp_relay]]
     mtu: Optional[int] = mtu_schema
     groups: List[str] = []
     devices: List[str] = []
 
+    @validator('ipv4_gw')
+    def vrf_required_if_ipv4_gw_set(cls, v, values, **kwargs):
+        if v:
+            if 'vrf' not in values or not values['vrf']:
+                raise ValueError('VRF is required when specifying ipv4_gw')
+        return v
+
 
 class f_underlay(BaseModel):
     infra_lo_net: str = ipv4_if_schema

From 44673d834e3a7ae4d4b0892af4e97a71b1a527b8 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 7 Apr 2020 13:45:52 +0200
Subject: [PATCH 035/102] update docs for vxlan settings

---
 docs/reporef/index.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reporef/index.rst b/docs/reporef/index.rst
index 9610e9e3..06cd4008 100644
--- a/docs/reporef/index.rst
+++ b/docs/reporef/index.rst
@@ -181,10 +181,10 @@ Contains a dictinary called "vxlans", which in turn has one dictinoary per vxlan
 name is the dictionary key and dictionaly values are:
 
   * vni: VXLAN ID, 1-16777215
-  * vrf: VRF name
+  * vrf: VRF name. Optional unless ipv4_gw is also specified.
   * vlan_id: VLAN ID, 1-4095
   * vlan_name: VLAN name, single word/no spaces, max 31 characters
-  * ipv4_gw: IPv4 address with CIDR netmask, ex: 192.168.0.1/24
+  * ipv4_gw: IPv4 address with CIDR netmask, ex: 192.168.0.1/24. Optional.
   * groups: List of group names where this VXLAN/VLAN should be provisioned. If you select an
     access switch the parent dist switch should be automatically provisioned.
 

From e397faca549c342310ac5edc0ba3176e007184cf Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 9 Apr 2020 09:14:01 +0200
Subject: [PATCH 036/102] Add get previous method to Job, and add API call to
 get previous config

---
 src/cnaas_nms/api/device.py | 48 +++++++++++++++++++++++++++++++
 src/cnaas_nms/db/job.py     | 56 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 103 insertions(+), 1 deletion(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 8c0810d3..ad8215b5 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -1,4 +1,5 @@
 import json
+import datetime
 from typing import Optional
 
 from flask import request, make_response
@@ -12,6 +13,7 @@
 import cnaas_nms.confpush.underlay
 from cnaas_nms.api.generic import build_filter, empty_result
 from cnaas_nms.db.device import Device, DeviceState, DeviceType
+from cnaas_nms.db.job import Job, JobNotFoundError, InvalidJobError
 from cnaas_nms.db.session import sqla_session
 from cnaas_nms.db.settings import get_groups
 from cnaas_nms.scheduler.scheduler import Scheduler
@@ -412,10 +414,56 @@ def get(self, hostname: str):
         return result
 
 
+class DevicePreviousConfigApi(Resource):
+    @jwt_required
+    @device_api.param('job_id')
+    @device_api.param('previous')
+    @device_api.param('before')
+    def get(self, hostname: str):
+        args = request.args
+        result = empty_result()
+        result['data'] = {'config': None}
+        if not Device.valid_hostname(hostname):
+            return empty_result(
+                status='error',
+                data=f"Invalid hostname specified"
+            ), 400
+
+        kwargs = {}
+        if 'job_id' in args:
+            try:
+                kwargs['job_id'] = int(args['job_id'])
+            except Exception:
+                return empty_result('error', "job_id must be an integer"), 400
+        elif 'previous' in args:
+            try:
+                kwargs['previous'] = int(args['previous'])
+            except Exception:
+                return empty_result('error', "previous must be an integer"), 400
+        elif 'before' in args:
+            try:
+                kwargs['before'] = datetime.datetime.fromisoformat(args['before'])
+            except Exception:
+                return empty_result('error', "before must be a valid ISO format date time string"), 400
+
+        with sqla_session() as session:
+            try:
+                result['data'] = Job.get_previous_config(session, hostname, **kwargs)
+            except JobNotFoundError as e:
+                return empty_result('error', str(e)), 404
+            except InvalidJobError as e:
+                return empty_result('error', str(e)), 500
+            except Exception as e:
+                return empty_result('error', "Unhandled exception: {}".format(e)), 500
+
+        return result
+
+
 # Devices
 device_api.add_resource(DeviceByIdApi, '/<int:device_id>')
 device_api.add_resource(DeviceByHostnameApi, '/<string:hostname>')
 device_api.add_resource(DeviceConfigApi, '/<string:hostname>/generate_config')
+device_api.add_resource(DevicePreviousConfigApi, '/<string:hostname>/previous_config')
 device_api.add_resource(DeviceApi, '')
 devices_api.add_resource(DevicesApi, '')
 device_init_api.add_resource(DeviceInitApi, '/<int:device_id>')
diff --git a/src/cnaas_nms/db/job.py b/src/cnaas_nms/db/job.py
index 4de89b15..99862b11 100644
--- a/src/cnaas_nms/db/job.py
+++ b/src/cnaas_nms/db/job.py
@@ -1,7 +1,7 @@
 import enum
 import datetime
 import json
-from typing import Optional
+from typing import Optional, Dict
 
 from sqlalchemy import Column, Integer, Unicode, SmallInteger
 from sqlalchemy import Enum, DateTime
@@ -21,6 +21,14 @@
 logger = get_logger()
 
 
+class JobNotFoundError(Exception):
+    pass
+
+
+class InvalidJobError(Exception):
+    pass
+
+
 class JobStatus(enum.Enum):
     UNKNOWN = 0
     SCHEDULED = 1
@@ -149,3 +157,49 @@ def clear_jobs(cls, session):
                     "Job found in past SCHEDULED state at startup moved to ABORTED, id: {}".
                     format(job.id))
                 job.status = JobStatus.ABORTED
+
+    @classmethod
+    def get_previous_config(cls, session, hostname: str, previous: Optional[int] = None,
+                            job_id: Optional[int] = None,
+                            before: Optional[datetime.datetime] = None) -> Dict[str, str]:
+        """
+
+        Args:
+            session:
+            hostname:
+            previous:
+            job_id:
+            before:
+
+        Returns:
+            Returns a result dict with keys: config, job_id and finish_time
+
+        """
+        result = {}
+        query_part = session.query(Job).filter(Job.function_name == 'sync_devices'). \
+            filter(Job.result.has_key('devices')).filter(Job.result['devices'].has_key(hostname))
+
+        if job_id and type(job_id) == int:
+            query_part = query_part.filter(Job.id == job_id)
+        elif previous and type(previous) == int:
+            query_part = query_part.order_by(Job.id.desc()).offset(previous)
+        elif before and type(before) == datetime.datetime:
+            query_part = query_part.filter(Job.finish_time < before).order_by(Job.id.desc())
+        else:
+            query_part = query_part.order_by(Job.id.desc())
+
+        job: Job = query_part.first()
+        if not job:
+            raise JobNotFoundError("No matching job found")
+
+        result['job_id'] = job.id
+        result['finish_time'] = job.finish_time.isoformat()
+
+        if 'job_tasks' not in job.result['devices'][hostname]:
+            raise InvalidJobError("Invalid job data found in database: missing job_tasks")
+
+        for task in job.result['devices'][hostname]['job_tasks']:
+            if task['task_name'] == 'Generate device config':
+                result['config'] = task['result']
+
+        return result

From 0638cebc68712912e36920d1be07189f96ee9bd2 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 9 Apr 2020 14:52:43 +0200
Subject: [PATCH 037/102] Start building features to apply old config to a
 device

---
 src/cnaas_nms/api/device.py            | 69 ++++++++++++++++++++++++++
 src/cnaas_nms/confpush/sync_devices.py | 69 ++++++++++++++++++++++++++
 src/cnaas_nms/db/job.py                | 15 +++---
 3 files changed, 145 insertions(+), 8 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index ad8215b5..1f40642e 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -74,6 +74,13 @@
     'resync': fields.Boolean(required=False)
 })
 
+device_restore_model = device_api.model('device_restore', {
+    'dry_run': fields.Boolean(required=False),
+    'job_id': fields.Integer(required=False),
+    'previous': fields.Integer(required=False),
+    'before': fields.DateTime(required=False)
+})
+
 
 class DeviceByIdApi(Resource):
     @jwt_required
@@ -458,6 +465,68 @@ def get(self, hostname: str):
 
         return result
 
+    @jwt_required
+    @device_api.expect(device_restore_model)
+    def post(self, hostname: str):
+        """Restore configuration to previous version"""
+        json_data = request.get_json()
+        if not Device.valid_hostname(hostname):
+            return empty_result(
+                status='error',
+                data=f"Invalid hostname specified"
+            ), 400
+
+        kwargs = {}
+        if 'job_id' in json_data:
+            try:
+                kwargs['job_id'] = int(json_data['job_id'])
+            except Exception:
+                return empty_result('error', "job_id must be an integer"), 400
+        elif 'previous' in json_data:
+            try:
+                kwargs['previous'] = int(json_data['previous'])
+            except Exception:
+                return empty_result('error', "previous must be an integer"), 400
+        elif 'before' in json_data:
+            try:
+                kwargs['before'] = datetime.datetime.fromisoformat(json_data['before'])
+            except Exception:
+                return empty_result('error', "before must be a valid ISO format date time string"), 400
+
+        with sqla_session() as session:
+            try:
+                config = Job.get_previous_config(session, hostname, **kwargs)['config']
+            except JobNotFoundError as e:
+                return empty_result('error', str(e)), 404
+            except InvalidJobError as e:
+                return empty_result('error', str(e)), 500
+            except Exception as e:
+                return empty_result('error', "Unhandled exception: {}".format(e)), 500
+
+        if not config:
+            return empty_result('error', "No config found in this job"), 500
+
+        if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \
+                and not json_data['dry_run']:
+            dry_run = False
+        else:
+            dry_run = True
+
+        scheduler = Scheduler()
+        job_id = scheduler.add_onetime_job(
+            'cnaas_nms.confpush.sync_devices:apply_config',
+            when=1,
+            scheduled_by=get_jwt_identity(),
+            hostname=hostname,
+            config=config,
+            dry_run=dry_run
+        )
+
+        res = empty_result(data=f"Scheduled job to restore {hostname}")
+        res['job_id'] = job_id
+
+        return res, 200
+
 
 # Devices
 device_api.add_resource(DeviceByIdApi, '/<int:device_id>')
diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 79618009..6e71b1e8 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -9,6 +9,7 @@
 from nornir.plugins.functions.text import print_result
 from nornir.core.filter import F
 from nornir.core.task import MultiResult
+from sqlalchemy import or_
 
 import cnaas_nms.db.helper
 import cnaas_nms.confpush.nornir_helper
@@ -584,3 +585,71 @@ def exclude_filter(host, exclude_list=failed_hosts+unchanged_hosts):
             )
 
     return NornirJobResult(nrresult=nrresult, next_job_id=next_job_id, change_score=total_change_score)
+
+
+def push_static_config(task, config: str, dry_run: bool = True,
+                       job_id: Optional[str] = None,
+                       scheduled_by: Optional[str] = None):
+    """
+    Nornir task to push static config to device
+
+    Args:
+        task: nornir task, sent by nornir when doing .run()
+        config: static config to apply
+        dry_run: Don't commit config to device, just do compare/diff
+        scheduled_by: username that triggered job
+
+    Returns:
+    """
+    set_thread_data(job_id)
+    logger = get_logger()
+
+    logger.debug("Push static config to device: {}".format(task.host.name))
+
+    task.run(task=networking.napalm_configure,
+             name="Push static config",
+             replace=True,
+             configuration=config,
+             dry_run=dry_run
+             )
+
+
+@job_wrapper
+def apply_config(hostname: str, config: str, dry_run: bool,
+                 job_id: Optional[int] = None,
+                 scheduled_by: Optional[str] = None) -> NornirJobResult:
+    """Apply a static configuration (from backup etc) to a device.
+
+    Args:
+        hostname: Specify a single host by hostname to synchronize
+        config: Static configuration to apply
+        dry_run: Don't commit config to device
+        job_id: Job ID number
+        scheduled_by: Username from JWT
+
+    Returns:
+        NornirJobResult
+    """
+    logger = get_logger()
+
+    with sqla_session() as session:
+        dev: Device = session.query(Device).filter(Device.hostname == hostname).one_or_none()
+        if not dev:
+            raise Exception("Device {} not found, apply_config aborting".format(hostname))
+        elif not (dev.state == DeviceState.MANAGED or dev.state == DeviceState.UNMANAGED):
+            raise Exception("Device {} is in invalid state: {}".format(hostname, dev.state))
+        dev.state = DeviceState.UNMANAGED
+
+    nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
+    nr_filtered = nr.filter(name=hostname).filter(managed=False)
+
+    try:
+        nrresult = nr_filtered.run(task=push_static_config,
+                                   config=config,
+                                   dry_run=dry_run,
+                                   job_id=job_id)
+    except Exception as e:
+        logger.exception("Exception in apply_config: {}".format(e))
+    
+    return NornirJobResult(nrresult=nrresult)
+
diff --git a/src/cnaas_nms/db/job.py b/src/cnaas_nms/db/job.py
index 99862b11..9eaa84a1 100644
--- a/src/cnaas_nms/db/job.py
+++ b/src/cnaas_nms/db/job.py
@@ -162,18 +162,17 @@ def clear_jobs(cls, session):
     def get_previous_config(cls, session, hostname: str, previous: Optional[int] = None,
                             job_id: Optional[int] = None,
                             before: Optional[datetime.datetime] = None) -> Dict[str, str]:
-        """
+        """Get full configuration for a device from a previous job.
 
         Args:
-            session:
-            hostname:
-            previous:
-            job_id:
-            before:
+            session: sqla_session
+            hostname: hostname of device to get config for
+            previous: number of revisions back to get config from
+            job_id: specific job to get config from
+            before: date to get config before
 
         Returns:
             Returns a result dict with keys: config, job_id and finish_time
-
         """
         result = {}
         query_part = session.query(Job).filter(Job.function_name == 'sync_devices'). \
@@ -193,7 +192,7 @@ def get_previous_config(cls, session, hostname: str, previous: Optional[int] = N
             raise JobNotFoundError("No matching job found")
 
         result['job_id'] = job.id
-        result['finish_time'] = job.finish_time.isoformat()
+        result['finish_time'] = job.finish_time.isoformat(timespec='seconds')
 
         if 'job_tasks' not in job.result['devices'][hostname]:
             raise InvalidJobError("Invalid job data found in database: missing job_tasks")

From 9305e73574e019e0c29e91e1849fc233fd405e19 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 10 Apr 2020 09:45:01 +0200
Subject: [PATCH 038/102] Only allow restore to a specific job_id since
 previous=n count might change during the process of restoring if other users
 are working with the system. Only allow revert to successful jobs.

---
 src/cnaas_nms/api/device.py            | 26 ++++++++++----------------
 src/cnaas_nms/confpush/sync_devices.py |  5 +++--
 src/cnaas_nms/db/job.py                |  5 ++++-
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 1f40642e..3ffe1d2c 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -76,9 +76,7 @@
 
 device_restore_model = device_api.model('device_restore', {
     'dry_run': fields.Boolean(required=False),
-    'job_id': fields.Integer(required=False),
-    'previous': fields.Integer(required=False),
-    'before': fields.DateTime(required=False)
+    'job_id': fields.Integer(required=True),
 })
 
 
@@ -476,26 +474,19 @@ def post(self, hostname: str):
                 data=f"Invalid hostname specified"
             ), 400
 
-        kwargs = {}
         if 'job_id' in json_data:
             try:
-                kwargs['job_id'] = int(json_data['job_id'])
+                job_id = int(json_data['job_id'])
             except Exception:
                 return empty_result('error', "job_id must be an integer"), 400
-        elif 'previous' in json_data:
-            try:
-                kwargs['previous'] = int(json_data['previous'])
-            except Exception:
-                return empty_result('error', "previous must be an integer"), 400
-        elif 'before' in json_data:
-            try:
-                kwargs['before'] = datetime.datetime.fromisoformat(json_data['before'])
-            except Exception:
-                return empty_result('error', "before must be a valid ISO format date time string"), 400
+        else:
+            return empty_result('error', "job_id must be specified"), 400
 
         with sqla_session() as session:
             try:
-                config = Job.get_previous_config(session, hostname, **kwargs)['config']
+                prev_config_result = Job.get_previous_config(session, hostname, job_id=job_id)
+                config = prev_config_result['config']
+                failed = prev_config_result['failed']
             except JobNotFoundError as e:
                 return empty_result('error', str(e)), 404
             except InvalidJobError as e:
@@ -506,6 +497,9 @@ def post(self, hostname: str):
         if not config:
             return empty_result('error', "No config found in this job"), 500
 
+        if failed:
+            return empty_result('error', "The specified job_id has a failed status"), 400
+
         if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \
                 and not json_data['dry_run']:
             dry_run = False
diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 6e71b1e8..a90ef276 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -635,10 +635,11 @@ def apply_config(hostname: str, config: str, dry_run: bool,
     with sqla_session() as session:
         dev: Device = session.query(Device).filter(Device.hostname == hostname).one_or_none()
         if not dev:
-            raise Exception("Device {} not found, apply_config aborting".format(hostname))
+            raise Exception("Device {} not found".format(hostname))
         elif not (dev.state == DeviceState.MANAGED or dev.state == DeviceState.UNMANAGED):
             raise Exception("Device {} is in invalid state: {}".format(hostname, dev.state))
         dev.state = DeviceState.UNMANAGED
+        dev.synchronized = False
 
     nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
     nr_filtered = nr.filter(name=hostname).filter(managed=False)
@@ -650,6 +651,6 @@ def apply_config(hostname: str, config: str, dry_run: bool,
                                    job_id=job_id)
     except Exception as e:
         logger.exception("Exception in apply_config: {}".format(e))
-    
+
     return NornirJobResult(nrresult=nrresult)
 
diff --git a/src/cnaas_nms/db/job.py b/src/cnaas_nms/db/job.py
index 9eaa84a1..1395e84f 100644
--- a/src/cnaas_nms/db/job.py
+++ b/src/cnaas_nms/db/job.py
@@ -194,11 +194,14 @@ def get_previous_config(cls, session, hostname: str, previous: Optional[int] = N
         result['job_id'] = job.id
         result['finish_time'] = job.finish_time.isoformat(timespec='seconds')
 
-        if 'job_tasks' not in job.result['devices'][hostname]:
+        if 'job_tasks' not in job.result['devices'][hostname] or \
+                'failed' not in job.result['devices'][hostname]:
             raise InvalidJobError("Invalid job data found in database: missing job_tasks")
 
         for task in job.result['devices'][hostname]['job_tasks']:
             if task['task_name'] == 'Generate device config':
                 result['config'] = task['result']
 
+        result['failed'] = job.result['devices'][hostname]['failed']
+
         return result

From e8fe49f9579ecc5977a6cebcc85014cddba79b42 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 10 Apr 2020 12:26:00 +0200
Subject: [PATCH 039/102] Fix authentication for devices in unmanaged state

---
 src/cnaas_nms/api/device.py                           | 11 ++++++-----
 .../confpush/nornir_plugins/cnaas_inventory.py        |  2 ++
 src/cnaas_nms/confpush/sync_devices.py                |  2 +-
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 3ffe1d2c..76ee0a74 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -468,6 +468,7 @@ def get(self, hostname: str):
     def post(self, hostname: str):
         """Restore configuration to previous version"""
         json_data = request.get_json()
+        apply_kwargs = {'hostname': hostname}
         if not Device.valid_hostname(hostname):
             return empty_result(
                 status='error',
@@ -502,18 +503,18 @@ def post(self, hostname: str):
 
         if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \
                 and not json_data['dry_run']:
-            dry_run = False
+            apply_kwargs['dry_run'] = False
         else:
-            dry_run = True
+            apply_kwargs['dry_run'] = True
+
+        apply_kwargs['config'] = config
 
         scheduler = Scheduler()
         job_id = scheduler.add_onetime_job(
             'cnaas_nms.confpush.sync_devices:apply_config',
             when=1,
             scheduled_by=get_jwt_identity(),
-            hostname=hostname,
-            config=config,
-            dry_run=dry_run
+            kwargs=apply_kwargs,
         )
 
         res = empty_result(data=f"Scheduled job to restore {hostname}")
diff --git a/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py b/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py
index 09be9292..33fd2d3b 100644
--- a/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py
+++ b/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py
@@ -82,6 +82,8 @@ def __init__(self, **kwargs):
         username, password = self._get_credentials('MANAGED')
         groups['S_MANAGED']['username'] = username
         groups['S_MANAGED']['password'] = password
+        groups['S_UNMANAGED']['username'] = username
+        groups['S_UNMANAGED']['password'] = password
 
         defaults = {'data': {'k': 'v'}}
         super().__init__(hosts=hosts, groups=groups, defaults=defaults,
diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index a90ef276..cff96edc 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -623,7 +623,7 @@ def apply_config(hostname: str, config: str, dry_run: bool,
     Args:
         hostname: Specify a single host by hostname to synchronize
         config: Static configuration to apply
-        dry_run: Don't commit config to device
+        dry_run: Set to false to actually apply config to device
         job_id: Job ID number
         scheduled_by: Username from JWT
 

From 24fc69b3ba0cb40314c0160b2bfc734f67170fd8 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 10 Apr 2020 12:51:04 +0200
Subject: [PATCH 040/102] Add docs for view / restore previous configuration

---
 docs/apiref/devices.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/apiref/devices.rst b/docs/apiref/devices.rst
index ed020992..926ee8f9 100644
--- a/docs/apiref/devices.rst
+++ b/docs/apiref/devices.rst
@@ -183,6 +183,34 @@ This will return both the generated configuration based on the template for
 this device type, and also a list of available vaiables that could be used
 in the template.
 
+View previous config
+--------------------
+
+You can also view previous versions of the configuration for a device. All
+previous configurations are saved in the job database and can be found using
+either a specific Job ID (using job_id=), a number of steps to walk backward
+to find a previous configuration (previous=), or using a date to find the last
+configuration applied to the device before that date.
+
+::
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/previous_config?before=2020-04-07T12:03:05"
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/previous_config?previous=1"
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/previous_config?job_id=12"
+
+If you want to restore a device to a previous configuration you can send a POST:
+
+::
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/previous_config" -X POST -d '{"job_id": 12, "dry_run": true}' -H "Content-Type: application/json"
+
+When sending a POST you must specify an exact job_id to restore. The job must
+have finished with a successful status for the specified device. The device
+will change to UNMANAGED state since it's no longer in sync with current
+templates and settings.
+
 Initialize device
 -----------------
 

From 840d5fa23c2593f1435dee3f0203211a24028967 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 13 Apr 2020 10:38:08 +0200
Subject: [PATCH 041/102] Better error when no config was found

---
 src/cnaas_nms/api/device.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 76ee0a74..33d19fe2 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -469,6 +469,7 @@ def post(self, hostname: str):
         """Restore configuration to previous version"""
         json_data = request.get_json()
         apply_kwargs = {'hostname': hostname}
+        config = None
         if not Device.valid_hostname(hostname):
             return empty_result(
                 status='error',
@@ -486,8 +487,9 @@ def post(self, hostname: str):
         with sqla_session() as session:
             try:
                 prev_config_result = Job.get_previous_config(session, hostname, job_id=job_id)
-                config = prev_config_result['config']
                 failed = prev_config_result['failed']
+                if not failed and 'config' in prev_config_result:
+                    config = prev_config_result['config']
             except JobNotFoundError as e:
                 return empty_result('error', str(e)), 404
             except InvalidJobError as e:
@@ -495,12 +497,12 @@ def post(self, hostname: str):
             except Exception as e:
                 return empty_result('error', "Unhandled exception: {}".format(e)), 500
 
-        if not config:
-            return empty_result('error', "No config found in this job"), 500
-
         if failed:
             return empty_result('error', "The specified job_id has a failed status"), 400
 
+        if not config:
+            return empty_result('error', "No config found in this job"), 500
+
         if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \
                 and not json_data['dry_run']:
             apply_kwargs['dry_run'] = False

From 651c0d10d4090f98af612d2a9cf9da81cd6b6e69 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 13 Apr 2020 10:39:22 +0200
Subject: [PATCH 042/102] Add integration test for previous_config (not
 unittest since job status needs to be polled)

---
 src/cnaas_nms/api/tests/test_api.py |  7 +------
 test/integrationtests.py            | 23 +++++++++++++++++++++++
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/cnaas_nms/api/tests/test_api.py b/src/cnaas_nms/api/tests/test_api.py
index 66cc2360..0262344d 100644
--- a/src/cnaas_nms/api/tests/test_api.py
+++ b/src/cnaas_nms/api/tests/test_api.py
@@ -1,8 +1,8 @@
-import pprint
 import shutil
 import yaml
 import pkg_resources
 import os
+import time
 
 import unittest
 import cnaas_nms.api.app
@@ -20,11 +20,6 @@ def setUp(self):
         self.app = cnaas_nms.api.app.app
         self.app.wsgi_app = TestAppWrapper(self.app.wsgi_app, self.jwt_auth_token)
         self.client = self.app.test_client()
-#        self.tmp_postgres = PostgresTemporaryInstance()
-
-    def tearDown(self):
-#        self.tmp_postgres.shutdown()
-        pass
 
     def test_get_single_device(self):
         hostname = "eosdist1"
diff --git a/test/integrationtests.py b/test/integrationtests.py
index 3d4cbb69..597e7e02 100644
--- a/test/integrationtests.py
+++ b/test/integrationtests.py
@@ -240,6 +240,29 @@ def test_8_sysversion(self):
         )
         self.assertEqual(r.status_code, 200, "Failed to get CNaaS-NMS version")
 
+    def test_9_get_prev_config(self):
+        hostname = "eosaccess"
+        r = requests.get(
+            f"{URL}/api/v1.0/device/{hostname}/previous_config?previous=1"
+        )
+        self.assertEqual(r.status_code, 200)
+        prev_job_id = r.json()['data']['job_id']
+        if r.json()['data']['failed']:
+            return
+        data = {
+            "job_id": prev_job_id,
+            "dry_run": True
+        }
+        r = requests.post(
+            "/api/v1.0/device/{}/previous_config".format(hostname),
+            headers=AUTH_HEADER,
+            verify=TLS_VERIFY,
+            json=data
+        )
+        self.assertEqual(r.status_code, 200)
+        restore_job_id = r.json()['job_id']
+        job = self.check_jobid(restore_job_id)
+        self.assertFalse(job['result']['devices'][hostname]['failed'])
 
 
 if __name__ == '__main__':

From ff40952ab902c0faa03b5f91789e16a8e7a15d63 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 13 Apr 2020 10:39:47 +0200
Subject: [PATCH 043/102] Don't update device status on dry_run

---
 src/cnaas_nms/confpush/sync_devices.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index cff96edc..004a6165 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -638,8 +638,9 @@ def apply_config(hostname: str, config: str, dry_run: bool,
             raise Exception("Device {} not found".format(hostname))
         elif not (dev.state == DeviceState.MANAGED or dev.state == DeviceState.UNMANAGED):
             raise Exception("Device {} is in invalid state: {}".format(hostname, dev.state))
-        dev.state = DeviceState.UNMANAGED
-        dev.synchronized = False
+        if not dry_run:
+            dev.state = DeviceState.UNMANAGED
+            dev.synchronized = False
 
     nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
     nr_filtered = nr.filter(name=hostname).filter(managed=False)

From 4f9d9c3499403d2a445c50b8eb681b78ab652587 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 13 Apr 2020 10:45:03 +0200
Subject: [PATCH 044/102] Fix integrationtest urls, headers

---
 test/integrationtests.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/integrationtests.py b/test/integrationtests.py
index 597e7e02..7ee457cd 100644
--- a/test/integrationtests.py
+++ b/test/integrationtests.py
@@ -243,7 +243,9 @@ def test_8_sysversion(self):
     def test_9_get_prev_config(self):
         hostname = "eosaccess"
         r = requests.get(
-            f"{URL}/api/v1.0/device/{hostname}/previous_config?previous=1"
+            f"{URL}/api/v1.0/device/{hostname}/previous_config?previous=1",
+            headers=AUTH_HEADER,
+            verify=TLS_VERIFY
         )
         self.assertEqual(r.status_code, 200)
         prev_job_id = r.json()['data']['job_id']
@@ -254,7 +256,7 @@ def test_9_get_prev_config(self):
             "dry_run": True
         }
         r = requests.post(
-            "/api/v1.0/device/{}/previous_config".format(hostname),
+            f"{URL}/api/v1.0/device/{hostname}/previous_config",
             headers=AUTH_HEADER,
             verify=TLS_VERIFY,
             json=data

From 9f60421eac46c23f35f5c23b7ac5815c817d6b54 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 13 Apr 2020 10:58:04 +0200
Subject: [PATCH 045/102] Make sure to wait for syncto jobs, so get_prev_config
 can do it's job

---
 test/integrationtests.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/integrationtests.py b/test/integrationtests.py
index 7ee457cd..f3773841 100644
--- a/test/integrationtests.py
+++ b/test/integrationtests.py
@@ -183,6 +183,7 @@ def test_3_syncto_access(self):
             verify=TLS_VERIFY
         )
         self.assertEqual(r.status_code, 200, "Failed to do sync_to access")
+        self.check_jobid(r.json()['job_id'])
         r = requests.post(
             f'{URL}/api/v1.0/device_syncto',
             headers=AUTH_HEADER,
@@ -190,6 +191,7 @@ def test_3_syncto_access(self):
             verify=TLS_VERIFY
         )
         self.assertEqual(r.status_code, 200, "Failed to do sync_to access")
+        self.check_jobid(r.json()['job_id'])
 
     def test_4_syncto_dist(self):
         r = requests.post(
@@ -199,6 +201,7 @@ def test_4_syncto_dist(self):
             verify=TLS_VERIFY
         )
         self.assertEqual(r.status_code, 200, "Failed to do sync_to dist")
+        self.check_jobid(r.json()['job_id'])
 
     def test_5_genconfig(self):
         r = requests.get(

From d70e07f3da3114168df91bc2bbcd62f14f4f9a1a Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 13 Apr 2020 11:13:05 +0200
Subject: [PATCH 046/102] Make setUp run only once. Trying to fix devicelock
 issue for refresh templates

---
 test/integrationtests.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/test/integrationtests.py b/test/integrationtests.py
index f3773841..595fc1a3 100644
--- a/test/integrationtests.py
+++ b/test/integrationtests.py
@@ -20,8 +20,9 @@
 
 
 class GetTests(unittest.TestCase):
-    def setUp(self):
-        self.assertTrue(self.wait_connect(), "Connection to API failed")
+    @classmethod
+    def setUpClass(cls):
+        cls.assertTrue(cls.wait_connect(), "Connection to API failed")
 
         r = requests.put(
             f'{URL}/api/v1.0/repository/templates',
@@ -30,7 +31,7 @@ def setUp(self):
             verify=TLS_VERIFY
         )
         print("Template refresh status: {}".format(r.status_code))
-        self.assertEqual(r.status_code, 200, "Failed to refresh templates")
+        cls.assertEqual(r.status_code, 200, "Failed to refresh templates")
         r = requests.put(
             f'{URL}/api/v1.0/repository/settings',
             headers=AUTH_HEADER,
@@ -38,9 +39,9 @@ def setUp(self):
             verify=TLS_VERIFY
         )
         print("Settings refresh status: {}".format(r.status_code))
-        self.assertEqual(r.status_code, 200, "Failed to refresh settings")
+        cls.assertEqual(r.status_code, 200, "Failed to refresh settings")
 
-    def wait_connect(self):
+    def wait_connect(self) -> bool:
         for i in range(100):
             try:
                 r = requests.get(

From ff60c1e32e3ca565b6407f3f054878154491a8a7 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 13 Apr 2020 12:13:11 +0200
Subject: [PATCH 047/102] Restructure setup/test of integrationtests

---
 test/integrationtests.py | 60 +++++++++++++++++++---------------------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/test/integrationtests.py b/test/integrationtests.py
index 595fc1a3..3f9020a4 100644
--- a/test/integrationtests.py
+++ b/test/integrationtests.py
@@ -22,26 +22,6 @@
 class GetTests(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.assertTrue(cls.wait_connect(), "Connection to API failed")
-
-        r = requests.put(
-            f'{URL}/api/v1.0/repository/templates',
-            headers=AUTH_HEADER,
-            json={"action": "refresh"},
-            verify=TLS_VERIFY
-        )
-        print("Template refresh status: {}".format(r.status_code))
-        cls.assertEqual(r.status_code, 200, "Failed to refresh templates")
-        r = requests.put(
-            f'{URL}/api/v1.0/repository/settings',
-            headers=AUTH_HEADER,
-            json={"action": "refresh"},
-            verify=TLS_VERIFY
-        )
-        print("Settings refresh status: {}".format(r.status_code))
-        cls.assertEqual(r.status_code, 200, "Failed to refresh settings")
-
-    def wait_connect(self) -> bool:
         for i in range(100):
             try:
                 r = requests.get(
@@ -58,7 +38,7 @@ def wait_connect(self) -> bool:
                 else:
                     print("Bad status code {}, retrying in 1 second...".format(r.status_code))
                     time.sleep(1)
-        return False
+        assert False, "Failed to test connection to API"
 
     def wait_for_discovered_device(self):
         for i in range(100):
@@ -91,7 +71,25 @@ def check_jobid(self, job_id):
             else:
                 raise Exception
 
-    def test_0_init_dist(self):
+    def test_00_sync(self):
+        r = requests.put(
+            f'{URL}/api/v1.0/repository/templates',
+            headers=AUTH_HEADER,
+            json={"action": "refresh"},
+            verify=TLS_VERIFY
+        )
+        print("Template refresh status: {}".format(r.status_code))
+        self.assertEqual(r.status_code, 200, "Failed to refresh templates")
+        r = requests.put(
+            f'{URL}/api/v1.0/repository/settings',
+            headers=AUTH_HEADER,
+            json={"action": "refresh"},
+            verify=TLS_VERIFY
+        )
+        print("Settings refresh status: {}".format(r.status_code))
+        self.assertEqual(r.status_code, 200, "Failed to refresh settings")
+
+    def test_01_init_dist(self):
         new_dist_data = {
             "hostname": "eosdist1",
             "management_ip": "10.100.3.101",
@@ -129,7 +127,7 @@ def test_0_init_dist(self):
         )
         self.assertEqual(r.status_code, 200, "Failed to add mgmtdomain")
 
-    def test_1_ztp(self):
+    def test_02_ztp(self):
         hostname, device_id = self.wait_for_discovered_device()
         print("Discovered hostname, id: {}, {}".format(hostname, device_id))
         self.assertTrue(hostname, "No device in state discovered found for ZTP")
@@ -152,7 +150,7 @@ def test_1_ztp(self):
         self.assertFalse(result_step2['devices']['eosaccess']['failed'],
                          "Could not reach device after ZTP")
 
-    def test_2_interfaces(self):
+    def test_03_interfaces(self):
         r = requests.get(
             f'{URL}/api/v1.0/device/eosaccess/interfaces',
             headers=AUTH_HEADER,
@@ -176,7 +174,7 @@ def test_2_interfaces(self):
         )
         self.assertEqual(r.status_code, 200, "Failed to update interface")
 
-    def test_3_syncto_access(self):
+    def test_04_syncto_access(self):
         r = requests.post(
             f'{URL}/api/v1.0/device_syncto',
             headers=AUTH_HEADER,
@@ -194,7 +192,7 @@ def test_3_syncto_access(self):
         self.assertEqual(r.status_code, 200, "Failed to do sync_to access")
         self.check_jobid(r.json()['job_id'])
 
-    def test_4_syncto_dist(self):
+    def test_05_syncto_dist(self):
         r = requests.post(
             f'{URL}/api/v1.0/device_syncto',
             headers=AUTH_HEADER,
@@ -204,7 +202,7 @@ def test_4_syncto_dist(self):
         self.assertEqual(r.status_code, 200, "Failed to do sync_to dist")
         self.check_jobid(r.json()['job_id'])
 
-    def test_5_genconfig(self):
+    def test_06_genconfig(self):
         r = requests.get(
             f'{URL}/api/v1.0/device/eosdist1/generate_config',
             headers=AUTH_HEADER,
@@ -212,7 +210,7 @@ def test_5_genconfig(self):
         )
         self.assertEqual(r.status_code, 200, "Failed to generate config for eosdist1")
 
-    def test_6_plugins(self):
+    def test_07_plugins(self):
         r = requests.get(
             f'{URL}/api/v1.0/plugins',
             headers=AUTH_HEADER,
@@ -228,7 +226,7 @@ def test_6_plugins(self):
         )
         self.assertEqual(r.status_code, 200, "Failed to run plugin selftests")
 
-    def test_7_firmware(self):
+    def test_08_firmware(self):
         r = requests.get(
             f'{URL}/api/v1.0/firmware',
             headers=AUTH_HEADER,
@@ -237,14 +235,14 @@ def test_7_firmware(self):
         # TODO: not working
         #self.assertEqual(r.status_code, 200, "Failed to list firmware")
 
-    def test_8_sysversion(self):
+    def test_09_sysversion(self):
         r = requests.get(
             f'{URL}/api/v1.0/system/version',
             verify=TLS_VERIFY
         )
         self.assertEqual(r.status_code, 200, "Failed to get CNaaS-NMS version")
 
-    def test_9_get_prev_config(self):
+    def test_10_get_prev_config(self):
         hostname = "eosaccess"
         r = requests.get(
             f"{URL}/api/v1.0/device/{hostname}/previous_config?previous=1",

From 1eb491f784a67a994b133ca6a9b802d52b75b17b Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 13 Apr 2020 12:26:10 +0200
Subject: [PATCH 048/102] Fix for ff40952ab902c0faa03b5f91789e16a8e7a15d63 ,
 match devices regardless of state

---
 src/cnaas_nms/confpush/sync_devices.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 004a6165..e00aca55 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -643,7 +643,7 @@ def apply_config(hostname: str, config: str, dry_run: bool,
             dev.synchronized = False
 
     nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
-    nr_filtered = nr.filter(name=hostname).filter(managed=False)
+    nr_filtered = nr.filter(name=hostname)
 
     try:
         nrresult = nr_filtered.run(task=push_static_config,

From 1778b37759e8f1614528a82f4315a81147d40934 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 23 Apr 2020 09:08:41 +0200
Subject: [PATCH 049/102] Begin work on update_facts job

---
 src/cnaas_nms/confpush/update.py | 44 ++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/src/cnaas_nms/confpush/update.py b/src/cnaas_nms/confpush/update.py
index 2f2b3f33..b1f4ff6b 100644
--- a/src/cnaas_nms/confpush/update.py
+++ b/src/cnaas_nms/confpush/update.py
@@ -1,11 +1,16 @@
 from typing import Optional, List
 
+from nornir.plugins.tasks import networking
+
 from cnaas_nms.db.session import sqla_session
 from cnaas_nms.db.device import Device, DeviceType, DeviceState
 from cnaas_nms.db.interface import Interface, InterfaceConfigType
 from cnaas_nms.confpush.get import get_interfaces_names, get_uplinks, \
     filter_interfaces, get_mlag_ifs
 from cnaas_nms.tools.log import get_logger
+from cnaas_nms.scheduler.wrapper import job_wrapper
+from cnaas_nms.confpush.nornir_helper import NornirJobResult
+import cnaas_nms.confpush.nornir_helper
 
 
 def update_interfacedb_worker(session, dev: Device, replace: bool, delete: bool,
@@ -93,3 +98,42 @@ def reset_interfacedb(hostname: str):
         return ret
 
 
+@job_wrapper
+def update_facts(hostname: str,
+                 job_id: Optional[str] = None,
+                 scheduled_by: Optional[str] = None):
+    logger = get_logger()
+    with sqla_session() as session:
+        dev: Device = session.query(Device).filter(Device.hostname == hostname).one_or_none()
+        if not dev:
+            raise ValueError("Device with hostname {} not found".format(hostname))
+        if not (dev.state == DeviceState.MANAGED or dev.state == DeviceState.UNMANAGED):
+            raise ValueError("Device with ztp_mac {} is in incorrect state: {}".format(
+                hostname, str(dev.state)
+            ))
+        hostname = dev.hostname
+
+    nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
+    nr_filtered = nr.filter(name=hostname)
+
+    nrresult = nr_filtered.run(task=networking.napalm_get, getters=["facts"])
+
+    if nrresult.failed:
+        logger.info("Could not contact device with hostname {}".format(hostname))
+        return NornirJobResult(nrresult=nrresult)
+    try:
+        facts = nrresult[hostname][0].result['facts']
+        with sqla_session() as session:
+            dev: Device = session.query(Device).filter(Device.hostname == hostname).one()
+            dev.serial = facts['serial_number']
+            dev.vendor = facts['vendor']
+            dev.model = facts['model']
+            dev.os_version = facts['os_version']
+    except Exception as e:
+        logger.exception("Could not update device with hostname {} with new facts: {}".format(
+            hostname, str(e)
+        ))
+        logger.debug("nrresult for hostname {}: {}".format(hostname, nrresult))
+        raise e
+
+    return NornirJobResult(nrresult=nrresult)

From 8ee345ea1a01ba8c0f70116439132e99696d360b Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 23 Apr 2020 09:12:13 +0200
Subject: [PATCH 050/102] Fix routing of log levels for websocket logs

---
 src/cnaas_nms/tools/log.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/cnaas_nms/tools/log.py b/src/cnaas_nms/tools/log.py
index a12d31ea..662f5f17 100644
--- a/src/cnaas_nms/tools/log.py
+++ b/src/cnaas_nms/tools/log.py
@@ -20,15 +20,15 @@ def socketio_emit(self, msg, rooms=[]):
     def emit(self, record):
         msg = self.format(record)
         if record.levelname == 'DEBUG':
-            self.socketio_emit(msg, rooms=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])
+            self.socketio_emit(msg, rooms=['DEBUG'])
         elif record.levelname == 'INFO':
-            self.socketio_emit(msg, rooms=['INFO', 'WARNING', 'ERROR', 'CRITICAL'])
+            self.socketio_emit(msg, rooms=['DEBUG', 'INFO'])
         elif record.levelname == 'WARNING':
-            self.socketio_emit(msg, rooms=['WARNING', 'ERROR', 'CRITICAL'])
+            self.socketio_emit(msg, rooms=['DEBUG', 'INFO', 'WARNING'])
         elif record.levelname == 'ERROR':
-            self.socketio_emit(msg, rooms=['ERROR', 'CRITICAL'])
+            self.socketio_emit(msg, rooms=['DEBUG', 'INFO', 'WARNING', 'ERROR'])
         elif record.levelname == 'CRITICAL':
-            self.socketio_emit(msg, rooms=['CRITICAL'])
+            self.socketio_emit(msg, rooms=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])
 
 
 def get_logger():

From 5d25d0363139ff9897e1b5f2ecb77c170895ffbd Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 24 Apr 2020 13:01:45 +0200
Subject: [PATCH 051/102] Default to charset utf-8 in redis

---
 src/cnaas_nms/db/session.py        | 2 +-
 src/cnaas_nms/scheduler/wrapper.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cnaas_nms/db/session.py b/src/cnaas_nms/db/session.py
index 20e024c5..cd202e76 100644
--- a/src/cnaas_nms/db/session.py
+++ b/src/cnaas_nms/db/session.py
@@ -61,5 +61,5 @@ def sqla_execute(**kwargs):
 @contextmanager
 def redis_session(**kwargs):
     db_data = get_dbdata(**kwargs)
-    with StrictRedis(host=db_data['redis_hostname'], port=6379) as conn:
+    with StrictRedis(host=db_data['redis_hostname'], port=6379, charset="utf-8", decode_responses=True) as conn:
         yield conn
diff --git a/src/cnaas_nms/scheduler/wrapper.py b/src/cnaas_nms/scheduler/wrapper.py
index 68ee8151..ea0d4a5c 100644
--- a/src/cnaas_nms/scheduler/wrapper.py
+++ b/src/cnaas_nms/scheduler/wrapper.py
@@ -30,7 +30,7 @@ def update_device_progress(job_id: int):
     new_finished_devices = []
     with redis_session() as db:
         while db.llen('finished_devices_' + str(job_id)) != 0:
-            last_finished = db.lpop('finished_devices_' + str(job_id)).decode('utf-8')
+            last_finished = db.lpop('finished_devices_' + str(job_id))
             new_finished_devices.append(last_finished)
 
     if new_finished_devices:

From b268a4c9fe868b2953400233b4055775e7f4c6c5 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 24 Apr 2020 13:02:41 +0200
Subject: [PATCH 052/102] Make logger send websocket messages to redis stream.
 Make run.py start thread to listen to log messages from redis stream.

---
 src/cnaas_nms/run.py       | 51 ++++++++++++++++++++++++++++++++++++++
 src/cnaas_nms/tools/log.py | 24 +++++-------------
 2 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/src/cnaas_nms/run.py b/src/cnaas_nms/run.py
index bd849c22..8b08d5b8 100644
--- a/src/cnaas_nms/run.py
+++ b/src/cnaas_nms/run.py
@@ -2,7 +2,11 @@
 import coverage
 import atexit
 import signal
+import threading
+import time
+from typing import List
 from gevent import monkey, signal as gevent_signal
+from redis import StrictRedis
 
 from cnaas_nms.tools.get_apidata import get_apidata
 # Do late imports for anything cnaas/flask related so we can do gevent monkey patch, see below
@@ -59,10 +63,57 @@ def get_app():
     return app.app
 
 
+def socketio_emit(event_name: str, msg: str, rooms: List[str]):
+    if not app.socketio:
+        return
+    for room in rooms:
+        app.socketio.emit(event_name, msg, room=room)
+
+
+def loglevel_to_rooms(levelname: str) -> List[str]:
+    if levelname == 'DEBUG':
+        return ['DEBUG']
+    elif levelname == 'INFO':
+        return ['DEBUG', 'INFO']
+    elif levelname == 'WARNING':
+        return ['DEBUG', 'INFO', 'WARNING']
+    elif levelname == 'ERROR':
+        return ['DEBUG', 'INFO', 'WARNING', 'ERROR']
+    elif levelname == 'CRITICAL':
+        return ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+
+
+def parse_redis_log_item(item):
+    try:
+        # [stream, [(messageid, {datadict})]
+        if item[0] == "log":
+            return item[1][0][1]
+    except Exception as e:
+        return None
+
+
+def thread_websocket_events():
+    redis: StrictRedis
+    with redis_session() as redis:
+        while True:
+            result = redis.xread({"log": b"$"}, count=1, block=200)
+            for item in result:
+                item = parse_redis_log_item(item)
+                if item:
+                    print("DEBUG01: {}".format(item))
+                    print(threading.get_ident())
+                    socketio_emit(
+                        'cnaas_log', item['message'], loglevel_to_rooms(item['level']))
+
+
 if __name__ == '__main__':
     # gevent monkey patching required if you start flask with the auto-reloader (debug mode)
     monkey.patch_all()
     from cnaas_nms.api import app
+    from cnaas_nms.db.session import redis_session
+
+    t_websocket_events = threading.Thread(target=thread_websocket_events)
+    t_websocket_events.start()
 
     apidata = get_apidata()
     if isinstance(apidata, dict) and 'host' in apidata:
diff --git a/src/cnaas_nms/tools/log.py b/src/cnaas_nms/tools/log.py
index 662f5f17..4d980075 100644
--- a/src/cnaas_nms/tools/log.py
+++ b/src/cnaas_nms/tools/log.py
@@ -1,34 +1,22 @@
 import logging
-import threading
 
 from flask import current_app
 
 from cnaas_nms.scheduler.thread_data import thread_data
+from cnaas_nms.db.session import redis_session
 
 
 class WebsocketHandler(logging.StreamHandler):
     def __init__(self):
         logging.StreamHandler.__init__(self)
 
-    def socketio_emit(self, msg, rooms=[]):
-        # late import to avoid circular dependency on import
-        import cnaas_nms.api.app
-        if cnaas_nms.api.app.socketio:
-            for room in rooms:
-                cnaas_nms.api.app.socketio.emit('cnaas_log', msg, room=room)
-
     def emit(self, record):
         msg = self.format(record)
-        if record.levelname == 'DEBUG':
-            self.socketio_emit(msg, rooms=['DEBUG'])
-        elif record.levelname == 'INFO':
-            self.socketio_emit(msg, rooms=['DEBUG', 'INFO'])
-        elif record.levelname == 'WARNING':
-            self.socketio_emit(msg, rooms=['DEBUG', 'INFO', 'WARNING'])
-        elif record.levelname == 'ERROR':
-            self.socketio_emit(msg, rooms=['DEBUG', 'INFO', 'WARNING', 'ERROR'])
-        elif record.levelname == 'CRITICAL':
-            self.socketio_emit(msg, rooms=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])
+        with redis_session() as redis:
+            try:
+                redis.xadd("log", {"message": msg, "level": record.levelname})
+            except Exception as e:
+                pass
 
 
 def get_logger():

From f4fa0cf5858e836de4ac05893a139cd32f24550a Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 24 Apr 2020 13:38:37 +0200
Subject: [PATCH 053/102] Start websocket event listen thread in uwsgi mode

---
 src/cnaas_nms/run.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/cnaas_nms/run.py b/src/cnaas_nms/run.py
index 8b08d5b8..85a38c54 100644
--- a/src/cnaas_nms/run.py
+++ b/src/cnaas_nms/run.py
@@ -107,6 +107,7 @@ def thread_websocket_events():
 
 
 if __name__ == '__main__':
+    # Starting via python run.py
     # gevent monkey patching required if you start flask with the auto-reloader (debug mode)
     monkey.patch_all()
     from cnaas_nms.api import app
@@ -120,9 +121,18 @@ def thread_websocket_events():
         app.socketio.run(get_app(), debug=True, host=apidata['host'])
     else:
         app.socketio.run(get_app(), debug=True)
+
     if 'COVERAGE' in os.environ:
         save_coverage()
 else:
+    # Starting via uwsgi
     from cnaas_nms.api import app
+    from cnaas_nms.db.session import redis_session
+
+    t_websocket_events = threading.Thread(target=thread_websocket_events)
+    t_websocket_events.start()
 
     cnaas_app = get_app()
+
+    if 'COVERAGE' in os.environ:
+        save_coverage()

From 548fd271338e6f7790ace5e17675cd689844d001 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 24 Apr 2020 13:42:53 +0200
Subject: [PATCH 054/102] Remove debug prints

---
 src/cnaas_nms/run.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/cnaas_nms/run.py b/src/cnaas_nms/run.py
index 85a38c54..10199b97 100644
--- a/src/cnaas_nms/run.py
+++ b/src/cnaas_nms/run.py
@@ -100,8 +100,6 @@ def thread_websocket_events():
             for item in result:
                 item = parse_redis_log_item(item)
                 if item:
-                    print("DEBUG01: {}".format(item))
-                    print(threading.get_ident())
                     socketio_emit(
                         'cnaas_log', item['message'], loglevel_to_rooms(item['level']))
 

From 2c2e7afecc5a96cc6b2fffaa1b807716c66e12f5 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 24 Apr 2020 14:11:09 +0200
Subject: [PATCH 055/102] Make sure to save max 100 messages in redis log
 stream

---
 src/cnaas_nms/tools/log.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cnaas_nms/tools/log.py b/src/cnaas_nms/tools/log.py
index 4d980075..2c84f178 100644
--- a/src/cnaas_nms/tools/log.py
+++ b/src/cnaas_nms/tools/log.py
@@ -14,7 +14,7 @@ def emit(self, record):
         msg = self.format(record)
         with redis_session() as redis:
             try:
-                redis.xadd("log", {"message": msg, "level": record.levelname})
+                redis.xadd("log", {"message": msg, "level": record.levelname}, maxlen=100)
             except Exception as e:
                 pass
 

From 58d872b11ddef7a448472c050469429381304a9d Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 27 Apr 2020 09:40:53 +0200
Subject: [PATCH 056/102] Rename socketio logs to events and add more
 event_types other than logs: job_id, device_id, update

---
 src/cnaas_nms/api/app.py     | 17 +++++++++++------
 src/cnaas_nms/run.py         | 25 ++++++++++++++++---------
 src/cnaas_nms/tools/event.py | 22 ++++++++++++++++++++++
 src/cnaas_nms/tools/log.py   |  8 ++------
 4 files changed, 51 insertions(+), 21 deletions(-)
 create mode 100644 src/cnaas_nms/tools/event.py

diff --git a/src/cnaas_nms/api/app.py b/src/cnaas_nms/api/app.py
index 74c54738..cdf57790 100644
--- a/src/cnaas_nms/api/app.py
+++ b/src/cnaas_nms/api/app.py
@@ -1,5 +1,6 @@
 import os
 import sys
+from typing import Optional
 
 from flask import Flask, render_template, request, g
 from flask_restx import Api
@@ -109,13 +110,17 @@ def handle_error(self, e):
 
 
 # SocketIO listen for new log messages
-@socketio.on('logs')
+@socketio.on('events')
 def ws_logs(data):
-    room: str = None
-    if 'level' in data and data['level'] in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
-        room = data['level']
-    elif 'jobid' in data and isinstance(data['jobid'], str):
-        room = data['jobid']
+    room: Optional[str] = None
+    if 'loglevel' in data and data['loglevel'] in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
+        room = data['loglevel']
+    elif 'job_id' in data and isinstance(data['job_id'], int):
+        room = "job_id_{}".format(data['job_id'])
+    elif 'device_id' in data and isinstance(data['device_id'], int):
+        room = "device_id_{}".format(data['device_id'])
+    elif 'update' in data and data['update'] in ['device', 'job']:
+        room = "update_{}".format(data['update'])
     else:
         return False  # TODO: how to send error message to client?
 
diff --git a/src/cnaas_nms/run.py b/src/cnaas_nms/run.py
index 10199b97..c609fcde 100644
--- a/src/cnaas_nms/run.py
+++ b/src/cnaas_nms/run.py
@@ -63,11 +63,11 @@ def get_app():
     return app.app
 
 
-def socketio_emit(event_name: str, msg: str, rooms: List[str]):
+def socketio_emit(message: str, rooms: List[str]):
     if not app.socketio:
         return
     for room in rooms:
-        app.socketio.emit(event_name, msg, room=room)
+        app.socketio.emit("events", message, room=room)
 
 
 def loglevel_to_rooms(levelname: str) -> List[str]:
@@ -83,10 +83,10 @@ def loglevel_to_rooms(levelname: str) -> List[str]:
         return ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
 
 
-def parse_redis_log_item(item):
+def parse_redis_event(item):
     try:
         # [stream, [(messageid, {datadict})]
-        if item[0] == "log":
+        if item[0] == "events":
             return item[1][0][1]
     except Exception as e:
         return None
@@ -96,12 +96,19 @@ def thread_websocket_events():
     redis: StrictRedis
     with redis_session() as redis:
         while True:
-            result = redis.xread({"log": b"$"}, count=1, block=200)
+            result = redis.xread({"events": b"$"}, count=10, block=200)
             for item in result:
-                item = parse_redis_log_item(item)
-                if item:
-                    socketio_emit(
-                        'cnaas_log', item['message'], loglevel_to_rooms(item['level']))
+                item = parse_redis_event(item)
+                if not item:
+                    continue
+                if item['type'] == "log":
+                    socketio_emit(item['message'], loglevel_to_rooms(item['level']))
+                elif item['type'] == "job_id":
+                    socketio_emit(item['message'], ["job_id_{}".format(item['job_id'])])
+                elif item['type'] == "device_id":
+                    socketio_emit(item['message'], ["device_id_{}".format(item['device_id'])])
+                elif item['type'] == "update":
+                    socketio_emit(item['message'], ["update_{}".format(item['update_type'])])
 
 
 if __name__ == '__main__':
diff --git a/src/cnaas_nms/tools/event.py b/src/cnaas_nms/tools/event.py
new file mode 100644
index 00000000..e11cb2de
--- /dev/null
+++ b/src/cnaas_nms/tools/event.py
@@ -0,0 +1,22 @@
+from typing import Optional
+
+from cnaas_nms.db.session import redis_session
+
+
+def add_event(message: str, event_type: str = "log", level: str = "INFO",
+              job_id: Optional[int] = None, device_id: Optional[int] = None,
+              update_type: Optional[str] = None):
+    with redis_session() as redis:
+        try:
+            data = {"type": event_type, "message": message, "level": level}
+            if event_type == "job_id":
+                data['job_id'] = job_id
+            elif event_type == "device_id":
+                data['device_id'] = device_id
+            elif event_type == "update":
+                data['update_type'] = update_type
+            redis.xadd("events",
+                       data,
+                       maxlen=100)
+        except Exception as e:
+            pass
diff --git a/src/cnaas_nms/tools/log.py b/src/cnaas_nms/tools/log.py
index 2c84f178..1dca5372 100644
--- a/src/cnaas_nms/tools/log.py
+++ b/src/cnaas_nms/tools/log.py
@@ -3,7 +3,7 @@
 from flask import current_app
 
 from cnaas_nms.scheduler.thread_data import thread_data
-from cnaas_nms.db.session import redis_session
+from cnaas_nms.tools.event import add_event
 
 
 class WebsocketHandler(logging.StreamHandler):
@@ -12,11 +12,7 @@ def __init__(self):
 
     def emit(self, record):
         msg = self.format(record)
-        with redis_session() as redis:
-            try:
-                redis.xadd("log", {"message": msg, "level": record.levelname}, maxlen=100)
-            except Exception as e:
-                pass
+        add_event(msg, level=record.levelname)
 
 
 def get_logger():

From 3d1a4aba452ed5be259e9d4f895660e5f1854fbb Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 27 Apr 2020 09:52:47 +0200
Subject: [PATCH 057/102] Add update and job_id events to Job class

---
 src/cnaas_nms/db/job.py | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/cnaas_nms/db/job.py b/src/cnaas_nms/db/job.py
index 1395e84f..dd8e5ca3 100644
--- a/src/cnaas_nms/db/job.py
+++ b/src/cnaas_nms/db/job.py
@@ -16,6 +16,7 @@
 from cnaas_nms.scheduler.jobresult import StrJobResult, DictJobResult
 from cnaas_nms.db.helper import json_dumper
 from cnaas_nms.tools.log import get_logger
+from cnaas_nms.tools.event import add_event
 
 
 logger = get_logger()
@@ -89,6 +90,12 @@ def start_job(self, function_name: str, scheduled_by: str):
         self.status = JobStatus.RUNNING
         self.finished_devices = []
         self.scheduled_by = scheduled_by
+        try:
+            event_msg = "Job #{} started".format(self.id)
+            add_event(event_msg, event_type="update", update_type="job")
+            add_event(event_msg, event_type="job_id", job_id=self.id)
+        except Exception as e:
+            pass
 
     def finish_success(self, res: dict, next_job_id: Optional[int]):
         try:
@@ -110,6 +117,14 @@ def finish_success(self, res: dict, next_job_id: Optional[int]):
         if next_job_id:
             # TODO: check if this exists in the db?
             self.next_job_id = next_job_id
+        try:
+            event_msg = "Job #{} finished".format(self.id)
+            if next_job_id:
+                event_msg += " (next job_id: {})".format(next_job_id)
+            add_event(event_msg, event_type="update", update_type="job")
+            add_event(event_msg, event_type="job_id", job_id=self.id)
+        except Exception as e:
+            pass
 
     def finish_exception(self, e: Exception, traceback: str):
         logger.warning("Job {} finished with exception: {}".format(self.id, str(e)))
@@ -127,13 +142,24 @@ def finish_exception(self, e: Exception, traceback: str):
             errmsg = "Unable to serialize exception or traceback: {}".format(str(e))
             logger.exception(errmsg)
             self.exception = {"error": errmsg}
+        try:
+            event_msg = "Job #{} encountered an exception: {}".format(self.id, str(e))
+            add_event(event_msg, event_type="update", update_type="job")
+            add_event(event_msg, event_type="job_id", job_id=self.id)
+        except Exception as e:
+            pass
 
     def finish_abort(self, message: str):
         logger.debug("Job {} aborted: {}".format(self.id, message))
         self.finish_time = datetime.datetime.utcnow()
         self.status = JobStatus.ABORTED
         self.result = {"message": message}
-
+        try:
+            event_msg = "Job #{} was aborted: {}".format(self.id, message)
+            add_event(event_msg, event_type="update", update_type="job")
+            add_event(event_msg, event_type="job_id", job_id=self.id)
+        except Exception as e:
+            pass
 
     @classmethod
     def clear_jobs(cls, session):

From afb5b97c13c8e26f39c33de137ae106180ee64b0 Mon Sep 17 00:00:00 2001
From: Kristofer Hallin <kristofer@sunet.se>
Date: Mon, 27 Apr 2020 12:34:58 +0200
Subject: [PATCH 058/102] Check if json_data is empty or not before checking if
 factory_default is in it.

Signed-off-by: Kristofer Hallin <kristofer@sunet.se>
---
 src/cnaas_nms/api/device.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 33d19fe2..58c28367 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -99,7 +99,7 @@ def delete(self, device_id):
         """ Delete device from ID """
         json_data = request.get_json()
 
-        if 'factory_default' in json_data:
+        if json_data is not None and 'factory_default' in json_data:
             if isinstance(json_data['factory_default'], bool) and json_data['factory_default'] is True:
                 scheduler = Scheduler()
                 job_id = scheduler.add_onetime_job(

From b5e07fa6d6163f804febc0d682ff9f494300d297 Mon Sep 17 00:00:00 2001
From: Kristofer Hallin <kristofer@sunet.se>
Date: Mon, 27 Apr 2020 12:35:36 +0200
Subject: [PATCH 059/102] Do a proper check if we have an access switch or not.

---
 src/cnaas_nms/confpush/firmware.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cnaas_nms/confpush/firmware.py b/src/cnaas_nms/confpush/firmware.py
index 98dee5bd..82a77458 100644
--- a/src/cnaas_nms/confpush/firmware.py
+++ b/src/cnaas_nms/confpush/firmware.py
@@ -77,7 +77,7 @@ def arista_firmware_download(task, filename: str, httpd_url: str) -> None:
                 filter(Device.hostname == task.host.name).one_or_none()
             device_type = dev.device_type
 
-        if device_type == 'ACCESS':
+        if device_type == DeviceType.ACCESS:
             firmware_download_cmd = 'copy {} flash:'.format(url)
         else:
             firmware_download_cmd = 'copy {} vrf MGMT flash:'.format(url)

From e76d3828ee40fbfb5374ff7055e7579c8c34c195 Mon Sep 17 00:00:00 2001
From: Kristofer Hallin <kristofer@sunet.se>
Date: Mon, 27 Apr 2020 13:32:33 +0200
Subject: [PATCH 060/102] No need to use 'is not None'.

---
 src/cnaas_nms/api/device.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 58c28367..7bb8dabf 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -99,7 +99,7 @@ def delete(self, device_id):
         """ Delete device from ID """
         json_data = request.get_json()
 
-        if json_data is not None and 'factory_default' in json_data:
+        if json_data and 'factory_default' in json_data:
             if isinstance(json_data['factory_default'], bool) and json_data['factory_default'] is True:
                 scheduler = Scheduler()
                 job_id = scheduler.add_onetime_job(
@@ -131,7 +131,7 @@ def put(self, device_id):
                 return empty_result(status='error', data=f"No device with id {device_id}")
 
             errors = dev.device_update(**json_data)
-            if errors is not None:
+            if errors:
                 return empty_result(status='error', data=errors), 404
             return empty_result(status='success', data={"updated_device": dev.as_dict()}), 200
 
@@ -166,7 +166,7 @@ def post(self):
         with sqla_session() as session:
             instance: Device = session.query(Device).filter(Device.hostname ==
                                                             data['hostname']).one_or_none()
-            if instance is not None:
+            if instance:
                 errors.append('Device already exists')
                 return empty_result(status='error', data=errors), 400
             if 'platform' not in data or data['platform'] not in supported_platforms:

From 70edb989616dbbff270f8cc92dad96dbce6d0eff Mon Sep 17 00:00:00 2001
From: Kristofer Hallin <kristofer@sunet.se>
Date: Tue, 28 Apr 2020 10:05:49 +0200
Subject: [PATCH 061/102] Change device state to INIT after the management
 configuration is pushed, otherwise we will try to connect with a user which
 is not yet available on the switch.

---
 src/cnaas_nms/confpush/init_device.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 8c75e997..54d27542 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -223,7 +223,6 @@ def init_access_device_step1(device_id: int, new_hostname: str,
         device_variables = {**device_variables, **mlag_vars}
         # Update device state
         dev = session.query(Device).filter(Device.id == device_id).one()
-        dev.state = DeviceState.INIT
         dev.hostname = new_hostname
         session.commit()
         hostname = dev.hostname
@@ -239,6 +238,7 @@ def init_access_device_step1(device_id: int, new_hostname: str,
     with sqla_session() as session:
         dev = session.query(Device).filter(Device.id == device_id).one()
         dev.management_ip = device_variables['mgmt_ip']
+        dev.state = DeviceState.INIT
         # Remove the reserved IP since it's now saved in the device database instead
         reserved_ip = session.query(ReservedIP).filter(ReservedIP.device == dev).one_or_none()
         if reserved_ip:

From b084ea1e2b7339a0352bb5b2386568d706b7b57f Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 28 Apr 2020 16:36:31 +0200
Subject: [PATCH 062/102] Add event for device updated. Send update_event as
 json object instead of just message. Remove event type job_id and device_id
 and send more data in update type instead.

---
 src/cnaas_nms/api/app.py     |  4 ----
 src/cnaas_nms/db/device.py   | 13 +++++++++++++
 src/cnaas_nms/db/job.py      |  7 ++-----
 src/cnaas_nms/run.py         | 31 ++++++++++++++++++-------------
 src/cnaas_nms/tools/event.py | 20 +++++++++-----------
 5 files changed, 42 insertions(+), 33 deletions(-)

diff --git a/src/cnaas_nms/api/app.py b/src/cnaas_nms/api/app.py
index cdf57790..0e3dd7ed 100644
--- a/src/cnaas_nms/api/app.py
+++ b/src/cnaas_nms/api/app.py
@@ -115,10 +115,6 @@ def ws_logs(data):
     room: Optional[str] = None
     if 'loglevel' in data and data['loglevel'] in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
         room = data['loglevel']
-    elif 'job_id' in data and isinstance(data['job_id'], int):
-        room = "job_id_{}".format(data['job_id'])
-    elif 'device_id' in data and isinstance(data['device_id'], int):
-        room = "device_id_{}".format(data['device_id'])
     elif 'update' in data and data['update'] in ['device', 'job']:
         room = "update_{}".format(data['update'])
     else:
diff --git a/src/cnaas_nms/db/device.py b/src/cnaas_nms/db/device.py
index 2a65c0a3..b3010023 100644
--- a/src/cnaas_nms/db/device.py
+++ b/src/cnaas_nms/db/device.py
@@ -4,11 +4,13 @@
 import datetime
 import enum
 import re
+import json
 from typing import Optional, List, Set
 
 from sqlalchemy import Column, Integer, Unicode, String, UniqueConstraint
 from sqlalchemy import Enum, DateTime, Boolean
 from sqlalchemy import ForeignKey
+from sqlalchemy import event
 from sqlalchemy.orm import relationship
 from sqlalchemy_utils import IPAddressType
 
@@ -17,6 +19,7 @@
 import cnaas_nms.db.linknet
 
 from cnaas_nms.db.interface import Interface, InterfaceConfigType
+from cnaas_nms.tools.event import add_event
 
 
 class DeviceException(Exception):
@@ -441,3 +444,13 @@ def validate(cls, new_entry=True, **kwargs):
         return data, errors
 
 
+@event.listens_for(Device, 'after_update')
+def after_update_device(mapper, connection, target: Device):
+    update_data = {
+        "action": "UPDATED",
+        "device_id": target.id,
+        "hostname": target.hostname,
+        "object": target.as_dict()
+    }
+    json_data = json.dumps(update_data)
+    add_event(json_data=json_data, event_type="update", update_type="device")
diff --git a/src/cnaas_nms/db/job.py b/src/cnaas_nms/db/job.py
index dd8e5ca3..9b4daa58 100644
--- a/src/cnaas_nms/db/job.py
+++ b/src/cnaas_nms/db/job.py
@@ -91,9 +91,9 @@ def start_job(self, function_name: str, scheduled_by: str):
         self.finished_devices = []
         self.scheduled_by = scheduled_by
         try:
-            event_msg = "Job #{} started".format(self.id)
+            event_msg = "Job #{} started (function_name: {}, scheduled_by: {})".format(
+                self.id, function_name, scheduled_by)
             add_event(event_msg, event_type="update", update_type="job")
-            add_event(event_msg, event_type="job_id", job_id=self.id)
         except Exception as e:
             pass
 
@@ -122,7 +122,6 @@ def finish_success(self, res: dict, next_job_id: Optional[int]):
             if next_job_id:
                 event_msg += " (next job_id: {})".format(next_job_id)
             add_event(event_msg, event_type="update", update_type="job")
-            add_event(event_msg, event_type="job_id", job_id=self.id)
         except Exception as e:
             pass
 
@@ -145,7 +144,6 @@ def finish_exception(self, e: Exception, traceback: str):
         try:
             event_msg = "Job #{} encountered an exception: {}".format(self.id, str(e))
             add_event(event_msg, event_type="update", update_type="job")
-            add_event(event_msg, event_type="job_id", job_id=self.id)
         except Exception as e:
             pass
 
@@ -157,7 +155,6 @@ def finish_abort(self, message: str):
         try:
             event_msg = "Job #{} was aborted: {}".format(self.id, message)
             add_event(event_msg, event_type="update", update_type="job")
-            add_event(event_msg, event_type="job_id", job_id=self.id)
         except Exception as e:
             pass
 
diff --git a/src/cnaas_nms/run.py b/src/cnaas_nms/run.py
index c609fcde..37ba660c 100644
--- a/src/cnaas_nms/run.py
+++ b/src/cnaas_nms/run.py
@@ -83,32 +83,35 @@ def loglevel_to_rooms(levelname: str) -> List[str]:
         return ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
 
 
-def parse_redis_event(item):
+def parse_redis_event(event):
     try:
         # [stream, [(messageid, {datadict})]
-        if item[0] == "events":
-            return item[1][0][1]
+        if event[0] == "events":
+            return event[1][0][1]
     except Exception as e:
         return None
 
 
+def emit_redis_event(event):
+    try:
+        if event['type'] == "log":
+            socketio_emit(event['message'], loglevel_to_rooms(event['level']))
+        elif event['type'] == "update":
+            socketio_emit(json.loads(event['json']), ["update_{}".format(event['update_type'])])
+    except Exception as e:
+        pass
+
+
 def thread_websocket_events():
     redis: StrictRedis
     with redis_session() as redis:
         while True:
             result = redis.xread({"events": b"$"}, count=10, block=200)
             for item in result:
-                item = parse_redis_event(item)
-                if not item:
+                event = parse_redis_event(item)
+                if not event:
                     continue
-                if item['type'] == "log":
-                    socketio_emit(item['message'], loglevel_to_rooms(item['level']))
-                elif item['type'] == "job_id":
-                    socketio_emit(item['message'], ["job_id_{}".format(item['job_id'])])
-                elif item['type'] == "device_id":
-                    socketio_emit(item['message'], ["device_id_{}".format(item['device_id'])])
-                elif item['type'] == "update":
-                    socketio_emit(item['message'], ["update_{}".format(item['update_type'])])
+                emit_redis_event(event)
 
 
 if __name__ == '__main__':
@@ -117,6 +120,7 @@ def thread_websocket_events():
     monkey.patch_all()
     from cnaas_nms.api import app
     from cnaas_nms.db.session import redis_session
+    import json
 
     t_websocket_events = threading.Thread(target=thread_websocket_events)
     t_websocket_events.start()
@@ -133,6 +137,7 @@ def thread_websocket_events():
     # Starting via uwsgi
     from cnaas_nms.api import app
     from cnaas_nms.db.session import redis_session
+    import json
 
     t_websocket_events = threading.Thread(target=thread_websocket_events)
     t_websocket_events.start()
diff --git a/src/cnaas_nms/tools/event.py b/src/cnaas_nms/tools/event.py
index e11cb2de..a7972566 100644
--- a/src/cnaas_nms/tools/event.py
+++ b/src/cnaas_nms/tools/event.py
@@ -3,20 +3,18 @@
 from cnaas_nms.db.session import redis_session
 
 
-def add_event(message: str, event_type: str = "log", level: str = "INFO",
-              job_id: Optional[int] = None, device_id: Optional[int] = None,
-              update_type: Optional[str] = None):
+def add_event(message: Optional[str] = None, event_type: str = "log", level: str = "INFO",
+              update_type: Optional[str] = None, json_data: Optional[str] = None):
     with redis_session() as redis:
         try:
-            data = {"type": event_type, "message": message, "level": level}
-            if event_type == "job_id":
-                data['job_id'] = job_id
-            elif event_type == "device_id":
-                data['device_id'] = device_id
+            send_data = {"type": event_type, "level": level}
+            if event_type == "log":
+                send_data['message'] = message
             elif event_type == "update":
-                data['update_type'] = update_type
+                send_data['update_type'] = update_type
+                send_data['json'] = json_data
             redis.xadd("events",
-                       data,
+                       send_data,
                        maxlen=100)
         except Exception as e:
-            pass
+            print("Error in add_event: {}".format(e))

From c67b21fa7879bb6c0f7e63d992c18fbb3d863cf4 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 29 Apr 2020 13:03:44 +0200
Subject: [PATCH 063/102] Make job events send json instead of text message

---
 src/cnaas_nms/db/job.py | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/src/cnaas_nms/db/job.py b/src/cnaas_nms/db/job.py
index 9b4daa58..64eb7d0c 100644
--- a/src/cnaas_nms/db/job.py
+++ b/src/cnaas_nms/db/job.py
@@ -91,9 +91,13 @@ def start_job(self, function_name: str, scheduled_by: str):
         self.finished_devices = []
         self.scheduled_by = scheduled_by
         try:
-            event_msg = "Job #{} started (function_name: {}, scheduled_by: {})".format(
-                self.id, function_name, scheduled_by)
-            add_event(event_msg, event_type="update", update_type="job")
+            json_data = json.dumps({
+                "job_id": self.id,
+                "status": "RUNNING",
+                "function_name": function_name,
+                "scheduled_by": scheduled_by
+            })
+            add_event(json_data=json_data, event_type="update", update_type="job")
         except Exception as e:
             pass
 
@@ -118,10 +122,14 @@ def finish_success(self, res: dict, next_job_id: Optional[int]):
             # TODO: check if this exists in the db?
             self.next_job_id = next_job_id
         try:
-            event_msg = "Job #{} finished".format(self.id)
+            event_data = {
+                "job_id": self.id,
+                "status": "FINISHED"
+            }
             if next_job_id:
-                event_msg += " (next job_id: {})".format(next_job_id)
-            add_event(event_msg, event_type="update", update_type="job")
+                event_data['next_job_id'] = next_job_id
+            json_data = json.dumps(event_data)
+            add_event(json_data=json_data, event_type="update", update_type="job")
         except Exception as e:
             pass
 
@@ -142,8 +150,12 @@ def finish_exception(self, e: Exception, traceback: str):
             logger.exception(errmsg)
             self.exception = {"error": errmsg}
         try:
-            event_msg = "Job #{} encountered an exception: {}".format(self.id, str(e))
-            add_event(event_msg, event_type="update", update_type="job")
+            json_data = json.dumps({
+                "job_id": self.id,
+                "status": "EXCEPTION",
+                "exception": str(e),
+            })
+            add_event(json_data=json_data, event_type="update", update_type="job")
         except Exception as e:
             pass
 
@@ -153,8 +165,12 @@ def finish_abort(self, message: str):
         self.status = JobStatus.ABORTED
         self.result = {"message": message}
         try:
-            event_msg = "Job #{} was aborted: {}".format(self.id, message)
-            add_event(event_msg, event_type="update", update_type="job")
+            json_data = json.dumps({
+                "job_id": self.id,
+                "status": "ABORTED",
+                "message": message,
+            })
+            add_event(json_data=json_data, event_type="update", update_type="job")
         except Exception as e:
             pass
 

From cb97b1b09057e146e08db6b7730a8d7f7ce78c1b Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 5 May 2020 13:30:35 +0200
Subject: [PATCH 064/102] Require jwt token via query string on socketio
 connect

---
 src/cnaas_nms/api/app.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/cnaas_nms/api/app.py b/src/cnaas_nms/api/app.py
index 0e3dd7ed..e6c37439 100644
--- a/src/cnaas_nms/api/app.py
+++ b/src/cnaas_nms/api/app.py
@@ -5,7 +5,7 @@
 from flask import Flask, render_template, request, g
 from flask_restx import Api
 from flask_socketio import SocketIO, join_room
-from flask_jwt_extended import JWTManager, decode_token
+from flask_jwt_extended import JWTManager, decode_token, jwt_required
 from flask_jwt_extended.exceptions import NoAuthorizationError
 
 from flask import jsonify
@@ -84,6 +84,7 @@ def handle_error(self, e):
 app.config['JWT_IDENTITY_CLAIM'] = 'sub'
 app.config['JWT_ALGORITHM'] = 'ES256'
 app.config['JWT_ACCESS_TOKEN_EXPIRES'] = False
+app.config['JWT_TOKEN_LOCATION'] = ('headers', 'query_string')
 
 jwt = JWTManager(app)
 api = CnaasApi(app, prefix='/api/{}'.format(__api_version__),
@@ -108,10 +109,15 @@ def handle_error(self, e):
 api.add_namespace(plugins_api)
 api.add_namespace(system_api)
 
+# SocketIO on connect
+@socketio.on('connect')
+@jwt_required
+def socketio_on_connect():
+    return True
 
-# SocketIO listen for new log messages
+# SocketIO join event rooms
 @socketio.on('events')
-def ws_logs(data):
+def socketio_on_events(data):
     room: Optional[str] = None
     if 'loglevel' in data and data['loglevel'] in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
         room = data['loglevel']

From a5941f8197ae50993f2c34712782f9d1a49513f0 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 5 May 2020 14:04:58 +0200
Subject: [PATCH 065/102] Make uwsgi skip log messages containing jwt tokens

---
 docker/api/cnaas-setup.sh              | 6 ++++--
 docker/api/config/supervisord_app.conf | 4 ++--
 docker/api/config/uwsgi.ini            | 3 +++
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/docker/api/cnaas-setup.sh b/docker/api/cnaas-setup.sh
index 69cbcbc9..85b1def7 100755
--- a/docker/api/cnaas-setup.sh
+++ b/docker/api/cnaas-setup.sh
@@ -25,9 +25,11 @@ apt-get update && \
       supervisor \
       libssl-dev \
       libpq-dev \
+      uwsgi \
+      uwsgi-plugin-python3 \
     && apt-get clean
 
-pip3 install uwsgi
+#pip3 install uwsgi
 
 # Start venv
 python3 -m venv /opt/cnaas/venv
@@ -46,7 +48,7 @@ python3 -m pip install -r requirements.txt
 #cd /opt/cnaas/venv/cnaas-nms/
 #git remote update
 #git fetch
-#git checkout --track origin/release-1.0.0
+#git checkout --track origin/feature.websocket_events_redis
 #python3 -m pip install -r requirements.txt
 
 chown -R www-data:www-data /opt/cnaas/settings
diff --git a/docker/api/config/supervisord_app.conf b/docker/api/config/supervisord_app.conf
index dcc3cd60..314647e0 100644
--- a/docker/api/config/supervisord_app.conf
+++ b/docker/api/config/supervisord_app.conf
@@ -7,9 +7,9 @@ pidfile=/tmp/supervisord.pid
 childlogdir=/tmp
 
 [program:uwsgi]
-command = /usr/local/bin/uwsgi --ini /opt/cnaas/venv/cnaas-nms/uwsgi.ini
+command = /usr/bin/uwsgi --ini /opt/cnaas/venv/cnaas-nms/uwsgi.ini
 autorestart=true
 
 [program:nginx]
 command=/usr/sbin/nginx -g "daemon off;"
-autorestart=true
\ No newline at end of file
+autorestart=true
diff --git a/docker/api/config/uwsgi.ini b/docker/api/config/uwsgi.ini
index d72ca1a6..34920bda 100644
--- a/docker/api/config/uwsgi.ini
+++ b/docker/api/config/uwsgi.ini
@@ -2,6 +2,7 @@
 uid=www-data
 gid=www-data
 chdir = /opt/cnaas/venv/cnaas-nms/src/
+plugins = python3
 callable = cnaas_app
 module = cnaas_nms.run
 socket = /tmp/uwsgi.sock
@@ -18,3 +19,5 @@ lazy-apps = true
 # websocket support
 http-websockets = true
 gevent = 1000
+# don't log jwt tokens
+log-drain = jwt=

From 012b45991c4cb5777c60bca62016de9103c361e5 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 5 May 2020 14:34:00 +0200
Subject: [PATCH 066/102] Add logging when user connects via socketio. Remove
 jwt token from request url before logging.

---
 src/cnaas_nms/api/app.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/cnaas_nms/api/app.py b/src/cnaas_nms/api/app.py
index e6c37439..de8e16e9 100644
--- a/src/cnaas_nms/api/app.py
+++ b/src/cnaas_nms/api/app.py
@@ -1,11 +1,12 @@
 import os
 import sys
+import re
 from typing import Optional
 
 from flask import Flask, render_template, request, g
 from flask_restx import Api
 from flask_socketio import SocketIO, join_room
-from flask_jwt_extended import JWTManager, decode_token, jwt_required
+from flask_jwt_extended import JWTManager, decode_token, jwt_required, get_jwt_identity
 from flask_jwt_extended.exceptions import NoAuthorizationError
 
 from flask import jsonify
@@ -45,6 +46,8 @@
     }
 }
 
+jwt_query_r = re.compile(r'jwt=[^ &]+')
+
 
 class CnaasApi(Api):
     def handle_error(self, e):
@@ -113,7 +116,12 @@ def handle_error(self, e):
 @socketio.on('connect')
 @jwt_required
 def socketio_on_connect():
-    return True
+    user = get_jwt_identity()
+    if user:
+        logger.info('User: {} connected via socketio'.format(user))
+        return True
+    else:
+        return False
 
 # SocketIO join event rooms
 @socketio.on('events')
@@ -137,5 +145,10 @@ def log_request(response):
         user = decode_token(token).get('sub')
     except Exception:
         user = 'unknown'
-    logger.info('User: {}, Method: {}, Status: {}, URL: {}, JSON: {}'.format(user, request.method, response.status_code, request.url, request.json))
+    try:
+        url = re.sub(jwt_query_r, '', request.url)
+        logger.info('User: {}, Method: {}, Status: {}, URL: {}, JSON: {}'.format(
+            user, request.method, response.status_code, url, request.json))
+    except Exception:
+        pass
     return response

From 4a203aeff10024e0d2f5e28b78cf26ccc1af03fc Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 14 May 2020 15:19:01 +0200
Subject: [PATCH 067/102] Handle sqlalchemy error on device delete. Since we
 commit() manually we need to manually handle rollback() as well.

---
 src/cnaas_nms/api/device.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 7bb8dabf..37b1a3ed 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -5,6 +5,7 @@
 from flask import request, make_response
 from flask_restx import Resource, Namespace, fields
 from sqlalchemy import func
+from sqlalchemy.exc import IntegrityError
 from nornir.core.filter import F
 
 import cnaas_nms.confpush.nornir_helper
@@ -111,12 +112,22 @@ def delete(self, device_id):
         else:
             with sqla_session() as session:
                 dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none()
-                if dev:
+                if not dev:
+                    return empty_result('error', "Device not found"), 404
+                try:
                     session.delete(dev)
                     session.commit()
-                    return empty_result(status="success", data={"deleted_device": dev.as_dict()}), 200
-                else:
-                    return empty_result('error', "Device not found"), 404
+                except IntegrityError as e:
+                    session.rollback()
+                    return empty_result(
+                        status='error',
+                        data="Could not remove device because existing references: {}".format(e))
+                except Exception as e:
+                    session.rollback()
+                    return empty_result(
+                        status='error',
+                        data="Could not remove device: {}".format(e))
+                return empty_result(status="success", data={"deleted_device": dev.as_dict()}), 200
 
     @jwt_required
     @device_api.expect(device_model)

From 539b2295623d59d345d474d3dd71403f7b82d632 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 25 May 2020 14:13:20 +0200
Subject: [PATCH 068/102] Add (dry_job) to function name in job log if argument
 dry_run was true. Add options to include comment and ticket_ref when starting
 a syncto job.

---
 docs/apiref/syncto.rst             |  4 ++++
 src/cnaas_nms/api/device.py        | 12 +++++++++++-
 src/cnaas_nms/scheduler/wrapper.py | 16 +++++++++++++++-
 3 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/docs/apiref/syncto.rst b/docs/apiref/syncto.rst
index e5e07a3b..7c15afd9 100644
--- a/docs/apiref/syncto.rst
+++ b/docs/apiref/syncto.rst
@@ -54,6 +54,10 @@ Arguments:
    re-synchronized, if you specify this option as true then all devices will be checked.
    This option does not affect syncto jobs with a specified hostname, when you select only
    a single device via hostname it's always re-synchronized. Defaults to false.
+ - comment: Optionally add a comment that is saved in the job log.
+   This should be a string with max 255 characters.
+ - ticket_ref: Optionally reference a service ticket associated with this job.
+   This should be a string with max 32 characters.
 
 If neither hostname or device_type is specified all devices that needs to be sycnhronized
 will be selected.
diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 7bb8dabf..ac67c7bf 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -305,7 +305,13 @@ class DeviceSyncApi(Resource):
     def post(self):
         """ Start sync of device(s) """
         json_data = request.get_json()
-        kwargs: dict = {}
+        # default args
+        kwargs: dict = {
+            'dry_run': True,
+            'auto_push': False,
+            'force': False,
+            'resync': False
+        }
 
         total_count: Optional[int] = None
 
@@ -372,6 +378,10 @@ def post(self):
             kwargs['auto_push'] = json_data['auto_push']
         if 'resync' in json_data and isinstance(json_data['resync'], bool):
             kwargs['resync'] = json_data['resync']
+        if 'comment' in json_data and isinstance(json_data['comment'], str):
+            kwargs['job_comment'] = json_data['comment']
+        if 'ticket_ref' in json_data and isinstance(json_data['ticket_ref'], str):
+            kwargs['job_ticket_ref'] = json_data['ticket_ref']
 
         scheduler = Scheduler()
         job_id = scheduler.add_onetime_job(
diff --git a/src/cnaas_nms/scheduler/wrapper.py b/src/cnaas_nms/scheduler/wrapper.py
index ea0d4a5c..6bca54d1 100644
--- a/src/cnaas_nms/scheduler/wrapper.py
+++ b/src/cnaas_nms/scheduler/wrapper.py
@@ -65,7 +65,21 @@ def wrapper(job_id: int, scheduled_by: str, *args, **kwargs):
             kwargs['kwargs']['job_id'] = job_id
             if scheduled_by is None:
                 scheduled_by = 'unknown'
-            job.start_job(function_name=func.__name__,
+            # Append (dry_run) to function name if set, so we can distinguish dry_run jobs
+            try:
+                if kwargs['kwargs']['dry_run']:
+                    function_name = "{} (dry_run)".format(func.__name__)
+                else:
+                    function_name = func.__name__
+            except Exception:
+                function_name = func.__name__
+            job_comment = kwargs['kwargs'].pop('job_comment', None)
+            if job_comment and isinstance(job_comment, str):
+                job.comment = job_comment[:255]
+            job_ticket_ref = kwargs['kwargs'].pop('job_ticket_ref', None)
+            if job_ticket_ref and isinstance(job_comment, str):
+                job.ticket_ref = job_ticket_ref[:32]
+            job.start_job(function_name=function_name,
                           scheduled_by=scheduled_by)
             if func.__name__ in progress_funcitons:
                 stop_event = threading.Event()

From fcbfb8636eba73501510ef70f55d9a1cc6b4d0cf Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 25 May 2020 14:41:30 +0200
Subject: [PATCH 069/102] Since get_previous_config looks for exact match
 'sync_devices' and dry_run jobs now has (dry_run) appended to the name
 previous config will not find old dry_run configs, which is a good thing.
 Update integrationtest to get available config.

---
 test/integrationtests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integrationtests.py b/test/integrationtests.py
index 3f9020a4..0df069d7 100644
--- a/test/integrationtests.py
+++ b/test/integrationtests.py
@@ -245,7 +245,7 @@ def test_09_sysversion(self):
     def test_10_get_prev_config(self):
         hostname = "eosaccess"
         r = requests.get(
-            f"{URL}/api/v1.0/device/{hostname}/previous_config?previous=1",
+            f"{URL}/api/v1.0/device/{hostname}/previous_config?previous=0",
             headers=AUTH_HEADER,
             verify=TLS_VERIFY
         )

From 19fffc761a3e05b06ac50fcddbdae46039e2edbc Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 28 May 2020 16:54:44 +0200
Subject: [PATCH 070/102] Start work on moving nornir inventory
 filtering/selection to separate reusable function

---
 src/cnaas_nms/api/device.py             | 64 +++++++++++--------------
 src/cnaas_nms/confpush/nornir_helper.py | 51 +++++++++++++++++---
 2 files changed, 72 insertions(+), 43 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index f8aeefcd..abd0394d 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -6,12 +6,11 @@
 from flask_restx import Resource, Namespace, fields
 from sqlalchemy import func
 from sqlalchemy.exc import IntegrityError
-from nornir.core.filter import F
 
-import cnaas_nms.confpush.nornir_helper
 import cnaas_nms.confpush.init_device
 import cnaas_nms.confpush.sync_devices
 import cnaas_nms.confpush.underlay
+from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector
 from cnaas_nms.api.generic import build_filter, empty_result
 from cnaas_nms.db.device import Device, DeviceState, DeviceType
 from cnaas_nms.db.job import Job, JobNotFoundError, InvalidJobError
@@ -324,7 +323,22 @@ def post(self):
             'resync': False
         }
 
+        if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \
+                and not json_data['dry_run']:
+            kwargs['dry_run'] = False
+        if 'force' in json_data and isinstance(json_data['force'], bool):
+            kwargs['force'] = json_data['force']
+        if 'auto_push' in json_data and isinstance(json_data['auto_push'], bool):
+            kwargs['auto_push'] = json_data['auto_push']
+        if 'resync' in json_data and isinstance(json_data['resync'], bool):
+            kwargs['resync'] = json_data['resync']
+        if 'comment' in json_data and isinstance(json_data['comment'], str):
+            kwargs['job_comment'] = json_data['comment']
+        if 'ticket_ref' in json_data and isinstance(json_data['ticket_ref'], str):
+            kwargs['job_ticket_ref'] = json_data['ticket_ref']
+
         total_count: Optional[int] = None
+        nr = cnaas_init()
 
         if 'hostname' in json_data:
             hostname = str(json_data['hostname'])
@@ -333,17 +347,14 @@ def post(self):
                     status='error',
                     data=f"Hostname '{hostname}' is not a valid hostname"
                 ), 400
-            with sqla_session() as session:
-                dev: Device = session.query(Device).\
-                    filter(Device.hostname == hostname).one_or_none()
-                if not dev or dev.state != DeviceState.MANAGED:
-                    return empty_result(
-                        status='error',
-                        data=f"Hostname '{hostname}' not found or is not a managed device"
-                    ), 400
+            _, total_count, _ = inventory_selector(nr, hostname=hostname)
+            if total_count != 1:
+                return empty_result(
+                    status='error',
+                    data=f"Hostname '{hostname}' not found or is not a managed device"
+                ), 400
             kwargs['hostname'] = hostname
             what = hostname
-            total_count = 1
         elif 'device_type' in json_data:
             devtype_str = str(json_data['device_type']).upper()
             if DeviceType.has_name(devtype_str):
@@ -354,46 +365,25 @@ def post(self):
                     data=f"Invalid device type '{json_data['device_type']}' specified"
                 ), 400
             what = f"{json_data['device_type']} devices"
-            with sqla_session() as session:
-                total_count = session.query(Device). \
-                    filter(Device.device_type == DeviceType[devtype_str]).count()
+            _, total_count, _ = inventory_selector(nr, resync=kwargs['resync'],
+                                                   device_type=devtype_str)
         elif 'group' in json_data:
             group_name = str(json_data['group'])
             if group_name not in get_groups():
                 return empty_result(status='error', data='Could not find a group with name {}'.format(group_name))
             kwargs['group'] = group_name
             what = 'group {}'.format(group_name)
-            nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
-            nr_filtered = nr.filter(F(groups__contains=group_name))
-            total_count = len(nr_filtered.inventory.hosts)
+            _, total_count, _ = inventory_selector(nr, resync=kwargs['resync'],
+                                                   group=group_name)
         elif 'all' in json_data and isinstance(json_data['all'], bool) and json_data['all']:
             what = "all devices"
-            with sqla_session() as session:
-                total_count_q = session.query(Device).filter(Device.state == DeviceState.MANAGED)
-                if 'resync' in json_data and isinstance(json_data['resync'], bool) and json_data['resync']:
-                    total_count = total_count_q.count()
-                else:
-                    total_count = total_count_q.filter(Device.synchronized == False).count()
+            _, total_count, _ = inventory_selector(nr, resync=kwargs['resync'])
         else:
             return empty_result(
                 status='error',
                 data=f"No devices to synchronize was specified"
             ), 400
 
-        if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \
-                and not json_data['dry_run']:
-            kwargs['dry_run'] = False
-        if 'force' in json_data and isinstance(json_data['force'], bool):
-            kwargs['force'] = json_data['force']
-        if 'auto_push' in json_data and isinstance(json_data['auto_push'], bool):
-            kwargs['auto_push'] = json_data['auto_push']
-        if 'resync' in json_data and isinstance(json_data['resync'], bool):
-            kwargs['resync'] = json_data['resync']
-        if 'comment' in json_data and isinstance(json_data['comment'], str):
-            kwargs['job_comment'] = json_data['comment']
-        if 'ticket_ref' in json_data and isinstance(json_data['ticket_ref'], str):
-            kwargs['job_ticket_ref'] = json_data['ticket_ref']
-
         scheduler = Scheduler()
         job_id = scheduler.add_onetime_job(
             'cnaas_nms.confpush.sync_devices:sync_devices',
diff --git a/src/cnaas_nms/confpush/nornir_helper.py b/src/cnaas_nms/confpush/nornir_helper.py
index baa2e2cf..1405f6f3 100644
--- a/src/cnaas_nms/confpush/nornir_helper.py
+++ b/src/cnaas_nms/confpush/nornir_helper.py
@@ -1,11 +1,12 @@
-from nornir import InitNornir
+from dataclasses import dataclass
+from typing import Optional, Tuple, List
 
-from nornir.core.task import AggregatedResult, MultiResult, Result
+from nornir import InitNornir
+from nornir.core import Nornir
+from nornir.core.task import AggregatedResult, MultiResult
+from nornir.core.filter import F
 from cnaas_nms.scheduler.jobresult import JobResult
 
-from dataclasses import dataclass
-from typing import Optional
-
 
 @dataclass
 class NornirJobResult(JobResult):
@@ -13,7 +14,7 @@ class NornirJobResult(JobResult):
     change_score: Optional[float] = None
 
 
-def cnaas_init():
+def cnaas_init() -> Nornir:
     nr = InitNornir(
         core={"num_workers": 50},
         inventory={
@@ -41,3 +42,41 @@ def nr_result_serialize(result: AggregatedResult):
             if res.failed:
                 hosts[host]['failed'] = True
     return hosts
+
+
+def inventory_selector(nr: Nornir, resync: bool = True,
+                       hostname: Optional[str] = None,
+                       device_type: Optional[str] = None,
+                       group: Optional[str] = None) -> Tuple[Nornir, int, List[str]]:
+    """Return a filtered Nornir inventory with only the selected devices
+
+    Args:
+        nr: Nornir object
+        resync: Set to false if you want to filter out devices that are synchronized
+        hostname: Select device by hostname (string)
+        device_type: Select device by device_type (string)
+        group: Select device by group (string)
+
+    Returns:
+        Tuple with: filtered Nornir inventory, total device count selected,
+                    list of hostnames that was skipped because of resync=False
+    """
+    skipped_devices = []
+    if hostname:
+        nr_filtered = nr.filter(name=hostname).filter(managed=True)
+    elif device_type:
+        nr_filtered = nr.filter(F(groups__contains='T_'+device_type)).filter(managed=True)
+    elif group:
+        nr_filtered = nr.filter(F(groups__contains=group)).filter(managed=True)
+    else:
+        # all devices
+        nr_filtered = nr.filter(managed=True)
+
+    if resync or hostname:
+        return nr_filtered, len(nr_filtered.inventory.hosts), skipped_devices
+    else:
+        pre_device_list = list(nr_filtered.inventory.hosts.keys())
+        nr_filtered = nr_filtered.filter(synchronized=False)
+        post_device_list = list(nr_filtered.inventory.hosts.keys())
+        skipped_devices = [x for x in pre_device_list if x not in post_device_list]
+        return nr_filtered, len(post_device_list), skipped_devices

From 5dc874772ff3581ff9aaccbfd0462b1a5dacc9a0 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 29 May 2020 10:05:20 +0200
Subject: [PATCH 071/102] Move sync_devices over to inventory selector

---
 src/cnaas_nms/confpush/sync_devices.py | 46 +++++++++++++-------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index e00aca55..11bb6051 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -7,12 +7,10 @@
 
 from nornir.plugins.tasks import networking, text
 from nornir.plugins.functions.text import print_result
-from nornir.core.filter import F
 from nornir.core.task import MultiResult
-from sqlalchemy import or_
 
 import cnaas_nms.db.helper
-import cnaas_nms.confpush.nornir_helper
+from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector
 from cnaas_nms.db.session import sqla_session, redis_session
 from cnaas_nms.confpush.get import calc_config_hash
 from cnaas_nms.confpush.changescore import calculate_score
@@ -329,8 +327,8 @@ def generate_only(hostname: str) -> (str, dict):
         (string with config, dict with available template variables)
     """
     logger = get_logger()
-    nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
-    nr_filtered = nr.filter(name=hostname).filter(managed=True)
+    nr = cnaas_init()
+    nr_filtered, _, _ = inventory_selector(nr, hostname=hostname)
     template_vars = {}
     if len(nr_filtered.inventory.hosts) != 1:
         raise ValueError("Invalid hostname: {}".format(hostname))
@@ -430,30 +428,32 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No
         NornirJobResult
     """
     logger = get_logger()
-    nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
+    nr = cnaas_init()
+    dev_count = 0
+    skipped_hostnames = []
     if hostname:
-        nr_filtered = nr.filter(name=hostname).filter(managed=True)
+        nr_filtered, dev_count, skipped_hostnames = \
+            inventory_selector(nr, hostname=hostname)
     else:
         if device_type:
-            nr_filtered = nr.filter(F(groups__contains='T_'+device_type)).filter(managed=True)
+            nr_filtered, dev_count, skipped_hostnames = \
+                inventory_selector(nr, resync=resync, device_type=device_type)
         elif group:
-            nr_filtered = nr.filter(F(groups__contains=group)).filter(managed=True)
+            nr_filtered, dev_count, skipped_hostnames = \
+                inventory_selector(nr, resync=resync, group=group)
         else:
             # all devices
-            nr_filtered = nr.filter(managed=True)
-        if not resync:
-            pre_device_list = list(nr_filtered.inventory.hosts.keys())
-            nr_filtered = nr_filtered.filter(synchronized=False)
-            post_device_list = list(nr_filtered.inventory.hosts.keys())
-            already_synced_device_list = [x for x in pre_device_list if x not in post_device_list]
-            if already_synced_device_list:
-                logger.info("Device(s) already synchronized, skipping: {}".format(
-                    already_synced_device_list
-                ))
+            nr_filtered, dev_count, skipped_hostnames = \
+                inventory_selector(nr, resync=resync)
+
+    if skipped_hostnames:
+        logger.info("Device(s) already synchronized, skipping ({}): {}".format(
+            len(skipped_hostnames), ", ".join(skipped_hostnames)
+        ))
 
     device_list = list(nr_filtered.inventory.hosts.keys())
-    logger.info("Device(s) selected for synchronization: {}".format(
-        device_list
+    logger.info("Device(s) selected for synchronization ({}): {}".format(
+        dev_count, ", ".join(device_list)
     ))
 
     try:
@@ -642,8 +642,8 @@ def apply_config(hostname: str, config: str, dry_run: bool,
             dev.state = DeviceState.UNMANAGED
             dev.synchronized = False
 
-    nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
-    nr_filtered = nr.filter(name=hostname)
+    nr = cnaas_init()
+    nr_filtered, _, _ = inventory_selector(nr, hostname=hostname)
 
     try:
         nrresult = nr_filtered.run(task=push_static_config,

From c6b5bf0094b1e32255b25fcc5b321077d3e3b6fc Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 29 May 2020 10:21:24 +0200
Subject: [PATCH 072/102] Change firmware upgrade over to inventory_selector.
 Fix bug(?) where if neither hostname nor group was selected all
 unsynchronized devices would have been upgraded.

---
 src/cnaas_nms/confpush/firmware.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/cnaas_nms/confpush/firmware.py b/src/cnaas_nms/confpush/firmware.py
index 82a77458..27281dea 100644
--- a/src/cnaas_nms/confpush/firmware.py
+++ b/src/cnaas_nms/confpush/firmware.py
@@ -1,16 +1,13 @@
-import cnaas_nms.confpush.nornir_helper
-
+from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector
 from cnaas_nms.tools.log import get_logger
-from cnaas_nms.scheduler.scheduler import Scheduler
 from cnaas_nms.scheduler.wrapper import job_wrapper
 from cnaas_nms.confpush.nornir_helper import NornirJobResult
 from cnaas_nms.db.session import sqla_session, redis_session
 from cnaas_nms.db.device import DeviceType, Device
 
 from nornir.plugins.functions.text import print_result
-from nornir.plugins.tasks.networking import napalm_cli, napalm_configure, napalm_get
+from nornir.plugins.tasks.networking import napalm_cli
 from nornir.plugins.tasks.networking import netmiko_send_command
-from nornir.core.filter import F
 from nornir.core.task import MultiResult
 
 from napalm.base.exceptions import CommandErrorException
@@ -253,17 +250,17 @@ def device_upgrade(download: Optional[bool] = False,
                    reboot: Optional[bool] = False,
                    scheduled_by: Optional[str] = None) -> NornirJobResult:
 
-    nr = cnaas_nms.confpush.nornir_helper.cnaas_init()
+    nr = cnaas_init()
     if hostname:
-        nr_filtered = nr.filter(name=hostname).filter(managed=True)
+        nr_filtered, dev_count, _ = inventory_selector(nr, hostname=hostname)
     elif group:
-        nr_filtered = nr.filter(F(groups__contains=group))
+        nr_filtered, dev_count, _ = inventory_selector(nr, group=group)
     else:
-        nr_filtered = nr.filter(synchronized=False).filter(managed=True)
+        raise ValueError("Neither hostname nor group specified for device_upgrade")
 
     device_list = list(nr_filtered.inventory.hosts.keys())
-    logger.info("Device(s) selected for firmware upgrade: {}".format(
-        device_list
+    logger.info("Device(s) selected for firmware upgrade ({}): {}".format(
+        dev_count, ", ".join(device_list)
     ))
 
     # Make sure we only upgrade Arista access switches

From d69c996b64a7be3351d04293688f92dbfa1e4648 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 4 Jun 2020 09:17:49 +0200
Subject: [PATCH 073/102] Add integration test for update facts

---
 test/integrationtests.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/integrationtests.py b/test/integrationtests.py
index 3d4cbb69..fb057b55 100644
--- a/test/integrationtests.py
+++ b/test/integrationtests.py
@@ -240,6 +240,14 @@ def test_8_sysversion(self):
         )
         self.assertEqual(r.status_code, 200, "Failed to get CNaaS-NMS version")
 
+    def test_9_update_facts_dist(self):
+        r = requests.post(
+            f'{URL}/api/v1.0/device_update_facts',
+            headers=AUTH_HEADER,
+            json={"hostname": "eosdist1"},
+            verify=TLS_VERIFY
+        )
+        self.assertEqual(r.status_code, 200, "Failed to do update facts for dist")
 
 
 if __name__ == '__main__':

From 64e9071dea868313bf34a51dac35755c9dde2b03 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 4 Jun 2020 09:18:32 +0200
Subject: [PATCH 074/102] Add docs for device_update_facts API call

---
 docs/apiref/devices.rst | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/docs/apiref/devices.rst b/docs/apiref/devices.rst
index ed020992..3192f588 100644
--- a/docs/apiref/devices.rst
+++ b/docs/apiref/devices.rst
@@ -202,4 +202,19 @@ To initialize a pair of ACCESS devices as an MLAG pair:
 
    curl https://localhost/api/v1.0/device_init/45 -d '{"hostname": "a1", "device_type": "ACCESS", "mlag_peer_id": 46, "mlag_peer_hostname": "a2"}' -X POST -H "Content-Type: application/json"
 
-For MLAG pairs the devices must be able to dectect it's peer via LLDP neighbors and compatible uplink devices for initialization to finish.
\ No newline at end of file
+For MLAG pairs the devices must be able to dectect it's peer via LLDP neighbors and compatible uplink devices for initialization to finish.
+
+Update facts
+------------
+
+To update the facts about a device (serial number, vendor, model and OS version)
+use this API call:
+
+::
+
+   curl https://localhost/api/v1.0/device_update_facts -d '{"hostname": "eosdist1"}' -X POST -H "Content-Type: application/json"
+
+This will schedule a job to log in to the device, get the facts and update the
+database. You can perform this action on both MANAGED and UNMANAGED devices.
+UNMANAGED devices might not be reachable so this could be a good test-call
+before moving the device back to the MANAGED state.
\ No newline at end of file

From cf3384706dab049aa55797129fd159845bc882d7 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 4 Jun 2020 09:18:49 +0200
Subject: [PATCH 075/102] Add API call for device_update_facts

---
 src/cnaas_nms/api/app.py         |  5 +--
 src/cnaas_nms/api/device.py      | 58 ++++++++++++++++++++++++++++++++
 src/cnaas_nms/confpush/update.py |  9 +++--
 3 files changed, 67 insertions(+), 5 deletions(-)

diff --git a/src/cnaas_nms/api/app.py b/src/cnaas_nms/api/app.py
index 74c54738..23243361 100644
--- a/src/cnaas_nms/api/app.py
+++ b/src/cnaas_nms/api/app.py
@@ -13,8 +13,8 @@
 from cnaas_nms.version import __api_version__
 from cnaas_nms.tools.log import get_logger
 
-from cnaas_nms.api.device import device_api, devices_api, \
-    device_init_api, device_syncto_api, device_discover_api
+from cnaas_nms.api.device import device_api, devices_api, device_init_api, \
+    device_syncto_api, device_discover_api, device_update_facts_api
 from cnaas_nms.api.linknet import api as links_api
 from cnaas_nms.api.firmware import api as firmware_api
 from cnaas_nms.api.interface import api as interfaces_api
@@ -94,6 +94,7 @@ def handle_error(self, e):
 api.add_namespace(device_init_api)
 api.add_namespace(device_syncto_api)
 api.add_namespace(device_discover_api)
+api.add_namespace(device_update_facts_api)
 api.add_namespace(links_api)
 api.add_namespace(firmware_api)
 api.add_namespace(interfaces_api)
diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 8c0810d3..64d3cb40 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -33,6 +33,9 @@
                               prefix='/api/{}'.format(__api_version__))
 device_discover_api = Namespace('device_discover', description='API to discover devices',
                                 prefix='/api/{}'.format(__api_version__))
+device_update_facts_api = Namespace('device_update_facts',
+                                    description='API to update facts about devices',
+                                    prefix='/api/{}'.format(__api_version__))
 
 
 device_model = device_api.model('device', {
@@ -72,6 +75,10 @@
     'resync': fields.Boolean(required=False)
 })
 
+device_update_facts_model = device_syncto_api.model('device_update_facts', {
+    'hostname': fields.String(required=False),
+})
+
 
 class DeviceByIdApi(Resource):
     @jwt_required
@@ -383,6 +390,56 @@ def post(self):
         return resp
 
 
+class DeviceUpdateFactsApi(Resource):
+    @jwt_required
+    @device_update_facts_api.expect(device_update_facts_model)
+    def post(self):
+        """ Start update facts of device(s) """
+        json_data = request.get_json()
+        kwargs: dict = {}
+
+        total_count: Optional[int] = None
+
+        if 'hostname' in json_data:
+            hostname = str(json_data['hostname'])
+            if not Device.valid_hostname(hostname):
+                return empty_result(
+                    status='error',
+                    data=f"Hostname '{hostname}' is not a valid hostname"
+                ), 400
+            with sqla_session() as session:
+                dev: Device = session.query(Device). \
+                    filter(Device.hostname == hostname).one_or_none()
+                if not dev or dev.state != DeviceState.MANAGED:
+                    return empty_result(
+                        status='error',
+                        data=f"Hostname '{hostname}' not found or is not a managed device"
+                    ), 400
+            kwargs['hostname'] = hostname
+            total_count = 1
+        else:
+            return empty_result(
+                status='error',
+                data="No target to be updated was specified"
+            ), 400
+
+        scheduler = Scheduler()
+        job_id = scheduler.add_onetime_job(
+            'cnaas_nms.confpush.update:update_facts',
+            when=0,
+            scheduled_by=get_jwt_identity(),
+            kwargs=kwargs)
+
+        res = empty_result(data=f"Scheduled job to update facts for {hostname}")
+        res['job_id'] = job_id
+
+        resp = make_response(json.dumps(res), 200)
+        if total_count:
+            resp.headers['X-Total-Count'] = total_count
+        resp.headers['Content-Type'] = "application/json"
+        return resp
+
+
 class DeviceConfigApi(Resource):
     @jwt_required
     def get(self, hostname: str):
@@ -421,4 +478,5 @@ def get(self, hostname: str):
 device_init_api.add_resource(DeviceInitApi, '/<int:device_id>')
 device_discover_api.add_resource(DeviceDiscoverApi, '')
 device_syncto_api.add_resource(DeviceSyncApi, '')
+device_update_facts_api.add_resource(DeviceUpdateFactsApi, '')
 # device/<string:hostname>/current_config
diff --git a/src/cnaas_nms/confpush/update.py b/src/cnaas_nms/confpush/update.py
index b1f4ff6b..965c101b 100644
--- a/src/cnaas_nms/confpush/update.py
+++ b/src/cnaas_nms/confpush/update.py
@@ -108,7 +108,7 @@ def update_facts(hostname: str,
         if not dev:
             raise ValueError("Device with hostname {} not found".format(hostname))
         if not (dev.state == DeviceState.MANAGED or dev.state == DeviceState.UNMANAGED):
-            raise ValueError("Device with ztp_mac {} is in incorrect state: {}".format(
+            raise ValueError("Device with hostname {} is in incorrect state: {}".format(
                 hostname, str(dev.state)
             ))
         hostname = dev.hostname
@@ -119,7 +119,7 @@ def update_facts(hostname: str,
     nrresult = nr_filtered.run(task=networking.napalm_get, getters=["facts"])
 
     if nrresult.failed:
-        logger.info("Could not contact device with hostname {}".format(hostname))
+        logger.error("Could not contact device with hostname {}".format(hostname))
         return NornirJobResult(nrresult=nrresult)
     try:
         facts = nrresult[hostname][0].result['facts']
@@ -129,11 +129,14 @@ def update_facts(hostname: str,
             dev.vendor = facts['vendor']
             dev.model = facts['model']
             dev.os_version = facts['os_version']
+        logger.debug("Updating facts for device {}: {}, {}, {}, {}".format(
+            hostname, facts['serial_number'], facts['vendor'], facts['model'], facts['os_version']
+        ))
     except Exception as e:
         logger.exception("Could not update device with hostname {} with new facts: {}".format(
             hostname, str(e)
         ))
-        logger.debug("nrresult for hostname {}: {}".format(hostname, nrresult))
+        logger.debug("Get facts nrresult for hostname {}: {}".format(hostname, nrresult))
         raise e
 
     return NornirJobResult(nrresult=nrresult)

From 9d0032c9fa1c76b4e64c7122cb19ea2bd0d4c4e0 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 4 Jun 2020 09:43:49 +0200
Subject: [PATCH 076/102] Start 1 sec into future

---
 src/cnaas_nms/api/device.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 19c6600b..84c6b239 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -444,7 +444,7 @@ def post(self):
         scheduler = Scheduler()
         job_id = scheduler.add_onetime_job(
             'cnaas_nms.confpush.update:update_facts',
-            when=0,
+            when=1,
             scheduled_by=get_jwt_identity(),
             kwargs=kwargs)
 

From 4899022059d9bedcad202daffa175fe7fa48670e Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Thu, 4 Jun 2020 15:24:16 +0200
Subject: [PATCH 077/102] Fix bug in check of device state for update facts

---
 src/cnaas_nms/api/device.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index 84c6b239..b16f27fd 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -428,10 +428,11 @@ def post(self):
             with sqla_session() as session:
                 dev: Device = session.query(Device). \
                     filter(Device.hostname == hostname).one_or_none()
-                if not dev or dev.state != DeviceState.MANAGED:
+                if not dev or (dev.state != DeviceState.MANAGED and
+                               dev.state != DeviceState.UNMANAGED):
                     return empty_result(
                         status='error',
-                        data=f"Hostname '{hostname}' not found or is not a managed device"
+                        data=f"Hostname '{hostname}' not found or is in invalid state"
                     ), 400
             kwargs['hostname'] = hostname
             total_count = 1

From 43ca2fd191d819ed9b360efe01c4d89bdac1b9f2 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 5 Jun 2020 14:37:28 +0200
Subject: [PATCH 078/102] Standalone script to render config from local
 templates and test dry_run without commiting to git. Used to help in template
 development

---
 src/cnaas_nms/tools/template_dry_run.py | 99 +++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100755 src/cnaas_nms/tools/template_dry_run.py

diff --git a/src/cnaas_nms/tools/template_dry_run.py b/src/cnaas_nms/tools/template_dry_run.py
new file mode 100755
index 00000000..1c004ce1
--- /dev/null
+++ b/src/cnaas_nms/tools/template_dry_run.py
@@ -0,0 +1,99 @@
+#!/bin/env python3
+
+import sys
+import os
+import requests
+import jinja2
+import yaml
+
+
+api_url = os.environ['CNAASURL']
+headers = {"Authorization": "Bearer "+os.environ['JWT_AUTH_TOKEN']}
+
+
+def get_entrypoint(platform, device_type):
+    mapfile = os.path.join(platform, 'mapping.yml')
+    if not os.path.isfile(mapfile):
+        raise Exception("File {} not found".format(mapfile))
+    with open(mapfile, 'r') as f:
+        mapping = yaml.safe_load(f)
+        template_file = mapping[device_type]['entrypoint']
+    return template_file
+
+
+def get_device_details(hostname):
+    r = requests.get(
+        f"{api_url}/api/v1.0/device/{hostname}",
+        headers=headers)
+    if r.status_code != 200:
+        raise Exception("Could not query device API")
+    device_data = r.json()['data']['devices'][0]
+
+    r = requests.get(
+        f"{api_url}/api/v1.0/device/{hostname}/generate_config",
+        headers=headers)
+    if r.status_code != 200:
+        raise Exception("Could not query generate_config API")
+    config_data = r.json()['data']['config']
+
+    return device_data['device_type'], device_data['platform'], \
+        config_data['available_variables'], config_data['generated_config']
+
+
+def render_template(platform, device_type, variables):
+    jinjaenv = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(platform),
+        undefined=jinja2.StrictUndefined, trim_blocks=True
+    )
+    template_secrets = {}
+    for env in os.environ:
+        if env.startswith('TEMPLATE_SECRET_'):
+            template_secrets[env] = os.environ[env]
+    template_vars = {**variables, **template_secrets}
+    template = jinjaenv.get_template(get_entrypoint(platform, device_type))
+    return template.render(**template_vars)
+
+
+def schedule_apply_dryrun(hostname, config):
+    data = {
+        'full_config': config,
+        'dry_run': True
+    }
+    r = requests.post(
+        f"{api_url}/api/v1.0/device/{hostname}/apply_config",
+        headers=headers,
+        json=data
+    )
+    if r.status_code != 200:
+        raise Exception("Could not schedule apply_config job via API")
+    return r.json()['job_id']
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: template_dry_run.py <hostname>")
+        sys.exit(1)
+
+    hostname = sys.argv[1]
+    try:
+        device_type, platform, variables, old_config = get_device_details(hostname)
+    except Exception as e:
+        print(e)
+        sys.exit(2)
+    variables['host'] = hostname
+    new_config = render_template(platform, device_type, variables)
+    print("OLD TEMPLATE CONFIG ==============================")
+    print(old_config)
+    print("NEW TEMPLATE CONFIG ==============================")
+    print(new_config)
+
+    try:
+        input("Start apply_config dry run? Ctrl-c to abort...")
+    except KeyboardInterrupt:
+        print("Exiting...")
+    else:
+        print("Apply config dry_run job: {}".format(schedule_apply_dryrun(hostname, new_config)))
+
+
+if __name__ == "__main__":
+    main()

From 3949060aa4349852c5befad9b3798a3405fd5551 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 5 Jun 2020 14:38:01 +0200
Subject: [PATCH 079/102] Add device/<hostname>/apply_config API call to apply
 exact specified config instead of using templates

---
 src/cnaas_nms/api/device.py | 48 +++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index b16f27fd..b048af8a 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -86,6 +86,11 @@
     'job_id': fields.Integer(required=True),
 })
 
+device_apply_config_model = device_api.model('device_apply_config', {
+    'dry_run': fields.Boolean(required=False),
+    'full_config': fields.String(required=True),
+})
+
 
 class DeviceByIdApi(Resource):
     @jwt_required
@@ -594,11 +599,54 @@ def post(self, hostname: str):
         return res, 200
 
 
+class DeviceApplyConfigApi(Resource):
+    @jwt_required
+    @device_api.expect(device_apply_config_model)
+    def post(self, hostname: str):
+        """Apply exact specified configuration to device without using templates"""
+        json_data = request.get_json()
+        apply_kwargs = {'hostname': hostname}
+        allow_live_run = False
+        if not Device.valid_hostname(hostname):
+            return empty_result(
+                status='error',
+                data=f"Invalid hostname specified"
+            ), 400
+
+        if 'full_config' not in json_data:
+            return empty_result('error', "full_config must be specified"), 400
+
+        if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \
+                and not json_data['dry_run']:
+            if allow_live_run:
+                apply_kwargs['dry_run'] = False
+            else:
+                return empty_result('error', "Apply config live_run is not allowed"), 400
+        else:
+            apply_kwargs['dry_run'] = True
+
+        apply_kwargs['config'] = json_data['full_config']
+
+        scheduler = Scheduler()
+        job_id = scheduler.add_onetime_job(
+            'cnaas_nms.confpush.sync_devices:apply_config',
+            when=1,
+            scheduled_by=get_jwt_identity(),
+            kwargs=apply_kwargs,
+        )
+
+        res = empty_result(data=f"Scheduled job to apply config {hostname}")
+        res['job_id'] = job_id
+
+        return res, 200
+
+
 # Devices
 device_api.add_resource(DeviceByIdApi, '/<int:device_id>')
 device_api.add_resource(DeviceByHostnameApi, '/<string:hostname>')
 device_api.add_resource(DeviceConfigApi, '/<string:hostname>/generate_config')
 device_api.add_resource(DevicePreviousConfigApi, '/<string:hostname>/previous_config')
+device_api.add_resource(DeviceApplyConfigApi, '/<string:hostname>/apply_config')
 device_api.add_resource(DeviceApi, '')
 devices_api.add_resource(DevicesApi, '')
 device_init_api.add_resource(DeviceInitApi, '/<int:device_id>')

From 878adfe14b4579f3652dffe9cc8667ed2140c99f Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 8 Jun 2020 09:54:44 +0200
Subject: [PATCH 080/102] Give error messages for missing modules and env
 variables

---
 src/cnaas_nms/tools/template_dry_run.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/cnaas_nms/tools/template_dry_run.py b/src/cnaas_nms/tools/template_dry_run.py
index 1c004ce1..f4dc1364 100755
--- a/src/cnaas_nms/tools/template_dry_run.py
+++ b/src/cnaas_nms/tools/template_dry_run.py
@@ -2,10 +2,17 @@
 
 import sys
 import os
-import requests
-import jinja2
-import yaml
-
+try:
+    import requests
+    import jinja2
+    import yaml
+except ModuleNotFoundError as e:
+    print("Please install python modules requests, jinja2 and yaml: {}".format(e))
+    sys.exit(3)
+
+if 'CNAASURL' not in os.environ or 'JWT_AUTH_TOKEN' not in os.environ:
+    print("Please export environment variables CNAASURL and JWT_AUTH_TOKEN")
+    sys.exit(4)
 
 api_url = os.environ['CNAASURL']
 headers = {"Authorization": "Bearer "+os.environ['JWT_AUTH_TOKEN']}
@@ -88,7 +95,7 @@ def main():
     print(new_config)
 
     try:
-        input("Start apply_config dry run? Ctrl-c to abort...")
+        input("Start apply_config dry run? Ctrl-c to abort or enter to continue...")
     except KeyboardInterrupt:
         print("Exiting...")
     else:

From c1a759419f3cae74e76ba1964361557e0045dc9e Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 8 Jun 2020 13:26:20 +0200
Subject: [PATCH 081/102] Add config option for allow apply_config liverun

---
 src/cnaas_nms/api/device.py        | 3 ++-
 src/cnaas_nms/tools/get_apidata.py | 7 +++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py
index b048af8a..63839850 100644
--- a/src/cnaas_nms/api/device.py
+++ b/src/cnaas_nms/api/device.py
@@ -20,6 +20,7 @@
 from cnaas_nms.tools.log import get_logger
 from flask_jwt_extended import jwt_required, get_jwt_identity
 from cnaas_nms.version import __api_version__
+from cnaas_nms.tools.get_apidata import get_apidata
 
 
 logger = get_logger()
@@ -606,7 +607,7 @@ def post(self, hostname: str):
         """Apply exact specified configuration to device without using templates"""
         json_data = request.get_json()
         apply_kwargs = {'hostname': hostname}
-        allow_live_run = False
+        allow_live_run = get_apidata()['allow_apply_config_liverun']
         if not Device.valid_hostname(hostname):
             return empty_result(
                 status='error',
diff --git a/src/cnaas_nms/tools/get_apidata.py b/src/cnaas_nms/tools/get_apidata.py
index d1efe848..073c8009 100644
--- a/src/cnaas_nms/tools/get_apidata.py
+++ b/src/cnaas_nms/tools/get_apidata.py
@@ -1,6 +1,9 @@
 import yaml
 
 
-def get_apidata(config='/etc/cnaas-nms/api.yml'):
+def get_apidata(config='/etc/cnaas-nms/api.yml') -> dict:
+    defaults = {
+        'allow_apply_config_liverun': False
+    }
     with open(config, 'r') as api_file:
-        return yaml.safe_load(api_file)
+        return {**defaults, **yaml.safe_load(api_file)}

From dd01614078fcaf3ac31eb4e8699bd5ae3c1049b6 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 8 Jun 2020 13:37:15 +0200
Subject: [PATCH 082/102] Add docs for apply_config API call and new config
 option allow_apply_config_liverun

---
 docs/apiref/devices.rst      | 16 ++++++++++++++++
 docs/configuration/index.rst |  1 +
 2 files changed, 17 insertions(+)

diff --git a/docs/apiref/devices.rst b/docs/apiref/devices.rst
index f98d9954..9afb39fd 100644
--- a/docs/apiref/devices.rst
+++ b/docs/apiref/devices.rst
@@ -211,6 +211,22 @@ have finished with a successful status for the specified device. The device
 will change to UNMANAGED state since it's no longer in sync with current
 templates and settings.
 
+Apply static config
+-------------------
+
+You can also test a static configuration specified in the API call directly
+instead of generating the configuration via templates and settings.
+This can be useful when developing new templates (see template_dry_run.py tool)
+when you don't wish to do the commit/push/refresh/sync workflow for every
+iteration. By default only dry_run are allowed, but you can configure api.yml
+to allow apply config live run as well.
+
+::
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/apply_config" -X POST -d '{"full_config": "hostname eosdist1\n...", "dry_run": True}' -H "Content-Type: application/json"
+
+This will schedule a job to send the configuration to the device.
+
 Initialize device
 -----------------
 
diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst
index 4f1d744a..8cc122a6 100644
--- a/docs/configuration/index.rst
+++ b/docs/configuration/index.rst
@@ -23,6 +23,7 @@ Defines parameters for the API:
 - jwtcert: Defines the path to the public JWT certificate used to verify JWT tokens
 - httpd_url: URL to the httpd container containing firmware images
 - verify_tls: Verify certificate for connections to httpd/firmware server
+- allow_apply_config_liverun: Allow liverun on apply_config API call. Defaults to False.
 
 /etc/cnaas-nms/repository.yml
 -----------------------------

From d8b008492196b53d2301c68184154fac72e731c8 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 9 Jun 2020 09:04:11 +0200
Subject: [PATCH 083/102] Add device_model and device_os_version template
 variables

---
 docs/reporef/index.rst                 | 6 ++++++
 src/cnaas_nms/confpush/init_device.py  | 4 +++-
 src/cnaas_nms/confpush/sync_devices.py | 4 +++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/docs/reporef/index.rst b/docs/reporef/index.rst
index 06cd4008..5f4bf999 100644
--- a/docs/reporef/index.rst
+++ b/docs/reporef/index.rst
@@ -44,6 +44,12 @@ that are exposed from CNaaS includes:
 
 - access_auto: A list of access_auto interfacs. Using same keys as uplinks.
 
+- device_model: Device model string, same as "model" in the device API. Can be
+  used if you need model specific configuration lines.
+
+- device_os_version: Device OS version string, same as "os_version" in the
+  device API. Can be used if you need OS version specific configuration lines.
+
 Additional variables available for distribution switches:
 
 - infra_ip: IPv4 infrastructure VRF address (ex 10.199.0.0)
diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py
index 54d27542..fb8feda1 100644
--- a/src/cnaas_nms/confpush/init_device.py
+++ b/src/cnaas_nms/confpush/init_device.py
@@ -206,7 +206,9 @@ def init_access_device_step1(device_id: int, new_hostname: str,
             'mgmt_prefixlen': int(mgmt_gw_ipif.network.prefixlen),
             'interfaces': [],
             'mgmt_vlan_id': mgmtdomain.vlan,
-            'mgmt_gw': mgmt_gw_ipif.ip
+            'mgmt_gw': mgmt_gw_ipif.ip,
+            'device_model': dev.model,
+            'device_os_version': dev.os_version
         }
         intfs = session.query(Interface).filter(Interface.device == dev).all()
         intf: Interface
diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 11bb6051..1c9bf558 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -129,7 +129,9 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False,
             raise ValueError("Unknown platform: {}".format(dev.platform))
         settings, settings_origin = get_settings(hostname, devtype)
         device_variables = {
-            'mgmt_ip': str(mgmt_ip)
+            'mgmt_ip': str(mgmt_ip),
+            'device_model': dev.model,
+            'device_os_version': dev.os_version
         }
 
         if devtype == DeviceType.ACCESS:

From cd25b31cb5dbf20f2bb1a36ba6f421cc68338ac2 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 9 Jun 2020 09:12:44 +0200
Subject: [PATCH 084/102] Change change_score to use individual max instead of
 average

---
 src/cnaas_nms/confpush/sync_devices.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 1c9bf558..0c3a8e96 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -557,13 +557,9 @@ def exclude_filter(host, exclude_list=failed_hosts+unchanged_hosts):
         total_change_score = 0
     elif not change_scores or total_change_score >= 100 or failed_hosts:
         total_change_score = 100
-    elif max(change_scores) > 1000:
-        # If some device has a score higher than this, disregard any averages
-        # and report max impact score
-        total_change_score = 100
     else:
-        # calculate median value and round up, use min value of 1 and max of 100
-        total_change_score = max(min(int(median(change_scores) + 0.5), 100), 1)
+        # use individual max as total_change_score, range 1-100
+        total_change_score = max(min(int(max(change_scores) + 0.5), 100), 1)
     logger.info(
         "Change impact score: {} (dry_run: {}, selected devices: {}, changed devices: {})".
             format(total_change_score, dry_run, len(device_list), len(changed_hosts)))

From c0aebab6c792734e54697afdd2a35f1d3964075e Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 10 Jun 2020 09:34:48 +0200
Subject: [PATCH 085/102] Add fallback to find mgmtdomain via any two core
 devices if connected dist devices does not have an mgmtdomain

---
 src/cnaas_nms/db/helper.py | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/cnaas_nms/db/helper.py b/src/cnaas_nms/db/helper.py
index c47def7c..e603832a 100644
--- a/src/cnaas_nms/db/helper.py
+++ b/src/cnaas_nms/db/helper.py
@@ -3,7 +3,7 @@
 from ipaddress import IPv4Interface, IPv4Address
 
 import netaddr
-from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 
 from cnaas_nms.db.device import Device, DeviceType
 from cnaas_nms.db.mgmtdomain import Mgmtdomain
@@ -49,12 +49,22 @@ def find_mgmtdomain(session, hostnames: List[str]) -> Optional[Mgmtdomain]:
             raise ValueError("Both uplink devices must be of same device type: {}, {}".format(
                 device0.hostname, device1.hostname
             ))
-        mgmtdomain: Mgmtdomain = session.query(Mgmtdomain).\
-            filter(
-                ((Mgmtdomain.device_a == device0) & (Mgmtdomain.device_b == device1))
-                |
-                ((Mgmtdomain.device_a == device1) & (Mgmtdomain.device_b == device0))
-            ).one_or_none()
+        try:
+            mgmtdomain: Mgmtdomain = session.query(Mgmtdomain).\
+                filter(
+                    ((Mgmtdomain.device_a == device0) & (Mgmtdomain.device_b == device1))
+                    |
+                    ((Mgmtdomain.device_a == device1) & (Mgmtdomain.device_b == device0))
+                ).one_or_none()
+            if not mgmtdomain:
+                mgmtdomain: Mgmtdomain = session.query(Mgmtdomain).filter(
+                    (Mgmtdomain.device_a.has(Device.device_type == DeviceType.CORE))
+                    |
+                    (Mgmtdomain.device_b.has(Device.device_type == DeviceType.CORE))
+                ).one_or_none()
+        except MultipleResultsFound:
+            raise Exception(
+                "Found multiple possible mgmtdomains, please remove any redundant mgmtdomains")
     elif device0.device_type == DeviceType.ACCESS or device1.device_type == DeviceType.ACCESS:
         if device0.device_type != DeviceType.ACCESS or device1.device_type != DeviceType.ACCESS:
             raise ValueError("Both uplink devices must be of same device type: {}, {}".format(

From 55d3d7bc3c545f0969a6a5f5bbe0dc229b458758 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 10 Jun 2020 09:48:32 +0200
Subject: [PATCH 086/102] Add mgmtdomains variables for core devices in syncto
 job

---
 src/cnaas_nms/confpush/sync_devices.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 0c3a8e96..2a77f357 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -210,15 +210,14 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False,
                             'config': intf['config'],
                             'indexnum': ifindexnum
                         })
-            if devtype == DeviceType.DIST:
-                for mgmtdom in cnaas_nms.db.helper.get_all_mgmtdomains(session, hostname):
-                    fabric_device_variables['mgmtdomains'].append({
-                        'id': mgmtdom.id,
-                        'ipv4_gw': mgmtdom.ipv4_gw,
-                        'vlan': mgmtdom.vlan,
-                        'description': mgmtdom.description,
-                        'esi_mac': mgmtdom.esi_mac
-                    })
+            for mgmtdom in cnaas_nms.db.helper.get_all_mgmtdomains(session, hostname):
+                fabric_device_variables['mgmtdomains'].append({
+                    'id': mgmtdom.id,
+                    'ipv4_gw': mgmtdom.ipv4_gw,
+                    'vlan': mgmtdom.vlan,
+                    'description': mgmtdom.description,
+                    'esi_mac': mgmtdom.esi_mac
+                })
             # find fabric neighbors
             fabric_links = []
             for neighbor_d in dev.get_neighbors(session):

From ec4182426d55e58b1926e7ee9fdc4c3dde74ab29 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 10 Jun 2020 14:38:20 +0200
Subject: [PATCH 087/102] Fix mgmtdomain API to be consistent with
 documentation and device API.

---
 docs/apiref/mgmtdomains.rst     |  2 +-
 src/cnaas_nms/api/app.py        |  3 ++-
 src/cnaas_nms/api/mgmtdomain.py | 16 +++++++++-------
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/docs/apiref/mgmtdomains.rst b/docs/apiref/mgmtdomains.rst
index b6435acd..bbf5a7ba 100644
--- a/docs/apiref/mgmtdomains.rst
+++ b/docs/apiref/mgmtdomains.rst
@@ -63,7 +63,7 @@ Example using CURL:
 
 ::
 
-   curl -s -H "Authorization: Bearer $JWT_AUTH_TOKEN" ${CNAASURL}/api/v1.0/mgmtdomains -H "Content-Type: application/json" -X POST -d '{"ipv4_gw": "10.0.6.1/24", "device_a": "dist1", "device_b": "dist2", "vlan": 600}'
+   curl -s -H "Authorization: Bearer $JWT_AUTH_TOKEN" ${CNAASURL}/api/v1.0/mgmtdomain -H "Content-Type: application/json" -X POST -d '{"ipv4_gw": "10.0.6.1/24", "device_a": "dist1", "device_b": "dist2", "vlan": 600}'
 
 
 Update management domain
diff --git a/src/cnaas_nms/api/app.py b/src/cnaas_nms/api/app.py
index 7a8ad3cd..a51be805 100644
--- a/src/cnaas_nms/api/app.py
+++ b/src/cnaas_nms/api/app.py
@@ -21,7 +21,7 @@
 from cnaas_nms.api.firmware import api as firmware_api
 from cnaas_nms.api.interface import api as interfaces_api
 from cnaas_nms.api.jobs import job_api, jobs_api, joblock_api
-from cnaas_nms.api.mgmtdomain import api as mgmtdomains_api
+from cnaas_nms.api.mgmtdomain import mgmtdomain_api, mgmtdomains_api
 from cnaas_nms.api.groups import api as groups_api
 from cnaas_nms.api.repository import api as repository_api
 from cnaas_nms.api.settings import api as settings_api
@@ -106,6 +106,7 @@ def handle_error(self, e):
 api.add_namespace(job_api)
 api.add_namespace(jobs_api)
 api.add_namespace(joblock_api)
+api.add_namespace(mgmtdomain_api)
 api.add_namespace(mgmtdomains_api)
 api.add_namespace(groups_api)
 api.add_namespace(repository_api)
diff --git a/src/cnaas_nms/api/mgmtdomain.py b/src/cnaas_nms/api/mgmtdomain.py
index 1483d21d..c61a180c 100644
--- a/src/cnaas_nms/api/mgmtdomain.py
+++ b/src/cnaas_nms/api/mgmtdomain.py
@@ -11,10 +11,12 @@
 from cnaas_nms.version import __api_version__
 
 
-api = Namespace('mgmtdomains', description='API for handling managemeent domains',
-                prefix='/api/{}'.format(__api_version__))
+mgmtdomains_api = Namespace('mgmtdomains', description='API for handling management domains',
+                            prefix='/api/{}'.format(__api_version__))
+mgmtdomain_api = Namespace('mgmtdomain', description='API for handling a single management domain',
+                           prefix='/api/{}'.format(__api_version__))
 
-mgmtdomain_model = api.model('mgmtdomain', {
+mgmtdomain_model = mgmtdomain_api.model('mgmtdomain', {
     'device_a': fields.String(required=True),
     'device_b': fields.String(required=True),
     'vlan': fields.Integer(required=True),
@@ -51,7 +53,7 @@ def delete(self, mgmtdomain_id):
                 return empty_result('error', "Management domain not found"), 404
 
     @jwt_required
-    @api.expect(mgmtdomain_model)
+    @mgmtdomain_api.expect(mgmtdomain_model)
     def put(self, mgmtdomain_id):
         """ Modify management domain """
         json_data = request.get_json()
@@ -105,7 +107,7 @@ def get(self):
         return result
 
     @jwt_required
-    @api.expect(mgmtdomain_model)
+    @mgmtdomain_api.expect(mgmtdomain_model)
     def post(self):
         """ Add management domain """
         json_data = request.get_json()
@@ -169,5 +171,5 @@ def post(self):
                 return empty_result('error', errors), 400
 
 
-api.add_resource(MgmtdomainsApi, '')
-api.add_resource(MgmtdomainByIdApi, '/<int:mgmtdomain_id>')
+mgmtdomains_api.add_resource(MgmtdomainsApi, '')
+mgmtdomain_api.add_resource(MgmtdomainByIdApi, '/<int:mgmtdomain_id>')

From aaddd70748096d211386c3beabfccbe0a5f3ff69 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 10 Jun 2020 14:52:41 +0200
Subject: [PATCH 088/102] Add some documentation for new use of mgmtdomain with
 two CORE devices

---
 docs/apiref/mgmtdomains.rst | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docs/apiref/mgmtdomains.rst b/docs/apiref/mgmtdomains.rst
index bbf5a7ba..9bceae72 100644
--- a/docs/apiref/mgmtdomains.rst
+++ b/docs/apiref/mgmtdomains.rst
@@ -6,7 +6,7 @@ Management domain can be retreived, added, updated an removed using this API.
 Get all managment domains
 -------------------------
 
-All management domain can be listed using CURL:
+All management domains can be listed using CURL:
 
 
 ::
@@ -39,6 +39,8 @@ That will return a JSON structured response which describes all domains availabl
       }
   }
 
+Note that some of these fields does not have a use case (yet).
+
 You can also specify one specifc mgmtdomain to query by using:
 
 ::
@@ -57,7 +59,15 @@ To add a new management domain we can to call the API with a few fields set in a
    * ipv4_gw (mandatory): The IPv4 gateway to be used, should be expressed with a prefix (10.0.0.1/24)
    * device_a (mandatory): Hostname of the first device
    * device_b (mandatory): Hostname of the second device
-   * vlan (mandatory): A VLAN
+   * vlan (mandatory): A VLAN ID
+
+device_a and device_b should be a pair of DIST devices that are connected to a
+specific set of access devices that should share the same management network.
+It's also possible to specify two CORE devices if there is a need to have the
+gateway/routing for all access switch management done in the CORE layer instead.
+In the case where two CORE devices are specified there should only be one single
+mgmtdomain defined for the entire NMS, and this mgmtdomain can only contain
+exactly two CORE devices even if there are more CORE devices in the network.
 
 Example using CURL:
 

From 50d4a34088b37581dfe7f3889b5b64c163a0a69d Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 12 Jun 2020 09:11:27 +0200
Subject: [PATCH 089/102] Update integrationtests api call

---
 test/integrationtests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integrationtests.py b/test/integrationtests.py
index 0b8a19f0..b5c74f63 100644
--- a/test/integrationtests.py
+++ b/test/integrationtests.py
@@ -120,7 +120,7 @@ def test_01_init_dist(self):
             "vlan": 600
         }
         r = requests.post(
-            f'{URL}/api/v1.0/mgmtdomains',
+            f'{URL}/api/v1.0/mgmtdomain',
             headers=AUTH_HEADER,
             json=new_mgmtdom_data,
             verify=TLS_VERIFY

From 19fa40e6a66ff9684480e02da9f1092cb9058ebf Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 12 Jun 2020 10:08:02 +0200
Subject: [PATCH 090/102] Revert POST to use mgmtdomains path, this is
 apparently how linknets API is and should maybe not be changed now?

---
 docs/apiref/mgmtdomains.rst | 2 +-
 test/integrationtests.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/apiref/mgmtdomains.rst b/docs/apiref/mgmtdomains.rst
index 9bceae72..cf9465c0 100644
--- a/docs/apiref/mgmtdomains.rst
+++ b/docs/apiref/mgmtdomains.rst
@@ -73,7 +73,7 @@ Example using CURL:
 
 ::
 
-   curl -s -H "Authorization: Bearer $JWT_AUTH_TOKEN" ${CNAASURL}/api/v1.0/mgmtdomain -H "Content-Type: application/json" -X POST -d '{"ipv4_gw": "10.0.6.1/24", "device_a": "dist1", "device_b": "dist2", "vlan": 600}'
+   curl -s -H "Authorization: Bearer $JWT_AUTH_TOKEN" ${CNAASURL}/api/v1.0/mgmtdomains -H "Content-Type: application/json" -X POST -d '{"ipv4_gw": "10.0.6.1/24", "device_a": "dist1", "device_b": "dist2", "vlan": 600}'
 
 
 Update management domain
diff --git a/test/integrationtests.py b/test/integrationtests.py
index b5c74f63..0b8a19f0 100644
--- a/test/integrationtests.py
+++ b/test/integrationtests.py
@@ -120,7 +120,7 @@ def test_01_init_dist(self):
             "vlan": 600
         }
         r = requests.post(
-            f'{URL}/api/v1.0/mgmtdomain',
+            f'{URL}/api/v1.0/mgmtdomains',
             headers=AUTH_HEADER,
             json=new_mgmtdom_data,
             verify=TLS_VERIFY

From 8ccd78a2ad326c44c7a1609b462924313219b842 Mon Sep 17 00:00:00 2001
From: Kristofer Hallin <kristofer@sunet.se>
Date: Tue, 16 Jun 2020 09:09:34 +0200
Subject: [PATCH 091/102] Accept firmware upgrade for all device types.

---
 src/cnaas_nms/confpush/firmware.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/cnaas_nms/confpush/firmware.py b/src/cnaas_nms/confpush/firmware.py
index 27281dea..9e45e79a 100644
--- a/src/cnaas_nms/confpush/firmware.py
+++ b/src/cnaas_nms/confpush/firmware.py
@@ -268,9 +268,9 @@ def device_upgrade(download: Optional[bool] = False,
         with sqla_session() as session:
             dev: Device = session.query(Device).\
                 filter(Device.hostname == device).one_or_none()
-            if not dev or dev.device_type != DeviceType.ACCESS:
-                raise Exception('Invalid device type: {}'.format(device))
-            if not dev or dev.platform != 'eos':
+            if not dev:
+                raise Exception('Could not find device: {}'.format(device))
+            if dev.platform != 'eos':
                 raise Exception('Invalid device platform: {}'.format(device))
 
     # Start tasks to take care of the upgrade

From 646e88e5811e7622934a528eac982497eb250e56 Mon Sep 17 00:00:00 2001
From: Kristofer Hallin <kristofer@sunet.se>
Date: Wed, 17 Jun 2020 09:23:44 +0200
Subject: [PATCH 092/102] Change exception error message.

---
 src/cnaas_nms/confpush/firmware.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cnaas_nms/confpush/firmware.py b/src/cnaas_nms/confpush/firmware.py
index 9e45e79a..2e1a29b2 100644
--- a/src/cnaas_nms/confpush/firmware.py
+++ b/src/cnaas_nms/confpush/firmware.py
@@ -271,7 +271,8 @@ def device_upgrade(download: Optional[bool] = False,
             if not dev:
                 raise Exception('Could not find device: {}'.format(device))
             if dev.platform != 'eos':
-                raise Exception('Invalid device platform: {}'.format(device))
+                raise Exception('Invalid device platform "{}" for device: {}'.format(
+                    dev.platform, device))
 
     # Start tasks to take care of the upgrade
     try:

From 32cd2af6b8a25db43e48ca3a1147ecc3e0708817 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 17 Jun 2020 09:28:29 +0200
Subject: [PATCH 093/102] Bump version to 1.1.0b1

---
 docs/changelog/index.rst | 19 +++++++++++++++++++
 src/cnaas_nms/version.py |  2 +-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst
index aa6c4818..a84b82d8 100644
--- a/docs/changelog/index.rst
+++ b/docs/changelog/index.rst
@@ -1,6 +1,25 @@
 Changelog
 =========
 
+Version 1.1.0
+-------------
+
+New features:
+
+- New options for connecting access switches:
+
+  - Two access switches as an MLAG pair
+  - Access switch connected to other access switch
+
+- New template variables:
+
+  - device_model: Hardware model of this device
+  - device_os_version: OS version of this device
+
+- Get/restore previous config versions for a device
+- API call to update facts (serial,os version etc) about device
+- Websocket event improvements for logs, jobs and device updates
+
 Version 1.0.0
 -------------
 
diff --git a/src/cnaas_nms/version.py b/src/cnaas_nms/version.py
index 02a84bff..a51ac9b8 100644
--- a/src/cnaas_nms/version.py
+++ b/src/cnaas_nms/version.py
@@ -1,3 +1,3 @@
-__version__ = '1.1.0dev0'
+__version__ = '1.1.0b1'
 __version_info__ = tuple([field for field in __version__.split('.')])
 __api_version__ = 'v1.0'

From ab6b7303b85732f7905908739d7912d36bfe4fae Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 17 Jun 2020 10:24:18 +0200
Subject: [PATCH 094/102] Mark affected devices as unsynchronized if config
 hash check fails

---
 src/cnaas_nms/confpush/sync_devices.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index 2a77f357..fe85ce85 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -467,6 +467,10 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No
         raise e
     else:
         if nrresult.failed:
+            # Mark devices as unsynchronized if config hash check failed
+            with sqla_session() as session:
+                session.query(Device).filter(Device.hostname.in_(nrresult.failed_hosts.keys())).\
+                    update({Device.synchronized: False}, synchronize_session=False)
             raise Exception('Configuration hash check failed for {}'.format(
                 ' '.join(nrresult.failed_hosts.keys())))
 

From 93efaac1c6f3be8a8767d76c9b89d8a007582838 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 29 Jun 2020 09:32:25 +0200
Subject: [PATCH 095/102] Log change score with one decimal point

---
 src/cnaas_nms/confpush/sync_devices.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index fe85ce85..aac522d3 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -514,7 +514,7 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No
             changed_hosts.append(host)
             if "change_score" in results[0].host:
                 change_scores.append(results[0].host["change_score"])
-                logger.debug("Change score for host {}: {}".format(
+                logger.debug("Change score for host {}: {:.1f}".format(
                     host, results[0].host["change_score"]))
         else:
             unchanged_hosts.append(host)
@@ -564,7 +564,7 @@ def exclude_filter(host, exclude_list=failed_hosts+unchanged_hosts):
         # use individual max as total_change_score, range 1-100
         total_change_score = max(min(int(max(change_scores) + 0.5), 100), 1)
     logger.info(
-        "Change impact score: {} (dry_run: {}, selected devices: {}, changed devices: {})".
+        "Change impact score: {:.1f} (dry_run: {}, selected devices: {}, changed devices: {})".
             format(total_change_score, dry_run, len(device_list), len(changed_hosts)))
 
     next_job_id = None

From 661228c447b207d8237559457ab5e9a8f25edb18 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Mon, 29 Jun 2020 09:33:35 +0200
Subject: [PATCH 096/102] Add a uniqueness ratio to lower score if many lines
 are exactly the same. Add some new modifiers for name, comment, ntp, snmp,
 vrf, neighbor, address-family, redistribute

---
 src/cnaas_nms/confpush/changescore.py | 56 ++++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/src/cnaas_nms/confpush/changescore.py b/src/cnaas_nms/confpush/changescore.py
index 8e758894..bd970809 100644
--- a/src/cnaas_nms/confpush/changescore.py
+++ b/src/cnaas_nms/confpush/changescore.py
@@ -11,6 +11,36 @@
         'regex': re.compile(str(line_start + r"description")),
         'modifier': 0.0
     },
+    {
+        'name': 'name',
+        'regex': re.compile(str(line_start + r"name")),
+        'modifier': 0.0
+    },
+    {
+        'name': 'comment',
+        'regex': re.compile(str(line_start + r"!")),
+        'modifier': 0.0
+    },
+    {
+        'name': 'dot1x',
+        'regex': re.compile(str(line_start + r"dot1x")),
+        'modifier': 0.5
+    },
+    {
+        'name': 'ntp',
+        'regex': re.compile(str(line_start + r"ntp")),
+        'modifier': 0.5
+    },
+    {
+        'name': 'snmp',
+        'regex': re.compile(str(line_start + r"snmp")),
+        'modifier': 0.5
+    },
+    {
+        'name': 'vrf',
+        'regex': re.compile(str(line_start + r"vrf")),
+        'modifier': 5.0
+    },
     {
         'name': 'removed ip address',
         'regex': re.compile(str(line_start_remove + r".*(ip address).*")),
@@ -21,16 +51,36 @@
         'regex': re.compile(str(line_start_remove + r"vlan")),
         'modifier': 10.0
     },
+    {
+        'name': 'spanning-tree mode',
+        'regex': re.compile(str(line_start + r"spanning-tree mode")),
+        'modifier': 50.0
+    },
     {
         'name': 'spanning-tree',
         'regex': re.compile(str(line_start + r"spanning-tree")),
-        'modifier': 50.0
+        'modifier': 5.0
     },
     {
         'name': 'removed routing',
         'regex': re.compile(str(line_start_remove + r".*(routing|router).*")),
         'modifier': 50.0
     },
+    {
+        'name': 'removed neighbor',
+        'regex': re.compile(str(line_start_remove + r"neighbor")),
+        'modifier': 10.0
+    },
+    {
+        'name': 'address-family',
+        'regex': re.compile(str(line_start + r"address-family")),
+        'modifier': 10.0
+    },
+    {
+        'name': 'redistribute',
+        'regex': re.compile(str(line_start + r"redistribute")),
+        'modifier': 10.0
+    },
 ]
 # TODO: multiline patterns / block-aware config
 
@@ -63,8 +113,10 @@ def calculate_score(config: str, diff: str) -> float:
             total_line_score += calculate_line_score(line)
 
     changed_ratio = changed_lines / float(len(config_lines))
+    unique_ratio = len(set(diff_lines)) / len(diff_lines)
 
     # Calculate score, 20% based on number of lines changed, 80% on individual
     # line score with applied modifiers
+    # Apply uniqueness ratio to lower score if many lines are the same
 
-    return (changed_ratio*100*0.2) + (total_line_score*0.8)
+    return ((changed_ratio*100*0.2) + (total_line_score*0.8)) * unique_ratio

From 169ce0a1e810969beb4fe374010feba84e6932ab Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 25 Aug 2020 09:06:11 +0200
Subject: [PATCH 097/102] Update requirements and set version 1.1rc1

---
 requirements.txt         | 30 +++++++++++++++---------------
 src/cnaas_nms/version.py |  2 +-
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index c6065f87..6f979a1b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,24 +1,24 @@
-alembic==1.4.0
+alembic==1.4.2
 APScheduler==3.6.3
-coverage==5.0.3
+coverage==5.2.1
 Flask-Cors==3.0.8
 Flask-JWT-Extended==3.24.1
-flask-restx==0.1.1
-Flask-SocketIO==4.2.1
-gevent==1.4.0
-GitPython==3.1.0
-mypy==0.761
+flask-restx==0.2.0
+Flask-SocketIO==4.3.1
+gevent==20.6.2
+GitPython==3.1.7
+mypy==0.782
 mypy-extensions==0.4.3
 nornir==2.4.0
 nose==1.3.7
 pluggy==0.13.1
-psycopg2==2.8.4
-psycopg2-binary==2.8.4
-redis==3.4.1
+psycopg2==2.8.5
+psycopg2-binary==2.8.5
+redis==3.5.3
 redis-lru==0.1.0
-Sphinx==2.4.3
-SQLAlchemy==1.3.13
+Sphinx==3.2.1
+SQLAlchemy==1.3.19
 sqlalchemy-stubs==0.3
-SQLAlchemy-Utils==0.36.1
-pydantic==1.4
-Werkzeug==0.16.1
+SQLAlchemy-Utils==0.36.8
+pydantic==1.6.1
+Werkzeug==1.0.1
diff --git a/src/cnaas_nms/version.py b/src/cnaas_nms/version.py
index a51ac9b8..792c1c78 100644
--- a/src/cnaas_nms/version.py
+++ b/src/cnaas_nms/version.py
@@ -1,3 +1,3 @@
-__version__ = '1.1.0b1'
+__version__ = '1.1.0rc1'
 __version_info__ = tuple([field for field in __version__.split('.')])
 __api_version__ = 'v1.0'

From 6895e2c47fbb39234fe21b5463b4275aa3871e00 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 25 Aug 2020 09:25:37 +0200
Subject: [PATCH 098/102] Bump napalm to 3.1.0

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 6f979a1b..850cfa54 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,6 +10,7 @@ GitPython==3.1.7
 mypy==0.782
 mypy-extensions==0.4.3
 nornir==2.4.0
+napalm==3.1.0
 nose==1.3.7
 pluggy==0.13.1
 psycopg2==2.8.5

From 299d10594e278651b1ab82d936e38d8d656e6e5e Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Tue, 25 Aug 2020 10:54:43 +0200
Subject: [PATCH 099/102] Update for new gevent signal api

---
 src/cnaas_nms/run.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cnaas_nms/run.py b/src/cnaas_nms/run.py
index 37ba660c..a73f0b0b 100644
--- a/src/cnaas_nms/run.py
+++ b/src/cnaas_nms/run.py
@@ -28,8 +28,8 @@ def save_coverage():
         cov.save()
 
     atexit.register(save_coverage)
-    gevent_signal(signal.SIGTERM, save_coverage)
-    gevent_signal(signal.SIGINT, save_coverage)
+    gevent_signal.signal(signal.SIGTERM, save_coverage)
+    gevent_signal.signal(signal.SIGINT, save_coverage)
 
 
 def get_app():

From a816a2f6a0a195333d04bc5a5170b6fb815432b8 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Wed, 26 Aug 2020 10:40:11 +0200
Subject: [PATCH 100/102] Avoid 'BlockingIOError: [Errno 11] write could not
 complete without blocking' by removing large prints

---
 src/cnaas_nms/confpush/sync_devices.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py
index aac522d3..9333b58c 100644
--- a/src/cnaas_nms/confpush/sync_devices.py
+++ b/src/cnaas_nms/confpush/sync_devices.py
@@ -461,7 +461,6 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No
         nrresult = nr_filtered.run(task=sync_check_hash,
                                    force=force,
                                    job_id=job_id)
-        print_result(nrresult)
     except Exception as e:
         logger.exception("Exception while checking config hash: {}".format(str(e)))
         raise e
@@ -483,7 +482,6 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No
     try:
         nrresult = nr_filtered.run(task=push_sync_device, dry_run=dry_run,
                                    job_id=job_id)
-        print_result(nrresult)
     except Exception as e:
         logger.exception("Exception while synchronizing devices: {}".format(str(e)))
         try:

From fab273a127beeab65aaa07302112fc39c1b260c9 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 25 Sep 2020 09:56:23 +0200
Subject: [PATCH 101/102] Fix issue where uwsgi/gevent does not work correctly
 causing websocket to block requests

(cherry picked from commit 6b0b2c81ef14a87a04280c24a4b4e1c92b1587f2)
---
 docker/api/Dockerfile                  |  3 ++-
 docker/api/cnaas-setup.sh              | 17 ++++++++++-------
 docker/api/config/supervisord_app.conf |  2 +-
 docker/api/config/uwsgi.ini            |  2 +-
 requirements.txt                       |  1 +
 5 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index 610f570d..86505e08 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -1,4 +1,5 @@
 FROM debian:buster
+ARG BUILDBRANCH=develop
 
 # Create directories
 RUN mkdir -p /opt/cnaas
@@ -14,7 +15,7 @@ COPY config/plugins.yml /etc/cnaas-nms/plugins.yml
 
 # Setup script
 COPY cnaas-setup.sh /opt/cnaas/cnaas-setup.sh
-RUN /opt/cnaas/cnaas-setup.sh
+RUN /opt/cnaas/cnaas-setup.sh $BUILDBRANCH
 
 # Prepare for supervisord, uwsgi, ngninx
 COPY nosetests.sh /opt/cnaas/
diff --git a/docker/api/cnaas-setup.sh b/docker/api/cnaas-setup.sh
index 85b1def7..d79193e8 100755
--- a/docker/api/cnaas-setup.sh
+++ b/docker/api/cnaas-setup.sh
@@ -25,11 +25,12 @@ apt-get update && \
       supervisor \
       libssl-dev \
       libpq-dev \
-      uwsgi \
+      libpcre2-dev \
+      libpcre3-dev \
       uwsgi-plugin-python3 \
     && apt-get clean
 
-#pip3 install uwsgi
+pip3 install uwsgi
 
 # Start venv
 python3 -m venv /opt/cnaas/venv
@@ -45,11 +46,13 @@ git checkout develop
 python3 -m pip install -r requirements.txt
 
 # Temporary for testing new branch
-#cd /opt/cnaas/venv/cnaas-nms/
-#git remote update
-#git fetch
-#git checkout --track origin/feature.websocket_events_redis
-#python3 -m pip install -r requirements.txt
+if [ "$1" != "develop" ] ; then
+	cd /opt/cnaas/venv/cnaas-nms/
+	git remote update
+	git fetch
+	git checkout --track origin/$1
+	python3 -m pip install -r requirements.txt
+fi
 
 chown -R www-data:www-data /opt/cnaas/settings
 chown -R www-data:www-data /opt/cnaas/templates
diff --git a/docker/api/config/supervisord_app.conf b/docker/api/config/supervisord_app.conf
index 314647e0..2805fb8b 100644
--- a/docker/api/config/supervisord_app.conf
+++ b/docker/api/config/supervisord_app.conf
@@ -7,7 +7,7 @@ pidfile=/tmp/supervisord.pid
 childlogdir=/tmp
 
 [program:uwsgi]
-command = /usr/bin/uwsgi --ini /opt/cnaas/venv/cnaas-nms/uwsgi.ini
+command = /usr/local/bin/uwsgi --ini /opt/cnaas/venv/cnaas-nms/uwsgi.ini
 autorestart=true
 
 [program:nginx]
diff --git a/docker/api/config/uwsgi.ini b/docker/api/config/uwsgi.ini
index 34920bda..05dcb178 100644
--- a/docker/api/config/uwsgi.ini
+++ b/docker/api/config/uwsgi.ini
@@ -2,7 +2,7 @@
 uid=www-data
 gid=www-data
 chdir = /opt/cnaas/venv/cnaas-nms/src/
-plugins = python3
+plugins = gevent
 callable = cnaas_app
 module = cnaas_nms.run
 socket = /tmp/uwsgi.sock
diff --git a/requirements.txt b/requirements.txt
index 850cfa54..80bc8299 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,3 +23,4 @@ sqlalchemy-stubs==0.3
 SQLAlchemy-Utils==0.36.8
 pydantic==1.6.1
 Werkzeug==1.0.1
+greenlet==0.4.16

From 0b53e9bef2f5934386f1b77ae136b54f8a83fa62 Mon Sep 17 00:00:00 2001
From: Johan Marcusson <johan.marcusson@sunet.se>
Date: Fri, 25 Sep 2020 12:46:26 +0200
Subject: [PATCH 102/102] Bump version to v1.1.0

---
 src/cnaas_nms/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cnaas_nms/version.py b/src/cnaas_nms/version.py
index 792c1c78..a676d27c 100644
--- a/src/cnaas_nms/version.py
+++ b/src/cnaas_nms/version.py
@@ -1,3 +1,3 @@
-__version__ = '1.1.0rc1'
+__version__ = '1.1.0'
 __version_info__ = tuple([field for field in __version__.split('.')])
 __api_version__ = 'v1.0'