diff --git a/alembic/versions/8a635012afa7_add_new_interface_config_types.py b/alembic/versions/8a635012afa7_add_new_interface_config_types.py new file mode 100644 index 00000000..dfb4687c --- /dev/null +++ b/alembic/versions/8a635012afa7_add_new_interface_config_types.py @@ -0,0 +1,31 @@ +"""add new interface config types + +Revision ID: 8a635012afa7 +Revises: 395427a732d6 +Create Date: 2020-03-26 09:21:15.439761 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '8a635012afa7' +down_revision = '395427a732d6' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + # ### end Alembic commands ### + op.execute("COMMIT") + op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'TEMPLATE' AFTER 'CUSTOM'") + op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'MLAG_PEER' AFTER 'TEMPLATE'") + op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'ACCESS_DOWNLINK' AFTER 'ACCESS_UPLINK'") + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index 610f570d..86505e08 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -1,4 +1,5 @@ FROM debian:buster +ARG BUILDBRANCH=develop # Create directories RUN mkdir -p /opt/cnaas @@ -14,7 +15,7 @@ COPY config/plugins.yml /etc/cnaas-nms/plugins.yml # Setup script COPY cnaas-setup.sh /opt/cnaas/cnaas-setup.sh -RUN /opt/cnaas/cnaas-setup.sh +RUN /opt/cnaas/cnaas-setup.sh $BUILDBRANCH # Prepare for supervisord, uwsgi, ngninx COPY nosetests.sh /opt/cnaas/ diff --git a/docker/api/cnaas-setup.sh b/docker/api/cnaas-setup.sh index 69cbcbc9..d79193e8 100755 --- a/docker/api/cnaas-setup.sh +++ b/docker/api/cnaas-setup.sh @@ -25,6 +25,9 @@ apt-get update && \ supervisor \ libssl-dev \ libpq-dev \ + libpcre2-dev \ + libpcre3-dev \ + uwsgi-plugin-python3 \ && apt-get clean pip3 install uwsgi @@ -43,11 +46,13 @@ git checkout develop python3 -m pip install -r requirements.txt # Temporary for testing new branch -#cd /opt/cnaas/venv/cnaas-nms/ -#git remote update -#git fetch -#git checkout --track origin/release-1.0.0 -#python3 -m pip install -r requirements.txt +if [ "$1" != "develop" ] ; then + cd /opt/cnaas/venv/cnaas-nms/ + git remote update + git fetch + git checkout --track origin/$1 + python3 -m pip install -r requirements.txt +fi chown -R www-data:www-data /opt/cnaas/settings chown -R www-data:www-data /opt/cnaas/templates diff --git a/docker/api/config/supervisord_app.conf b/docker/api/config/supervisord_app.conf index dcc3cd60..2805fb8b 100644 --- a/docker/api/config/supervisord_app.conf +++ b/docker/api/config/supervisord_app.conf @@ -12,4 +12,4 @@ autorestart=true [program:nginx] command=/usr/sbin/nginx -g "daemon off;" -autorestart=true \ No newline at end of file +autorestart=true diff --git a/docker/api/config/uwsgi.ini b/docker/api/config/uwsgi.ini index d72ca1a6..05dcb178 100644 --- a/docker/api/config/uwsgi.ini +++ b/docker/api/config/uwsgi.ini @@ -2,6 +2,7 @@ uid=www-data gid=www-data chdir = /opt/cnaas/venv/cnaas-nms/src/ +plugins = gevent callable = cnaas_app module = cnaas_nms.run socket = /tmp/uwsgi.sock @@ -18,3 +19,5 @@ lazy-apps = true # websocket support http-websockets = true gevent = 1000 +# don't log jwt tokens +log-drain = jwt= diff --git a/docker/dhcpd/dhcpd.conf b/docker/dhcpd/dhcpd.conf index 1c40e8a4..6422ca9e 100644 --- a/docker/dhcpd/dhcpd.conf +++ b/docker/dhcpd/dhcpd.conf @@ -52,6 +52,7 @@ class "JUNIPER" { option tftp-server-name "10.0.1.3"; } +# This empty subnet is required for dhcpd to start in the cnaas docker-compose network, do not remove subnet 10.0.1.0 netmask 255.255.255.0 { } diff --git a/docs/apiref/devices.rst b/docs/apiref/devices.rst index 2c32363f..9afb39fd 100644 --- a/docs/apiref/devices.rst +++ b/docs/apiref/devices.rst @@ -160,6 +160,15 @@ To remove a device, pass the device ID in a DELTE call: curl -X DELETE https://hostname/api/v1.0/device/10 +There is also the option to factory default and reboot the device +when removing it. This can be done like this: + +:: + + curl -H "Content-Type: application/json" -X DELETE -d + '{"factory_default": true}' https://hostname/api/v1.0/device/10 + + Preview config -------------- @@ -173,3 +182,83 @@ touching the device use generate_config: This will return both the generated configuration based on the template for this device type, and also a list of available vaiables that could be used in the template. + +View previous config +-------------------- + +You can also view previous versions of the configuration for a device. All +previous configurations are saved in the job database and can be found using +either a specific Job ID (using job_id=), a number of steps to walk backward +to find a previous configuration (previous=), or using a date to find the last +configuration applied to the device before that date. + +:: + + curl "https://hostname/api/v1.0/device//previous_config?before=2020-04-07T12:03:05" + + curl "https://hostname/api/v1.0/device//previous_config?previous=1" + + curl "https://hostname/api/v1.0/device//previous_config?job_id=12" + +If you want to restore a device to a previous configuration you can send a POST: + +:: + + curl "https://hostname/api/v1.0/device//previous_config" -X POST -d '{"job_id": 12, "dry_run": true}' -H "Content-Type: application/json" + +When sending a POST you must specify an exact job_id to restore. The job must +have finished with a successful status for the specified device. The device +will change to UNMANAGED state since it's no longer in sync with current +templates and settings. + +Apply static config +------------------- + +You can also test a static configuration specified in the API call directly +instead of generating the configuration via templates and settings. +This can be useful when developing new templates (see template_dry_run.py tool) +when you don't wish to do the commit/push/refresh/sync workflow for every +iteration. By default only dry_run are allowed, but you can configure api.yml +to allow apply config live run as well. + +:: + + curl "https://hostname/api/v1.0/device//apply_config" -X POST -d '{"full_config": "hostname eosdist1\n...", "dry_run": True}' -H "Content-Type: application/json" + +This will schedule a job to send the configuration to the device. + +Initialize device +----------------- + +For a more detailed explanation see documentation under Howto :ref:`ztp_intro`. + +To initialize a single ACCESS type device: + +:: + + curl https://localhost/api/v1.0/device_init/45 -d '{"hostname": "ex2300-top", "device_type": "ACCESS"}' -X POST -H "Content-Type: application/json" + +The device must be in state DISCOVERED to start initialization. The device must be able to detect compatible uplink devices via LLDP for initialization to finish. + +To initialize a pair of ACCESS devices as an MLAG pair: + +:: + + curl https://localhost/api/v1.0/device_init/45 -d '{"hostname": "a1", "device_type": "ACCESS", "mlag_peer_id": 46, "mlag_peer_hostname": "a2"}' -X POST -H "Content-Type: application/json" + +For MLAG pairs the devices must be able to dectect it's peer via LLDP neighbors and compatible uplink devices for initialization to finish. + +Update facts +------------ + +To update the facts about a device (serial number, vendor, model and OS version) +use this API call: + +:: + + curl https://localhost/api/v1.0/device_update_facts -d '{"hostname": "eosdist1"}' -X POST -H "Content-Type: application/json" + +This will schedule a job to log in to the device, get the facts and update the +database. You can perform this action on both MANAGED and UNMANAGED devices. +UNMANAGED devices might not be reachable so this could be a good test-call +before moving the device back to the MANAGED state. \ No newline at end of file diff --git a/docs/apiref/mgmtdomains.rst b/docs/apiref/mgmtdomains.rst index b6435acd..cf9465c0 100644 --- a/docs/apiref/mgmtdomains.rst +++ b/docs/apiref/mgmtdomains.rst @@ -6,7 +6,7 @@ Management domain can be retreived, added, updated an removed using this API. Get all managment domains ------------------------- -All management domain can be listed using CURL: +All management domains can be listed using CURL: :: @@ -39,6 +39,8 @@ That will return a JSON structured response which describes all domains availabl } } +Note that some of these fields does not have a use case (yet). + You can also specify one specifc mgmtdomain to query by using: :: @@ -57,7 +59,15 @@ To add a new management domain we can to call the API with a few fields set in a * ipv4_gw (mandatory): The IPv4 gateway to be used, should be expressed with a prefix (10.0.0.1/24) * device_a (mandatory): Hostname of the first device * device_b (mandatory): Hostname of the second device - * vlan (mandatory): A VLAN + * vlan (mandatory): A VLAN ID + +device_a and device_b should be a pair of DIST devices that are connected to a +specific set of access devices that should share the same management network. +It's also possible to specify two CORE devices if there is a need to have the +gateway/routing for all access switch management done in the CORE layer instead. +In the case where two CORE devices are specified there should only be one single +mgmtdomain defined for the entire NMS, and this mgmtdomain can only contain +exactly two CORE devices even if there are more CORE devices in the network. Example using CURL: diff --git a/docs/apiref/syncto.rst b/docs/apiref/syncto.rst index e5e07a3b..7c15afd9 100644 --- a/docs/apiref/syncto.rst +++ b/docs/apiref/syncto.rst @@ -54,6 +54,10 @@ Arguments: re-synchronized, if you specify this option as true then all devices will be checked. This option does not affect syncto jobs with a specified hostname, when you select only a single device via hostname it's always re-synchronized. Defaults to false. + - comment: Optionally add a comment that is saved in the job log. + This should be a string with max 255 characters. + - ticket_ref: Optionally reference a service ticket associated with this job. + This should be a string with max 32 characters. If neither hostname or device_type is specified all devices that needs to be sycnhronized will be selected. diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst index aa6c4818..a84b82d8 100644 --- a/docs/changelog/index.rst +++ b/docs/changelog/index.rst @@ -1,6 +1,25 @@ Changelog ========= +Version 1.1.0 +------------- + +New features: + +- New options for connecting access switches: + + - Two access switches as an MLAG pair + - Access switch connected to other access switch + +- New template variables: + + - device_model: Hardware model of this device + - device_os_version: OS version of this device + +- Get/restore previous config versions for a device +- API call to update facts (serial,os version etc) about device +- Websocket event improvements for logs, jobs and device updates + Version 1.0.0 ------------- diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst index 4f1d744a..8cc122a6 100644 --- a/docs/configuration/index.rst +++ b/docs/configuration/index.rst @@ -23,6 +23,7 @@ Defines parameters for the API: - jwtcert: Defines the path to the public JWT certificate used to verify JWT tokens - httpd_url: URL to the httpd container containing firmware images - verify_tls: Verify certificate for connections to httpd/firmware server +- allow_apply_config_liverun: Allow liverun on apply_config API call. Defaults to False. /etc/cnaas-nms/repository.yml ----------------------------- diff --git a/docs/howto/index.rst b/docs/howto/index.rst index f3b67af7..a0d30aa1 100644 --- a/docs/howto/index.rst +++ b/docs/howto/index.rst @@ -24,6 +24,8 @@ with dry_run to preview changes:: The API call to device_syncto will start a job running in the background on the API server. To show the progress/output of the job run the last command (/job) until you get a finished result. +.. _ztp_intro: + Zero-touch provisioning of access switch ---------------------------------------- diff --git a/docs/reporef/index.rst b/docs/reporef/index.rst index 9610e9e3..5f4bf999 100644 --- a/docs/reporef/index.rst +++ b/docs/reporef/index.rst @@ -44,6 +44,12 @@ that are exposed from CNaaS includes: - access_auto: A list of access_auto interfacs. Using same keys as uplinks. +- device_model: Device model string, same as "model" in the device API. Can be + used if you need model specific configuration lines. + +- device_os_version: Device OS version string, same as "os_version" in the + device API. Can be used if you need OS version specific configuration lines. + Additional variables available for distribution switches: - infra_ip: IPv4 infrastructure VRF address (ex 10.199.0.0) @@ -181,10 +187,10 @@ Contains a dictinary called "vxlans", which in turn has one dictinoary per vxlan name is the dictionary key and dictionaly values are: * vni: VXLAN ID, 1-16777215 - * vrf: VRF name + * vrf: VRF name. Optional unless ipv4_gw is also specified. * vlan_id: VLAN ID, 1-4095 * vlan_name: VLAN name, single word/no spaces, max 31 characters - * ipv4_gw: IPv4 address with CIDR netmask, ex: 192.168.0.1/24 + * ipv4_gw: IPv4 address with CIDR netmask, ex: 192.168.0.1/24. Optional. * groups: List of group names where this VXLAN/VLAN should be provisioned. If you select an access switch the parent dist switch should be automatically provisioned. diff --git a/requirements.txt b/requirements.txt index c6065f87..80bc8299 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,26 @@ -alembic==1.4.0 +alembic==1.4.2 APScheduler==3.6.3 -coverage==5.0.3 +coverage==5.2.1 Flask-Cors==3.0.8 Flask-JWT-Extended==3.24.1 -flask-restx==0.1.1 -Flask-SocketIO==4.2.1 -gevent==1.4.0 -GitPython==3.1.0 -mypy==0.761 +flask-restx==0.2.0 +Flask-SocketIO==4.3.1 +gevent==20.6.2 +GitPython==3.1.7 +mypy==0.782 mypy-extensions==0.4.3 nornir==2.4.0 +napalm==3.1.0 nose==1.3.7 pluggy==0.13.1 -psycopg2==2.8.4 -psycopg2-binary==2.8.4 -redis==3.4.1 +psycopg2==2.8.5 +psycopg2-binary==2.8.5 +redis==3.5.3 redis-lru==0.1.0 -Sphinx==2.4.3 -SQLAlchemy==1.3.13 +Sphinx==3.2.1 +SQLAlchemy==1.3.19 sqlalchemy-stubs==0.3 -SQLAlchemy-Utils==0.36.1 -pydantic==1.4 -Werkzeug==0.16.1 +SQLAlchemy-Utils==0.36.8 +pydantic==1.6.1 +Werkzeug==1.0.1 +greenlet==0.4.16 diff --git a/src/cnaas_nms/api/app.py b/src/cnaas_nms/api/app.py index 74c54738..a51be805 100644 --- a/src/cnaas_nms/api/app.py +++ b/src/cnaas_nms/api/app.py @@ -1,10 +1,12 @@ import os import sys +import re +from typing import Optional from flask import Flask, render_template, request, g from flask_restx import Api from flask_socketio import SocketIO, join_room -from flask_jwt_extended import JWTManager, decode_token +from flask_jwt_extended import JWTManager, decode_token, jwt_required, get_jwt_identity from flask_jwt_extended.exceptions import NoAuthorizationError from flask import jsonify @@ -13,13 +15,13 @@ from cnaas_nms.version import __api_version__ from cnaas_nms.tools.log import get_logger -from cnaas_nms.api.device import device_api, devices_api, \ - device_init_api, device_syncto_api, device_discover_api +from cnaas_nms.api.device import device_api, devices_api, device_init_api, \ + device_syncto_api, device_discover_api, device_update_facts_api from cnaas_nms.api.linknet import api as links_api from cnaas_nms.api.firmware import api as firmware_api from cnaas_nms.api.interface import api as interfaces_api from cnaas_nms.api.jobs import job_api, jobs_api, joblock_api -from cnaas_nms.api.mgmtdomain import api as mgmtdomains_api +from cnaas_nms.api.mgmtdomain import mgmtdomain_api, mgmtdomains_api from cnaas_nms.api.groups import api as groups_api from cnaas_nms.api.repository import api as repository_api from cnaas_nms.api.settings import api as settings_api @@ -44,6 +46,8 @@ } } +jwt_query_r = re.compile(r'jwt=[^ &]+') + class CnaasApi(Api): def handle_error(self, e): @@ -83,6 +87,7 @@ def handle_error(self, e): app.config['JWT_IDENTITY_CLAIM'] = 'sub' app.config['JWT_ALGORITHM'] = 'ES256' app.config['JWT_ACCESS_TOKEN_EXPIRES'] = False +app.config['JWT_TOKEN_LOCATION'] = ('headers', 'query_string') jwt = JWTManager(app) api = CnaasApi(app, prefix='/api/{}'.format(__api_version__), @@ -94,12 +99,14 @@ def handle_error(self, e): api.add_namespace(device_init_api) api.add_namespace(device_syncto_api) api.add_namespace(device_discover_api) +api.add_namespace(device_update_facts_api) api.add_namespace(links_api) api.add_namespace(firmware_api) api.add_namespace(interfaces_api) api.add_namespace(job_api) api.add_namespace(jobs_api) api.add_namespace(joblock_api) +api.add_namespace(mgmtdomain_api) api.add_namespace(mgmtdomains_api) api.add_namespace(groups_api) api.add_namespace(repository_api) @@ -107,15 +114,25 @@ def handle_error(self, e): api.add_namespace(plugins_api) api.add_namespace(system_api) - -# SocketIO listen for new log messages -@socketio.on('logs') -def ws_logs(data): - room: str = None - if 'level' in data and data['level'] in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: - room = data['level'] - elif 'jobid' in data and isinstance(data['jobid'], str): - room = data['jobid'] +# SocketIO on connect +@socketio.on('connect') +@jwt_required +def socketio_on_connect(): + user = get_jwt_identity() + if user: + logger.info('User: {} connected via socketio'.format(user)) + return True + else: + return False + +# SocketIO join event rooms +@socketio.on('events') +def socketio_on_events(data): + room: Optional[str] = None + if 'loglevel' in data and data['loglevel'] in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: + room = data['loglevel'] + elif 'update' in data and data['update'] in ['device', 'job']: + room = "update_{}".format(data['update']) else: return False # TODO: how to send error message to client? @@ -130,5 +147,10 @@ def log_request(response): user = decode_token(token).get('sub') except Exception: user = 'unknown' - logger.info('User: {}, Method: {}, Status: {}, URL: {}, JSON: {}'.format(user, request.method, response.status_code, request.url, request.json)) + try: + url = re.sub(jwt_query_r, '', request.url) + logger.info('User: {}, Method: {}, Status: {}, URL: {}, JSON: {}'.format( + user, request.method, response.status_code, url, request.json)) + except Exception: + pass return response diff --git a/src/cnaas_nms/api/device.py b/src/cnaas_nms/api/device.py index c6b7b91f..63839850 100644 --- a/src/cnaas_nms/api/device.py +++ b/src/cnaas_nms/api/device.py @@ -1,23 +1,26 @@ import json +import datetime from typing import Optional from flask import request, make_response from flask_restx import Resource, Namespace, fields from sqlalchemy import func -from nornir.core.filter import F +from sqlalchemy.exc import IntegrityError -import cnaas_nms.confpush.nornir_helper import cnaas_nms.confpush.init_device import cnaas_nms.confpush.sync_devices import cnaas_nms.confpush.underlay +from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector from cnaas_nms.api.generic import build_filter, empty_result from cnaas_nms.db.device import Device, DeviceState, DeviceType +from cnaas_nms.db.job import Job, JobNotFoundError, InvalidJobError from cnaas_nms.db.session import sqla_session from cnaas_nms.db.settings import get_groups from cnaas_nms.scheduler.scheduler import Scheduler from cnaas_nms.tools.log import get_logger from flask_jwt_extended import jwt_required, get_jwt_identity from cnaas_nms.version import __api_version__ +from cnaas_nms.tools.get_apidata import get_apidata logger = get_logger() @@ -33,6 +36,9 @@ prefix='/api/{}'.format(__api_version__)) device_discover_api = Namespace('device_discover', description='API to discover devices', prefix='/api/{}'.format(__api_version__)) +device_update_facts_api = Namespace('device_update_facts', + description='API to update facts about devices', + prefix='/api/{}'.format(__api_version__)) device_model = device_api.model('device', { @@ -72,6 +78,20 @@ 'resync': fields.Boolean(required=False) }) +device_update_facts_model = device_syncto_api.model('device_update_facts', { + 'hostname': fields.String(required=False), +}) + +device_restore_model = device_api.model('device_restore', { + 'dry_run': fields.Boolean(required=False), + 'job_id': fields.Integer(required=True), +}) + +device_apply_config_model = device_api.model('device_apply_config', { + 'dry_run': fields.Boolean(required=False), + 'full_config': fields.String(required=True), +}) + class DeviceByIdApi(Resource): @jwt_required @@ -90,14 +110,36 @@ def get(self, device_id): @jwt_required def delete(self, device_id): """ Delete device from ID """ - with sqla_session() as session: - dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none() - if dev: - session.delete(dev) - session.commit() + json_data = request.get_json() + + if json_data and 'factory_default' in json_data: + if isinstance(json_data['factory_default'], bool) and json_data['factory_default'] is True: + scheduler = Scheduler() + job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.erase:device_erase', + when=1, + scheduled_by=get_jwt_identity(), + kwargs={'device_id': device_id}) + return empty_result(data='Scheduled job {} to factory default device'.format(job_id)) + else: + with sqla_session() as session: + dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none() + if not dev: + return empty_result('error', "Device not found"), 404 + try: + session.delete(dev) + session.commit() + except IntegrityError as e: + session.rollback() + return empty_result( + status='error', + data="Could not remove device because existing references: {}".format(e)) + except Exception as e: + session.rollback() + return empty_result( + status='error', + data="Could not remove device: {}".format(e)) return empty_result(status="success", data={"deleted_device": dev.as_dict()}), 200 - else: - return empty_result('error', "Device not found"), 404 @jwt_required @device_api.expect(device_model) @@ -112,7 +154,7 @@ def put(self, device_id): return empty_result(status='error', data=f"No device with id {device_id}") errors = dev.device_update(**json_data) - if errors is not None: + if errors: return empty_result(status='error', data=errors), 404 return empty_result(status='success', data={"updated_device": dev.as_dict()}), 200 @@ -147,7 +189,7 @@ def post(self): with sqla_session() as session: instance: Device = session.query(Device).filter(Device.hostname == data['hostname']).one_or_none() - if instance is not None: + if instance: errors.append('Device already exists') return empty_result(status='error', data=errors), 400 if 'platform' not in data or data['platform'] not in supported_platforms: @@ -219,14 +261,34 @@ def post(self, device_id: int): if not DeviceType.has_name(device_type): return empty_result(status='error', data="Invalid 'device_type' provided"), 400 + job_kwargs = { + 'device_id': device_id, + 'new_hostname': new_hostname + } + + if 'mlag_peer_id' in json_data or 'mlag_peer_hostname' in json_data: + if 'mlag_peer_id' not in json_data or 'mlag_peer_hostname' not in json_data: + return empty_result( + status='error', + data="Both 'mlag_peer_id' and 'mlag_peer_hostname' must be specified"), 400 + if not isinstance(json_data['mlag_peer_id'], int): + return empty_result(status='error', data="'mlag_peer_id' must be an integer"), 400 + if not Device.valid_hostname(json_data['mlag_peer_hostname']): + return empty_result( + status='error', + data="Provided 'mlag_peer_hostname' is not valid"), 400 + job_kwargs['mlag_peer_id'] = json_data['mlag_peer_id'] + job_kwargs['mlag_peer_new_hostname'] = json_data['mlag_peer_hostname'] + if device_type == DeviceType.ACCESS.name: scheduler = Scheduler() job_id = scheduler.add_onetime_job( 'cnaas_nms.confpush.init_device:init_access_device_step1', when=1, scheduled_by=get_jwt_identity(), - kwargs={'device_id': device_id, - 'new_hostname': new_hostname}) + kwargs=job_kwargs) + else: + return empty_result(status='error', data="Unsupported 'device_type' provided"), 400 res = empty_result(data=f"Scheduled job to initialize device_id { device_id }") res['job_id'] = job_id @@ -266,9 +328,30 @@ class DeviceSyncApi(Resource): def post(self): """ Start sync of device(s) """ json_data = request.get_json() - kwargs: dict = {} + # default args + kwargs: dict = { + 'dry_run': True, + 'auto_push': False, + 'force': False, + 'resync': False + } + + if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \ + and not json_data['dry_run']: + kwargs['dry_run'] = False + if 'force' in json_data and isinstance(json_data['force'], bool): + kwargs['force'] = json_data['force'] + if 'auto_push' in json_data and isinstance(json_data['auto_push'], bool): + kwargs['auto_push'] = json_data['auto_push'] + if 'resync' in json_data and isinstance(json_data['resync'], bool): + kwargs['resync'] = json_data['resync'] + if 'comment' in json_data and isinstance(json_data['comment'], str): + kwargs['job_comment'] = json_data['comment'] + if 'ticket_ref' in json_data and isinstance(json_data['ticket_ref'], str): + kwargs['job_ticket_ref'] = json_data['ticket_ref'] total_count: Optional[int] = None + nr = cnaas_init() if 'hostname' in json_data: hostname = str(json_data['hostname']) @@ -277,17 +360,14 @@ def post(self): status='error', data=f"Hostname '{hostname}' is not a valid hostname" ), 400 - with sqla_session() as session: - dev: Device = session.query(Device).\ - filter(Device.hostname == hostname).one_or_none() - if not dev or dev.state != DeviceState.MANAGED: - return empty_result( - status='error', - data=f"Hostname '{hostname}' not found or is not a managed device" - ), 400 + _, total_count, _ = inventory_selector(nr, hostname=hostname) + if total_count != 1: + return empty_result( + status='error', + data=f"Hostname '{hostname}' not found or is not a managed device" + ), 400 kwargs['hostname'] = hostname what = hostname - total_count = 1 elif 'device_type' in json_data: devtype_str = str(json_data['device_type']).upper() if DeviceType.has_name(devtype_str): @@ -298,42 +378,25 @@ def post(self): data=f"Invalid device type '{json_data['device_type']}' specified" ), 400 what = f"{json_data['device_type']} devices" - with sqla_session() as session: - total_count = session.query(Device). \ - filter(Device.device_type == DeviceType[devtype_str]).count() + _, total_count, _ = inventory_selector(nr, resync=kwargs['resync'], + device_type=devtype_str) elif 'group' in json_data: group_name = str(json_data['group']) if group_name not in get_groups(): return empty_result(status='error', data='Could not find a group with name {}'.format(group_name)) kwargs['group'] = group_name what = 'group {}'.format(group_name) - nr = cnaas_nms.confpush.nornir_helper.cnaas_init() - nr_filtered = nr.filter(F(groups__contains=group_name)) - total_count = len(nr_filtered.inventory.hosts) + _, total_count, _ = inventory_selector(nr, resync=kwargs['resync'], + group=group_name) elif 'all' in json_data and isinstance(json_data['all'], bool) and json_data['all']: what = "all devices" - with sqla_session() as session: - total_count_q = session.query(Device).filter(Device.state == DeviceState.MANAGED) - if 'resync' in json_data and isinstance(json_data['resync'], bool) and json_data['resync']: - total_count = total_count_q.count() - else: - total_count = total_count_q.filter(Device.synchronized == False).count() + _, total_count, _ = inventory_selector(nr, resync=kwargs['resync']) else: return empty_result( status='error', data=f"No devices to synchronize was specified" ), 400 - if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \ - and not json_data['dry_run']: - kwargs['dry_run'] = False - if 'force' in json_data and isinstance(json_data['force'], bool): - kwargs['force'] = json_data['force'] - if 'auto_push' in json_data and isinstance(json_data['auto_push'], bool): - kwargs['auto_push'] = json_data['auto_push'] - if 'resync' in json_data and isinstance(json_data['resync'], bool): - kwargs['resync'] = json_data['resync'] - scheduler = Scheduler() job_id = scheduler.add_onetime_job( 'cnaas_nms.confpush.sync_devices:sync_devices', @@ -351,6 +414,57 @@ def post(self): return resp +class DeviceUpdateFactsApi(Resource): + @jwt_required + @device_update_facts_api.expect(device_update_facts_model) + def post(self): + """ Start update facts of device(s) """ + json_data = request.get_json() + kwargs: dict = {} + + total_count: Optional[int] = None + + if 'hostname' in json_data: + hostname = str(json_data['hostname']) + if not Device.valid_hostname(hostname): + return empty_result( + status='error', + data=f"Hostname '{hostname}' is not a valid hostname" + ), 400 + with sqla_session() as session: + dev: Device = session.query(Device). \ + filter(Device.hostname == hostname).one_or_none() + if not dev or (dev.state != DeviceState.MANAGED and + dev.state != DeviceState.UNMANAGED): + return empty_result( + status='error', + data=f"Hostname '{hostname}' not found or is in invalid state" + ), 400 + kwargs['hostname'] = hostname + total_count = 1 + else: + return empty_result( + status='error', + data="No target to be updated was specified" + ), 400 + + scheduler = Scheduler() + job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.update:update_facts', + when=1, + scheduled_by=get_jwt_identity(), + kwargs=kwargs) + + res = empty_result(data=f"Scheduled job to update facts for {hostname}") + res['job_id'] = job_id + + resp = make_response(json.dumps(res), 200) + if total_count: + resp.headers['X-Total-Count'] = total_count + resp.headers['Content-Type'] = "application/json" + return resp + + class DeviceConfigApi(Resource): @jwt_required def get(self, hostname: str): @@ -380,13 +494,164 @@ def get(self, hostname: str): return result +class DevicePreviousConfigApi(Resource): + @jwt_required + @device_api.param('job_id') + @device_api.param('previous') + @device_api.param('before') + def get(self, hostname: str): + args = request.args + result = empty_result() + result['data'] = {'config': None} + if not Device.valid_hostname(hostname): + return empty_result( + status='error', + data=f"Invalid hostname specified" + ), 400 + + kwargs = {} + if 'job_id' in args: + try: + kwargs['job_id'] = int(args['job_id']) + except Exception: + return empty_result('error', "job_id must be an integer"), 400 + elif 'previous' in args: + try: + kwargs['previous'] = int(args['previous']) + except Exception: + return empty_result('error', "previous must be an integer"), 400 + elif 'before' in args: + try: + kwargs['before'] = datetime.datetime.fromisoformat(args['before']) + except Exception: + return empty_result('error', "before must be a valid ISO format date time string"), 400 + + with sqla_session() as session: + try: + result['data'] = Job.get_previous_config(session, hostname, **kwargs) + except JobNotFoundError as e: + return empty_result('error', str(e)), 404 + except InvalidJobError as e: + return empty_result('error', str(e)), 500 + except Exception as e: + return empty_result('error', "Unhandled exception: {}".format(e)), 500 + + return result + + @jwt_required + @device_api.expect(device_restore_model) + def post(self, hostname: str): + """Restore configuration to previous version""" + json_data = request.get_json() + apply_kwargs = {'hostname': hostname} + config = None + if not Device.valid_hostname(hostname): + return empty_result( + status='error', + data=f"Invalid hostname specified" + ), 400 + + if 'job_id' in json_data: + try: + job_id = int(json_data['job_id']) + except Exception: + return empty_result('error', "job_id must be an integer"), 400 + else: + return empty_result('error', "job_id must be specified"), 400 + + with sqla_session() as session: + try: + prev_config_result = Job.get_previous_config(session, hostname, job_id=job_id) + failed = prev_config_result['failed'] + if not failed and 'config' in prev_config_result: + config = prev_config_result['config'] + except JobNotFoundError as e: + return empty_result('error', str(e)), 404 + except InvalidJobError as e: + return empty_result('error', str(e)), 500 + except Exception as e: + return empty_result('error', "Unhandled exception: {}".format(e)), 500 + + if failed: + return empty_result('error', "The specified job_id has a failed status"), 400 + + if not config: + return empty_result('error', "No config found in this job"), 500 + + if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \ + and not json_data['dry_run']: + apply_kwargs['dry_run'] = False + else: + apply_kwargs['dry_run'] = True + + apply_kwargs['config'] = config + + scheduler = Scheduler() + job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.sync_devices:apply_config', + when=1, + scheduled_by=get_jwt_identity(), + kwargs=apply_kwargs, + ) + + res = empty_result(data=f"Scheduled job to restore {hostname}") + res['job_id'] = job_id + + return res, 200 + + +class DeviceApplyConfigApi(Resource): + @jwt_required + @device_api.expect(device_apply_config_model) + def post(self, hostname: str): + """Apply exact specified configuration to device without using templates""" + json_data = request.get_json() + apply_kwargs = {'hostname': hostname} + allow_live_run = get_apidata()['allow_apply_config_liverun'] + if not Device.valid_hostname(hostname): + return empty_result( + status='error', + data=f"Invalid hostname specified" + ), 400 + + if 'full_config' not in json_data: + return empty_result('error', "full_config must be specified"), 400 + + if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \ + and not json_data['dry_run']: + if allow_live_run: + apply_kwargs['dry_run'] = False + else: + return empty_result('error', "Apply config live_run is not allowed"), 400 + else: + apply_kwargs['dry_run'] = True + + apply_kwargs['config'] = json_data['full_config'] + + scheduler = Scheduler() + job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.sync_devices:apply_config', + when=1, + scheduled_by=get_jwt_identity(), + kwargs=apply_kwargs, + ) + + res = empty_result(data=f"Scheduled job to apply config {hostname}") + res['job_id'] = job_id + + return res, 200 + + # Devices device_api.add_resource(DeviceByIdApi, '/') device_api.add_resource(DeviceByHostnameApi, '/') device_api.add_resource(DeviceConfigApi, '//generate_config') +device_api.add_resource(DevicePreviousConfigApi, '//previous_config') +device_api.add_resource(DeviceApplyConfigApi, '//apply_config') device_api.add_resource(DeviceApi, '') devices_api.add_resource(DevicesApi, '') device_init_api.add_resource(DeviceInitApi, '/') device_discover_api.add_resource(DeviceDiscoverApi, '') device_syncto_api.add_resource(DeviceSyncApi, '') +device_update_facts_api.add_resource(DeviceUpdateFactsApi, '') # device//current_config diff --git a/src/cnaas_nms/api/firmware.py b/src/cnaas_nms/api/firmware.py index 62214938..58d4af09 100644 --- a/src/cnaas_nms/api/firmware.py +++ b/src/cnaas_nms/api/firmware.py @@ -139,7 +139,7 @@ def get(self) -> tuple: try: res = requests.get(get_httpd_url(), verify=verify_tls()) - json_data = json.loads(res.content) + json_data = json.loads(res.content)['data'] except Exception as e: logger.exception(f"Exception when getting images: {e}") return empty_result(status='error', diff --git a/src/cnaas_nms/api/mgmtdomain.py b/src/cnaas_nms/api/mgmtdomain.py index 1483d21d..c61a180c 100644 --- a/src/cnaas_nms/api/mgmtdomain.py +++ b/src/cnaas_nms/api/mgmtdomain.py @@ -11,10 +11,12 @@ from cnaas_nms.version import __api_version__ -api = Namespace('mgmtdomains', description='API for handling managemeent domains', - prefix='/api/{}'.format(__api_version__)) +mgmtdomains_api = Namespace('mgmtdomains', description='API for handling management domains', + prefix='/api/{}'.format(__api_version__)) +mgmtdomain_api = Namespace('mgmtdomain', description='API for handling a single management domain', + prefix='/api/{}'.format(__api_version__)) -mgmtdomain_model = api.model('mgmtdomain', { +mgmtdomain_model = mgmtdomain_api.model('mgmtdomain', { 'device_a': fields.String(required=True), 'device_b': fields.String(required=True), 'vlan': fields.Integer(required=True), @@ -51,7 +53,7 @@ def delete(self, mgmtdomain_id): return empty_result('error', "Management domain not found"), 404 @jwt_required - @api.expect(mgmtdomain_model) + @mgmtdomain_api.expect(mgmtdomain_model) def put(self, mgmtdomain_id): """ Modify management domain """ json_data = request.get_json() @@ -105,7 +107,7 @@ def get(self): return result @jwt_required - @api.expect(mgmtdomain_model) + @mgmtdomain_api.expect(mgmtdomain_model) def post(self): """ Add management domain """ json_data = request.get_json() @@ -169,5 +171,5 @@ def post(self): return empty_result('error', errors), 400 -api.add_resource(MgmtdomainsApi, '') -api.add_resource(MgmtdomainByIdApi, '/') +mgmtdomains_api.add_resource(MgmtdomainsApi, '') +mgmtdomain_api.add_resource(MgmtdomainByIdApi, '/') diff --git a/src/cnaas_nms/api/tests/test_api.py b/src/cnaas_nms/api/tests/test_api.py index 66cc2360..0262344d 100644 --- a/src/cnaas_nms/api/tests/test_api.py +++ b/src/cnaas_nms/api/tests/test_api.py @@ -1,8 +1,8 @@ -import pprint import shutil import yaml import pkg_resources import os +import time import unittest import cnaas_nms.api.app @@ -20,11 +20,6 @@ def setUp(self): self.app = cnaas_nms.api.app.app self.app.wsgi_app = TestAppWrapper(self.app.wsgi_app, self.jwt_auth_token) self.client = self.app.test_client() -# self.tmp_postgres = PostgresTemporaryInstance() - - def tearDown(self): -# self.tmp_postgres.shutdown() - pass def test_get_single_device(self): hostname = "eosdist1" diff --git a/src/cnaas_nms/confpush/changescore.py b/src/cnaas_nms/confpush/changescore.py index 8e758894..bd970809 100644 --- a/src/cnaas_nms/confpush/changescore.py +++ b/src/cnaas_nms/confpush/changescore.py @@ -11,6 +11,36 @@ 'regex': re.compile(str(line_start + r"description")), 'modifier': 0.0 }, + { + 'name': 'name', + 'regex': re.compile(str(line_start + r"name")), + 'modifier': 0.0 + }, + { + 'name': 'comment', + 'regex': re.compile(str(line_start + r"!")), + 'modifier': 0.0 + }, + { + 'name': 'dot1x', + 'regex': re.compile(str(line_start + r"dot1x")), + 'modifier': 0.5 + }, + { + 'name': 'ntp', + 'regex': re.compile(str(line_start + r"ntp")), + 'modifier': 0.5 + }, + { + 'name': 'snmp', + 'regex': re.compile(str(line_start + r"snmp")), + 'modifier': 0.5 + }, + { + 'name': 'vrf', + 'regex': re.compile(str(line_start + r"vrf")), + 'modifier': 5.0 + }, { 'name': 'removed ip address', 'regex': re.compile(str(line_start_remove + r".*(ip address).*")), @@ -21,16 +51,36 @@ 'regex': re.compile(str(line_start_remove + r"vlan")), 'modifier': 10.0 }, + { + 'name': 'spanning-tree mode', + 'regex': re.compile(str(line_start + r"spanning-tree mode")), + 'modifier': 50.0 + }, { 'name': 'spanning-tree', 'regex': re.compile(str(line_start + r"spanning-tree")), - 'modifier': 50.0 + 'modifier': 5.0 }, { 'name': 'removed routing', 'regex': re.compile(str(line_start_remove + r".*(routing|router).*")), 'modifier': 50.0 }, + { + 'name': 'removed neighbor', + 'regex': re.compile(str(line_start_remove + r"neighbor")), + 'modifier': 10.0 + }, + { + 'name': 'address-family', + 'regex': re.compile(str(line_start + r"address-family")), + 'modifier': 10.0 + }, + { + 'name': 'redistribute', + 'regex': re.compile(str(line_start + r"redistribute")), + 'modifier': 10.0 + }, ] # TODO: multiline patterns / block-aware config @@ -63,8 +113,10 @@ def calculate_score(config: str, diff: str) -> float: total_line_score += calculate_line_score(line) changed_ratio = changed_lines / float(len(config_lines)) + unique_ratio = len(set(diff_lines)) / len(diff_lines) # Calculate score, 20% based on number of lines changed, 80% on individual # line score with applied modifiers + # Apply uniqueness ratio to lower score if many lines are the same - return (changed_ratio*100*0.2) + (total_line_score*0.8) + return ((changed_ratio*100*0.2) + (total_line_score*0.8)) * unique_ratio diff --git a/src/cnaas_nms/confpush/erase.py b/src/cnaas_nms/confpush/erase.py new file mode 100644 index 00000000..04c8a2d6 --- /dev/null +++ b/src/cnaas_nms/confpush/erase.py @@ -0,0 +1,93 @@ +import cnaas_nms.confpush.nornir_helper + +from cnaas_nms.tools.log import get_logger +from cnaas_nms.scheduler.scheduler import Scheduler +from cnaas_nms.scheduler.wrapper import job_wrapper +from cnaas_nms.confpush.nornir_helper import NornirJobResult +from cnaas_nms.db.session import sqla_session +from cnaas_nms.db.device import DeviceType, Device + +from nornir.plugins.functions.text import print_result +from nornir.plugins.tasks.networking import netmiko_send_command + + +logger = get_logger() + + +def device_erase_task(task, hostname: str) -> str: + try: + res = task.run(netmiko_send_command, command_string='enable', + expect_string='.*#', + name='Enable') + print_result(res) + + res = task.run(netmiko_send_command, + command_string='write erase now', + expect_string='.*#', + name='Write rase') + print_result(res) + except Exception as e: + logger.info('Failed to factory default device {}, reason: {}'.format( + task.host.name, e)) + raise Exception('Factory default device') + + try: + res = task.run(netmiko_send_command, command_string='reload force', + max_loops=2, + expect_string='.*') + print_result(res) + except Exception as e: + pass + + return "Device factory defaulted" + + +@job_wrapper +def device_erase(device_id: int = None, job_id: int = None) -> NornirJobResult: + + with sqla_session() as session: + dev: Device = session.query(Device).filter(Device.id == + device_id).one_or_none() + if dev: + hostname = dev.hostname + device_type = dev.device_type + else: + raise Exception('Could not find a device with ID {}'.format( + device_id)) + + if device_type != DeviceType.ACCESS: + raise Exception('Can only do factory default on access') + + nr = cnaas_nms.confpush.nornir_helper.cnaas_init() + nr_filtered = nr.filter(name=hostname).filter(managed=True) + + device_list = list(nr_filtered.inventory.hosts.keys()) + logger.info("Device selected: {}".format( + device_list + )) + + try: + nrresult = nr_filtered.run(task=device_erase_task, + hostname=hostname) + print_result(nrresult) + except Exception as e: + logger.exception('Exception while erasing device: {}'.format( + str(e))) + return NornirJobResult(nrresult=nrresult) + + failed_hosts = list(nrresult.failed_hosts.keys()) + for hostname in failed_hosts: + logger.error("Failed to factory default device '{}' failed".format( + hostname)) + + if nrresult.failed: + logger.error("Factory default failed") + + if failed_hosts == []: + with sqla_session() as session: + dev: Device = session.query(Device).filter(Device.id == + device_id).one_or_none() + session.delete(dev) + session.commit() + + return NornirJobResult(nrresult=nrresult) diff --git a/src/cnaas_nms/confpush/firmware.py b/src/cnaas_nms/confpush/firmware.py index c5219dfc..2e1a29b2 100644 --- a/src/cnaas_nms/confpush/firmware.py +++ b/src/cnaas_nms/confpush/firmware.py @@ -1,16 +1,13 @@ -import cnaas_nms.confpush.nornir_helper - +from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector from cnaas_nms.tools.log import get_logger -from cnaas_nms.scheduler.scheduler import Scheduler from cnaas_nms.scheduler.wrapper import job_wrapper from cnaas_nms.confpush.nornir_helper import NornirJobResult from cnaas_nms.db.session import sqla_session, redis_session from cnaas_nms.db.device import DeviceType, Device from nornir.plugins.functions.text import print_result -from nornir.plugins.tasks.networking import napalm_cli, napalm_configure, napalm_get +from nornir.plugins.tasks.networking import napalm_cli from nornir.plugins.tasks.networking import netmiko_send_command -from nornir.core.filter import F from nornir.core.task import MultiResult from napalm.base.exceptions import CommandErrorException @@ -69,13 +66,15 @@ def arista_firmware_download(task, filename: str, httpd_url: str) -> None: """ logger.info('Downloading firmware for {}'.format(task.host.name)) + url = httpd_url + '/' + filename + try: with sqla_session() as session: dev: Device = session.query(Device).\ filter(Device.hostname == task.host.name).one_or_none() device_type = dev.device_type - if device_type == 'ACCESS': + if device_type == DeviceType.ACCESS: firmware_download_cmd = 'copy {} flash:'.format(url) else: firmware_download_cmd = 'copy {} vrf MGMT flash:'.format(url) @@ -251,17 +250,17 @@ def device_upgrade(download: Optional[bool] = False, reboot: Optional[bool] = False, scheduled_by: Optional[str] = None) -> NornirJobResult: - nr = cnaas_nms.confpush.nornir_helper.cnaas_init() + nr = cnaas_init() if hostname: - nr_filtered = nr.filter(name=hostname).filter(managed=True) + nr_filtered, dev_count, _ = inventory_selector(nr, hostname=hostname) elif group: - nr_filtered = nr.filter(F(groups__contains=group)) + nr_filtered, dev_count, _ = inventory_selector(nr, group=group) else: - nr_filtered = nr.filter(synchronized=False).filter(managed=True) + raise ValueError("Neither hostname nor group specified for device_upgrade") device_list = list(nr_filtered.inventory.hosts.keys()) - logger.info("Device(s) selected for firmware upgrade: {}".format( - device_list + logger.info("Device(s) selected for firmware upgrade ({}): {}".format( + dev_count, ", ".join(device_list) )) # Make sure we only upgrade Arista access switches @@ -269,10 +268,11 @@ def device_upgrade(download: Optional[bool] = False, with sqla_session() as session: dev: Device = session.query(Device).\ filter(Device.hostname == device).one_or_none() - if not dev or dev.device_type != DeviceType.ACCESS: - raise Exception('Invalid device type: {}'.format(device)) - if not dev or dev.platform != 'eos': - raise Exception('Invalid device platform: {}'.format(device)) + if not dev: + raise Exception('Could not find device: {}'.format(device)) + if dev.platform != 'eos': + raise Exception('Invalid device platform "{}" for device: {}'.format( + dev.platform, device)) # Start tasks to take care of the upgrade try: diff --git a/src/cnaas_nms/confpush/get.py b/src/cnaas_nms/confpush/get.py index e4c0c405..b0aa4f18 100644 --- a/src/cnaas_nms/confpush/get.py +++ b/src/cnaas_nms/confpush/get.py @@ -2,7 +2,7 @@ import re import hashlib -from typing import Optional, Tuple, List +from typing import Optional, Tuple, List, Dict from nornir.core.deserializer.inventory import Inventory from nornir.core.filter import F @@ -16,7 +16,7 @@ from cnaas_nms.db.device import Device, DeviceType from cnaas_nms.db.linknet import Linknet from cnaas_nms.tools.log import get_logger -from cnaas_nms.db.interface import Interface +from cnaas_nms.db.interface import Interface, InterfaceConfigType def get_inventory(): @@ -93,23 +93,53 @@ def get_neighbors(hostname: Optional[str] = None, group: Optional[str] = None)\ return result -def get_uplinks(session, hostname: str) -> Tuple[List, List]: +def get_uplinks(session, hostname: str) -> Dict[str, str]: + """Returns dict with mapping of interface -> neighbor hostname""" logger = get_logger() # TODO: check if uplinks are already saved in database? - uplinks = [] - neighbor_hostnames = [] + uplinks = {} dev = session.query(Device).filter(Device.hostname == hostname).one() + neighbor_d: Device for neighbor_d in dev.get_neighbors(session): if neighbor_d.device_type == DeviceType.DIST: local_if = dev.get_neighbor_local_ifname(session, neighbor_d) + # TODO: check that dist interface is configured as downlink if local_if: - uplinks.append({'ifname': local_if}) - neighbor_hostnames.append(neighbor_d.hostname) - logger.debug("Uplinks for device {} detected: {} neighbor_hostnames: {}". \ - format(hostname, uplinks, neighbor_hostnames)) + uplinks[local_if] = neighbor_d.hostname + elif neighbor_d.device_type == DeviceType.ACCESS: + intfs: Interface = session.query(Interface).filter(Interface.device == neighbor_d). \ + filter(InterfaceConfigType == InterfaceConfigType.ACCESS_DOWNLINK).all() + local_if = dev.get_neighbor_local_ifname(session, neighbor_d) + remote_if = neighbor_d.get_neighbor_local_ifname(session, dev) + + intf: Interface + for intf in intfs: + if intf.name == remote_if: + uplinks[local_if] = neighbor_d.hostname + + logger.debug("Uplinks for device {} detected: {}". + format(hostname, ', '.join(["{}: {}".format(ifname, hostname) + for ifname, hostname in uplinks.items()]))) - return (uplinks, neighbor_hostnames) + return uplinks + + +def get_mlag_ifs(session, hostname, mlag_peer_hostname) -> Dict[str, int]: + """Returns dict with mapping of interface -> neighbor id + Return id instead of hostname since mlag peer will change hostname during init""" + logger = get_logger() + mlag_ifs = {} + + dev = session.query(Device).filter(Device.hostname == hostname).one() + for neighbor_d in dev.get_neighbors(session): + if neighbor_d.hostname == mlag_peer_hostname: + for local_if in dev.get_neighbor_local_ifnames(session, neighbor_d): + mlag_ifs[local_if] = neighbor_d.id + logger.debug("MLAG peer interfaces for device {} detected: {}". + format(hostname, ', '.join(["{}: {}".format(ifname, hostname) + for ifname, hostname in mlag_ifs.items()]))) + return mlag_ifs def get_interfaces(hostname: str) -> AggregatedResult: @@ -206,7 +236,7 @@ def update_inventory(hostname: str, site='default') -> dict: return diff -def update_linknets(hostname): +def update_linknets(session, hostname): """Update linknet data for specified device using LLDP neighbor data. """ logger = get_logger() @@ -217,60 +247,60 @@ def update_linknets(hostname): ret = [] - with sqla_session() as session: - local_device_inst = session.query(Device).filter(Device.hostname == hostname).one() - logger.debug("Updating linknets for device {} ...".format(local_device_inst.id)) - - for local_if, data in neighbors.items(): - logger.debug(f"Local: {local_if}, remote: {data[0]['hostname']} {data[0]['port']}") - remote_device_inst = session.query(Device).\ - filter(Device.hostname == data[0]['hostname']).one_or_none() - if not remote_device_inst: - logger.info(f"Unknown connected device: {data[0]['hostname']}") + local_device_inst = session.query(Device).filter(Device.hostname == hostname).one() + logger.debug("Updating linknets for device {} ...".format(local_device_inst.id)) + + for local_if, data in neighbors.items(): + logger.debug(f"Local: {local_if}, remote: {data[0]['hostname']} {data[0]['port']}") + remote_device_inst = session.query(Device).\ + filter(Device.hostname == data[0]['hostname']).one_or_none() + if not remote_device_inst: + logger.info(f"Unknown connected device: {data[0]['hostname']}") + continue + logger.debug(f"Remote device found, device id: {remote_device_inst.id}") + + # Check if linknet object already exists in database + local_devid = local_device_inst.id + check_linknet = session.query(Linknet).\ + filter( + ((Linknet.device_a_id == local_devid) & (Linknet.device_a_port == local_if)) + | + ((Linknet.device_b_id == local_devid) & (Linknet.device_b_port == local_if)) + | + ((Linknet.device_a_id == remote_device_inst.id) & + (Linknet.device_a_port == data[0]['port'])) + | + ((Linknet.device_b_id == remote_device_inst.id) & + (Linknet.device_b_port == data[0]['port'])) + ).one_or_none() + if check_linknet: + logger.debug(f"Found entry: {check_linknet.id}") + if ( + ( check_linknet.device_a_id == local_devid + and check_linknet.device_a_port == local_if + and check_linknet.device_b_id == remote_device_inst.id + and check_linknet.device_b_port == data[0]['port'] + ) + or + ( check_linknet.device_a_id == local_devid + and check_linknet.device_a_port == local_if + and check_linknet.device_b_id == remote_device_inst.id + and check_linknet.device_b_port == data[0]['port'] + ) + ): + # All info is the same, no update required continue - logger.debug(f"Remote device found, device id: {remote_device_inst.id}") - - # Check if linknet object already exists in database - local_devid = local_device_inst.id - check_linknet = session.query(Linknet).\ - filter( - ((Linknet.device_a_id == local_devid) & (Linknet.device_a_port == local_if)) - | - ((Linknet.device_b_id == local_devid) & (Linknet.device_b_port == local_if)) - | - ((Linknet.device_a_id == remote_device_inst.id) & - (Linknet.device_a_port == data[0]['port'])) - | - ((Linknet.device_b_id == remote_device_inst.id) & - (Linknet.device_b_port == data[0]['port'])) - ).one_or_none() - if check_linknet: - logger.debug(f"Found entry: {check_linknet.id}") - if ( - ( check_linknet.device_a_id == local_devid - and check_linknet.device_a_port == local_if - and check_linknet.device_b_id == remote_device_inst.id - and check_linknet.device_b_port == data[0]['port'] - ) - or - ( check_linknet.device_a_id == local_devid - and check_linknet.device_a_port == local_if - and check_linknet.device_b_id == remote_device_inst.id - and check_linknet.device_b_port == data[0]['port'] - ) - ): - # All info is the same, no update required - continue - else: - # TODO: update instead of delete+new insert? - session.delete(check_linknet) - session.commit() - - new_link = Linknet() - new_link.device_a = local_device_inst - new_link.device_a_port = local_if - new_link.device_b = remote_device_inst - new_link.device_b_port = data[0]['port'] - session.add(new_link) - ret.append(new_link.as_dict()) + else: + # TODO: update instead of delete+new insert? + session.delete(check_linknet) + session.commit() + + new_link = Linknet() + new_link.device_a = local_device_inst + new_link.device_a_port = local_if + new_link.device_b = remote_device_inst + new_link.device_b_port = data[0]['port'] + session.add(new_link) + ret.append(new_link.as_dict()) + session.commit() return ret diff --git a/src/cnaas_nms/confpush/init_device.py b/src/cnaas_nms/confpush/init_device.py index 1d828e03..fb8feda1 100644 --- a/src/cnaas_nms/confpush/init_device.py +++ b/src/cnaas_nms/confpush/init_device.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, List from ipaddress import IPv4Interface from nornir.plugins.tasks import networking, text @@ -14,10 +14,12 @@ import cnaas_nms.db.helper from cnaas_nms.db.session import sqla_session from cnaas_nms.db.device import Device, DeviceState, DeviceType, DeviceStateException +from cnaas_nms.db.interface import Interface, InterfaceConfigType from cnaas_nms.scheduler.scheduler import Scheduler from cnaas_nms.scheduler.wrapper import job_wrapper from cnaas_nms.confpush.nornir_helper import NornirJobResult -from cnaas_nms.confpush.update import update_interfacedb +from cnaas_nms.confpush.update import update_interfacedb_worker +from cnaas_nms.confpush.sync_devices import get_mlag_vars from cnaas_nms.db.git import RepoStructureException from cnaas_nms.db.settings import get_settings from cnaas_nms.plugins.pluginmanager import PluginManagerHandler @@ -73,25 +75,75 @@ def push_base_management_access(task, device_variables, job_id): task.host["config"] = r.result # Use extra low timeout for this since we expect to loose connectivity after changing IP - task.host.connection_options["napalm"] = ConnectionOptions(extras={"timeout": 5}) + task.host.connection_options["napalm"] = ConnectionOptions(extras={"timeout": 30}) - task.run(task=networking.napalm_configure, - name="Push base management config", - replace=True, - configuration=task.host["config"], - dry_run=False # TODO: temp for testing - ) + try: + task.run(task=networking.napalm_configure, + name="Push base management config", + replace=True, + configuration=task.host["config"], + dry_run=False + ) + except Exception: + task.run(task=networking.napalm_get, getters=["facts"]) + if not task.results[-1].failed: + raise InitError("Device {} did not commit new base management config".format( + task.host.name + )) + + +def pre_init_checks(session, device_id) -> Device: + # Check that we can find device and that it's in the correct state to start init + dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none() + if not dev: + raise ValueError(f"No device with id {device_id} found") + if dev.state != DeviceState.DISCOVERED: + raise DeviceStateException("Device must be in state DISCOVERED to begin init") + old_hostname = dev.hostname + # Perform connectivity check + nr = cnaas_nms.confpush.nornir_helper.cnaas_init() + nr_old_filtered = nr.filter(name=old_hostname) + try: + nrresult_old = nr_old_filtered.run(task=networking.napalm_get, getters=["facts"]) + except Exception as e: + raise ConnectionCheckError(f"Failed to connect to device_id {device_id}: {str(e)}") + if nrresult_old.failed: + print_result(nrresult_old) + raise ConnectionCheckError(f"Failed to connect to device_id {device_id}") + return dev + + +def pre_init_check_mlag(session, dev, mlag_peer_dev): + intfs: Interface = session.query(Interface).filter(Interface.device == dev).\ + filter(InterfaceConfigType == InterfaceConfigType.MLAG_PEER).all() + intf: Interface + for intf in intfs: + if intf.data['neighbor_id'] == mlag_peer_dev.id: + continue + else: + raise Exception("Inconsistent MLAG peer {} detected for device {}".format( + intf.data['neighbor'], dev.hostname + )) @job_wrapper def init_access_device_step1(device_id: int, new_hostname: str, + mlag_peer_id: Optional[int] = None, + mlag_peer_new_hostname: Optional[str] = None, + uplink_hostnames_arg: Optional[List[str]] = [], job_id: Optional[str] = None, scheduled_by: Optional[str] = None) -> NornirJobResult: - """Initialize access device for management by CNaaS-NMS + """Initialize access device for management by CNaaS-NMS. + If a MLAG/MC-LAG pair is to be configured both mlag_peer_id and + mlag_peer_new_hostname must be set. Args: device_id: Device to select for initialization - new_hostname: Hostname to configure for the new device + new_hostname: Hostname to configure on this device + mlag_peer_id: Device ID of MLAG peer device (optional) + mlag_peer_new_hostname: Hostname to configure on peer device (optional) + uplink_hostnames_arg: List of hostnames of uplink peer devices (optional) + Used when initializing MLAG peer device job_id: job_id provided by scheduler when adding job scheduled_by: Username from JWT. @@ -100,47 +152,43 @@ def init_access_device_step1(device_id: int, new_hostname: str, Raises: DeviceStateException + ValueError """ logger = get_logger() - # Check that we can find device and that it's in the correct state to start init with sqla_session() as session: - dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none() - if not dev: - raise ValueError(f"No device with id {device_id} found") - if dev.state != DeviceState.DISCOVERED: - raise DeviceStateException("Device must be in state DISCOVERED to begin init") - old_hostname = dev.hostname - # Perform connectivity check - nr = cnaas_nms.confpush.nornir_helper.cnaas_init() - nr_old_filtered = nr.filter(name=old_hostname) - try: - nrresult_old = nr_old_filtered.run(task=networking.napalm_get, getters=["facts"]) - except Exception as e: - raise ConnectionCheckError(f"Failed to connect to device_id {device_id}: {str(e)}") - if nrresult_old.failed: - print_result(nrresult_old) - raise ConnectionCheckError(f"Failed to connect to device_id {device_id}") + dev = pre_init_checks(session, device_id) + + cnaas_nms.confpush.get.update_linknets(session, dev.hostname) # update linknets using LLDP data + + # If this is the first device in an MLAG pair + if mlag_peer_id and mlag_peer_new_hostname: + mlag_peer_dev = pre_init_checks(session, mlag_peer_id) + cnaas_nms.confpush.get.update_linknets(session, mlag_peer_dev.hostname) + update_interfacedb_worker(session, dev, replace=True, delete=False, + mlag_peer_hostname=mlag_peer_dev.hostname) + update_interfacedb_worker(session, mlag_peer_dev, replace=True, delete=False, + mlag_peer_hostname=dev.hostname) + uplink_hostnames = dev.get_uplink_peer_hostnames(session) + uplink_hostnames += mlag_peer_dev.get_uplink_peer_hostnames(session) + # check that both devices see the correct MLAG peer + pre_init_check_mlag(session, dev, mlag_peer_dev) + pre_init_check_mlag(session, mlag_peer_dev, dev) + # If this is the second device in an MLAG pair + elif uplink_hostnames_arg: + uplink_hostnames = uplink_hostnames_arg + elif mlag_peer_id or mlag_peer_new_hostname: + raise ValueError("mlag_peer_id and mlag_peer_new_hostname must be specified together") + # If this device is not part of an MLAG pair + else: + update_interfacedb_worker(session, dev, replace=True, delete=False) + uplink_hostnames = dev.get_uplink_peer_hostnames(session) - cnaas_nms.confpush.get.update_linknets(old_hostname) - uplinks = [] - neighbor_hostnames = [] - with sqla_session() as session: - # Find management domain to use for this access switch - dev = session.query(Device).filter(Device.hostname == old_hostname).one() - for neighbor_d in dev.get_neighbors(session): - if neighbor_d.device_type == DeviceType.DIST: - local_if = dev.get_neighbor_local_ifname(session, neighbor_d) - if local_if: - uplinks.append({'ifname': local_if}) - neighbor_hostnames.append(neighbor_d.hostname) - logger.debug("Uplinks for device {} detected: {} neighbor_hostnames: {}".\ - format(device_id, uplinks, neighbor_hostnames)) # TODO: check compatability, same dist pair and same ports on dists - mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, neighbor_hostnames) + mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, uplink_hostnames) if not mgmtdomain: raise Exception( "Could not find appropriate management domain for uplink peer devices: {}".format( - neighbor_hostnames)) + uplink_hostnames)) # Select a new management IP for the device ReservedIP.clean_reservations(session, device=dev) session.commit() @@ -158,16 +206,25 @@ def init_access_device_step1(device_id: int, new_hostname: str, 'mgmt_prefixlen': int(mgmt_gw_ipif.network.prefixlen), 'interfaces': [], 'mgmt_vlan_id': mgmtdomain.vlan, - 'mgmt_gw': mgmt_gw_ipif.ip + 'mgmt_gw': mgmt_gw_ipif.ip, + 'device_model': dev.model, + 'device_os_version': dev.os_version } - for uplink in uplinks: + intfs = session.query(Interface).filter(Interface.device == dev).all() + intf: Interface + for intf in intfs: + intfdata = None + if intf.data: + intfdata = dict(intf.data) device_variables['interfaces'].append({ - 'name': uplink['ifname'], - 'ifclass': 'ACCESS_UPLINK', + 'name': intf.name, + 'ifclass': intf.configtype.name, + 'data': intfdata }) + mlag_vars = get_mlag_vars(session, dev) + device_variables = {**device_variables, **mlag_vars} # Update device state dev = session.query(Device).filter(Device.id == device_id).one() - dev.state = DeviceState.INIT dev.hostname = new_hostname session.commit() hostname = dev.hostname @@ -176,25 +233,14 @@ def init_access_device_step1(device_id: int, new_hostname: str, nr_filtered = nr.filter(name=hostname) # step2. push management config - try: - nrresult = nr_filtered.run(task=push_base_management_access, - device_variables=device_variables, - job_id=job_id) - except SessionLockedException as e: - # TODO: Handle this somehow? - pass - except Exception as e: - # Ignore exception, we expect to loose connectivity. - # Sometimes we get no exception here, but it's saved in result - # other times we get socket.timeout, pyeapi.eapilib.ConnectionError or - # napalm.base.exceptions.ConnectionException to handle here? - pass - if not nrresult.failed: - raise Exception # we don't expect success here + nrresult = nr_filtered.run(task=push_base_management_access, + device_variables=device_variables, + job_id=job_id) with sqla_session() as session: dev = session.query(Device).filter(Device.id == device_id).one() dev.management_ip = device_variables['mgmt_ip'] + dev.state = DeviceState.INIT # Remove the reserved IP since it's now saved in the device database instead reserved_ip = session.query(ReservedIP).filter(ReservedIP.device == dev).one_or_none() if reserved_ip: @@ -214,11 +260,28 @@ def init_access_device_step1(device_id: int, new_hostname: str, scheduler = Scheduler() next_job_id = scheduler.add_onetime_job( 'cnaas_nms.confpush.init_device:init_access_device_step2', - when=0, + when=30, scheduled_by=scheduled_by, kwargs={'device_id': device_id, 'iteration': 1}) - logger.debug(f"Step 2 scheduled as ID {next_job_id}") + logger.info("Init step 2 for {} scheduled as job # {}".format( + new_hostname, next_job_id + )) + + if mlag_peer_id and mlag_peer_new_hostname: + mlag_peer_job_id = scheduler.add_onetime_job( + 'cnaas_nms.confpush.init_device:init_access_device_step1', + when=60, + scheduled_by=scheduled_by, + kwargs={ + 'device_id': mlag_peer_id, + 'new_hostname': mlag_peer_new_hostname, + 'uplink_hostnames_arg': uplink_hostnames, + 'scheduled_by': scheduled_by + }) + logger.info("MLAG peer (id {}) init scheduled as job # {}".format( + mlag_peer_id, mlag_peer_job_id + )) return NornirJobResult( nrresult=nrresult, @@ -306,13 +369,6 @@ def init_access_device_step2(device_id: int, iteration: int = -1, except Exception as e: logger.exception("Error while running plugin hooks for new_managed_device: ".format(str(e))) - try: - update_interfacedb(hostname, replace=True) - except Exception as e: - logger.exception( - "Exception while updating interface database for device {}: {}".\ - format(hostname, str(e))) - return NornirJobResult( nrresult = nrresult ) @@ -334,6 +390,28 @@ def schedule_discover_device(ztp_mac: str, dhcp_ip: str, iteration: int, return None +def set_hostname_task(task, new_hostname: str): + with open('/etc/cnaas-nms/repository.yml', 'r') as db_file: + repo_config = yaml.safe_load(db_file) + local_repo_path = repo_config['templates_local'] + template_vars = {} # host is already set by nornir + r = task.run( + task=text.template_file, + name="Generate hostname config", + template="hostname.j2", + path=f"{local_repo_path}/{task.host.platform}", + **template_vars + ) + task.host["config"] = r.result + task.run( + task=networking.napalm_configure, + name="Configure hostname", + replace=False, + configuration=task.host["config"], + ) + task.host.close_connection("napalm") + + @job_wrapper def discover_device(ztp_mac: str, dhcp_ip: str, iteration=-1, job_id: Optional[str] = None, @@ -378,6 +456,7 @@ def discover_device(ztp_mac: str, dhcp_ip: str, iteration=-1, dev.model = facts['model'] dev.os_version = facts['os_version'] dev.state = DeviceState.DISCOVERED + new_hostname = dev.hostname logger.info(f"Device with ztp_mac {ztp_mac} successfully scanned, " + "moving to DISCOVERED state") except Exception as e: @@ -387,4 +466,10 @@ def discover_device(ztp_mac: str, dhcp_ip: str, iteration=-1, logger.debug("nrresult for ztp_mac {}: {}".format(ztp_mac, nrresult)) raise e + nrresult_hostname = nr_filtered.run(task=set_hostname_task, new_hostname=new_hostname) + if nrresult_hostname.failed: + logger.info("Could not set hostname for ztp_mac: {}".format( + ztp_mac + )) + return NornirJobResult(nrresult=nrresult) diff --git a/src/cnaas_nms/confpush/nornir_helper.py b/src/cnaas_nms/confpush/nornir_helper.py index baa2e2cf..1405f6f3 100644 --- a/src/cnaas_nms/confpush/nornir_helper.py +++ b/src/cnaas_nms/confpush/nornir_helper.py @@ -1,11 +1,12 @@ -from nornir import InitNornir +from dataclasses import dataclass +from typing import Optional, Tuple, List -from nornir.core.task import AggregatedResult, MultiResult, Result +from nornir import InitNornir +from nornir.core import Nornir +from nornir.core.task import AggregatedResult, MultiResult +from nornir.core.filter import F from cnaas_nms.scheduler.jobresult import JobResult -from dataclasses import dataclass -from typing import Optional - @dataclass class NornirJobResult(JobResult): @@ -13,7 +14,7 @@ class NornirJobResult(JobResult): change_score: Optional[float] = None -def cnaas_init(): +def cnaas_init() -> Nornir: nr = InitNornir( core={"num_workers": 50}, inventory={ @@ -41,3 +42,41 @@ def nr_result_serialize(result: AggregatedResult): if res.failed: hosts[host]['failed'] = True return hosts + + +def inventory_selector(nr: Nornir, resync: bool = True, + hostname: Optional[str] = None, + device_type: Optional[str] = None, + group: Optional[str] = None) -> Tuple[Nornir, int, List[str]]: + """Return a filtered Nornir inventory with only the selected devices + + Args: + nr: Nornir object + resync: Set to false if you want to filter out devices that are synchronized + hostname: Select device by hostname (string) + device_type: Select device by device_type (string) + group: Select device by group (string) + + Returns: + Tuple with: filtered Nornir inventory, total device count selected, + list of hostnames that was skipped because of resync=False + """ + skipped_devices = [] + if hostname: + nr_filtered = nr.filter(name=hostname).filter(managed=True) + elif device_type: + nr_filtered = nr.filter(F(groups__contains='T_'+device_type)).filter(managed=True) + elif group: + nr_filtered = nr.filter(F(groups__contains=group)).filter(managed=True) + else: + # all devices + nr_filtered = nr.filter(managed=True) + + if resync or hostname: + return nr_filtered, len(nr_filtered.inventory.hosts), skipped_devices + else: + pre_device_list = list(nr_filtered.inventory.hosts.keys()) + nr_filtered = nr_filtered.filter(synchronized=False) + post_device_list = list(nr_filtered.inventory.hosts.keys()) + skipped_devices = [x for x in pre_device_list if x not in post_device_list] + return nr_filtered, len(post_device_list), skipped_devices diff --git a/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py b/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py index 09be9292..33fd2d3b 100644 --- a/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py +++ b/src/cnaas_nms/confpush/nornir_plugins/cnaas_inventory.py @@ -82,6 +82,8 @@ def __init__(self, **kwargs): username, password = self._get_credentials('MANAGED') groups['S_MANAGED']['username'] = username groups['S_MANAGED']['password'] = password + groups['S_UNMANAGED']['username'] = username + groups['S_UNMANAGED']['password'] = password defaults = {'data': {'k': 'v'}} super().__init__(hosts=hosts, groups=groups, defaults=defaults, diff --git a/src/cnaas_nms/confpush/sync_devices.py b/src/cnaas_nms/confpush/sync_devices.py index 353ed42c..9333b58c 100644 --- a/src/cnaas_nms/confpush/sync_devices.py +++ b/src/cnaas_nms/confpush/sync_devices.py @@ -7,13 +7,12 @@ from nornir.plugins.tasks import networking, text from nornir.plugins.functions.text import print_result -from nornir.core.filter import F from nornir.core.task import MultiResult import cnaas_nms.db.helper -import cnaas_nms.confpush.nornir_helper +from cnaas_nms.confpush.nornir_helper import cnaas_init, inventory_selector from cnaas_nms.db.session import sqla_session, redis_session -from cnaas_nms.confpush.get import get_uplinks, calc_config_hash +from cnaas_nms.confpush.get import calc_config_hash from cnaas_nms.confpush.changescore import calculate_score from cnaas_nms.tools.log import get_logger from cnaas_nms.db.settings import get_settings @@ -81,6 +80,24 @@ def resolve_vlanid_list(vlan_name_list: List[str], vxlans: dict) -> List[int]: return ret +def get_mlag_vars(session, dev: Device) -> dict: + ret = { + 'mlag_peer': False, + 'mlag_peer_hostname': None, + 'mlag_peer_low': None + } + mlag_peer: Device = dev.get_mlag_peer(session) + if not mlag_peer: + return ret + ret['mlag_peer'] = True + ret['mlag_peer_hostname'] = mlag_peer.hostname + if dev.id < mlag_peer.id: + ret['mlag_peer_low'] = True + else: + ret['mlag_peer_low'] = False + return ret + + def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, job_id: Optional[str] = None, scheduled_by: Optional[str] = None): @@ -112,19 +129,17 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, raise ValueError("Unknown platform: {}".format(dev.platform)) settings, settings_origin = get_settings(hostname, devtype) device_variables = { - 'mgmt_ip': str(mgmt_ip) + 'mgmt_ip': str(mgmt_ip), + 'device_model': dev.model, + 'device_os_version': dev.os_version } if devtype == DeviceType.ACCESS: - neighbor_hostnames = dev.get_uplink_peers(session) - if not neighbor_hostnames: - raise Exception("Could not find any uplink neighbors for device {}".format( - hostname)) - mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, neighbor_hostnames) + mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain_by_ip(session, dev.management_ip) if not mgmtdomain: raise Exception( - "Could not find appropriate management domain for uplink peer devices: {}". - format(neighbor_hostnames)) + "Could not find appropriate management domain for management_ip: {}". + format(dev.management_ip)) mgmt_gw_ipif = IPv4Interface(mgmtdomain.ipv4_gw) access_device_variables = { @@ -156,8 +171,8 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, 'tagged_vlan_list': tagged_vlan_list, 'data': intfdata }) - - device_variables = {**access_device_variables, **device_variables} + mlag_vars = get_mlag_vars(session, dev) + device_variables = {**access_device_variables, **device_variables, **mlag_vars} elif devtype == DeviceType.DIST or devtype == DeviceType.CORE: asn = generate_asn(infra_ip) fabric_device_variables = { @@ -195,19 +210,19 @@ def push_sync_device(task, dry_run: bool = True, generate_only: bool = False, 'config': intf['config'], 'indexnum': ifindexnum }) - if devtype == DeviceType.DIST: - for mgmtdom in cnaas_nms.db.helper.get_all_mgmtdomains(session, hostname): - fabric_device_variables['mgmtdomains'].append({ - 'id': mgmtdom.id, - 'ipv4_gw': mgmtdom.ipv4_gw, - 'vlan': mgmtdom.vlan, - 'description': mgmtdom.description, - 'esi_mac': mgmtdom.esi_mac - }) + for mgmtdom in cnaas_nms.db.helper.get_all_mgmtdomains(session, hostname): + fabric_device_variables['mgmtdomains'].append({ + 'id': mgmtdom.id, + 'ipv4_gw': mgmtdom.ipv4_gw, + 'vlan': mgmtdom.vlan, + 'description': mgmtdom.description, + 'esi_mac': mgmtdom.esi_mac + }) # find fabric neighbors fabric_links = [] for neighbor_d in dev.get_neighbors(session): if neighbor_d.device_type == DeviceType.DIST or neighbor_d.device_type == DeviceType.CORE: + # TODO: support multiple links to the same neighbor? local_if = dev.get_neighbor_local_ifname(session, neighbor_d) local_ipif = dev.get_neighbor_local_ipif(session, neighbor_d) neighbor_ip = dev.get_neighbor_ip(session, neighbor_d) @@ -313,8 +328,8 @@ def generate_only(hostname: str) -> (str, dict): (string with config, dict with available template variables) """ logger = get_logger() - nr = cnaas_nms.confpush.nornir_helper.cnaas_init() - nr_filtered = nr.filter(name=hostname).filter(managed=True) + nr = cnaas_init() + nr_filtered, _, _ = inventory_selector(nr, hostname=hostname) template_vars = {} if len(nr_filtered.inventory.hosts) != 1: raise ValueError("Invalid hostname: {}".format(hostname)) @@ -414,42 +429,47 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No NornirJobResult """ logger = get_logger() - nr = cnaas_nms.confpush.nornir_helper.cnaas_init() + nr = cnaas_init() + dev_count = 0 + skipped_hostnames = [] if hostname: - nr_filtered = nr.filter(name=hostname).filter(managed=True) + nr_filtered, dev_count, skipped_hostnames = \ + inventory_selector(nr, hostname=hostname) else: if device_type: - nr_filtered = nr.filter(F(groups__contains='T_'+device_type)).filter(managed=True) + nr_filtered, dev_count, skipped_hostnames = \ + inventory_selector(nr, resync=resync, device_type=device_type) elif group: - nr_filtered = nr.filter(F(groups__contains=group)).filter(managed=True) + nr_filtered, dev_count, skipped_hostnames = \ + inventory_selector(nr, resync=resync, group=group) else: # all devices - nr_filtered = nr.filter(managed=True) - if not resync: - pre_device_list = list(nr_filtered.inventory.hosts.keys()) - nr_filtered = nr_filtered.filter(synchronized=False) - post_device_list = list(nr_filtered.inventory.hosts.keys()) - already_synced_device_list = [x for x in pre_device_list if x not in post_device_list] - if already_synced_device_list: - logger.info("Device(s) already synchronized, skipping: {}".format( - already_synced_device_list - )) + nr_filtered, dev_count, skipped_hostnames = \ + inventory_selector(nr, resync=resync) + + if skipped_hostnames: + logger.info("Device(s) already synchronized, skipping ({}): {}".format( + len(skipped_hostnames), ", ".join(skipped_hostnames) + )) device_list = list(nr_filtered.inventory.hosts.keys()) - logger.info("Device(s) selected for synchronization: {}".format( - device_list + logger.info("Device(s) selected for synchronization ({}): {}".format( + dev_count, ", ".join(device_list) )) try: nrresult = nr_filtered.run(task=sync_check_hash, force=force, job_id=job_id) - print_result(nrresult) except Exception as e: logger.exception("Exception while checking config hash: {}".format(str(e))) raise e else: if nrresult.failed: + # Mark devices as unsynchronized if config hash check failed + with sqla_session() as session: + session.query(Device).filter(Device.hostname.in_(nrresult.failed_hosts.keys())).\ + update({Device.synchronized: False}, synchronize_session=False) raise Exception('Configuration hash check failed for {}'.format( ' '.join(nrresult.failed_hosts.keys()))) @@ -462,7 +482,6 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No try: nrresult = nr_filtered.run(task=push_sync_device, dry_run=dry_run, job_id=job_id) - print_result(nrresult) except Exception as e: logger.exception("Exception while synchronizing devices: {}".format(str(e))) try: @@ -493,7 +512,7 @@ def sync_devices(hostname: Optional[str] = None, device_type: Optional[str] = No changed_hosts.append(host) if "change_score" in results[0].host: change_scores.append(results[0].host["change_score"]) - logger.debug("Change score for host {}: {}".format( + logger.debug("Change score for host {}: {:.1f}".format( host, results[0].host["change_score"])) else: unchanged_hosts.append(host) @@ -539,15 +558,11 @@ def exclude_filter(host, exclude_list=failed_hosts+unchanged_hosts): total_change_score = 0 elif not change_scores or total_change_score >= 100 or failed_hosts: total_change_score = 100 - elif max(change_scores) > 1000: - # If some device has a score higher than this, disregard any averages - # and report max impact score - total_change_score = 100 else: - # calculate median value and round up, use min value of 1 and max of 100 - total_change_score = max(min(int(median(change_scores) + 0.5), 100), 1) + # use individual max as total_change_score, range 1-100 + total_change_score = max(min(int(max(change_scores) + 0.5), 100), 1) logger.info( - "Change impact score: {} (dry_run: {}, selected devices: {}, changed devices: {})". + "Change impact score: {:.1f} (dry_run: {}, selected devices: {}, changed devices: {})". format(total_change_score, dry_run, len(device_list), len(changed_hosts))) next_job_id = None @@ -569,3 +584,73 @@ def exclude_filter(host, exclude_list=failed_hosts+unchanged_hosts): ) return NornirJobResult(nrresult=nrresult, next_job_id=next_job_id, change_score=total_change_score) + + +def push_static_config(task, config: str, dry_run: bool = True, + job_id: Optional[str] = None, + scheduled_by: Optional[str] = None): + """ + Nornir task to push static config to device + + Args: + task: nornir task, sent by nornir when doing .run() + config: static config to apply + dry_run: Don't commit config to device, just do compare/diff + scheduled_by: username that triggered job + + Returns: + """ + set_thread_data(job_id) + logger = get_logger() + + logger.debug("Push static config to device: {}".format(task.host.name)) + + task.run(task=networking.napalm_configure, + name="Push static config", + replace=True, + configuration=config, + dry_run=dry_run + ) + + +@job_wrapper +def apply_config(hostname: str, config: str, dry_run: bool, + job_id: Optional[int] = None, + scheduled_by: Optional[str] = None) -> NornirJobResult: + """Apply a static configuration (from backup etc) to a device. + + Args: + hostname: Specify a single host by hostname to synchronize + config: Static configuration to apply + dry_run: Set to false to actually apply config to device + job_id: Job ID number + scheduled_by: Username from JWT + + Returns: + NornirJobResult + """ + logger = get_logger() + + with sqla_session() as session: + dev: Device = session.query(Device).filter(Device.hostname == hostname).one_or_none() + if not dev: + raise Exception("Device {} not found".format(hostname)) + elif not (dev.state == DeviceState.MANAGED or dev.state == DeviceState.UNMANAGED): + raise Exception("Device {} is in invalid state: {}".format(hostname, dev.state)) + if not dry_run: + dev.state = DeviceState.UNMANAGED + dev.synchronized = False + + nr = cnaas_init() + nr_filtered, _, _ = inventory_selector(nr, hostname=hostname) + + try: + nrresult = nr_filtered.run(task=push_static_config, + config=config, + dry_run=dry_run, + job_id=job_id) + except Exception as e: + logger.exception("Exception in apply_config: {}".format(e)) + + return NornirJobResult(nrresult=nrresult) + diff --git a/src/cnaas_nms/confpush/tests/test_get.py b/src/cnaas_nms/confpush/tests/test_get.py index aec4d645..79851bd6 100644 --- a/src/cnaas_nms/confpush/tests/test_get.py +++ b/src/cnaas_nms/confpush/tests/test_get.py @@ -6,6 +6,8 @@ import yaml import os +from cnaas_nms.db.session import sqla_session + class GetTests(unittest.TestCase): def setUp(self): data_dir = pkg_resources.resource_filename(__name__, 'data') @@ -33,8 +35,10 @@ def test_update_inventory(self): pprint.pprint(diff) def test_update_links(self): - new_links = cnaas_nms.confpush.get.update_linknets(self.testdata['update_hostname']) + with sqla_session() as session: + new_links = cnaas_nms.confpush.get.update_linknets(session, self.testdata['update_hostname']) pprint.pprint(new_links) + if __name__ == '__main__': unittest.main() diff --git a/src/cnaas_nms/confpush/update.py b/src/cnaas_nms/confpush/update.py index 08e7da2f..965c101b 100644 --- a/src/cnaas_nms/confpush/update.py +++ b/src/cnaas_nms/confpush/update.py @@ -1,15 +1,70 @@ from typing import Optional, List +from nornir.plugins.tasks import networking + from cnaas_nms.db.session import sqla_session from cnaas_nms.db.device import Device, DeviceType, DeviceState from cnaas_nms.db.interface import Interface, InterfaceConfigType from cnaas_nms.confpush.get import get_interfaces_names, get_uplinks, \ - filter_interfaces, get_interfacedb_ifs + filter_interfaces, get_mlag_ifs from cnaas_nms.tools.log import get_logger +from cnaas_nms.scheduler.wrapper import job_wrapper +from cnaas_nms.confpush.nornir_helper import NornirJobResult +import cnaas_nms.confpush.nornir_helper + + +def update_interfacedb_worker(session, dev: Device, replace: bool, delete: bool, + mlag_peer_hostname: Optional[str] = None) -> List[dict]: + """Perform actual work of updating database for update_interfacedb""" + logger = get_logger() + ret = [] + + iflist = get_interfaces_names(dev.hostname) + uplinks = get_uplinks(session, dev.hostname) + if mlag_peer_hostname: + mlag_ifs = get_mlag_ifs(session, dev.hostname, mlag_peer_hostname) + else: + mlag_ifs = {} + phy_interfaces = filter_interfaces(iflist, platform=dev.platform, include='physical') + + for intf_name in phy_interfaces: + intf: Interface = session.query(Interface).filter(Interface.device == dev). \ + filter(Interface.name == intf_name).one_or_none() + if intf: + new_intf = False + else: + new_intf = True + intf: Interface = Interface() + if not new_intf and delete: # 'not new_intf' means interface exists in database + logger.debug("Deleting interface {} on device {} from interface DB".format( + intf_name, dev.hostname + )) + session.delete(intf) + continue + elif not new_intf and not replace: + continue + logger.debug("New/updated physical interface found on device {}: {}".format( + dev.hostname, intf_name + )) + if intf_name in uplinks.keys(): + intf.configtype = InterfaceConfigType.ACCESS_UPLINK + intf.data = {'neighbor': uplinks[intf_name]} + elif intf_name in mlag_ifs.keys(): + intf.configtype = InterfaceConfigType.MLAG_PEER + intf.data = {'neighbor_id': mlag_ifs[intf_name]} + else: + intf.configtype = InterfaceConfigType.ACCESS_AUTO + intf.name = intf_name + intf.device = dev + if new_intf: + session.add(intf) + ret.append(intf.as_dict()) + session.commit() + return ret def update_interfacedb(hostname: str, replace: bool = False, delete: bool = False) \ - -> Optional[List[dict]]: + -> List[dict]: """Update interface DB with any new physical interfaces for specified device. If replace is set, any existing records in the database will get overwritten. If delete is set, all entries in database for this device will be removed. @@ -17,8 +72,6 @@ def update_interfacedb(hostname: str, replace: bool = False, delete: bool = Fals Returns: List of interfaces that was added to DB """ - logger = get_logger() - ret = [] with sqla_session() as session: dev: Device = session.query(Device).filter(Device.hostname == hostname).one_or_none() if not dev: @@ -27,48 +80,12 @@ def update_interfacedb(hostname: str, replace: bool = False, delete: bool = Fals raise ValueError(f"Hostname {hostname} is not a managed device") if dev.device_type != DeviceType.ACCESS: raise ValueError("This function currently only supports access devices") - # TODO: add support for dist/core devices? - - iflist = get_interfaces_names(hostname) - uplinks, neighbor_hostnames = get_uplinks(session, hostname) - uplinks_ifnames = [x['ifname'] for x in uplinks] - phy_interfaces = filter_interfaces(iflist, platform=dev.platform, include='physical') -# existing_ifs = get_interfacedb_ifs(session, hostname) - - updated = False - for intf_name in phy_interfaces: - intf: Interface = session.query(Interface).filter(Interface.device == dev).\ - filter(Interface.name == intf_name).one_or_none() - if intf: - new_intf = False - else: - new_intf = True - intf: Interface = Interface() - if not new_intf and delete: - logger.debug("Deleting interface {} on device {} from interface DB".format( - intf_name, dev.hostname - )) - session.delete(intf) - continue - elif not new_intf and not replace: - continue - updated = True - logger.debug("New/updated physical interface found on device {}: {}".format( - dev.hostname, intf_name - )) - if intf_name in uplinks_ifnames: - intf.configtype = InterfaceConfigType.ACCESS_UPLINK - intf.data = {'neighbor': neighbor_hostnames[uplinks_ifnames.index(intf_name)]} - else: - intf.configtype = InterfaceConfigType.ACCESS_AUTO - intf.name = intf_name - intf.device = dev - if new_intf: - session.add(intf) - ret.append(intf.as_dict()) - if updated: + + result = update_interfacedb_worker(session, dev, replace, delete) + + if result: dev.synchronized = False - return ret + return result def reset_interfacedb(hostname: str): @@ -81,3 +98,45 @@ def reset_interfacedb(hostname: str): return ret +@job_wrapper +def update_facts(hostname: str, + job_id: Optional[str] = None, + scheduled_by: Optional[str] = None): + logger = get_logger() + with sqla_session() as session: + dev: Device = session.query(Device).filter(Device.hostname == hostname).one_or_none() + if not dev: + raise ValueError("Device with hostname {} not found".format(hostname)) + if not (dev.state == DeviceState.MANAGED or dev.state == DeviceState.UNMANAGED): + raise ValueError("Device with hostname {} is in incorrect state: {}".format( + hostname, str(dev.state) + )) + hostname = dev.hostname + + nr = cnaas_nms.confpush.nornir_helper.cnaas_init() + nr_filtered = nr.filter(name=hostname) + + nrresult = nr_filtered.run(task=networking.napalm_get, getters=["facts"]) + + if nrresult.failed: + logger.error("Could not contact device with hostname {}".format(hostname)) + return NornirJobResult(nrresult=nrresult) + try: + facts = nrresult[hostname][0].result['facts'] + with sqla_session() as session: + dev: Device = session.query(Device).filter(Device.hostname == hostname).one() + dev.serial = facts['serial_number'] + dev.vendor = facts['vendor'] + dev.model = facts['model'] + dev.os_version = facts['os_version'] + logger.debug("Updating facts for device {}: {}, {}, {}, {}".format( + hostname, facts['serial_number'], facts['vendor'], facts['model'], facts['os_version'] + )) + except Exception as e: + logger.exception("Could not update device with hostname {} with new facts: {}".format( + hostname, str(e) + )) + logger.debug("Get facts nrresult for hostname {}: {}".format(hostname, nrresult)) + raise e + + return NornirJobResult(nrresult=nrresult) diff --git a/src/cnaas_nms/db/device.py b/src/cnaas_nms/db/device.py index 81cfd4ab..b3010023 100644 --- a/src/cnaas_nms/db/device.py +++ b/src/cnaas_nms/db/device.py @@ -4,11 +4,13 @@ import datetime import enum import re -from typing import Optional, List +import json +from typing import Optional, List, Set from sqlalchemy import Column, Integer, Unicode, String, UniqueConstraint from sqlalchemy import Enum, DateTime, Boolean from sqlalchemy import ForeignKey +from sqlalchemy import event from sqlalchemy.orm import relationship from sqlalchemy_utils import IPAddressType @@ -17,6 +19,7 @@ import cnaas_nms.db.linknet from cnaas_nms.db.interface import Interface, InterfaceConfigType +from cnaas_nms.tools.event import add_event class DeviceException(Exception): @@ -145,9 +148,8 @@ def get_linknet_localif_mapping(self, session) -> dict[str, str]: )) return ret - def get_link_to(self, session, peer_device: Device) -> Optional[cnaas_nms.db.linknet.Linknet]: + def get_links_to(self, session, peer_device: Device) -> List[cnaas_nms.db.linknet.Linknet]: """Return linknet connecting to device peer_device.""" - # TODO: support multiple links to the same neighbor? return session.query(cnaas_nms.db.linknet.Linknet).\ filter( ((cnaas_nms.db.linknet.Linknet.device_a_id == self.id) & @@ -155,23 +157,44 @@ def get_link_to(self, session, peer_device: Device) -> Optional[cnaas_nms.db.lin | ((cnaas_nms.db.linknet.Linknet.device_b_id == self.id) & (cnaas_nms.db.linknet.Linknet.device_a_id == peer_device.id)) - ).one_or_none() + ).all() def get_neighbor_local_ifname(self, session, peer_device: Device) -> Optional[str]: """Get the local interface name on this device that links to peer_device.""" - linknet = self.get_link_to(session, peer_device) - if not linknet: + linknets = self.get_links_to(session, peer_device) + if not linknets: return None + elif len(linknets) > 1: + raise ValueError("Multiple linknets between devices not supported") + else: + linknet = linknets[0] if linknet.device_a_id == self.id: return linknet.device_a_port elif linknet.device_b_id == self.id: return linknet.device_b_port + def get_neighbor_local_ifnames(self, session, peer_device: Device) -> List[str]: + """Get the local interface name on this device that links to peer_device.""" + linknets = self.get_links_to(session, peer_device) + ifnames = [] + if not linknets: + return ifnames + for linknet in linknets: + if linknet.device_a_id == self.id: + ifnames.append(linknet.device_a_port) + elif linknet.device_b_id == self.id: + ifnames.append(linknet.device_b_port) + return ifnames + def get_neighbor_local_ipif(self, session, peer_device: Device) -> Optional[str]: """Get the local interface IP on this device that links to peer_device.""" - linknet = self.get_link_to(session, peer_device) - if not linknet: + linknets = self.get_links_to(session, peer_device) + if not linknets: return None + elif len(linknets) > 1: + raise ValueError("Multiple linknets between devices not supported") + else: + linknet = linknets[0] if linknet.device_a_id == self.id: return "{}/{}".format(linknet.device_a_ip, ipaddress.IPv4Network(linknet.ipv4_network).prefixlen) elif linknet.device_b_id == self.id: @@ -179,15 +202,19 @@ def get_neighbor_local_ipif(self, session, peer_device: Device) -> Optional[str] def get_neighbor_ip(self, session, peer_device: Device): """Get the remote peer IP address for the linknet going towards device.""" - linknet = self.get_link_to(session, peer_device) - if not linknet: + linknets = self.get_links_to(session, peer_device) + if not linknets: return None + elif len(linknets) > 1: + raise ValueError("Multiple linknets between devices not supported") + else: + linknet = linknets[0] if linknet.device_a_id == self.id: return linknet.device_b_ip elif linknet.device_b_id == self.id: return linknet.device_a_ip - def get_uplink_peers(self, session): + def get_uplink_peer_hostnames(self, session) -> List[str]: intfs = session.query(Interface).filter(Interface.device == self).\ filter(Interface.configtype == InterfaceConfigType.ACCESS_UPLINK).all() peer_hostnames = [] @@ -197,6 +224,29 @@ def get_uplink_peers(self, session): peer_hostnames.append(intf.data['neighbor']) return peer_hostnames + def get_mlag_peer(self, session) -> Optional[Device]: + intfs = session.query(Interface).filter(Interface.device == self). \ + filter(Interface.configtype == InterfaceConfigType.MLAG_PEER).all() + peers: Set[Device] = set() + linknets = self.get_linknets(session) + intf: Interface = Interface() + for intf in intfs: + for linknet in linknets: + if linknet.device_a == self and linknet.device_a_port == intf.name: + peers.add(linknet.device_b) + elif linknet.device_b == self and linknet.device_b_port == intf.name: + peers.add(linknet.device_a) + if len(peers) > 1: + raise DeviceException("More than one MLAG peer found: {}".format( + [x.hostname for x in peers] + )) + elif len(peers) == 1: + if self.device_type != next(iter(peers)).device_type: + raise DeviceException("MLAG peers are not the same device type") + return next(iter(peers)) + else: + return None + @classmethod def valid_hostname(cls, hostname: str) -> bool: if not isinstance(hostname, str): @@ -394,3 +444,13 @@ def validate(cls, new_entry=True, **kwargs): return data, errors +@event.listens_for(Device, 'after_update') +def after_update_device(mapper, connection, target: Device): + update_data = { + "action": "UPDATED", + "device_id": target.id, + "hostname": target.hostname, + "object": target.as_dict() + } + json_data = json.dumps(update_data) + add_event(json_data=json_data, event_type="update", update_type="device") diff --git a/src/cnaas_nms/db/helper.py b/src/cnaas_nms/db/helper.py index e70cbe83..e603832a 100644 --- a/src/cnaas_nms/db/helper.py +++ b/src/cnaas_nms/db/helper.py @@ -1,10 +1,11 @@ import datetime from typing import List, Optional +from ipaddress import IPv4Interface, IPv4Address import netaddr -from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound -from cnaas_nms.db.device import Device +from cnaas_nms.db.device import Device, DeviceType from cnaas_nms.db.mgmtdomain import Mgmtdomain @@ -27,27 +28,68 @@ def find_mgmtdomain(session, hostnames: List[str]) -> Optional[Mgmtdomain]: ValueError: On invalid hostnames etc """ if not isinstance(hostnames, list) or not len(hostnames) == 2: - raise ValueError("hostnames argument must be a list with two device hostnames") + raise ValueError( + "hostnames argument must be a list with two device hostnames, got: {}".format( + hostnames + )) for hostname in hostnames: if not Device.valid_hostname(hostname): raise ValueError(f"Argument {hostname} is not a valid hostname") try: - device0 = session.query(Device).filter(Device.hostname == hostnames[0]).one() + device0: Device = session.query(Device).filter(Device.hostname == hostnames[0]).one() except NoResultFound: raise ValueError(f"hostname {hostnames[0]} not found in device database") try: - device1 = session.query(Device).filter(Device.hostname == hostnames[1]).one() + device1: Device = session.query(Device).filter(Device.hostname == hostnames[1]).one() except NoResultFound: raise ValueError(f"hostname {hostnames[1]} not found in device database") - mgmtdomain = session.query(Mgmtdomain).\ - filter( - ((Mgmtdomain.device_a == device0) & (Mgmtdomain.device_b == device1)) - | - ((Mgmtdomain.device_a == device1) & (Mgmtdomain.device_b == device0)) - ).one_or_none() + + if device0.device_type == DeviceType.DIST or device1.device_type == DeviceType.DIST: + if device0.device_type != DeviceType.DIST or device1.device_type != DeviceType.DIST: + raise ValueError("Both uplink devices must be of same device type: {}, {}".format( + device0.hostname, device1.hostname + )) + try: + mgmtdomain: Mgmtdomain = session.query(Mgmtdomain).\ + filter( + ((Mgmtdomain.device_a == device0) & (Mgmtdomain.device_b == device1)) + | + ((Mgmtdomain.device_a == device1) & (Mgmtdomain.device_b == device0)) + ).one_or_none() + if not mgmtdomain: + mgmtdomain: Mgmtdomain = session.query(Mgmtdomain).filter( + (Mgmtdomain.device_a.has(Device.device_type == DeviceType.CORE)) + | + (Mgmtdomain.device_b.has(Device.device_type == DeviceType.CORE)) + ).one_or_none() + except MultipleResultsFound: + raise Exception( + "Found multiple possible mgmtdomains, please remove any redundant mgmtdomains") + elif device0.device_type == DeviceType.ACCESS or device1.device_type == DeviceType.ACCESS: + if device0.device_type != DeviceType.ACCESS or device1.device_type != DeviceType.ACCESS: + raise ValueError("Both uplink devices must be of same device type: {}, {}".format( + device0.hostname, device1.hostname + )) + mgmtdomain: Mgmtdomain = find_mgmtdomain_by_ip(session, device0.management_ip) + if mgmtdomain.id != find_mgmtdomain_by_ip(session, device1.management_ip).id: + raise Exception("Uplink access devices have different mgmtdomains: {}, {}".format( + device0.hostname, device1.hostname + )) + else: + raise Exception("Unknown uplink device type") return mgmtdomain +def find_mgmtdomain_by_ip(session, ipv4_address: IPv4Address) -> Optional[Mgmtdomain]: + mgmtdomains = session.query(Mgmtdomain).all() + mgmtdom: Mgmtdomain + for mgmtdom in mgmtdomains: + mgmtdom_ipv4_network = IPv4Interface(mgmtdom.ipv4_gw).network + if ipv4_address in mgmtdom_ipv4_network: + return mgmtdom + return None + + def get_all_mgmtdomains(session, hostname: str) -> List[Mgmtdomain]: """ Get all mgmtdomains for a specific distribution switch. diff --git a/src/cnaas_nms/db/interface.py b/src/cnaas_nms/db/interface.py index eae0bccf..533c4d00 100644 --- a/src/cnaas_nms/db/interface.py +++ b/src/cnaas_nms/db/interface.py @@ -17,10 +17,12 @@ class InterfaceConfigType(enum.Enum): CONFIGFILE = 2 CUSTOM = 3 TEMPLATE = 4 + MLAG_PEER = 5 ACCESS_AUTO = 10 ACCESS_UNTAGGED = 11 ACCESS_TAGGED = 12 ACCESS_UPLINK = 13 + ACCESS_DOWNLINK = 14 @classmethod def has_value(cls, value): diff --git a/src/cnaas_nms/db/job.py b/src/cnaas_nms/db/job.py index efb1a7f9..64eb7d0c 100644 --- a/src/cnaas_nms/db/job.py +++ b/src/cnaas_nms/db/job.py @@ -1,7 +1,7 @@ import enum import datetime import json -from typing import Optional +from typing import Optional, Dict from sqlalchemy import Column, Integer, Unicode, SmallInteger from sqlalchemy import Enum, DateTime @@ -16,11 +16,20 @@ from cnaas_nms.scheduler.jobresult import StrJobResult, DictJobResult from cnaas_nms.db.helper import json_dumper from cnaas_nms.tools.log import get_logger +from cnaas_nms.tools.event import add_event logger = get_logger() +class JobNotFoundError(Exception): + pass + + +class InvalidJobError(Exception): + pass + + class JobStatus(enum.Enum): UNKNOWN = 0 SCHEDULED = 1 @@ -81,6 +90,16 @@ def start_job(self, function_name: str, scheduled_by: str): self.status = JobStatus.RUNNING self.finished_devices = [] self.scheduled_by = scheduled_by + try: + json_data = json.dumps({ + "job_id": self.id, + "status": "RUNNING", + "function_name": function_name, + "scheduled_by": scheduled_by + }) + add_event(json_data=json_data, event_type="update", update_type="job") + except Exception as e: + pass def finish_success(self, res: dict, next_job_id: Optional[int]): try: @@ -102,6 +121,17 @@ def finish_success(self, res: dict, next_job_id: Optional[int]): if next_job_id: # TODO: check if this exists in the db? self.next_job_id = next_job_id + try: + event_data = { + "job_id": self.id, + "status": "FINISHED" + } + if next_job_id: + event_data['next_job_id'] = next_job_id + json_data = json.dumps(event_data) + add_event(json_data=json_data, event_type="update", update_type="job") + except Exception as e: + pass def finish_exception(self, e: Exception, traceback: str): logger.warning("Job {} finished with exception: {}".format(self.id, str(e))) @@ -119,6 +149,30 @@ def finish_exception(self, e: Exception, traceback: str): errmsg = "Unable to serialize exception or traceback: {}".format(str(e)) logger.exception(errmsg) self.exception = {"error": errmsg} + try: + json_data = json.dumps({ + "job_id": self.id, + "status": "EXCEPTION", + "exception": str(e), + }) + add_event(json_data=json_data, event_type="update", update_type="job") + except Exception as e: + pass + + def finish_abort(self, message: str): + logger.debug("Job {} aborted: {}".format(self.id, message)) + self.finish_time = datetime.datetime.utcnow() + self.status = JobStatus.ABORTED + self.result = {"message": message} + try: + json_data = json.dumps({ + "job_id": self.id, + "status": "ABORTED", + "message": message, + }) + add_event(json_data=json_data, event_type="update", update_type="job") + except Exception as e: + pass @classmethod def clear_jobs(cls, session): @@ -142,3 +196,51 @@ def clear_jobs(cls, session): "Job found in past SCHEDULED state at startup moved to ABORTED, id: {}". format(job.id)) job.status = JobStatus.ABORTED + + @classmethod + def get_previous_config(cls, session, hostname: str, previous: Optional[int] = None, + job_id: Optional[int] = None, + before: Optional[datetime.datetime] = None) -> Dict[str, str]: + """Get full configuration for a device from a previous job. + + Args: + session: sqla_session + hostname: hostname of device to get config for + previous: number of revisions back to get config from + job_id: specific job to get config from + before: date to get config before + + Returns: + Returns a result dict with keys: config, job_id and finish_time + """ + result = {} + query_part = session.query(Job).filter(Job.function_name == 'sync_devices'). \ + filter(Job.result.has_key('devices')).filter(Job.result['devices'].has_key(hostname)) + + if job_id and type(job_id) == int: + query_part = query_part.filter(Job.id == job_id) + elif previous and type(previous) == int: + query_part = query_part.order_by(Job.id.desc()).offset(previous) + elif before and type(before) == datetime.datetime: + query_part = query_part.filter(Job.finish_time < before).order_by(Job.id.desc()) + else: + query_part = query_part.order_by(Job.id.desc()) + + job: Job = query_part.first() + if not job: + raise JobNotFoundError("No matching job found") + + result['job_id'] = job.id + result['finish_time'] = job.finish_time.isoformat(timespec='seconds') + + if 'job_tasks' not in job.result['devices'][hostname] or \ + 'failed' not in job.result['devices'][hostname]: + raise InvalidJobError("Invalid job data found in database: missing job_tasks") + + for task in job.result['devices'][hostname]['job_tasks']: + if task['task_name'] == 'Generate device config': + result['config'] = task['result'] + + result['failed'] = job.result['devices'][hostname]['failed'] + + return result diff --git a/src/cnaas_nms/db/session.py b/src/cnaas_nms/db/session.py index 20e024c5..cd202e76 100644 --- a/src/cnaas_nms/db/session.py +++ b/src/cnaas_nms/db/session.py @@ -61,5 +61,5 @@ def sqla_execute(**kwargs): @contextmanager def redis_session(**kwargs): db_data = get_dbdata(**kwargs) - with StrictRedis(host=db_data['redis_hostname'], port=6379) as conn: + with StrictRedis(host=db_data['redis_hostname'], port=6379, charset="utf-8", decode_responses=True) as conn: yield conn diff --git a/src/cnaas_nms/db/settings_fields.py b/src/cnaas_nms/db/settings_fields.py index 505b201b..a15ba264 100644 --- a/src/cnaas_nms/db/settings_fields.py +++ b/src/cnaas_nms/db/settings_fields.py @@ -1,6 +1,6 @@ from typing import List, Optional, Dict -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, validator # HOSTNAME_REGEX = r'([a-z0-9-]{1,63}\.?)+' @@ -29,7 +29,7 @@ ipv4_schema = Field(..., regex=f"^{IPV4_REGEX}$", description="IPv4 address") IPV4_IF_REGEX = f"{IPV4_REGEX}" + r"\/[0-9]{1,2}" -ipv4_if_schema = Field(..., regex=f"^{IPV4_IF_REGEX}$", +ipv4_if_schema = Field(None, regex=f"^{IPV4_IF_REGEX}$", description="IPv4 address in CIDR/prefix notation (0.0.0.0/0)") ipv6_schema = Field(..., regex=f"^{IPV6_REGEX}$", description="IPv6 address") @@ -40,7 +40,7 @@ # VLAN name is alphanumeric max 32 chars on Cisco # should not start with number according to some Juniper doc VLAN_NAME_REGEX = r'^[a-zA-Z][a-zA-Z0-9-_]{0,31}$' -vlan_name_schema = Field(..., regex=VLAN_NAME_REGEX, +vlan_name_schema = Field(None, regex=VLAN_NAME_REGEX, description="Max 32 alphanumeric chars, " + "beginning with a non-numeric character") vlan_id_schema = Field(..., gt=0, lt=4096, description="Numeric 802.1Q VLAN ID, 1-4095") @@ -176,15 +176,22 @@ class f_extroute_bgp(BaseModel): class f_vxlan(BaseModel): description: str = None vni: int = vxlan_vni_schema - vrf: str = vlan_name_schema + vrf: Optional[str] = vlan_name_schema vlan_id: int = vlan_id_schema vlan_name: str = vlan_name_schema - ipv4_gw: str = ipv4_if_schema + ipv4_gw: Optional[str] = ipv4_if_schema dhcp_relays: Optional[List[f_dhcp_relay]] mtu: Optional[int] = mtu_schema groups: List[str] = [] devices: List[str] = [] + @validator('ipv4_gw') + def vrf_required_if_ipv4_gw_set(cls, v, values, **kwargs): + if v: + if 'vrf' not in values or not values['vrf']: + raise ValueError('VRF is required when specifying ipv4_gw') + return v + class f_underlay(BaseModel): infra_lo_net: str = ipv4_if_schema diff --git a/src/cnaas_nms/db/tests/test_mgmtdomain.py b/src/cnaas_nms/db/tests/test_mgmtdomain.py index 0c0fe112..339da5ed 100644 --- a/src/cnaas_nms/db/tests/test_mgmtdomain.py +++ b/src/cnaas_nms/db/tests/test_mgmtdomain.py @@ -5,26 +5,21 @@ import yaml import os import pprint +from ipaddress import IPv4Address, IPv4Network, IPv4Interface import cnaas_nms.db.helper from cnaas_nms.db.device import Device, DeviceState, DeviceType from cnaas_nms.db.session import sqla_session from cnaas_nms.db.mgmtdomain import Mgmtdomain -from cnaas_nms.tools.testsetup import PostgresTemporaryInstance - class MgmtdomainTests(unittest.TestCase): def setUp(self): data_dir = pkg_resources.resource_filename(__name__, 'data') with open(os.path.join(data_dir, 'testdata.yml'), 'r') as f_testdata: self.testdata = yaml.safe_load(f_testdata) - self.tmp_postgres = PostgresTemporaryInstance() - - def tearDown(self): - self.tmp_postgres.shutdown() - def add_mgmt_domain(self): + def add_mgmtdomain(self): with sqla_session() as session: d_a = session.query(Device).filter(Device.hostname == 'eosdist1').one() d_b = session.query(Device).filter(Device.hostname == 'eosdist2').one() @@ -46,7 +41,7 @@ def add_mgmt_domain(self): # 1, # len(result['hosts'].items())) - def delete_mgmt_domain(self): + def delete_mgmtdomain(self): with sqla_session() as session: d_a = session.query(Device).filter(Device.hostname == 'eosdist1').one() instance = session.query(Mgmtdomain).filter(Mgmtdomain.device_a == d_a).first() @@ -56,19 +51,24 @@ def delete_mgmt_domain(self): else: print(f"Mgmtdomain for device {d_a.hostname} not found") - def test_find_mgmt_domain(self): + def test_find_mgmt_omain(self): with sqla_session() as session: mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, ['eosdist1', 'eosdist2']) if mgmtdomain: pprint.pprint(mgmtdomain.as_dict()) def test_find_free_mgmt_ip(self): - mgmtdomain_id = 2 + mgmtdomain_id = 1 with sqla_session() as session: mgmtdomain = session.query(Mgmtdomain).filter(Mgmtdomain.id == mgmtdomain_id).one() if mgmtdomain: print(mgmtdomain.find_free_mgmt_ip(session)) + def test_find_mgmtdomain_by_ip(self): + with sqla_session() as session: + mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain_by_ip(session, IPv4Address('10.0.6.6')) + self.assertEqual(IPv4Interface(mgmtdomain.ipv4_gw).network, IPv4Network('10.0.6.0/24')) + if __name__ == '__main__': unittest.main() diff --git a/src/cnaas_nms/run.py b/src/cnaas_nms/run.py index bd849c22..a73f0b0b 100644 --- a/src/cnaas_nms/run.py +++ b/src/cnaas_nms/run.py @@ -2,7 +2,11 @@ import coverage import atexit import signal +import threading +import time +from typing import List from gevent import monkey, signal as gevent_signal +from redis import StrictRedis from cnaas_nms.tools.get_apidata import get_apidata # Do late imports for anything cnaas/flask related so we can do gevent monkey patch, see below @@ -24,8 +28,8 @@ def save_coverage(): cov.save() atexit.register(save_coverage) - gevent_signal(signal.SIGTERM, save_coverage) - gevent_signal(signal.SIGINT, save_coverage) + gevent_signal.signal(signal.SIGTERM, save_coverage) + gevent_signal.signal(signal.SIGINT, save_coverage) def get_app(): @@ -59,19 +63,86 @@ def get_app(): return app.app +def socketio_emit(message: str, rooms: List[str]): + if not app.socketio: + return + for room in rooms: + app.socketio.emit("events", message, room=room) + + +def loglevel_to_rooms(levelname: str) -> List[str]: + if levelname == 'DEBUG': + return ['DEBUG'] + elif levelname == 'INFO': + return ['DEBUG', 'INFO'] + elif levelname == 'WARNING': + return ['DEBUG', 'INFO', 'WARNING'] + elif levelname == 'ERROR': + return ['DEBUG', 'INFO', 'WARNING', 'ERROR'] + elif levelname == 'CRITICAL': + return ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] + + +def parse_redis_event(event): + try: + # [stream, [(messageid, {datadict})] + if event[0] == "events": + return event[1][0][1] + except Exception as e: + return None + + +def emit_redis_event(event): + try: + if event['type'] == "log": + socketio_emit(event['message'], loglevel_to_rooms(event['level'])) + elif event['type'] == "update": + socketio_emit(json.loads(event['json']), ["update_{}".format(event['update_type'])]) + except Exception as e: + pass + + +def thread_websocket_events(): + redis: StrictRedis + with redis_session() as redis: + while True: + result = redis.xread({"events": b"$"}, count=10, block=200) + for item in result: + event = parse_redis_event(item) + if not event: + continue + emit_redis_event(event) + + if __name__ == '__main__': + # Starting via python run.py # gevent monkey patching required if you start flask with the auto-reloader (debug mode) monkey.patch_all() from cnaas_nms.api import app + from cnaas_nms.db.session import redis_session + import json + + t_websocket_events = threading.Thread(target=thread_websocket_events) + t_websocket_events.start() apidata = get_apidata() if isinstance(apidata, dict) and 'host' in apidata: app.socketio.run(get_app(), debug=True, host=apidata['host']) else: app.socketio.run(get_app(), debug=True) + if 'COVERAGE' in os.environ: save_coverage() else: + # Starting via uwsgi from cnaas_nms.api import app + from cnaas_nms.db.session import redis_session + import json + + t_websocket_events = threading.Thread(target=thread_websocket_events) + t_websocket_events.start() cnaas_app = get_app() + + if 'COVERAGE' in os.environ: + save_coverage() diff --git a/src/cnaas_nms/scheduler/wrapper.py b/src/cnaas_nms/scheduler/wrapper.py index 68ee8151..6bca54d1 100644 --- a/src/cnaas_nms/scheduler/wrapper.py +++ b/src/cnaas_nms/scheduler/wrapper.py @@ -30,7 +30,7 @@ def update_device_progress(job_id: int): new_finished_devices = [] with redis_session() as db: while db.llen('finished_devices_' + str(job_id)) != 0: - last_finished = db.lpop('finished_devices_' + str(job_id)).decode('utf-8') + last_finished = db.lpop('finished_devices_' + str(job_id)) new_finished_devices.append(last_finished) if new_finished_devices: @@ -65,7 +65,21 @@ def wrapper(job_id: int, scheduled_by: str, *args, **kwargs): kwargs['kwargs']['job_id'] = job_id if scheduled_by is None: scheduled_by = 'unknown' - job.start_job(function_name=func.__name__, + # Append (dry_run) to function name if set, so we can distinguish dry_run jobs + try: + if kwargs['kwargs']['dry_run']: + function_name = "{} (dry_run)".format(func.__name__) + else: + function_name = func.__name__ + except Exception: + function_name = func.__name__ + job_comment = kwargs['kwargs'].pop('job_comment', None) + if job_comment and isinstance(job_comment, str): + job.comment = job_comment[:255] + job_ticket_ref = kwargs['kwargs'].pop('job_ticket_ref', None) + if job_ticket_ref and isinstance(job_comment, str): + job.ticket_ref = job_ticket_ref[:32] + job.start_job(function_name=function_name, scheduled_by=scheduled_by) if func.__name__ in progress_funcitons: stop_event = threading.Event() diff --git a/src/cnaas_nms/scheduler_mule.py b/src/cnaas_nms/scheduler_mule.py index d9e02902..627b2b43 100644 --- a/src/cnaas_nms/scheduler_mule.py +++ b/src/cnaas_nms/scheduler_mule.py @@ -9,6 +9,7 @@ from cnaas_nms.plugins.pluginmanager import PluginManagerHandler from cnaas_nms.db.session import sqla_session from cnaas_nms.db.joblock import Joblock +from cnaas_nms.db.job import Job, JobStatus from cnaas_nms.tools.log import get_logger @@ -31,6 +32,26 @@ def save_coverage(): signal.signal(signal.SIGINT, save_coverage) +def pre_schedule_checks(scheduler, kwargs): + check_ok = True + message = "" + for job in scheduler.get_scheduler().get_jobs(): + # Only allow scheduling of one discover_device job at the same time + if job.name == 'cnaas_nms.confpush.init_device:discover_device': + if job.kwargs['kwargs']['ztp_mac'] == kwargs['kwargs']['ztp_mac']: + message = ("There is already another scheduled job to discover device {}, skipping ". + format(kwargs['kwargs']['ztp_mac'])) + check_ok = False + + if not check_ok: + logger.debug(message) + with sqla_session() as session: + job_entry: Job = session.query(Job).filter(Job.id == kwargs['job_id']).one_or_none() + job_entry.finish_abort(message) + + return check_ok + + def main_loop(): try: import uwsgi @@ -62,8 +83,15 @@ def main_loop(): for k, v in data.items(): if k not in ['func', 'trigger', 'id', 'run_date']: kwargs[k] = v + # Perform pre-schedule job checks + try: + if not pre_schedule_checks(scheduler, kwargs): + continue + except Exception as e: + logger.exception("Unable to perform pre-schedule job checks: {}".format(e)) + scheduler.add_job(data['func'], trigger=data['trigger'], kwargs=kwargs, - id=data['id'], run_date=data['run_date']) + id=data['id'], run_date=data['run_date'], name=data['func']) if __name__ == '__main__': diff --git a/src/cnaas_nms/tools/event.py b/src/cnaas_nms/tools/event.py new file mode 100644 index 00000000..a7972566 --- /dev/null +++ b/src/cnaas_nms/tools/event.py @@ -0,0 +1,20 @@ +from typing import Optional + +from cnaas_nms.db.session import redis_session + + +def add_event(message: Optional[str] = None, event_type: str = "log", level: str = "INFO", + update_type: Optional[str] = None, json_data: Optional[str] = None): + with redis_session() as redis: + try: + send_data = {"type": event_type, "level": level} + if event_type == "log": + send_data['message'] = message + elif event_type == "update": + send_data['update_type'] = update_type + send_data['json'] = json_data + redis.xadd("events", + send_data, + maxlen=100) + except Exception as e: + print("Error in add_event: {}".format(e)) diff --git a/src/cnaas_nms/tools/get_apidata.py b/src/cnaas_nms/tools/get_apidata.py index d1efe848..073c8009 100644 --- a/src/cnaas_nms/tools/get_apidata.py +++ b/src/cnaas_nms/tools/get_apidata.py @@ -1,6 +1,9 @@ import yaml -def get_apidata(config='/etc/cnaas-nms/api.yml'): +def get_apidata(config='/etc/cnaas-nms/api.yml') -> dict: + defaults = { + 'allow_apply_config_liverun': False + } with open(config, 'r') as api_file: - return yaml.safe_load(api_file) + return {**defaults, **yaml.safe_load(api_file)} diff --git a/src/cnaas_nms/tools/log.py b/src/cnaas_nms/tools/log.py index a12d31ea..1dca5372 100644 --- a/src/cnaas_nms/tools/log.py +++ b/src/cnaas_nms/tools/log.py @@ -1,34 +1,18 @@ import logging -import threading from flask import current_app from cnaas_nms.scheduler.thread_data import thread_data +from cnaas_nms.tools.event import add_event class WebsocketHandler(logging.StreamHandler): def __init__(self): logging.StreamHandler.__init__(self) - def socketio_emit(self, msg, rooms=[]): - # late import to avoid circular dependency on import - import cnaas_nms.api.app - if cnaas_nms.api.app.socketio: - for room in rooms: - cnaas_nms.api.app.socketio.emit('cnaas_log', msg, room=room) - def emit(self, record): msg = self.format(record) - if record.levelname == 'DEBUG': - self.socketio_emit(msg, rooms=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']) - elif record.levelname == 'INFO': - self.socketio_emit(msg, rooms=['INFO', 'WARNING', 'ERROR', 'CRITICAL']) - elif record.levelname == 'WARNING': - self.socketio_emit(msg, rooms=['WARNING', 'ERROR', 'CRITICAL']) - elif record.levelname == 'ERROR': - self.socketio_emit(msg, rooms=['ERROR', 'CRITICAL']) - elif record.levelname == 'CRITICAL': - self.socketio_emit(msg, rooms=['CRITICAL']) + add_event(msg, level=record.levelname) def get_logger(): diff --git a/src/cnaas_nms/tools/template_dry_run.py b/src/cnaas_nms/tools/template_dry_run.py new file mode 100755 index 00000000..f4dc1364 --- /dev/null +++ b/src/cnaas_nms/tools/template_dry_run.py @@ -0,0 +1,106 @@ +#!/bin/env python3 + +import sys +import os +try: + import requests + import jinja2 + import yaml +except ModuleNotFoundError as e: + print("Please install python modules requests, jinja2 and yaml: {}".format(e)) + sys.exit(3) + +if 'CNAASURL' not in os.environ or 'JWT_AUTH_TOKEN' not in os.environ: + print("Please export environment variables CNAASURL and JWT_AUTH_TOKEN") + sys.exit(4) + +api_url = os.environ['CNAASURL'] +headers = {"Authorization": "Bearer "+os.environ['JWT_AUTH_TOKEN']} + + +def get_entrypoint(platform, device_type): + mapfile = os.path.join(platform, 'mapping.yml') + if not os.path.isfile(mapfile): + raise Exception("File {} not found".format(mapfile)) + with open(mapfile, 'r') as f: + mapping = yaml.safe_load(f) + template_file = mapping[device_type]['entrypoint'] + return template_file + + +def get_device_details(hostname): + r = requests.get( + f"{api_url}/api/v1.0/device/{hostname}", + headers=headers) + if r.status_code != 200: + raise Exception("Could not query device API") + device_data = r.json()['data']['devices'][0] + + r = requests.get( + f"{api_url}/api/v1.0/device/{hostname}/generate_config", + headers=headers) + if r.status_code != 200: + raise Exception("Could not query generate_config API") + config_data = r.json()['data']['config'] + + return device_data['device_type'], device_data['platform'], \ + config_data['available_variables'], config_data['generated_config'] + + +def render_template(platform, device_type, variables): + jinjaenv = jinja2.Environment( + loader=jinja2.FileSystemLoader(platform), + undefined=jinja2.StrictUndefined, trim_blocks=True + ) + template_secrets = {} + for env in os.environ: + if env.startswith('TEMPLATE_SECRET_'): + template_secrets[env] = os.environ[env] + template_vars = {**variables, **template_secrets} + template = jinjaenv.get_template(get_entrypoint(platform, device_type)) + return template.render(**template_vars) + + +def schedule_apply_dryrun(hostname, config): + data = { + 'full_config': config, + 'dry_run': True + } + r = requests.post( + f"{api_url}/api/v1.0/device/{hostname}/apply_config", + headers=headers, + json=data + ) + if r.status_code != 200: + raise Exception("Could not schedule apply_config job via API") + return r.json()['job_id'] + + +def main(): + if len(sys.argv) != 2: + print("Usage: template_dry_run.py ") + sys.exit(1) + + hostname = sys.argv[1] + try: + device_type, platform, variables, old_config = get_device_details(hostname) + except Exception as e: + print(e) + sys.exit(2) + variables['host'] = hostname + new_config = render_template(platform, device_type, variables) + print("OLD TEMPLATE CONFIG ==============================") + print(old_config) + print("NEW TEMPLATE CONFIG ==============================") + print(new_config) + + try: + input("Start apply_config dry run? Ctrl-c to abort or enter to continue...") + except KeyboardInterrupt: + print("Exiting...") + else: + print("Apply config dry_run job: {}".format(schedule_apply_dryrun(hostname, new_config))) + + +if __name__ == "__main__": + main() diff --git a/src/cnaas_nms/version.py b/src/cnaas_nms/version.py index 6b998fcf..a676d27c 100644 --- a/src/cnaas_nms/version.py +++ b/src/cnaas_nms/version.py @@ -1,3 +1,3 @@ -__version__ = '1.0.0' +__version__ = '1.1.0' __version_info__ = tuple([field for field in __version__.split('.')]) __api_version__ = 'v1.0' diff --git a/test/integrationtests.py b/test/integrationtests.py index 3d4cbb69..0b8a19f0 100644 --- a/test/integrationtests.py +++ b/test/integrationtests.py @@ -20,27 +20,8 @@ class GetTests(unittest.TestCase): - def setUp(self): - self.assertTrue(self.wait_connect(), "Connection to API failed") - - r = requests.put( - f'{URL}/api/v1.0/repository/templates', - headers=AUTH_HEADER, - json={"action": "refresh"}, - verify=TLS_VERIFY - ) - print("Template refresh status: {}".format(r.status_code)) - self.assertEqual(r.status_code, 200, "Failed to refresh templates") - r = requests.put( - f'{URL}/api/v1.0/repository/settings', - headers=AUTH_HEADER, - json={"action": "refresh"}, - verify=TLS_VERIFY - ) - print("Settings refresh status: {}".format(r.status_code)) - self.assertEqual(r.status_code, 200, "Failed to refresh settings") - - def wait_connect(self): + @classmethod + def setUpClass(cls): for i in range(100): try: r = requests.get( @@ -57,7 +38,7 @@ def wait_connect(self): else: print("Bad status code {}, retrying in 1 second...".format(r.status_code)) time.sleep(1) - return False + assert False, "Failed to test connection to API" def wait_for_discovered_device(self): for i in range(100): @@ -90,7 +71,25 @@ def check_jobid(self, job_id): else: raise Exception - def test_0_init_dist(self): + def test_00_sync(self): + r = requests.put( + f'{URL}/api/v1.0/repository/templates', + headers=AUTH_HEADER, + json={"action": "refresh"}, + verify=TLS_VERIFY + ) + print("Template refresh status: {}".format(r.status_code)) + self.assertEqual(r.status_code, 200, "Failed to refresh templates") + r = requests.put( + f'{URL}/api/v1.0/repository/settings', + headers=AUTH_HEADER, + json={"action": "refresh"}, + verify=TLS_VERIFY + ) + print("Settings refresh status: {}".format(r.status_code)) + self.assertEqual(r.status_code, 200, "Failed to refresh settings") + + def test_01_init_dist(self): new_dist_data = { "hostname": "eosdist1", "management_ip": "10.100.3.101", @@ -128,7 +127,7 @@ def test_0_init_dist(self): ) self.assertEqual(r.status_code, 200, "Failed to add mgmtdomain") - def test_1_ztp(self): + def test_02_ztp(self): hostname, device_id = self.wait_for_discovered_device() print("Discovered hostname, id: {}, {}".format(hostname, device_id)) self.assertTrue(hostname, "No device in state discovered found for ZTP") @@ -151,7 +150,7 @@ def test_1_ztp(self): self.assertFalse(result_step2['devices']['eosaccess']['failed'], "Could not reach device after ZTP") - def test_2_interfaces(self): + def test_03_interfaces(self): r = requests.get( f'{URL}/api/v1.0/device/eosaccess/interfaces', headers=AUTH_HEADER, @@ -175,7 +174,7 @@ def test_2_interfaces(self): ) self.assertEqual(r.status_code, 200, "Failed to update interface") - def test_3_syncto_access(self): + def test_04_syncto_access(self): r = requests.post( f'{URL}/api/v1.0/device_syncto', headers=AUTH_HEADER, @@ -183,6 +182,7 @@ def test_3_syncto_access(self): verify=TLS_VERIFY ) self.assertEqual(r.status_code, 200, "Failed to do sync_to access") + self.check_jobid(r.json()['job_id']) r = requests.post( f'{URL}/api/v1.0/device_syncto', headers=AUTH_HEADER, @@ -190,8 +190,9 @@ def test_3_syncto_access(self): verify=TLS_VERIFY ) self.assertEqual(r.status_code, 200, "Failed to do sync_to access") + self.check_jobid(r.json()['job_id']) - def test_4_syncto_dist(self): + def test_05_syncto_dist(self): r = requests.post( f'{URL}/api/v1.0/device_syncto', headers=AUTH_HEADER, @@ -199,8 +200,9 @@ def test_4_syncto_dist(self): verify=TLS_VERIFY ) self.assertEqual(r.status_code, 200, "Failed to do sync_to dist") + self.check_jobid(r.json()['job_id']) - def test_5_genconfig(self): + def test_06_genconfig(self): r = requests.get( f'{URL}/api/v1.0/device/eosdist1/generate_config', headers=AUTH_HEADER, @@ -208,7 +210,7 @@ def test_5_genconfig(self): ) self.assertEqual(r.status_code, 200, "Failed to generate config for eosdist1") - def test_6_plugins(self): + def test_07_plugins(self): r = requests.get( f'{URL}/api/v1.0/plugins', headers=AUTH_HEADER, @@ -224,7 +226,7 @@ def test_6_plugins(self): ) self.assertEqual(r.status_code, 200, "Failed to run plugin selftests") - def test_7_firmware(self): + def test_08_firmware(self): r = requests.get( f'{URL}/api/v1.0/firmware', headers=AUTH_HEADER, @@ -233,13 +235,51 @@ def test_7_firmware(self): # TODO: not working #self.assertEqual(r.status_code, 200, "Failed to list firmware") - def test_8_sysversion(self): + def test_09_sysversion(self): r = requests.get( f'{URL}/api/v1.0/system/version', verify=TLS_VERIFY ) self.assertEqual(r.status_code, 200, "Failed to get CNaaS-NMS version") + def test_10_get_prev_config(self): + hostname = "eosaccess" + r = requests.get( + f"{URL}/api/v1.0/device/{hostname}/previous_config?previous=0", + headers=AUTH_HEADER, + verify=TLS_VERIFY + ) + self.assertEqual(r.status_code, 200) + prev_job_id = r.json()['data']['job_id'] + if r.json()['data']['failed']: + return + data = { + "job_id": prev_job_id, + "dry_run": True + } + r = requests.post( + f"{URL}/api/v1.0/device/{hostname}/previous_config", + headers=AUTH_HEADER, + verify=TLS_VERIFY, + json=data + ) + self.assertEqual(r.status_code, 200) + restore_job_id = r.json()['job_id'] + job = self.check_jobid(restore_job_id) + self.assertFalse(job['result']['devices'][hostname]['failed']) + + def test_11_update_facts_dist(self): + hostname = "eosdist1" + r = requests.post( + f'{URL}/api/v1.0/device_update_facts', + headers=AUTH_HEADER, + json={"hostname": hostname}, + verify=TLS_VERIFY + ) + self.assertEqual(r.status_code, 200, "Failed to do update facts for dist") + restore_job_id = r.json()['job_id'] + job = self.check_jobid(restore_job_id) + self.assertFalse(job['result']['devices'][hostname]['failed']) if __name__ == '__main__':