Merge pull request #133 from SUNET/release.1.1

Release.1.1

alembic/versions/8a635012afa7_add_new_interface_config_types.py

@@ -0,0 +1,31 @@
+"""add new interface config types
+
+Revision ID: 8a635012afa7
+Revises: 395427a732d6
+Create Date: 2020-03-26 09:21:15.439761
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8a635012afa7'
+down_revision = '395427a732d6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # ### end Alembic commands ###
+    op.execute("COMMIT")
+    op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'TEMPLATE' AFTER 'CUSTOM'")
+    op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'MLAG_PEER' AFTER 'TEMPLATE'")
+    op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'ACCESS_DOWNLINK' AFTER 'ACCESS_UPLINK'")
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

docker/api/Dockerfile

@@ -1,4 +1,5 @@
 FROM debian:buster
+ARG BUILDBRANCH=develop
 
 # Create directories
 RUN mkdir -p /opt/cnaas
@@ -14,7 +15,7 @@ COPY config/plugins.yml /etc/cnaas-nms/plugins.yml
 
 # Setup script
 COPY cnaas-setup.sh /opt/cnaas/cnaas-setup.sh
-RUN /opt/cnaas/cnaas-setup.sh
+RUN /opt/cnaas/cnaas-setup.sh $BUILDBRANCH
 
 # Prepare for supervisord, uwsgi, ngninx
 COPY nosetests.sh /opt/cnaas/

docker/api/cnaas-setup.sh

@@ -25,6 +25,9 @@ apt-get update && \
       supervisor \
       libssl-dev \
       libpq-dev \
+      libpcre2-dev \
+      libpcre3-dev \
+      uwsgi-plugin-python3 \
     && apt-get clean
 
 pip3 install uwsgi
@@ -43,11 +46,13 @@ git checkout develop
 python3 -m pip install -r requirements.txt
 
 # Temporary for testing new branch
-#cd /opt/cnaas/venv/cnaas-nms/
-#git remote update
-#git fetch
-#git checkout --track origin/release-1.0.0
-#python3 -m pip install -r requirements.txt
+if [ "$1" != "develop" ] ; then
+	cd /opt/cnaas/venv/cnaas-nms/
+	git remote update
+	git fetch
+	git checkout --track origin/$1
+	python3 -m pip install -r requirements.txt
+fi
 
 chown -R www-data:www-data /opt/cnaas/settings
 chown -R www-data:www-data /opt/cnaas/templates

docker/api/config/supervisord_app.conf

@@ -12,4 +12,4 @@ autorestart=true
 
 [program:nginx]
 command=/usr/sbin/nginx -g "daemon off;"
-autorestart=true
+autorestart=true

docker/api/config/uwsgi.ini

@@ -2,6 +2,7 @@
 uid=www-data
 gid=www-data
 chdir = /opt/cnaas/venv/cnaas-nms/src/
+plugins = gevent
 callable = cnaas_app
 module = cnaas_nms.run
 socket = /tmp/uwsgi.sock
@@ -18,3 +19,5 @@ lazy-apps = true
 # websocket support
 http-websockets = true
 gevent = 1000
+# don't log jwt tokens
+log-drain = jwt=

docker/dhcpd/dhcpd.conf

@@ -52,6 +52,7 @@ class "JUNIPER" {
   option tftp-server-name "10.0.1.3";
 }
 
+# This empty subnet is required for dhcpd to start in the cnaas docker-compose network, do not remove
 subnet 10.0.1.0 netmask 255.255.255.0 {
 }
 

docs/apiref/devices.rst

@@ -160,6 +160,15 @@ To remove a device, pass the device ID in a DELTE call:
    curl -X DELETE https://hostname/api/v1.0/device/10
 
 
+There is also the option to factory default and reboot the device
+when removing it. This can be done like this:
+
+::
+
+   curl -H "Content-Type: application/json" -X DELETE -d
+   '{"factory_default": true}' https://hostname/api/v1.0/device/10
+
+
 Preview config
 --------------
 
@@ -173,3 +182,83 @@ touching the device use generate_config:
 This will return both the generated configuration based on the template for
 this device type, and also a list of available vaiables that could be used
 in the template.
+
+View previous config
+--------------------
+
+You can also view previous versions of the configuration for a device. All
+previous configurations are saved in the job database and can be found using
+either a specific Job ID (using job_id=), a number of steps to walk backward
+to find a previous configuration (previous=), or using a date to find the last
+configuration applied to the device before that date.
+
+::
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/previous_config?before=2020-04-07T12:03:05"
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/previous_config?previous=1"
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/previous_config?job_id=12"
+
+If you want to restore a device to a previous configuration you can send a POST:
+
+::
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/previous_config" -X POST -d '{"job_id": 12, "dry_run": true}' -H "Content-Type: application/json"
+
+When sending a POST you must specify an exact job_id to restore. The job must
+have finished with a successful status for the specified device. The device
+will change to UNMANAGED state since it's no longer in sync with current
+templates and settings.
+
+Apply static config
+-------------------
+
+You can also test a static configuration specified in the API call directly
+instead of generating the configuration via templates and settings.
+This can be useful when developing new templates (see template_dry_run.py tool)
+when you don't wish to do the commit/push/refresh/sync workflow for every
+iteration. By default only dry_run are allowed, but you can configure api.yml
+to allow apply config live run as well.
+
+::
+
+   curl "https://hostname/api/v1.0/device/<device_hostname>/apply_config" -X POST -d '{"full_config": "hostname eosdist1\n...", "dry_run": True}' -H "Content-Type: application/json"
+
+This will schedule a job to send the configuration to the device.
+
+Initialize device
+-----------------
+
+For a more detailed explanation see documentation under Howto :ref:`ztp_intro`.
+
+To initialize a single ACCESS type device:
+
+::
+
+   curl https://localhost/api/v1.0/device_init/45 -d '{"hostname": "ex2300-top", "device_type": "ACCESS"}' -X POST -H "Content-Type: application/json"
+
+The device must be in state DISCOVERED to start initialization. The device must be able to detect compatible uplink devices via LLDP for initialization to finish.
+
+To initialize a pair of ACCESS devices as an MLAG pair:
+
+::
+
+   curl https://localhost/api/v1.0/device_init/45 -d '{"hostname": "a1", "device_type": "ACCESS", "mlag_peer_id": 46, "mlag_peer_hostname": "a2"}' -X POST -H "Content-Type: application/json"
+
+For MLAG pairs the devices must be able to dectect it's peer via LLDP neighbors and compatible uplink devices for initialization to finish.
+
+Update facts
+------------
+
+To update the facts about a device (serial number, vendor, model and OS version)
+use this API call:
+
+::
+
+   curl https://localhost/api/v1.0/device_update_facts -d '{"hostname": "eosdist1"}' -X POST -H "Content-Type: application/json"
+
+This will schedule a job to log in to the device, get the facts and update the
+database. You can perform this action on both MANAGED and UNMANAGED devices.
+UNMANAGED devices might not be reachable so this could be a good test-call
+before moving the device back to the MANAGED state.

docs/apiref/mgmtdomains.rst

@@ -6,7 +6,7 @@ Management domain can be retreived, added, updated an removed using this API.
 Get all managment domains
 -------------------------
 
-All management domain can be listed using CURL:
+All management domains can be listed using CURL:
 
 
 ::
@@ -39,6 +39,8 @@ That will return a JSON structured response which describes all domains availabl
       }
   }
 
+Note that some of these fields does not have a use case (yet).
+
 You can also specify one specifc mgmtdomain to query by using:
 
 ::
@@ -57,7 +59,15 @@ To add a new management domain we can to call the API with a few fields set in a
    * ipv4_gw (mandatory): The IPv4 gateway to be used, should be expressed with a prefix (10.0.0.1/24)
    * device_a (mandatory): Hostname of the first device
    * device_b (mandatory): Hostname of the second device
-   * vlan (mandatory): A VLAN
+   * vlan (mandatory): A VLAN ID
+
+device_a and device_b should be a pair of DIST devices that are connected to a
+specific set of access devices that should share the same management network.
+It's also possible to specify two CORE devices if there is a need to have the
+gateway/routing for all access switch management done in the CORE layer instead.
+In the case where two CORE devices are specified there should only be one single
+mgmtdomain defined for the entire NMS, and this mgmtdomain can only contain
+exactly two CORE devices even if there are more CORE devices in the network.
 
 Example using CURL:
 

docs/apiref/syncto.rst

@@ -54,6 +54,10 @@ Arguments:
    re-synchronized, if you specify this option as true then all devices will be checked.
    This option does not affect syncto jobs with a specified hostname, when you select only
    a single device via hostname it's always re-synchronized. Defaults to false.
+ - comment: Optionally add a comment that is saved in the job log.
+   This should be a string with max 255 characters.
+ - ticket_ref: Optionally reference a service ticket associated with this job.
+   This should be a string with max 32 characters.
 
 If neither hostname or device_type is specified all devices that needs to be sycnhronized
 will be selected.

docs/changelog/index.rst

@@ -1,6 +1,25 @@
 Changelog
 =========
 
+Version 1.1.0
+-------------
+
+New features:
+
+- New options for connecting access switches:
+
+  - Two access switches as an MLAG pair
+  - Access switch connected to other access switch
+
+- New template variables:
+
+  - device_model: Hardware model of this device
+  - device_os_version: OS version of this device
+
+- Get/restore previous config versions for a device
+- API call to update facts (serial,os version etc) about device
+- Websocket event improvements for logs, jobs and device updates
+
 Version 1.0.0
 -------------
 

docs/configuration/index.rst

@@ -23,6 +23,7 @@ Defines parameters for the API:
 - jwtcert: Defines the path to the public JWT certificate used to verify JWT tokens
 - httpd_url: URL to the httpd container containing firmware images
 - verify_tls: Verify certificate for connections to httpd/firmware server
+- allow_apply_config_liverun: Allow liverun on apply_config API call. Defaults to False.
 
 /etc/cnaas-nms/repository.yml
 -----------------------------

docs/howto/index.rst

@@ -24,6 +24,8 @@ with dry_run to preview changes::
 The API call to device_syncto will start a job running in the background on the API server. To
 show the progress/output of the job run the last command (/job) until you get a finished result.
 
+.. _ztp_intro:
+
 Zero-touch provisioning of access switch
 ----------------------------------------
 

docs/reporef/index.rst

@@ -44,6 +44,12 @@ that are exposed from CNaaS includes:
 
 - access_auto: A list of access_auto interfacs. Using same keys as uplinks.
 
+- device_model: Device model string, same as "model" in the device API. Can be
+  used if you need model specific configuration lines.
+
+- device_os_version: Device OS version string, same as "os_version" in the
+  device API. Can be used if you need OS version specific configuration lines.
+
 Additional variables available for distribution switches:
 
 - infra_ip: IPv4 infrastructure VRF address (ex 10.199.0.0)
@@ -181,10 +187,10 @@ Contains a dictinary called "vxlans", which in turn has one dictinoary per vxlan
 name is the dictionary key and dictionaly values are:
 
   * vni: VXLAN ID, 1-16777215
-  * vrf: VRF name
+  * vrf: VRF name. Optional unless ipv4_gw is also specified.
   * vlan_id: VLAN ID, 1-4095
   * vlan_name: VLAN name, single word/no spaces, max 31 characters
-  * ipv4_gw: IPv4 address with CIDR netmask, ex: 192.168.0.1/24
+  * ipv4_gw: IPv4 address with CIDR netmask, ex: 192.168.0.1/24. Optional.
   * groups: List of group names where this VXLAN/VLAN should be provisioned. If you select an
     access switch the parent dist switch should be automatically provisioned.
 

requirements.txt

@@ -1,24 +1,26 @@
-alembic==1.4.0
+alembic==1.4.2
 APScheduler==3.6.3
-coverage==5.0.3
+coverage==5.2.1
 Flask-Cors==3.0.8
 Flask-JWT-Extended==3.24.1
-flask-restx==0.1.1
-Flask-SocketIO==4.2.1
-gevent==1.4.0
-GitPython==3.1.0
-mypy==0.761
+flask-restx==0.2.0
+Flask-SocketIO==4.3.1
+gevent==20.6.2
+GitPython==3.1.7
+mypy==0.782
 mypy-extensions==0.4.3
 nornir==2.4.0
+napalm==3.1.0
 nose==1.3.7
 pluggy==0.13.1
-psycopg2==2.8.4
-psycopg2-binary==2.8.4
-redis==3.4.1
+psycopg2==2.8.5
+psycopg2-binary==2.8.5
+redis==3.5.3
 redis-lru==0.1.0
-Sphinx==2.4.3
-SQLAlchemy==1.3.13
+Sphinx==3.2.1
+SQLAlchemy==1.3.19
 sqlalchemy-stubs==0.3
-SQLAlchemy-Utils==0.36.1
-pydantic==1.4
-Werkzeug==0.16.1
+SQLAlchemy-Utils==0.36.8
+pydantic==1.6.1
+Werkzeug==1.0.1
+greenlet==0.4.16