Merge branch 'dev'

Conflicts:
	docs/INSTALLATION.rst

.travis.yml

@@ -5,30 +5,102 @@ sudo: required
 services:
   - docker
 
-before_install:
-  - docker pull pierky/bird:1.6.3 
+script: tox
 
-install:
-  - pip install tox
-  - pip install coveralls
+jobs:
+  include:
 
-env:
-   - TOXENV=py34-syntax
-   - TOXENV=py27-flake8
-   - TOXENV=py34-flake8
-   - TOXENV=py27
-   - TOXENV=py34
-   - TOXENV=py27-coverage
+    # All the "standard" tests, run on push/PR.
+    - stage: test
+      env: TOXENV=py34-syntax
+      python: "3.4"
+      install:
+        - pip install tox
+    - env: TOXENV=py27-flake8
+      python: "2.7"
+      install:
+        - pip install tox
+    - env: TOXENV=py34-flake8
+      python: "3.4"
+      install:
+        - pip install tox
+    - env: TOXENV=py27
+      python: "2.7"
+      install:
+        - pip install tox
+        - docker pull pierky/bird:1.6.3
+    - env: TOXENV=py34
+      python: "3.4"
+      install:
+        - pip install tox
+        - docker pull pierky/bird:1.6.3
+    - env: TOXENV=py27-coverage
+      python: "2.7"
+      install:
+        - pip install tox
+        - pip install coveralls
+      after_success:
+        - coveralls
 
-script: tox
+    # External resources tests, run on push/PR and via cron job.
+    - stage: test external resources
+      env: TOXENV=py34-extres
+      python: "3.4"
+      install:
+        - pip install tox
+      before_script:
+        - ./tests/external_resources/install_bgpq3.sh
+    - env: TOXENV=py27-extres
+      python: "2.7"
+      install:
+        - pip install tox
+      before_script:
+        - ./tests/external_resources/install_bgpq3.sh
 
-after_success:
- - coveralls
+    # Installation tests on Ubuntu, Debian, CentOS, run on push/PR with tags.
 
-jobs:
-  include:
+    # CentOS, latest
+    - &installation-stage
+      stage: installation tests
+      install: skip
+      script: ./tests/docker/build
+      env: DISTRO=centos-latest PYVER=2
+
+    # Ubuntu, latest
+    - <<: *installation-stage
+      env: DISTRO=ubuntu-latest PYVER=2
+    - <<: *installation-stage
+      env: DISTRO=ubuntu-latest PYVER=3
+
+    # Ubuntu, rolling
+    - <<: *installation-stage
+      env: DISTRO=ubuntu-rolling PYVER=2
+    - <<: *installation-stage
+      env: DISTRO=ubuntu-rolling PYVER=3
+
+    # Ubuntu, Trusty
+    - <<: *installation-stage
+      env: DISTRO=ubuntu-trusty PYVER=2
+    - <<: *installation-stage
+      env: DISTRO=ubuntu-trusty PYVER=3
+
+    # Debian, stable
+    - <<: *installation-stage
+      env: DISTRO=debian-stable PYVER=2
+    - <<: *installation-stage
+      env: DISTRO=debian-stable PYVER=3
+
+    # Debian, oldstable
+    - <<: *installation-stage
+      env: DISTRO=debian-oldstable PYVER=2
+    - <<: *installation-stage
+      env: DISTRO=debian-oldstable PYVER=3
+
+    # Deploy to the test instance of PyPI.
+    # Only "vX.Y.Z-alphaN" releases from 'dev' branch go here.
     - stage: deploy test
       script: skip
+      install: skip
       deploy:
         # Test
         - provider: pypi
@@ -40,8 +112,12 @@ jobs:
           on:
             branch: dev
             tags: true
+
+    # Deploy to the production instance of PyPI.
+    # Only "vX.Y.Z" releases from 'master' branch go here.
     - stage: deploy production
       script: skip
+      install: skip
       deploy:
         # Production
         - provider: pypi
@@ -54,8 +130,13 @@ jobs:
             tags: true
 
 stages:
-  - test
+  - name: test
+    if: type IN (push, pull_request)
+  - name: test external resources
+    if: type IN (push, pull_request, cron)
+  - name: installation tests
+    if: type IN (push, pull_request) AND tag IS present
   - name: deploy test
-    if: tag IS present AND tag =~ ^v\d+\.\d+\.\d+-alpha\d+\S*$
+    if: type IN (push, pull_request) AND tag IS present AND tag =~ ^v\d+\.\d+\.\d+-alpha\d+\S*$
   - name: deploy production
-    if: tag IS present AND tag =~ ^v\d+\.\d+\.\d+$
+    if: type IN (push, pull_request) AND tag IS present AND tag =~ ^v\d+\.\d+\.\d+$

CHANGES.rst

@@ -246,7 +246,7 @@ v0.5.0
 
 - Fix: avoid the use of standard communities in the range 65535:x.
 - Improvement: option to set max-prefix restart timer for OpenBGPD.
-- Deleted feature: tagging of routes à la RPKI-Light has been removed.
+- Deleted feature: tagging of routes a' la RPKI-Light has been removed.
 
   - The ``reject_invalid`` flag, that previously was on general scope only, now can be set on a client-by-client basis.
   - The ``roa_valid``, ``roa_invalid``, and ``roa_unknown`` communities no longer exist.

COPYRIGHT

@@ -1,4 +1,4 @@
-# Copyright (C) 2017 Pier Carlo Chiodi
+# Copyright (C) 2017-2018 Pier Carlo Chiodi
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -12,4 +12,3 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-

MANIFEST.in

@@ -5,9 +5,7 @@ include MANIFEST.in
 include requirements.txt
 include setup.cfg
 recursive-include pierky *.py
-recursive-include pierky/arouteserver/config.d
 recursive-include pierky/arouteserver/config.d *
-recursive-include pierky/arouteserver/templates
 recursive-include pierky/arouteserver/templates *
 recursive-include pierky/arouteserver/tests/live_tests/skeleton *.py
 recursive-include pierky/arouteserver/tests/live_tests/skeleton *.j2

docs/EXAMPLES.rst

@@ -87,6 +87,8 @@ Some clients files automatically built from `Euro-IX member list JSON files <htt
 
 https://github.com/pierky/arouteserver/blob/master/examples/clients-from-euroix
 
+.. DO NOT EDIT: this file is automatically created by /utils/build_doc
+
 ``configure`` command output
 ----------------------------
 
@@ -96,7 +98,7 @@ A list of BGP communities is also automatically built.
 
 .. code-block:: console
 
-   $ arouteserver configure --output general.yml
+   $ arouteserver configure --output examples/auto-config/README.rst
    
    BGP daemon
    ==========
@@ -109,17 +111,14 @@ A list of BGP communities is also automatically built.
    limitations
    
    Which BGP daemon will be used? [bird/openbgpd] bird
-   
    Router server's ASN
    ===================
    
    What's the ASN of the route server? 64496
-   
    Route server's BGP router-id
    ============================
    
    Please enter the route server BGP router-id: 192.0.2.1
-   
    List of local networks
    ======================
    
@@ -130,8 +129,10 @@ A list of BGP communities is also automatically built.
    
    
    Route server policy definition file generated successfully!
+   ===========================================================
    
-   The content of the general configuration file will now be written to general.yml
+   The content of the general configuration file will now be written to examples
+   /auto-config/bird-general.yml
    
    Some notes:
    
@@ -143,9 +144,11 @@ A list of BGP communities is also automatically built.
     - PeeringDB is used to fetch AS-SETs for those clients that are not explicitly
    configured.
     - RPKI ROAs are used as if they were route objects to further enrich IRR data.
+    - ARIN Whois database dump is fetched from NLNOG to further enrich IRR data.
+    - PeeringDB is used to fetch networks prefix count.
     - Routes tagged with the GRACEFUL_SHUTDOWN well-known community (65535:0) are
    processed accordingly to draft-ietf-grow-bgp-gshut.
-
+   
 The textual description (HTML) generated on the basis of the *general.yml* files produced by this command is also reported here.
 
 https://github.com/pierky/arouteserver/blob/master/examples/auto-config

docs/INSTALLATION.rst

@@ -1,70 +1,95 @@
 Installation
 ============
 
-1. Strongly suggested: install ``pip`` and setup a `Virtualenv <https://virtualenv.pypa.io/en/latest/installation.html>`_:
+Dependencies
+------------
 
-  .. code:: bash
+Some components used by ARouteServer need Python dev header files and static libraries: some distributions have them already included, others may need to manually install them:
+
+.. code:: bash
+
+   # Debian Jessie, Ubuntu Trusty
+   apt-get install python-dev  # for Python 2
+   apt-get install python3-dev # for Python 3
 
-    # on Debian/Ubuntu:
-    sudo apt-get install python-virtualenv
+   # CentOS
+   yum -y install gcc python-devel
 
-    # on CentOS:
-    sudo yum install epel-release
-    sudo yum install python-pip python-virtualenv 
+Please note that ARouteServer also needs `bgpq3 <https://github.com/snar/bgpq3>`_ to build IRR-based filters: details on its installation can be found within the :ref:`External programs` section.
 
-    # setup a virtualenv
-    mkdir -p ~/.virtualenvs/arouteserver
-    virtualenv ~/.virtualenvs/arouteserver
-    source ~/.virtualenvs/arouteserver/bin/activate
+Install using ``pip`` (suggested)
+---------------------------------
 
-  More: ``virtualenv`` `installation <https://virtualenv.pypa.io/en/latest/installation.html>`_ and `usage <https://virtualenv.pypa.io/en/latest/userguide.html>`_.
+If you plan to just use the program to build configurations or to run your own live tests scenarios, you can install it using ``pip``.
+
+Strongly suggested: setup a `Virtualenv <https://virtualenv.pypa.io/>`_.
+
+.. code:: bash
 
-2. Install the program.
-
-        - If you plan to run built-in :doc:`Live tests <LIVETESTS>` on your own or to contribute to the project, clone the GitHub repository locally and install dependencies:
+   # on Debian/Ubuntu:
+   sudo apt-get install python-pip python-virtualenv
 
-        .. code:: bash
+   # on CentOS:
+   sudo yum install epel-release
+   sudo yum install python-pip python-virtualenv
 
-            # from within the previously created arouteserver directory
-            git clone https://github.com/pierky/arouteserver.git ./
-            export PYTHONPATH="`pwd`"
-            pip install -r requirements.txt
+   # setup a virtualenv
+   mkdir -p ~/.virtualenvs/arouteserver
+   virtualenv ~/.virtualenvs/arouteserver
+   source ~/.virtualenvs/arouteserver/bin/activate
 
+   # install the program
+   pip install arouteserver
 
-        - If you plan to just use the program to build configurations or to run your own live tests scenarios, you can install it using ``pip``:
+More: ``virtualenv`` `installation <https://virtualenv.pypa.io/en/latest/installation.html>`_ and `usage <https://virtualenv.pypa.io/en/latest/userguide.html>`_.
+
+.. note:: If you receive the following error while installing the program (or its requirements): **error in setup command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers** then please upgrade the *setuptools* package that is used in your virtualenv: ``pip install --upgrade setuptools``.
+
+.. note:: In the case the pip installation process breaks with the **Failed building wheel for py-radix / fatal error: Python.h: No such file or directory** error, please verify that the dependencies are satisfied.
+
+Install from GitHub
+-------------------
+
+If you plan to run built-in :doc:`Live tests <LIVETESTS>` on your own or to contribute to the project, clone the GitHub repository locally and install dependencies:
+
+.. code:: bash
 
-        .. code:: bash
+   mkdir -p ~/src/arouteserver
+   cd ~/src/arouteserver
 
-           pip install arouteserver
+   # use the URL of your fork here:
+   git clone https://github.com/USERNAME/arouteserver.git ./
 
-        .. note:: If you receive the following error while installing the program (or its requirements) using pip: **error in setup command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers** then please upgrade the *setuptools* package that is used in your virtualenv: ``pip install --upgrade setuptools``.
+   export PYTHONPATH="`pwd`"
+   pip install -r requirements.txt
 
-        .. note:: Python dev header files and static libraries are needed to compile a module on which ARouteServer depends on: in the case the pip installation process breaks with the **Failed building wheel for py-radix / fatal error: Python.h: No such file or directory** error, please install the appropriate python-dev package: ``sudo apt-get install python-dev`` for Python2, ``sudo apt-get install python3-dev`` for Python 3.
+Setup and initialization
+------------------------
 
-3. Setup your system layout (confirmation will be asked before each action):
+- Setup your system layout (confirmation will be asked before each action):
 
   .. code:: bash
 
+    # if you used pip
+    arouteserver setup
+
     # if you installed from GitHub
     export PYTHONPATH="`pwd`"
     ./scripts/arouteserver setup
 
-    # if you used pip
-    arouteserver setup
-
   The program will ask you to create some directories (under ``~/arouteserver`` by default) and to copy some files there.
   These paths can be changed by editing the ``arouteserver.yml`` program configuration file or by using command line arguments. More information in the :doc:`configuration section <CONFIG>`.
 
-4. Define the route server configuration policies, using the ``configure`` command or manually by editing the ``general.yml`` file:
+- Define the route server configuration policies, using the ``configure`` command or manually by editing the ``general.yml`` file:
 
   .. code:: bash
 
-    # if you installed from GitHub
-    ./scripts/arouteserver configure
-
     # if you used pip
     arouteserver configure
 
+    # if you installed from GitHub
+    ./scripts/arouteserver configure
+
   The ``configure`` command asks some questions about the route server environment (ASN, router ID, local subnets) and then it builds a policy definition file based on best practices and suggestions which also includes a rich BGP communities list.
 
 External programs

docs/USAGE.rst

@@ -190,7 +190,7 @@ To get a list of all the available options, run the ``arouteserver clients-from-
 Integration with IXP-Manager
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Since the popular `IXP-Manager <https://github.com/inex/IXP-Manager>`_ allows to `export the list of members in Euro-IX JSON format <https://github.com/inex/IXP-Manager/wiki/Euro-IX-Member-Data-Export>`_, this ARouteServer's command can also be used to integrate the two tools:
+Since the popular `IXP-Manager <https://github.com/inex/IXP-Manager>`_ allows to `export the list of members in Euro-IX JSON format <https://github.com/inex/IXP-Manager/wiki/Euro-IX-Member-Data-Export>`_, the ``arouteserver clients-from-euroix`` command can also be used to integrate the two tools and to build ARouteServer's list of clients automatically:
 
 .. code:: bash