diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 92e0eb30b..000000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: CI - -on: - push: - branches: [ main, 4.2, 3.4 ] - pull_request: - branches: [ main ] - -permissions: - contents: read - -jobs: - build: - - # runs-on: ubuntu-latest - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: [3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12] - os: [ubuntu-latest] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5.1.0 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f qa/requirements.txt ]; then pip install -r qa/requirements.txt; fi - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --exclude src/exabgp/vendoring/ --exclude build/ --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Configuration Parsing Checks - run: | - ./qa/bin/functional parsing - - - name: Functional Checks - run: | - ./qa/bin/functional encoding - - - name: Decoding Checks - run: | - ./qa/bin/functional decoding - - - name: Test Coverage - run: | - env PYTHONPATH=src exabgp_log_enable=false pytest --cov --cov-reset ./tests/*_test.py - - # - name: Coveralls - # run: | - # coveralls - -# - name: Test with pytest -# run: | -# pytest diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yaml similarity index 72% rename from .github/workflows/codeql-analysis.yml rename to .github/workflows/codeql-analysis.yaml index 8670e9f00..cafe464e2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yaml @@ -1,14 +1,3 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "CodeQL" on: @@ -33,8 +22,6 @@ jobs: fail-fast: false matrix: language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support steps: - name: Checkout repository diff --git a/.github/workflows/functional-3.6.yaml b/.github/workflows/functional-3.6.yaml new file mode 100644 index 000000000..cad9455e1 --- /dev/null +++ b/.github/workflows/functional-3.6.yaml @@ -0,0 +1,46 @@ +name: Functional Legacy + +on: + push: + branches: [ main, 4.2, 3.4 ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +jobs: + build: + + # runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.6" ] + os: [ "ubuntu-20.04" ] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: ${{ matrix.python-version }} + + - name: run python version + run: | + python --version + + - name: Install dependencies + run: | + python -m pip install --no-cache-dir --upgrade pip + pip install --no-cache-dir -r requirements.txt + pip install psutil + + - name: change ownership to exa user + run : | + echo "EXABGP_DAEMON_USER=$(whoami)" >> $GITHUB_ENV + + - name: Python 3.6 Coverage + run: | + ./qa/bin/functional-3.6 run diff --git a/.github/workflows/functional-testing.yaml b/.github/workflows/functional-testing.yaml new file mode 100644 index 000000000..d4a8deeeb --- /dev/null +++ b/.github/workflows/functional-testing.yaml @@ -0,0 +1,50 @@ +name: Functional Testing + +on: + push: + branches: [ main, 4.2, 3.4 ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +jobs: + build: + + # runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12" ] + os: [ "ubuntu-latest" ] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: ${{ matrix.python-version }} + + - name: run python version + run: | + python --version + + - name: Install dependencies + run: | + python -m pip install --no-cache-dir --upgrade pip + pip install --no-cache-dir -r requirements.txt + pip install psutil + + - name: Configuration + run: | + ./qa/bin/functional parsing + + - name: Functional + run: | + ./qa/bin/functional encoding + + - name: Decoding + run: | + ./qa/bin/functional decoding diff --git a/.github/workflows/linting.yaml b/.github/workflows/linting.yaml new file mode 100644 index 000000000..e3dd9a70e --- /dev/null +++ b/.github/workflows/linting.yaml @@ -0,0 +1,39 @@ +name: Linting + +on: + push: + branches: [ main, 4.2, 3.4 ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +jobs: + build: + + # runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + os: [ "ubuntu-latest" ] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install psutil + pip install flake8 + + - name: flake8 + run: | + flake8 . --exclude src/exabgp/vendoring/ --exclude build/ --exclude site-packages --count --select=E9,F63,F7,F82 --show-source --statistics diff --git a/.github/workflows/unit-testing.yaml b/.github/workflows/unit-testing.yaml new file mode 100644 index 000000000..d7b3644ad --- /dev/null +++ b/.github/workflows/unit-testing.yaml @@ -0,0 +1,41 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Unit Testing + +on: + push: + branches: [ main, 4.2, 3.4 ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +jobs: + build: + + # runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12" ] + os: [ "ubuntu-latest" ] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest pytest-cov + + - name: pytest + run: | + env PYTHONPATH=src exabgp_log_enable=false pytest --cov --cov-reset ./tests/*_test.py diff --git a/qa/bin/functional b/qa/bin/functional index 764618586..58c6d0e1c 100755 --- a/qa/bin/functional +++ b/qa/bin/functional @@ -98,6 +98,7 @@ class Exec(object): self.code = -1 self.stdout = b'' self.stderr = b'' + self.message = '' self._process = None self.command = [] @@ -130,6 +131,7 @@ class Exec(object): print(f'return: {self.code}') print(f'stdout: {self.stdout}') print(f'stderr: {self.stderr}') + print(f'message: {self.message}') return False def collect(self): @@ -142,13 +144,15 @@ class Exec(object): signal.signal(signal.SIGALRM, alarm_handler) try: - signal.alarm(5) + signal.alarm(15) self.stdout, self.stderr = self._process.communicate() self.code = self._process.returncode signal.alarm(0) - except ValueError: # I/O operation on closed file + except ValueError as exc: # I/O operation on closed file + self.message = str(exc) pass except Alarm: + self.message = str(exc) pass def terminate(self): @@ -327,7 +331,7 @@ class EncodingTests(Tests): if self._check in self.stderr: return True - return self.failed() + return self.failed('completed successfully') API = re.compile(r'^\s*run\s+(.*)\s*?;\s*?$') @@ -498,10 +502,10 @@ class DecodingTests(Tests): def success(self): self.collect() - if not self.stdout: - return self.failed() if self.stderr: - return self.failed() + return self.failed('stderr is \n' + self.stderr) + if not self.stdout: + return self.failed('no stdout received') try: decoded = json.loads(self.stdout) self._cleanup(decoded) @@ -601,7 +605,7 @@ class ParsingTests(Tests): def success(self): self.collect() if self.code != 0: - return self.failed() + return self.failed('return code is not zero') return self.code == 0 diff --git a/qa/bin/functional-3.6 b/qa/bin/functional-3.6 new file mode 100755 index 000000000..8806b820c --- /dev/null +++ b/qa/bin/functional-3.6 @@ -0,0 +1,526 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +""" +cache.py + +Created by Thomas Mangin +Copyright (c) 2013-2017 Exa Networks. All rights reserved. +License: 3-clause BSD. (See the COPYRIGHT file) +""" + +import os +import re +import sys +import glob +import time +import signal +import argparse +import itertools +import subprocess + +PROGRAM = os.path.realpath(__file__) +ROOT = os.path.abspath(os.path.join(os.path.dirname(PROGRAM), os.path.join('..', '..'))) +LIBRARY = os.path.join(ROOT, 'src') + +EXPLAIN = """ +ExaBGP command line +======================================================= + +%(client)s + + +bgp daemon command line +======================================================= + +%(server)s + + +The following extra configuration options could be used +======================================================= + +export exabgp_debug_rotate=true +export exabgp_debug_defensive=true +""" + + +class Color(object): + NONE = '\033[0m' + '\033[0m' + ' ' # NONE + STARTING = '\033[0m' + '\033[96m' + '~' # LIGHT BLUE + READY = '\033[0m' + '\033[94m' + '=' # PENDING + FAIL = '\033[0m' + '\033[91m' + '-' # RED + SUCCESS = '\033[1m' + '\033[92m' + '+' # GREEN + + +class Identifier(dict): + _listing = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzαβγδεζηθικλμνξοπρςστυφχψω' + _from_name = {} + _from_nick = {} + _next = 0 + _nl = 3 + + @classmethod + def get(cls, name): + letter = cls._listing[cls._next] + cls._from_name[name] = letter + cls._from_nick[letter] = name + cls._next += 1 + return letter + + @classmethod + def identifiers(cls): + for n in range(0, cls._next): + yield cls._listing[n], not (n + 1) % cls._nl + + @classmethod + def nick(cls, name): + return cls._from_name[name] + + @classmethod + def name(cls, nick): + return cls._from_nick[nick] + + +class Port(object): + base = 1790 + + @classmethod + def get(cls): + current = cls.base + cls.base += 1 + return current + + +class Path(object): + ETC = os.path.join(ROOT, 'etc', 'exabgp') + EXABGP = os.path.join(ROOT, 'sbin', 'exabgp') + BGP = os.path.join(ROOT, 'qa', 'sbin', 'bgp-3.6') + CI = os.path.join(os.path.join(ROOT, 'qa', 'encoding')) + ALL_CI = glob.glob(os.path.join(CI, '*.ci')) + ALL_CI.sort() + + @classmethod + def validate(cls): + if not os.path.isdir(cls.ETC): + sys.exit('could not find etc folder') + + if not os.path.isdir(cls.CI): + sys.exit('could not find tests in the qa/ci folder') + + if not os.path.isfile(cls.EXABGP): + sys.exit('could not find exabgp') + + if not os.path.isfile(cls.BGP): + sys.exit('could not find the sequence daemon') + + +class CI(dict): + API = re.compile(r'^\s*run\s+(.*)\s*?;\s*?$') + _content = {} + _status = {} + _tests = [] + + @classmethod + def make(cls): + for filename in Path.ALL_CI: + name, extension = os.path.splitext(filename.split('/')[-1]) + if name in ['api-reload', 'api-notification', 'conf-ebgp', 'conf-ipself6']: + continue + nick = Identifier.get(name) + with open(filename, 'r') as reader: + content = reader.readline() + cls._content[nick] = { + 'name': name, + 'confs': [os.path.join(Path.ETC, _) for _ in content.split()], + 'ci': os.path.join(Path.CI, name) + '.ci', + 'msg': os.path.join(Path.CI, name) + '.msg', + 'port': Port.get(), + } + cls._tests.extend(sorted(cls._content.keys())) + + @classmethod + def get(cls, k): + return cls._content.get(k, None) + + @classmethod + def state(cls, name): + if name not in cls._status: + cls._status[name] = Color.NONE + elif cls._status[name] == Color.NONE: + cls._status[name] = Color.STARTING + elif cls._status[name] == Color.STARTING: + cls._status[name] = Color.READY + + @classmethod + def color(cls, name): + return cls._status.get(name, Color.NONE) + + @classmethod + def reset(cls, name): + cls._status[name] = Color.NONE + + @classmethod + def passed(cls, name): + cls._status[name] = Color.SUCCESS + + @classmethod + def failed(cls, name): + cls._status[name] = Color.FAIL + + @classmethod + def files(cls, k): + test = cls._content.get(k, None) + if not test: + return [] + files = [ + test['msg'], + ] + for f in test['confs']: + files.append(f) + with open(f) as reader: + for line in reader: + found = cls.API.match(line) + if not found: + continue + name = found.group(1) + if not name.startswith('/'): + name = os.path.abspath(os.path.join(Path.ETC, name)) + if name not in files: + files.append(name) + return [f for f in files if os.path.isfile(f)] + + @classmethod + def display(cls): + # sys.stdout.write('\r') + for k in cls._tests: + sys.stdout.write('%s%s ' % (CI.color(k), k)) + sys.stdout.write(Color.NONE) + # same line printing now buggy + sys.stdout.write('\r') + sys.stdout.flush() + + @classmethod + def listing(cls): + sys.stdout.write('\n') + sys.stdout.write('The available functional tests are:\n') + sys.stdout.write('\n') + for index, nl in Identifier.identifiers(): + name = cls._content[index]['name'] + sys.stdout.write(' %-2s %s%s' % (index, name, ' ' * (25 - len(name)))) + sys.stdout.write('\n' if nl else '') + sys.stdout.write('\n') + sys.stdout.write('\n') + sys.stdout.write('\n') + sys.stdout.write('checking\n') + sys.stdout.write('\n') + sys.stdout.flush() + + +Path.validate() +CI.make() +# CI.display() + + +class Alarm(Exception): + pass + + +def alarm_handler(number, frame): # pylint: disable=W0613 + raise Alarm() + + +class Process(object): + _running = {} + _result = {} + + @classmethod + def add(cls, name, side, process): + cls._running.setdefault(name, {})[side] = process + for std in ('in', 'out'): + cls._result.setdefault(name, {}).setdefault(side, {})[std] = b'' + + @classmethod + def success(cls, name): + return b'successful' in cls._result[name]['server']['out'] + + @classmethod + def _ready(cls, side, name): + try: + signal.alarm(1) + polled = cls._running[side][name].poll() + signal.alarm(0) + except Alarm: + return False + except (IOError, OSError, ValueError): + return True + if polled is None: + return False + return True + + @classmethod + def collect(cls, name, side): + try: + signal.alarm(1) + stdout, stderr = cls._running[name][side].communicate() + signal.alarm(0) + cls._result[name][side]['out'] = stdout + cls._result[name][side]['err'] = stderr + except ValueError: # I/O operation on closed file + pass + except Alarm: + pass + + @classmethod + def output(cls, name, side): + return cls._result[name][side]['out'] + + def error(cls, name, side): + return cls._result[name][side]['err'] + + @classmethod + def _terminate(cls, name, side): + try: + cls._running[name][side].send_signal(signal.SIGTERM) + except OSError: # No such process, Errno 3 + pass + + @classmethod + def terminate(cls): + for name in cls._running: + for side in cls._running[name]: + if cls.output(name, side) != '' or cls.error(name, side) != '': + continue + cls._terminate(name, side) + cls.collect(name, side) + + +class Command(dict): + @staticmethod + def execute(cmd): + print('starting: %s' % ' '.join(cmd)) + popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + for line in itertools.chain(iter(popen.stdout.readline, ''), iter(popen.stderr.readline, '')): + yield line + popen.stdout.close() + return_code = popen.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, cmd) + + @classmethod + def explain(cls, index): + print( + EXPLAIN % {'client': cls.client(index), 'server': cls.server(index),} + ) + sys.exit(1) + + @staticmethod + def client(index): + test = CI.get(index) + if not test: + sys.exit("can not find any test called '%s'" % index) + + if os.getuid() and os.getgid() and test['port'] <= 1024: + sys.exit('you need to have root privileges to bind to port 79') + + config = { + 'env': ' \\\n '.join( + [ + 'exabgp_tcp_once=true', + 'exabgp_api_cli=false', + 'exabgp_debug_rotate=true', + 'exabgp_debug_configuration=true', + 'exabgp_tcp_bind=\'\'', + 'exabgp_tcp_port=%d' % test['port'], + 'INTERPRETER=%s ' % os.environ.get('__PYVENV_LAUNCHER__', sys.executable), + ] + ), + 'exabgp': Path.EXABGP, + 'confs': ' \\\n '.join(test['confs']), + } + return 'env \\\n %(env)s \\\n %(exabgp)s -d -p \\\n %(confs)s' % config + + @staticmethod + def server(index): + test = CI.get(index) + + if not test: + sys.exit("can not find any test called '%s'" % index) + + if os.getuid() and os.getgid() and test['port'] <= 1024: + sys.exit('you need to have root privileges to bind to port 79') + + config = { + 'env': ' \\\n '.join(['exabgp_tcp_port=%d' % test['port'],]), + 'interpreter': os.environ.get('__PYVENV_LAUNCHER__', sys.executable), + 'bgp': Path.BGP, + 'msg': test['msg'], + } + + return 'env \\\n %(env)s \\\n %(interpreter)s %(bgp)s \\\n %(msg)s' % config + + @staticmethod + def dispatch(running, timeout): + completed = True + names = [] + for name in running: + if CI.get(name) is None: + sys.exit("can not find any test called '%s'" % name) + CI.state(name) + names.append(name) + + for side in ['server', 'client']: + for name in running: + process = subprocess.Popen( + [sys.argv[0], side, name, '--port', str(CI.get(name)['port'])], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + Process.add(name, side, process) + CI.state(name) + CI.display() + time.sleep(0.02) + + exit_time = time.time() + timeout + + while names and time.time() < exit_time: + CI.display() + for name in list(names): + for side in ('server', 'client'): + if not Process._ready(name, side): + continue + + Process.collect(name, side) + + if side == 'server': + names.remove(name) + + if Process.success(name): + CI.passed(name) + else: + CI.failed(name) + completed = False + + CI.display() + time.sleep(0.2) + + Process.terminate() + + for name in names: + print('server stderr\n------\n%s' % str(Process.output(name, 'server')).replace('\\n', '\n')) + print('client stdout\n------\n%s' % str(Process.output(name, 'client')).replace('\\n', '\n')) + + CI.display() + return completed + + +def _run(to_run, chunk, timeout): + success = True + while to_run and success: + running, to_run = to_run[:chunk], to_run[chunk:] + success = Command.dispatch(running, timeout) + sys.stdout.write('\n') + + sys.exit(0 if success else 1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='The BGP swiss army knife of networking functional testing tool') + subparsers = parser.add_subparsers() + + def all(parsed): + to_run = [index for index, _ in Identifier.identifiers()] + chunk = 1 + _run(to_run, chunk, parsed.timeout) + + sub = subparsers.add_parser('all', help='run all available test') + sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) + sub.add_argument('--port', help='base port to use', type=int, default=1790) + sub.set_defaults(func=all) + + def run(parsed): + Port.base = parsed.port + if parsed.test: + to_run = [ + parsed.test, + ] + else: + to_run = [index for index, _ in Identifier.identifiers()] + chunk = len(to_run) if not parsed.steps else parsed.steps + _run(to_run, chunk, parsed.timeout) + + sub = subparsers.add_parser('run', help='run a particular test') + sub.add_argument('test', help='name of the test to run', nargs='?', default=None) + sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) + sub.add_argument('--port', help='base port to use', type=int, default=1790) + sub.add_argument('--steps', help='number of test to run simultaneously', type=int, default=0) + sub.set_defaults(func=run) + + def client(parsed): + command = Command.client(parsed.test) + print(f'> {command}') + if not parsed.dry: + sys.exit(os.system(command)) + sys.exit(0) + + sub = subparsers.add_parser('client', help='start the client for a specific test') + sub.add_argument('test', help='name of the test to run') + sub.add_argument('-d', '--dry', help='show what command would be run but does nothing', action='store_true') + sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) + sub.add_argument('--port', help='base port to use', type=int, default=1790) + sub.set_defaults(func=client) + + def server(parsed): + command = Command.server(parsed.test) + print(f'> {command}') + if not parsed.dry: + sys.exit(os.system(command)) + sys.exit(0) + + sub = subparsers.add_parser('server', help='start the server for a specific test') + sub.add_argument('test', help='name of the test to run') + sub.add_argument('-d', '--dry', help='show what command would be run but does nothing', action='store_true') + sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) + sub.add_argument('--port', help='base port to use', type=int, default=1790) + sub.set_defaults(func=server) + + def explain(parsed): + Command.explain(parsed.test) + sys.exit(0) + + sub = subparsers.add_parser('explain', help='show what command for a test are run') + sub.add_argument('test', help='name of the test to explain') + sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) + sub.add_argument('--port', help='base port to use', type=int, default=1790) + sub.set_defaults(func=explain) + + def edit(parsed): + files = CI.files(parsed.test) + if not files: + sys.exit('no such test') + editor = os.environ.get('EDITOR', 'vi') + os.system('%s %s' % (editor, ' '.join(files))) + sys.exit(0) + + sub = subparsers.add_parser('edit', help='start $EDITOR to edit a specific test') + sub.add_argument('test', help='name of the test to edit') + sub.set_defaults(func=edit) + + def decode(parsed): + test = CI.get(parsed.test) + command = '%s decode %s "%s"' % (Path.EXABGP, test['confs'][0], ''.join(parsed.payload)) + print('> %s' % command) + os.system(command) + sys.exit(0) + + sub = subparsers.add_parser('decode', help='use the test configuration to decode a packet') + sub.add_argument('test', help='name of the test to use to know the BGP configuration') + sub.add_argument('payload', nargs='+', help='the hexadecimal representation of the packet') + sub.set_defaults(func=decode) + + sub = subparsers.add_parser('listing', help='list all functional test available') + sub.set_defaults(func=lambda _: CI.listing()) + + parsed = parser.parse_args() + if vars(parsed): + parsed.func(parsed) + else: + parser.print_help() \ No newline at end of file diff --git a/qa/sbin/bgp-3.6 b/qa/sbin/bgp-3.6 new file mode 100755 index 000000000..b0f42f63e --- /dev/null +++ b/qa/sbin/bgp-3.6 @@ -0,0 +1,592 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +""" +bgp + +Created by Thomas Mangin +Copyright (c) 2013-2017 Exa Networks. All rights reserved. +License: 3-clause BSD. (See the COPYRIGHT file) +""" + +import os +import pwd +import sys +import time +import errno +import socket +import threading +import signal +import asyncore +import subprocess +from struct import unpack + +SIGNAL = dict([(name, getattr(signal, name)) for name in dir(signal) if name.startswith('SIG')]) + + +def flushed(*output): + print(' '.join(str(_) for _ in output)) + sys.stdout.flush() + + +def bytestream(value): + return ''.join(['%02X' % _ for _ in value]) + + +def dump(value): + def spaced(value): + even = None + for v in value: + if even is False: + yield ' ' + yield '%02X' % v + even = not even + + return ''.join(spaced(value)) + + +def cdr_to_length(cidr): + if cidr > 24: + return 4 + if cidr > 16: + return 3 + if cidr > 8: + return 2 + if cidr > 0: + return 1 + return 0 + + +class BGPHandler(asyncore.dispatcher_with_send): + counter = 0 + + keepalive = bytearray([0xFF,] * 16 + [0x0, 0x13, 0x4]) + + _name = { + b'\x01': 'OPEN', + b'\x02': 'UPDATE', + b'\x03': 'NOTIFICATION', + b'\x04': 'KEEPALIVE', + } + + def signal(self, myself, signal_name='SIGUSR1'): + signal_number = SIGNAL.get(signal_name, '') + if not signal_number: + self.announce('invalid signal name in configuration : %s' % signal_name) + self.announce('options are: %s' % ','.join(SIGNAL.keys())) + sys.exit(1) + + conf_name = sys.argv[1].split('/')[-1].split('.')[0] + + processes = [] + + for line in os.popen("/bin/ps x"): + low = line.strip().lower() + if not low: + continue + if 'python' not in low and 'pypy' not in low: + continue + + cmdline = line.strip().split()[4:] + pid = line.strip().split()[0] + + if len(cmdline) > 1 and not cmdline[1].endswith('/bgp.py'): + continue + + if conf_name not in cmdline[-1]: + continue + + if not cmdline[-1].endswith('.conf'): + continue + + processes.append(pid) + + if len(processes) == 0: + self.announce('no running process found, this should not happend, quitting') + sys.exit(1) + + if len(processes) > 1: + self.announce('more than one process running, this should not happend, quitting') + sys.exit(1) + + try: + self.announce('sending signal %s to ExaBGP (pid %s)\n' % (signal_name, processes[0])) + os.kill(int(processes[0]), signal_number) + except Exception as exc: + self.announce('\n failed: %s' % str(exc)) + + def kind(self, header): + return header[18] + + def isupdate(self, header): + return header[18] == 2 + + def isnotification(self, header): + return header[18] == 4 + + def name(self, header): + return self._name.get(header[18], 'SOME WEIRD RFC PACKET') + + def routes(self, header, body): + len_w = unpack('!H', body[0:2])[0] + withdrawn = bytearray([_ for _ in body[2 : 2 + len_w]]) + len_a = unpack('!H', body[2 + len_w : 2 + len_w + 2])[0] + announced = bytearray([_ for _ in body[2 + len_w + 2 + len_a :]]) + + if not withdrawn and not announced: + if len(body) == 4: + yield 'eor:1:1' + elif len(body) == 11: + yield 'eor:%d:%d' % (body[-2], body[-1]) + else: # undecoded MP route + yield 'mp:' + return + + while withdrawn: + cdr, withdrawn = withdrawn[0], withdrawn[1:] + size = cdr_to_length(cdr) + r = [0, 0, 0, 0] + for index in range(size): + r[index], withdrawn = withdrawn[0], withdrawn[1:] + yield 'withdraw:%s' % '.'.join(str(_) for _ in r) + '/' + str(cdr) + + while announced: + cdr, announced = announced[0], announced[1:] + size = cdr_to_length(cdr) + r = [0, 0, 0, 0] + for index in range(size): + r[index], announced = announced[0], announced[1:] + yield 'announce:%s' % '.'.join(str(_) for _ in r) + '/' + str(cdr) + + def notification(self, header, body): + yield 'notification:%d,%d' % (body[0], body[1]), bytestream(body) + + def announce(self, *args): + flushed(' ', self.ip, self.port, ' '.join(str(_) for _ in args) if len(args) > 1 else args[0]) + + def check_signal(self): + if self.messages and self.messages[0].startswith('signal:'): + name = self.messages.pop(0).split(':')[-1] + self.signal(os.getppid(), name) + + def setup(self, ip, port, messages, options): + self.ip = ip + self.port = port + self.options = options + self.handle_read = self.handle_open + self.sequence = {} + self.raw = False + for rule in messages: + sequence, announcement = rule.split(':', 1) + if announcement.startswith('raw:'): + self.raw = True + announcement = ''.join(announcement[4:].replace(':', '')) + self.sequence.setdefault(sequence, []).append(announcement) + self.update_sequence() + return self + + def update_sequence(self): + if self.options['sink'] or self.options['echo']: + self.messages = [] + return True + keys = sorted(list(self.sequence)) + if keys: + key = keys[0] + self.messages = self.sequence[key] + self.step = key + del self.sequence[key] + + self.check_signal() + # we had a list with only one signal + if not self.messages: + return self.update_sequence() + return True + return False + + def read_message(self): + header = b'' + while len(header) != 19: + try: + left = 19 - len(header) + header += self.recv(left) + if left == 19 - len(header): # ugly + # the TCP session is gone. + return None, None + except socket.error as exc: + if exc.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): + continue + raise exc + + length = unpack('!H', header[16:18])[0] - 19 + + body = b'' + while len(body) != length: + try: + left = length - len(body) + body += self.recv(left) + except socket.error as exc: + if exc.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): + continue + raise exc + + return bytearray(header), bytearray(body) + + def handle_open(self): + # reply with a IBGP response with the same capability (just changing routerID) + header, body = self.read_message() + routerid = bytearray([body[8] + 1 & 0xFF]) + o = header + body[:8] + routerid + body[9:] + + if self.options['send-unknown-capability']: + # hack capability 66 into the message + + content = b'loremipsum' + cap66 = bytearray([66, len(content)]) + content + param = bytearray([2, len(cap66)]) + cap66 + o = o[:17] + bytearray([o[17] + len(param)]) + o[18:28] + bytearray([o[28] + len(param)]) + o[29:] + param + + self.send(o) + self.send(self.keepalive) + + if self.options['send-default-route']: + self.send( + bytearray( + [0xFF,] * 16 + + [0x00, 0x31] + + [0x02,] + + [0x00, 0x00] + + [0x00, 0x15] + + [] + + [0x40, 0x01, 0x01, 0x00] + + [] + + [0x40, 0x02, 0x00] + + [] + + [0x40, 0x03, 0x04, 0x7F, 0x00, 0x00, 0x01] + + [] + + [0x40, 0x05, 0x04, 0x00, 0x00, 0x00, 0x64] + + [0x20, 0x00, 0x00, 0x00, 0x00] + ) + ) + self.announce('sending default-route\n') + + self.handle_read = self.handle_keepalive + + def handle_keepalive(self): + header, body = self.read_message() + + if header is None: + self.announce('connection closed') + self.close() + if self.options['send-notification']: + self.announce('successful') + sys.exit(0) + return + + if self.raw: + + def parser(self, header, body): + if body: + yield bytestream(header + body) + + else: + parser = self._decoder.get(self.kind(header), None) + + if self.options['sink']: + self.announce( + 'received %d: %s' + % ( + self.counter, + '%s:%s:%s:%s' + % (bytestream(header[:16]), bytestream(header[16:18]), bytestream(header[18:]), bytestream(body)), + ) + ) + self.send(self.keepalive) + return + + if self.options['echo']: + self.announce( + 'received %d: %s' + % ( + self.counter, + '%s:%s:%s:%s' + % (bytestream(header[:16]), bytestream(header[16:18]), bytestream(header[18:]), bytestream(body)), + ) + ) + self.send(header + body) + self.announce( + 'sent %d: %s' + % ( + self.counter, + '%s:%s:%s:%s' + % (bytestream(header[:16]), bytestream(header[16:18]), bytestream(header[18:]), bytestream(body)), + ) + ) + return + + if parser: + for announcement in parser(self, header, body): + self.send(self.keepalive) + if announcement.startswith('eor:'): # skip EOR + self.announce('skipping eor', announcement) + continue + + if announcement.startswith('mp:'): # skip unparsed MP + self.announce('skipping multiprotocol :', dump(body)) + continue + + self.counter += 1 + + if announcement in self.messages: + self.messages.remove(announcement) + if self.raw: + self.announce( + 'received %d (%1s%s):' % (self.counter, self.options['letter'], self.step), + '%s:%s:%s:%s' + % (announcement[:32], announcement[32:36], announcement[36:38], announcement[38:]), + ) + else: + self.announce( + 'received %d (%1s%s):' % (self.counter, self.options['letter'], self.step), announcement + ) + self.check_signal() + else: + if self.raw: + self.announce( + 'received %d (%1s%s):' % (self.counter, self.options['letter'], self.step), + '%s:%s:%s:%s' + % ( + bytestream(header[:16]), + bytestream(header[16:18]), + bytestream(header[18:]), + bytestream(body), + ), + ) + else: + self.announce('received %d :' % self.counter, announcement) + + if len(self.messages) > 1: + self.announce('expected one of the following :') + for message in self.messages: + if message.startswith('F' * 32): + self.announce( + ' %s:%s:%s:%s' + % (message[:32], message[32:36], message[36:38], message[38:]) + ) + else: + self.announce(' %s' % message) + elif self.messages: + message = self.messages[0].upper() + if message.startswith('F' * 32): + self.announce('expected : %s:%s:%s:%s' % (message[:32], message[32:36], message[36:38], message[38:])) + else: + self.announce('expected : %s' % message) + else: + # can happen when the thread is still running + self.announce('extra data') + sys.exit(1) + + sys.exit(1) + + if not self.messages: + if self.options['single-shot']: + self.announce('successful (partial test)') + sys.exit(0) + + if not self.update_sequence(): + if self.options['exit']: + self.announce('successful') + sys.exit(0) + else: + self.send(self.keepalive) + + if self.options['send-notification']: + notification = b'closing session because we can' + self.send( + bytearray([0xFF,] * 16 + [0x00, 19 + 2 + len(notification)] + [0x03] + [0x06] + [0x00]) + notification + ) + + _decoder = { + 2: routes, + 3: notification, + } + + +class BGPServer(asyncore.dispatcher): + def announce(self, *args): + flushed(' ' + ' '.join(str(_) for _ in args) if len(args) > 1 else args[0]) + + def __init__(self, host, options): + asyncore.dispatcher.__init__(self) + + if ':' in host: + self.create_socket(socket.AF_INET6, socket.SOCK_STREAM) + else: + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.set_reuse_addr() + self.bind((host, options['port'])) + self.listen(5) + + self.messages = {} + + self.options = { + 'send-unknown-capability': False, # add an unknown capability to the open message + 'send-default-route': False, # send a default route to the peer + 'send-notification': False, # send notification messages to the backend + 'signal-SIGUSR1': 0, # send SIGUSR1 after X seconds + 'single-shot': False, # we can not test signal on python 2.6 + 'sink': False, # just accept whatever is sent + 'echo': False, # just accept whatever is sent + } + self.options.update(options) + + for message in options['messages']: + if message.strip() == 'option:open:send-unknown-capability': + self.options['send-unknown-capability'] = True + continue + if message.strip() == 'option:update:send-default-route': + self.options['send-default-route'] = True + continue + if message.strip() == 'option:notification:send-notification': + self.options['send-notification'] = True + continue + if message.strip().startswith('option:SIGUSR1:'): + + def notify(delay, myself): + time.sleep(delay) + self.signal(myself) + time.sleep(10) + + # Python 2.6 can not perform this test as it misses the function + if 'check_output' in dir(subprocess): + # thread.start_new_thread(notify,(int(message.split(':')[-1]),os.getpid())) + threading.Thread(target=notify, args=(int(message.split(':')[-1]), os.getpid())) + else: + self.options['single-shot'] = True + continue + + if message[0].isalpha(): + index, content = message[:1].upper(), message[1:] + else: + index, content = 'A', message + self.messages.setdefault(index, []).append(content) + + def handle_accept(self): + messages = None + for number in range(ord('A'), ord('Z') + 1): + letter = chr(number) + if letter in self.messages: + messages = self.messages[letter] + del self.messages[letter] + break + + if self.options['sink']: + flushed('\nsink mode - send us whatever, we can take it ! :p\n') + messages = [] + elif self.options['echo']: + flushed('\necho mode - send us whatever, we can parrot it ! :p\n') + messages = [] + elif not messages: + self.announce('we used all the test data available, can not handle this new connection') + sys.exit(1) + else: + flushed('using :\n ', '\n '.join(messages), '\n\nconversation:\n') + + self.options['exit'] = not len(self.messages.keys()) + self.options['letter'] = letter + + pair = self.accept() + if pair is not None: + sock, addr = pair + handler = BGPHandler(sock).setup(*addr[:2], messages=messages, options=self.options) + + +def drop(): + uid = os.getuid() + gid = os.getgid() + + if uid and gid: + return + + for name in [ + 'nobody', + ]: + try: + user = pwd.getpwnam(name) + nuid = int(user.pw_uid) + ngid = int(user.pw_uid) + except KeyError: + pass + + if not gid: + os.setgid(ngid) + if not uid: + os.setuid(nuid) + + +def main(): + port = os.environ.get('exabgp.tcp.port', os.environ.get('exabgp_tcp_port', '179')) + + if not port.isdigit() and port > 0 and port <= 65535 or len(sys.argv) <= 1: + flushed('--sink accept any BGP messages and reply with a keepalive') + flushed('--echo accept any BGP messages send it back to the emiter') + flushed('--port port to bind to') + flushed( + 'a list of expected route announcement/withdrawl in the format :announce: :withdraw: :raw:' + ) + flushed('for example:', sys.argv[0], '1:announce:10.0.0.0/8 1:announce:192.0.2.0/24 2:withdraw:10.0.0.0/8 ') + flushed('routes with the same can arrive in any order') + sys.exit(1) + + options = {'sink': False, 'echo': False, 'port': int(port), 'messages': []} + + for arg in sys.argv[1:]: + if arg == '--sink': + messages = [] + options['sink'] = True + continue + + if arg == '--echo': + messages = [] + options['echo'] = True + continue + + if arg == '--port': + args = sys.argv[1:] + [ + '', + ] + port = args[args.index('--port') + 1] + if port.isdigit() and int(port) > 0: + options['port'] = int(port) + continue + print('invalid port %s' % port) + sys.exit(1) + + if arg == str(options['port']): + continue + + try: + with open(sys.argv[1]) as content: + options['messages'] = [_.strip() for _ in content.readlines() if _.strip() and '#' not in _] + except IOError: + flushed('could not open file', sys.argv[1]) + sys.exit(1) + + try: + BGPServer('127.0.0.1', options) + try: + BGPServer('::1', options) + except: + # does not work on travis-ci + pass + drop() + asyncore.loop() + except socket.error as exc: + if exc.errno == errno.EACCES: + flushed('failure: could not bind to port %s - most likely not run as root' % port) + elif exc.errno == errno.EADDRINUSE: + flushed('failure: could not bind to port %s - port already in use' % port) + else: + flushed('failure', str(exc)) + + +if __name__ == '__main__': + main() diff --git a/src/exabgp/cli/command.py b/src/exabgp/cli/command.py index 0ba2dffd9..4e13b38a6 100644 --- a/src/exabgp/cli/command.py +++ b/src/exabgp/cli/command.py @@ -6,6 +6,11 @@ from vyos.util import call from vyos.util import popen +if sys.version_info[:3] < (3,7): + def breakpoint(): + import pdb; + pdb.set_trace() + pass def _nop(config, path): order = path[0] diff --git a/src/exabgp/conf/yang/code.py b/src/exabgp/conf/yang/code.py index 5a0f8a4d5..bdd9f2e8e 100644 --- a/src/exabgp/conf/yang/code.py +++ b/src/exabgp/conf/yang/code.py @@ -22,6 +22,12 @@ from exabgp.conf.yang.datatypes import kw from exabgp.conf.yang.datatypes import ranges +import sys +if sys.version_info[:3] < (3,7): + def breakpoint(): + import pdb; + pdb.set_trace() + pass class Code(object): def __init__(self, tree): diff --git a/src/exabgp/conf/yang/parser.py b/src/exabgp/conf/yang/parser.py index 52f65b5af..f95525955 100644 --- a/src/exabgp/conf/yang/parser.py +++ b/src/exabgp/conf/yang/parser.py @@ -16,6 +16,12 @@ from exabgp.conf.yang.datatypes import words from exabgp.conf.yang.datatypes import types +import sys +if sys.version_info[:3] < (3,7): + def breakpoint(): + import pdb; + pdb.set_trace() + pass DEBUG = True