diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a8b4da4f2..3ae839035 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,7 +43,7 @@ jobs: - name: Configuration Parsing Checks run: | - ./qa/bin/parsing + ./qa/bin/functional parsing - name: Functional Checks run: | diff --git a/README.md b/README.md index 9e43d8076..bc69ffa73 100644 --- a/README.md +++ b/README.md @@ -285,18 +285,18 @@ If you want to check any code changes, the repository comes with a `qa` folder, ExaBGP comes with a set of functional tests, each test starts an IBGP daemon expecting a number of per recorded UPDATEs for the matching configuration file. -You can see all the existing tests running `./qa/bin/functional listing`. Each test is numbered and can be run independently (please note that 03 is not the same as 3). +You can see all the existing tests running `./qa/bin/functional encoding --list`. Each test is numbered and can be run independently (please note that 03 is not the same as 3). ```sh -# ./qa/bin/functional encoding # (run all the test) -# ./qa/bin/functional encoding 03 # (run test 03 as reported by listing) +# ./qa/bin/functional encoding # (run all the test) +# ./qa/bin/functional encoding A # (run test 03 as reported by listing) ``` You can also manually run both the server and client for any given test: ```sh -shell1# ./qa/bin/functional server 03 -shell2# ./qa/bin/functional client 03 +shell1# ./qa/bin/functional encoding --server A +shell2# ./qa/bin/functional encoding --client A ``` A test suite is also present to complement the functional testing. diff --git a/qa/bin/conversation.serial b/qa/bin/conversation.serial deleted file mode 100755 index 97b404dab..000000000 --- a/qa/bin/conversation.serial +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 -""" -cache.py - -Created by Thomas Mangin -Copyright (c) 2013-2017 Exa Networks. All rights reserved. -License: 3-clause BSD. (See the COPYRIGHT file) -""" - -import os -import sys -import glob -import time -import signal -import subprocess - -from os import path - - -class Alarm(Exception): - pass - - -def alarm_handler(number, frame): # pylint: disable=W0613 - raise Alarm() - - -def main(): - location = path.realpath(__file__) - - exabgp = path.abspath(path.join(location, '..', '..', '..', 'sbin', 'exabgp')) - if not path.isfile(exabgp): - print "could not find exabgp" - - etc = path.abspath(path.join(location, '..', '..', '..', 'etc', 'exabgp')) - if not path.isdir(etc): - sys.exit('could not find etc folder') - - conf = path.abspath(path.join(location, '..', '..', 'ci')) - if not path.isdir(conf): - sys.exit('could not find conf folder') - - sequence_daemon = path.abspath(path.join(location, '..', '..', 'sbin', 'bgp')) - if not path.isfile(sequence_daemon): - sys.exit('could not find the sequence daemon') - - match = '*' if len(sys.argv) < 2 else sys.argv[1].split('/')[-1].split('.')[0] - if match == 'all': - match = '*' - - if len(sys.argv) >= 3: - reverse = reversed if sys.argv[2].startswith('rev') else lambda _: _ - skip = 0 if not sys.argv[2].isdigit() else int(sys.argv[2]) - show = (sys.argv[2] == 'show') - else: - reverse = lambda _: _ - skip = 0 - show = False - - groups = sorted(glob.glob(path.join(conf, '%s.ci' % match.replace('all', '*')))) - groups = reverse(groups) - - success = True - - for group in groups: - if skip: - print 'skiped ', group - skip -= 1 - continue - - configurations = [os.path.join(etc, name) for name in open(group).read().strip().split()] - sequence_file = group.replace('.ci', '.msg') - - if path.isfile(sequence_file): - if check_sequence(group, exabgp, configurations, sequence_daemon, sequence_file, show) == False: - success = False - else: - print "checking %s\nskipped (no sequence data)\n" % group.split('/')[-1] - continue - - sys.exit(0 if success else 1) - - -def check_sequence(group, exabgp, configurations, daemon, fname, show): - os.environ['exabgp.tcp.once'] = 'true' - os.environ['exabgp.debug.defensive'] = 'true' - os.environ['exabgp.debug.rotate'] = 'true' - - timeout = 10240 # seconds - - class Exit(Exception): - pass - - try: - command_daemon = [daemon, fname] - command_exabgp = [exabgp, '-d'] + configurations - - if show: - raise Exit() - - signal.signal(signal.SIGALRM, alarm_handler) - signal.alarm(timeout) - - print "%-15s %s\n" % ('checking', group.split('/')[-1].split('.')[0]), - daemon = subprocess.Popen(command_daemon, stdout=subprocess.PIPE) - time.sleep(5) - exabgp = subprocess.Popen(command_exabgp, stdout=subprocess.PIPE) - - exabgp_output = exabgp.communicate()[0] - daemon_output = daemon.communicate()[0] - except Alarm: - exabgp_output = '(killed) exabgp was still running\n' - daemon_output = '(killed) still waiting for data after %d seconds\n' % timeout - - for pid in [daemon.pid, exabgp.pid]: - try: - os.kill(pid, signal.SIGTERM) - except OSError: - pass - except Exit: - exabgp_output = '' - daemon_output = '' - - def commands(command_exabgp, command_daemon): - r = "" - r += "command lines are:\n" - r += 'export exabgp_debug_defensive=true\n' - r += ( - "> env exabgp.tcp.port=1790 exabgp.tcp.once=true exabgp.debug.rotate=true exabgp.tcp.bind='' " - + ' '.join(command_exabgp) - + '\n' - ) - r += '> env exabgp.tcp.port=1790 ' + ' '.join(command_daemon) - r += '\n' - return r - - if show: - print commands(command_exabgp, command_daemon) - return None - elif 'successful' in daemon_output: - print "successful\n" - return True - else: - print "failure" - print - print commands(command_exabgp, command_daemon) - print - print 'exabgp:\n', '\n'.join((' %s' % _ for _ in exabgp_output.split('\n'))) - print 'daemon:\n', '\n'.join((' %s' % _ for _ in daemon_output.split('\n'))) - return False - - -def check(main): - os.environ['exabgp.tcp.bind'] = '' - - if os.environ.get('exabgp.tcp.port', '').isdigit(): - port = os.environ.get('exabgp.tcp.port') - elif os.environ.get('exabgp_tcp_port', '').isdigit(): - port = os.environ.get('exabgp_tcp_port') - else: - port = '1790' - - os.environ['exabgp.tcp.port'] = port - os.environ['exabgp_tcp_port'] = port - - uid = os.getuid() - gid = os.getgid() - - port = os.environ['exabgp.tcp.port'] - if not port.isdigit(): - print 'invalid port value %s' % port - - if uid and gid and int(port) <= 1024: - print 'you need to have root privileges to bind to port 79' - sys.exit(1) - main() - - -check(main) diff --git a/qa/bin/functional b/qa/bin/functional index c01467f8a..6f29a7e82 100755 --- a/qa/bin/functional +++ b/qa/bin/functional @@ -56,6 +56,8 @@ class Path: BGP = os.path.join(ROOT, 'qa', 'sbin', 'bgp') CI = os.path.join(os.path.join(ROOT, 'qa', 'ci')) JSON = os.path.join(os.path.join(ROOT, 'qa', 'json')) + ALL_ETC = glob.glob(os.path.join(ETC, '*.conf')) + ALL_ETC.sort() ALL_JSON = glob.glob(os.path.join(JSON, '*')) ALL_JSON.sort() ALL_CI = glob.glob(os.path.join(CI, '*.ci')) @@ -101,6 +103,7 @@ class Sequence: class Exec(object): def __init__(self): + self.code = -1 self.stdout = b'' self.stderr = b'' self._process = None @@ -138,12 +141,11 @@ class Exec(object): signal.alarm(2) self.stdout, self.stderr = self._process.communicate() signal.alarm(0) + self.code = self._process.returncode except ValueError: # I/O operation on closed file pass except Alarm: pass - # self.stdout = self.stdout.replace('\\n', '\n') - # self.stderr = self.stderr.replace('\\n', '\n') def terminate(self): try: @@ -225,17 +227,6 @@ class Record: return success -class EncodingTest(Record, Exec): - def __init__(self, nick, name): - Record.__init__(self, nick, name) - Exec.__init__(self) - self._check = b'successful' - - def success(self): - self.collect() - return self._check in self.stdout or self._check in self.stderr - - class Tests: def __init__(self, klass): self.klass = klass @@ -305,10 +296,20 @@ class Tests: class EncodingTests(Tests): + class Test(Record, Exec): + def __init__(self, nick, name): + Record.__init__(self, nick, name) + Exec.__init__(self) + self._check = b'successful' + + def success(self): + self.collect() + return self._check in self.stdout or self._check in self.stderr + API = re.compile(r'^\s*run\s+(.*)\s*?;\s*?$') def __init__(self): - super().__init__(EncodingTest) + super().__init__(self.Test) for filename in Path.ALL_CI: name = os.path.basename(filename)[:-3] @@ -352,7 +353,6 @@ class EncodingTests(Tests): 'client': self.client(index), 'server': self.server(index), }) - sys.exit(1) def client(self, index): test = self.select(index) @@ -400,7 +400,7 @@ class EncodingTests(Tests): self.display() test.run([ sys.argv[0], - 'server', test.nick, + 'encoding', '--server', test.nick, '--port', f'{test.conf["port"]}' ]) time.sleep(0.005) @@ -412,7 +412,7 @@ class EncodingTests(Tests): self.display() client[test.nick] = Exec().run([ sys.argv[0], - 'client', test.nick, + 'encoding', '--client', test.nick, '--port', f'{test.conf["port"]}' ]) time.sleep(0.005) @@ -438,50 +438,49 @@ class EncodingTests(Tests): return success -class DecodingTest(Record, Exec): - def __init__(self, nick, name): - Record.__init__(self, nick, name) - Exec.__init__(self) - - def _cleanup(self, decoded): - decoded.pop('exabgp', None) - decoded.pop('host', None) - decoded.pop('pid', None) - decoded.pop('ppid', None) - decoded.pop('time', None) - decoded.pop('version', None) - return decoded - - def success(self): - self.collect() - if self.stderr: - print('issue, got data on stderr from exabgp') - print(self.command) - print(self.stderr) - return False - if not self.stdout: - print('issue, not data on stdout from exabgp') - print(self.command) - return False - try: - decoded = json.loads(self.stdout) - self._cleanup(decoded) - except Exception: - print('issue, failed to decode the JSON') - print(self.command) - print(self.stdout) - return False - if decoded != self.conf['json']: - print('issue, JSON does not match') - print(decoded) - print(self.conf['json']) - return False - return True - - class DecodingTests(Tests): + class Test(Record, Exec): + def __init__(self, nick, name): + Record.__init__(self, nick, name) + Exec.__init__(self) + + def _cleanup(self, decoded): + decoded.pop('exabgp', None) + decoded.pop('host', None) + decoded.pop('pid', None) + decoded.pop('ppid', None) + decoded.pop('time', None) + decoded.pop('version', None) + return decoded + + def success(self): + self.collect() + if self.stderr: + print('issue, got data on stderr from exabgp') + print(self.command) + print(self.stderr) + return False + if not self.stdout: + print('issue, no data on stdout from exabgp') + print(self.command) + return False + try: + decoded = json.loads(self.stdout) + self._cleanup(decoded) + except Exception: + print('issue, failed to decode the JSON') + print(self.command) + print(self.stdout) + return False + if decoded != self.conf['json']: + print('issue, JSON does not match') + print(decoded) + print(self.conf['json']) + return False + return True + def __init__(self): - super().__init__(DecodingTest) + super().__init__(self.Test) for filename in Path.ALL_JSON: name = os.path.basename(filename).split('.')[0] @@ -495,7 +494,7 @@ class DecodingTests(Tests): expected = reader.readline().strip() decoded = json.loads(expected) test.conf['json'] = test._cleanup(decoded) - test.files.extend(filename) + test.files.append(filename) def listing(self): sys.stdout.write('\n') @@ -528,121 +527,132 @@ class DecodingTests(Tests): return success -if __name__ == '__main__': - Path.validate() - decoding_tests = DecodingTests() - encoding_tests = EncodingTests() +class ParsingTests(Tests): + class Test(Record, Exec): + def __init__(self, nick, name): + Record.__init__(self, nick, name) + Exec.__init__(self) - parser = argparse.ArgumentParser(description='The BGP swiss army knife of networking functional testing tool') - subparsers = parser.add_subparsers() + def success(self): + self.collect() + if self.code != 0: + print('issue, got error code from exabgp') + print(self.command) + return False - def decoding(parsed): - parsed.timeout = 0 - decoding_tests.select(parsed.test) - decoding_tests.run_active(parsed.timeout) - sys.stdout.write('\n') + return self.code == 0 - sub = subparsers.add_parser('decoding', help='run decoding test') - sub.add_argument('test', help='name of the test to run', nargs='?', default=None) - sub.set_defaults(func=decoding) + def __init__(self): + super().__init__(self.Test) - def all(parsed): - encoding_tests.select(parsed.test) - encoding_tests.run_active(parsed.timeout) + for filename in Path.ALL_ETC: + name = os.path.basename(filename).split('.')[0] + test = self.new(name) + test.conf['fname'] = filename + test.files.append(filename) + + def listing(self): + sys.stdout.write('\n') + sys.stdout.write('The available tests are:\n') + sys.stdout.write('\n') + for nick, name, nl in self._iterate(): + sys.stdout.write(f' {nick:2} {name:25}') + sys.stdout.write('\n' if nl else '') sys.stdout.write('\n') + sys.stdout.flush() - sub = subparsers.add_parser('all', help='run encoding test') - sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) - sub.add_argument('--port', help='base port to use', type=int, default=1790) - sub.set_defaults(func=all) + def run_active(self, timeout): + success = True - def encoding(parsed): - encoding_tests.select(parsed.test) - encoding_tests.run_active(parsed.timeout) - sys.stdout.write('\n') + for test in self.selected(): + test.running() + self.display() + test.run([ + Path.EXABGP, + 'validate', '-nrv', test.conf['fname'] + ]) + + for test in self.selected(): + self.display() + success = test.result(test.success()) and success + time.sleep(0.005) + + self.display() + return success - sub = subparsers.add_parser('encoding', help='run a particular test') + +def add_test(subparser, name, tests, extra): + boolean = argparse.BooleanOptionalAction + sub = subparser.add_parser(name, help=f'run {name} test') + if 'dry' in extra: + sub.add_argument('--dry', help='show the action', action=boolean) + if 'server' in extra: + sub.add_argument('--server', help='start the server for a test', action=boolean) + if 'client' in extra: + sub.add_argument('--client', help='start the client for a test', action=boolean) + if 'list' in extra: + sub.add_argument('--list', help='list the files making a test', action=boolean) + if 'edit' in extra: + sub.add_argument('--edit', help='edit the files making a test', action=boolean) + if 'timeout' in extra: + sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) + if 'port' in extra: + sub.add_argument('--port', help='base port to use', type=int, default=1790) sub.add_argument('test', help='name of the test to run', nargs='?', default=None) - sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) - sub.add_argument('--port', help='base port to use', type=int, default=1790) - sub.add_argument('--steps', help='number of test to run simultaneously', type=int, default=0) - sub.set_defaults(func=encoding) - - def client(parsed): - command = encoding_tests.client(parsed.test) - print(f'> {command}') - if not parsed.dry: - sys.exit(os.system(command)) - sys.exit(0) - - sub = subparsers.add_parser('client', help='start the client for a specific test') - sub.add_argument('test', help='name of the test to run') - sub.add_argument('-d', '--dry', help='show what command would be run but does nothing', action='store_true') - sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) - sub.add_argument('--port', help='base port to use', type=int, default=1790) - sub.set_defaults(func=client) - - def server(parsed): - command = encoding_tests.server(parsed.test) - print(f'> {command}') - if not parsed.dry: - sys.exit(os.system(command)) - sys.exit(0) - - sub = subparsers.add_parser('server', help='start the server for a specific test') - sub.add_argument('test', help='name of the test to run') - sub.add_argument('-d', '--dry', help='show what command would be run but does nothing', action='store_true') - sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) - sub.add_argument('--port', help='base port to use', type=int, default=1790) - sub.set_defaults(func=server) - - def explain(parsed): - encoding_tests.explain(parsed.test) - sys.exit(0) - - sub = subparsers.add_parser('explain', help='show what command for a test are run') - sub.add_argument('test', help='name of the test to explain') - sub.add_argument('--timeout', help='timeout for test failure', type=int, default=60) - sub.add_argument('--port', help='base port to use', type=int, default=1790) - sub.set_defaults(func=explain) - - def edit(parsed): - test = encoding_tests.select(parsed.test) or Record('', '') - if not test.files: - sys.exit('no such test') - editor = os.environ.get('EDITOR', 'vi') - os.system('%s %s' % (editor, ' '.join(test.files))) - sys.exit(0) - - sub = subparsers.add_parser('edit', help='start $EDITOR to edit a specific test') - sub.add_argument('test', help='name of the test to edit') - sub.set_defaults(func=edit) - - def decode(parsed): - test = encoding_tests.get(parsed.test) - command = '%s decode %s "%s"' % (Path.EXABGP, test['confs'][0], ''.join(parsed.payload)) - print('> %s' % command) - os.system(command) - sys.exit(0) - - sub = subparsers.add_parser('decode', help='use the test configuration to decode a packet') - sub.add_argument('test', help='name of the test to use to know the BGP configuration') - sub.add_argument('payload', nargs='+', help='the hexadecimal representation of the packet') - sub.set_defaults(func=decode) - - def listing(parsed): - if parsed.test in ('encoding', ''): - EncodingTests().listing() + + def func(parsed): + if 'edit' in extra and parsed.edit: + test = tests.select(parsed.test) or Record('', '') + if not test.files: + sys.exit('no such test') + editor = os.environ.get('EDITOR', 'vi') + command = '%s %s' % (editor, ' '.join(test.files)) + print(f'> {command}') + if not parsed.dry: + sys.exit(os.system(command)) return - if parsed.test == 'decoding': - DecodingTests().listing() + + if 'list' in extra and parsed.list: + tests.listing() + return + + if 'client' in extra and parsed.client: + command = tests.client(parsed.test) + print(f'> {command}') + if not parsed.dry: + sys.exit(os.system(command)) return - print('option [%s]' % parsed.test) - sys.exit('invalid option') - sub = subparsers.add_parser('list', help='list all functional test available') - sub.add_argument('test', help='name of the test to explain (encoding, decoding)') - sub.set_defaults(func=listing) + if 'server' in extra and parsed.server: + command = tests.server(parsed.test) + print(f'> {command}') + if not parsed.dry: + sys.exit(os.system(command)) + return + + if 'timeout' not in parsed: + parsed.timeout = 0 + tests.select(parsed.test) + tests.run_active(parsed.timeout) + sys.stdout.write('\n') + return + + sub.set_defaults(func=func) + + +if __name__ == '__main__': + Path.validate() + + decoding = DecodingTests() + encoding = EncodingTests() + parsing = ParsingTests() + + parser = argparse.ArgumentParser(description='The BGP swiss army knife of networking functional testing tool') + subparser = parser.add_subparsers() + + add_test(subparser, 'decoding', decoding, ['list', 'edit', 'dry', 'timeout', 'port']) + add_test(subparser, 'encoding', encoding, ['list', 'edit', 'dry', 'timeout', 'port', 'server', 'client']) + add_test(subparser, 'parsing', parsing, ['list', 'dry', 'edit']) parsed = parser.parse_args() if vars(parsed): diff --git a/qa/bin/parsing b/qa/bin/parsing deleted file mode 100755 index 41bfa84e5..000000000 --- a/qa/bin/parsing +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh - -set +e - -dirname=`dirname $0` -retval=0 - -case $dirname in - /*) - cd $dirname/../.. > /dev/null - path=`pwd` - cd - > /dev/null - ;; - *) - cd `pwd`/$dirname/../.. > /dev/null - path=`pwd` - cd - > /dev/null - ;; -esac - -export PYTHONPATH=$path/src - -cd $path/etc/exabgp > /dev/null -names=`ls *.conf` -cd - > /dev/null - -for conf in $names -do - printf "%-50s " $conf - result=`$path/sbin/exabgp validate -nrv $path/etc/exabgp/$conf 2>&1` - retcode=$? - problem=`echo $result | grep 'Problem with the configuration file' || true` - - if [ $retcode -eq 0 ] && [ "$problem" = "" ] - then - printf "ok\n" - else - printf "failed\n" - printf "\n" - printf "$path/sbin/exabgp validate -nrvp $path/etc/exabgp/$conf 2>&1" - printf "\n\n" - printf "$result" - printf "\n\n" - retval=1 - fi -done - -exit $retval