From bc61ff86a1a9ea247c096ca1eb05b7f81b422d68 Mon Sep 17 00:00:00 2001 From: Thomas Mangin Date: Mon, 31 Aug 2020 19:42:49 +0100 Subject: [PATCH] integrate yang within exabgp main code --- src/exabgp/conf/__init__.py | 2 + src/exabgp/conf/generate.py | 76 +++ src/exabgp/conf/yang.py | 991 ------------------------------ src/exabgp/conf/yang/__init__.py | 13 + src/exabgp/conf/yang/code.py | 356 +++++++++++ src/exabgp/conf/yang/datatypes.py | 131 ++++ src/exabgp/conf/yang/model.py | 128 ++++ src/exabgp/conf/yang/tree.py | 359 +++++++++++ 8 files changed, 1065 insertions(+), 991 deletions(-) create mode 100644 src/exabgp/conf/__init__.py create mode 100644 src/exabgp/conf/generate.py delete mode 100755 src/exabgp/conf/yang.py create mode 100755 src/exabgp/conf/yang/__init__.py create mode 100644 src/exabgp/conf/yang/code.py create mode 100644 src/exabgp/conf/yang/datatypes.py create mode 100644 src/exabgp/conf/yang/model.py create mode 100644 src/exabgp/conf/yang/tree.py diff --git a/src/exabgp/conf/__init__.py b/src/exabgp/conf/__init__.py new file mode 100644 index 000000000..222558dac --- /dev/null +++ b/src/exabgp/conf/__init__.py @@ -0,0 +1,2 @@ +# from exabgp.conf.config import Config +# from exabgp.conf.local import init_modules diff --git a/src/exabgp/conf/generate.py b/src/exabgp/conf/generate.py new file mode 100644 index 000000000..2981526a8 --- /dev/null +++ b/src/exabgp/conf/generate.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# encoding: utf-8 + +import os +import astunparse + +from exabgp.conf.yang import Tree +from exabgp.conf.yang import Code + + +class Generate(object): + intro = '# yang model structure and validation\n# autogenerate by exabgp\n\n' + variable = '{name} = {data}\n' + + def __init__(self, fname): + self.fname = fname + self.dicts = [] + self.codes = [] + + def add_dict(self, name, data): + self.dicts.append((name, data)) + + def add_code(self, block): + self.codes.append(block) + + def _generate(self): + returned = self.intro + for name, data in self.dicts: + returned += self.variable.format(name=name, data=data) + returned += '\n' + for section in self.codes: + returned += section + returned += '\n' + return returned + + def save(self): + print(f'generating {self.fname}') + with open(self.fname, 'w') as w: + w.write(self._generate()) + + def output(self): + # for name, data in self.dicts: + # pprint.pprint(name) + # pprint.pprint(data) + for section in self.codes: + print(section) + + +def main(): + folder = os.path.dirname(__file__) + data = os.path.join(folder, '..', '..', '..', 'data') + os.chdir(os.path.abspath(data)) + + library = 'yang-library-data.json' + module = 'exabgp' + models = 'models' + fname = '{module}-yang.py' + + gen = Generate(fname) + + tree = Tree(library, models, module).parse() + + gen.add_dict('model', tree) + # gen.output() + + code = Code(tree) + ast = code.generate(module) + block = astunparse.unparse(ast) + gen.add_code(block) + + gen.output() + # # gen.save() + + +if __name__ == "__main__": + main() diff --git a/src/exabgp/conf/yang.py b/src/exabgp/conf/yang.py deleted file mode 100755 index 0437b3551..000000000 --- a/src/exabgp/conf/yang.py +++ /dev/null @@ -1,991 +0,0 @@ -#!/usr/bin/env python3 - -import re -import os -import sys -import json -import decimal -import shutil -import pprint -import urllib.request -from glob import glob - -from pygments.token import Token -from yanglexer import yanglexer - - -DEBUG = True - - -def write(string): - if not string.startswith('\n'): - fill = ' ' * shutil.get_terminal_size().columns - sys.stdout.write(f'\r{fill}\r') - sys.stdout.write(f'{string}') - sys.stdout.flush() - - -class Boolean(int): - def __new__(cls, value): - return int.__new__(cls, value not in ('false', False, 0)) - - def __init__(self, boolean): - self.string = boolean - - def __str__(self): - return self.string - - -class Decimal64(decimal.Decimal): - def __init__(cls, value, frac=0): - raise RuntimeError() - # look at https://github.com/CZ-NIC/yangson/blob/master/yangson/datatype.py#L682 - return super().__init__(decimal.Decimal(value)) - - -class yang: - restriction = { - 'binary': ['length'], - 'bits': ['bit'], - 'boolean': [], - 'decimal64': ['range'], - 'empty': [], - 'enumeration': ['enum'], - 'identityref': [], - 'instance-identifier': ['require-instance'], - 'int8': [], - 'int16': [], - 'int32': [], - 'int64': [], - 'leafref': ['path', 'require-instance'], - 'string': ['pattern', 'length'], - 'uint8': [], - 'uint16': [], - 'uint32': [], - 'uint64': [], - 'union': [], - } - - klass = { - 'binary': ['length'], - 'bits': ['bit'], - 'boolean': Boolean, - 'decimal64': Decimal64, - 'empty': None, - 'enumeration': None, - 'identityref': str, - 'instance-identifier': str, - 'int8': int, - 'int16': int, - 'int32': int, - 'int64': int, - 'leafref': str, - 'string': str, - 'uint8': int, - 'uint16': int, - 'uint32': int, - 'uint64': int, - 'union': None, - } - - types = list(restriction.keys()) - - words = ( - 'extension', - 'yang-version', - 'namespace', - 'prefix', - 'description', - 'import', - 'organization', - 'contact', - 'description', - 'revision', - 'typedef', - 'type', - 'enumeration', - 'range', - 'length', - 'grouping', - 'leaf', - 'leaf-list', - 'enum', - 'default', - 'key', - 'mandatory', - 'refine', - 'uses', - 'list', - 'container', - 'union', - 'value', - 'reference', - 'pattern', - ) - - # the yang keywords - kw = dict((w, f'[{w}]') for w in words) - # the yang module loaded - kw['loaded'] = '[loaded]' - # the root of the configuration - kw['root'] = '[root]' - # strings can be pattern but is assume to be a match otherwise, make it explicit - kw['match'] = '[match]' - - ranges = { - 'int8': (0, pow(2, 8)-1), - 'int16': (0, pow(2, 16)-1), - 'int32': (0, pow(2, 32)-1), - 'int64': (0, pow(2, 64)-1), - 'uint8': (-pow(2, 7), pow(2, 7)-1), - 'uint16': (-pow(2, 7), pow(2, 15)-1), - 'uint32': (-pow(2, 7), pow(2, 31)-1), - 'uint64': (-pow(2, 7), pow(2, 64)-1), - } - - namespaces = { - 'ietf': 'https://raw.githubusercontent.com/YangModels/yang/master/standard/ietf/RFC', - } - - # initialised by load - # this is a namespace / singleton, and should not be used as instance - models = {} - folder = '' - - @classmethod - def load(cls, library, folder): - cls.folder = folder - if not cls.models: - models = json.loads(open(library).read()) - - for m in models['ietf-yang-library:modules-state']['module']: - cls.models[m['name']] = m - - @classmethod - def fetch_models(cls, folder): - print('downloading models') - - for module in cls.models: - cls.fetch_model(folder, module) - - print('done.\n') - - @classmethod - def fetch_model(cls, folder, name): - if not os.path.exists('models'): - os.mkdir('models') - - if name not in cls.models: - sys.exit(f'{module} imported but not defined in yang-library-data.json') - - module = cls.models[name] - - revision = module['revision'] - yang = f'{name}@{revision}.yang' - save = f'{folder}/{name}.yang' - - if 'schema' in module: - url = module['schema'] - - elif 'namespace' in module: - namespace = module['namespace'].split(':') - site = cls.namespaces.get(namespace[1], '') - if not site: - raise RuntimeError('unimplemented namespace case') - - url = f"{site}/{yang}" - else: - raise RuntimeError('unimplemented yang-library case') - - if os.path.exists(save): - write(f'šŸ‘Œ skipping {name} (already downloaded)') - if cls._verify(name, save): - write('\n') - return - - write(f'šŸ‘ļø retrieve {name}@{revision} ({url})') - - try: - urllib.request.urlretrieve(url, save) - # indirect = urllib.request.urlopen(schema).read() - except urllib.error.HTTPError as exc: - write(f'\nšŸ„ŗ failure attempting to retrieve {url}\n{exc}') - return - - if not cls._verify(name, save): - sys.exit(f'\ninvalid yang content for {name}@{revision}') - - write(f'šŸ‘ retrieve {name}@{revision}\n') - - @staticmethod - def _verify(name, save): - # simple but should be enough - write(f'šŸ” checking {name} for correct yaml') - if not open(save).readline().startswith('module'): - write(f'šŸ„µ not-yang {name} does not contain a yang module') - return False - return True - - @classmethod - def clean_models(cls): - print(f'cleaning {cls.folder}') - for file in glob(f'{cls.folder}/*.yang'): - print(f'cleanup: {file}') - os.remove(file) - print('done.\n') - - -class Lexer(object): - ignore = (Token.Text, Token.Comment.Singleline) - - @staticmethod - def formated(string): - returned = '' - for line in string.strip().split('\n'): - line = line.strip() - - if line.endswith('+'): - line = line[:-1].strip() - if line.startswith('+'): - line = line[1:].strip() - - if line and line[0] == line[-1]: - if line[0] in ('"', "'"): - line = line[1:-1] - returned += line - return returned - - def __init__(self, yangfile): - name = yangfile.split('/')[-1].split('.')[0] - # the yang file parsed tokens (configuration block according to the syntax) - self.tokens = [] - # the name of the module being parsed - self.module = '' - # module can declare a "prefix" (nickname), which can be used to make the syntax shorter - self.prefix = '' - # the parsed yang tree - # - at the root are the namespace (the module names) and within - # * a key for all the typedef - # * a key for all the grouping - # * a key for the root of the configuration - # - a key [loaded] with a list of the module loaded (first is the one parsed) - # the names of all the configuration sections - self.tree = {} - # the current namespace (module) we are parsing - self.ns = {} - # where the grouping for this section are stored - self.grouping = {} - # where the typedef for this section are stored - self.typedef = {} - # where the configuration parsed is stored - self.root = {} - self.load(name, yangfile) - - def tokenise(self, name): - lexer = yanglexer.YangLexer() - content = open(name).read() - tokens = lexer.get_tokens(content) - return [(t, n) for (t, n) in tokens if t not in self.ignore] - - def unexpected(self, string): - pprint.pprint(f'unexpected data: {string}') - for t in self.tokens[:15]: - print(t) - breakpoint() - pass - - def pop(self, what=None, expected=None): - token, string = self.tokens[0] - if what is not None and not str(token).startswith(str(what)): - self.unexpected(string) - if expected is not None and string.strip() != expected: - self.unexpected(string) - self.tokens.pop(0) - return string - - def peek(self, position, ponctuation=None): - token, string = self.tokens[position] - # the self includes a last ' ' - if ponctuation and ponctuation != token: - self.unexpected(string) - return token, string.rstrip() - - def skip_keyword_block(self, name): - count = 0 - while True: - t, v = self.tokens.pop(0) - if t != Token.Punctuation: - continue - if v.strip() == '{': - count += 1 - if v.strip() == '}': - count -= 1 - if not count: - break - - def set_subtrees(self): - """ - to make the core more redeable the tree[module] structure - is presented as subtrees, this reset all the subtree - for the current module - """ - self.ns = self.tree[self.module] - self.grouping = self.ns[yang.kw['grouping']] - self.typedef = self.ns[yang.kw['typedef']] - self.root = self.ns[yang.kw['root']] - - def imports(self, module, prefix): - """ - load, and if missing and defined download, a yang module - - module: the name of the yang module to find - prefix: how it is called (prefix) - """ - fname = os.path.join(yang.folder, module) + '.yang' - if not os.path.exists(fname): - yang.fetch_model('models', module) - - backup = (self.tokens, self.module, self.prefix) - self.load(prefix, fname) - self.tokens, self.module, self.prefix = backup - self.set_subtrees() - - def load(self, module, fname): - """ - add a new yang module/namespace to the tree - this _function is used when initialising the - root module, as it does not perform backups - """ - self.tree.setdefault(yang.kw['loaded'], []).append(module) - self.tokens = self.tokenise(fname) - self.module = module - self.prefix = module - self.tree[module] = { - yang.kw['typedef']: {}, - yang.kw['grouping']: {}, - yang.kw['root']: {}, - } - self.set_subtrees() - self.parse() - - def parse(self): - self._parse([], self.root) - return self.tree - - def _parse(self, inside, tree): - while self.tokens: - token, string = self.peek(0) - - if token == Token.Punctuation and string == '}': - # it is clearer to pop it in the caller - return - - self._parse_one(inside, tree, token, string) - - def _parse_one(self, inside, tree, token, string): - if token == Token.Comment.Multiline: - # ignore multiline comments - self.pop(Token.Comment.Multiline) - return - - if token == Token.Keyword.Namespace: - self.pop(Token.Keyword.Namespace, 'module') - self.pop(Token.Literal.String) - self.pop(Token.Punctuation, '{') - self._parse(inside, tree) - self.pop(Token.Punctuation, '}') - return - - if token != Token.Keyword or string not in yang.words: - if ':' not in string: - self.unknown(string, '') - return - - self.pop(Token.Keyword, string) - name = self.formated(self.pop(Token.Literal.String)) - - if string == 'prefix': - self.prefix = name - self.pop(Token.Punctuation, ';') - return - - if string in ('namespace', 'organization', 'contact', 'yang-version'): - self.pop(Token.Punctuation, ';') - return - - if string in ('revision', 'extension'): - self.skip_keyword_block(Token.Punctuation) - return - - if string in ('range', 'length'): - self.pop(Token.Punctuation, ';') - tree[yang.kw[string]] = [_ for _ in name.replace(' ', '').replace('..',' ').split()] - return - - if string == 'import': - token, string = self.peek(0, Token.Punctuation) - if string == ';': - self.pop(Token.Punctuation, ';') - self.imports(name, name) - if string == '{': - self.pop(Token.Punctuation, '{') - self.pop(Token.Keyword, 'prefix') - prefix = self.formated(self.pop(Token.Literal.String)) - self.pop(Token.Punctuation, ';') - self.pop(Token.Punctuation, '}') - self.imports(name, prefix) - return - - if string in ('description', 'reference'): - self.pop(Token.Punctuation, ';') - # XXX: not saved during debugging - if DEBUG: - return - tree[yang.kw[string]] = name - return - - if string in ('pattern', 'value', 'default', 'mandatory'): - self.pop(Token.Punctuation, ';') - tree[yang.kw[string]] = name - return - - if string == 'key': - self.pop(Token.Punctuation, ';') - tree[yang.kw[string]] = name.split() - return - - if string == 'typedef': - self.pop(Token.Punctuation, '{') - sub = self.typedef.setdefault(name, {}) - self._parse(inside + [name], sub) - self.pop(Token.Punctuation, '}') - return - - if string == 'enum': - option = self.pop(Token.Punctuation) - if option == ';': - tree.setdefault(yang.kw[string], []).append(name) - return - if option == '{': - sub = tree.setdefault(name, {}) - self._parse(inside + [name], sub) - self.pop(Token.Punctuation, '}') - return - - if string == 'type': - # make sure the use module name and not prefix, as prefix is not global - if ':' in name: - module, typeref = name.split(':', 1) - if module == self.prefix: - name = f'{self.module}:{typeref}' - module = self.module - - if module not in self.tree: - self.unexpected(f'referenced non-included module {name}') - elif name not in yang.types: - name = f'{self.module}:{name}' - - option = self.pop(Token.Punctuation) - if option == ';': - if name in yang.types: - tree.setdefault(yang.kw[string], {name: {}}) - return - - if ':' not in name: - # not dealing with refine - # breakpoint() - tree.setdefault(yang.kw[string], {name: {}}) - return - - tree.setdefault(yang.kw[string], {name: {}}) - return - - if option == '{': - if name == 'union': - sub = tree.setdefault(yang.kw[string], {}).setdefault(name, []) - while True: - what, name = self.peek(0) - name = self.formated(name) - if name == 'type': - union_type = {} - self._parse_one(inside + [name], union_type, what, name) - sub.append(union_type[yang.kw['type']]) - continue - if name == '}': - self.pop(Token.Punctuation, '}') - break - self.unexpected(f'did not expect this in an union: {what}') - return - - if name == 'enumeration': - sub = tree.setdefault(yang.kw[string], {}).setdefault(name, {}) - self._parse(inside + [name], sub) - self.pop(Token.Punctuation, '}') - return - - if name in yang.types: - sub = tree.setdefault(yang.kw[string], {}).setdefault(name, {}) - self._parse(inside + [name], sub) - self.pop(Token.Punctuation, '}') - return - - if ':' in name: - sub = tree.setdefault(yang.kw[string], {}).setdefault(name, {}) - self._parse(inside + [name], sub) - self.pop(Token.Punctuation, '}') - return - - if string == 'uses': - if name not in self.grouping: - self.unexpected(f'could not find grouping calle {name}') - tree.update(self.grouping[name]) - option = self.pop(Token.Punctuation) - if option == ';': - return - if option == '{': - sub = tree.setdefault(name, {}) - self._parse(inside + [name], sub) - self.pop(Token.Punctuation, '}') - return - - if string == 'grouping': - self.pop(Token.Punctuation, '{') - sub = self.grouping.setdefault(name, {}) - self._parse(inside + [name], sub) - self.pop(Token.Punctuation, '}') - return - - if string in ('container', 'list', 'refine', 'leaf', 'leaf-list'): - self.pop(Token.Punctuation, '{') - sub = tree.setdefault(name, {}) - self._parse(inside + [name], sub) - self.pop(Token.Punctuation, '}') - return - - self.unknown(string, name) - - def unknown(self, string, name): - # catch unknown keyword so we can implement them - pprint.pprint(self.ns) - pprint.pprint('\n') - pprint.pprint(string) - pprint.pprint(name) - pprint.pprint('\n') - for t in self.tokens[:15]: - pprint.pprint(t) - breakpoint() - # good luck! - pass - - -# Used https://github.com/asottile/astpretty -# to understand how the python AST works -# Python 3.9 will have ast.unparse but until then -# https://github.com/simonpercivall/astunparse -# is used to generate code from the AST created - -from ast import Module, Import, FunctionDef, arguments, arg, alias -from ast import Load, Call, Return, Name, Attribute, Constant, Param -from ast import Add, If, Compare, Gt, Lt, GtE, LtE, And, Or -from ast import BoolOp, UnaryOp, Not, USub - -import astunparse - - -class Code(object): - def __init__(self, tree): - # the modules (import) required within the generated function - self.imported = set() - # type/function referenced in other types (union, ...) - # which should therefore also be generated - self.referenced = set() - # the parsed yang as a tree - self.tree = tree - # the main yang namespace/module - self.module = tree[yang.kw['loaded']][0] - # the namespace we are working in - self.ns = self.module - - @staticmethod - def _missing(**kargs): - print(' '.join(f'{k}={v}' for k, v in kargs.items())) - - # this code path was not handled - breakpoint() - - @staticmethod - def _unique(name, counter={}): - unique = counter.get(name, 0) - unique += 1 - counter[name] = unique - return f'{name}_{unique}' - - @staticmethod - def _return_boolean(value): - return [ - Return( - value=Constant(value=value, kind=None), - ) - ] - - def _python_name(self, name): - if self.ns != self.tree[yang.kw['loaded']][0] and ':' not in name: - # XXX: could this lead to function shadowing? - name = f'{self.ns}:{name}' - # XXX: could this lead to function shadowing? - return name.replace(':', '__').replace('-', '_') - - def _if_pattern(self, pattern): - self.imported.add('re') - return [ - If( - test=UnaryOp( - op=Not(), - operand=Call( - func=Attribute( - value=Name(id='re', ctx=Load()), - attr='match', - ctx=Load(), - ), - args=[ - Constant(value=pattern, kind=None), - Name(id='value', ctx=Load()), - ], - keywords=[], - ), - ), - body=[ - Return( - value=Constant(value=False, kind=None), - ), - ], - orelse=[], - ), - ] - - def _if_length(self, minimum, maximum): - self.imported.add('re') - return [ - If( - test=Compare( - left=Constant(value=int(minimum), kind=None), - ops=[ - Gt(), - Gt(), - ], - comparators=[ - Call( - func=Name(id='len', ctx=Load()), - args=[Name(id='value', ctx=Load())], - keywords=[], - ), - Constant(value=int(maximum), kind=None), - ], - ), - body=[ - Return( - value=Constant(value=False, kind=None), - ), - ], - orelse=[], - ), - ] - - def _iter_if_string(self, node): - for what, sub in node.items(): - if what == yang.kw['pattern']: - yield self._if_pattern(sub) - continue - - if what == yang.kw['match']: - self._missing(if_type=what, node=node) - continue - - if what == yang.kw['length']: - yield self._if_length(*sub) - continue - - self._missing(if_type=what, node=node) - - @staticmethod - def _if_digit(): - return [ - If( - test=UnaryOp( - op=Not(), - operand=Call( - func=Attribute( - value=Name(id='value', ctx=Load()), - attr='isdigit', - ctx=Load(), - ), - args=[], - keywords=[], - ), - ), - body=[ - Return( - value=Constant(value=False, kind=None), - ), - ], - orelse=[], - ) - ] - - @staticmethod - def _if_lt(value): - if value >= 0: - comparators = [ - Constant(value=value, kind=None) - ] - else: - comparators = [ - UnaryOp( - op=USub(), - operand=Constant(value=abs(value), kind=None), - ), - ] - - return [ - If( - test=Compare( - left=Call( - func=Name(id='int', ctx=Load()), - args=[Name(id='value', ctx=Load())], - keywords=[], - ), - ops=[Lt()], - comparators=comparators, - keywords=[], - ), - body=[ - Return( - value=Constant(value=False, kind=None), - ), - ], - orelse=[], - ) - ] - - @staticmethod - def _if_gt(value): - if value >= 0: - comparators = [ - Constant(value=value, kind=None) - ] - else: - comparators = [ - UnaryOp( - op=USub(), - operand=Constant(value=abs(value), kind=None), - ), - ] - return [ - If( - test=Compare( - left=Call( - func=Name(id='int', ctx=Load()), - args=[Name(id='value', ctx=Load())], - keywords=[], - ), - ops=[Gt()], - comparators=comparators, - ), - body=[ - Return( - value=Constant(value=False, kind=None), - ), - ], - orelse=[], - ) - ] - - def _if_range(self, minimum, maximum): - return self._if_digit() + self._if_lt(minimum) + self._if_gt(maximum) - - def _union(self, node): - values = [] - generated = [] - - for union in node: - for what, sub in union.items(): - if ':' in what: - if what in generated: - # only generate any imported function once - continue - generated.append(what) - name = what - yield self._type(what, name, sub) - else: - # this is a build_in type (and my have been refined) - # therefore generate one function per type - name = self._unique(what) - yield self._function(name, self._type(what, what, sub)) - - values += [ - UnaryOp( - op=Not(), - operand=Call( - func=Name(id=self._python_name(name), ctx=Load()), - args=[Name(id='value', ctx=Load())], - keywords=[], - ), - ), - ] - - yield [ - If( - test=BoolOp( - op=And(), - values=values, - ), - body=[ - Return( - value=Constant(value=False, kind=None), - ), - ], - orelse=[], - ), - ] - - def _imported(self): - for imported in self.imported: - yield Import(names=[alias(name=imported, asname=None)]) - - def _type(self, what, name, node): - if what == 'union': - return list(self._union(node)) - - if what in ('int8', 'int16', 'int16', 'int32', 'uint8', 'uint16', 'uint16', 'uint32'): - # not dealing with refine - minimum, maximum = yang.ranges[what] - return self._if_range(minimum, maximum) - - if what == 'string': - return list(self._iter_if_string(node)) - - if ':' in what: - ns, name = what.split(':', 1) - backup_ns, self.ns = self.ns, ns - answer = list(self._typedef(ns, name)) - self.ns = backup_ns - return answer - - self._missing(what=what, name=name, node=node) - - def _iter(self, node): - for keyword, content in node.items(): - yield self._type(keyword, keyword, content) - - def _function(self, name, body): - # XXX: could this lead to function shadowing? - return [ - FunctionDef( - name=self._python_name(name), - args=arguments( - posonlyargs=[], - args=[arg(arg='value', annotation=None, type_comment=None)], - vararg=None, - kwonlyargs=[], - kw_defaults=[], - kwarg=None, - defaults=[], - ), - body=body + self._return_boolean(True), - decorator_list=[], - returns=None, - type_comment=None, - ) - ] - - def _typedef(self, module, only): - td = self.tree[module][yang.kw['typedef']] - - for name in td: - if only and only != name: - continue - body = list(self._iter(td[name][yang.kw['type']])) - yield self._function(name, body) - - def _module(self, module, only=''): - generated = list(self._typedef(module, only)) - # while self.referenced: - # module, check = self.referenced.pop(0) - # generated += list(self._typedef(module, check)) - return generated - - def generate(self, module): - # this must be run first so that the imported module can be generated - body = list(self._module(module)) - ast = Module( - body=list(self._imported()) + body, - ) - return ast - - -class Conf(object): - intro = '# yang model structure and validation\n# autogenerate by exabgp\n\n' - variable = '{name} = {data}\n' - - def __init__(self, fname): - self.fname = fname - self.dicts = [] - self.codes = [] - - def add_dict(self, name, data): - self.dicts.append((name, data)) - - def add_code(self, block): - self.codes.append(block) - - def _generate(self): - returned = self.intro - for name, data in self.dicts: - returned += self.variable.format(name=name, data=data) - returned += '\n' - for section in self.codes: - returned += section - returned += '\n' - return returned - - def save(self): - print(f'generating {self.fname}') - with open(self.fname, 'w') as w: - w.write(self._generate()) - - def output(self): - # for name, data in self.dicts: - # pprint.pprint(name) - # pprint.pprint(data) - for section in self.codes: - print(section) - - -def main(): - folder = os.path.dirname(__file__) - os.chdir(os.path.abspath(folder)) - - library = 'yang-library-data.json' - module = 'exabgp' - models = 'models' - fname = '{module}-yang.py' - - conf = Conf(fname) - - yang.load(library, models) - tree = Lexer(f'{module}.yang').parse() - - conf.add_dict('model', tree) - # conf.output() - - code = Code(tree) - ast = code.generate(module) - block = astunparse.unparse(ast) - conf.add_code(block) - - conf.output() - # # conf.save() - - -if __name__ == "__main__": - main() diff --git a/src/exabgp/conf/yang/__init__.py b/src/exabgp/conf/yang/__init__.py new file mode 100755 index 000000000..8271e6797 --- /dev/null +++ b/src/exabgp/conf/yang/__init__.py @@ -0,0 +1,13 @@ +# encoding: utf-8 +""" +yang/__init__.py + +Created by Thomas Mangin on 2020-09-01. +Copyright (c) 2020 Exa Networks. All rights reserved. +""" + +from exabgp.conf.yang.model import Model +from exabgp.conf.yang.code import Code +from exabgp.conf.yang.tree import Tree + +from exabgp.conf.yang.datatypes import kw diff --git a/src/exabgp/conf/yang/code.py b/src/exabgp/conf/yang/code.py new file mode 100644 index 000000000..b15596c0a --- /dev/null +++ b/src/exabgp/conf/yang/code.py @@ -0,0 +1,356 @@ +# encoding: utf-8 +""" +code.py + +Created by Thomas Mangin on 2020-09-01. +Copyright (c) 2020 Exa Networks. All rights reserved. +""" + +# Used https://github.com/asottile/astpretty +# to understand how the python AST works +# Python 3.9 will have ast.unparse but until then +# https://github.com/simonpercivall/astunparse +# is used to generate code from the AST created + +from ast import Module, Import, FunctionDef, arguments, arg, alias +from ast import Load, Call, Return, Name, Attribute, Constant, Param +from ast import Add, If, Compare, Gt, Lt, GtE, LtE, And, Or +from ast import BoolOp, UnaryOp, Not, USub + +import astunparse + +from exabgp.conf.yang.datatypes import kw +from exabgp.conf.yang.datatypes import ranges + + +class Code(object): + def __init__(self, tree): + # the modules (import) required within the generated function + self.imported = set() + # type/function referenced in other types (union, ...) + # which should therefore also be generated + self.referenced = set() + # the parsed yang as a tree + self.tree = tree + # the main yang namespace/module + self.module = tree[kw['loaded']][0] + # the namespace we are working in + self.ns = self.module + + @staticmethod + def _missing(**kargs): + print(' '.join(f'{k}={v}' for k, v in kargs.items())) + + # this code path was not handled + breakpoint() + + @staticmethod + def _unique(name, counter={}): + unique = counter.get(name, 0) + unique += 1 + counter[name] = unique + return f'{name}_{unique}' + + @staticmethod + def _return_boolean(value): + return [ + Return( + value=Constant(value=value, kind=None), + ) + ] + + def _python_name(self, name): + if self.ns != self.tree[kw['loaded']][0] and ':' not in name: + # XXX: could this lead to function shadowing? + name = f'{self.ns}:{name}' + # XXX: could this lead to function shadowing? + return name.replace(':', '__').replace('-', '_') + + def _if_pattern(self, pattern): + self.imported.add('re') + return [ + If( + test=UnaryOp( + op=Not(), + operand=Call( + func=Attribute( + value=Name(id='re', ctx=Load()), + attr='match', + ctx=Load(), + ), + args=[ + Constant(value=pattern, kind=None), + Name(id='value', ctx=Load()), + ], + keywords=[], + ), + ), + body=[ + Return( + value=Constant(value=False, kind=None), + ), + ], + orelse=[], + ), + ] + + def _if_length(self, minimum, maximum): + self.imported.add('re') + return [ + If( + test=Compare( + left=Constant(value=int(minimum), kind=None), + ops=[ + Gt(), + Gt(), + ], + comparators=[ + Call( + func=Name(id='len', ctx=Load()), + args=[Name(id='value', ctx=Load())], + keywords=[], + ), + Constant(value=int(maximum), kind=None), + ], + ), + body=[ + Return( + value=Constant(value=False, kind=None), + ), + ], + orelse=[], + ), + ] + + def _iter_if_string(self, node): + for what, sub in node.items(): + if what == kw['pattern']: + yield self._if_pattern(sub) + continue + + if what == kw['match']: + self._missing(if_type=what, node=node) + continue + + if what == kw['length']: + yield self._if_length(*sub) + continue + + self._missing(if_type=what, node=node) + + @staticmethod + def _if_digit(): + return [ + If( + test=UnaryOp( + op=Not(), + operand=Call( + func=Attribute( + value=Name(id='value', ctx=Load()), + attr='isdigit', + ctx=Load(), + ), + args=[], + keywords=[], + ), + ), + body=[ + Return( + value=Constant(value=False, kind=None), + ), + ], + orelse=[], + ) + ] + + @staticmethod + def _if_lt(value): + if value >= 0: + comparators = [ + Constant(value=value, kind=None) + ] + else: + comparators = [ + UnaryOp( + op=USub(), + operand=Constant(value=abs(value), kind=None), + ), + ] + + return [ + If( + test=Compare( + left=Call( + func=Name(id='int', ctx=Load()), + args=[Name(id='value', ctx=Load())], + keywords=[], + ), + ops=[Lt()], + comparators=comparators, + keywords=[], + ), + body=[ + Return( + value=Constant(value=False, kind=None), + ), + ], + orelse=[], + ) + ] + + @staticmethod + def _if_gt(value): + if value >= 0: + comparators = [ + Constant(value=value, kind=None) + ] + else: + comparators = [ + UnaryOp( + op=USub(), + operand=Constant(value=abs(value), kind=None), + ), + ] + return [ + If( + test=Compare( + left=Call( + func=Name(id='int', ctx=Load()), + args=[Name(id='value', ctx=Load())], + keywords=[], + ), + ops=[Gt()], + comparators=comparators, + ), + body=[ + Return( + value=Constant(value=False, kind=None), + ), + ], + orelse=[], + ) + ] + + def _if_range(self, minimum, maximum): + return self._if_digit() + self._if_lt(minimum) + self._if_gt(maximum) + + def _union(self, node): + values = [] + generated = [] + + for union in node: + for what, sub in union.items(): + if ':' in what: + if what in generated: + # only generate any imported function once + continue + generated.append(what) + name = what + yield self._type(what, name, sub) + else: + # this is a build_in type (and my have been refined) + # therefore generate one function per type + name = self._unique(what) + yield self._function(name, self._type(what, what, sub)) + + values += [ + UnaryOp( + op=Not(), + operand=Call( + func=Name(id=self._python_name(name), ctx=Load()), + args=[Name(id='value', ctx=Load())], + keywords=[], + ), + ), + ] + + yield [ + If( + test=BoolOp( + op=And(), + values=values, + ), + body=[ + Return( + value=Constant(value=False, kind=None), + ), + ], + orelse=[], + ), + ] + + def _imported(self): + for imported in self.imported: + yield Import(names=[alias(name=imported, asname=None)]) + + def _type(self, what, name, node): + if what == 'union': + return list(self._union(node)) + + if what in ('int8', 'int16', 'int16', 'int32', 'uint8', 'uint16', 'uint16', 'uint32'): + # not dealing with refine + minimum, maximum = ranges[what] + return self._if_range(minimum, maximum) + + if what == 'string': + return list(self._iter_if_string(node)) + + if ':' in what: + ns, name = what.split(':', 1) + backup_ns, self.ns = self.ns, ns + answer = list(self._typedef(ns, name)) + self.ns = backup_ns + return answer + + self._missing(what=what, name=name, node=node) + + def _iter(self, node): + for keyword, content in node.items(): + yield self._type(keyword, keyword, content) + + def _function(self, name, body): + # XXX: could this lead to function shadowing? + return [ + FunctionDef( + name=self._python_name(name), + args=arguments( + posonlyargs=[], + args=[arg(arg='value', annotation=None, type_comment=None)], + vararg=None, + kwonlyargs=[], + kw_defaults=[], + kwarg=None, + defaults=[], + ), + body=body + self._return_boolean(True), + decorator_list=[], + returns=None, + type_comment=None, + ) + ] + + def _typedef(self, module, only): + td = self.tree[module][kw['typedef']] + + for name in td: + if only and only != name: + continue + body = list(self._iter(td[name][kw['type']])) + yield self._function(name, body) + + def _module(self, module, only=''): + generated = list(self._typedef(module, only)) + # while self.referenced: + # module, check = self.referenced.pop(0) + # generated += list(self._typedef(module, check)) + return generated + + def generate(self, module): + # this must be run first so that the imported module can be generated + body = list(self._module(module)) + ast = Module( + body=list(self._imported()) + body, + ) + return ast + + diff --git a/src/exabgp/conf/yang/datatypes.py b/src/exabgp/conf/yang/datatypes.py new file mode 100644 index 000000000..b9bc4d8ab --- /dev/null +++ b/src/exabgp/conf/yang/datatypes.py @@ -0,0 +1,131 @@ +# encoding: utf-8 +""" +yang/datatypes.py + +Created by Thomas Mangin on 2020-09-01. +Copyright (c) 2020 Exa Networks. All rights reserved. +""" + +import decimal + + +words = ( + 'bit', + 'boolean', + 'contact', + 'container', + 'default', + 'description', + 'enumeration', + 'key', + 'import', + 'enum', + 'extension', + 'grouping', + 'leaf', + 'leaf-list', + 'length', + 'list', + 'mandatory', + 'namespace', + 'organization', + 'revision', + 'path', + 'pattern', + 'prefix', + 'refine', + 'type', + 'typedef', + 'union', + 'uses', + 'range', + 'reference', + 'require-instance', + 'value', + 'yang-version', +) + +restriction = { + 'binary': ['length'], + 'bits': ['bit'], + 'boolean': [], + 'decimal64': ['range'], + 'empty': [], + 'enumeration': ['enum'], + 'identityref': [], + 'instance-identifier': ['require-instance'], + 'int8': [], + 'int16': [], + 'int32': [], + 'int64': [], + 'leafref': ['path', 'require-instance'], + 'string': ['pattern', 'length'], + 'uint8': [], + 'uint16': [], + 'uint32': [], + 'uint64': [], + 'union': [], +} + +types = list(restriction.keys()) + +# the yang keywords +kw = dict((w, f'[{w}]') for w in words) +# the yang module loaded +kw['loaded'] = '[loaded]' +# the root of the configuration +kw['root'] = '[root]' +# to differenciate with pattern +kw['match'] = '[match]' + +ranges = { + 'int8': (0, pow(2, 8)-1), + 'int16': (0, pow(2, 16)-1), + 'int32': (0, pow(2, 32)-1), + 'int64': (0, pow(2, 64)-1), + 'uint8': (-pow(2, 7), pow(2, 7)-1), + 'uint16': (-pow(2, 7), pow(2, 15)-1), + 'uint32': (-pow(2, 7), pow(2, 31)-1), + 'uint64': (-pow(2, 7), pow(2, 64)-1), +} + + +class Boolean(int): + def __new__(cls, value): + return int.__new__(cls, value not in ('false', False, 0)) + + def __init__(self, boolean): + self.string = boolean + + def __str__(self): + return self.string + + +class Decimal64(decimal.Decimal): + def __init__(cls, value, frac=0): + raise RuntimeError() + # look at https://github.com/CZ-NIC/yangson/blob/master/yangson/datatype.py#L682 + # return super().__init__(decimal.Decimal(value)) + + +klass = { + 'binary': ['length'], + 'bits': ['bit'], + 'boolean': Boolean, + 'decimal64': Decimal64, + 'empty': None, + 'enumeration': None, + 'identityref': str, + 'instance-identifier': str, + 'int8': int, + 'int16': int, + 'int32': int, + 'int64': int, + 'leafref': str, + 'string': str, + 'uint8': int, + 'uint16': int, + 'uint32': int, + 'uint64': int, + 'union': None, +} diff --git a/src/exabgp/conf/yang/model.py b/src/exabgp/conf/yang/model.py new file mode 100644 index 000000000..a52ffd11a --- /dev/null +++ b/src/exabgp/conf/yang/model.py @@ -0,0 +1,128 @@ + +# encoding: utf-8 +""" +yang/model.py + +Created by Thomas Mangin on 2020-09-01. +Copyright (c) 2020 Exa Networks. All rights reserved. +""" + +import os +import sys +import json +import glob +import shutil +import urllib +import urllib.request + + +class Model(object): + namespaces = { + 'ietf': 'https://raw.githubusercontent.com/YangModels/yang/master/standard/ietf/RFC', + } + + models = {} + + def __init__(self, library, folder, module): + self.library = library + self.folder = folder + + models = json.loads(open(self.library).read()) + + for m in models['ietf-yang-library:modules-state']['module']: + self.models[m['name']] = m + + if not os.path.exists('models'): + os.mkdir('models') + + def _write(self, string): + if not string.startswith('\n'): + fill = ' ' * shutil.get_terminal_size().columns + sys.stdout.write(f'\r{fill}\r') + sys.stdout.write(f'{string}') + sys.stdout.flush() + + # @classmethod + # def fetch_models(self, folder): + # print('downloading models') + + # for module in self.models: + # self.fetch_model(folder, module) + + # print('done.\n') + + def load(self, module, infolder=False): + fname = f'{module}.yang' + if infolder: + fname = os.path.join(self.folder, fname) + + if not os.path.exists(fname): + self.fetch(module) + + return open(fname).read() + + def fetch(self, module): + if module not in self.models: + sys.exit(f'{module} imported but not defined in yang-library-data.json') + + module = self.models[module] + + revision = module['revision'] + yang = f'{module}@{revision}.yang' + save = f'{self.models}/{module}.yang' + + if 'schema' in module: + url = module['schema'] + + elif 'namespace' in module: + namespace = module['namespace'].split(':') + site = self.namespaces.get(namespace[1], '') + if not site: + raise RuntimeError('unimplemented namespace case') + + url = f"{site}/{yang}" + else: + raise RuntimeError('unimplemented yang-library case') + + if os.path.exists(save): + self._write(f'šŸ‘Œ skipping {module} (already downloaded)') + if self._verify(module, save): + self._write('\n') + return + + self._write(f'šŸ‘ļø retrieve {module}@{revision} ({url})') + + try: + urllib.request.urlretrieve(url, save) + # indirect = urllib.request.urlopen(schema).read() + except urllib.error.HTTPError as exc: + self._write(f'\nšŸ„ŗ failure attempting to retrieve {url}\n{exc}') + return + + if not self._verify(module, save): + sys.exit(f'\ninvalid yang content for {module}@{revision}') + + self._write(f'šŸ‘ retrieve {module}@{revision}\n') + + def _verify(self, name, save): + # simple but should be enough + self._write(f'šŸ” checking {name} for correct yaml') + if not open(save).readline().startswith('module'): + self._write(f'šŸ„µ not-yang {name} does not contain a yang module') + return False + + # XXX: removed tests - check later + return True + + self._write(f'šŸ” checking {name} for correct yaml') + if not open(save).readline().startswith('module'): + self._write(f'šŸ„µ not-yang {name} does not contain a yang module') + return False + return True + + def clean_models(self): + print(f'cleaning {self.folder}') + for file in glob.glob(f'{self.folder}/*.yang'): + print(f'cleanup: {file}') + os.remove(file) + print('done.\n') diff --git a/src/exabgp/conf/yang/tree.py b/src/exabgp/conf/yang/tree.py new file mode 100644 index 000000000..a7e376a05 --- /dev/null +++ b/src/exabgp/conf/yang/tree.py @@ -0,0 +1,359 @@ +# encoding: utf-8 +""" +yang/tree.py + +Created by Thomas Mangin on 2020-09-01. +Copyright (c) 2020 Exa Networks. All rights reserved. +""" + +import os + +from pygments.token import Token +from yanglexer import yanglexer + +from exabgp.conf.yang.model import Model +from exabgp.conf.yang.datatypes import kw +from exabgp.conf.yang.datatypes import words +from exabgp.conf.yang.datatypes import types + + +DEBUG = True + + +class Tree(object): + ignore = (Token.Text, Token.Comment.Singleline) + + @staticmethod + def formated(string): + returned = '' + for line in string.strip().split('\n'): + line = line.strip() + + if line.endswith('+'): + line = line[:-1].strip() + if line.startswith('+'): + line = line[1:].strip() + + if line and line[0] == line[-1]: + if line[0] in ('"', "'"): + line = line[1:-1] + returned += line + return returned + + def __init__(self, library, models, yang): + self.model = Model(library, models, yang) + # the yang file parsed tokens (configuration block according to the syntax) + self.tokens = [] + # the name of the module being parsed + self.module = '' + # module can declare a "prefix" (nickname), which can be used to make the syntax shorter + self.prefix = '' + # the parsed yang tree + # - at the root are the namespace (the module names) and within + # * a key for all the typedef + # * a key for all the grouping + # * a key for the root of the configuration + # - a key [loaded] with a list of the module loaded (first is the one parsed) + # the names of all the configuration sections + self.tree = {} + # the current namespace (module) we are parsing + self.ns = {} + # where the grouping for this section are stored + self.grouping = {} + # where the typedef for this section are stored + self.typedef = {} + # where the configuration parsed is stored + self.root = {} + self.load(yang) + + def tokenise(self, module, ismodel): + lexer = yanglexer.YangLexer() + tokens = lexer.get_tokens(self.model.load(module, ismodel)) + return [(t, n) for (t, n) in tokens if t not in self.ignore] + + def unexpected(self, string): + pprint.pprint(f'unexpected data: {string}') + for t in self.tokens[:15]: + print(t) + breakpoint() + pass + + def pop(self, what=None, expected=None): + token, string = self.tokens[0] + if what is not None and not str(token).startswith(str(what)): + self.unexpected(string) + if expected is not None and string.strip() != expected: + self.unexpected(string) + self.tokens.pop(0) + return string + + def peek(self, position, ponctuation=None): + token, string = self.tokens[position] + # the self includes a last ' ' + if ponctuation and ponctuation != token: + self.unexpected(string) + return token, string.rstrip() + + def skip_keyword_block(self, name): + count = 0 + while True: + t, v = self.tokens.pop(0) + if t != Token.Punctuation: + continue + if v.strip() == '{': + count += 1 + if v.strip() == '}': + count -= 1 + if not count: + break + + def set_subtrees(self): + """ + to make the core more redeable the tree[module] structure + is presented as subtrees, this reset all the subtree + for the current module + """ + self.ns = self.tree[self.module] + self.grouping = self.ns[kw['grouping']] + self.typedef = self.ns[kw['typedef']] + self.root = self.ns[kw['root']] + + def imports(self, module): + """ + load, and if missing and defined download, a yang module + + module: the name of the yang module to find + prefix: how it is called (prefix) + """ + backup = (self.tokens, self.module, self.prefix) + # XXX: should it be module ?? + self.load(module, ismodel=True) + self.tokens, self.module, self.prefix = backup + self.set_subtrees() + + def load(self, module, ismodel=False): + """ + add a new yang module/namespace to the tree + this _function is used when initialising the + root module, as it does not perform backups + """ + self.tree.setdefault(kw['loaded'], []).append(module) + self.tokens = self.tokenise(module, ismodel) + self.module = module + self.prefix = module + self.tree[module] = { + kw['typedef']: {}, + kw['grouping']: {}, + kw['root']: {}, + } + self.set_subtrees() + self.parse() + + def parse(self): + self._parse([], self.root) + return self.tree + + def _parse(self, inside, tree): + while self.tokens: + token, string = self.peek(0) + + if token == Token.Punctuation and string == '}': + # it is clearer to pop it in the caller + return + + self._parse_one(inside, tree, token, string) + + def _parse_one(self, inside, tree, token, string): + if token == Token.Comment.Multiline: + # ignore multiline comments + self.pop(Token.Comment.Multiline) + return + + if token == Token.Keyword.Namespace: + self.pop(Token.Keyword.Namespace, 'module') + self.pop(Token.Literal.String) + self.pop(Token.Punctuation, '{') + self._parse(inside, tree) + self.pop(Token.Punctuation, '}') + return + + if token != Token.Keyword or string not in words: + if ':' not in string: + self.unknown(string, '') + return + + self.pop(Token.Keyword, string) + name = self.formated(self.pop(Token.Literal.String)) + + if string == 'prefix': + self.prefix = name + self.pop(Token.Punctuation, ';') + return + + if string in ('namespace', 'organization', 'contact', 'yang-version'): + self.pop(Token.Punctuation, ';') + return + + if string in ('revision', 'extension'): + self.skip_keyword_block(Token.Punctuation) + return + + if string in ('range', 'length'): + self.pop(Token.Punctuation, ';') + tree[kw[string]] = [_ for _ in name.replace(' ', '').replace('..',' ').split()] + return + + if string == 'import': + token, string = self.peek(0, Token.Punctuation) + if string == ';': + self.pop(Token.Punctuation, ';') + self.imports(name) + if string == '{': + self.pop(Token.Punctuation, '{') + self.pop(Token.Keyword, 'prefix') + prefix = self.formated(self.pop(Token.Literal.String)) + self.pop(Token.Punctuation, ';') + self.pop(Token.Punctuation, '}') + self.imports(name) + return + + if string in ('description', 'reference'): + self.pop(Token.Punctuation, ';') + # XXX: not saved during debugging + if DEBUG: + return + tree[kw[string]] = name + return + + if string in ('pattern', 'value', 'default', 'mandatory'): + self.pop(Token.Punctuation, ';') + tree[kw[string]] = name + return + + if string == 'key': + self.pop(Token.Punctuation, ';') + tree[kw[string]] = name.split() + return + + if string == 'typedef': + self.pop(Token.Punctuation, '{') + sub = self.typedef.setdefault(name, {}) + self._parse(inside + [name], sub) + self.pop(Token.Punctuation, '}') + return + + if string == 'enum': + option = self.pop(Token.Punctuation) + if option == ';': + tree.setdefault(kw[string], []).append(name) + return + if option == '{': + sub = tree.setdefault(name, {}) + self._parse(inside + [name], sub) + self.pop(Token.Punctuation, '}') + return + + if string == 'type': + # make sure the use module name and not prefix, as prefix is not global + if ':' in name: + module, typeref = name.split(':', 1) + if module == self.prefix: + name = f'{self.module}:{typeref}' + module = self.module + + if module not in self.tree: + self.unexpected(f'referenced non-included module {name}') + elif name not in types: + name = f'{self.module}:{name}' + + option = self.pop(Token.Punctuation) + if option == ';': + if name in types: + tree.setdefault(kw[string], {name: {}}) + return + + if ':' not in name: + # not dealing with refine + # breakpoint() + tree.setdefault(kw[string], {name: {}}) + return + + tree.setdefault(kw[string], {name: {}}) + return + + if option == '{': + if name == 'union': + sub = tree.setdefault(kw[string], {}).setdefault(name, []) + while True: + what, name = self.peek(0) + name = self.formated(name) + if name == 'type': + union_type = {} + self._parse_one(inside + [name], union_type, what, name) + sub.append(union_type[kw['type']]) + continue + if name == '}': + self.pop(Token.Punctuation, '}') + break + self.unexpected(f'did not expect this in an union: {what}') + return + + if name == 'enumeration': + sub = tree.setdefault(kw[string], {}).setdefault(name, {}) + self._parse(inside + [name], sub) + self.pop(Token.Punctuation, '}') + return + + if name in types: + sub = tree.setdefault(kw[string], {}).setdefault(name, {}) + self._parse(inside + [name], sub) + self.pop(Token.Punctuation, '}') + return + + if ':' in name: + sub = tree.setdefault(kw[string], {}).setdefault(name, {}) + self._parse(inside + [name], sub) + self.pop(Token.Punctuation, '}') + return + + if string == 'uses': + if name not in self.grouping: + self.unexpected(f'could not find grouping calle {name}') + tree.update(self.grouping[name]) + option = self.pop(Token.Punctuation) + if option == ';': + return + if option == '{': + sub = tree.setdefault(name, {}) + self._parse(inside + [name], sub) + self.pop(Token.Punctuation, '}') + return + + if string == 'grouping': + self.pop(Token.Punctuation, '{') + sub = self.grouping.setdefault(name, {}) + self._parse(inside + [name], sub) + self.pop(Token.Punctuation, '}') + return + + if string in ('container', 'list', 'refine', 'leaf', 'leaf-list'): + self.pop(Token.Punctuation, '{') + sub = tree.setdefault(name, {}) + self._parse(inside + [name], sub) + self.pop(Token.Punctuation, '}') + return + + self.unknown(string, name) + + def unknown(self, string, name): + # catch unknown keyword so we can implement them + pprint.pprint(self.ns) + pprint.pprint('\n') + pprint.pprint(string) + pprint.pprint(name) + pprint.pprint('\n') + for t in self.tokens[:15]: + pprint.pprint(t) + breakpoint() + # good luck! + pass