From ba14f1b05d5543cc1d4d213fc11fcca4495fbe4c Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Mon, 13 Jan 2020 01:45:27 -0600 Subject: [PATCH] Rewrite itl2py with Jinja, Generate Plain Python - Make pyopendds.dev.itl2dev an entry point for itl2py - Rewrote itl2py to use Jinja templates for all the gernated files. - Generate straightforward Python from ITL. See https://github.com/iguessthislldo/pyopendds/issues/2 for details. --- pyopendds/IDL.py | 81 --- pyopendds/dev/itl2py/CppOutput.py | 472 +++--------------- pyopendds/dev/itl2py/Output.py | 26 + pyopendds/dev/itl2py/PythonOutput.py | 109 ++-- pyopendds/dev/itl2py/__init__.py | 14 - pyopendds/dev/itl2py/__main__.py | 54 ++ pyopendds/dev/itl2py/ast.py | 48 +- pyopendds/dev/itl2py/build_files.py | 58 --- pyopendds/dev/itl2py/generate.py | 89 ++++ pyopendds/dev/itl2py/templates/CMakeLists.txt | 17 + pyopendds/dev/itl2py/templates/setup.py | 14 + pyopendds/dev/itl2py/templates/user.cpp | 356 +++++++++++++ pyopendds/dev/itl2py/templates/user.py | 26 + scripts/itl2py | 68 --- setup.cfg | 5 + setup.py | 7 +- tests/basic_test/subscriber.py | 2 +- 17 files changed, 757 insertions(+), 689 deletions(-) delete mode 100644 pyopendds/IDL.py create mode 100644 pyopendds/dev/itl2py/Output.py create mode 100644 pyopendds/dev/itl2py/__main__.py delete mode 100644 pyopendds/dev/itl2py/build_files.py create mode 100644 pyopendds/dev/itl2py/generate.py create mode 100644 pyopendds/dev/itl2py/templates/CMakeLists.txt create mode 100644 pyopendds/dev/itl2py/templates/setup.py create mode 100644 pyopendds/dev/itl2py/templates/user.cpp create mode 100644 pyopendds/dev/itl2py/templates/user.py delete mode 100755 scripts/itl2py diff --git a/pyopendds/IDL.py b/pyopendds/IDL.py deleted file mode 100644 index d55b931..0000000 --- a/pyopendds/IDL.py +++ /dev/null @@ -1,81 +0,0 @@ -class IDL: - def __init__(self): - self.definitions = {} - self.types = {} - self.root_contents = set() - - def add_definition(self, module, name, type, **kw): - if module: - full_name = module + '.' + name - self.definitions[module]['contents'] |= {full_name} - else: - full_name = name - self.root_contents |= {full_name} - d = { - 'module': module, - 'name': name, - 'type': type, - } - d.update(kw) - self.definitions[full_name] = d - - def add_struct(self, module, name, ts_package, members): - self.add_definition(module, name, 'struct', - ts_package=ts_package, members=members) - - def get_struct(self, full_name): - if full_name not in self.types: - d = self.definitions[full_name] - from dataclasses import make_dataclass - t = make_dataclass(d['name'], d['members']) - t.__module__ = d['module'] - if d['ts_package']: - t._pyopendds_typesupport_packge_name = d['ts_package'] - self.types[full_name] = t - else: - t = self.types[full_name] - return t - - def add_enum(self, module, name, members): - self.add_definition(module, name, 'enum', members=members) - - def get_enum(self, full_name): - if full_name not in self.types: - d = self.definitions[full_name] - from enum import IntFlag - t = IntFlag(d['name'], d['members']) - t.__module__ = d['module'] - self.types[full_name] = t - else: - t = self.types[full_name] - return t - - def add_module(self, module, name): - self.add_definition(module, name, 'module', contents=set()) - - def inject_i(self, python_module, contents): - item_types = { - 'struct': self.get_struct, - 'enum': self.get_enum, - 'module': self.get_module, - } - for item_name in contents: - item_def = self.definitions[item_name] - setattr(python_module, item_def['name'], - item_types[item_def['type']](item_name)) - - def get_module(self, full_name): - if full_name not in self.types: - d = self.definitions[full_name] - from types import ModuleType - m = ModuleType(d['name']) - m.__package__ = d['module'] - self.inject_i(m, d['contents']) - self.types[full_name] = m - else: - m = self.types[full_name] - return m - - def inject(self, python_name): - from sys import modules - self.inject_i(modules[python_name], self.root_contents) diff --git a/pyopendds/dev/itl2py/CppOutput.py b/pyopendds/dev/itl2py/CppOutput.py index bc749e1..481a472 100644 --- a/pyopendds/dev/itl2py/CppOutput.py +++ b/pyopendds/dev/itl2py/CppOutput.py @@ -1,428 +1,94 @@ -from pathlib import Path -from typing import List +from jinja2 import Environment -from .ast import Output, PrimitiveType, EnumType +from .ast import PrimitiveType, EnumType +from .Output import Output class CppOutput(Output): - def __init__( - self, - output_path: Path, - python_package_name: str, - native_package_name: str, - idl_names: List[str], - default_encoding: str): - self.python_package_name = python_package_name - self.native_package_name = native_package_name - self.default_encoding = default_encoding - super().__init__(output_path / (native_package_name + '.cpp')) - - self.topic_types = [] - - self.append('#include \n\n') - - for name in idl_names: - self.append('#include <' + name + 'TypeSupportImpl.h>') - self.append(''' -#include -#include - -#include -#include -#include - -namespace { - -/// Get Contents of Capsule from a PyObject -template -T* get_capsule(PyObject* obj) -{ - T* rv = nullptr; - PyObject* capsule = PyObject_GetAttrString(obj, "_var"); - if (capsule && PyCapsule_IsValid(capsule, nullptr)) { - rv = static_cast(PyCapsule_GetPointer(capsule, nullptr)); - } - return rv; -} - -// Python Objects To Keep -PyObject* pyopendds; -PyObject* PyOpenDDS_Error; -PyObject* ReturnCodeError; - -bool cache_python_objects() -{ - // Get pyopendds - pyopendds = PyImport_ImportModule("pyopendds"); - if (!pyopendds) return true; - - // Get PyOpenDDS_Error - PyOpenDDS_Error = PyObject_GetAttrString(pyopendds, "PyOpenDDS_Error"); - if (!PyOpenDDS_Error) return true; - Py_INCREF(PyOpenDDS_Error); - - // Get ReturnCodeError - ReturnCodeError = PyObject_GetAttrString(pyopendds, "ReturnCodeError"); - if (!ReturnCodeError) return true; - Py_INCREF(ReturnCodeError); - - return false; -} - -bool check_rc(DDS::ReturnCode_t rc) -{ - return !PyObject_CallMethod(ReturnCodeError, "check", "k", rc); -} - -class Exception : public std::exception { -public: - Exception(const char* message) - : message_(message) - { - } - - virtual const char* what() const noexcept - { - return message_; - } - -private: - const char* message_; -}; - -class TypeBase { -public: - virtual void register_type(PyObject* pyparticipant) = 0; - virtual PyObject* get_python_class() = 0; - virtual const char* type_name() = 0; - virtual PyObject* read(PyObject* pyreader) = 0; -}; - -template -class TemplatedTypeBase : public TypeBase { -public: - typedef typename OpenDDS::DCPS::DDSTraits Traits; - - typedef T IdlType; - typedef typename Traits::MessageSequenceType IdlTypeSequence; - - typedef typename Traits::TypeSupportType TypeSupport; - typedef typename Traits::TypeSupportTypeImpl TypeSupportImpl; - typedef typename Traits::DataWriterType DataWriter; - typedef typename Traits::DataReaderType DataReader; - - const char* type_name() - { - return Traits::type_name(); - } - - /** - * Callback for Python to call when the TypeSupport capsule is deleted - */ - static void delete_typesupport(PyObject* capsule) - { - if (PyCapsule_CheckExact(capsule)) { - delete static_cast( - PyCapsule_GetPointer(capsule, NULL)); - } - } - - void register_type(PyObject* pyparticipant) - { - // Get DomainParticipant_var - DDS::DomainParticipant* participant = - get_capsule(pyparticipant); - if (!participant) { - throw Exception("Could not get native particpant"); - } - - // Register with OpenDDS - TypeSupportImpl* type_support = new TypeSupportImpl; - if (type_support->register_type(participant, "") != DDS::RETCODE_OK) { - delete type_support; - type_support = 0; - throw Exception("Could not create register type"); - } - - // Store TypeSupport in Python Participant - PyObject* capsule = PyCapsule_New(participant, NULL, delete_typesupport); - if (!capsule) { - throw Exception("Could not create ts capsule"); - } - PyObject* list = PyObject_GetAttrString( - pyparticipant, "_registered_typesupport"); - if (!list || !PyList_Check(list)) { - throw Exception("Could not get ts list"); - } - if (PyList_Append(list, capsule)) { - PyErr_Print(); - throw Exception("Could not append ts to list"); - } - } - - virtual void to_python(const T& cpp, PyObject*& py) = 0; - - PyObject* read(PyObject* pyreader) - { - DDS::DataReader* reader = get_capsule(pyreader); - if (!reader) { - PyErr_SetString(PyOpenDDS_Error, "Could not get datareader"); - return nullptr; - } - - DataReader* reader_impl = DataReader::_narrow(reader); - if (!reader_impl) { - PyErr_SetString(PyOpenDDS_Error, "Could not narrow reader implementation"); - return nullptr; - } - - DDS::ReturnCode_t rc; - DDS::ReadCondition_var read_condition = reader_impl->create_readcondition( - DDS::ANY_SAMPLE_STATE, DDS::ANY_VIEW_STATE, DDS::ANY_SAMPLE_STATE); - DDS::WaitSet_var ws = new DDS::WaitSet; - ws->attach_condition(read_condition); - DDS::ConditionSeq active; - const DDS::Duration_t max_wait_time = {10, 0}; - rc = ws->wait(active, max_wait_time); - ws->detach_condition(read_condition); - reader_impl->delete_readcondition(read_condition); - - T sample; - DDS::SampleInfo info; - if (check_rc(reader_impl->take_next_sample(sample, info))) return nullptr; - PyObject *rv = nullptr; - to_python(sample, rv); - return rv; - } - - virtual T from_python(PyObject* py) = 0; -}; -template class Type; - -typedef std::shared_ptr TypePtr; -typedef std::map Types; -Types types; - -template -void init_type() -{ - TypePtr type{new Type}; - types.insert(Types::value_type(type->get_python_class(), type)); -} - -long get_python_long_attr(PyObject* py, const char* attr_name) -{ - PyObject* attr = PyObject_GetAttrString(py, attr_name); - if (!attr) { - PyErr_Print(); - throw Exception("python error occured"); - } - if (!PyLong_Check(attr)) { - throw Exception("python attribute isn't an int"); - } - long long_value = PyLong_AsLong(attr); - if (long_value == -1 && PyErr_Occurred()) { - PyErr_Print(); - throw Exception("python error occured"); - } - return long_value; -} -''') + def __init__(self, context: dict): + new_context = context.copy() + jinja_start = '/*{' + jinja_end = '}*/' + new_context.update(dict( + idl_names=[ + itl_file.name[:-len('.itl')] for itl_file in context['itl_files']], + types=[], + topic_types=[], + jinja=Environment( + loader=context['jinja_loader'], + block_start_string=jinja_start + '%', + block_end_string='%' + jinja_end, + variable_start_string=jinja_start + '{', + variable_end_string='}' + jinja_end, + comment_start_string=jinja_start + '#', + comment_end_string='#' + jinja_end, + ) + )) + super().__init__(new_context, context['output'], + {context['native_package_name'] + '.cpp': 'user.cpp'}) def visit_struct(self, struct_type): cpp_name = '::' + struct_type.name.join('::') if struct_type.is_topic_type: - self.topic_types.append(cpp_name) - self.append('''\ -template<> -class Type<''' + cpp_name + '''> : public TemplatedTypeBase<''' + cpp_name + '''> { -public: - PyObject* get_python_class() - { - if (!python_class_) { - PyObject* module = PyImport_ImportModule("''' + self.python_package_name + '''"); - if (!module) return 0;''') - - for name in struct_type.parent_name().parts: - self.append('''\ - module = PyObject_GetAttrString(module, "''' + name + '''"); - if (!module) return 0;''') - - self.append('''\ - python_class_ = PyObject_GetAttrString(module, "''' + struct_type.local_name() + '''"); - } - return python_class_; - } + self.context['topic_types'].append(cpp_name) - void to_python(const ''' + cpp_name + '''& cpp, PyObject*& py) - { - PyObject* cls = get_python_class(); - if (py) { - if (PyObject_IsInstance(cls, py) != 1) { - throw Exception("Python object is not a valid type"); - } - } else { - py = PyObject_CallObject(cls, nullptr); - if (!py) { - PyErr_Print(); - throw Exception("Could not call __init__ for new class"); - } - } - - PyObject* field_value; -''') - - for field_name, (field_type, _) in struct_type.fields.items(): + struct_to_lines = [] + struct_from_lines = [] + nop = '// {field_name} was left unimplemented' + for field_name, field_node in struct_type.fields.items(): implemented = True - if isinstance(field_type, PrimitiveType): - if field_type.is_int(): - self.append(''' - field_value = PyLong_FromLong(cpp.''' + field_name + ');') - - elif field_type.is_string(): - self.append(''' - field_value = PyUnicode_Decode(cpp.''' + field_name + ', ' - + 'strlen(cpp.' + field_name + '), "' + self.default_encoding + '", ' - + '"strict");') + to_lines = [] + from_lines = [] + if isinstance(field_node.type_node, PrimitiveType): + if field_node.type_node.is_int(): + to_lines.append( + 'field_value = PyLong_FromLong(cpp.{field_name});') + from_lines.append( + 'rv.{field_name} = get_python_long_attr(py, "{field_name}");') + + elif field_node.type_node.is_string(): + to_lines.append('field_value = PyUnicode_Decode(cpp.{field_name}, ' + 'strlen(cpp.{field_name}), "{default_encoding}", "strict");') + from_lines.append(nop) else: implemented = False - elif isinstance(field_type, EnumType): + elif isinstance(field_node.type_node, EnumType): implemented = False else: implemented = False if implemented: - self.append('''\ - if (!field_value || PyObject_SetAttrString(py, "''' + field_name + '''", field_value)) { - py = nullptr; - }''') + to_lines.extend([ + 'if (!field_value || PyObject_SetAttrString(' + 'py, "{field_name}", field_value)) {{', + ' py = nullptr;', + '}}' + ]) else: - self.append(' // ' + field_name + ' was left unimplemented') - - self.append('''\ - } - - ''' + cpp_name + ''' from_python(PyObject* py) - { - ''' + cpp_name + ''' rv; - - PyObject* cls = get_python_class(); - if (PyObject_IsInstance(py, cls) != 1) { - throw Exception("Python object is not a valid type"); - } -''') - - for field_name, (field_type, _) in struct_type.fields.items(): - if isinstance(field_type, PrimitiveType) and field_type.is_int(): - self.append('''\ - rv.''' + field_name + ''' = get_python_long_attr(py, "''' + field_name + '''");''') - else: - self.append(' // ' + field_name + ' was left unimplemented') - - self.append(''' - return rv; - } - -private: - PyObject* python_class_ = nullptr; -}; -''') + to_lines.append(nop) + from_lines.append(nop) + + def line_process(lines): + return [ + s.format( + field_name=field_name, + default_encoding=self.context['default_encoding'], + ) for s in lines + ] + struct_to_lines.extend(line_process(to_lines)) + struct_from_lines.extend(line_process(from_lines)) + + self.context['types'].append({ + 'cpp_name': cpp_name, + 'name_parts': struct_type.parent_name().parts, + 'local_name': struct_type.local_name(), + 'to_lines': '\n'.join(struct_to_lines), + 'from_lines': '\n'.join(struct_from_lines), + }) def visit_enum(self, enum_type): pass - - def after(self): - lines = '''\ -} // Anonymous Namespace\n - -static PyObject* pyregister_type(PyObject* self, PyObject* args) -{ - // Get Arguments - PyObject* pyparticipant; - PyObject* pytype; - if (!PyArg_ParseTuple(args, "OO", &pyparticipant, &pytype)) { - PyErr_SetString(PyExc_TypeError, "Invalid Arguments"); - return NULL; - } - - Types::iterator i = types.find(pytype); - if (i != types.end()) { - i->second->register_type(pyparticipant); - Py_RETURN_NONE; - } - - PyErr_SetString(PyExc_TypeError, "Invalid Type"); - return NULL; -} - -static PyObject* pytype_name(PyObject* self, PyObject* args) -{ - // Get Arguments - PyObject* pytype; - if (!PyArg_ParseTuple(args, "O", &pytype)) { - PyErr_SetString(PyExc_TypeError, "Invalid Arguments"); - return NULL; - } - - Types::iterator i = types.find(pytype); - if (i != types.end()) { - return PyUnicode_FromString(i->second->type_name()); - } - - PyErr_SetString(PyExc_TypeError, "Invalid Type"); - return nullptr; -} - -static PyObject* pyread(PyObject* self, PyObject* args) -{ - // Get Arguments - PyObject* pyreader; - if (!PyArg_ParseTuple(args, "O", &pyreader)) return nullptr; - - // Try to Get Topic Type and Do Read - PyObject* pytopic = PyObject_GetAttrString(pyreader, "topic"); - if (!pytopic) return nullptr; - PyObject* pytype = PyObject_GetAttrString(pytopic, "type"); - if (!pytype) return nullptr; - Types::iterator i = types.find(pytype); - if (i != types.end()) { - return i->second->read(pyreader); - } - - PyErr_SetString(PyExc_TypeError, "Invalid Type"); - return NULL; -} - -static PyMethodDef ''' + self.native_package_name + '''_Methods[] = { - {"register_type", pyregister_type, METH_VARARGS, ""}, - {"type_name", pytype_name, METH_VARARGS, ""}, - {"read", pyread, METH_VARARGS, ""}, - {NULL, NULL, 0, NULL} -}; - -static struct PyModuleDef ''' + self.native_package_name + '''_Module = { - PyModuleDef_HEAD_INIT, - "''' + self.native_package_name + '''", "", - -1, // Global State Module, because OpenDDS uses Singletons - ''' + self.native_package_name + '''_Methods -}; - -PyMODINIT_FUNC PyInit_''' + self.native_package_name + '''() -{ - PyObject* module = PyModule_Create(&''' + self.native_package_name + '''_Module); - if (!module || cache_python_objects()) return nullptr; - -''' - for topic_type in self.topic_types: - lines += ' init_type<' + topic_type + '>();\n' - - lines += '''\ - - return module; -} -''' - return lines diff --git a/pyopendds/dev/itl2py/Output.py b/pyopendds/dev/itl2py/Output.py new file mode 100644 index 0000000..696cdcb --- /dev/null +++ b/pyopendds/dev/itl2py/Output.py @@ -0,0 +1,26 @@ +from pathlib import Path + +from .ast import NodeVisitor + + +class Output(NodeVisitor): + + def __init__(self, context: dict, path: Path, templates: dict): + self.context = context + self.path = path + self.templates = {} + for filename, template in templates.items(): + self.templates[path / filename] = context['jinja'].get_template(template) + + def write(self): + if self.context['dry_run']: + print('######################################## Create Dir', self.path) + else: + self.path.mkdir(exist_ok=True) + for path, template in self.templates.items(): + content = template.render(self.context) + if self.context['dry_run']: + print('======================================== Write file', path) + print(content) + else: + path.write_text(content) diff --git a/pyopendds/dev/itl2py/PythonOutput.py b/pyopendds/dev/itl2py/PythonOutput.py index 99edbdd..9cb8b10 100644 --- a/pyopendds/dev/itl2py/PythonOutput.py +++ b/pyopendds/dev/itl2py/PythonOutput.py @@ -1,9 +1,13 @@ -from pathlib import Path - -from .ast import Output, PrimitiveType +from .ast import PrimitiveType, StructType, EnumType +from .Output import Output class PythonOutput(Output): + '''Manages Output of Python Bindings + + Using a self nesting structure, a PythonOutput is created for each IDL + module. + ''' primitive_types = { # (Python Type, Default Default Value) PrimitiveType.Kind.u8: ('int', '0'), @@ -20,50 +24,73 @@ class PythonOutput(Output): PrimitiveType.Kind.s8: ('str', "''"), } - def __init__(self, - output_path: Path, - python_package_name: str, - native_package_name: str): - self.python_package_name = python_package_name - self.native_package_name = native_package_name - path = output_path / python_package_name - path.mkdir(parents=True, exist_ok=True) - super().__init__(path / '__init__.py') - self.append('''\ -def _pyopendds_inject_idl(module_name): - from pyopendds.IDL import IDL - from dataclasses import field - idl = IDL() -''') + def __init__(self, context: dict, name: str): + self.submodules = [] + self.module = None + new_context = context.copy() + new_context.update(dict( + output=context['output'] / name, + types=[] + )) + super().__init__(new_context, new_context['output'], + {'__init__.py': 'user.py'}) def write(self): - self.append(''' - idl.inject(module_name) - - -_pyopendds_inject_idl(__name__) -del _pyopendds_inject_idl''') super().write() + for submodule in self.submodules: + submodule.write() def visit_module(self, module): - if module.name.parts: - self.append(" idl.add_module('{}', '{}')".format( - module.parent_name().join(), module.local_name())) - super().visit_module(module) + if self.module: # We already have a module, this is a submodule + submodule = PythonOutput(self.context, module.local_name()) + self.submodules.append(submodule) + submodule.visit_module(module) + else: # This is our module + self.module = module + super().visit_module(module) + + def get_python_type_string(self, field_type): + if isinstance(field_type, PrimitiveType): + return self.primitive_types[field_type.kind][0] + elif field_type in self.module.types.values(): + return field_type.local_name() + else: + raise NotImplementedError + + def get_python_default_value_string(self, field_type): + if isinstance(field_type, PrimitiveType): + return self.primitive_types[field_type.kind][1] + elif field_type in self.module.types.values(): + if isinstance(field_type, StructType): + return field_type.local_name() + '()' + elif isinstance(field_type, EnumType): + return field_type.local_name() + '.' + field_type.default_member + else: + raise NotImplementedError + else: + raise NotImplementedError def visit_struct(self, struct_type): - self.append(" idl.add_struct('{}', '{}', '{}', [".format( - struct_type.parent_name().join(), - struct_type.local_name(), - self.native_package_name if struct_type.is_topic_type else None)) - for name in struct_type.fields: - self.append(" ('{}', 'typing.Any', field(default=None)),".format(name)) - self.append(" ])") + self.context['has_struct'] = True + self.context['types'].append(dict( + local_name=struct_type.local_name(), + type_support=self.context['native_package_name'] if struct_type.is_topic_type else None, + struct=dict( + fields=[dict( + name=name, + type=self.get_python_type_string(node.type_node), + default_value=self.get_python_default_value_string(node.type_node), + ) for name, node in struct_type.fields.items()], + ), + )) def visit_enum(self, enum_type): - self.append(" idl.add_enum('{}', '{}', [".format( - enum_type.parent_name().join(), - enum_type.local_name())) - for name in enum_type.members: - self.append(" '{}',".format(name)) - self.append(" ])") + self.context['has_enum'] = True + self.context['types'].append(dict( + local_name=enum_type.local_name(), + enum=dict( + members=[ + dict(name=name, value=value) for name, value in enum_type.members.items() + ], + ), + )) diff --git a/pyopendds/dev/itl2py/__init__.py b/pyopendds/dev/itl2py/__init__.py index a7c85b5..e69de29 100644 --- a/pyopendds/dev/itl2py/__init__.py +++ b/pyopendds/dev/itl2py/__init__.py @@ -1,14 +0,0 @@ -from .itl import parse_itl -from .ast import get_ast -from .PythonOutput import PythonOutput -from .CppOutput import CppOutput -from .build_files import write_cmakelists_txt, write_setup_py - -__all__ = [ - "parse_itl", - "get_ast", - "PythonOutput", - "CppOutput", - "write_cmakelists_txt", - "write_setup_py", -] diff --git a/pyopendds/dev/itl2py/__main__.py b/pyopendds/dev/itl2py/__main__.py new file mode 100644 index 0000000..ae912bb --- /dev/null +++ b/pyopendds/dev/itl2py/__main__.py @@ -0,0 +1,54 @@ +import sys +from argparse import ArgumentParser +from pathlib import Path + +from .generate import generate + + +def main(): + # Parse Arguments + argparser = ArgumentParser(description='ITL to Python Mapping Generator') + argparser.add_argument('idl_library_cmake_name', + metavar='IDL_LIBRARY_CMAKE_NAME', type=str, + help='Name of the exported CMake library to use.') + argparser.add_argument('itl_files', + metavar='ITL_FILE', type=Path, nargs='+', + help='Files that contain the types definitions.') + argparser.add_argument('-o', '--output', + type=Path, default=Path('.'), + help='Destination directory. The current directory by default.') + argparser.add_argument('-n', '--package-name', + type=str, + help='''\ +Name of the Python package to create. If there is only one ITL file, then by +default this will be \'py\' and the name of the ITL file with the .itl +extension (my_types.itl -> pymy_types). If there are are multiple ITL files, +then this option becomes required.''') + argparser.add_argument('--native-package-name', + type=str, + help='''\ +Name of the native Python extension package to create. By default this is \'_\' +and the name of the Python package.''') + argparser.add_argument('--default-encoding', + type=str, default='utf_8', + help='Default encoding of strings. By default this is UTF-8.') + argparser.add_argument('--dry-run', action='store_true', + help='Don\'t create any files or directories, print out what would be done.') + argparser.add_argument('--dump-ast', action='store_true', + help='Print the AST before processing it') + args = argparser.parse_args() + + # Fill in any missing arguments + if args.package_name is None: + if len(args.itl_files) > 1: + sys.exit('--package-name is required when using multiple ITL files') + args.package_name = 'py' + args.itl_files[0].stem + if args.native_package_name is None: + args.native_package_name = '_' + args.package_name + + # Generate The Python Package + generate(vars(args)) + + +if __name__ == "__main__": + main() diff --git a/pyopendds/dev/itl2py/ast.py b/pyopendds/dev/itl2py/ast.py index 683bed3..62d5cee 100644 --- a/pyopendds/dev/itl2py/ast.py +++ b/pyopendds/dev/itl2py/ast.py @@ -1,4 +1,3 @@ -from pathlib import Path from enum import Enum, auto @@ -41,6 +40,9 @@ def parent_name(self): def accept(self, visitor): raise NotImplementedError + def __repr__(self): + return '<{}: {}>'.format(self.__class__.__name__, self.name.join()) + class Module(Node): @@ -62,6 +64,12 @@ def accept(self, visitor): for submodule in self.submodules.values(): visitor.visit_module(submodule) + def __repr__(self): + if self.name.parts: + return super().__repr__() + else: + return '' + class PrimitiveType(Node): @@ -100,6 +108,21 @@ def is_int(self): def is_string(self): return self.kind == self.Kind.s8 + def __repr__(self): + return '<{} PrimitiveType>'.format(self.kind.name) + + +class FieldType(Node): + + def __init__(self, name, type_node, optional): + super().__init__(None) + self.name = name + self.type_node = type_node + self.optional = optional + + def __repr__(self): + return ''.format(self.name, repr(self.type_node)) + class StructType(Node): @@ -108,7 +131,7 @@ def __init__(self, note): self.fields = {} def add_field(self, name, type_node, optional): - self.fields[name] = (type_node, optional) + self.fields[name] = FieldType(name, type_node, optional) def accept(self, visitor): visitor.visit_struct(self) @@ -142,26 +165,7 @@ def visit_enum(self, enum_type): raise NotImplementedError -class Output(NodeVisitor): - - def __init__(self, path): - self.path = Path(path) - self.contents = [] - - def before(self): - return '' - - def after(self): - return '' - - def write(self): - self.path.write_text(self.before() + ''.join(self.contents) + self.after()) - - def append(self, what): - self.contents.append(what + '\n') - - -def get_ast(types): +def get_ast(types: dict) -> Module: root_module = Module(None, '') for type_node in types.values(): module = root_module diff --git a/pyopendds/dev/itl2py/build_files.py b/pyopendds/dev/itl2py/build_files.py deleted file mode 100644 index 468a6b2..0000000 --- a/pyopendds/dev/itl2py/build_files.py +++ /dev/null @@ -1,58 +0,0 @@ -from pathlib import Path - - -def get_setup_py(python_module_name, native_module_name): - return '''\ -from setuptools import setup, find_packages - -from pyopendds.dev.cmake import CMakeWrapperExtension, CMakeWrapperBuild - - -setup( - name=\'''' + python_module_name + '''\', - packages=find_packages(), - ext_modules=[CMakeWrapperExtension( - name=\'''' + native_module_name + '''\', - cmakelists_dir='.', - )], - cmdclass={'build_ext': CMakeWrapperBuild}, -) -''' - - -def write_setup_py( - output_dir: Path, - python_module_name: str, native_module_name: str, - filename: str = 'setup.py'): - (output_dir / filename).write_text( - get_setup_py(python_module_name, native_module_name)) - - -def get_cmakelists_txt(native_module_name: str, idl_library_name: str): - return '''\ -cmake_minimum_required(VERSION 3.12) -project(''' + native_module_name + ''') - -find_package(Python3 COMPONENTS Development REQUIRED) -find_package(OpenDDS REQUIRED) -find_package(''' + idl_library_name + ''' REQUIRED) - -add_library(''' + native_module_name + ''' SHARED ''' + native_module_name + '''.cpp) -target_link_libraries(''' + native_module_name + ''' PRIVATE Python3::Python) -target_link_libraries(''' + native_module_name + ''' PRIVATE ''' + idl_library_name + ''') - -# Set filename to exactly what Python is expecting -set_target_properties(''' + native_module_name + ''' PROPERTIES - PREFIX "" - LIBRARY_OUTPUT_NAME ${PYOPENDDS_NATIVE_FILENAME} - SUFFIX "" -) -''' - - -def write_cmakelists_txt( - output_dir: Path, - native_module_name: str, idl_library_name: str, - filename: str = 'CMakeLists.txt'): - (output_dir / filename).write_text( - get_cmakelists_txt(native_module_name, idl_library_name)) diff --git a/pyopendds/dev/itl2py/generate.py b/pyopendds/dev/itl2py/generate.py new file mode 100644 index 0000000..97b033c --- /dev/null +++ b/pyopendds/dev/itl2py/generate.py @@ -0,0 +1,89 @@ +import sys +from pathlib import Path +from typing import List +import codecs +import json + +from jinja2 import Environment, PackageLoader + +from .itl import parse_itl +from .ast import get_ast, Module +from .Output import Output +from .PythonOutput import PythonOutput +from .CppOutput import CppOutput + + +def parse_itl_files(itl_files: List[Path]) -> Module: + '''Read and parse a list of ITL file paths, collecting the results and + return an assembled AST. + ''' + + types = {} + for itl_file in itl_files: + with itl_file.open() as f: + parse_itl(types, json.load(f)) + return get_ast(types) + + +class PackageOutput(Output): + '''Wraps the other Outputs and manages build files for the Python package + ''' + + def __init__(self, context: dict): + super().__init__(context, context['output'], { + 'CMakeLists.txt': 'CMakeLists.txt', + 'setup.py': 'setup.py', + }) + self.pyout = PythonOutput(context, context['package_name']) + self.cppout = CppOutput(context) + + def visit_module(self, module): + if self.context['dump_ast']: + print(repr(module)) + super().visit_module(module) + if not module.name.parts: + self.pyout.visit_module(module) + self.cppout.visit_module(module) + + def write(self): + super().write() + self.pyout.write() + self.cppout.write() + + def visit_struct(self, struct_type): + print(repr(struct_type)) + for field_node in struct_type.fields.values(): + print(' ', repr(field_node)) + + def visit_enum(self, enum_type): + print(repr(enum_type)) + for name, value in enum_type.members.items(): + print(' ', name, ':', value) + + +def generate(context: dict) -> None: + '''Generate a Python IDL binding package given a dict of arguments. The + arguments are the following: + - idl_library_cmake_name + - itl_files + - output + - package_name + - native_package_name + - default_encoding + - dry_run + - dump_ast + ''' + + try: + codecs.lookup(context['default_encoding']) + except LookupError: + sys.exit('Invalid Python codec: "{}"'.format(context['default_encoding'])) + + context['jinja_loader'] = PackageLoader('pyopendds.dev.itl2py', 'templates') + context['jinja'] = Environment( + loader=context['jinja_loader'], + ) + + out = PackageOutput(context) + out.visit_module(parse_itl_files(context['itl_files'])) + out.write() diff --git a/pyopendds/dev/itl2py/templates/CMakeLists.txt b/pyopendds/dev/itl2py/templates/CMakeLists.txt new file mode 100644 index 0000000..e54066f --- /dev/null +++ b/pyopendds/dev/itl2py/templates/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.12) +project({{ native_package_name }}) + +find_package(Python3 COMPONENTS Development REQUIRED) +find_package(OpenDDS REQUIRED) +find_package({{ idl_library_cmake_name }} REQUIRED) + +add_library({{ native_package_name }} SHARED {{ native_package_name }}.cpp) +target_link_libraries({{ native_package_name }} PRIVATE Python3::Python) +target_link_libraries({{ native_package_name }} PRIVATE {{ idl_library_cmake_name }}) + +# Set filename to exactly what Python is expecting +set_target_properties({{ native_package_name }} PROPERTIES + PREFIX "" + LIBRARY_OUTPUT_NAME ${PYOPENDDS_NATIVE_FILENAME} + SUFFIX "" +) diff --git a/pyopendds/dev/itl2py/templates/setup.py b/pyopendds/dev/itl2py/templates/setup.py new file mode 100644 index 0000000..b615e17 --- /dev/null +++ b/pyopendds/dev/itl2py/templates/setup.py @@ -0,0 +1,14 @@ +from setuptools import setup, find_packages + +from pyopendds.dev.cmake import CMakeWrapperExtension, CMakeWrapperBuild + + +setup( + name='{{ package_name }}', + packages=find_packages(), + ext_modules=[CMakeWrapperExtension( + name='{{ native_package_name }}', + cmakelists_dir='.', + )], + cmdclass={'build_ext': CMakeWrapperBuild}, +) diff --git a/pyopendds/dev/itl2py/templates/user.cpp b/pyopendds/dev/itl2py/templates/user.cpp new file mode 100644 index 0000000..81f01e7 --- /dev/null +++ b/pyopendds/dev/itl2py/templates/user.cpp @@ -0,0 +1,356 @@ +// Python.h should always be first +#define PY_SSIZE_T_CLEAN +#include + +/*{% for name in idl_names -%}*/ +#include +/*{%- endfor %}*/ + +#include +#include + +#include +#include +#include + +namespace { + +/// Get Contents of Capsule from a PyObject +template +T* get_capsule(PyObject* obj) +{ + T* rv = nullptr; + PyObject* capsule = PyObject_GetAttrString(obj, "_var"); + if (capsule && PyCapsule_IsValid(capsule, nullptr)) { + rv = static_cast(PyCapsule_GetPointer(capsule, nullptr)); + } + return rv; +} + +// Python Objects To Keep +PyObject* pyopendds; +PyObject* PyOpenDDS_Error; +PyObject* ReturnCodeError; + +bool cache_python_objects() +{ + // Get pyopendds + pyopendds = PyImport_ImportModule("pyopendds"); + if (!pyopendds) return true; + + // Get PyOpenDDS_Error + PyOpenDDS_Error = PyObject_GetAttrString(pyopendds, "PyOpenDDS_Error"); + if (!PyOpenDDS_Error) return true; + Py_INCREF(PyOpenDDS_Error); + + // Get ReturnCodeError + ReturnCodeError = PyObject_GetAttrString(pyopendds, "ReturnCodeError"); + if (!ReturnCodeError) return true; + Py_INCREF(ReturnCodeError); + + return false; +} + +bool check_rc(DDS::ReturnCode_t rc) +{ + return !PyObject_CallMethod(ReturnCodeError, "check", "k", rc); +} + +class Exception : public std::exception { +public: + Exception(const char* message) + : message_(message) + { + } + + virtual const char* what() const noexcept + { + return message_; + } + +private: + const char* message_; +}; + +class TypeBase { +public: + virtual void register_type(PyObject* pyparticipant) = 0; + virtual PyObject* get_python_class() = 0; + virtual const char* type_name() = 0; + virtual PyObject* read(PyObject* pyreader) = 0; +}; + +template +class TemplatedTypeBase : public TypeBase { +public: + typedef typename OpenDDS::DCPS::DDSTraits Traits; + + typedef T IdlType; + typedef typename Traits::MessageSequenceType IdlTypeSequence; + + typedef typename Traits::TypeSupportType TypeSupport; + typedef typename Traits::TypeSupportTypeImpl TypeSupportImpl; + typedef typename Traits::DataWriterType DataWriter; + typedef typename Traits::DataReaderType DataReader; + + const char* type_name() + { + return Traits::type_name(); + } + + /** + * Callback for Python to call when the TypeSupport capsule is deleted + */ + static void delete_typesupport(PyObject* capsule) + { + if (PyCapsule_CheckExact(capsule)) { + delete static_cast( + PyCapsule_GetPointer(capsule, NULL)); + } + } + + void register_type(PyObject* pyparticipant) + { + // Get DomainParticipant_var + DDS::DomainParticipant* participant = + get_capsule(pyparticipant); + if (!participant) { + throw Exception("Could not get native particpant"); + } + + // Register with OpenDDS + TypeSupportImpl* type_support = new TypeSupportImpl; + if (type_support->register_type(participant, "") != DDS::RETCODE_OK) { + delete type_support; + type_support = 0; + throw Exception("Could not create register type"); + } + + // Store TypeSupport in Python Participant + PyObject* capsule = PyCapsule_New(participant, NULL, delete_typesupport); + if (!capsule) { + throw Exception("Could not create ts capsule"); + } + PyObject* list = PyObject_GetAttrString( + pyparticipant, "_registered_typesupport"); + if (!list || !PyList_Check(list)) { + throw Exception("Could not get ts list"); + } + if (PyList_Append(list, capsule)) { + PyErr_Print(); + throw Exception("Could not append ts to list"); + } + } + + virtual void to_python(const T& cpp, PyObject*& py) = 0; + + PyObject* read(PyObject* pyreader) + { + DDS::DataReader* reader = get_capsule(pyreader); + if (!reader) { + PyErr_SetString(PyOpenDDS_Error, "Could not get datareader"); + return nullptr; + } + + DataReader* reader_impl = DataReader::_narrow(reader); + if (!reader_impl) { + PyErr_SetString(PyOpenDDS_Error, "Could not narrow reader implementation"); + return nullptr; + } + + DDS::ReturnCode_t rc; + DDS::ReadCondition_var read_condition = reader_impl->create_readcondition( + DDS::ANY_SAMPLE_STATE, DDS::ANY_VIEW_STATE, DDS::ANY_SAMPLE_STATE); + DDS::WaitSet_var ws = new DDS::WaitSet; + ws->attach_condition(read_condition); + DDS::ConditionSeq active; + const DDS::Duration_t max_wait_time = {10, 0}; + rc = ws->wait(active, max_wait_time); + ws->detach_condition(read_condition); + reader_impl->delete_readcondition(read_condition); + + T sample; + DDS::SampleInfo info; + if (check_rc(reader_impl->take_next_sample(sample, info))) return nullptr; + PyObject *rv = nullptr; + to_python(sample, rv); + return rv; + } + + virtual T from_python(PyObject* py) = 0; +}; +template class Type; + +typedef std::shared_ptr TypePtr; +typedef std::map Types; +Types types; + +template +void init_type() +{ + TypePtr type{new Type}; + types.insert(Types::value_type(type->get_python_class(), type)); +} + +long get_python_long_attr(PyObject* py, const char* attr_name) +{ + PyObject* attr = PyObject_GetAttrString(py, attr_name); + if (!attr) { + PyErr_Print(); + throw Exception("python error occured"); + } + if (!PyLong_Check(attr)) { + throw Exception("python attribute isn't an int"); + } + long long_value = PyLong_AsLong(attr); + if (long_value == -1 && PyErr_Occurred()) { + PyErr_Print(); + throw Exception("python error occured"); + } + return long_value; +} + +/*{% for type in types -%}*/ +template<> +class Type : public TemplatedTypeBase { +public: + PyObject* get_python_class() + { + if (!python_class_) { + PyObject* module = PyImport_ImportModule("/*{{ package_name }}*/"); + if (!module) return nullptr; + + /*{% for name in type.name_parts -%}*/ + module = PyObject_GetAttrString(module, "/*{{ name }}*/"); + if (!module) return nullptr; + /*{%- endfor %}*/ + + python_class_ = PyObject_GetAttrString(module, "/*{{ type.local_name }}*/"); + } + return python_class_; + } + + void to_python(const /*{{ type.cpp_name }}*/& cpp, PyObject*& py) + { + PyObject* cls = get_python_class(); + if (py) { + if (PyObject_IsInstance(cls, py) != 1) { + throw Exception("Python object is not a valid type"); + } + } else { + py = PyObject_CallObject(cls, nullptr); + if (!py) { + PyErr_Print(); + throw Exception("Could not call __init__ for new class"); + } + } + + PyObject* field_value; + + /*{{ type.to_lines | indent }}*/ + } + + /*{{ type.cpp_name }}*/ from_python(PyObject* py) + { + /*{{ type.cpp_name }}*/ rv; + + PyObject* cls = get_python_class(); + if (PyObject_IsInstance(py, cls) != 1) { + throw Exception("Python object is not a valid type"); + } + + /*{{ type.from_lines | indent }}*/ + + return rv; + } + +private: + PyObject* python_class_ = nullptr; +}; +/*{%- endfor %}*/ + +} // Anonymous Namespace\n + +static PyObject* pyregister_type(PyObject* self, PyObject* args) +{ + // Get Arguments + PyObject* pyparticipant; + PyObject* pytype; + if (!PyArg_ParseTuple(args, "OO", &pyparticipant, &pytype)) { + PyErr_SetString(PyExc_TypeError, "Invalid Arguments"); + return NULL; + } + + Types::iterator i = types.find(pytype); + if (i != types.end()) { + i->second->register_type(pyparticipant); + Py_RETURN_NONE; + } + + PyErr_SetString(PyExc_TypeError, "Invalid Type"); + return NULL; +} + +static PyObject* pytype_name(PyObject* self, PyObject* args) +{ + // Get Arguments + PyObject* pytype; + if (!PyArg_ParseTuple(args, "O", &pytype)) { + PyErr_SetString(PyExc_TypeError, "Invalid Arguments"); + return NULL; + } + + Types::iterator i = types.find(pytype); + if (i != types.end()) { + return PyUnicode_FromString(i->second->type_name()); + } + + PyErr_SetString(PyExc_TypeError, "Invalid Type"); + return nullptr; +} + +static PyObject* pyread(PyObject* self, PyObject* args) +{ + // Get Arguments + PyObject* pyreader; + if (!PyArg_ParseTuple(args, "O", &pyreader)) return nullptr; + + // Try to Get Topic Type and Do Read + PyObject* pytopic = PyObject_GetAttrString(pyreader, "topic"); + if (!pytopic) return nullptr; + PyObject* pytype = PyObject_GetAttrString(pytopic, "type"); + if (!pytype) return nullptr; + Types::iterator i = types.find(pytype); + if (i != types.end()) { + return i->second->read(pyreader); + } + + PyErr_SetString(PyExc_TypeError, "Invalid Type"); + return NULL; +} + +static PyMethodDef /*{{ native_package_name }}*/_Methods[] = { + {"register_type", pyregister_type, METH_VARARGS, ""}, + {"type_name", pytype_name, METH_VARARGS, ""}, + {"read", pyread, METH_VARARGS, ""}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef /*{{ native_package_name }}*/_Module = { + PyModuleDef_HEAD_INIT, + "/*{{ native_package_name }}*/", "", + -1, // Global State Module, because OpenDDS uses Singletons + /*{{ native_package_name }}*/_Methods +}; + +PyMODINIT_FUNC PyInit_/*{{ native_package_name }}*/() +{ + PyObject* module = PyModule_Create(&/*{{ native_package_name }}*/_Module); + if (!module || cache_python_objects()) return nullptr; + + /*{% for topic_type in topic_types -%}*/ + init_type(); + /*{%- endfor %}*/ + + return module; +} diff --git a/pyopendds/dev/itl2py/templates/user.py b/pyopendds/dev/itl2py/templates/user.py new file mode 100644 index 0000000..2573822 --- /dev/null +++ b/pyopendds/dev/itl2py/templates/user.py @@ -0,0 +1,26 @@ +{% if has_struct -%} +from dataclasses import dataclass as _pyopendds_struct +{%- endif %} +{% if has_enum -%} +from enum import IntEnum as _pyopendds_enum +{%- endif %} +{% for type in types -%} +{%- if type.struct %} + +@_pyopendds_struct +class {{ type.local_name }}: +{%- if type.type_support %} + _pyopendds_typesupport_packge_name = '{{ type.type_support }}' +{% endif -%} +{%- for field in type.struct.fields %} + {{ field.name }}: {{ field.type }} = {{ field.default_value }} +{%- endfor %} +{%- elif type.enum %} +class {{ type.local_name }}(_pyopendds_enum): +{%- for member in type.enum.members %} + {{ member.name }} = {{ member.value }} +{%- endfor %} +{%- else %} +# {{ type.local_name }} was left unimplmented +{% endif -%} +{%- endfor -%} diff --git a/scripts/itl2py b/scripts/itl2py deleted file mode 100755 index fe637b8..0000000 --- a/scripts/itl2py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 - -# A Prototype for Generating a Python Mapping from an ITL File - -import sys -from argparse import ArgumentParser -from pathlib import Path -import json -import codecs - -from pyopendds.dev.itl2py import * - -if __name__ == "__main__": - # Parse Arguments - argparser = ArgumentParser(description='ITL to Python Mapping') - argparser.add_argument('itl_files', metavar='ITL_FILE', type=Path, nargs='+') - argparser.add_argument('-n', '--package-name', type=str) - argparser.add_argument('-o', '--output', type=Path, default=Path('.')) - argparser.add_argument('--cmake-idl-name', type=str, default=None) - argparser.add_argument('--default-encoding', type=str, default='utf_8') - args = argparser.parse_args() - - try: - codecs.lookup(args.default_encoding) - except LookupError: - sys.exit('Invalid Python codec: "{}"'.format(args.default_encoding)) - - if args.package_name is None: - if len(args.itl_files) > 1: - sys.exit('--package-name is required when using multiple ITL files') - args.package_name = 'py' + args.itl_files[0].stem - if args.cmake_idl_name is None: - args.cmake_idl_name = args.itl_files[0].stem + '_idl' - - # Parse ITL Files for Types - types = {} - idl_names = [] - for itl_file in args.itl_files: - idl_names.append(itl_file.name[:-len('.itl')]) - with itl_file.open() as f: - parse_itl(types, json.load(f)) - ast = get_ast(types) - - # TODO - # # If the package only contains a single module of the same name, eliminate - # # that module. - # if len(ast.submodules) == 1: - # submodule = next(iter(ast.submodules.values())) - # if submodule.local_name() == args.package_name: - # submodule.parent = None - # submodule.set_name(parts=[]) - # ast = submodule - - python_package_name = args.package_name - native_package_name = '_' + args.package_name - - pyout = PythonOutput(args.output, python_package_name, native_package_name) - pyout.visit_module(ast) - pyout.write() - - cppout = CppOutput( - args.output, python_package_name, native_package_name, idl_names, - args.default_encoding) - cppout.visit_module(ast) - cppout.write() - - write_setup_py(args.output, python_package_name, native_package_name) - write_cmakelists_txt(args.output, native_package_name, args.cmake_idl_name) diff --git a/setup.cfg b/setup.cfg index f9ee6f6..9643c78 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,9 +20,14 @@ classifiers = Topic :: Communications Topic :: Software Development :: Code Generators Topic :: Software Development :: Libraries +install_requires = + jinja2 [flake8] max-line-length = 100 ignore = E128, W503, +exclude = + pyopendds/dev/itl2py/templates/* + tests/basic_test/build/* diff --git a/setup.py b/setup.py index f221a46..73991fa 100644 --- a/setup.py +++ b/setup.py @@ -9,5 +9,10 @@ cmakelists_dir='pyopendds/ext', )], cmdclass={'build_ext': CMakeWrapperBuild}, - scripts=['scripts/itl2py'], + entry_points={ + 'console_scripts': [ + 'itl2py=pyopendds.dev.itl2py.__main__:main', + ], + }, + package_data={'pyopendds.dev.itl2py': ['templates/*']}, ) diff --git a/tests/basic_test/subscriber.py b/tests/basic_test/subscriber.py index 18d4066..7541d2f 100644 --- a/tests/basic_test/subscriber.py +++ b/tests/basic_test/subscriber.py @@ -1,7 +1,7 @@ import sys from pyopendds import Config, DomainParticipant, StatusKind, PyOpenDDS_Error -import pybasic +import pybasic.basic if __name__ == "__main__": try: