From 7e8010a909b719f95cf43ad6e2b0a6c4c15dc21d Mon Sep 17 00:00:00 2001 From: "Ralph J. Steinhagen" <46007894+RalphSteinhagen@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:43:27 +0200 Subject: [PATCH] new Feature: PythonBlock and PythonInterpreter * added PythonBlock ... needed to open-up 'applyChangedSettings()' to allow throwing of exceptions. * review-comment: added guards to prevent #port size changes while being connected. Signed-off-by: Ralph J. Steinhagen --- CMakeLists.txt | 29 ++ .../gnuradio-4.0/basic/PythonBlock.hpp | 418 +++++++++++++++++ .../gnuradio-4.0/basic/PythonInterpreter.hpp | 422 ++++++++++++++++++ blocks/basic/test/CMakeLists.txt | 6 +- blocks/basic/test/qa_PythonBlock.cpp | 273 +++++++++++ cmake/CheckNumPy.py | 6 + core/include/gnuradio-4.0/Settings.hpp | 4 +- core/test/CMakeLists.txt | 6 +- 8 files changed, 1160 insertions(+), 4 deletions(-) create mode 100644 blocks/basic/include/gnuradio-4.0/basic/PythonBlock.hpp create mode 100644 blocks/basic/include/gnuradio-4.0/basic/PythonInterpreter.hpp create mode 100644 blocks/basic/test/qa_PythonBlock.cpp create mode 100644 cmake/CheckNumPy.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 7111f724..f1c29602 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,6 +193,35 @@ target_include_directories(fftw INTERFACE ${FFTW_PREFIX}/install/include ${PROJE target_link_directories(fftw INTERFACE ${FFTW_PREFIX}/install/lib ${FFTW_PREFIX}/install/lib64) add_dependencies(fftw fftw_ext) + +## check for CPython and Numpy dependencies +set(PYTHON_FORCE_INCLUDE OFF) +if(PYTHON_FORCE_INCLUDE) + find_package(Python3 3.12 REQUIRED COMPONENTS Interpreter Development NumPy) +else() + find_package(Python3 3.12 COMPONENTS Interpreter Development NumPy) +endif() + +set(PYTHON_AVAILABLE OFF) +if(Python3_FOUND) + execute_process( + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/CheckNumPy.py" + RESULT_VARIABLE NUMPY_NOT_FOUND + OUTPUT_VARIABLE NUMPY_INCLUDE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + # If NumPy is found, set PYTHON_AVAILABLE to ON + if(NOT NUMPY_NOT_FOUND) + set(PYTHON_AVAILABLE ON) + include_directories(${Python3_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR}) + add_definitions(-DPYTHON_AVAILABLE) + message(STATUS "Using Python Include Dirs: ${Python3_INCLUDE_DIRS} and ${NUMPY_INCLUDE_DIR}") + else() + message(STATUS "Python and Numpy Include headers not found!!") + endif() +endif() + option(ENABLE_EXAMPLES "Enable Example Builds" ${GR_TOPLEVEL_PROJECT}) option(ENABLE_TESTING "Enable Test Builds" ${GR_TOPLEVEL_PROJECT}) diff --git a/blocks/basic/include/gnuradio-4.0/basic/PythonBlock.hpp b/blocks/basic/include/gnuradio-4.0/basic/PythonBlock.hpp new file mode 100644 index 00000000..a37d7a5e --- /dev/null +++ b/blocks/basic/include/gnuradio-4.0/basic/PythonBlock.hpp @@ -0,0 +1,418 @@ +#ifndef GNURADIO_PYTHONBLOCK_HPP +#define GNURADIO_PYTHONBLOCK_HPP + +#include "PythonInterpreter.hpp" + +#include +#include + +// Forward declaration of PythonBlock method definition, needed for CPython's C-API wrapping +template +inline PyModuleDef * +myBlockPythonDefinitions(void); + +namespace gr::basic { + +using namespace gr; + +template + requires std::is_arithmetic_v /* || std::is_same_v> || std::is_same_v> */ +struct PythonBlock : public Block> { + using Description = Doc + +// [...] +int main() { +// Python script that processes input data arrays and modifies output arrays +std::string pythonScript = R"( +# usual import etc. +counter = 0 # exemplary global state, kept between each invocation + +def process_bulk(ins, outs): + # [..] + settings = this_block.getSettings() + print("Current settings:", settings) + + if this_block.tagAvailable(): # tag handling + tag = this_block.getTag() + print('Tag:', tag) + + counter += 1 + # process the input->output samples, here: double each input element + for i in range(len(ins)): + outs[i][:] = ins[i] * 2 + + # update settings with the counter + settings["counter"] = str(counter) + this_block.setSettings(settings) + + # [..] +)"; + +// C++ side: instantiate PythonBlock with the script +PythonBlock myBlock(pythonScript); // nominal +myBlock.pythonScript = pythonScript; // alt: only for unit-testing + +// example for unit-test +std::vector data1 = { 1, 2, 3 }; +std::vector data2 = { 4, 5, 6 }; +std::vector out1(3); // need std::vector as backing storage +std::vector out2(3); +std::vector> ins = { data1, data2 }; +std::vector> outs = { out1, out2 }; + +// process data using the Python script +myBlock.processBulk(ins, outs); +// check values of outs +} +@endcode +)"">; + // optional shortening + template + using A = Annotated; + using poc_property_map = std::map>; // TODO: needs to be replaced with 'property_map` aka. 'pmtv::map_t' + using tag_type = std::string; + + std::vector> inputs{}; + std::vector> outputs{}; + A, Limits<1U, 32U>> n_inputs = 0U; + A, Limits<1U, 32U>> n_outputs = 0U; + std::string pythonScript = ""; + + PyModuleDef *_moduleDefinitions = myBlockPythonDefinitions(); + python::Interpreter _interpreter{ this, _moduleDefinitions }; + std::string _prePythonDefinition = fmt::format(R"p(import {0} +import warnings + +class WarningException(Exception): + """Custom exception class for handling warnings as exceptions with detailed messages.""" + def __init__(self, message, filename=None, lineno=None, category_name=None): + super().__init__(message) + self.filename = filename + self.lineno = lineno + self.category_name = category_name + +def custom_showwarning(message, category, filename, lineno, file=None, line=None): + raise WarningException(f"{{filename}}:{{lineno}}: {{category.__name__}}: {{message}}", filename=filename, lineno=lineno, category_name=category.__name__) # raise warning as an exception +warnings.showwarning = custom_showwarning +warnings.simplefilter('always') # trigger on all warnings, can be adjusted as needed + +class PythonBlockWrapper: ## helper class to make the C++ class appear as a Python class + def __init__(self, capsule): + self.capsule = capsule + def tagAvailable(self): + return {0}.tagAvailable(self.capsule) + def getTag(self): + return {0}.getTag(self.capsule) + def getSettings(self): + return {0}.getSettings(self.capsule) + def setSettings(self, settings): + {0}.setSettings(self.capsule, settings) + +this_block = PythonBlockWrapper(capsule))p", + _moduleDefinitions->m_name); + poc_property_map _settingsMap{ { "key1", "value1" }, { "key2", "value2" } }; + bool _tagAvailable = false; + tag_type _tag = "Simulated Tag"; + + void + settingsChanged(const gr::property_map &old_settings, const gr::property_map &new_settings) { + if (new_settings.contains("n_inputs") || new_settings.contains("n_outputs")) { + + fmt::print("{}: configuration changed: n_inputs {} -> {}, n_outputs {} -> {}\n", this->name, old_settings.at("n_inputs"), + new_settings.contains("n_inputs") ? new_settings.at("n_inputs") : "same", old_settings.at("n_outputs"), + new_settings.contains("n_outputs") ? new_settings.at("n_outputs") : "same"); + if (std::any_of(inputs.begin(), inputs.end(), [](const auto &port) { return port.isConnected(); })) { + throw gr::exception("Number of input ports cannot be changed after Graph initialization."); + } + if (std::any_of(outputs.begin(), outputs.end(), [](const auto &port) { return port.isConnected(); })) { + throw gr::exception("Number of output ports cannot be changed after Graph initialization."); + } + inputs.resize(n_inputs); + outputs.resize(n_outputs); + } + + if (new_settings.contains("pythonScript")) { + _interpreter.invoke( + [this] { + if (python::PyObjectGuard testImport(PyRun_StringFlags(_prePythonDefinition.data(), Py_file_input, _interpreter.getDictionary(), _interpreter.getDictionary(), nullptr)); + !testImport) { + python::throwCurrentPythonError(fmt::format("{}(aka. {})::settingsChanged(...) - testImport", this->unique_name, this->name), std::source_location::current(), + _prePythonDefinition); + } + + // Retrieve the PythonBlockWrapper class object + PyObject *pPythonBlockWrapperClass = PyDict_GetItemString(_interpreter.getDictionary(), "PythonBlockWrapper"); // borrowed reference + if (!pPythonBlockWrapperClass) { + python::throwCurrentPythonError(fmt::format("{}(aka. {})::settingsChanged(...) - failed to retrieve PythonBlockWrapper class", this->unique_name, this->name), + std::source_location::current(), _prePythonDefinition); + } + + // Retrieve the this_block + PyObject *pInstance = PyDict_GetItemString(_interpreter.getDictionary(), "this_block"); // borrowed reference + if (!pInstance) { + python::throwCurrentPythonError(fmt::format("{}(aka. {})::settingsChanged(...) - failed to retrieve 'this_block'", this->unique_name, this->name), + std::source_location::current(), _prePythonDefinition); + } + + // Check if pInstance is an instance of PythonBlockWrapper + if (PyObject_IsInstance(pInstance, pPythonBlockWrapperClass) != 1) { + python::throwCurrentPythonError(fmt::format("{}(aka. {})::settingsChanged(...) - 'this_block' is not an instance of PythonBlockWrapper", this->unique_name, this->name), + std::source_location::current(), _prePythonDefinition); + } + + if (const python::PyObjectGuard result(PyRun_StringFlags(pythonScript.data(), Py_file_input, _interpreter.getDictionary(), _interpreter.getDictionary(), nullptr)); !result) { + python::throwCurrentPythonError(fmt::format("{}(aka. '{}')::settingsChanged(...) - script parsing error", this->unique_name, this->name), std::source_location::current(), + pythonScript); + } + + python::PyObjectGuard pyFunc(PyObject_GetAttrString(_interpreter.getModule(), "process_bulk")); + if (!pyFunc.get() || !PyCallable_Check(pyFunc.get())) { + python::throwCurrentPythonError(fmt::format("{}(aka. {})::settingsChanged(...) Python function process_bulk not found or is not callable", this->unique_name, this->name), + std::source_location::current(), pythonScript); + } + }, + pythonScript); + } + } + + const poc_property_map & + getSettings() const { + // TODO: replace with this->settings().get() once the property_map is Python wrapped + return _settingsMap; + } + + bool + setSettings(const poc_property_map &newSettings) { + // TODO: replace with this->settings().set(newSettings) once the property_map is Python wrapped + if (newSettings.empty()) { + return false; + } + for (const auto &[key, value] : newSettings) { + _settingsMap.insert_or_assign(key, value); + } + return true; + } + + bool + tagAvailable() { + _tagAvailable = !_tagAvailable; + return _tagAvailable; + } + + tag_type + getTag() { + return _tag; + } + + template + work::Status + processBulk(std::span ins, std::span outs) { + _interpreter.invoke([this, ins, outs] { callPythonFunction(ins, outs); }, pythonScript); + return work::Status::OK; + } + + // block life-cycle methods + // clang-format off + void start() { _interpreter.invokeFunction("start"); } + void stop() { _interpreter.invokeFunction("stop"); } + void pause() { _interpreter.invokeFunction("pause"); } + void resume() { _interpreter.invokeFunction("resume"); } + void reset() { _interpreter.invokeFunction("reset"); } + // clang-format on + +private: + template + void + callPythonFunction(std::span ins, std::span outs) { + PyObject *pIns = PyList_New(static_cast(ins.size())); + for (size_t i = 0; i < ins.size(); ++i) { + PyList_SetItem(pIns, Py_ssize_t(i), python::toPyArray(ins[i].data(), { ins[i].size() })); + } + + PyObject *pOuts = PyList_New(static_cast(outs.size())); + for (size_t i = 0; i < outs.size(); ++i) { + PyList_SetItem(pOuts, Py_ssize_t(i), python::toPyArray(outs[i].data(), { outs[i].size() })); + } + + python::PyObjectGuard pyArgs(PyTuple_New(2)); + PyTuple_SetItem(pyArgs, 0, pIns); + PyTuple_SetItem(pyArgs, 1, pOuts); + + if (python::PyObjectGuard pyValue = _interpreter.invokeFunction("process_bulk", pyArgs); !pyValue) { + python::throwCurrentPythonError(fmt::format("{}(aka. {})::callPythonFunction(..) Python function call failed", this->unique_name, this->name), std::source_location::current(), + pythonScript); + } + } +}; + +} // namespace gr::basic +ENABLE_REFLECTION_FOR_TEMPLATE(gr::basic::PythonBlock, inputs, outputs, n_inputs, n_outputs, pythonScript) + +template +gr::basic::PythonBlock * +getPythonBlockFromCapsule(PyObject *capsule) { + static std::string pyBlockName = gr::python::sanitizedPythonBlockName>(); + if (void *objPointer = PyCapsule_GetPointer(capsule, pyBlockName.c_str()); objPointer != nullptr) { + return static_cast *>(objPointer); + } + gr::python::throwCurrentPythonError("could not retrieve obj pointer from capsule"); + return nullptr; +} + +template +PyObject * +PythonBlock_TagAvailable_Template(PyObject * /*self*/, PyObject *args) { + PyObject *capsule; + if (!PyArg_ParseTuple(args, "O", &capsule)) { + return nullptr; + } + gr::basic::PythonBlock *myBlock = getPythonBlockFromCapsule(capsule); + return myBlock->tagAvailable() ? gr::python::TrueObj : gr::python::FalseObj; +} + +template +PyObject * +PythonBlock_GetTag_Template(PyObject * /*self*/, PyObject *args) { + PyObject *capsule; + if (!PyArg_ParseTuple(args, "O", &capsule)) { + return nullptr; + } + gr::basic::PythonBlock *myBlock = getPythonBlockFromCapsule(capsule); + return PyUnicode_FromString(myBlock->getTag().c_str()); +} + +template +PyObject * +PythonBlock_GetSettings_Template(PyObject * /*self*/, PyObject *args) { + PyObject *capsule; + if (!PyArg_ParseTuple(args, "O", &capsule)) { + return nullptr; + } + const gr::basic::PythonBlock *myBlock = getPythonBlockFromCapsule(capsule); + if (myBlock == nullptr) { + gr::python::throwCurrentPythonError(fmt::format("could not retrieve myBLock<{}> {}", gr::meta::type_name(), gr::python::toString(capsule))); + return nullptr; + } + const auto &settings = myBlock->getSettings(); + + PyObject *dict = PyDict_New(); // returns owning reference + if (!dict) { + return PyErr_NoMemory(); + } + try { + for (const auto &[key, value] : settings) { + gr::python::PyObjectGuard py_value(PyUnicode_FromString(value.c_str())); + if (!py_value) { // Failed to convert string to Python Unicode + gr::python::PyDecRef(dict); + return PyErr_NoMemory(); + } + // PyDict_SetItemString does not steal reference, so no need to decref py_value + if (PyDict_SetItemString(dict, key.c_str(), py_value) != 0) { + gr::python::PyDecRef(dict); + return nullptr; + } + } + } catch (const std::exception &e) { + PyErr_SetString(PyExc_RuntimeError, e.what()); + gr::python::PyDecRef(dict); + return nullptr; + } + return dict; +} + +template +PyObject * +PythonBlock_SetSettings_Template(PyObject * /*self*/, PyObject *args) { + PyObject *capsule; + PyObject *settingsDict; + if (!PyArg_ParseTuple(args, "OO", &capsule, &settingsDict)) { + return nullptr; + } + gr::basic::PythonBlock *myBlock = getPythonBlockFromCapsule(capsule); + if (!gr::python::isPyDict(settingsDict)) { + PyErr_SetString(PyExc_TypeError, "Settings must be a dictionary"); + return nullptr; + } + + typename gr::basic::PythonBlock::poc_property_map newSettings; + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(settingsDict, &pos, &key, &value)) { + const char *keyStr = PyUnicode_AsUTF8(key); + const char *valueStr = PyUnicode_AsUTF8(value); + newSettings[keyStr] = valueStr; + } + + myBlock->setSettings(newSettings); + return gr::python::NoneObj; +} + +template +PyMethodDef * +blockMethods() { + static PyMethodDef methods[] = { + { "tagAvailable", reinterpret_cast(PythonBlock_TagAvailable_Template), METH_VARARGS, "Check if a tag is available" }, + { "getTag", reinterpret_cast(PythonBlock_GetTag_Template), METH_VARARGS, "Get the current tag" }, + { "getSettings", reinterpret_cast(PythonBlock_GetSettings_Template), METH_VARARGS, "Get the settings" }, + { "setSettings", reinterpret_cast(PythonBlock_SetSettings_Template), METH_VARARGS, "Set the settings" }, + { nullptr, nullptr, 0, nullptr } // Sentinel + }; + static_assert(gr::meta::always_false, "type not defined"); + return methods; +} + +#define DEFINE_PYTHON_WRAPPER(T, NAME) \ + extern "C" inline PyObject *NAME##_##T(PyObject *self, PyObject *args) { return NAME##_Template(self, args); } + +#define DEFINE_PYTHON_TYPE_FUNCTIONS_AND_METHODS(type) \ + DEFINE_PYTHON_WRAPPER(type, PythonBlock_TagAvailable) \ + DEFINE_PYTHON_WRAPPER(type, PythonBlock_GetTag) \ + DEFINE_PYTHON_WRAPPER(type, PythonBlock_GetSettings) \ + DEFINE_PYTHON_WRAPPER(type, PythonBlock_SetSettings) \ + template<> \ + PyMethodDef *blockMethods() { \ + static PyMethodDef methods[] = { \ + { "tagAvailable", reinterpret_cast(PythonBlock_TagAvailable_##type), METH_VARARGS, "Check if a tag is available" }, \ + { "getTag", reinterpret_cast(PythonBlock_GetTag_##type), METH_VARARGS, "Get the current tag" }, \ + { "getSettings", reinterpret_cast(PythonBlock_GetSettings_##type), METH_VARARGS, "Get the settings" }, \ + { "setSettings", reinterpret_cast(PythonBlock_SetSettings_##type), METH_VARARGS, "Set the settings" }, \ + { nullptr, nullptr, 0, nullptr } /* Sentinel */ \ + }; \ + return methods; \ + } + +DEFINE_PYTHON_TYPE_FUNCTIONS_AND_METHODS(int32_t) +DEFINE_PYTHON_TYPE_FUNCTIONS_AND_METHODS(float) + +// add more types as needed + +template +inline PyModuleDef * +myBlockPythonDefinitions(void) { + static std::string pyBlockName = gr::python::sanitizedPythonBlockName>(); + static PyMethodDef *pyBlockMethods = blockMethods(); + + constexpr auto blockDescription = static_cast(gr::basic::PythonBlock::Description::value); + static struct PyModuleDef myBlockModule = { .m_base = PyModuleDef_HEAD_INIT, + .m_name = pyBlockName.c_str(), + .m_doc = blockDescription.data(), + .m_size = -1, + .m_methods = pyBlockMethods, + .m_slots = nullptr, + .m_traverse = nullptr, + .m_clear = nullptr, + .m_free = nullptr }; + return &myBlockModule; +} + +#endif // GNURADIO_PYTHONBLOCK_HPP diff --git a/blocks/basic/include/gnuradio-4.0/basic/PythonInterpreter.hpp b/blocks/basic/include/gnuradio-4.0/basic/PythonInterpreter.hpp new file mode 100644 index 00000000..fdb93a1f --- /dev/null +++ b/blocks/basic/include/gnuradio-4.0/basic/PythonInterpreter.hpp @@ -0,0 +1,422 @@ +#ifndef GNURADIO_PYTHONINTERPRETER_HPP +#define GNURADIO_PYTHONINTERPRETER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" +#ifndef __clang__ +#pragma GCC diagnostic ignored "-Wuseless-cast" +#endif +#endif +#include + +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include + +namespace gr::python { + +inline static PyObject *TrueObj = Py_True; +inline static PyObject *FalseObj = Py_False; +inline static PyObject *NoneObj = Py_None; + +constexpr inline bool +isPyDict(const PyObject *obj) { + return PyDict_Check(obj); +} + +constexpr inline void +PyDecRef(PyObject *obj) { // wrapper to isolate unsafe warning on C-API casts + Py_XDECREF(obj); +} + +constexpr inline void +PyIncRef(PyObject *obj) { // wrapper to isolate unsafe warning on C-API casts + Py_XINCREF(obj); +} + +constexpr inline std::string +PyBytesAsString(PyObject *op) { + return PyBytes_AsString(op); +} + +class PyObjectGuard { + PyObject *_ptr; + + void + move(PyObjectGuard &&other) noexcept { + PyDecRef(_ptr); + std::swap(_ptr, other._ptr); + } + +public: + explicit PyObjectGuard(PyObject *ptr = nullptr) : _ptr(ptr) {} + + explicit PyObjectGuard(PyObjectGuard &&other) noexcept : _ptr(other._ptr) { move(std::move(other)); } + + ~PyObjectGuard() { PyDecRef(_ptr); } + + PyObjectGuard & + operator=(PyObjectGuard &&other) noexcept { + if (this != &other) { + move(std::move(other)); + } + return *this; + } + + PyObjectGuard(const PyObjectGuard& other) : _ptr(other._ptr) { // copy constructor + PyIncRef(_ptr); + } + + PyObjectGuard& operator=(const PyObjectGuard& other) { + if (this == &other) { + return *this; + } + _ptr = other._ptr; + PyIncRef(_ptr); + return *this; + } + + + operator PyObject *() const { return _ptr; } + + PyObject * + get() const { + return _ptr; + } +}; + +class PyGILGuard { + PyGILState_STATE _state; + +public: + PyGILGuard() : _state(PyGILState_Ensure()) {} + + ~PyGILGuard() { PyGILState_Release(_state); } + + PyGILGuard(const PyGILGuard &) = delete; + PyGILGuard & + operator=(const PyGILGuard &) + = delete; +}; + +[[nodiscard]] inline std::string +toString(PyObject *object) { + PyObjectGuard strObj(PyObject_Repr(object)); + PyObjectGuard bytesObj(PyUnicode_AsEncodedString(strObj.get(), "utf-8", "strict")); + return python::PyBytesAsString(bytesObj.get()); +} + +[[nodiscard]] inline std::string +toLineCountAnnotated(std::string_view code, std::size_t min = 0UZ, std::size_t max = std::numeric_limits::max(), std::size_t marker = std::numeric_limits::max() - 1UZ) { + if (code.empty()) { + return ""; + } + auto splitLines = [](std::string_view str) { + std::istringstream stream{ std::string(str) }; // Convert string_view to string + std::vector lines; + std::string line; + while (std::getline(stream, line)) { + lines.push_back(line); + } + return lines; + }; + + auto lines = splitLines(code); + std::string annotatedCode; + annotatedCode.reserve(code.size() + lines.size() * 4UZ /*sizeof "123:"*/); + for (std::size_t i = std::max(0UZ, min); i < std::min(lines.size(), max); i++) { + // N.B. Python starts counting from '1' not '0' + annotatedCode += std::format("{:3}:{}{}\n", i, lines[i], i == marker - 1UZ ? " ####### <== here's your problem #######" : ""); + } + return annotatedCode; +} + +[[nodiscard]] inline std::string +getDebugPythonObjectAttributes(PyObject *obj) { + if (obj == nullptr) { + return "The provided PyObject is null.\n"; + } + + PyObjectGuard dirList(PyObject_Dir(obj)); + if (!dirList) { + PyErr_Print(); + return "Failed to get attribute list from object.\n"; + } + + // iterate over the list of attribute names + std::string ret; + Py_ssize_t size = PyList_Size(dirList); + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *attrName = PyList_GetItem(dirList, i); // borrowed reference, no need to decref + PyObjectGuard attrValue(PyObject_GetAttr(obj, attrName)); + ret += std::format("item {:3}: key: {} value: {}\n", i, toString(attrName), attrValue ? toString(attrValue) : ""); + } + return ret; +} + +inline void +throwCurrentPythonError(std::string_view msg, std::source_location location = std::source_location::current(), std::string_view pythonCode = "") { + PyObjectGuard exception(PyErr_GetRaisedException()); + if (!exception) { + throw gr::exception(std::format("{}\nPython error: \ntrace-back: {}", msg, toLineCountAnnotated(pythonCode)), location); + } + // fmt::println("detailed debug info: {}", getDebugPythonObjectAttributes(exception)) + + std::size_t min = 0UZ; + std::size_t max = std::numeric_limits::max(); + std::size_t marker = std::numeric_limits::max() - 1UZ; + if (PyObjectGuard lineStr(PyObject_GetAttrString(exception.get(), "lineno")); lineStr) { + marker = PyLong_AsSize_t(lineStr); + min = marker > 5UZ ? marker - 5UZ : 0; + max = marker < (std::numeric_limits::max() - 5UZ) ? marker + 5UZ : marker < std::numeric_limits::max(); + } + + throw gr::exception(std::format("{}\nPython error: {}\n{}", msg, toString(exception), toLineCountAnnotated(pythonCode, min, max, marker)), location); +} + +[[nodiscard]] inline std::string +getDictionary(std::string_view moduleName) { + PyObject *module = PyDict_GetItemString(PyImport_GetModuleDict(), moduleName.data()); + if (module == nullptr) { + return ""; + } + + if (PyObject *module_dict = PyModule_GetDict(module); module_dict != nullptr) { + PyObjectGuard dictGuard(PyObject_Repr(module_dict)); + return PyUnicode_AsUTF8(dictGuard); + } + return ""; +} + +template +concept NoParamNoReturn = requires(T t) { + { t() } -> std::same_as; +}; + +template +int +numpyType() noexcept { + // clang-format off + if constexpr (std::is_same_v) return NPY_BOOL; + else if constexpr (std::is_same_v) return NPY_BYTE; + else if constexpr (std::is_same_v) return NPY_UBYTE; + else if constexpr (std::is_same_v) return NPY_SHORT; + else if constexpr (std::is_same_v) return NPY_USHORT; + else if constexpr (std::is_same_v) return NPY_INT; + else if constexpr (std::is_same_v) return NPY_UINT; + else if constexpr (std::is_same_v) return NPY_LONG; + else if constexpr (std::is_same_v) return NPY_ULONG; + else if constexpr (std::is_same_v) return NPY_FLOAT; + else if constexpr (std::is_same_v) return NPY_DOUBLE; + else if constexpr (std::is_same_v>) return NPY_CFLOAT; + else if constexpr (std::is_same_v>) return NPY_CDOUBLE; + else if constexpr (std::is_same_v || std::is_same_v) return NPY_STRING; + else return NPY_NOTYPE; + // clang-format on +} + +template + requires std::is_arithmetic_v || std::is_same_v> || std::is_same_v> +constexpr inline PyObject * +toPyArray(T *arrayData, std::initializer_list dimensions) { + assert(dimensions.size() >= 1 && "nDim needs to be >= 1"); + + std::vector npyDims(dimensions.begin(), dimensions.end()); + // N.B. reinterpret cast is needed to access NumPy's unsafe C-API + void *data = const_cast(reinterpret_cast(arrayData)); + PyObject *npArray = PyArray_SimpleNewFromData(static_cast(dimensions.size()), npyDims.data(), python::numpyType>(), data); + if (!npArray) { + python::throwCurrentPythonError("Unable to create NumPy array"); + } + PyArray_CLEARFLAGS(reinterpret_cast(npArray), NPY_ARRAY_OWNDATA); + + if constexpr (!std::is_const_v) { + PyArray_ENABLEFLAGS(reinterpret_cast(npArray), NPY_ARRAY_WRITEABLE); + } else { + PyArray_CLEARFLAGS(reinterpret_cast(npArray), NPY_ARRAY_WRITEABLE); + } + return npArray; +} + +template +std::string +sanitizedPythonBlockName() { + std::string str = gr::meta::type_name(); + std::replace(str.begin(), str.end(), ':', '_'); + std::replace(str.begin(), str.end(), '<', '_'); + std::replace(str.begin(), str.end(), '>', '_'); + str.erase(std::remove_if(str.begin(), str.end(), [](unsigned char c) { return std::isalnum(static_cast(c)) == 0 && c != '_'; }), str.end()); + return str; +} + +} // namespace gr::python +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +#include +#include +#include + +namespace gr::python { + +enum class EnforceFunction { MANDATORY, OPTIONAL }; + +class Interpreter { + static std::atomic _nInterpreters; + static std::atomic _nNumPyInit; + static PyThreadState *_interpreterThreadState; + PyModuleDef *_moduleDefinitions; + PyObject *_pMainModule; // borrowed reference + PyObject *_pMainDict; // borrowed reference + PyObjectGuard _pCapsule; + +public: + template + explicit(false) Interpreter(T *classReference, PyModuleDef *moduleDefinitions = nullptr, std::source_location location = std::source_location::current()) : _moduleDefinitions(moduleDefinitions) { + if (_nInterpreters.fetch_add(1UZ, std::memory_order_relaxed) == 0UZ) { + Py_Initialize(); + if (PyErr_Occurred()) { + PyErr_Print(); + } + + python::PyGILGuard guard; + _interpreterThreadState = PyThreadState_Get(); + assert(_interpreterThreadState && "internal thread state is a nullptr"); + if (_nNumPyInit.fetch_add(1UZ, std::memory_order_relaxed) == 0UZ && _import_array() < 0) { + // NumPy keeps internal state and does not allow to be re-initialised after 'Py_Finalize()' has been called. + + // initialise NumPy -- N.B. NumPy does not support sub-interpreters (as of Python 3.12): + // "sys:1: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. + // This will likely work for most users but might cause hard to track down issues or subtle bugs. + // A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. + // Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve." + python::throwCurrentPythonError("failed to initialize NumPy", location); + } + } + assert(Py_IsInitialized() && "Python isn't properly initialised"); + // Ensure the Python GIL is initialized for this instance + python::PyGILGuard localGuard; + + // need to be executed after the Python environment has been initialised + _pMainModule = PyImport_AddModule("__main__"); + _pMainDict = PyModule_GetDict(_pMainModule); + if (classReference == nullptr || moduleDefinitions == nullptr) { + return; + } + _pCapsule = PyObjectGuard(PyCapsule_New(static_cast(classReference), _moduleDefinitions->m_name, nullptr)); + if (!_pCapsule) { + python::throwCurrentPythonError(fmt::format("Interpreter(*{}) - failed to create a capsule", gr::meta::type_name())); + } + PyDict_SetItemString(_pMainDict, "capsule", _pCapsule); + python::PyIncRef(_pCapsule); // need to explicitly increas count for the Python interpreter not to delete the reference by 'accident' + + // replaces the 'PyImport_AppendInittab("ClassName", &classDefinition)' to allow for other blocks being added + // after the global Python interpreter is already being initialised + PyObject *m = PyModule_Create(moduleDefinitions); + if (m) { + int ret = PyDict_SetItemString(PyImport_GetModuleDict(), moduleDefinitions->m_name, m); + python::PyDecRef(m); // The module dict holds a reference now. + if (ret != 0) { + python::throwCurrentPythonError(fmt::format("Error inserting module {}.", _moduleDefinitions->m_name), location); + } + } else { + python::throwCurrentPythonError(fmt::format("failed to create the module {}.", _moduleDefinitions->m_name), location); + } + if (PyDict_GetItemString(PyImport_GetModuleDict(), moduleDefinitions->m_name)) { // module successfully inserted - performing some additional checks + assert(python::getDictionary(moduleDefinitions->m_name).size() > 0 && "dictionary exist for module"); + + if (PyObject *imported_module = PyImport_ImportModule(moduleDefinitions->m_name); imported_module != nullptr) { + python::PyDecRef(imported_module); + } else { + python::throwCurrentPythonError(fmt::format("Check import of {} failed.", _moduleDefinitions->m_name), location); + } + } else { + python::throwCurrentPythonError(fmt::format("Manual import of {} failed.", _moduleDefinitions->m_name), location); + } + } + + ~Interpreter() { + if (_nInterpreters.fetch_sub(1UZ, std::memory_order_acq_rel) == 1UZ && Py_IsInitialized()) { + Py_Finalize(); + } + } + + // Prevent copying and moving + Interpreter(const Interpreter &) = delete; + Interpreter & + operator=(const Interpreter &) + = delete; + Interpreter(Interpreter &&) = delete; + Interpreter & + operator=(Interpreter &&) + = delete; + + PyObject * + getModule() { + return _pMainModule; + } + + PyObject * + getDictionary() { + return _pMainDict; + } + + template + void + invoke(Func func, std::string_view pythonCode = "", std::source_location location = std::source_location::current()) { + assert(Py_IsInitialized()); + PyGILGuard localGuard; + if (_interpreterThreadState != PyThreadState_Get()) { + python::throwCurrentPythonError("detected sub-interpreter change which is not supported by NumPy", location, pythonCode); + } + if (PyErr_Occurred()) { + python::throwCurrentPythonError("python::Interpreter::invoke() -- uncleared Python error before executing func", location, pythonCode); + } + + func(); + + if (PyErr_Occurred()) { + python::throwCurrentPythonError("python::Interpreter::invoke() -- uncleared Python error after executing func", location, pythonCode); + } + } + + template + python::PyObjectGuard invokeFunction(std::string_view functionName, PyObject *functionArguments = nullptr, std::source_location location = std::source_location::current()) { + PyGILGuard localGuard; + const bool hasFunction = PyObject_HasAttrString(getModule(), functionName.data()); + if constexpr (forced == EnforceFunction::MANDATORY) { + if (!hasFunction) { + python::throwCurrentPythonError(fmt::format("getFunction('{}', '{}') Python function not found or is not callable", functionName, python::toString(functionArguments)), location); + } + } else { + if (!hasFunction) { + return python::PyObjectGuard(nullptr); + } + } + python::PyObjectGuard pyFunc(PyObject_GetAttrString(getModule(), functionName.data())); + return python::PyObjectGuard(PyObject_CallObject(pyFunc, functionArguments)); + } +}; + +std::atomic Interpreter::_nInterpreters{ 0UZ }; +std::atomic Interpreter::_nNumPyInit{ 0UZ }; +PyThreadState *Interpreter::_interpreterThreadState = nullptr; + +} // namespace gr::python + +#endif // GNURADIO_PYTHONINTERPRETER_HPP diff --git a/blocks/basic/test/CMakeLists.txt b/blocks/basic/test/CMakeLists.txt index 3a98ca09..6c48d9a8 100644 --- a/blocks/basic/test/CMakeLists.txt +++ b/blocks/basic/test/CMakeLists.txt @@ -1,5 +1,9 @@ add_ut_test(qa_Selector) add_ut_test(qa_sources) add_ut_test(qa_DataSink) - add_ut_test(qa_BasicKnownBlocks) + +message(STATUS "###Python Include Dirs: ${Python3_INCLUDE_DIRS}") +if(PYTHON_AVAILABLE) + add_ut_test(qa_PythonBlock) +endif() diff --git a/blocks/basic/test/qa_PythonBlock.cpp b/blocks/basic/test/qa_PythonBlock.cpp new file mode 100644 index 00000000..fef93619 --- /dev/null +++ b/blocks/basic/test/qa_PythonBlock.cpp @@ -0,0 +1,273 @@ +#include + +#include +#include + +#include +#include + +const boost::ut::suite<"python::"> pythonInterfaceTests = [] { + using namespace boost::ut; + using namespace gr::python; + + "numpyType()"_test = [] { + expect(numpyType() == NPY_BOOL); + expect(numpyType() == NPY_BYTE); + expect(numpyType() == NPY_UBYTE); + expect(numpyType() == NPY_SHORT); + expect(numpyType() == NPY_USHORT); + expect(numpyType() == NPY_INT); + expect(numpyType() == NPY_UINT); + expect(numpyType() == NPY_LONG); + expect(numpyType() == NPY_ULONG); + expect(numpyType() == NPY_FLOAT); + expect(numpyType() == NPY_DOUBLE); + expect(numpyType>() == NPY_CFLOAT); + expect(numpyType>() == NPY_CDOUBLE); + expect(numpyType() == NPY_STRING); + expect(numpyType() == NPY_STRING); + expect(numpyType() == NPY_NOTYPE); + }; +}; + +const boost::ut::suite<"PythonBlock"> pythonBlockTests = [] { + using namespace boost::ut; + using namespace gr::basic; + using namespace std::string_literals; + using namespace std::string_view_literals; + + static_assert(gr::HasRequiredProcessFunction>); + static_assert(gr::HasProcessBulkFunction>); + static_assert(gr::HasRequiredProcessFunction>); + static_assert(gr::HasProcessBulkFunction>); + + "nominal PoC"_test = [] { + // Your Python script + std::string pythonScript = R"(import time; +counter = 0 + +def process_bulk(ins, outs): + global counter + start = time.time() + print('Start Python processing iteration: {}'.format(counter)) + # Print current settings + settings = this_block.getSettings() + print("Current settings:", settings) + + # tag handling + if this_block.tagAvailable(): + tag = this_block.getTag() + print('Tag:', tag) + + counter += 1 + # process the input->output samples + for i in range(len(ins)): + outs[i][:] = ins[i] * 2 + + # Update settings with the counter + settings["counter"] = str(counter) + this_block.setSettings(settings) + + print('Stop Python processing - time: {} seconds'.format(time.time() - start)) +)"; + + PythonBlock myBlock({ { "n_inputs", 3U }, { "n_outputs", 3U }, { "pythonScript", pythonScript } }); + myBlock.applyChangedSettings(); // needed for unit-test only when executed outside a Scheduler/Graph + + int count = 0; + std::vector data1 = { 1, 2, 3 }; + std::vector data2 = { 4, 5, 6 }; + std::vector out1(3); + std::vector out2(3); + std::vector> outs = { out1, out2 }; + std::vector> ins = { data1, data2 }; + std::span> spanIns = ins; + for (const auto &span : ins) { + fmt::println("InPort[{}] : [{}]", count++, fmt::join(span, ", ")); + } + fmt::println(""); + + for (std::size_t i = 0; i < 3; i++) { + fmt::println("C++ processing iteration: {}", i); + std::vector> constOuts(outs.begin(), outs.end()); + std::span> constSpanOuts = constOuts; + std::span> spanOuts = outs; + + try { + if (i == 0) { + myBlock.processBulk(spanIns, spanOuts); + } else { + myBlock.processBulk(constSpanOuts, spanOuts); + } + } catch (const std::exception &ex) { + fmt::println(stderr, "myBlock.processBulk(...) - threw unexpected exception:\n {}", ex.what()); + expect(false) << "nominal example should not throw"; + } + + fmt::println("C++ side got:"); + fmt::println("settings: {}", myBlock._settingsMap); + for (const auto &span : outs) { + fmt::println("OutPort[{}] : [{}]", count++, fmt::join(span, ", ")); + } + fmt::println(""); + } + + expect(eq(outs[0][0], 8)) << "out1[0] should be 8"; + expect(eq(outs[0][1], 16)) << "out1[1] should be 16"; + expect(eq(outs[0][2], 24)) << "out1[2] should be 24"; + + expect(eq(outs[1][0], 32)) << "out2[0] should be 32"; + expect(eq(outs[1][1], 40)) << "out2[1] should be 40"; + expect(eq(outs[1][2], 48)) << "out2[2] should be 48"; + + expect(eq(myBlock.getSettings().at("counter"), "3"s)); + }; + + "Python SyntaxError"_test = [] { + // Your Python script + std::string pythonScript = R"(def process_bulk(ins, outs): + + # process the input->output samples + for i in range(len(ins)) # <- (N.B. missing ':') + outs[i][:] = ins[i] * 2 +)"; + + PythonBlock myBlock({ { "n_inputs", 3U }, { "n_outputs", 3U }, { "pythonScript", pythonScript } }); + + bool throws = false; + try { + myBlock.applyChangedSettings(); // needed for unit-test only when executed outside a Scheduler/Graph + } catch (const std::exception &ex) { + throws = true; + fmt::println("myBlock.processBulk(...) - correctly threw SyntaxError exception:\n {}", ex.what()); + } + expect(throws) << "SyntaxError should throw"; + }; + + "Python RuntimeWarning as exception"_test = [] { + // Your Python script + std::string pythonScript = R"(def process_bulk(ins, outs): + + # process the input->output samples + for i in range(len(ins)): + outs[i][:] = ins[i] * 2/0 # <- (N.B. division by zero) +)"; + + PythonBlock myBlock({ { "n_inputs", 3U }, { "n_outputs", 3U }, { "pythonScript", pythonScript } }); + myBlock.applyChangedSettings(); // needed for unit-test only when executed outside a Scheduler/Graph + + std::vector data1 = { 1, 2, 3 }; + std::vector data2 = { 4, 5, 6 }; + std::vector out1(3); + std::vector out2(3); + std::vector> outs = { out1, out2 }; + std::vector> ins = { data1, data2 }; + + bool throws = false; + try { + myBlock.processBulk(std::span(ins), std::span(outs)); + } catch (const std::exception &ex) { + throws = true; + fmt::println("myBlock.processBulk(...) - correctly threw RuntimeWarning as exception:\n {}", ex.what()); + } + expect(throws) << "RuntimeWarning should throw"; + }; + + "Python Execution via Scheduler/Graph"_test = [] { + std::string pythonScript = R"(def process_bulk(ins, outs): + + # process the input->output samples + for i in range(len(ins)): + outs[i][:] = ins[i] * 2 +)"; + + using namespace gr::testing; + Graph graph; + auto &src = graph.emplaceBlock>({ { "n_samples_max", 5U }, { "mark_tag", false } }); + auto &block = graph.emplaceBlock>({ { "n_inputs", 1U }, { "n_outputs", 1U }, { "pythonScript", pythonScript } }); + auto &sink = graph.emplaceBlock>({ { "n_samples_expected", 5U }, { "verbose_console", true } }); + + expect(gr::ConnectionResult::SUCCESS == graph.connect(src, { "out", gr::meta::invalid_index }, block, { "inputs", 0U })); + expect(gr::ConnectionResult::SUCCESS == graph.connect(block, { "outputs", 0U }, sink, { "in", gr::meta::invalid_index })); + + scheduler::Simple sched{ std::move(graph) }; + bool throws = false; + try { + expect(sched.runAndWait().has_value()); + } catch (const std::exception &ex) { + throws = true; + fmt::println("sched.runAndWait() unexpectedly threw an exception:\n {}", ex.what()); + } + expect(!throws); + + expect(eq(sink.n_samples_produced, 5U)) << "sinkOne did not consume enough input samples"; + expect(eq(sink.samples, std::vector{ 0, 2, 4, 6, 8 })) << fmt::format("mismatch of vector {}", sink.samples); + }; + + "Python Execution - Lifecycle method tests"_test = [] { + std::string pythonScript = R"x(import os +counter = 0 + +# optional life-cycle methods - can be used to inform the block of the scheduling state +def start(): + global counter + print("Python: invoked start") + counter += 1 + +def stop(): + global counter + print("Python: invoked stop") + counter += 1 + +def pause(): + global counter + counter += 1 + +def resume(): + global counter + counter += 1 + +def reset(): + global counter + counter += 1 + +# stream-based processing +def process_bulk(ins, outs): + global counter + assert counter == 4, "Counter is not equal to 4 (N.B. having called start(), pause(), resume(), reset() callback functions" + + print("Python: invoked process_bulk(..)") + # process the input->output samples + for i in range(len(ins)): + outs[i][:] = ins[i] * 2 +)x"; + + using namespace gr::testing; + Graph graph; + auto &src = graph.emplaceBlock>({ { "n_samples_max", 5U }, { "mark_tag", false } }); + auto &block = graph.emplaceBlock>({ { "n_inputs", 1U }, { "n_outputs", 1U }, { "pythonScript", pythonScript } }); + auto &sink = graph.emplaceBlock>({ { "n_samples_expected", 5U }, { "verbose_console", true } }); + + expect(gr::ConnectionResult::SUCCESS == graph.connect(src, { "out", gr::meta::invalid_index }, block, { "inputs", 0U })); + expect(gr::ConnectionResult::SUCCESS == graph.connect(block, { "outputs", 0U }, sink, { "in", gr::meta::invalid_index })); + + scheduler::Simple sched{ std::move(graph) }; + block.pause(); // simplified calling + block.resume(); // simplified calling + block.reset(); // simplified calling + bool throws = false; + try { + expect(sched.runAndWait().has_value()); + } catch (const std::exception &ex) { + throws = true; + fmt::println("sched.runAndWait() unexpectedly threw an exception:\n {}", ex.what()); + } + expect(!throws); + + expect(eq(sink.n_samples_produced, 5U)) << "sinkOne did not consume enough input samples"; + expect(eq(sink.samples, std::vector{ 0.f, 2.f, 4.f, 6.f, 8.f })) << fmt::format("mismatch of vector {}", sink.samples); + }; +}; + +int +main() { /* tests are statically executed */ } diff --git a/cmake/CheckNumPy.py b/cmake/CheckNumPy.py new file mode 100644 index 00000000..9cb2605b --- /dev/null +++ b/cmake/CheckNumPy.py @@ -0,0 +1,6 @@ +import sys +try: + import numpy + print(numpy.get_include()) +except ImportError: + sys.exit(1) diff --git a/core/include/gnuradio-4.0/Settings.hpp b/core/include/gnuradio-4.0/Settings.hpp index 95c869b4..33e0bd06 100644 --- a/core/include/gnuradio-4.0/Settings.hpp +++ b/core/include/gnuradio-4.0/Settings.hpp @@ -239,7 +239,7 @@ struct SettingsBase { * to dependent/child blocks. */ [[nodiscard]] virtual ApplyStagedParametersResult - applyStagedParameters() noexcept + applyStagedParameters() = 0; /** @@ -505,7 +505,7 @@ class BasicSettings : public SettingsBase { * - appliedParameters -- map with peoperties that were successfully set */ [[nodiscard]] ApplyStagedParametersResult - applyStagedParameters() noexcept override { + applyStagedParameters() override { ApplyStagedParametersResult result; if constexpr (refl::is_reflectable()) { std::lock_guard lg(_lock); diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index 3d7b60ee..86b9574a 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -10,7 +10,11 @@ function(setup_test_no_asan TEST_NAME) endfunction() function(setup_test TEST_NAME) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # limited to gcc due to a Ubuntu packaging bug of libc++, see https://github.com/llvm/llvm-project/issues/59432 + if(PYTHON_AVAILABLE) + target_include_directories(${TEST_NAME} PRIVATE ${Python3_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR}) + target_link_libraries(${TEST_NAME} PRIVATE ${Python3_LIBRARIES}) + endif() + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # limited to gcc due to a Ubuntu packaging bug of libc++, see https://github.com/llvm/llvm-project/issues/59432 target_compile_options(${TEST_NAME} PRIVATE -fsanitize=address) # for testing consider enabling -D_GLIBCXX_DEBUG and -D_GLIBCXX_SANITIZE_VECTOR target_link_options(${TEST_NAME} PRIVATE -fsanitize=address) # for testing consider enabling -D_GLIBCXX_DEBUG and -D_GLIBCXX_SANITIZE_VECTOR endif()