Improve Godot config loading and pythonscript init process

src/_pythonscript.pyx

@@ -27,17 +27,21 @@ include "_pythonscript_extension_class_script.pxi"
 # )
 # from godot.hazmat.internal cimport set_pythonscript_verbose, get_pythonscript_verbose
 
-cdef object ProjectSettings = None
-def _setup_config_entry(name, default_value):
-    global ProjectSettings
-    if ProjectSettings is None:
-        ProjectSettings = _load_singleton("ProjectSettings")
+def _setup_config_entry(name: str, default_value: object):
+    # Cannot use `None` as config value since it is used in Godot to erase
+    # custom project settings.
+    # see https://docs.godotengine.org/en/stable/classes/class_projectsettings.html#class-projectsettings-method-set-setting
+    assert default_value is not None
 
+    ProjectSettings = _load_singleton("ProjectSettings")
     gdname = GDString(name)
+
     if not ProjectSettings.has_setting(gdname):
         ProjectSettings.set_setting(gdname, default_value)
+
     ProjectSettings.set_initial_value(gdname, default_value)
-    # TODO: `set_builtin_order` is not exposed by gdnative... but is it useful ?
+    ProjectSettings.set_restart_if_changed(gdname, True)
+
     return ProjectSettings.get_setting(gdname)
 
 # include "_pythonscript_script.pxi"
@@ -102,7 +106,7 @@ cdef _testbench():
 
 
 # Early init: register `PythonScriptLanguage` & `PythonScript` classes in Godot
-cdef api void _pythonscript_early_init() noexcept with gil:
+cdef void _register_pythonscript_classes():
     # Here is how we register Python into Godot:
     #
     # GDExtension API allows us to register "extension classes", those will be seen from
@@ -120,19 +124,25 @@ cdef api void _pythonscript_early_init() noexcept with gil:
     #
     # see: https://docs.godotengine.org/en/latest/classes/class_scriptlanguageextension.html
 
-    # # 1) Register `PythonScript` class into Godot
     # See `scripts/gdextension_cython_preprocessor.py` for the detail of
     # `__godot_extension_register_class`'s implementation.
 
     PythonScriptLanguage._PythonScriptLanguage__godot_extension_register_class()
     PythonScript._PythonScript__godot_extension_register_class()
 
-    # # OS and ProjectSettings are singletons exposed as global python objects,
-    # # hence there are not available from a cimport
-    # from godot.bindings import OS, ProjectSettings
 
-    # # Provide argv arguments
-    # sys.argv = ["godot"] + [str(x) for x in OS.get_cmdline_args()]
+cdef void _customize_config():
+    import sys
+    ProjectSettings = _load_singleton("ProjectSettings")
+    OS = _load_singleton("OS")
+
+    # Provide argv arguments
+
+    args = OS.get_cmdline_args()
+    sys.argv = ["godot"]
+    # TODO: iteration on `PackedStringArray` not supported yet !
+    for i in range(args.size()):
+        sys.argv.append(str(args[i]))
 
     # # Redirect stdout/stderr to have it in the Godot editor console
     # if _setup_config_entry("python/io_streams_capture", True):
@@ -144,24 +154,85 @@ cdef api void _pythonscript_early_init() noexcept with gil:
     # if _setup_config_entry("python/verbose", False):
     #     set_pythonscript_verbose(True)
 
-    # # Finally proudly print banner ;-)
-    # if _setup_config_entry("python/print_startup_info", True):
-    #     cooked_sys_version = '.'.join(map(str, sys.version_info))
-    #     print(f"Pythonscript {pythonscript_version} (CPython {cooked_sys_version})")
+    # Update PYTHONPATH according to configuration
+    pythonpath = str(_setup_config_entry("python/path", "res://;res://lib"))
+    for p in pythonpath.split(";"):
+        p = ProjectSettings.globalize_path(GDString(p))
+        sys.path.insert(0, str(p))
 
-    # if get_pythonscript_verbose():
-    #     print(f"PYTHONPATH: {sys.path}")
 
-    import sys
-    from godot._version import __version__ as pythonscript_version
+cdef object _initialize_callback = None
+cdef object _initialize_callback_hook(int p_level):
+    global _initialize_callback
 
-    cooked_sys_version = '.'.join(map(str, sys.version_info))
-    print(f"Pythonscript {pythonscript_version} (CPython {cooked_sys_version})", flush=True)
-    print(f"PYTHONPATH: {sys.path}", flush=True)
+    if _initialize_callback is None:
+        config = _setup_config_entry("python/initialize_callback", "")
+
+        if not isinstance(config, GDString):
+            raise ValueError("Invalid value for config `python/initialize_callback`: expected a string in format `<module>:<function>`")
+
+        if config.is_empty():
+            _initialize_callback = lambda _level: None  # Dummy callback
+
+        else:
+            try:
+                module, function = str(config).split(":")
+            except ValueError:
+                raise ValueError("Invalid value for config `python/initialize_callback`: expected a string in format `<module>:<function>`") from None
+
+            import importlib
+            try:
+                module = importlib.import_module(module)
+            except ModuleNotFoundError:
+                raise ValueError(f"Invalid value for config `python/initialize_callback`: cannot load module `{module}`")
+            try:
+                _initialize_callback = getattr(module, function)
+            except AttributeError:
+                raise ValueError(f"Invalid value for config `python/initialize_callback`: module `{module}` has no attribute `{function}`")
+
+    try:
+        _initialize_callback(p_level)
+    except Exception as exc:
+        raise ValueError(f"Invalid value for config `python/initialize_callback`: callback `{module}:{function}` call has failed") from exc
+
+
+cdef object _deinitialize_callback = None
+cdef object _deinitialize_callback_hook(int p_level):
+    global _deinitialize_callback
+
+    if _deinitialize_callback is None:
+        config = _setup_config_entry("python/deinitialize_callback", "")
+
+        if not isinstance(config, GDString):
+            raise ValueError("Invalid value for config `python/deinitialize_callback`: expected a string in format `<module>:<function>`")
+
+        if config.is_empty():
+            _deinitialize_callback = lambda _level: None  # Dummy callback
+
+        else:
+            try:
+                module, function = str(config).split(":")
+            except ValueError:
+                raise ValueError("Invalid value for config `python/deinitialize_callback`: expected a string in format `<module>:<function>`")
+
+            import importlib
+            try:
+                module = importlib.import_module(module)
+            except ModuleNotFoundError:
+                raise ValueError(f"Invalid value for config `python/deinitialize_callback`: cannot load module `{module}`")
+            try:
+                _deinitialize_callback = getattr(module, function)
+            except AttributeError:
+                raise ValueError(f"Invalid value for config `python/deinitialize_callback`: module `{module}` has no attribute `{function}`")
+
+    try:
+        _deinitialize_callback(p_level)
+    except Exception as exc:
+        raise ValueError(f"Invalid value for config `python/deinitialize_callback`: callback `{module}:{function}` call has failed") from exc
 
 
 # Late init: instantiate `PythonScriptLanguage`
-cdef api void _pythonscript_late_init() noexcept with gil:
+cdef void _register_pythonscript_language():
     global _pythons_script_language
     cdef GDExtensionObjectPtr singleton
     cdef GDExtensionMethodBindPtr bind
@@ -170,44 +241,6 @@ cdef api void _pythonscript_late_init() noexcept with gil:
     cdef StringName gdname_register_script_language
     cdef gd_int_t ret
 
-    TODOOOOOOOOOOOOOOOOOOOOOO !!!
-    # TODO: configure sys.path from the Godot config
-
-    # Update PYTHONPATH according to configuration
-    pythonpath = str(_setup_config_entry("python/path", "res://;res://lib"))
-    import sys
-    for p in pythonpath.split(";"):
-        p = ProjectSettings.globalize_path(GDString(p))
-        sys.path.insert(0, str(p))
-
-    # # _testbench()
-    print("------------ ZOI -------------", flush=True)
-    initialize_callback = _setup_config_entry("python/initialize_callback", None)
-    if initialize_callback is not None:
-        if not isinstance(initialize_callback, GDString):
-            raise ValueError("Invalid value for config `python/initialize_callback`: expected a string in format `<module>:<function>`")
-        try:
-            module, function = str(initialize_callback).split(":")
-        except ValueError:
-            raise ValueError("Invalid value for config `python/initialize_callback`: expected a string in format `<module>:<function>`")
-
-        import importlib
-        try:
-            module = importlib.import_module(module)
-        except ModuleNotFoundError:
-            raise ValueError(f"Invalid value for config `python/initialize_callback`: cannot load module `{module}`")
-        try:
-            function = getattr(module, function)
-        except AttributeError:
-            raise ValueError(f"Invalid value for config `python/initialize_callback`: module `{module}` has no attribute `{function}`")
-
-        try:
-            function()
-        except Exception as exc:
-            raise ValueError(f"Invalid value for config `python/initialize_callback`: callback `{module}:{function}` call has failed") from exc
-
-    print("------------ END ZOI -------------", flush=True)
-
     if _pythons_script_language is None:
 
         # 2) Create the instance of `PythonScriptLanguage` class...
@@ -246,18 +279,34 @@ cdef api void _pythonscript_late_init() noexcept with gil:
             return
 
 
+cdef void _print_banner():
+    import sys
+    ProjectSettings = _load_singleton("ProjectSettings")
+
+    if _setup_config_entry("python/print_startup_info", True):
+        from godot._version import __version__ as pythonscript_version
+        cooked_sys_version = '.'.join(map(str, sys.version_info))
+        print(f"Pythonscript {pythonscript_version} (CPython {cooked_sys_version})", flush=True)
+
+    if _setup_config_entry("python/verbose", True):
+        print(f"PYTHONPATH: {sys.path}", flush=True)
+
+
 cdef api void _pythonscript_initialize(int p_level) noexcept with gil:
-    print(f"_pythonscript_initialize {p_level}")
     if p_level == GDEXTENSION_INITIALIZATION_SERVERS:
-        print("!!!! early init", flush=True)
-        _pythonscript_early_init()
+        _register_pythonscript_classes()
 
     # Language registration must be done at `GDEXTENSION_INITIALIZATION_SERVERS` level which
     # is too early to have have everything we need for (e.g. `ClassDB` & `OS` singletons).
     # So we have to do another init step at `GDEXTENSION_INITIALIZATION_SCENE` level.
     if p_level == GDEXTENSION_INITIALIZATION_SCENE:
-        print("!!!! late init", flush=True)
-        _pythonscript_late_init()
+        _customize_config()
+        _register_pythonscript_language()
+        # Finally proudly print banner ;-)
+        _print_banner()
+
+    if p_level >= GDEXTENSION_INITIALIZATION_SCENE:
+        _initialize_callback_hook(p_level)
 
 
 cdef api void _pythonscript_deinitialize(int p_level) noexcept with gil:
@@ -275,6 +324,9 @@ cdef api void _pythonscript_deinitialize(int p_level) noexcept with gil:
     cdef StringName gdname_register_script_language
     cdef gd_int_t ret
 
+    if p_level >= GDEXTENSION_INITIALIZATION_SCENE:
+        _deinitialize_callback_hook(p_level)
+
     if p_level == GDEXTENSION_INITIALIZATION_SCENE and _pythons_script_language is not None:
 
         # Unregister Python from Godot

src/godot/classes.pyx.j2

@@ -84,13 +84,25 @@ cdef class {{ cls.cy_type }}
 {# {% endfor %} #}
 
 
+cdef object _loaded_singletons = {}
+cdef object _loaded_classes = {}
+
+
 cdef object _load_singleton(str name):
+    try:
+        return _loaded_singletons[name]
+    except KeyError:
+        pass
+
     cdef object cls = _load_class(name)
 {# cdef GDExtensionObjectPtr gdobj = pythonscript_gdextension.global_get_singleton(&(<StringName>cls._gd_name)._gd_data) #}
     cdef gd_string_name_t gdname = gd_string_name_from_utf8(name.encode())
     cdef GDExtensionObjectPtr gdobj = pythonscript_gdextension.global_get_singleton(&gdname)
     gd_string_name_del(&gdname)
-    return cls._from_ptr(<size_t>gdobj)
+    cdef object singleton = cls._from_ptr(<size_t>gdobj)
+
+    _loaded_singletons[name] = singleton
+    return singleton
 {# {% for singleton in api.singletons %}
     cdef {{ singleton.type.py_type }} singleton_{{ singleton.name }} = {{ singleton.type.py_type }}.__new__({{ singleton.type.py_type }})
     singleton_{{ singleton.name }}._gd_ptr = pythonscript_gdextension.global_get_singleton("{{ singleton.original_name }}")
@@ -111,6 +123,11 @@ cdef inline object _meth_call(object obj, object name, object args):
 
 
 cdef object _load_class(str name):
+    try:
+        return _loaded_classes[name]
+    except KeyError:
+        pass
+
     cdef StringName gdname = StringName(name)
 
     # Load our good friend ClassDB
@@ -120,7 +137,6 @@ cdef object _load_class(str name):
     if not _object_call(classdb, "class_exists", [gdname]):
         raise RuntimeError(f"Class `{name}` doesn't exist in Godot !")
 
-    print("loading", name, flush=True)
     gdparent = _object_call(classdb, "get_parent_class", [gdname])
     parent = str(gdparent)
     if parent:
@@ -135,7 +151,6 @@ cdef object _load_class(str name):
     def _generate_method(spec, py_meth_name):
         gd_meth_name = spec["name"]
         def _meth(self, *args):
-            print(f"CALL {gd_meth_name!r} {args!r}", flush=True)
             return _meth_call(self, gd_meth_name, args)
         _meth.__name__ = py_meth_name
         return _meth
@@ -149,11 +164,9 @@ cdef object _load_class(str name):
         propname = spec["name"]
         @property
         def _property(self):
-            print(f"GET {propname}", flush=True)
             return _property_getter(self, propname)
         @_property.setter
         def _property(self, value):
-            print(f"GET {propname} {value!r}", flush=True)
             _property_setter(self, propname, value)
         _property.fget.__name__ = str(spec["name"])
         _property.fset.__name__ = str(spec["name"])
@@ -166,7 +179,10 @@ cdef object _load_class(str name):
     signals = _object_call(classdb, "class_get_signal_list", [gdname])
     # TODO
 
-    return type(name, bases, attrs)
+    cdef object klass = type(name, bases, attrs)
+
+    _loaded_classes[name] = klass
+    return klass
 
 
 cdef object _object_call(GDExtensionObjectPtr obj, str meth, args):
@@ -196,8 +212,6 @@ cdef object _object_call(GDExtensionObjectPtr obj, str meth, args):
         if not gd_variant_steal_from_pyobj(arg, &variant_args[i]):
             raise TypeError(f"Parameter `{arg!r}` cannot be converted into a Godot Variant !")
 
-    print(f"About to call {meth} {args!r}", flush=True)
-
     pythonscript_gdextension.object_method_bind_call(
         Object_call,
         obj,

src/pythonscript.c

@@ -24,6 +24,10 @@
 # define DLL_IMPORT
 #endif
 
+#ifdef __linux__
+#include <dlfcn.h>
+#endif
+
 // Just like any Godot builtin classes, GDString's size is defined in `extension_api.json`
 // and is platform-dependant (e.g. 4 bytes on float_32, 8 on double_64).
 // So in theory we should retrieve the value from the json file, convert it into a C
@@ -239,6 +243,23 @@ static void _initialize_python() {
     //     goto error;
     // }
 
+    // When embedding Python, the symbols from `libpython3.so` are not made available.
+    //
+    // This is an issue when loading native modules (typically error message
+    // `undefined symbol: PyExc_SystemError`) since they use those symbols while
+    // not explicitly being linked to `libpython3.so`.
+    //
+    // So the solution is to force those symbols with an explicit RTLD_GLOBAL dlopen.
+    //
+    // See: https://stackoverflow.com/a/50489814
+    #ifdef __linux__
+    void*const libpython_handle = dlopen("libpython3.so", RTLD_LAZY | RTLD_GLOBAL);
+    if (!libpython_handle) {
+        GD_PRINT_ERROR("Pythonscript: Cannot dlopen libpython3.so");
+        goto error;
+    }
+    #endif
+
     {
         PyStatus status = Py_InitializeFromConfig(&config);
         if (PyStatus_Exception(status)) {

tests/2-pythonscript-init/project.godot

@@ -20,4 +20,4 @@ main_scene="res://main.tscn"
 [python]
 
 io_streams_capture=false
-verbose=true
+verbose=false