Merge pull request #1641 from rstudio/allow-background-python-threads

Allow Python backgrounds threads to run in parallel with R

NEWS.md

@@ -1,5 +1,8 @@
 # reticulate (development version)
 
+- Python background threads can now run in parallel with
+  the R session (#1641).
+
 - Fixed error when importing a module named `config` (#1628)
 
 - `conda_run2()` is now exported (#1637, contributed by @dramanica)
@@ -9,14 +12,14 @@
 - Python Exceptions converted to R conditions are now R lists instead
   of R environments, for compatability with {rlang} and {purrr}.
   (tidyverse/purrr#1104, r-lib/rlang#1664, #1617)
-  
+
 - Internal updates for NumPy 2.0 (#1621)
 
 - Added support for converting NumPy StringDType arrays to R character arrays. (#1623)
 
 - Internal updates for compliance with R's upcoming formalized C API. (#1625)
 
-- Fixed an issue where attempting to convert a NumPy array with a non-simple 
+- Fixed an issue where attempting to convert a NumPy array with a non-simple
   dtype to R would signal an error. (#1613, fixed in #1614).
 
 # reticulate 1.37.0

R/RcppExports.R

@@ -365,6 +365,10 @@ py_iterate <- function(x, f, simplify = TRUE) {
     .Call(`_reticulate_py_iterate`, x, f, simplify)
 }
 
+py_allow_threads_impl <- function(allow = TRUE) {
+    .Call(`_reticulate_py_allow_threads_impl`, allow)
+}
+
 readline <- function(prompt) {
     .Call(`_reticulate_readline`, prompt)
 }

R/package.R

@@ -255,6 +255,14 @@ initialize_python <- function(required_module = NULL, use_environment = NULL) {
   # https://github.com/rstudio/reticulate/issues/586
   py_set_qt_qpa_platform_plugin_path(config)
 
+  if (was_python_initialized_by_reticulate()) {
+    allow_threads <- Sys.getenv("RETICULATE_ALLOW_THREADS", "true")
+    allow_threads <- tolower(allow_threads) %in% c("true", "1", "yes")
+    if (allow_threads) {
+      py_allow_threads_impl(TRUE)
+    }
+  }
+
   # return config
   config
 }

R/threads.R

@@ -0,0 +1,19 @@
+
+
+py_allow_threads <- function(allow = TRUE) {
+  if (allow) {
+    reticulate_ns <- environment(sys.function())
+    for (f in sys.frames()) {
+      if (identical(parent.env(f), reticulate_ns) &&
+          !identical(f, environment()))
+        # Can't release the gil as unlocked while we're holding it
+        # elsewhere on the callstack.
+        stop("Python threads can only be unblocked from a top-level reticulate call")
+    }
+  }
+
+  if (!was_python_initialized_by_reticulate())
+    stop("Can't safely unblock threads when R is running embedded")
+
+  invisible(py_allow_threads_impl(allow))
+}

inst/python/rpytools/run.py

@@ -0,0 +1,50 @@
+import sys
+import os
+
+
+def run_file(path):
+    with open(path, "r") as file:
+        file_content = file.read()
+
+    d = sys.modules["__main__"].__dict__
+
+    exec(file_content, d, d)
+
+
+class RunMainScriptContext:
+    def __init__(self, path, args):
+        self.path = path
+        self.args = tuple(args)
+
+    def __enter__(self):
+        sys.path.insert(0, os.path.dirname(self.path))
+
+        self._orig_sys_argv = sys.argv
+        sys.argv = [self.path] + list(self.args)
+
+    def __exit__(self, *_):
+        # try restore sys.path
+        try:
+            sys.path.remove(os.path.dirname(self.path))
+        except ValueError:
+            pass
+        # restore sys.argv if it's been unmodified
+        # otherwise, leave it as-is.
+        set_argv = [self.path] + list(self.args)
+        if sys.argv == set_argv:
+            sys.argv = self._orig_sys_argv
+
+
+def _run_file_on_thread(path, args=None):
+
+    import _thread
+
+    _thread.start_new_thread(run_file, (path, ))
+
+
+def _launch_lsp_server_on_thread(path, args):
+    # for now, leave sys.argv and sys.path permanently modified.
+    # Later, revisit if it's desirable/safe to restore after the initial
+    # lsp event loop startup.
+    RunMainScriptContext(path, args).__enter__()
+    _run_file_on_thread(path)

src/RcppExports.cpp

@@ -812,6 +812,17 @@ BEGIN_RCPP
     return rcpp_result_gen;
 END_RCPP
 }
+// py_allow_threads_impl
+bool py_allow_threads_impl(bool allow);
+RcppExport SEXP _reticulate_py_allow_threads_impl(SEXP allowSEXP) {
+BEGIN_RCPP
+    Rcpp::RObject rcpp_result_gen;
+    Rcpp::RNGScope rcpp_rngScope_gen;
+    Rcpp::traits::input_parameter< bool >::type allow(allowSEXP);
+    rcpp_result_gen = Rcpp::wrap(py_allow_threads_impl(allow));
+    return rcpp_result_gen;
+END_RCPP
+}
 // readline
 SEXP readline(const std::string& prompt);
 RcppExport SEXP _reticulate_readline(SEXP promptSEXP) {
@@ -895,6 +906,7 @@ static const R_CallMethodDef CallEntries[] = {
     {"_reticulate_as_iterator", (DL_FUNC) &_reticulate_as_iterator, 1},
     {"_reticulate_py_iter_next", (DL_FUNC) &_reticulate_py_iter_next, 2},
     {"_reticulate_py_iterate", (DL_FUNC) &_reticulate_py_iterate, 3},
+    {"_reticulate_py_allow_threads_impl", (DL_FUNC) &_reticulate_py_allow_threads_impl, 1},
     {"_reticulate_readline", (DL_FUNC) &_reticulate_readline, 1},
     {NULL, NULL, 0}
 };

src/libpython.cpp

@@ -293,6 +293,8 @@ bool LibPython::loadSymbols(bool python3, std::string* pError)
   LOAD_PYTHON_SYMBOL(PyGILState_Ensure)
   LOAD_PYTHON_SYMBOL(PyGILState_Release)
   LOAD_PYTHON_SYMBOL(PyThreadState_Next)
+  LOAD_PYTHON_SYMBOL(PyEval_SaveThread)
+  LOAD_PYTHON_SYMBOL(PyEval_RestoreThread)
   LOAD_PYTHON_SYMBOL(PyObject_CallMethod)
   LOAD_PYTHON_SYMBOL(PySequence_GetItem)
   LOAD_PYTHON_SYMBOL(PyObject_IsTrue)

src/libpython.h

@@ -790,6 +790,8 @@ LIBPYTHON_EXTERN PyThreadState* (*PyGILState_GetThisThreadState)(void);
 LIBPYTHON_EXTERN PyGILState_STATE (*PyGILState_Ensure)(void);
 LIBPYTHON_EXTERN void (*PyGILState_Release)(PyGILState_STATE);
 LIBPYTHON_EXTERN PyThreadState* (*PyThreadState_Next)(PyThreadState*);
+LIBPYTHON_EXTERN PyThreadState* (*PyEval_SaveThread)();
+LIBPYTHON_EXTERN void (*PyEval_RestoreThread)(PyThreadState*);
 
 /* End PyFrameObject */