Fix Python module thread model to allow for reusing interpreters

test/library/packages/Python/correctness/compareIterationPatterns.chpl

@@ -27,12 +27,9 @@ proc serialPythonApply(interp: borrowed, type t, arr, l) {
 proc parallelPythonApply(interp: borrowed, type t, arr, l) {
   var lambdaFunc = new Python.Function(interp, l);
   var res: [arr.domain] t;
-  var ts = new threadState();
-  ts.save();
-  forall i in arr.domain with (var gil = new Python.GIL()) {
+  forall i in arr.domain {
     res(i) = lambdaFunc(t, arr(i));
   }
-  ts.restore();
   return res;
 }
 //

test/library/packages/Python/correctness/loadingCode/fromCloudPickle.chpl

@@ -2,13 +2,15 @@ use Python, IO;
 
 var interp = new Interpreter();
 
-
+// must manually hold the GIL, since `loadPickle` is an internal function
+var g = new GIL();
 var MyClass =
   new Class(interp,
             interp.loadPickle(openReader("MyClass.pkl").readAll(bytes)));
 var my_function =
   new Function(interp,
                interp.loadPickle(openReader("my_function.pkl").readAll(bytes)));
+g.release();
 
 // run my_function
 var res = my_function();

test/library/packages/Python/correctness/loadingCode/fromPickle.chpl

@@ -3,6 +3,9 @@ use Python, IO;
 var interp = new Interpreter();
 
 
+// must manually hold the GIL, since `loadPickle` is an internal function
+var g = new GIL();
 var x = new Value(interp,
             interp.loadPickle(openReader("x.pkl").readAll(bytes)));
+g.release();
 writeln(x);

test/library/packages/Python/correctness/loadingCode/moduleFromBytes.chpl

@@ -4,7 +4,10 @@ config const pycPath: string;
 use Python;
 
 var interp = new Interpreter();
+// must manually hold the GIL, since `loadPycFile` is an internal function
+var g = new GIL();
 var modBytes = interp.loadPycFile(pycPath);
+g.release();
 
 var mod = interp.importModule("mod", modBytes);
 // get MyClass and my_function

test/library/packages/Python/correctness/reuseInterpreter.chpl

@@ -0,0 +1,39 @@
+use Python;
+use BlockDist;
+
+var interpreters = blockDist.createArray(0..<numLocales, owned Interpreter?);
+
+inline proc getInterpreter(): borrowed Interpreter {
+  return interpreters[here.id]!;
+}
+proc initPythonInterpreters() throws {
+  forall interp in interpreters {
+    interp = new Interpreter();
+  }
+}
+
+proc distributedApply(const ref x: [?d] ?t, funcStr: string): [d] t throws {
+  var ret = blockDist.createArray(d, t);
+
+  coforall l in d.targetLocales() do on l {
+    var func = new Function(getInterpreter(), funcStr);
+    for sd in d.localSubdomains() {
+      for i in sd {
+        ret[i] = func(t, x[i]);
+      }
+    }
+  }
+  return ret;
+}
+
+config const n = 100;
+proc main() {
+
+  initPythonInterpreters();
+
+  var arr = blockDist.createArray(0..<n, int);
+  arr = arr.domain;
+  writeln("before: ", arr);
+  var res = distributedApply(arr, "lambda x,: x + 1");
+  writeln("after: ", res);
+}

test/library/packages/Python/correctness/reuseInterpreter.good

@@ -0,0 +1,2 @@
+before: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
+after: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100

test/library/packages/Python/correctness/reuseInterpreter.numlocales

@@ -0,0 +1 @@
+4