Skip to content

Commit 733846c

Browse files
hoodmanehoodmane
committed
gh-124621: Emscripten: Support pyrepl
Basic support for pyrepl in Emscripten. Limitations: * requires JSPI * no signal handling implemented As followup work, it would be nice to implement a webworker variant for when JSPI is not available and proper signal handling. Because it requires JSPI, it doesn't work in Safari. Firefox requires setting an experimental flag. All the Chromiums have full support since May. Until we make it work without JSPI, let's keep the original web_example around.
1 parent 28153fe commit 733846c

File tree

11 files changed

+482
-36
lines changed

11 files changed

+482
-36
lines changed

‎Lib/_pyrepl/trace.py

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33
import os
4+
import sys
45

56
# types
67
if False:
@@ -12,10 +13,22 @@
1213
trace_file = open(trace_filename, "a")
1314

1415

15-
def trace(line: str, *k: object, **kw: object) -> None:
16-
if trace_file is None:
17-
return
18-
if k or kw:
19-
line = line.format(*k, **kw)
20-
trace_file.write(line + "\n")
21-
trace_file.flush()
16+
17+
if sys.platform == "emscripten":
18+
from posix import _emscripten_log
19+
20+
def trace(line: str, *k: object, **kw: object) -> None:
21+
if "PYREPL_TRACE" not in os.environ:
22+
return
23+
if k or kw:
24+
line = line.format(*k, **kw)
25+
_emscripten_log(line)
26+
27+
else:
28+
def trace(line: str, *k: object, **kw: object) -> None:
29+
if trace_file is None:
30+
return
31+
if k or kw:
32+
line = line.format(*k, **kw)
33+
trace_file.write(line + "\n")
34+
trace_file.flush()

‎Makefile.pre.in

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -804,7 +804,7 @@ build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \
804804
python-config checksharedmods
805805

806806
.PHONY: build_emscripten
807-
build_emscripten: build_wasm web_example
807+
build_emscripten: build_wasm web_example web_example_pyrepl_jspi
808808

809809
# Check that the source is clean when building out of source.
810810
.PHONY: check-clean-src
@@ -1095,26 +1095,28 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
10951095

10961096
# wasm32-emscripten browser web example
10971097

1098-
WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/
1098+
EMSCRIPTEN_DIR=$(srcdir)/Tools/wasm/emscripten
1099+
WEBEX_DIR=$(EMSCRIPTEN_DIR)/web_example/
1100+
1101+
ZIP_STDLIB=python$(VERSION)$(ABI_THREAD).zip
1102+
$(ZIP_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
1103+
$(EMSCRIPTEN_DIR)/wasm_assets.py \
1104+
Makefile pybuilddir.txt Modules/Setup.local
1105+
$(PYTHON_FOR_BUILD) $(EMSCRIPTEN_DIR)/wasm_assets.py \
1106+
--buildroot . --prefix $(prefix) -o $@
1107+
10991108
web_example/index.html: $(WEBEX_DIR)/index.html
11001109
@mkdir -p web_example
11011110
@cp $< $@
11021111

1103-
web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs
1112+
web_example/server.py: $(WEBEX_DIR)/server.py
11041113
@mkdir -p web_example
11051114
@cp $< $@
11061115

1107-
web_example/server.py: $(WEBEX_DIR)/server.py
1116+
web_example/$(ZIP_STDLIB): $(ZIP_STDLIB)
11081117
@mkdir -p web_example
11091118
@cp $< $@
11101119

1111-
WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip
1112-
$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
1113-
$(WEBEX_DIR)/wasm_assets.py \
1114-
Makefile pybuilddir.txt Modules/Setup.local
1115-
$(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \
1116-
--buildroot . --prefix $(prefix) -o $@
1117-
11181120
web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON)
11191121
@if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \
11201122
echo "Can only build web_example when target is Emscripten" ;\
@@ -1124,7 +1126,39 @@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON)
11241126
cp python.wasm web_example/python.wasm
11251127

11261128
.PHONY: web_example
1127-
web_example: web_example/python.mjs web_example/python.worker.mjs web_example/index.html web_example/server.py $(WEB_STDLIB)
1129+
web_example: web_example/python.mjs web_example/index.html web_example/src.mjs web_example/server.py web_example/$(ZIP_STDLIB)
1130+
1131+
WEBEX2=web_example_pyrepl_jspi
1132+
WEBEX2_DIR=$(EMSCRIPTEN_DIR)/$(WEBEX2)/
1133+
1134+
$(WEBEX2)/python.mjs $(WEBEX2)/python.wasm: $(BUILDPYTHON)
1135+
@if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \
1136+
echo "Can only build web_example when target is Emscripten" ;\
1137+
exit 1 ;\
1138+
fi
1139+
@mkdir -p $(WEBEX2)
1140+
@cp python.mjs $(WEBEX2)/python.mjs
1141+
@cp python.wasm $(WEBEX2)/python.wasm
1142+
1143+
$(WEBEX2)/index.html: $(WEBEX2_DIR)/index.html
1144+
@mkdir -p $(WEBEX2)
1145+
@cp $< $@
1146+
1147+
$(WEBEX2)/src.mjs: $(WEBEX2_DIR)/src.mjs
1148+
@mkdir -p $(WEBEX2)
1149+
@cp $< $@
1150+
1151+
$(WEBEX2)/$(ZIP_STDLIB): $(ZIP_STDLIB)
1152+
@mkdir -p $(WEBEX2)
1153+
@cp $< $@
1154+
1155+
$(WEBEX2)/src.mjs: $(WEBEX2_DIR)/src.mjs
1156+
@mkdir -p $(WEBEX2)
1157+
@cp $< $@
1158+
1159+
.PHONY: web_example_pyrepl_jspi
1160+
web_example_pyrepl_jspi: $(WEBEX2)/python.mjs $(WEBEX2)/index.html $(WEBEX2)/src.mjs $(WEBEX2)/$(ZIP_STDLIB)
1161+
11281162

11291163
############################################################################
11301164
# Header files

‎Modules/clinic/posixmodule.c.h

Lines changed: 79 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Modules/posixmodule.c

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16971,6 +16971,25 @@ os__emscripten_debugger_impl(PyObject *module)
1697116971
emscripten_debugger();
1697216972
Py_RETURN_NONE;
1697316973
}
16974+
16975+
EM_JS(void, emscripten_log_impl_js, (const char* arg), {
16976+
console.warn(UTF8ToString(arg));
16977+
});
16978+
16979+
/*[clinic input]
16980+
os._emscripten_log
16981+
arg: str
16982+
16983+
Log something to the JS console. Emscripten only.
16984+
[clinic start generated code]*/
16985+
16986+
static PyObject *
16987+
os__emscripten_log_impl(PyObject *module, const char *arg)
16988+
/*[clinic end generated code: output=9749e5e293c42784 input=350aa1f70bc1e905]*/
16989+
{
16990+
emscripten_log_impl_js(arg);
16991+
Py_RETURN_NONE;
16992+
}
1697416993
#endif /* __EMSCRIPTEN__ */
1697516994

1697616995

@@ -17190,6 +17209,7 @@ static PyMethodDef posix_methods[] = {
1719017209
OS__IS_INPUTHOOK_INSTALLED_METHODDEF
1719117210
OS__CREATE_ENVIRON_METHODDEF
1719217211
OS__EMSCRIPTEN_DEBUGGER_METHODDEF
17212+
OS__EMSCRIPTEN_LOG_METHODDEF
1719317213
{NULL, NULL} /* Sentinel */
1719417214
};
1719517215

‎Python/emscripten_syscalls.c

Lines changed: 87 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "emscripten.h"
2+
#include "stdio.h"
23

34
// If we're running in node, report the UID of the user in the native system as
45
// the UID of the user. Since the nodefs will report the uid correctly, if we
@@ -40,6 +41,7 @@ int __syscall_umask(int mask) {
4041

4142
#include <wasi/api.h>
4243
#include <errno.h>
44+
#include <fcntl.h>
4345
#undef errno
4446

4547
// Variant of EM_JS that does C preprocessor substitution on the body
@@ -100,7 +102,7 @@ EM_JS_MACROS(void, _emscripten_promising_main_js, (void), {
100102
return;
101103
}
102104
const origResolveGlobalSymbol = resolveGlobalSymbol;
103-
if (!Module.onExit && globalThis?.process?.exit) {
105+
if (ENVIRONMENT_IS_NODE && !Module.onExit) {
104106
Module.onExit = (code) => process.exit(code);
105107
}
106108
// * wrap the main symbol with WebAssembly.promising,
@@ -115,7 +117,7 @@ EM_JS_MACROS(void, _emscripten_promising_main_js, (void), {
115117
orig.sym = (...args) => {
116118
(async () => {
117119
const ret = await main(...args);
118-
process?.exit?.(ret);
120+
Module.onExit?.(ret);
119121
})();
120122
_emscripten_exit_with_live_runtime();
121123
};
@@ -199,23 +201,101 @@ __wasi_errno_t __wasi_fd_read_orig(__wasi_fd_t fd, const __wasi_iovec_t *iovs,
199201

200202
// Take a promise that resolves to __wasi_errno_t and suspend until it resolves,
201203
// get the output.
202-
EM_JS(__wasi_errno_t, __block_for_errno, (__externref_t p), {
204+
EM_JS(int, __block_for_int, (__externref_t p), {
203205
return p;
204206
}
205207
if (WebAssembly.Suspending) {
206-
__block_for_errno = new WebAssembly.Suspending(__block_for_errno);
208+
__block_for_int = new WebAssembly.Suspending(__block_for_int);
207209
}
208210
)
209211

210212
// Replacement for fd_read syscall. Call __maybe_fd_read_async. If it returned
211-
// null, delegate back to __wasi_fd_read_orig. Otherwise, use __block_for_errno
213+
// null, delegate back to __wasi_fd_read_orig. Otherwise, use __block_for_int
212214
// to get the result.
213215
__wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs,
214216
size_t iovs_len, __wasi_size_t *nread) {
215217
__externref_t p = __maybe_fd_read_async(fd, iovs, iovs_len, nread);
216218
if (__builtin_wasm_ref_is_null_extern(p)) {
217219
return __wasi_fd_read_orig(fd, iovs, iovs_len, nread);
218220
}
219-
__wasi_errno_t res = __block_for_errno(p);
220-
return res;
221+
return __block_for_int(p);
222+
}
223+
224+
#include <poll.h>
225+
#define POLLFD_FD 0
226+
#define POLLFD_EVENTS 4
227+
#define POLLFD_REVENTS 6
228+
#define POLLFD_SIZE 8
229+
_Static_assert(offsetof(struct pollfd, fd) == 0, "Unepxected pollfd struct layout");
230+
_Static_assert(offsetof(struct pollfd, events) == 4, "Unepxected pollfd struct layout");
231+
_Static_assert(offsetof(struct pollfd, revents) == 6, "Unepxected pollfd struct layout");
232+
_Static_assert(sizeof(struct pollfd) == 8, "Unepxected pollfd struct layout");
233+
234+
EM_JS_MACROS(__externref_t, __maybe_poll_async, (intptr_t fds, int nfds, int timeout), {
235+
if (!WebAssembly.promising) {
236+
return null;
237+
}
238+
return (async function() {
239+
try {
240+
var nonzero = 0;
241+
var promises = [];
242+
for (var i = 0; i < nfds; i++) {
243+
var pollfd = fds + POLLFD_SIZE * i;
244+
var fd = HEAP32[(pollfd + POLLFD_FD)/4];
245+
var events = HEAP16[(pollfd + POLLFD_EVENTS)/2];
246+
var mask = POLLNVAL;
247+
var stream = FS.getStream(fd);
248+
if (stream) {
249+
mask = POLLIN | POLLOUT;
250+
if (stream.stream_ops.pollAsync) {
251+
promises.push(stream.stream_ops.pollAsync(stream, timeout).then((mask) => {
252+
mask &= events | POLLERR | POLLHUP;
253+
HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask;
254+
if (mask) {
255+
nonzero ++;
256+
}
257+
}));
258+
} else if (stream.stream_ops.poll) {
259+
var mask = stream.stream_ops.poll(stream, timeout);
260+
mask &= events | POLLERR | POLLHUP;
261+
HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask;
262+
if (mask) {
263+
nonzero ++;
264+
}
265+
}
266+
}
267+
}
268+
await Promise.all(promises);
269+
return nonzero;
270+
} catch(e) {
271+
if (e?.name !== "ErrnoError") throw e;
272+
return -e.errno;
273+
}
274+
})();
275+
});
276+
277+
// Bind original poll syscall to syscall_poll_orig().
278+
int syscall_poll_orig(intptr_t fds, int nfds, int timeout)
279+
__attribute__((__import_module__("env"),
280+
__import_name__("__syscall_poll"), __warn_unused_result__));
281+
282+
int __syscall_poll(intptr_t fds, int nfds, int timeout) {
283+
__externref_t p = __maybe_poll_async(fds, nfds, timeout);
284+
if (__builtin_wasm_ref_is_null_extern(p)) {
285+
return syscall_poll_orig(fds, nfds, timeout);
286+
}
287+
return __block_for_int(p);
288+
}
289+
290+
#include <sys/ioctl.h>
291+
292+
int syscall_ioctl_orig(int fd, int request, void* varargs)
293+
__attribute__((__import_module__("env"),
294+
__import_name__("__syscall_ioctl"), __warn_unused_result__));
295+
296+
int __syscall_ioctl(int fd, int request, void* varargs) {
297+
if (request == FIOCLEX || request == FIONCLEX) {
298+
return 0;
299+
}
300+
return syscall_ioctl_orig(fd, request, varargs);
221301
}

‎Tools/wasm/config.site-wasm32-emscripten

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@ ac_cv_func_posix_fallocate=no
6969

7070
# Syscalls that resulted in a segfault
7171
ac_cv_func_utimensat=no
72-
ac_cv_header_sys_ioctl_h=no
7372

7473
# sockets are supported, but only AF_INET / AF_INET6 in non-blocking mode.
7574
# Disable AF_UNIX and AF_PACKET support, see socketmodule.h.

0 commit comments

Comments
 (0)