From cdf2f46c076eaabd3bbc262b5628138e9e9b04b4 Mon Sep 17 00:00:00 2001 From: Nicolau Leal Werneck Date: Tue, 4 Feb 2025 15:49:59 +0100 Subject: [PATCH 01/31] Add reference to time_ns in time (#57142) I couldn't find `time_ns` when I was looking for it, nice to make clear the "monotonic" clock is also available in Base. (cherry picked from commit a52de835af96c2f5f3939fa3342d985e613f5a30) --- base/libc.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/libc.jl b/base/libc.jl index 7364f6e6677fe..fc0cc774cab7a 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -288,6 +288,8 @@ time(tm::TmStruct) = Float64(ccall(:mktime, Int, (Ref{TmStruct},), tm)) time() -> Float64 Get the system time in seconds since the epoch, with fairly high (typically, microsecond) resolution. + +See also [`time_ns`](@ref). """ time() = ccall(:jl_clock_now, Float64, ()) From 033492e766656dc86c037a036b3a8a95aa338f4a Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 4 Feb 2025 11:42:09 -0500 Subject: [PATCH 02/31] Handle `waitpid` race condition when `SIGCHLD` is set to `SIG_IGN` (#57241) The `fork()` we do here relies on `SIGCHLD` to make sure that we don't race against the child. This is easy to see in an embedding application that dynamically links `libjulia`: ```c int main(int argc, char *argv[]) { signal(SIGCHLD, SIG_IGN); void *handle = dlopen("path/to/libjulia.so", RTLD_LAZY); return 0; } ``` Without this change, this fails with an error message: ``` Error during libstdcxxprobe in parent process: waitpid: No child processes ``` Resolves #57240 (cherry picked from commit daf865e8605344764c1f26b5fc4412f100888791) --- cli/loader_lib.c | 12 +++++++++--- test/embedding/Makefile | 14 +++++++++++--- test/embedding/libdl_embedding.c | 12 ++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 test/embedding/libdl_embedding.c diff --git a/cli/loader_lib.c b/cli/loader_lib.c index af2a36cfce8ab..4d75cfd9563cb 100644 --- a/cli/loader_lib.c +++ b/cli/loader_lib.c @@ -349,10 +349,16 @@ static char *libstdcxxprobe(void) pid_t npid = waitpid(pid, &wstatus, 0); if (npid == -1) { if (errno == EINTR) continue; - if (errno != EINTR) { - perror("Error during libstdcxxprobe in parent process:\nwaitpid"); - exit(1); + if (errno == ECHILD) { + // SIGCHLD is set to SIG_IGN or has flag SA_NOCLDWAIT, so the child + // did not become a zombie and wait for `waitpid` - it just exited. + // + // Assume that it exited successfully and use whatever libpath we + // got out of the pipe, if any. + break; } + perror("Error during libstdcxxprobe in parent process:\nwaitpid"); + exit(1); } else if (!WIFEXITED(wstatus)) { const char *err_str = "Error during libstdcxxprobe in parent process:\n" diff --git a/test/embedding/Makefile b/test/embedding/Makefile index df31c3735c9de..4be4974e864cd 100644 --- a/test/embedding/Makefile +++ b/test/embedding/Makefile @@ -21,6 +21,7 @@ EXE := $(suffix $(abspath $(JULIA))) # get compiler and linker flags. (see: `contrib/julia-config.jl`) JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "julia-config.jl"))' -- +JULIA_LIBDIR := $(shell $(JULIA) -e 'println(joinpath(Sys.BINDIR, "..", "lib"))' --) CPPFLAGS_ADD := CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags) LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) @@ -29,8 +30,8 @@ DEBUGFLAGS += -g #============================================================================= -release: $(BIN)/embedding$(EXE) -debug: $(BIN)/embedding-debug$(EXE) +release: $(BIN)/embedding$(EXE) $(BIN)/libdl-embedding$(EXE) +debug: $(BIN)/embedding-debug$(EXE) $(BIN)/libdl-embedding$(EXE) $(BIN)/embedding$(EXE): $(SRCDIR)/embedding.c $(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) @@ -38,6 +39,12 @@ $(BIN)/embedding$(EXE): $(SRCDIR)/embedding.c $(BIN)/embedding-debug$(EXE): $(SRCDIR)/embedding.c $(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS) +$(BIN)/libdl-embedding$(EXE): $(SRCDIR)/libdl_embedding.c + $(CC) $^ -o $@ $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -ldl -DLIBJULIA_PATH=\"$(JULIA_LIBDIR)/libjulia.so\" + +$(BIN)/libdl-embedding-debug$(EXE): $(SRCDIR)/libdl_embedding.c + $(CC) $^ -o $@ $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(DEBUGFLAGS) -ldl -DLIBJULIA_PATH=\"$(JULIA_LIBDIR)/libjulia.so\" + ifneq ($(abspath $(BIN)),$(abspath $(SRCDIR))) # for demonstration purposes, our demo code is also installed # in $BIN, although this would likely not be typical @@ -45,7 +52,8 @@ $(BIN)/LocalModule.jl: $(SRCDIR)/LocalModule.jl cp $< $@ endif -check: $(BIN)/embedding$(EXE) $(BIN)/LocalModule.jl +check: $(BIN)/embedding$(EXE) $(BIN)/libdl-embedding$(EXE) $(BIN)/LocalModule.jl + $(BIN)/libdl-embedding$(EXE) # run w/o error $(JULIA) --depwarn=error $(SRCDIR)/embedding-test.jl $< @echo SUCCESS diff --git a/test/embedding/libdl_embedding.c b/test/embedding/libdl_embedding.c new file mode 100644 index 0000000000000..6cd040d5f9abf --- /dev/null +++ b/test/embedding/libdl_embedding.c @@ -0,0 +1,12 @@ +#include +#include +#include + +int main(int argc, char *argv[]) +{ + // This test doesn't do much yet, except check + // https://github.com/JuliaLang/julia/issues/57240 + signal(SIGCHLD, SIG_IGN); + void *handle = dlopen(LIBJULIA_PATH, RTLD_LAZY); + return 0; +} From 171ce73659d0192efab850cdb0a1eb4364d16e28 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 4 Feb 2025 13:39:13 -0500 Subject: [PATCH 03/31] restore non-freebsd-unix fix for profiling (#57249) Restores #57035, undo #57089 for non-FreeBSD. While I suggested doing this change for all platforms, I forgot that means non-FreeBSD platforms become vulnerable again to the very deadlock problems that #57035 was required to prevent. That fix seems to not be viable on FreeBSD due to known libc implementation problems on that platform. However, upon closer inspection of the questionable design implementation decisions they seem to have made here, the platform is likely not currently vulnerable to this libunwind bug in the first place: https://github.com/lattera/freebsd/blob/master/libexec/rtld-elf/rtld_lock.c#L120 (cherry picked from commit 2f0a5239f00697914128e7bb7b58f7dadec0d519) --- src/signals-unix.c | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/signals-unix.c b/src/signals-unix.c index c730f27f16def..1f4ad647a87af 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -310,14 +310,34 @@ int exc_reg_is_write_fault(uintptr_t esr) { #include #include +#ifndef _OS_FREEBSD_ +typedef struct { + void (*f)(void*) JL_NOTSAFEPOINT; + void *ctx; +} callback_t; +static int with_dl_iterate_phdr_lock(struct dl_phdr_info *info, size_t size, void *data) +{ + jl_lock_profile(); + callback_t *callback = (callback_t*)data; + callback->f(callback->ctx); + jl_unlock_profile(); + return 1; // only call this once +} +#endif + void jl_with_stackwalk_lock(void (*f)(void*), void *ctx) { - sigset_t sset, oset; - sigemptyset(&sset); - sigaddset(&sset, SIGUSR2); - pthread_sigmask(SIG_BLOCK, &sset, &oset); +#ifndef _OS_FREEBSD_ + callback_t callback = {f, ctx}; + dl_iterate_phdr(with_dl_iterate_phdr_lock, &callback); +#else + // FreeBSD makes the questionable decisions to use a terrible implementation of a spin + // lock and to block all signals while a lock is held. However, that also means it is + // not currently vulnerable to this libunwind bug that other platforms can encounter. + jl_lock_profile(); f(ctx); - pthread_sigmask(SIG_SETMASK, &oset, NULL); + jl_unlock_profile(); +#endif } #if defined(_OS_LINUX_) && (defined(_CPU_X86_64_) || defined(_CPU_X86_)) From 6b059f9faa9e96b91d1b1d4b43d4afd79a2b9d7b Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Tue, 4 Feb 2025 13:55:11 -0700 Subject: [PATCH 04/31] Ensure read/readavailable for BufferStream are threadsafe (#57211) It looks like these methods were just missed while overloading for BufferStream. There's also `readbytes!` where the current implementation will fallback to the `LibuvStream` implementation that is currently not threadsafe. What's the best approach there since the implementation is quite a bit more involved? Just duplicate the code but for BufferStream? Should we take the BufferStream lock and invoke the LibuvStream method? Open to ideas there. Also open to suggestions for having tests here? Not easy to simulate the data race of writing and calling readavailable. The fix here will unblock https://github.com/JuliaWeb/HTTP.jl/pull/1213 (I'll probably do some compat shim there until this is fully released). Thanks to @oscardssmith for rubber ducking this issue with me. Probably most helpfully reviewed by @vtjnash. --------- Co-authored-by: Jameson Nash (cherry picked from commit ffc96bc7e14a80b22f3a7bbfd4fc60adaa120f1a) --- base/stream.jl | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/base/stream.jl b/base/stream.jl index e81f65685df72..33d884018d5ad 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -1559,6 +1559,64 @@ function wait_readnb(s::BufferStream, nb::Int) end end +function readavailable(this::BufferStream) + bytes = lock(this.cond) do + wait_readnb(this, 1) + buf = this.buffer + @assert buf.seekable == false + take!(buf) + end + return bytes +end + +function read(stream::BufferStream) + bytes = lock(stream.cond) do + wait_close(stream) + take!(stream.buffer) + end + return bytes +end + +function readbytes!(s::BufferStream, a::Vector{UInt8}, nb::Int) + sbuf = s.buffer + @assert sbuf.seekable == false + @assert sbuf.maxsize >= nb + + function wait_locked(s, buf, nb) + while bytesavailable(buf) < nb + s.readerror === nothing || throw(s.readerror) + isopen(s) || break + s.status != StatusEOF || break + wait_readnb(s, nb) + end + end + + bytes = lock(s.cond) do + if nb <= SZ_UNBUFFERED_IO # Under this limit we are OK with copying the array from the stream's buffer + wait_locked(s, sbuf, nb) + end + if bytesavailable(sbuf) >= nb + nread = readbytes!(sbuf, a, nb) + else + initsize = length(a) + newbuf = PipeBuffer(a, maxsize=nb) + newbuf.size = newbuf.offset # reset the write pointer to the beginning + nread = try + s.buffer = newbuf + write(newbuf, sbuf) + wait_locked(s, newbuf, nb) + bytesavailable(newbuf) + finally + s.buffer = sbuf + end + _take!(a, _unsafe_take!(newbuf)) + length(a) >= initsize || resize!(a, initsize) + end + return nread + end + return bytes +end + show(io::IO, s::BufferStream) = print(io, "BufferStream(bytes waiting=", bytesavailable(s.buffer), ", isopen=", isopen(s), ")") function readuntil(s::BufferStream, c::UInt8; keep::Bool=false) From 75f3690176c9fee8a787eef22b54bb09a97a3eda Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 5 Feb 2025 08:24:14 -0500 Subject: [PATCH 05/31] edit NEWS for v1.12 (#57262) (cherry picked from commit 99fd5d9a92190e826bc462d5739e7be948a3bf44) --- NEWS.md | 310 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 172 insertions(+), 138 deletions(-) diff --git a/NEWS.md b/NEWS.md index 53643ee2c0954..403d832ee6f68 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,83 +4,74 @@ Julia v1.12 Release Notes New language features --------------------- -- New option `--trim` creates smaller binaries by removing code that was not proven to be reachable from - the entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). -- A new keyword argument `usings::Bool` has been added to `names`. By using this, we can now - find all the names available in module `A` by `names(A; all=true, imported=true, usings=true)`. ([#54609]) -- the `@atomic(...)` macro family supports now the reference assignment syntax, e.g. - `@atomic :monotonic v[3] += 4` modifies `v[3]` atomically with monotonic ordering semantics. ([#54707]) +* New option `--trim` creates smaller binaries by removing code that was not proven to be reachable from + entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). +* A new keyword argument `usings::Bool` has been added to `names`, returning all names visible + via `using` ([#54609]). +* The `@atomic` macro family now supports reference assignment syntax, e.g. `@atomic :monotonic v[3] += 4`, + which modifies `v[3]` atomically with monotonic ordering semantics ([#54707]). The supported syntax allows - - atomic fetch (`x = @atomic v[3]`), - - atomic set (`@atomic v[3] = 4`), - - atomic modify (`@atomic v[3] += 2`), - - atomic set once (`@atomiconce v[3] = 2`), - - atomic swap (`x = @atomicswap v[3] = 2`), and - - atomic replace (`x = @atomicreplace v[3] 2=>5`). -- New option `--task-metrics=yes` to enable the collection of per-task timing information, - which can also be enabled/disabled at runtime with `Base.Experimental.task_metrics(::Bool)`. ([#56320]) + * atomic fetch (`x = @atomic v[3]`), + * atomic set (`@atomic v[3] = 4`), + * atomic modify (`@atomic v[3] += 2`), + * atomic set once (`@atomiconce v[3] = 2`), + * atomic swap (`x = @atomicswap v[3] = 2`), and + * atomic replace (`x = @atomicreplace v[3] 2=>5`). +* New option `--task-metrics=yes` to enable the collection of per-task timing information, + which can also be enabled/disabled at runtime with `Base.Experimental.task_metrics(::Bool)` ([#56320]). The available metrics are: - - actual running time for the task (`Base.Experimental.task_running_time_ns`), and - - wall-time for the task (`Base.Experimental.task_wall_time_ns`). -- Support for Unicode 16 ([#56925]). -- `Threads.@spawn` now takes a `:samepool` argument to specify the same threadpool as the caller. - `Threads.@spawn :samepool foo()` which is shorthand for `Threads.@spawn Threads.threadpool() foo()` ([#57109]) + * actual running time for the task (`Base.Experimental.task_running_time_ns`), and + * wall-time for the task (`Base.Experimental.task_wall_time_ns`). +* Support for Unicode 16 ([#56925]). +* `Threads.@spawn` now takes a `:samepool` argument to specify the same threadpool as the caller. + `Threads.@spawn :samepool foo()` which is shorthand for `Threads.@spawn Threads.threadpool() foo()` ([#57109]). Language changes ---------------- - - Julia now defaults to 1 "interactive" thread, in addition to the 1 "default" worker thread. i.e. `-t1,1` +* Julia now defaults to 1 "interactive" thread, in addition to the 1 default "worker" thread. i.e. `-t1,1`. This means in default configuration the main task and repl (when in interactive mode), which both run on - thread 1, now run within the `interactive` threadpool. Also the libuv IO loop runs on thread 1, - helping efficient utilization of the "default" worker threadpool, which is what `Threads.@threads` and a bare - `Threads.@spawn` uses. Use `0` to disable the interactive thread i.e. `-t1,0` or `JULIA_NUM_THREADS=1,0`, or - `-tauto,0` etc. The zero is explicitly required to disable it, `-t2` will set the equivalent of `-t2,1` ([#57087]) - - - When methods are replaced with exactly equivalent ones, the old method is no - longer deleted implicitly simultaneously, although the new method does take - priority and become more specific than the old method. Thus if the new - method is deleted later, the old method will resume operating. This can be - useful to mocking frameworks (such as in SparseArrays, Pluto, and Mocking, - among others), as they do not need to explicitly restore the old method. - While inference and compilation still must be repeated with this, it also - may pave the way for inference to be able to intelligently re-use the old - results, once the new method is deleted. ([#53415]) - - - Macro expansion will no longer eagerly recurse into `Expr(:toplevel)` - expressions returned from macros. Instead, macro expansion of `:toplevel` - expressions will be delayed until evaluation time. This allows a later - expression within a given `:toplevel` expression to make use of macros - defined earlier in the same `:toplevel` expression. ([#53515]) - - - Trivial infinite loops (like `while true; end`) are no longer undefined - behavior. Infinite loops that actually do things (e.g. have side effects - or sleep) were never and are still not undefined behavior. ([#52999]) - - - It is now an error to mark a binding as both `public` and `export`ed. - ([#53664]) + thread 1, now run within the `interactive` threadpool. The libuv IO loop also runs on thread 1, + helping efficient utilization of the worker threadpool used by `Threads.@spawn`. Pass `0` to disable the + interactive thread i.e. `-t1,0` or `JULIA_NUM_THREADS=1,0`, or `-tauto,0` etc. The zero is explicitly + required to disable it, `-t2` will set the equivalent of `-t2,1` ([#57087]). +* When a method is replaced with an exactly equivalent one, the old method is not deleted. Instead, the + new method takes priority and becomes more specific than the old method. Thus if the new method is deleted + later, the old method will resume operating. This can be useful in mocking frameworks (as in SparseArrays, + Pluto, and Mocking, among others), as they do not need to explicitly restore the old method. + At this time, inference and compilation must be repeated in this situation, but we may eventually be + able to re-use the old results ([#53415]). +* Macro expansion will no longer eagerly recurse into `Expr(:toplevel)` expressions returned from macros. + Instead, macro expansion of `:toplevel` expressions will be delayed until evaluation time. This allows a + later expression within a given `:toplevel` expression to make use of macros defined earlier in the same + `:toplevel` expression ([#53515]). +* Trivial infinite loops (like `while true; end`) are no longer undefined behavior. Infinite loops that + do things (e.g. have side effects or sleep) were never and are still not undefined behavior ([#52999]). +* It is now an error to mark a binding as both `public` and `export`ed ([#53664]). +* Errors during `getfield` now raise a new `FieldError` exception type instead of the generic + `ErrorException` ([#54504]). Compiler/Runtime improvements ----------------------------- -- Generated LLVM IR now uses actual pointer types instead of passing pointers as integers. +* Generated LLVM IR now uses pointer types instead of passing pointers as integers. This affects `llvmcall`: Inline LLVM IR should be updated to use `i8*` or `ptr` instead of `i32` or `i64`, and remove unneeded `ptrtoint`/`inttoptr` conversions. For compatibility, - IR with integer pointers is still supported, but generates a deprecation warning. ([#53687]) - -- A new exception `FieldError` is now introduced to raise/handle `getfield` exceptions. Previously `getfield` exception was captured by fallback generic exception `ErrorException`. Now that `FieldError` is more specific `getfield` related exceptions that can occur should use `FieldError` exception instead. ([#54504]) + IR with integer pointers is still supported, but generates a deprecation warning ([#53687]). Command-line option changes --------------------------- * The `-m/--module` flag can be passed to run the `main` function inside a package with a set of arguments. - This `main` function should be declared using `@main` to indicate that it is an entry point. ([#52103]) + This `main` function should be declared using `@main` to indicate that it is an entry point ([#52103]). * Enabling or disabling color text in Julia can now be controlled with the [`NO_COLOR`](https://no-color.org/) or [`FORCE_COLOR`](https://force-color.org/) environment variables. These variables are also honored by Julia's build system ([#53742], [#56346]). -* `--project=@temp` starts Julia with a temporary environment. ([#51149]) +* `--project=@temp` starts Julia with a temporary environment ([#51149]). * New `--trace-compile-timing` option to report how long each method reported by `--trace-compile` took - to compile, in ms. ([#54662]) -* `--trace-compile` now prints recompiled methods in yellow or with a trailing comment if color is not supported ([#55763]) + to compile, in ms ([#54662]). +* `--trace-compile` now prints recompiled methods in yellow or with a trailing comment if color is not + supported ([#55763]). * New `--trace-dispatch` option to report methods that are dynamically dispatched ([#55848]). Multi-threading changes @@ -90,45 +81,49 @@ Multi-threading changes a `OncePerProcess{T}` type, which allows defining a function that should be run exactly once the first time it is called, and then always return the same result value of type `T` every subsequent time afterwards. There are also `OncePerThread{T}` and `OncePerTask{T}` types for - similar usage with threads or tasks. ([#55793]) + similar usage with threads or tasks ([#55793]). Build system changes -------------------- -* There are new `Makefile`s to build Julia and LLVM using the Binary Optimization and Layout Tool (BOLT), see `contrib/bolt` and `contrib/pgo-lto-bolt` ([#54107]). +* There are new `Makefile`s to build Julia and LLVM using the Binary Optimization and Layout Tool (BOLT). + See `contrib/bolt` and `contrib/pgo-lto-bolt` ([#54107]). New library functions --------------------- -* `logrange(start, stop; length)` makes a range of constant ratio, instead of constant step ([#39071]) -* The new `isfull(c::Channel)` function can be used to check if `put!(c, some_value)` will block. ([#53159]) -* `waitany(tasks; throw=false)` and `waitall(tasks; failfast=false, throw=false)` which wait multiple tasks at once ([#53341]). +* `logrange(start, stop; length)` makes a range of constant ratio, instead of constant step ([#39071]). +* The new `isfull(c::Channel)` function can be used to check if `put!(c, some_value)` will block ([#53159]). +* `waitany(tasks; throw=false)` and `waitall(tasks; failfast=false, throw=false)` which wait for multiple tasks + at once ([#53341]). * `uuid7()` creates an RFC 9652 compliant UUID with version 7 ([#54834]). -* `insertdims(array; dims)` allows to insert singleton dimensions into an array which is the inverse operation to `dropdims`. ([#45793]) -* The new `Fix` type is a generalization of `Fix1/Fix2` for fixing a single argument ([#54653]). -* `Sys.detectwsl()` allows to testing if Julia is running inside WSL at runtime. ([#57069]) +* `insertdims(array; dims)` inserts singleton dimensions into an array --- the inverse operation of + `dropdims` ([#45793]). +* A new `Fix` type generalizes `Fix1/Fix2` for fixing a single argument ([#54653]). +* `Sys.detectwsl()` tests whether Julia is running inside WSL at runtime ([#57069]). New library features -------------------- -* `escape_string` takes additional keyword arguments `ascii=true` (to escape all - non-ASCII characters) and `fullhex=true` (to require full 4/8-digit hex numbers - for u/U escapes, e.g. for C compatibility) ([#55099]). +* `escape_string` takes additional keyword arguments `ascii=true` (to escape all non-ASCII characters) and + `fullhex=true` (to require full 4/8-digit hex numbers for u/U escapes, e.g. for C compatibility) ([#55099]). * `tempname` can now take a suffix string to allow the file name to include a suffix and include that suffix in - the uniquing checking ([#53474]) -* `RegexMatch` objects can now be used to construct `NamedTuple`s and `Dict`s ([#50988]) -* `Lockable` is now exported ([#54595]) -* `Base.require_one_based_indexing` and `Base.has_offset_axes` are now public ([#56196]) -* New `ltruncate`, `rtruncate` and `ctruncate` functions for truncating strings to text width, accounting for char widths ([#55351]) -* `isless` (and thus `cmp`, sorting, etc.) is now supported for zero-dimensional `AbstractArray`s ([#55772]) -* `invoke` now supports passing a Method instead of a type signature making this interface somewhat more flexible for certain uncommon use cases ([#56692]). -* `Timer(f, ...)` will now match the stickiness of the parent task when creating timer tasks, which can be overridden - by the new `spawn` kwarg. This avoids the issue where sticky tasks i.e. `@async` make their parent sticky ([#56745]) -* `invoke` now supports passing a CodeInstance instead of a type, which can enable -certain compiler plugin workflows ([#56660]). -* `sort` now supports `NTuple`s ([#54494]) -* `map!(f, A)` now stores the results in `A`, like `map!(f, A, A)`. or `A .= f.(A)` ([#40632]). -* `Timer` now has readable `timeout` and `interval` properties, and a more descriptive show method ([#57081]) + the uniquing checking ([#53474]). +* `RegexMatch` objects can now be used to construct `NamedTuple`s and `Dict`s ([#50988]). +* `Lockable` is now exported ([#54595]). +* `Base.require_one_based_indexing` and `Base.has_offset_axes` are now public ([#56196]). +* New `ltruncate`, `rtruncate` and `ctruncate` functions for truncating strings to text width, accounting for + char widths ([#55351]). +* `isless` (and thus `cmp`, sorting, etc.) is now supported for zero-dimensional `AbstractArray`s ([#55772]). +* `invoke` now supports passing a `Method` instead of a type signature ([#56692]). +* `invoke` now supports passing a `CodeInstance` instead of a type, which can enable certain compiler plugin + workflows ([#56660]). +* `Timer(f, ...)` will now match the stickiness of the parent task when creating timer tasks, which can be + overridden by the new `spawn` keyword argument. This avoids the issue where sticky tasks (i.e. `@async`) + make their parent sticky ([#56745]). +* `Timer` now has readable `timeout` and `interval` properties, and a more descriptive `show` method ([#57081]). +* `sort` now supports `NTuple`s ([#54494]). +* `map!(f, A)` now stores the results in `A`, like `map!(f, A, A)` or `A .= f.(A)` ([#40632]). Standard library changes ------------------------ @@ -136,27 +131,23 @@ Standard library changes * `gcdx(0, 0)` now returns `(0, 0, 0)` instead of `(0, 1, 0)` ([#40989]). * `fd` returns a `RawFD` instead of an `Int` ([#55080]). -#### StyledStrings - #### JuliaSyntaxHighlighting -* A new standard library for applying syntax highlighting to Julia code, this - uses `JuliaSyntax` and `StyledStrings` to implement a `highlight` function - that creates an `AnnotatedString` with syntax highlighting applied. ([#51810]) - -#### Package Manager +* A new standard library for applying syntax highlighting to Julia code, this uses `JuliaSyntax` and + `StyledStrings` to implement a `highlight` function that creates an `AnnotatedString` with syntax highlighting + applied ([#51810]). #### LinearAlgebra * `rank` can now take a `QRPivoted` matrix to allow rank estimation via QR factorization ([#54283]). -* Added keyword argument `alg` to `eigen`, `eigen!`, `eigvals` and `eigvals!` for self-adjoint - matrix types (i.e., the type union `RealHermSymComplexHerm`) that allows one to switch - between different eigendecomposition algorithms ([#49355]). -* Added a generic version of the (unblocked) pivoted Cholesky decomposition - (callable via `cholesky[!](A, RowMaximum())`) ([#54619]). -* The number of default BLAS threads now respects process affinity, instead of - using total number of logical threads available on the system ([#55574]). -* A new function `zeroslike` is added that is used to generate the zero elements for matrix-valued banded matrices. +* Added keyword argument `alg` to `eigen`, `eigen!`, `eigvals` and `eigvals!` for self-adjoint matrix types + (i.e., the type union `RealHermSymComplexHerm`) that allows one to switch between different eigendecomposition + algorithms ([#49355]). +* Added a generic version of the (unblocked) pivoted Cholesky decomposition (callable via + `cholesky[!](A, RowMaximum())`) ([#54619]). +* The number of default BLAS threads now respects process affinity, instead of using the total number of logical + threads available on the system ([#55574]). +* A new function `zeroslike` is added that generates the zero elements for matrix-valued banded matrices. Custom array types may specialize this function to return an appropriate result ([#55252]). * The matrix multiplication `A * B` calls `matprod_dest(A, B, T::Type)` to generate the destination. This function is now public ([#55537]). @@ -164,41 +155,33 @@ Standard library changes This is now public ([#56223]). * A new function `diagview` is added that returns a view into a specific band of an `AbstractMatrix` ([#56175]). -#### Logging - -#### Printf - #### Profile -* `Profile.take_heap_snapshot` takes a new keyword argument, `redact_data::Bool`, - that is `true` by default. When set, the contents of Julia objects are not emitted - in the heap snapshot. This currently only applies to strings. ([#55326]) +* `Profile.take_heap_snapshot` takes a new keyword argument, `redact_data::Bool`, which is `true` by default. + When set, the contents of Julia objects are not emitted in the heap snapshot. This currently only applies to + strings ([#55326]). * `Profile.print()` now colors Base/Core/Package modules similarly to how they are in stacktraces. Also paths, even if truncated, are now clickable in terminals that support URI links - to take you to the specified `JULIA_EDITOR` for the given file & line number. ([#55335]) - -#### Random + to take you to the specified `JULIA_EDITOR` for the given file & line number ([#55335]). #### REPL -- Using the new `usings=true` feature of the `names()` function, REPL completions can now - complete names that have been explicitly `using`-ed. ([#54610]) -- REPL completions can now complete input lines like `[import|using] Mod: xxx|` e.g. - complete `using Base.Experimental: @op` to `using Base.Experimental: @opaque`. ([#54719]) -- the REPL will now warn if it detects a name is being accessed from a module which does not define it (nor has a submodule which defines it), - and for which the name is not public in that module. For example, `map` is defined in Base, and executing `LinearAlgebra.map` - in the REPL will now issue a warning the first time occurs. ([#54872]) -- When an object is printed automatically (by being returned in the REPL), its display is now truncated after printing 20 KiB. - This does not affect manual calls to `show`, `print`, and so forth. ([#53959]) -- Backslash completions now print the respective glyph or emoji next to each matching backslash shortcode. ([#54800]) - -#### SuiteSparse - -#### SparseArrays +* Using the new `usings=true` feature of the `names()` function, REPL completions can now + complete names visible via `using` ([#54610]). +* REPL completions can now complete input lines like `[import|using] Mod: xxx|` e.g. + complete `using Base.Experimental: @op` to `using Base.Experimental: @opaque` ([#54719]). +* The REPL will now warn if it detects a name is being accessed via a module which does not define it (nor has + a submodule which defines it), and for which the name is not public in that module. For example, `map` is + defined in Base, and executing `LinearAlgebra.map` in the REPL will now issue a warning the first time it + occurs ([#54872]). +* When the result of a REPL input is printed, the output is now truncated to 20 KiB. + This does not affect manual calls to `show`, `print`, etc. ([#53959]). +* Backslash completions now print the respective glyph or emoji next to each matching backslash shortcode ([#54800]). #### Test -* A failing `DefaultTestSet` now prints to screen the random number generator (RNG) of the failed test, to help reproducing a stochastic failure which only depends on the state of the RNG. +* A failing `DefaultTestSet` now prints to screen the random number generator (RNG) of the failed test, to help + reproducing a stochastic failure which only depends on the state of the RNG. It is also possible seed a test set by passing the `rng` keyword argument to `@testset`: ```julia using Test, Random @@ -207,35 +190,86 @@ Standard library changes end ``` -#### Dates - -#### Statistics - -#### Distributed - -#### Unicode - -#### DelimitedFiles - #### InteractiveUtils * New macros `@trace_compile` and `@trace_dispatch` for running an expression with - `--trace-compile=stderr --trace-compile-timing` and `--trace-dispatch=stderr` respectively enabled. - ([#55915]) - -Deprecated or removed ---------------------- + `--trace-compile=stderr --trace-compile-timing` and `--trace-dispatch=stderr` respectively enabled ([#55915]). External dependencies --------------------- -- The terminal info database, `terminfo`, is now vendored by default, providing a better +* The terminal info database, `terminfo`, is now vendored by default, providing a better REPL user experience when `terminfo` is not available on the system. Julia can be built - without vendoring the database using the Makefile option `WITH_TERMINFO=0`. ([#55411]) + without vendoring the database using the Makefile option `WITH_TERMINFO=0` ([#55411]). Tooling Improvements -------------------- -- A wall-time profiler is now available for users who need a sampling profiler that captures tasks regardless of their scheduling or running state. This type of profiler enables profiling of I/O-heavy tasks and helps detect areas of heavy contention in the system ([#55889]). +* A wall-time profiler is now available for users who need a sampling profiler that captures tasks regardless + of their scheduling or running state. This type of profiler enables profiling of I/O-heavy tasks and helps + detect areas of heavy contention in the system ([#55889]). +[#39071]: https://github.com/JuliaLang/julia/issues/39071 +[#40632]: https://github.com/JuliaLang/julia/issues/40632 +[#40989]: https://github.com/JuliaLang/julia/issues/40989 +[#45793]: https://github.com/JuliaLang/julia/issues/45793 +[#49355]: https://github.com/JuliaLang/julia/issues/49355 +[#50988]: https://github.com/JuliaLang/julia/issues/50988 +[#51149]: https://github.com/JuliaLang/julia/issues/51149 +[#51810]: https://github.com/JuliaLang/julia/issues/51810 +[#52103]: https://github.com/JuliaLang/julia/issues/52103 +[#52999]: https://github.com/JuliaLang/julia/issues/52999 +[#53159]: https://github.com/JuliaLang/julia/issues/53159 +[#53341]: https://github.com/JuliaLang/julia/issues/53341 +[#53415]: https://github.com/JuliaLang/julia/issues/53415 +[#53474]: https://github.com/JuliaLang/julia/issues/53474 +[#53515]: https://github.com/JuliaLang/julia/issues/53515 +[#53664]: https://github.com/JuliaLang/julia/issues/53664 +[#53687]: https://github.com/JuliaLang/julia/issues/53687 +[#53742]: https://github.com/JuliaLang/julia/issues/53742 +[#53959]: https://github.com/JuliaLang/julia/issues/53959 +[#54107]: https://github.com/JuliaLang/julia/issues/54107 +[#54283]: https://github.com/JuliaLang/julia/issues/54283 +[#54494]: https://github.com/JuliaLang/julia/issues/54494 +[#54504]: https://github.com/JuliaLang/julia/issues/54504 +[#54595]: https://github.com/JuliaLang/julia/issues/54595 +[#54609]: https://github.com/JuliaLang/julia/issues/54609 +[#54610]: https://github.com/JuliaLang/julia/issues/54610 +[#54619]: https://github.com/JuliaLang/julia/issues/54619 +[#54653]: https://github.com/JuliaLang/julia/issues/54653 +[#54662]: https://github.com/JuliaLang/julia/issues/54662 +[#54707]: https://github.com/JuliaLang/julia/issues/54707 +[#54719]: https://github.com/JuliaLang/julia/issues/54719 +[#54800]: https://github.com/JuliaLang/julia/issues/54800 +[#54834]: https://github.com/JuliaLang/julia/issues/54834 +[#54872]: https://github.com/JuliaLang/julia/issues/54872 +[#55047]: https://github.com/JuliaLang/julia/issues/55047 +[#55080]: https://github.com/JuliaLang/julia/issues/55080 +[#55099]: https://github.com/JuliaLang/julia/issues/55099 +[#55252]: https://github.com/JuliaLang/julia/issues/55252 +[#55326]: https://github.com/JuliaLang/julia/issues/55326 +[#55335]: https://github.com/JuliaLang/julia/issues/55335 +[#55351]: https://github.com/JuliaLang/julia/issues/55351 +[#55411]: https://github.com/JuliaLang/julia/issues/55411 +[#55537]: https://github.com/JuliaLang/julia/issues/55537 +[#55574]: https://github.com/JuliaLang/julia/issues/55574 +[#55763]: https://github.com/JuliaLang/julia/issues/55763 +[#55772]: https://github.com/JuliaLang/julia/issues/55772 +[#55793]: https://github.com/JuliaLang/julia/issues/55793 +[#55848]: https://github.com/JuliaLang/julia/issues/55848 +[#55889]: https://github.com/JuliaLang/julia/issues/55889 +[#55915]: https://github.com/JuliaLang/julia/issues/55915 +[#56175]: https://github.com/JuliaLang/julia/issues/56175 +[#56196]: https://github.com/JuliaLang/julia/issues/56196 +[#56223]: https://github.com/JuliaLang/julia/issues/56223 +[#56320]: https://github.com/JuliaLang/julia/issues/56320 +[#56346]: https://github.com/JuliaLang/julia/issues/56346 +[#56660]: https://github.com/JuliaLang/julia/issues/56660 +[#56692]: https://github.com/JuliaLang/julia/issues/56692 +[#56745]: https://github.com/JuliaLang/julia/issues/56745 +[#56925]: https://github.com/JuliaLang/julia/issues/56925 +[#57069]: https://github.com/JuliaLang/julia/issues/57069 +[#57081]: https://github.com/JuliaLang/julia/issues/57081 +[#57087]: https://github.com/JuliaLang/julia/issues/57087 +[#57109]: https://github.com/JuliaLang/julia/issues/57109 From 2b91d9c600d430b00251de013f99f2ff70b2486a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 5 Feb 2025 23:45:19 -0500 Subject: [PATCH 06/31] cfunction: reimplement, as originally planned, for reliable performance (#57226) (cherry picked from commit ca7cf30d6310469c335e7d2ed95c786cf8b93881) --- Compiler/src/typeinfer.jl | 12 +- src/aotcompile.cpp | 179 +++++++++- src/ccall.cpp | 2 + src/cgutils.cpp | 54 ++- src/codegen.cpp | 706 +++++++++++++++++++++++++------------- src/gf.c | 3 +- src/intrinsics.cpp | 3 + src/jitlayers.cpp | 124 ++++--- src/jitlayers.h | 25 ++ src/jl_exported_funcs.inc | 1 + src/julia_internal.h | 9 +- src/runtime_ccall.cpp | 130 +++++++ src/staticdata.c | 3 + 13 files changed, 926 insertions(+), 325 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 96f58943e3255..f1da2e8a75a67 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1266,6 +1266,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: tocompile = Vector{CodeInstance}() codeinfos = [] # first compute the ABIs of everything + latest = true # whether this_world == world_counter() for this_world in reverse(sort!(worlds)) interp = NativeInterpreter(this_world) for i = 1:length(methods) @@ -1278,18 +1279,18 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: # then we want to compile and emit this if item.def.primary_world <= this_world <= item.def.deleted_world ci = typeinf_ext(interp, item, SOURCE_MODE_NOT_REQUIRED) - ci isa CodeInstance && !use_const_api(ci) && push!(tocompile, ci) + ci isa CodeInstance && push!(tocompile, ci) end - elseif item isa SimpleVector + elseif item isa SimpleVector && latest (rt::Type, sig::Type) = item # make a best-effort attempt to enqueue the relevant code for the ccallable ptr = ccall(:jl_get_specialization1, #= MethodInstance =# Ptr{Cvoid}, (Any, Csize_t, Cint), sig, this_world, #= mt_cache =# 0) if ptr !== C_NULL - mi = unsafe_pointer_to_objref(ptr) + mi = unsafe_pointer_to_objref(ptr)::MethodInstance ci = typeinf_ext(interp, mi, SOURCE_MODE_NOT_REQUIRED) - ci isa CodeInstance && !use_const_api(ci) && push!(tocompile, ci) + ci isa CodeInstance && push!(tocompile, ci) end # additionally enqueue the ccallable entrypoint / adapter, which implicitly # invokes the above ci @@ -1305,7 +1306,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: mi = get_ci_mi(callee) def = mi.def if use_const_api(callee) - src = codeinfo_for_const(interp, mi, code.rettype_const) + src = codeinfo_for_const(interp, mi, callee.rettype_const) elseif haskey(interp.codegen, callee) src = interp.codegen[callee] elseif isa(def, Method) && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 && !trim @@ -1327,6 +1328,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: println("warning: failed to get code for ", mi) end end + latest = false end return codeinfos end diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 5524518da46fa..d91da9c64cda9 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -423,7 +423,7 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root if (decls.functionObject == "jl_fptr_args") { preal_decl = decls.specFunctionObject; } - else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") { + else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call" && decls.functionObject != "jl_fptr_const_return") { preal_decl = decls.specFunctionObject; preal_specsig = true; } @@ -439,6 +439,13 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root Module *mod = proto.decl->getParent(); assert(proto.decl->isDeclaration()); Function *pinvoke = nullptr; + if (preal_decl.empty() && jl_atomic_load_relaxed(&codeinst->invoke) == jl_fptr_const_return_addr) { + std::string gf_thunk_name = emit_abi_constreturn(mod, params, proto.specsig, codeinst); + preal_specsig = proto.specsig; + if (invokeName.empty()) + invokeName = "jl_fptr_const_return"; + preal_decl = mod->getNamedValue(gf_thunk_name)->getName(); + } if (preal_decl.empty()) { if (invokeName.empty() && params.params->trim) { jl_safe_printf("warning: bailed out to invoke when compiling: "); @@ -483,6 +490,7 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root ocinvokeDecl = pinvoke->getName(); assert(!ocinvokeDecl.empty()); assert(ocinvokeDecl != "jl_fptr_args"); + assert(ocinvokeDecl != "jl_fptr_const_return"); assert(ocinvokeDecl != "jl_fptr_sparam"); // merge and/or rename this prototype to the real function if (Value *specfun = mod->getNamedValue(ocinvokeDecl)) { @@ -499,6 +507,134 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root JL_GC_POP(); } +/// Link the function in the source module into the destination module if +/// needed, setting up mapping information. +/// Similar to orc::cloneFunctionDecl, but more complete for greater correctness +Function *IRLinker_copyFunctionProto(Module *DstM, Function *SF) { + // If there is no linkage to be performed or we are linking from the source, + // bring SF over, if we haven't already. + if (SF->getParent() == DstM) + return SF; + if (auto *F = DstM->getNamedValue(SF->getName())) + return cast(F); + auto *F = Function::Create(SF->getFunctionType(), SF->getLinkage(), + SF->getAddressSpace(), SF->getName(), DstM); + F->copyAttributesFrom(SF); + F->IsNewDbgInfoFormat = SF->IsNewDbgInfoFormat; + + // Remove these copied constants since they point to the source module. + F->setPersonalityFn(nullptr); + F->setPrefixData(nullptr); + F->setPrologueData(nullptr); + return F; +} + +static Function *aot_abi_converter(jl_codegen_params_t ¶ms, Module *M, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Module *defM, StringRef func, StringRef specfunc, bool target_specsig) +{ + std::string gf_thunk_name; + if (!specfunc.empty()) { + Value *llvmtarget = IRLinker_copyFunctionProto(M, defM->getFunction(specfunc)); + gf_thunk_name = emit_abi_converter(M, params, declrt, sigt, nargs, specsig, codeinst, llvmtarget, target_specsig); + } + else { + Value *llvmtarget = func.empty() ? nullptr : IRLinker_copyFunctionProto(M, defM->getFunction(func)); + gf_thunk_name = emit_abi_dispatcher(M, params, declrt, sigt, nargs, specsig, codeinst, llvmtarget); + } + auto F = M->getFunction(gf_thunk_name); + assert(F); + return F; +} + +static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_functions_t &compiled_functions) +{ + DenseMap compiled_mi; + for (auto &def : compiled_functions) { + jl_code_instance_t *this_code = def.first; + jl_method_instance_t *mi = jl_get_ci_mi(this_code); + if (this_code->owner == jl_nothing && jl_atomic_load_relaxed(&this_code->max_world) == ~(size_t)0 && this_code->def == (jl_value_t*)mi) + compiled_mi[mi] = this_code; + } + size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); + for (cfunc_decl_t &cfunc : params.cfuncs) { + Module *M = cfunc.theFptr->getParent(); + jl_value_t *sigt = cfunc.sigt; + JL_GC_PROMISE_ROOTED(sigt); + jl_value_t *declrt = cfunc.declrt; + JL_GC_PROMISE_ROOTED(declrt); + Function *unspec = aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, nullptr, nullptr, "", "", false); + jl_code_instance_t *codeinst = nullptr; + auto assign_fptr = [¶ms, &cfunc, &codeinst, &unspec](Function *f) { + ConstantArray *init = cast(cfunc.cfuncdata->getInitializer()); + SmallVector initvals; + for (unsigned i = 0; i < init->getNumOperands(); ++i) + initvals.push_back(init->getOperand(i)); + assert(initvals.size() == 6); + assert(initvals[0]->isNullValue()); + if (codeinst) { + Constant *llvmcodeinst = literal_pointer_val_slot(params, f->getParent(), (jl_value_t*)codeinst); + initvals[0] = llvmcodeinst; // plast_codeinst + } + assert(initvals[2]->isNullValue()); + initvals[2] = unspec; + cfunc.cfuncdata->setInitializer(ConstantArray::get(init->getType(), initvals)); + cfunc.theFptr->setInitializer(f); + }; + Module *defM = nullptr; + StringRef func; + jl_method_instance_t *mi = jl_get_specialization1((jl_tupletype_t*)sigt, latestworld, 0); + if (mi) { + auto it = compiled_mi.find(mi); + if (it != compiled_mi.end()) { + codeinst = it->second; + JL_GC_PROMISE_ROOTED(codeinst); + auto defs = compiled_functions.find(codeinst); + defM = std::get<0>(defs->second).getModuleUnlocked(); + const jl_llvm_functions_t &decls = std::get<1>(defs->second); + func = decls.functionObject; + StringRef specfunc = decls.specFunctionObject; + jl_value_t *astrt = codeinst->rettype; + if (astrt != (jl_value_t*)jl_bottom_type && + jl_type_intersection(astrt, declrt) == jl_bottom_type) { + // Do not warn if the function never returns since it is + // occasionally required by the C API (typically error callbacks) + // even though we're likely to encounter memory errors in that case + jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); + } + if (func == "jl_fptr_const_return") { + std::string gf_thunk_name = emit_abi_constreturn(M, params, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst->rettype_const); + auto F = M->getFunction(gf_thunk_name); + assert(F); + assign_fptr(F); + continue; + } + else if (func == "jl_fptr_args") { + assert(!specfunc.empty()); + if (!cfunc.specsig && jl_subtype(astrt, declrt)) { + assign_fptr(IRLinker_copyFunctionProto(M, defM->getFunction(specfunc))); + continue; + } + assign_fptr(aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst, defM, func, specfunc, false)); + continue; + } + else if (func == "jl_fptr_sparam" || func == "jl_f_opaque_closure_call") { + func = ""; // use jl_invoke instead for these, since we don't declare these prototypes + } + else { + assert(!specfunc.empty()); + if (jl_egal(mi->specTypes, sigt) && jl_egal(declrt, astrt)) { + assign_fptr(IRLinker_copyFunctionProto(M, defM->getFunction(specfunc))); + continue; + } + assign_fptr(aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst, defM, func, specfunc, true)); + continue; + } + } + } + Function *f = codeinst ? aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst, defM, func, "", false) : unspec; + return assign_fptr(f); + } +} + // takes the running content that has collected in the shadow module and dump it to disk // this builds the object file portion of the sysimage files for fast startup @@ -651,7 +787,11 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, clone.getModuleUnlocked()->getDataLayout(), Triple(clone.getModuleUnlocked()->getTargetTriple())); - jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params); + jl_llvm_functions_t decls; + if (jl_atomic_load_relaxed(&codeinst->invoke) == jl_fptr_const_return_addr) + decls.functionObject = "jl_fptr_const_return"; + else + decls = jl_emit_codeinst(result_m, codeinst, src, params); record_method_roots(method_roots, jl_get_ci_mi(codeinst)); if (result_m) compiled_functions[codeinst] = {std::move(result_m), std::move(decls)}; @@ -671,6 +811,8 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm } // finally, make sure all referenced methods get fixed up, particularly if the user declined to compile them resolve_workqueue(params, method_roots, compiled_functions); + // including generating cfunction thunks + generate_cfunc_thunks(params, compiled_functions); aot_optimize_roots(params, method_roots, compiled_functions); params.temporary_roots = nullptr; JL_GC_POP(); @@ -728,9 +870,12 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm else if (func == "jl_fptr_sparam") { func_id = -2; } - else if (decls.functionObject == "jl_f_opaque_closure_call") { + else if (func == "jl_f_opaque_closure_call") { func_id = -4; } + else if (func == "jl_fptr_const_return") { + func_id = -5; + } else { //Safe b/c context is locked by params data->jl_sysimg_fvars.push_back(cast(clone.getModuleUnlocked()->getNamedValue(func))); @@ -2201,7 +2346,7 @@ extern "C" JL_DLLEXPORT_CODEGEN jl_code_info_t *jl_gdbdumpcode(jl_method_instanc // for use in reflection from Julia. // This is paired with jl_dump_function_ir and jl_dump_function_asm, either of which will free all memory allocated here extern "C" JL_DLLEXPORT_CODEGEN -void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, jl_code_info_t *src, char getwrapper, char optimize, const jl_cgparams_t params) +void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_code_info_t *src, char getwrapper, char optimize, const jl_cgparams_t params) { // emit this function into a new llvm module dump->F = nullptr; @@ -2223,7 +2368,31 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, jl_ output.imaging_mode = jl_options.image_codegen; output.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); JL_GC_PUSH1(&output.temporary_roots); - auto decls = jl_emit_code(m, mi, src, mi->specTypes, src->rettype, output); + jl_llvm_functions_t decls = jl_emit_code(m, mi, src, mi->specTypes, src->rettype, output); + // while not required, also emit the cfunc thunks, based on the + // inferred ABIs of their targets in the current latest world, + // since otherwise it is challenging to see all relevant codes + jl_compiled_functions_t compiled_functions; + size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); + for (cfunc_decl_t &cfunc : output.cfuncs) { + jl_value_t *sigt = cfunc.sigt; + JL_GC_PROMISE_ROOTED(sigt); + jl_method_instance_t *mi = jl_get_specialization1((jl_tupletype_t*)sigt, latestworld, 0); + if (mi == nullptr) + continue; + jl_code_instance_t *codeinst = jl_type_infer(mi, latestworld, SOURCE_MODE_NOT_REQUIRED); + if (codeinst == nullptr || compiled_functions.count(codeinst)) + continue; + orc::ThreadSafeModule decl_m = jl_create_ts_module("extern", ctx); + jl_llvm_functions_t decls; + if (jl_atomic_load_relaxed(&codeinst->invoke) == jl_fptr_const_return_addr) + decls.functionObject = "jl_fptr_const_return"; + else + decls = jl_emit_codedecls(decl_m, codeinst, output); + compiled_functions[codeinst] = {std::move(decl_m), std::move(decls)}; + } + generate_cfunc_thunks(output, compiled_functions); + compiled_functions.clear(); output.temporary_roots = nullptr; JL_GC_POP(); // GC the global_targets array contents now since reflection doesn't need it diff --git a/src/ccall.cpp b/src/ccall.cpp index eb64adef447f4..6c03f9532d9a8 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1970,6 +1970,8 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) return retval; } +static inline Constant *literal_static_pointer_val(const void *p, Type *T); + jl_cgval_t function_sig_t::emit_a_ccall( jl_codectx_t &ctx, const native_sym_arg_t &symarg, diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 98c5627578b80..39879503596fe 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -391,19 +391,17 @@ static llvm::SmallVector get_gc_roots_for(jl_codectx_t &ctx, const jl_ static void jl_temporary_root(jl_codegen_params_t &ctx, jl_value_t *val); static void jl_temporary_root(jl_codectx_t &ctx, jl_value_t *val); -static inline Constant *literal_static_pointer_val(const void *p, Type *T); -static Constant *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr) +static Constant *julia_pgv(jl_codegen_params_t ¶ms, Module *M, const char *cname, void *addr) { // emit a GlobalVariable for a jl_value_t named "cname" // store the name given so we can reuse it (facilitating merging later) // so first see if there already is a GlobalVariable for this address - GlobalVariable* &gv = ctx.emission_context.global_targets[addr]; - Module *M = jl_Module; + GlobalVariable* &gv = params.global_targets[addr]; StringRef localname; std::string gvname; if (!gv) { - uint64_t id = jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); // TODO: use ctx.emission_context.global_targets.size() + uint64_t id = jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); // TODO: use params.global_targets.size() raw_string_ostream(gvname) << cname << id; localname = StringRef(gvname); } @@ -413,9 +411,9 @@ static Constant *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr) gv = cast_or_null(M->getNamedValue(localname)); } if (gv == nullptr) - gv = new GlobalVariable(*M, ctx.types().T_pjlvalue, + gv = new GlobalVariable(*M, getPointerTy(M->getContext()), false, GlobalVariable::ExternalLinkage, - NULL, localname); + nullptr, localname); // LLVM passes sometimes strip metadata when moving load around // since the load at the new location satisfy the same condition as the original one. // Mark the global as constant to LLVM code using our own metadata @@ -426,7 +424,7 @@ static Constant *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr) return gv; } -static Constant *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name, jl_module_t *mod, void *addr) +static Constant *julia_pgv(jl_codegen_params_t ¶ms, Module *M, const char *prefix, jl_sym_t *name, jl_module_t *mod, void *addr) { // emit a GlobalVariable for a jl_value_t, using the prefix, name, and module to // to create a readable name of the form prefixModA.ModB.name# @@ -451,53 +449,49 @@ static Constant *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name finalname.resize(orig_end + prefix_name.size()); std::reverse_copy(prefix_name.begin(), prefix_name.end(), finalname.begin() + orig_end); std::reverse(finalname.begin(), finalname.end()); - return julia_pgv(ctx, finalname.c_str(), addr); + return julia_pgv(params, M, finalname.c_str(), addr); } static JuliaVariable *julia_const_gv(jl_value_t *val); -static Constant *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p) +Constant *literal_pointer_val_slot(jl_codegen_params_t ¶ms, Module *M, jl_value_t *p) { // emit a pointer to a jl_value_t* which will allow it to be valid across reloading code // also, try to give it a nice name for gdb, for easy identification if (JuliaVariable *gv = julia_const_gv(p)) { // if this is a known special object, use the existing GlobalValue - return prepare_global_in(jl_Module, gv); + return prepare_global_in(M, gv); } if (jl_is_datatype(p)) { jl_datatype_t *addr = (jl_datatype_t*)p; if (addr->smalltag) { // some common builtin datatypes have a special pool for accessing them by smalltag id - Constant *tag = ConstantInt::get(getInt32Ty(ctx.builder.getContext()), addr->smalltag << 4); - Constant *smallp = ConstantExpr::getInBoundsGetElementPtr(getInt8Ty(ctx.builder.getContext()), prepare_global_in(jl_Module, jl_small_typeof_var), tag); - auto ty = ctx.types().T_ppjlvalue; - if (ty->getPointerAddressSpace() == smallp->getType()->getPointerAddressSpace()) - return ConstantExpr::getBitCast(smallp, ty); - else { - Constant *newsmallp = ConstantExpr::getAddrSpaceCast(smallp, ty); - return ConstantExpr::getBitCast(newsmallp, ty); - } + Constant *tag = ConstantInt::get(getInt32Ty(M->getContext()), addr->smalltag << 4); + Constant *smallp = ConstantExpr::getInBoundsGetElementPtr(getInt8Ty(M->getContext()), prepare_global_in(M, jl_small_typeof_var), tag); + if (smallp->getType()->getPointerAddressSpace() != 0) + smallp = ConstantExpr::getAddrSpaceCast(smallp, getPointerTy(M->getContext())); + return smallp; } // DataTypes are prefixed with a + - return julia_pgv(ctx, "+", addr->name->name, addr->name->module, p); + return julia_pgv(params, M, "+", addr->name->name, addr->name->module, p); } if (jl_is_method(p)) { jl_method_t *m = (jl_method_t*)p; // functions are prefixed with a - - return julia_pgv(ctx, "-", m->name, m->module, p); + return julia_pgv(params, M, "-", m->name, m->module, p); } if (jl_is_method_instance(p)) { jl_method_instance_t *linfo = (jl_method_instance_t*)p; // Type-inferred functions are also prefixed with a - if (jl_is_method(linfo->def.method)) - return julia_pgv(ctx, "-", linfo->def.method->name, linfo->def.method->module, p); + return julia_pgv(params, M, "-", linfo->def.method->name, linfo->def.method->module, p); } if (jl_is_symbol(p)) { jl_sym_t *addr = (jl_sym_t*)p; // Symbols are prefixed with jl_sym# - return julia_pgv(ctx, "jl_sym#", addr, NULL, p); + return julia_pgv(params, M, "jl_sym#", addr, NULL, p); } // something else gets just a generic name - return julia_pgv(ctx, "jl_global#", p); + return julia_pgv(params, M, "jl_global#", p); } static size_t dereferenceable_size(jl_value_t *jt) @@ -570,7 +564,7 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p) { if (p == NULL) return Constant::getNullValue(ctx.types().T_pjlvalue); - Value *pgv = literal_pointer_val_slot(ctx, p); + Value *pgv = literal_pointer_val_slot(ctx.emission_context, jl_Module, p); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); auto load = ai.decorateInst(maybe_mark_load_dereferenceable( ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, pgv, Align(sizeof(void*))), @@ -610,7 +604,7 @@ static Value *julia_binding_gv(jl_codectx_t &ctx, jl_binding_t *b) // emit a literal_pointer_val to a jl_binding_t // binding->value are prefixed with * jl_globalref_t *gr = b->globalref; - Value *pgv = gr ? julia_pgv(ctx, "*", gr->name, gr->mod, b) : julia_pgv(ctx, "*jl_bnd#", b); + Value *pgv = gr ? julia_pgv(ctx.emission_context, jl_Module, "*", gr->name, gr->mod, b) : julia_pgv(ctx.emission_context, jl_Module, "*jl_bnd#", b); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); auto load = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, pgv, Align(sizeof(void*)))); setName(ctx.emission_context, load, pgv->getName()); @@ -1352,7 +1346,7 @@ static Value *emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull ptr = get_pointer_to_constant(ctx.emission_context, ConstantInt::get(expr_type, jt->smalltag << 4), Align(sizeof(jl_value_t*)), StringRef("_j_smalltag_") + jl_symbol_name(jt->name->name), *jl_Module); } else { - ptr = ConstantExpr::getBitCast(literal_pointer_val_slot(ctx, (jl_value_t*)jt), datatype_or_p->getType()); + ptr = ConstantExpr::getBitCast(literal_pointer_val_slot(ctx.emission_context, jl_Module, (jl_value_t*)jt), datatype_or_p->getType()); } datatype_or_p = ctx.builder.CreateSelect(cmp, ptr, datatype_or_p); setName(ctx.emission_context, datatype_or_p, "typetag_ptr"); @@ -2284,7 +2278,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, const jl_cgval_t argv[3] = { cmp, lhs, rhs }; jl_cgval_t ret; if (modifyop) { - ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, nullptr); + ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); } else { if (trim_may_error(ctx.params->trim)) { @@ -4023,7 +4017,7 @@ static jl_cgval_t union_store(jl_codectx_t &ctx, emit_lockstate_value(ctx, needlock, false); const jl_cgval_t argv[3] = { cmp, oldval, rhs }; if (modifyop) { - rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, nullptr); + rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); } else { if (trim_may_error(ctx.params->trim)) { diff --git a/src/codegen.cpp b/src/codegen.cpp index e9e4275672c7e..7787b684fbd12 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -278,16 +278,16 @@ extern void _chkstk(void); // types struct jl_typecache_t { - Type *T_ptr; + PointerType *T_ptr; Type *T_size; Type *T_jlvalue; - Type *T_pjlvalue; - Type *T_prjlvalue; - Type *T_ppjlvalue; - Type *T_pprjlvalue; + PointerType *T_pjlvalue; + PointerType *T_prjlvalue; + PointerType *T_ppjlvalue; + PointerType *T_pprjlvalue; StructType *T_jlgenericmemory; StructType *T_jlarray; - Type *T_pjlarray; + PointerType *T_pjlarray; FunctionType *T_jlfunc; FunctionType *T_jlfuncparams; @@ -990,20 +990,12 @@ static const auto jlapplygeneric_func = new JuliaFunction<>{ static const auto jlinvoke_func = new JuliaFunction<>{ XSTR(jl_invoke), get_func2_sig, - [](LLVMContext &C) { return AttributeList::get(C, - AttributeSet(), - Attributes(C, {Attribute::NonNull}), - {AttributeSet(), - Attributes(C, {Attribute::ReadOnly, Attribute::NoCapture})}); }, + get_func_attrs, }; static const auto jlinvokeoc_func = new JuliaFunction<>{ XSTR(jl_invoke_oc), get_func2_sig, - [](LLVMContext &C) { return AttributeList::get(C, - AttributeSet(), - Attributes(C, {Attribute::NonNull}), - {AttributeSet(), - Attributes(C, {Attribute::ReadOnly, Attribute::NoCapture})}); }, + get_func_attrs, }; static const auto jlopaque_closure_call_func = new JuliaFunction<>{ XSTR(jl_f_opaque_closure_call), @@ -1396,6 +1388,14 @@ static const auto jlgetcfunctiontrampoline_func = new JuliaFunction<>{ Attributes(C, {Attribute::NonNull}), None); }, }; +static const auto jlgetabiconverter_func = new JuliaFunction{ + XSTR(jl_get_abi_converter), + [](LLVMContext &C, Type *T_size) { + Type *T_ptr = getPointerTy(C); + return FunctionType::get(T_ptr, + {T_ptr, T_ptr, T_ptr, T_ptr}, false); }, + nullptr, +}; static const auto diff_gc_total_bytes_func = new JuliaFunction<>{ XSTR(jl_gc_diff_total_bytes), [](LLVMContext &C) { return FunctionType::get(getInt64Ty(C), false); }, @@ -1462,7 +1462,6 @@ static const auto jlgetbuiltinfptr_func = new JuliaFunction<>{ nullptr, }; - // placeholder functions static const auto gcroot_flush_func = new JuliaFunction<>{ "julia.gcroot_flush", @@ -2095,7 +2094,7 @@ jl_aliasinfo_t jl_aliasinfo_t::fromTBAA(jl_codectx_t &ctx, MDNode *tbaa) { } static Type *julia_type_to_llvm(jl_codectx_t &ctx, jl_value_t *jt, bool *isboxed = NULL); -static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value *fval, StringRef name, jl_value_t *sig, jl_value_t *jlrettype, bool is_opaque_closure, bool gcstack_arg, +static jl_returninfo_t get_specsig_function(jl_codegen_params_t &ctx, Module *M, Value *fval, StringRef name, jl_value_t *sig, jl_value_t *jlrettype, bool is_opaque_closure, ArrayRef ArgNames=None, unsigned nreq=0); static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval = -1); static jl_cgval_t emit_checked_var(jl_codectx_t &ctx, Value *bp, jl_sym_t *name, jl_value_t *scope, bool isvol, MDNode *tbaa); @@ -2107,14 +2106,14 @@ static Value *get_tls_world_age(jl_codectx_t &ctx); static Value *get_scope_field(jl_codectx_t &ctx); static Value *get_tls_world_age_field(jl_codectx_t &ctx); static void CreateTrap(IRBuilder<> &irbuilder, bool create_new_block = true); -static CallInst *emit_jlcall(jl_codectx_t &ctx, FunctionCallee theFptr, Value *theF, +static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, ArrayRef args, size_t nargs, JuliaFunction<> *trampoline); static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value *theF, ArrayRef args, size_t nargs, JuliaFunction<> *trampoline); static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2, Value *nullcheck1 = nullptr, Value *nullcheck2 = nullptr); static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, ArrayRef argv, bool is_promotable=false); -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, Value *age_ok); +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt); static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static unsigned julia_alignment(jl_value_t *jt); @@ -2914,7 +2913,7 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ return jl_cgval_t(v, typ, new_tindex); } -std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &context, const DataLayout &DL, const Triple &triple) +std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &context, const DataLayout &DL, const Triple &triple) JL_NOTSAFEPOINT { ++ModulesCreated; auto m = std::make_unique(name, context); @@ -2941,14 +2940,16 @@ std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &conte return m; } -static void jl_name_jlfunc_args(jl_codegen_params_t ¶ms, Function *F) { +static void jl_name_jlfunc_args(jl_codegen_params_t ¶ms, Function *F) JL_NOTSAFEPOINT +{ assert(F->arg_size() == 3); F->getArg(0)->setName("function::Core.Function"); F->getArg(1)->setName("args::Any[]"); F->getArg(2)->setName("nargs::UInt32"); } -static void jl_name_jlfuncparams_args(jl_codegen_params_t ¶ms, Function *F) { +static void jl_name_jlfuncparams_args(jl_codegen_params_t ¶ms, Function *F) JL_NOTSAFEPOINT +{ assert(F->arg_size() == 4); F->getArg(0)->setName("function::Core.Function"); F->getArg(1)->setName("args::Any[]"); @@ -2956,7 +2957,7 @@ static void jl_name_jlfuncparams_args(jl_codegen_params_t ¶ms, Function *F) F->getArg(3)->setName("sparams::Any"); } -void jl_init_function(Function *F, const Triple &TT) +void jl_init_function(Function *F, const Triple &TT) JL_NOTSAFEPOINT { // set any attributes that *must* be set on all functions AttrBuilder attr(F->getContext()); @@ -3020,6 +3021,7 @@ static bool uses_specsig(jl_value_t *sig, bool needsparams, jl_value_t *rettype, bool allSingleton = true; for (size_t i = 0; i < jl_nparams(sig); i++) { jl_value_t *sigt = jl_tparam(sig, i); + // TODO: sigt = unwrap_va(sigt) bool issing = jl_is_datatype(sigt) && jl_is_datatype_singleton((jl_datatype_t*)sigt); allSingleton &= issing; if (!deserves_argbox(sigt) && !issing) { @@ -5311,14 +5313,14 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } // Returns ctx.types().T_prjlvalue -static CallInst *emit_jlcall(jl_codectx_t &ctx, FunctionCallee theFptr, Value *theF, +static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, ArrayRef argv, size_t nargs, JuliaFunction<> *trampoline) { ++EmittedJLCalls; Function *TheTrampoline = prepare_call(trampoline); // emit arguments SmallVector theArgs; - theArgs.push_back(theFptr.getCallee()); + theArgs.push_back(theFptr); if (theF) theArgs.push_back(theF); for (size_t i = 0; i < nargs; i++) { @@ -5485,12 +5487,11 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, llvm::Value *callee, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *nreturn_roots, jl_value_t *inferred_retty, Value *age_ok) + ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *nreturn_roots, jl_value_t *inferred_retty) { ++EmittedSpecfunCalls; // emit specialized call site - bool gcstack_arg = JL_FEAT_TEST(ctx, gcstack_arg); - jl_returninfo_t returninfo = get_specsig_function(ctx, jl_Module, callee, specFunctionObject, specTypes, jlretty, is_opaque_closure, gcstack_arg); + jl_returninfo_t returninfo = get_specsig_function(ctx.emission_context, jl_Module, callee, specFunctionObject, specTypes, jlretty, is_opaque_closure); *cc = returninfo.cc; *nreturn_roots = returninfo.return_roots; if (fromexternal) { @@ -5510,31 +5511,17 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos setName(ctx.emission_context, TheCallee, namep); returninfo.decl = FunctionCallee(returninfo.decl.getFunctionType(), TheCallee); } - if (age_ok) { - std::string funcName(specFunctionObject); - funcName += "_gfthunk"; - Function *gf_thunk = Function::Create(returninfo.decl.getFunctionType(), - GlobalVariable::InternalLinkage, funcName, jl_Module); - jl_init_function(gf_thunk, ctx.emission_context.TargetTriple); - gf_thunk->setAttributes(AttributeList::get(gf_thunk->getContext(), {returninfo.attrs, gf_thunk->getAttributes()})); - // build a specsig -> jl_apply_generic converter thunk - // this builds a method that calls jl_apply_generic (as a closure over a singleton function pointer), - // but which has the signature of a specsig - emit_specsig_to_fptr1(gf_thunk, returninfo.cc, returninfo.return_roots, specTypes, jlretty, is_opaque_closure, nargs, ctx.emission_context, - prepare_call(jlapplygeneric_func)); - returninfo.decl = FunctionCallee(returninfo.decl.getFunctionType(), ctx.builder.CreateSelect(age_ok, returninfo.decl.getCallee(), gf_thunk)); - } jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, specTypes, jlretty, returninfo, argv, nargs); // see if inference has a different / better type for the call than the lambda return update_julia_type(ctx, retval, inferred_retty); } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_t *mi, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty, Value *age_ok) + ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) { bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; return emit_call_specfun_other(ctx, is_opaque_closure, mi->specTypes, jlretty, NULL, - specFunctionObject, fromexternal, argv, nargs, cc, return_roots, inferred_retty, age_ok); + specFunctionObject, fromexternal, argv, nargs, cc, return_roots, inferred_retty); } static jl_value_t *get_ci_abi(jl_code_instance_t *ci) @@ -5545,16 +5532,16 @@ static jl_value_t *get_ci_abi(jl_code_instance_t *ci) } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_code_instance_t *ci, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty, Value *age_ok) + ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) { jl_method_instance_t *mi = jl_get_ci_mi(ci); bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; return emit_call_specfun_other(ctx, is_opaque_closure, get_ci_abi(ci), ci->rettype, NULL, - specFunctionObject, fromexternal, argv, nargs, cc, return_roots, inferred_retty, age_ok); + specFunctionObject, fromexternal, argv, nargs, cc, return_roots, inferred_retty); } static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_value_t *inferred_retty, Value *age_ok) + ArrayRef argv, size_t nargs, jl_value_t *inferred_retty) { Value *theFptr; if (fromexternal) { @@ -5577,9 +5564,7 @@ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty theFptr = jl_Module->getOrInsertFunction(specFunctionObject, ctx.types().T_jlfunc).getCallee(); addRetAttr(cast(theFptr), Attribute::NonNull); } - if (age_ok) - theFptr = ctx.builder.CreateSelect(age_ok, theFptr, prepare_call(jlapplygeneric_func)); - Value *ret = emit_jlcall(ctx, FunctionCallee(ctx.types().T_jlfunc, theFptr), nullptr, argv, nargs, julia_call); + Value *ret = emit_jlcall(ctx, theFptr, nullptr, argv, nargs, julia_call); return update_julia_type(ctx, mark_julia_type(ctx, ret, true, jlretty), inferred_retty); } @@ -5597,10 +5582,10 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) if (argv[i].typ == jl_bottom_type) return jl_cgval_t(); } - return emit_invoke(ctx, lival, argv, nargs, rt, nullptr); + return emit_invoke(ctx, lival, argv, nargs, rt); } -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, Value *age_ok) +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt) { ++EmittedInvokes; bool handled = false; @@ -5633,7 +5618,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR unsigned return_roots = 0; jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; StringRef protoname = f->getName(); - result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, &cc, &return_roots, rt, age_ok); + result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, &cc, &return_roots, rt); } handled = true; } @@ -5699,9 +5684,9 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; unsigned return_roots = 0; if (specsig) - result = emit_call_specfun_other(ctx, codeinst, protoname, external ? codeinst : nullptr, argv, nargs, &cc, &return_roots, rt, age_ok); + result = emit_call_specfun_other(ctx, codeinst, protoname, external ? codeinst : nullptr, argv, nargs, &cc, &return_roots, rt); else - result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt, age_ok); + result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt); if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig}; @@ -5726,8 +5711,8 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR print_stacktrace(ctx, ctx.params->trim); } } - Value *r = age_ok ? emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs, julia_call) : emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); - result = mark_julia_type(ctx, r, true, age_ok ? (jl_value_t*)jl_any_type : rt); + Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); + result = mark_julia_type(ctx, r, true, rt); } if (result.typ == jl_bottom_type) { #ifndef JL_NDEBUG @@ -5818,9 +5803,10 @@ static jl_cgval_t emit_specsig_oc_call(jl_codectx_t &ctx, jl_value_t *oc_type, j jl_cgval_t &theArg = argv[0]; jl_cgval_t closure_specptr = emit_getfield_knownidx(ctx, theArg, 4, (jl_datatype_t*)oc_type, jl_memory_order_notatomic); Value *specptr = emit_unbox(ctx, ctx.types().T_size, closure_specptr, (jl_value_t*)jl_long_type); + specptr = emit_inttoptr(ctx, specptr, ctx.types().T_ptr); JL_GC_PUSH1(&sigtype); jl_cgval_t r = emit_call_specfun_other(ctx, true, sigtype, oc_rett, specptr, "", NULL, argv, nargs, - &cc, &return_roots, oc_rett, nullptr); + &cc, &return_roots, oc_rett); JL_GC_POP(); return r; } @@ -5898,14 +5884,14 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo return mark_julia_type(ctx, ret, true, rt); } } - FunctionCallee fptr; + Value *fptr; JuliaFunction<> *cc; if (f.typ == (jl_value_t*)jl_intrinsic_type) { fptr = prepare_call(jlintrinsic_func); cc = julia_call3; } else { - fptr = FunctionCallee(get_func_sig(ctx.builder.getContext()), ctx.builder.CreateCall(prepare_call(jlgetbuiltinfptr_func), {emit_typeof(ctx, f)})); + fptr = ctx.builder.CreateCall(prepare_call(jlgetbuiltinfptr_func), {emit_typeof(ctx, f)}); cc = julia_call; } if (trim_may_error(ctx.params->trim)) { @@ -6723,8 +6709,7 @@ static std::pair get_oc_function(jl_codectx_t &ctx, jl_met bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; assert(is_opaque_closure); if (specsig) { - bool gcstack_arg = JL_FEAT_TEST(ctx, gcstack_arg); - jl_returninfo_t returninfo = get_specsig_function(ctx, jl_Module, nullptr, protoname, mi->specTypes, rettype, is_opaque_closure, gcstack_arg); + jl_returninfo_t returninfo = get_specsig_function(ctx.emission_context, jl_Module, nullptr, protoname, mi->specTypes, rettype, is_opaque_closure); cc = returninfo.cc; return_roots = returninfo.return_roots; specF = cast(returninfo.decl.getCallee()); @@ -7192,7 +7177,7 @@ JL_GCC_IGNORE_STOP // --- generate function bodies --- // gc frame emission -static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0, bool or_new=false) +static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0, bool or_new=false) JL_NOTSAFEPOINT { // allocate a placeholder gc instruction // this will require the runtime, but it gets deleted later if unused @@ -7244,12 +7229,34 @@ static Value *get_scope_field(jl_codectx_t &ctx) return emit_ptrgep(ctx, ct, offsetof(jl_task_t, scope), "scope"); } +static std::string get_function_name(bool specsig, bool needsparams, const char *unadorned_name, const Triple &TargetTriple) +{ + std::string _funcName; + raw_string_ostream funcName(_funcName); + // try to avoid conflicts in the global symbol table + if (specsig) + funcName << "julia_"; // api 5 + else if (needsparams) + funcName << "japi3_"; + else + funcName << "japi1_"; + if (TargetTriple.isOSLinux()) { + if (unadorned_name[0] == '@') + unadorned_name++; + } + funcName << unadorned_name << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); + return funcName.str(); +} + +static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *abi, jl_value_t *jlretty, jl_value_t *declrt, jl_returninfo_t &f, unsigned nargs, int retarg, bool is_opaque_closure, StringRef funcName, + Module *M, jl_codegen_params_t ¶ms); + Function *get_or_emit_fptr1(StringRef preal_decl, Module *M) { return cast(M->getOrInsertFunction(preal_decl, get_func_sig(M->getContext()), get_func_attrs(M->getContext())).getCallee()); } -Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT +Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Value *theFunc, Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT { ++EmittedToJLInvokes; jl_codectx_t ctx(M->getContext(), params, codeinst); @@ -7265,14 +7272,11 @@ Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, M jl_name_jlfunc_args(params, f); //f->setAlwaysInline(); ctx.f = f; // for jl_Module - BasicBlock *b0 = BasicBlock::Create(ctx.builder.getContext(), "top", f); + BasicBlock *b0 = BasicBlock::Create(M->getContext(), "top", f); ctx.builder.SetInsertPoint(b0); - Function *theFunc; Value *theFarg; - if (!theFptrName.empty()) { - theFunc = cast( - M->getOrInsertFunction(theFptrName, jlinvoke_func->_type(ctx.builder.getContext())).getCallee()); + if (theFunc) { theFarg = literal_pointer_val(ctx, (jl_value_t*)codeinst); } else { @@ -7283,12 +7287,20 @@ Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, M } theFarg = track_pjlvalue(ctx, theFarg); auto args = f->arg_begin(); - CallInst *r = ctx.builder.CreateCall(theFunc, { &*args, &*++args, &*++args, theFarg }); - r->setAttributes(theFunc->getAttributes()); + CallInst *r = ctx.builder.CreateCall(FunctionCallee(jlinvoke_func->_type(M->getContext()), theFunc), { &*args, &*++args, &*++args, theFarg }); + r->setAttributes(jlinvoke_func->_attrs(M->getContext())); ctx.builder.CreateRet(r); return f; } +Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT +{ + Value *theFunc = nullptr; + if (!theFptrName.empty()) + theFunc = M->getOrInsertFunction(theFptrName, jlinvoke_func->_type(M->getContext()), jlinvoke_func->_attrs(M->getContext())).getCallee(); + return emit_tojlinvoke(codeinst, theFunc, M, params); +} + static jl_value_t *get_oc_type(jl_value_t *calltype, jl_value_t *rettype) JL_ALWAYS_LEAFTYPE { jl_value_t *argtype = jl_argtype_without_function((jl_value_t*)calltype); @@ -7299,12 +7311,16 @@ static jl_value_t *get_oc_type(jl_value_t *calltype, jl_value_t *rettype) JL_ALW return oc_type; } -void emit_specsig_to_fptr1( +static void emit_specsig_to_specsig( Function *gf_thunk, jl_returninfo_t::CallingConv cc, unsigned return_roots, jl_value_t *calltype, jl_value_t *rettype, bool is_for_opaque_closure, size_t nargs, jl_codegen_params_t ¶ms, - Function *target) + Value *target, + jl_value_t *targetsig, + jl_value_t *targetrt, + jl_returninfo_t *targetspec, + jl_value_t *rettype_const) { ++EmittedCFuncInvalidates; jl_codectx_t ctx(gf_thunk->getParent()->getContext(), params, 0, 0); @@ -7321,7 +7337,7 @@ void emit_specsig_to_fptr1( ++AI; if (return_roots) ++AI; - if (JL_FEAT_TEST(ctx,gcstack_arg)){ + if (JL_FEAT_TEST(ctx,gcstack_arg)) { ++AI; // gcstack_arg } for (size_t i = 0; i < nargs; i++) { @@ -7372,16 +7388,23 @@ void emit_specsig_to_fptr1( } } assert(AI == gf_thunk->arg_end()); - Value *gf_ret = emit_jlcall(ctx, target, nullptr, myargs, nargs, julia_call); - jl_cgval_t gf_retbox = mark_julia_type(ctx, gf_ret, true, jl_any_type); - if (cc != jl_returninfo_t::Boxed) { - emit_typecheck(ctx, gf_retbox, rettype, "cfunction"); - gf_retbox = update_julia_type(ctx, gf_retbox, rettype); + jl_cgval_t gf_retval; + if (target || targetspec) { + if (targetspec == nullptr) + gf_retval = mark_julia_type(ctx, emit_jlcall(ctx, target, nullptr, myargs, nargs, julia_call), true, targetrt); + else + gf_retval = emit_call_specfun_other(ctx, is_for_opaque_closure, targetsig, targetrt, *targetspec, myargs, nargs); + } + if (rettype_const) + gf_retval = mark_julia_const(ctx, rettype_const); + if (targetrt != rettype) { + emit_typecheck(ctx, gf_retval, rettype, "cfunction"); + gf_retval = update_julia_type(ctx, gf_retval, rettype); } switch (cc) { case jl_returninfo_t::Boxed: - ctx.builder.CreateRet(gf_ret); + ctx.builder.CreateRet(boxed(ctx, gf_retval)); break; case jl_returninfo_t::Register: { Type *gfrt = gf_thunk->getReturnType(); @@ -7389,7 +7412,7 @@ void emit_specsig_to_fptr1( ctx.builder.CreateRetVoid(); } else { - ctx.builder.CreateRet(ctx.builder.CreateAlignedLoad(gfrt, gf_ret, Align(julia_alignment(rettype)))); + ctx.builder.CreateRet(emit_unbox(ctx, gfrt, gf_retval, rettype)); } break; } @@ -7398,65 +7421,273 @@ void emit_specsig_to_fptr1( Align align(julia_alignment(rettype)); if (return_roots) { Value *roots = gf_thunk->arg_begin() + 1; // root1 has type [n x {}*]* - split_value_into(ctx, gf_retbox, align, sret, align, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), roots, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe)); + split_value_into(ctx, gf_retval, align, sret, align, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), roots, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe)); } else { - emit_unbox_store(ctx, gf_retbox, sret, ctx.tbaa().tbaa_stack, align); + emit_unbox_store(ctx, gf_retval, sret, ctx.tbaa().tbaa_stack, align); } ctx.builder.CreateRetVoid(); break; } case jl_returninfo_t::Union: { + Value *gf_ret = boxed(ctx, gf_retval); // TODO: this is not the most optimal way to emit this Type *retty = gf_thunk->getReturnType(); - Value *gf_retval = UndefValue::get(retty); - Value *tindex = compute_box_tindex(ctx, emit_typeof(ctx, gf_retbox, false, true), (jl_value_t*)jl_any_type, rettype); + Value *retval = UndefValue::get(retty); + Value *tindex = compute_box_tindex(ctx, emit_typeof(ctx, gf_retval, false, true), (jl_value_t*)jl_any_type, rettype); tindex = ctx.builder.CreateOr(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), UNION_BOX_MARKER)); - gf_retval = ctx.builder.CreateInsertValue(gf_retval, gf_ret, 0); - gf_retval = ctx.builder.CreateInsertValue(gf_retval, tindex, 1); - ctx.builder.CreateRet(gf_retval); + retval = ctx.builder.CreateInsertValue(retval, gf_ret, 0); + retval = ctx.builder.CreateInsertValue(retval, tindex, 1); + ctx.builder.CreateRet(retval); break; } case jl_returninfo_t::Ghosts: { - Value *gf_retval = compute_tindex_unboxed(ctx, gf_retbox, rettype); - ctx.builder.CreateRet(gf_retval); + Value *retval = compute_tindex_unboxed(ctx, gf_retval, rettype); + ctx.builder.CreateRet(retval); break; } } } +void emit_specsig_to_fptr1( + Function *gf_thunk, jl_returninfo_t::CallingConv cc, unsigned return_roots, + jl_value_t *calltype, jl_value_t *rettype, bool is_for_opaque_closure, + size_t nargs, + jl_codegen_params_t ¶ms, + Function *target) +{ + emit_specsig_to_specsig(gf_thunk, cc, return_roots, calltype, rettype, is_for_opaque_closure, nargs, params, target, calltype, rettype, nullptr, nullptr); +} + +static void emit_fptr1_wrapper(Module *M, StringRef gf_thunk_name, Value *target, jl_value_t *rettype_const, jl_value_t *declrt, jl_value_t *jlrettype, jl_codegen_params_t ¶ms) +{ + Function *w = Function::Create(get_func_sig(M->getContext()), GlobalVariable::ExternalLinkage, gf_thunk_name, M); + jl_init_function(w, params.TargetTriple); + w->setAttributes(AttributeList::get(M->getContext(), {get_func_attrs(M->getContext()), w->getAttributes()})); + w->addFnAttr(Attribute::OptimizeNone); + w->addFnAttr(Attribute::NoInline); + + jl_codectx_t ctx(M->getContext(), params, 0, 0); + ctx.f = w; + ctx.rettype = declrt; + + BasicBlock *b0 = BasicBlock::Create(ctx.builder.getContext(), "top", w); + ctx.builder.SetInsertPoint(b0); + DebugLoc noDbg; + ctx.builder.SetCurrentDebugLocation(noDbg); + allocate_gc_frame(ctx, b0); + + jl_cgval_t gf_retval; + if (target) { + FunctionCallee theFunc(w->getFunctionType(), target); + auto args = w->arg_begin(); + CallInst *r = ctx.builder.CreateCall(theFunc, { &*args, &*++args, &*++args }); // cf emit_tojlinvoke + assert(++args == w->arg_end()); + r->setAttributes(w->getAttributes()); + gf_retval = mark_julia_type(ctx, r, true, jlrettype); + } + if (rettype_const) + gf_retval = mark_julia_const(ctx, rettype_const); + if (jlrettype != declrt) + emit_typecheck(ctx, gf_retval, declrt, "cfunction"); + ctx.builder.CreateRet(boxed(ctx, gf_retval)); +} + +static void emit_specsig_to_specsig( + Module *M, StringRef gf_thunk_name, + jl_value_t *calltype, jl_value_t *rettype, bool is_for_opaque_closure, + size_t nargs, + jl_codegen_params_t ¶ms, + Value *target, + jl_value_t *targetsig, + jl_value_t *targetrt, + jl_returninfo_t *targetspec, + jl_value_t *rettype_const) +{ + jl_returninfo_t returninfo = get_specsig_function(params, M, nullptr, gf_thunk_name, calltype, rettype, is_for_opaque_closure); + Function *gf_thunk = cast(returninfo.decl.getCallee()); + jl_init_function(gf_thunk, params.TargetTriple); + gf_thunk->setAttributes(AttributeList::get(gf_thunk->getContext(), {returninfo.attrs, gf_thunk->getAttributes()})); + emit_specsig_to_specsig(gf_thunk, returninfo.cc, returninfo.return_roots, calltype, rettype, is_for_opaque_closure, nargs, params, target, targetsig, targetrt, targetspec, rettype_const); +} + +std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *target, bool target_specsig) +{ + // this builds a method that calls a method with the same arguments but a different specsig + // build a specsig -> specsig converter thunk + // build a specsig -> arg1 converter thunk + // build a args1 -> specsig converter thunk (gen_invoke_wrapper) + // build a args1 -> args1 converter thunk (to add typeassert on result) + bool needsparams = false; + bool is_opaque_closure = false; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + std::string gf_thunk_name = get_function_name(specsig, needsparams, name_from_method_instance(mi), params.TargetTriple); + gf_thunk_name += "_gfthunk"; + if (target_specsig) { + jl_value_t *abi = get_ci_abi(codeinst); + jl_returninfo_t targetspec = get_specsig_function(params, M, target, "", abi, codeinst->rettype, is_opaque_closure); + if (specsig) + emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, + target, mi->specTypes, codeinst->rettype, &targetspec, nullptr); + else + gen_invoke_wrapper(mi, abi, codeinst->rettype, declrt, targetspec, nargs, -1, is_opaque_closure, gf_thunk_name, M, params); + } + else { + if (specsig) + emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, + target, mi->specTypes, codeinst->rettype, nullptr, nullptr); + else + emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, declrt, codeinst->rettype, params); + } + return gf_thunk_name; +} + +std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *invoke) +{ + // this builds a method that calls a method with the same arguments but a different specsig + // build a specsig -> args1 (apply_generic) or invoke (emit_tojlinvoke) call + // build a args1 -> args1 call (emit_fptr1_wrapper) + // build a args1 -> invoke call (emit_tojlinvoke) + bool is_opaque_closure = false; + Value *target; + if (!codeinst) + target = prepare_call_in(M, jlapplygeneric_func); + else + target = emit_tojlinvoke(codeinst, invoke, M, params); // TODO: inline this call? + std::string gf_thunk_name; + if (codeinst) + raw_string_ostream(gf_thunk_name) << "jfptr_" << name_from_method_instance(jl_get_ci_mi(codeinst)) << "_"; + else + raw_string_ostream(gf_thunk_name) << "j_"; + raw_string_ostream(gf_thunk_name) << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1) << "_gfthunk"; + if (specsig) + emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, + target, sigt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, nullptr, nullptr); + else + emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, declrt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, params); + return gf_thunk_name; +} + +std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_value_t *rettype_const) +{ + bool is_opaque_closure = false; + std::string gf_thunk_name; + raw_string_ostream(gf_thunk_name) << "jconst_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); + if (specsig) { + emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, + nullptr, sigt, jl_typeof(rettype_const), nullptr, rettype_const); + } + else { + emit_fptr1_wrapper(M, gf_thunk_name, nullptr, rettype_const, declrt, jl_typeof(rettype_const), params); + } + return gf_thunk_name; +} + +std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, bool specsig, jl_code_instance_t *codeinst) +{ + jl_value_t *abi = get_ci_abi(codeinst); + return emit_abi_constreturn(M, params, codeinst->rettype, abi, specsig ? jl_nparams(abi) : 0, specsig, codeinst->rettype_const); +} + +// release jl_world_counter +// store theFptr +// release last_world_v +// +// acquire last_world_v +// read theFptr +// acquire jl_world_counter +// if (last_world_v != jl_world_counter) +// fptr = compute_new_fptr(&last_world_v) +// return fptr() +static jl_cgval_t emit_abi_call(jl_codectx_t &ctx, jl_value_t *declrt, jl_value_t *sigt, ArrayRef inputargs, size_t nargs, Value *world_age_field) +{ + jl_cgval_t retval; + if (sigt) { + jl_temporary_root(ctx, declrt); + jl_temporary_root(ctx, sigt); + assert(nargs == jl_nparams(sigt)); + bool needsparams = false; + bool is_opaque_closure = false; + bool specsig = uses_specsig(sigt, needsparams, declrt, ctx.params->prefer_specsig); + PointerType *T_ptr = ctx.types().T_ptr; + Type *T_size = ctx.types().T_size; + Constant *Vnull = ConstantPointerNull::get(T_ptr); + Module *M = jl_Module; + GlobalVariable *theFptr = new GlobalVariable(*M, T_ptr, false, + GlobalVariable::PrivateLinkage, + Vnull); + GlobalVariable *last_world_p = new GlobalVariable(*M, T_size, false, + GlobalVariable::PrivateLinkage, + ConstantInt::get(T_size, 0)); + ArrayType *T_cfuncdata = ArrayType::get(T_ptr, 6); + size_t flags = specsig; + GlobalVariable *cfuncdata = new GlobalVariable(*M, T_cfuncdata, false, + GlobalVariable::PrivateLinkage, + ConstantArray::get(T_cfuncdata, { + Vnull, + Vnull, + Vnull, + literal_pointer_val_slot(ctx.emission_context, M, declrt), + literal_pointer_val_slot(ctx.emission_context, M, sigt), + literal_static_pointer_val((void*)flags, T_ptr)})); + LoadInst *last_world_v = ctx.builder.CreateAlignedLoad(T_size, last_world_p, ctx.types().alignof_ptr); + last_world_v->setOrdering(AtomicOrdering::Acquire); + LoadInst *callee = ctx.builder.CreateAlignedLoad(T_ptr, theFptr, ctx.types().alignof_ptr); + callee->setOrdering(AtomicOrdering::Monotonic); + LoadInst *world_v = ctx.builder.CreateAlignedLoad(ctx.types().T_size, + prepare_global_in(M, jlgetworld_global), ctx.types().alignof_ptr); + world_v->setOrdering(AtomicOrdering::Acquire); + ctx.builder.CreateStore(world_v, world_age_field); + Value *age_not_ok = ctx.builder.CreateICmpNE(last_world_v, world_v); + Value *target = emit_guarded_test(ctx, age_not_ok, callee, [&] { + Function *getcaller = prepare_call(jlgetabiconverter_func); + CallInst *cw = ctx.builder.CreateCall(getcaller, { + get_current_task(ctx), + theFptr, + last_world_p, + cfuncdata}); + cw->setAttributes(getcaller->getAttributes()); + return cw; + }); + ctx.emission_context.cfuncs.push_back({declrt, sigt, nargs, specsig, theFptr, cfuncdata}); + if (specsig) { + // TODO: could we force this to guarantee passing a box for `f` here (since we + // know we had it here) and on the receiver end (emit_abi_converter / + // emit_abi_dispatcher), force it to know that it can simply use this pointer + // instead of re-boxing it if it needs to the boxed copy of it. This comes up + // very rarely since usually the ABI calls are concrete and match exactly and + // aren't closures, but sometimes there are cases like that because of + // `::Function` de-specialization heuristics, such as for the `Returns` callable + // given that it is `@nospecialize`. + jl_returninfo_t targetspec = get_specsig_function(ctx.emission_context, M, target, "", sigt, declrt, is_opaque_closure); + retval = emit_call_specfun_other(ctx, is_opaque_closure, sigt, declrt, targetspec, inputargs, nargs); + } + else { + retval = mark_julia_type(ctx, emit_jlcall(ctx, target, nullptr, inputargs, nargs, julia_call), true, declrt); + } + } + else { + // emit a dispatch + Value *ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, inputargs, nargs, julia_call); + retval = mark_julia_type(ctx, ret, true, jl_any_type); + // inline a call to typeassert here + emit_typecheck(ctx, retval, declrt, "cfunction"); + retval = update_julia_type(ctx, retval, declrt); + } + return retval; +} + static Function *gen_cfun_wrapper( Module *into, jl_codegen_params_t ¶ms, const function_sig_t &sig, jl_value_t *ff, const char *aliasname, - jl_value_t *declrt, jl_method_instance_t *lam, + jl_value_t *declrt, jl_value_t *sigt, jl_unionall_t *unionall_env, jl_svec_t *sparam_vals, jl_array_t **closure_types) { ++GeneratedCFuncWrappers; // Generate a c-callable wrapper assert(into); size_t nargs = sig.nccallargs; - const char *name = "cfunction"; - size_t world = jl_atomic_load_acquire(&jl_world_counter); + const char *name = aliasname ? aliasname : "cfunction"; bool nest = (!ff || unionall_env); - jl_value_t *astrt = (jl_value_t*)jl_any_type; - if (aliasname) - name = aliasname; - else if (lam) - name = jl_symbol_name(lam->def.method->name); - - jl_code_instance_t *codeinst = NULL; - if (lam) { - // TODO: this isn't ideal to be unconditionally calling type inference from here - codeinst = jl_type_infer(lam, world, SOURCE_MODE_NOT_REQUIRED); - if (codeinst) - astrt = codeinst->rettype; - if (astrt != (jl_value_t*)jl_bottom_type && - jl_type_intersection(astrt, declrt) == jl_bottom_type) { - // Do not warn if the function never returns since it is - // occasionally required by the C API (typically error callbacks) - // even though we're likely to encounter memory errors in that case - jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name); - } - } std::string funcName; raw_string_ostream(funcName) << "jlcapi_" << name << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); @@ -7538,19 +7769,6 @@ static Function *gen_cfun_wrapper( jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); ctx.world_age_at_entry = ai.decorateInst( ctx.builder.CreateAlignedLoad(ctx.types().T_size, world_age_field, ctx.types().alignof_ptr)); - Value *world_v = ctx.builder.CreateAlignedLoad(ctx.types().T_size, - prepare_global_in(jl_Module, jlgetworld_global), ctx.types().alignof_ptr); - cast(world_v)->setOrdering(AtomicOrdering::Acquire); - - Value *age_ok = nullptr; - if (codeinst) { - LoadInst *lam_max = ctx.builder.CreateAlignedLoad( - ctx.types().T_size, - emit_ptrgep(ctx, literal_pointer_val(ctx, (jl_value_t*)codeinst), offsetof(jl_code_instance_t, max_world)), - ctx.types().alignof_ptr); - age_ok = ctx.builder.CreateICmpUGE(lam_max, world_v); - } - ctx.builder.CreateStore(world_v, world_age_field); // first emit code to record the arguments Function::arg_iterator AI = cw->arg_begin(); @@ -7705,30 +7923,8 @@ static Function *gen_cfun_wrapper( assert(AI == cw->arg_end()); // Create the call - bool jlfunc_sret; - jl_cgval_t retval; - if (codeinst) { - retval = emit_invoke(ctx, mark_julia_const(ctx, (jl_value_t*)codeinst), inputargs, nargs + 1, astrt, age_ok); - jlfunc_sret = retval.V && isa(retval.V) && !retval.TIndex && retval.inline_roots.empty(); - if (jlfunc_sret && sig.sret) { - // fuse the two sret together - assert(retval.ispointer()); - AllocaInst *result = cast(retval.V); - retval.V = sretPtr; - result->replaceAllUsesWith(sretPtr); - result->eraseFromParent(); - } - } - else { - // emit a dispatch - jlfunc_sret = false; - Value *ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, inputargs, nargs + 1, julia_call); - retval = mark_julia_type(ctx, ret, true, astrt); - } - - // inline a call to typeassert here, if required - emit_typecheck(ctx, retval, declrt, "cfunction"); - retval = update_julia_type(ctx, retval, declrt); + jl_cgval_t retval = emit_abi_call(ctx, declrt, sigt, inputargs, nargs + 1, world_age_field); + bool jlfunc_sret = retval.V && isa(retval.V) && !retval.TIndex && retval.inline_roots.empty(); // Prepare the return value Value *r; @@ -7738,7 +7934,12 @@ static Function *gen_cfun_wrapper( r = boxed(ctx, retval); } else if (sig.sret && jlfunc_sret) { - // nothing to do + // fuse the two sret together + assert(retval.ispointer()); + AllocaInst *result = cast(retval.V); + retval.V = sretPtr; + result->replaceAllUsesWith(sretPtr); + result->eraseFromParent(); r = NULL; } else if (!type_is_ghost(sig.lrt)) { @@ -7761,14 +7962,6 @@ static Function *gen_cfun_wrapper( ctx.builder.SetCurrentDebugLocation(noDbg); ctx.builder.ClearInsertionPoint(); - if (aliasname) { - auto alias = GlobalAlias::create(cw->getValueType(), cw->getType()->getAddressSpace(), - GlobalValue::ExternalLinkage, aliasname, cw, M); - if(ctx.emission_context.TargetTriple.isOSBinFormatCOFF()) { - alias->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::DLLExportStorageClass); - } - } - if (nest) { funcName += "make"; Function *cw_make = Function::Create( @@ -7797,6 +7990,26 @@ static Function *gen_cfun_wrapper( return cw; } +static const char *derive_sigt_name(jl_value_t *jargty) +{ + jl_datatype_t *dt = (jl_datatype_t*)jl_argument_datatype(jargty); + if ((jl_value_t*)dt == jl_nothing) + return NULL; + jl_sym_t *name = dt->name->name; + // if we have a kwcall, use that as the name anyways + jl_methtable_t *mt = dt->name->mt; + if (mt == jl_type_type_mt || mt == jl_nonfunction_mt || mt == NULL) { + // our value for `name` from MethodTable is not good, try to come up with something better + if (jl_is_type_type((jl_value_t*)dt)) { + dt = (jl_datatype_t*)jl_argument_datatype(jl_tparam0(dt)); + if ((jl_value_t*)dt != jl_nothing) { + name = dt->name->name; + } + } + } + return jl_symbol_name(name); +} + // Get the LLVM Function* for the C-callable entry point for a certain function // and argument types. // here argt does not include the leading function type argument @@ -7896,13 +8109,11 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con return jl_cgval_t(); } } - size_t world = jl_atomic_load_acquire(&jl_world_counter); - // try to look up this function for direct invoking - jl_method_instance_t *lam = sigt ? jl_get_specialization1((jl_tupletype_t*)sigt, world, 0) : NULL; + const char *name = derive_sigt_name(fexpr_rt.typ); Value *F = gen_cfun_wrapper( jl_Module, ctx.emission_context, - sig, fexpr_rt.constant, NULL, - declrt, lam, + sig, fexpr_rt.constant, name, + declrt, sigt, unionall_env, sparam_vals, &closure_types); bool outboxed; if (nest) { @@ -7967,6 +8178,7 @@ const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value { ++GeneratedCCallables; jl_datatype_t *ft = (jl_datatype_t*)jl_tparam0(sigt); + assert(jl_is_datatype(ft)); jl_value_t *ff = ft->instance; assert(ff); const char *name = jl_symbol_name(ft->name->mt->name); @@ -7991,7 +8203,6 @@ const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value function_sig_t sig("cfunction", lcrt, crt, toboxed, argtypes, NULL, false, CallingConv::C, false, ¶ms); if (sig.err_msg.empty()) { - size_t world = jl_atomic_load_acquire(&jl_world_counter); if (sysimg_handle) { // restore a ccallable from the system image void *addr; @@ -8004,9 +8215,13 @@ const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value } } else { - jl_method_instance_t *lam = jl_get_specialization1((jl_tupletype_t*)sigt, world, 0); //Safe b/c params holds context lock - gen_cfun_wrapper(llvmmod, params, sig, ff, name, declrt, lam, NULL, NULL, NULL); + Function *cw = gen_cfun_wrapper(llvmmod, params, sig, ff, name, declrt, sigt, NULL, NULL, NULL); + auto alias = GlobalAlias::create(cw->getValueType(), cw->getType()->getAddressSpace(), + GlobalValue::ExternalLinkage, name, cw, llvmmod); + if (params.TargetTriple.isOSBinFormatCOFF()) { + alias->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::DLLExportStorageClass); + } } JL_GC_POP(); return name; @@ -8018,7 +8233,7 @@ const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value // generate a julia-callable function that calls f (AKA lam) // if is_opaque_closure, then generate the OC invoke, rather than a real invoke -static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *abi, jl_value_t *jlretty, jl_returninfo_t &f, unsigned nargs, int retarg, bool is_opaque_closure, StringRef funcName, +static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *abi, jl_value_t *jlretty, jl_value_t *declrt, jl_returninfo_t &f, unsigned nargs, int retarg, bool is_opaque_closure, StringRef funcName, Module *M, jl_codegen_params_t ¶ms) { ++GeneratedInvokeWrappers; @@ -8069,6 +8284,10 @@ static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *abi, jl_va argv[i] = mark_julia_type(ctx, theArg, true, ty); } jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, abi, jlretty, f, argv, nargs); + if (declrt != jlretty) { + emit_typecheck(ctx, retval, declrt, "cfunction"); + retval = update_julia_type(ctx, retval, declrt); + } if (retarg != -1) { Value *theArg; if (retarg == 0) @@ -8085,20 +8304,22 @@ static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *abi, jl_va ctx.builder.CreateRet(boxed(ctx, retval)); } -static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value *fval, StringRef name, jl_value_t *sig, jl_value_t *jlrettype, bool is_opaque_closure, bool gcstack_arg, +static jl_returninfo_t get_specsig_function(jl_codegen_params_t ¶ms, Module *M, Value *fval, StringRef name, jl_value_t *sig, jl_value_t *jlrettype, bool is_opaque_closure, ArrayRef ArgNames, unsigned nreq) { + bool gcstack_arg = params.params->gcstack_arg; jl_returninfo_t props = {}; SmallVector fsig; SmallVector argnames; Type *rt = NULL; Type *srt = NULL; + Type *T_prjlvalue = PointerType::get(M->getContext(), AddressSpace::Tracked); if (jlrettype == (jl_value_t*)jl_bottom_type) { - rt = getVoidTy(ctx.builder.getContext()); + rt = getVoidTy(M->getContext()); props.cc = jl_returninfo_t::Register; } else if (jl_is_structtype(jlrettype) && jl_is_datatype_singleton((jl_datatype_t*)jlrettype)) { - rt = getVoidTy(ctx.builder.getContext()); + rt = getVoidTy(M->getContext()); props.cc = jl_returninfo_t::Register; } else if (jl_is_uniontype(jlrettype)) { @@ -8106,25 +8327,25 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value union_alloca_type((jl_uniontype_t*)jlrettype, allunbox, props.union_bytes, props.union_align, props.union_minalign); if (props.union_bytes) { props.cc = jl_returninfo_t::Union; - Type *AT = ArrayType::get(getInt8Ty(ctx.builder.getContext()), props.union_bytes); + Type *AT = ArrayType::get(getInt8Ty(M->getContext()), props.union_bytes); fsig.push_back(AT->getPointerTo()); argnames.push_back("union_bytes_return"); - Type *pair[] = { ctx.types().T_prjlvalue, getInt8Ty(ctx.builder.getContext()) }; - rt = StructType::get(ctx.builder.getContext(), ArrayRef(pair)); + Type *pair[] = { T_prjlvalue, getInt8Ty(M->getContext()) }; + rt = StructType::get(M->getContext(), ArrayRef(pair)); } else if (allunbox) { props.cc = jl_returninfo_t::Ghosts; - rt = getInt8Ty(ctx.builder.getContext()); + rt = getInt8Ty(M->getContext()); } else { - rt = ctx.types().T_prjlvalue; + rt = T_prjlvalue; } } else if (!deserves_retbox(jlrettype)) { bool retboxed; - rt = julia_type_to_llvm(ctx, jlrettype, &retboxed); + rt = _julia_type_to_llvm(¶ms, M->getContext(), jlrettype, &retboxed); assert(!retboxed); - if (rt != getVoidTy(ctx.builder.getContext()) && deserves_sret(jlrettype, rt)) { + if (rt != getVoidTy(M->getContext()) && deserves_sret(jlrettype, rt)) { auto tracked = CountTrackedPointers(rt, true); assert(!tracked.derived); if (tracked.count && !tracked.all) { @@ -8139,53 +8360,53 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value fsig.push_back(rt->getPointerTo(M->getDataLayout().getAllocaAddrSpace())); argnames.push_back("sret_return"); srt = rt; - rt = getVoidTy(ctx.builder.getContext()); + rt = getVoidTy(M->getContext()); } else { props.cc = jl_returninfo_t::Register; } } else { - rt = ctx.types().T_prjlvalue; + rt = T_prjlvalue; } SmallVector attrs; // function declaration attributes if (props.cc == jl_returninfo_t::SRet) { assert(srt); - AttrBuilder param(ctx.builder.getContext()); + AttrBuilder param(M->getContext()); param.addStructRetAttr(srt); param.addAttribute(Attribute::NoAlias); param.addAttribute(Attribute::NoCapture); param.addAttribute(Attribute::NoUndef); - attrs.push_back(AttributeSet::get(ctx.builder.getContext(), param)); + attrs.push_back(AttributeSet::get(M->getContext(), param)); assert(fsig.size() == 1); } if (props.cc == jl_returninfo_t::Union) { - AttrBuilder param(ctx.builder.getContext()); + AttrBuilder param(M->getContext()); param.addAttribute(Attribute::NoAlias); param.addAttribute(Attribute::NoCapture); param.addAttribute(Attribute::NoUndef); - attrs.push_back(AttributeSet::get(ctx.builder.getContext(), param)); + attrs.push_back(AttributeSet::get(M->getContext(), param)); assert(fsig.size() == 1); } if (props.return_roots) { - AttrBuilder param(ctx.builder.getContext()); + AttrBuilder param(M->getContext()); param.addAttribute(Attribute::NoAlias); param.addAttribute(Attribute::NoCapture); param.addAttribute(Attribute::NoUndef); - attrs.push_back(AttributeSet::get(ctx.builder.getContext(), param)); - fsig.push_back(ctx.types().T_ptr); + attrs.push_back(AttributeSet::get(M->getContext(), param)); + fsig.push_back(getPointerTy(M->getContext())); argnames.push_back("return_roots"); } - if (gcstack_arg){ - AttrBuilder param(ctx.builder.getContext()); - if (ctx.emission_context.use_swiftcc) + if (gcstack_arg) { + AttrBuilder param(M->getContext()); + if (params.use_swiftcc) param.addAttribute(Attribute::SwiftSelf); param.addAttribute(Attribute::NonNull); - attrs.push_back(AttributeSet::get(ctx.builder.getContext(), param)); - fsig.push_back(PointerType::get(JuliaType::get_ppjlvalue_ty(ctx.builder.getContext()), 0)); + attrs.push_back(AttributeSet::get(M->getContext(), param)); + fsig.push_back(PointerType::get(M->getContext(), 0)); argnames.push_back("pgcstack_arg"); } @@ -8198,16 +8419,16 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value if (is_uniquerep_Type(jt)) continue; isboxed = deserves_argbox(jt); - et = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jt); + et = isboxed ? T_prjlvalue : _julia_type_to_llvm(¶ms, M->getContext(), jt, nullptr); if (type_is_ghost(et)) continue; } - AttrBuilder param(ctx.builder.getContext()); + AttrBuilder param(M->getContext()); Type *ty = et; if (et == nullptr || et->isAggregateType()) { // aggregate types are passed by pointer param.addAttribute(Attribute::NoCapture); param.addAttribute(Attribute::ReadOnly); - ty = ctx.builder.getPtrTy(AddressSpace::Derived); + ty = PointerType::get(M->getContext(), AddressSpace::Derived); } else if (isboxed && jl_is_immutable_datatype(jt)) { param.addAttribute(Attribute::ReadOnly); @@ -8217,7 +8438,7 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value Attribute::AttrKind attr = issigned ? Attribute::SExt : Attribute::ZExt; param.addAttribute(attr); } - attrs.push_back(AttributeSet::get(ctx.builder.getContext(), param)); + attrs.push_back(AttributeSet::get(M->getContext(), param)); fsig.push_back(ty); size_t argno = i < nreq ? i : nreq; std::string genname; @@ -8233,8 +8454,8 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value if (et && et->isAggregateType()) { auto tracked = CountTrackedPointers(et); if (tracked.count && !tracked.all) { - attrs.push_back(AttributeSet::get(ctx.builder.getContext(), param)); - fsig.push_back(ctx.builder.getPtrTy(M->getDataLayout().getAllocaAddrSpace())); + attrs.push_back(AttributeSet::get(M->getContext(), param)); + fsig.push_back(PointerType::get(M->getContext(), M->getDataLayout().getAllocaAddrSpace())); if (!genname.empty()) argnames.push_back((Twine(".roots.") + genname).str()); } @@ -8244,25 +8465,26 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value AttributeSet FnAttrs; AttributeSet RetAttrs; if (jlrettype == (jl_value_t*)jl_bottom_type) - FnAttrs = FnAttrs.addAttribute(ctx.builder.getContext(), Attribute::NoReturn); - else if (rt == ctx.types().T_prjlvalue) - RetAttrs = RetAttrs.addAttribute(ctx.builder.getContext(), Attribute::NonNull); - AttributeList attributes = AttributeList::get(ctx.builder.getContext(), FnAttrs, RetAttrs, attrs); + FnAttrs = FnAttrs.addAttribute(M->getContext(), Attribute::NoReturn); + else if (rt == T_prjlvalue) + RetAttrs = RetAttrs.addAttribute(M->getContext(), Attribute::NonNull); + AttributeList attributes = AttributeList::get(M->getContext(), FnAttrs, RetAttrs, attrs); FunctionType *ftype = FunctionType::get(rt, fsig, false); if (fval == NULL) { Function *f = M ? cast_or_null(M->getNamedValue(name)) : NULL; if (f == NULL) { f = Function::Create(ftype, GlobalVariable::ExternalLinkage, name, M); - jl_init_function(f, ctx.emission_context.TargetTriple); - if (ctx.emission_context.params->debug_info_level >= 2) { + jl_init_function(f, params.TargetTriple); + if (params.params->debug_info_level >= 2) { ios_t sigbuf; ios_mem(&sigbuf, 0); jl_static_show_func_sig((JL_STREAM*) &sigbuf, sig); - f->setAttributes(AttributeList::get(f->getContext(), {attributes.addFnAttribute(ctx.builder.getContext(),"julia.fsig", StringRef(sigbuf.buf, sigbuf.size)), f->getAttributes()})); + f->setAttributes(AttributeList::get(f->getContext(), {attributes.addFnAttribute(M->getContext(),"julia.fsig", StringRef(sigbuf.buf, sigbuf.size)), f->getAttributes()})); ios_close(&sigbuf); - } else + } else { f->setAttributes(AttributeList::get(f->getContext(), {attributes, f->getAttributes()})); + } } else { assert(f->getFunctionType() == ftype); @@ -8270,11 +8492,10 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value fval = f; } else { - if (fval->getType()->isIntegerTy()) - fval = emit_inttoptr(ctx, fval, ftype->getPointerTo()); + assert(fval->getType()->isPointerTy()); } if (auto F = dyn_cast(fval)) { - if (gcstack_arg && ctx.emission_context.use_swiftcc) + if (gcstack_arg && params.use_swiftcc) F->setCallingConv(CallingConv::Swift); assert(F->arg_size() >= argnames.size()); for (size_t i = 0; i < argnames.size(); i++) { @@ -8321,25 +8542,6 @@ static jl_datatype_t *compute_va_type(jl_value_t *sig, size_t nreq) return (jl_datatype_t*)typ; } -static std::string get_function_name(bool specsig, bool needsparams, const char *unadorned_name, const Triple &TargetTriple) -{ - std::string _funcName; - raw_string_ostream funcName(_funcName); - // try to avoid conflicts in the global symbol table - if (specsig) - funcName << "julia_"; // api 5 - else if (needsparams) - funcName << "japi3_"; - else - funcName << "japi1_"; - if (TargetTriple.isOSLinux()) { - if (unadorned_name[0] == '@') - unadorned_name++; - } - funcName << unadorned_name << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); - return funcName.str(); -} - // Compile to LLVM IR, using a specialized signature if applicable. static jl_llvm_functions_t emit_function( @@ -8555,8 +8757,8 @@ static jl_llvm_functions_t ArgNames[i] = name; } } - returninfo = get_specsig_function(ctx, M, NULL, declarations.specFunctionObject, abi, - jlrettype, ctx.is_opaque_closure, JL_FEAT_TEST(ctx,gcstack_arg), + returninfo = get_specsig_function(params, M, NULL, declarations.specFunctionObject, abi, + jlrettype, ctx.is_opaque_closure, ArgNames, nreq); f = cast(returninfo.decl.getCallee()); has_sret = (returninfo.cc == jl_returninfo_t::SRet || returninfo.cc == jl_returninfo_t::Union); @@ -8590,7 +8792,7 @@ static jl_llvm_functions_t raw_string_ostream(wrapName) << "jfptr_" << ctx.name << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); declarations.functionObject = wrapName; size_t nparams = jl_nparams(abi); - gen_invoke_wrapper(lam, abi, jlrettype, returninfo, nparams, retarg, ctx.is_opaque_closure, declarations.functionObject, M, ctx.emission_context); + gen_invoke_wrapper(lam, abi, jlrettype, jlrettype, returninfo, nparams, retarg, ctx.is_opaque_closure, declarations.functionObject, M, ctx.emission_context); // TODO: add attributes: maybe_mark_argument_dereferenceable(Arg, argType) // TODO: add attributes: dereferenceable // TODO: (if needsparams) add attributes: dereferenceable, readonly, nocapture @@ -9958,6 +10160,39 @@ static jl_llvm_functions_t // --- entry point --- +jl_llvm_functions_t jl_emit_codedecls( + orc::ThreadSafeModule &M, + jl_code_instance_t *codeinst, + jl_codegen_params_t ¶ms) +{ + jl_llvm_functions_t decls = {}; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + bool specsig, needsparams; + std::tie(specsig, needsparams) = uses_specsig(get_ci_abi(codeinst), mi, codeinst->rettype, params.params->prefer_specsig); + const char *name = name_from_method_instance(mi); + if (specsig) + raw_string_ostream(decls.functionObject) << "jfptr_" << name << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); + else if (needsparams) + decls.functionObject = "jl_fptr_sparam"; + else + decls.functionObject = "jl_fptr_args"; + raw_string_ostream(decls.specFunctionObject) << (specsig ? "j_" : "j1_") << name << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); + M.withModuleDo([&](Module &M) { + bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; + if (specsig) { + get_specsig_function(params, &M, nullptr, decls.specFunctionObject, get_ci_abi(codeinst), codeinst->rettype, is_opaque_closure); + } + else { + Function *f = Function::Create(needsparams ? JuliaType::get_jlfuncparams_ty(M.getContext()) : JuliaType::get_jlfunc_ty(M.getContext()), + GlobalVariable::ExternalLinkage, + decls.specFunctionObject, M); + jl_init_function(f, params.TargetTriple); + f->setAttributes(AttributeList::get(M.getContext(), {get_func_attrs(M.getContext()), f->getAttributes()})); + } + }); + return decls; +} + jl_llvm_functions_t jl_emit_code( orc::ThreadSafeModule &m, jl_method_instance_t *li, @@ -10012,7 +10247,7 @@ static jl_llvm_functions_t jl_emit_oc_wrapper(orc::ThreadSafeModule &m, jl_codeg jl_codectx_t ctx(M->getContext(), params, 0, 0); ctx.name = M->getModuleIdentifier().data(); std::string funcName = get_function_name(true, false, ctx.name, ctx.emission_context.TargetTriple); - jl_returninfo_t returninfo = get_specsig_function(ctx, M, NULL, funcName, mi->specTypes, rettype, true, JL_FEAT_TEST(ctx,gcstack_arg)); + jl_returninfo_t returninfo = get_specsig_function(params, M, NULL, funcName, mi->specTypes, rettype, true); Function *gf_thunk = cast(returninfo.decl.getCallee()); jl_init_function(gf_thunk, ctx.emission_context.TargetTriple); size_t nrealargs = jl_nparams(mi->specTypes); @@ -10032,8 +10267,8 @@ jl_llvm_functions_t jl_emit_codeinst( { JL_TIMING(CODEGEN, CODEGEN_Codeinst); jl_timing_show_method_instance(jl_get_ci_mi(codeinst), JL_TIMING_DEFAULT_BLOCK); + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); if (!src) { - jl_method_instance_t *mi = jl_get_ci_mi(codeinst); // Assert that this this is the generic method for opaque closure wrappers: // this signals to instead compile specptr such that it holds the specptr -> invoke wrapper // to satisfy the dispatching implementation requirements of jl_f_opaque_closure_call @@ -10044,7 +10279,7 @@ jl_llvm_functions_t jl_emit_codeinst( return jl_llvm_functions_t(); // user error } //assert(jl_egal((jl_value_t*)jl_atomic_load_relaxed(&codeinst->debuginfo), (jl_value_t*)src->debuginfo) && "trying to generate code for a codeinst for an incompatible src"); - jl_llvm_functions_t decls = jl_emit_code(m, jl_get_ci_mi(codeinst), src, get_ci_abi(codeinst), codeinst->rettype, params); + jl_llvm_functions_t decls = jl_emit_code(m, mi, src, get_ci_abi(codeinst), codeinst->rettype, params); return decls; } @@ -10152,6 +10387,7 @@ static void init_jit_functions(void) add_named_global(jlunlockvalue_func, &jl_unlock_value); add_named_global(jllockfield_func, &jl_lock_field); add_named_global(jlunlockfield_func, &jl_unlock_field); + add_named_global(jlgetabiconverter_func, &jl_get_abi_converter); #ifdef _OS_WINDOWS_ #if defined(_CPU_X86_64_) diff --git a/src/gf.c b/src/gf.c index 710dda208f0b2..82e1e43333eb4 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3165,6 +3165,7 @@ jl_method_instance_t *jl_normalize_to_compilable_mi(jl_method_instance_t *mi JL_ JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache) { jl_method_t *m = match->method; + JL_GC_PROMISE_ROOTED(m); jl_svec_t *env = match->sparams; jl_tupletype_t *ti = match->spec_types; jl_method_instance_t *mi = NULL; @@ -3200,7 +3201,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *matc } // compile-time method lookup -jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, int mt_cache) +jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, int mt_cache) { if (jl_has_free_typevars((jl_value_t*)types)) return NULL; // don't poison the cache due to a malformed query diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index e4dc2459e8db6..4006397d08ea1 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -454,6 +454,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va Constant *c = x.constant ? julia_const_to_llvm(ctx, x.constant) : nullptr; if ((x.inline_roots.empty() && !x.ispointer()) || c != nullptr) { // already unboxed, but sometimes need conversion Value *unboxed = c ? c : x.V; + assert(unboxed); // clang-sa doesn't know that !x.ispointer() implies x.V does have a value return emit_unboxed_coercion(ctx, to, unboxed); } @@ -461,6 +462,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va Value *p = x.constant ? literal_pointer_val(ctx, x.constant) : x.V; if (jt == (jl_value_t*)jl_bool_type || to->isIntegerTy(1)) { + assert(p && x.inline_roots.empty()); // clang-sa doesn't know that x.ispointer() implied these are true jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); Instruction *unbox_load = ai.decorateInst(ctx.builder.CreateLoad(getInt8Ty(ctx.builder.getContext()), p)); setName(ctx.emission_context, unbox_load, p->getName() + ".unbox"); @@ -486,6 +488,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va p = combined; ai = combined_ai; } + assert(p); // clang-sa doesn't know that x.ispointer() implied this is true Instruction *load = ctx.builder.CreateAlignedLoad(to, p, Align(alignment)); setName(ctx.emission_context, load, p->getName() + ".unbox"); return ai.decorateInst(load); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 0acb7beaca9ab..80642bef95619 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -213,13 +213,84 @@ static void jl_optimize_roots(jl_codegen_params_t ¶ms, jl_method_instance_t JL_UNLOCK(&m->writelock); } -void jl_jit_globals(std::map &globals) JL_NOTSAFEPOINT +static void finish_params(Module *M, jl_codegen_params_t ¶ms, SmallVector &sharedmodules) JL_NOTSAFEPOINT { - for (auto &global : globals) { - jl_link_global(global.second, global.first); + if (params._shared_module) { + sharedmodules.push_back(orc::ThreadSafeModule(std::move(params._shared_module), params.tsctx)); + } + + // In imaging mode, we can't inline global variable initializers in order to preserve + // the fiction that we don't know what loads from the global will return. Thus, we + // need to emit a separate module for the globals before any functions are compiled, + // to ensure that the globals are defined when they are compiled. + if (jl_options.image_codegen) { + if (!params.global_targets.empty()) { + void **globalslots = new void*[params.global_targets.size()]; + void **slot = globalslots; + for (auto &global : params.global_targets) { + auto GV = global.second; + *slot = global.first; + jl_ExecutionEngine->addGlobalMapping(GV->getName(), (uintptr_t)slot); + slot++; + } +#ifdef __clang_analyzer__ + static void **leaker = globalslots; // for the purpose of the analyzer, we need to expressly leak this variable or it thinks we forgot to free it +#endif + } + } + else { + StringMap NewGlobals; + for (auto &global : params.global_targets) { + NewGlobals[global.second->getName()] = global.first; + } + for (auto &GV : M->globals()) { + auto InitValue = NewGlobals.find(GV.getName()); + if (InitValue != NewGlobals.end()) { + jl_link_global(&GV, InitValue->second); + } + } } } +extern "C" JL_DLLEXPORT_CODEGEN +void *jl_jit_abi_converter_impl(jl_task_t *ct, void *unspecialized, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, int specsig, + jl_code_instance_t *codeinst, jl_callptr_t invoke, void *target, int target_specsig) +{ + if (codeinst == nullptr && unspecialized != nullptr) + return unspecialized; + orc::ThreadSafeModule result_m; + std::string gf_thunk_name; + { + jl_codegen_params_t params(std::make_unique(), jl_ExecutionEngine->getDataLayout(), jl_ExecutionEngine->getTargetTriple()); // Locks the context + params.getContext().setDiscardValueNames(true); + params.cache = true; + params.imaging_mode = 0; + result_m = jl_create_ts_module("gfthunk", params.tsctx, params.DL, params.TargetTriple); + Module *M = result_m.getModuleUnlocked(); + if (target) { + Value *llvmtarget = literal_static_pointer_val((void*)target, PointerType::get(M->getContext(), 0)); + gf_thunk_name = emit_abi_converter(M, params, declrt, sigt, nargs, specsig, codeinst, llvmtarget, target_specsig); + } + else if (invoke == jl_fptr_const_return_addr) { + gf_thunk_name = emit_abi_constreturn(M, params, declrt, sigt, nargs, specsig, codeinst->rettype_const); + } + else { + Value *llvminvoke = invoke ? literal_static_pointer_val((void*)invoke, PointerType::get(M->getContext(), 0)) : nullptr; + gf_thunk_name = emit_abi_dispatcher(M, params, declrt, sigt, nargs, specsig, codeinst, llvminvoke); + } + SmallVector sharedmodules; + finish_params(M, params, sharedmodules); + assert(sharedmodules.empty()); + } + int8_t gc_state = jl_gc_safe_enter(ct->ptls); + jl_ExecutionEngine->addModule(std::move(result_m)); + uintptr_t Addr = jl_ExecutionEngine->getFunctionAddress(gf_thunk_name); + jl_gc_safe_leave(ct->ptls, gc_state); + assert(Addr); + return (void*)Addr; +} + + // lock for places where only single threaded behavior is implemented, so we need GC support static jl_mutex_t jitlock; // locks for adding external code to the JIT atomically @@ -262,45 +333,6 @@ static DenseMap> incompl // as materialization may need to acquire TSC locks. -static void finish_params(Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT -{ - if (params._shared_module) { - sharedmodules.push_back(orc::ThreadSafeModule(std::move(params._shared_module), params.tsctx)); - } - - // In imaging mode, we can't inline global variable initializers in order to preserve - // the fiction that we don't know what loads from the global will return. Thus, we - // need to emit a separate module for the globals before any functions are compiled, - // to ensure that the globals are defined when they are compiled. - if (jl_options.image_codegen) { - if (!params.global_targets.empty()) { - void **globalslots = new void*[params.global_targets.size()]; - void **slot = globalslots; - for (auto &global : params.global_targets) { - auto GV = global.second; - *slot = global.first; - jl_ExecutionEngine->addGlobalMapping(GV->getName(), (uintptr_t)slot); - slot++; - } -#ifdef __clang_analyzer__ - static void **leaker = globalslots; // for the purpose of the analyzer, we need to expressly leak this variable or it thinks we forgot to free it -#endif - } - } - else { - StringMap NewGlobals; - for (auto &global : params.global_targets) { - NewGlobals[global.second->getName()] = global.first; - } - for (auto &GV : M->globals()) { - auto InitValue = NewGlobals.find(GV.getName()); - if (InitValue != NewGlobals.end()) { - jl_link_global(&GV, InitValue->second); - } - } - } -} - static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t ¶ms, bool forceall=false) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER { jl_task_t *ct = jl_current_task; @@ -516,7 +548,7 @@ static void prepare_compile(jl_code_instance_t *codeinst) JL_NOTSAFEPOINT_LEAVE waiting = jl_analyze_workqueue(codeinst, params, true); // may safepoint assert(!waiting); (void)waiting; Module *M = emittedmodules[codeinst].getModuleUnlocked(); - finish_params(M, params); + finish_params(M, params, sharedmodules); incompletemodules.erase(it); } // and then indicate this should be compiled now @@ -548,7 +580,7 @@ static void complete_emit(jl_code_instance_t *edge) JL_NOTSAFEPOINT_LEAVE JL_NOT int waiting = jl_analyze_workqueue(callee, params); // may safepoint assert(!waiting); (void)waiting; Module *M = emittedmodules[callee].getModuleUnlocked(); - finish_params(M, params); + finish_params(M, params, sharedmodules); incompletemodules.erase(it); } } @@ -764,7 +796,7 @@ void jl_emit_codeinst_to_jit_impl( incompletemodules.try_emplace(codeinst, std::move(params), waiting); } else { - finish_params(result_m.getModuleUnlocked(), params); + finish_params(result_m.getModuleUnlocked(), params, sharedmodules); } emittedmodules[codeinst] = std::move(result_m); } @@ -832,7 +864,7 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void * } jl_analyze_workqueue(nullptr, params, true); assert(params.workqueue.empty()); - finish_params(&M, params); + finish_params(&M, params, sharedmodules); } } pparams = nullptr; diff --git a/src/jitlayers.h b/src/jitlayers.h index 4637670ec588c..139137d0ca477 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -212,6 +212,16 @@ struct jl_codegen_call_target_t { bool specsig; }; +// reification of a call to jl_jit_abi_convert, so that it isn't necessary to parse the Modules to recover this info +struct cfunc_decl_t { + jl_value_t *declrt; + jl_value_t *sigt; + size_t nargs; + bool specsig; + llvm::GlobalVariable *theFptr; + llvm::GlobalVariable *cfuncdata; +}; + typedef SmallVector, 0> jl_workqueue_t; typedef std::list> CallFrames; @@ -227,6 +237,7 @@ struct jl_codegen_params_t { typedef StringMap SymMapGV; // outputs jl_workqueue_t workqueue; + SmallVector cfuncs; std::map global_targets; jl_array_t *temporary_roots = nullptr; std::map, GlobalVariable*> external_fns; @@ -288,6 +299,11 @@ jl_llvm_functions_t jl_emit_codeinst( jl_code_info_t *src, jl_codegen_params_t ¶ms); +jl_llvm_functions_t jl_emit_codedecls( + orc::ThreadSafeModule &M, + jl_code_instance_t *codeinst, + jl_codegen_params_t ¶ms); + enum CompilationPolicy { Default = 0, Extern = 1, @@ -296,6 +312,13 @@ enum CompilationPolicy { Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t *argt, jl_codegen_params_t ¶ms); +extern "C" JL_DLLEXPORT_CODEGEN +void *jl_jit_abi_convert(jl_task_t *ct, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data); +std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *invoke); +std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *target, bool target_specsig); +std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_value_t *rettype_const); +std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, bool specsig, jl_code_instance_t *codeinst); + Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT; void emit_specsig_to_fptr1( Function *gf_thunk, jl_returninfo_t::CallingConv cc, unsigned return_roots, @@ -308,6 +331,8 @@ void jl_init_function(Function *F, const Triple &TT) JL_NOTSAFEPOINT; void add_named_global(StringRef name, void *addr) JL_NOTSAFEPOINT; +Constant *literal_pointer_val_slot(jl_codegen_params_t ¶ms, Module *M, jl_value_t *p); + static inline Constant *literal_static_pointer_val(const void *p, Type *T) JL_NOTSAFEPOINT { // this function will emit a static pointer into the generated code diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 4d1ab94644e39..21b5afa74c5f4 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -547,6 +547,7 @@ YY(jl_getUnwindInfo) \ YY(jl_get_libllvm) \ YY(jl_register_passbuilder_callbacks) \ + YY(jl_jit_abi_converter) \ YY(JLJITGetLLVMOrcExecutionSession) \ YY(JLJITGetJuliaOJIT) \ YY(JLJITGetExternalJITDylib) \ diff --git a/src/julia_internal.h b/src/julia_internal.h index 9817c8cc8263b..41a650969ae01 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1229,15 +1229,15 @@ _Atomic(jl_value_t*) *jl_table_peek_bp(jl_genericmemory_t *a, jl_value_t *key) J JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); -JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, int mt_cache); -jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp); +JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, int mt_cache); +jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp) JL_PROPAGATES_ROOT; JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_value_t *owner, jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t min_world, size_t max_world); JL_DLLEXPORT jl_value_t *jl_rettype_inferred_native(jl_method_instance_t *mi, size_t min_world, size_t max_world) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt JL_PROPAGATES_ROOT, jl_value_t *type, size_t world); JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo( jl_method_t *m JL_PROPAGATES_ROOT, jl_value_t *type, jl_svec_t *sparams); -jl_method_instance_t *jl_specializations_get_or_insert(jl_method_instance_t *mi_ins); +jl_method_instance_t *jl_specializations_get_or_insert(jl_method_instance_t *mi_ins JL_PROPAGATES_ROOT); JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_value_t *invokesig, jl_code_instance_t *caller); JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_code_instance_t *caller); JL_DLLEXPORT void jl_mi_cache_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, @@ -1557,6 +1557,9 @@ JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline( jl_value_t *fobj, jl_datatype_t *result, htable_t *cache, jl_svec_t *fill, void *(*init_trampoline)(void *tramp, void **nval), jl_unionall_t *env, jl_value_t **vals); +JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data); +JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, void *unspecialized, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, int specsig, + jl_code_instance_t *codeinst, jl_callptr_t invoke, void *target, int target_specsig); // Special filenames used to refer to internal julia libraries diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 2a6cb00961594..dd5ceb2c6ad90 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -326,6 +326,136 @@ jl_value_t *jl_get_cfunction_trampoline( } JL_GCC_IGNORE_STOP +struct cfuncdata_t { + jl_code_instance_t** plast_codeinst; + jl_code_instance_t* last_codeinst; + void *unspecialized; + jl_value_t *const *const declrt; + jl_value_t *const *const sigt; + size_t flags; +}; + +extern "C" JL_DLLEXPORT +void *jl_jit_abi_converter_fallback(jl_task_t *ct, void *unspecialized, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, int specsig, + jl_code_instance_t *codeinst, jl_callptr_t invoke, void *target, int target_specsig) +{ + if (unspecialized) + return unspecialized; + jl_errorf("cfunction not available in this build of Julia"); +} + +static const inline char *name_from_method_instance(jl_method_instance_t *li) JL_NOTSAFEPOINT +{ + return jl_is_method(li->def.method) ? jl_symbol_name(li->def.method->name) : "top-level scope"; +} + +static jl_mutex_t cfun_lock; +// release jl_world_counter +// store theFptr +// release last_world_v +// +// acquire last_world_v +// read theFptr +// acquire jl_world_counter +extern "C" JL_DLLEXPORT +void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data) +{ + cfuncdata_t *cfuncdata = (cfuncdata_t*)data; + jl_value_t *sigt = *cfuncdata->sigt; + JL_GC_PROMISE_ROOTED(sigt); + jl_value_t *declrt = *cfuncdata->declrt; + JL_GC_PROMISE_ROOTED(declrt); + bool specsig = cfuncdata->flags & 1; + size_t nargs = jl_nparams(sigt); + jl_method_instance_t *mi; + jl_code_instance_t *codeinst; + size_t world; + // check first, while behind this lock, of the validity of the current contents of this cfunc thunk + JL_LOCK(&cfun_lock); + do { + size_t last_world_v = jl_atomic_load_relaxed(last_world); + void *f = jl_atomic_load_relaxed(fptr); + jl_code_instance_t *last_ci = cfuncdata->plast_codeinst ? *cfuncdata->plast_codeinst : nullptr; + world = jl_atomic_load_acquire(&jl_world_counter); + ct->world_age = world; + if (world == last_world_v) { + JL_UNLOCK(&cfun_lock); + return f; + } + mi = jl_get_specialization1((jl_tupletype_t*)sigt, world, 0); + if (f != nullptr) { + if (last_ci == nullptr) { + if (mi == nullptr) { + jl_atomic_store_release(last_world, world); + JL_UNLOCK(&cfun_lock); + return f; + } + } + else { + if (jl_get_ci_mi(last_ci) == mi && jl_atomic_load_relaxed(&last_ci->max_world) >= world) { // same dispatch and source + jl_atomic_store_release(last_world, world); + JL_UNLOCK(&cfun_lock); + return f; + } + } + } + JL_UNLOCK(&cfun_lock); + // next, try to figure out what the target should look like (outside of the lock since this is very slow) + codeinst = mi ? jl_type_infer(mi, world, SOURCE_MODE_ABI) : nullptr; + // relock for the remainder of the function + JL_LOCK(&cfun_lock); + } while (jl_atomic_load_acquire(&jl_world_counter) != world); // restart entirely, since jl_world_counter changed thus jl_get_specialization1 might have changed + // double-check if the values were set on another thread + size_t last_world_v = jl_atomic_load_relaxed(last_world); + void *f = jl_atomic_load_relaxed(fptr); + if (world == last_world_v) { + JL_UNLOCK(&cfun_lock); + return f; // another thread fixed this up while we were away + } + auto assign_fptr = [fptr, last_world, cfuncdata, world, codeinst](void *f) { + cfuncdata->plast_codeinst = &cfuncdata->last_codeinst; + cfuncdata->last_codeinst = codeinst; + jl_atomic_store_relaxed(fptr, f); + jl_atomic_store_release(last_world, world); + JL_UNLOCK(&cfun_lock); + return f; + }; + jl_callptr_t invoke = nullptr; + if (codeinst != NULL) { + jl_value_t *astrt = codeinst->rettype; + if (astrt != (jl_value_t*)jl_bottom_type && + jl_type_intersection(astrt, declrt) == jl_bottom_type) { + // Do not warn if the function never returns since it is + // occasionally required by the C API (typically error callbacks) + // even though we're likely to encounter memory errors in that case + jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); + } + uint8_t specsigflags; + jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &f, 1); + if (invoke != nullptr) { + if (invoke == jl_fptr_const_return_addr) { + return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, nullptr, false)); + } + else if (invoke == jl_fptr_args_addr) { + assert(f); + if (!specsig && jl_subtype(astrt, declrt)) + return assign_fptr(f); + return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, f, false)); + } + else if (specsigflags & 0b1) { + assert(f); + if (specsig && jl_egal(mi->specTypes, sigt) && jl_egal(declrt, astrt)) + return assign_fptr(f); + return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, f, true)); + } + } + } + f = jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, nullptr, false); + if (codeinst == nullptr) + cfuncdata->unspecialized = f; + return assign_fptr(f); +} + void jl_init_runtime_ccall(void) { JL_MUTEX_INIT(&libmap_lock, "libmap_lock"); diff --git a/src/staticdata.c b/src/staticdata.c index cb1dc54d26d50..ec77c771bd880 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1859,6 +1859,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED else if (invokeptr_id == -4) { fptr_id = JL_API_OC_CALL; } + else if (invokeptr_id == -5) { + abort(); + } else { assert(invokeptr_id > 0); ios_ensureroom(s->fptr_record, invokeptr_id * sizeof(void*)); From ae78058fa1ff8f05fe8d1e05cecb3f65bc71a957 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 6 Feb 2025 01:14:47 -0500 Subject: [PATCH 07/31] bpart: Fully switch to partitioned semantics (#57253) This is the final PR in the binding partitions series (modulo bugs and tweaks), i.e. it closes #54654 and thus closes #40399, which was the original design sketch. This thus activates the full designed semantics for binding partitions, in particular allowing safe replacement of const bindings. It in particular allows struct redefinitions. This thus closes timholy/Revise.jl#18 and also closes #38584. The biggest semantic change here is probably that this gets rid of the notion of "resolvedness" of a binding. Previously, a lot of the behavior of our implementation depended on when bindings were "resolved", which could happen at basically an arbitrary point (in the compiler, in REPL completion, in a different thread), making a lot of the semantics around bindings ill- or at least implementation-defined. There are several related issues in the bugtracker, so this closes #14055 closes #44604 closes #46354 closes #30277 It is also the last step to close #24569. It also supports bindings for undef->defined transitions and thus closes #53958 closes #54733 - however, this is not activated yet for performance reasons and may need some further optimization. Since resolvedness no longer exists, we need to replace it with some hopefully more well-defined semantics. I will describe the semantics below, but before I do I will make two notes: 1. There are a number of cases where these semantics will behave slightly differently than the old semantics absent some other task going around resolving random bindings. 2. The new behavior (except for the replacement stuff) was generally permissible under the old semantics if the bindings happened to be resolved at the right time. With all that said, there are essentially three "strengths" of bindings: 1. Implicit Bindings: Anything implicitly obtained from `using Mod`, "no binding", plus slightly more exotic corner cases around conflicts 2. Weakly declared bindings: Declared using `global sym` and nothing else 3. Strongly declared bindings: Declared using `global sym::T`, `const sym=val`, `import Mod: sym`, `using Mod: sym` or as an implicit strong global declaration in `sym=val`, where `sym` is known to be global (either by being at toplevle or as `global sym=val` inside a function). In general, you always allowed to syntactically replace a weaker binding by a stronger one (although the runtime permits arbitrary binding deletion now, this is just a syntactic constraint to catch errors). Second, any implicit binding can be replaced by other implicit bindings as the result of changing the `using`'ed module. And lastly, any constants may be replaced by any other constants (irrespective of type). We do not currently allow replacing globals, but may consider changing that in 1.13. This is mostly how things used to work, as well in the absence of any stray external binding resolutions. The most prominent difference is probably this one: ``` set_foo!() = global foo = 1 ``` In the above terminology, this now always declares a "strongly declared binding", whereas before it declared a "weakly declared binding" that would become strongly declared on first write to the global (unless of course somebody had created a different strongly declared global in the meantime). To see the difference, this is now disallowed: ``` julia> set_foo!() = global foo = 1 set_foo! (generic function with 1 method) julia> const foo = 1 ERROR: cannot declare Main.foo constant; it was already declared global Stacktrace: [1] top-level scope @ REPL[2]:1 ``` Before it would depend on the order of binding resolution (although it just crashes on current master for some reason - whoops, probably my fault). Another major change is the ambiguousness of imports. In: ``` module M1; export x; x=1; end module M2; export x; x=2; end using .M1, .M2 ``` the binding `Main.x` is now always ambiguous and will throw on access. Before which binding you get, would depend on resolution order. To choose one, use an explicit import (which was the behavior you would previously get if neither binding was resolved before both imports). (cherry picked from commit 888cf0350668a677ef27c87ab80a8562f3bc211e) --- Compiler/src/Compiler.jl | 2 +- Compiler/src/abstractinterpretation.jl | 32 +- Compiler/src/ssair/ir.jl | 2 +- Compiler/src/ssair/verify.jl | 1 - Compiler/src/validation.jl | 2 +- Compiler/test/effects.jl | 31 +- Compiler/test/inference.jl | 2 +- base/boot.jl | 2 +- base/client.jl | 3 +- base/deprecated.jl | 25 + base/invalidation.jl | 64 +- base/reflection.jl | 86 --- base/runtime_internals.jl | 118 ++- base/show.jl | 26 +- contrib/generate_precompile.jl | 7 +- doc/src/manual/variables-and-scoping.md | 53 +- doc/src/manual/variables.md | 10 +- src/builtins.c | 76 +- src/codegen.cpp | 15 +- src/gc-stock.c | 3 + src/interpreter.c | 9 +- src/jl_exported_funcs.inc | 4 +- src/jlapi.c | 40 + src/julia-syntax.scm | 60 +- src/julia.h | 77 +- src/julia_internal.h | 13 +- src/method.c | 45 +- src/module.c | 719 ++++++++++-------- src/staticdata.c | 124 ++- src/toplevel.c | 161 ++-- .../InteractiveUtils/src/InteractiveUtils.jl | 4 +- stdlib/REPL/src/REPL.jl | 1 - stdlib/REPL/src/REPLCompletions.jl | 3 +- stdlib/REPL/src/docview.jl | 10 +- stdlib/REPL/test/replcompletions.jl | 2 - stdlib/Test/src/Test.jl | 4 +- test/ambiguous.jl | 4 - test/atomics.jl | 18 +- test/core.jl | 120 ++- test/docs.jl | 2 +- test/errorshow.jl | 7 +- test/misc.jl | 4 +- test/precompile.jl | 494 ++++++------ test/rebinding.jl | 93 ++- test/reflection.jl | 4 - test/show.jl | 9 - test/staged.jl | 2 +- test/syntax.jl | 12 +- test/worlds.jl | 25 +- 49 files changed, 1557 insertions(+), 1073 deletions(-) diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index 0fc7bd6e328e7..ba6d1607ded55 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -49,7 +49,7 @@ using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstanc using Base using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospecializeinfer, - BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST, + BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST, BINDING_KIND_DECLARED, Base, BitVector, Bottom, Callable, DataTypeFieldDesc, EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES, OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME, diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index fe7cc1dbf5680..8a7e8aee715a6 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3464,14 +3464,7 @@ world_range(ci::CodeInfo) = WorldRange(ci.min_world, ci.max_world) world_range(ci::CodeInstance) = WorldRange(ci.min_world, ci.max_world) world_range(compact::IncrementalCompact) = world_range(compact.ir) -function force_binding_resolution!(g::GlobalRef, world::UInt) - # Force resolution of the binding - # TODO: This will go away once we switch over to fully partitioned semantics - ccall(:jl_force_binding_resolution, Cvoid, (Any, Csize_t), g, world) - return nothing -end - -function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}, retry_after_resolve::Bool=true) +function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}) worlds = world_range(src) partition = lookup_binding_partition(min_world(worlds), g) partition.max_world < max_world(worlds) && return Any @@ -3480,25 +3473,18 @@ function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, partition = lookup_binding_partition(min_world(worlds), imported_binding) partition.max_world < max_world(worlds) && return Any end - if is_some_guard(binding_kind(partition)) - if retry_after_resolve - # This method is surprisingly hot. For performance, don't ask the runtime to resolve - # the binding unless necessary - doing so triggers an additional lookup, which though - # not super expensive is hot enough to show up in benchmarks. - force_binding_resolution!(g, min_world(worlds)) - return abstract_eval_globalref_type(g, src, false) - end + kind = binding_kind(partition) + if is_some_guard(kind) # return Union{} return Any end - if is_some_const_binding(binding_kind(partition)) + if is_some_const_binding(kind) return Const(partition_restriction(partition)) end - return partition_restriction(partition) + return kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition) end function lookup_binding_partition!(interp::AbstractInterpreter, g::GlobalRef, sv::AbsIntState) - force_binding_resolution!(g, get_inference_world(interp)) partition = lookup_binding_partition(get_inference_world(interp), g) update_valid_age!(sv, WorldRange(partition.min_world, partition.max_world)) partition @@ -3541,7 +3527,11 @@ function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Co return RTEffects(rt, Union{}, Effects(EFFECTS_TOTAL, inaccessiblememonly=is_mutation_free_argtype(rt) ? ALWAYS_TRUE : ALWAYS_FALSE)) end - rt = partition_restriction(partition) + if kind == BINDING_KIND_DECLARED + rt = Any + else + rt = partition_restriction(partition) + end return RTEffects(rt, UndefVarError, generic_getglobal_effects) end @@ -3580,7 +3570,7 @@ function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partitio elseif is_some_const_binding(kind) return Pair{Any,Any}(Bottom, ErrorException) end - ty = partition_restriction(partition) + ty = kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition) wnewty = widenconst(newty) if !hasintersect(wnewty, ty) return Pair{Any,Any}(Bottom, TypeError) diff --git a/Compiler/src/ssair/ir.jl b/Compiler/src/ssair/ir.jl index f86ada2309ddc..e6c8f3a6d2c78 100644 --- a/Compiler/src/ssair/ir.jl +++ b/Compiler/src/ssair/ir.jl @@ -581,7 +581,7 @@ function is_relevant_expr(e::Expr) :foreigncall, :isdefined, :copyast, :throw_undef_if_not, :cfunction, :method, :pop_exception, - :leave, + :leave, :const, :globaldecl, :new_opaque_closure) end diff --git a/Compiler/src/ssair/verify.jl b/Compiler/src/ssair/verify.jl index 2d6d59cc4e22b..974690885a2e4 100644 --- a/Compiler/src/ssair/verify.jl +++ b/Compiler/src/ssair/verify.jl @@ -61,7 +61,6 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int, raise_error() end elseif isa(op, GlobalRef) - force_binding_resolution!(op, min_world(ir.valid_worlds)) bpart = lookup_binding_partition(min_world(ir.valid_worlds), op) while is_some_imported(binding_kind(bpart)) && max_world(ir.valid_worlds) <= bpart.max_world imported_binding = partition_restriction(bpart)::Core.Binding diff --git a/Compiler/src/validation.jl b/Compiler/src/validation.jl index 6700aa8d4508f..9bde405a49956 100644 --- a/Compiler/src/validation.jl +++ b/Compiler/src/validation.jl @@ -22,7 +22,7 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}( :copyast => 1:1, :meta => 0:typemax(Int), :global => 1:1, - :globaldecl => 2:2, + :globaldecl => 1:2, :foreigncall => 5:typemax(Int), # name, RT, AT, nreq, (cconv, effects), args..., roots... :cfunction => 5:5, :isdefined => 1:2, diff --git a/Compiler/test/effects.jl b/Compiler/test/effects.jl index b8a841b6b74b7..082d954b4a9bc 100644 --- a/Compiler/test/effects.jl +++ b/Compiler/test/effects.jl @@ -378,32 +378,27 @@ let effects = Base.infer_effects(; optimize=false) do end # we should taint `nothrow` if the binding doesn't exist and isn't fixed yet, -# as the cached effects can be easily wrong otherwise -# since the inference currently doesn't track "world-age" of global variables -@eval global_assignment_undefinedyet() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET)) = 42 setglobal!_nothrow_undefinedyet() = setglobal!(@__MODULE__, :UNDEFINEDYET, 42) -let effects = Base.infer_effects() do - global_assignment_undefinedyet() - end +let effects = Base.infer_effects(setglobal!_nothrow_undefinedyet) @test !Compiler.is_nothrow(effects) end -let effects = Base.infer_effects() do - setglobal!_nothrow_undefinedyet() - end - @test !Compiler.is_nothrow(effects) +@test_throws ErrorException setglobal!_nothrow_undefinedyet() +# This declares the binding as ::Any +@eval global_assignment_undefinedyet() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET)) = 42 +let effects = Base.infer_effects(global_assignment_undefinedyet) + @test Compiler.is_nothrow(effects) end -global UNDEFINEDYET::String = "0" -let effects = Base.infer_effects() do - global_assignment_undefinedyet() - end +# Again with type mismatch +global UNDEFINEDYET2::String = "0" +setglobal!_nothrow_undefinedyet2() = setglobal!(@__MODULE__, :UNDEFINEDYET2, 42) +@eval global_assignment_undefinedyet2() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET2)) = 42 +let effects = Base.infer_effects(global_assignment_undefinedyet2) @test !Compiler.is_nothrow(effects) end -let effects = Base.infer_effects() do - setglobal!_nothrow_undefinedyet() - end +let effects = Base.infer_effects(setglobal!_nothrow_undefinedyet2) @test !Compiler.is_nothrow(effects) end -@test_throws Union{ErrorException,TypeError} setglobal!_nothrow_undefinedyet() # TODO: what kind of error should this be? +@test_throws TypeError setglobal!_nothrow_undefinedyet2() # Nothrow for setfield! mutable struct SetfieldNothrow diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index d4ea990e7d148..563828ac77296 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -6160,7 +6160,7 @@ end === Int swapglobal!(@__MODULE__, :swapglobal!_xxx, x) end === Union{} -global swapglobal!_must_throw +eval(Expr(:const, :swapglobal!_must_throw)) @newinterp SwapGlobalInterp Compiler.InferenceParams(::SwapGlobalInterp) = Compiler.InferenceParams(; assume_bindings_static=true) function func_swapglobal!_must_throw(x) diff --git a/base/boot.jl b/base/boot.jl index e50d74659d399..26a405f92f884 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -229,7 +229,7 @@ export Expr, QuoteNode, LineNumberNode, GlobalRef, # object model functions fieldtype, getfield, setfield!, swapfield!, modifyfield!, replacefield!, setfieldonce!, - nfields, throw, tuple, ===, isdefined, eval, + nfields, throw, tuple, ===, isdefined, # access to globals getglobal, setglobal!, swapglobal!, modifyglobal!, replaceglobal!, setglobalonce!, isdefinedglobal, # ifelse, sizeof # not exported, to avoid conflicting with Base diff --git a/base/client.jl b/base/client.jl index 2527d382c695d..3449e2fbf2e62 100644 --- a/base/client.jl +++ b/base/client.jl @@ -267,9 +267,8 @@ function exec_options(opts) let Distributed = require(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed")) Core.eval(MainInclude, :(const Distributed = $Distributed)) Core.eval(Main, :(using Base.MainInclude.Distributed)) + invokelatest(Distributed.process_opts, opts) end - - invokelatest(Main.Distributed.process_opts, opts) end interactiveinput = (repl || is_interactive::Bool) && isa(stdin, TTY) diff --git a/base/deprecated.jl b/base/deprecated.jl index cffff05d954d1..f72698ad47008 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -531,4 +531,29 @@ end # BEGIN 1.12 deprecations +@deprecate isbindingresolved(m::Module, var::Symbol) true false + +""" + isbindingresolved(m::Module, s::Symbol) -> Bool + +Returns whether the binding of a symbol in a module is resolved. + +See also: [`isexported`](@ref), [`ispublic`](@ref), [`isdeprecated`](@ref) + +```jldoctest +julia> module Mod + foo() = 17 + end +Mod + +julia> Base.isbindingresolved(Mod, :foo) +true +``` + +!!! warning + This function is deprecated. The concept of binding "resolvedness" was removed in Julia 1.12. + The function now always returns `true`. +""" +isbindingresolved + # END 1.12 deprecations diff --git a/base/invalidation.jl b/base/invalidation.jl index 36b867ede2868..c0aed35aa90a0 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -113,32 +113,56 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid end end -function invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_max_world::UInt) - b = convert(Core.Binding, gr) - try - valid_in_valuepos = false - foreach_module_mtable(gr.mod, new_max_world) do mt::Core.MethodTable - for method in MethodList(mt) - invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) +const BINDING_FLAG_EXPORTP = 0x2 + +function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core.BindingPartition, new_bpart::Union{Core.BindingPartition, Nothing}, new_max_world::UInt) + gr = b.globalref + if is_some_guard(binding_kind(invalidated_bpart)) + # TODO: We may want to invalidate for these anyway, since they have performance implications + return + end + foreach_module_mtable(gr.mod, new_max_world) do mt::Core.MethodTable + for method in MethodList(mt) + invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) + end + return true + end + if isdefined(b, :backedges) + for edge in b.backedges + if isa(edge, CodeInstance) + ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world) + elseif isa(edge, Core.Binding) + isdefined(edge, :partitions) || continue + latest_bpart = edge.partitions + latest_bpart.max_world == typemax(UInt) || continue + is_some_imported(binding_kind(latest_bpart)) || continue + partition_restriction(latest_bpart) === b || continue + invalidate_code_for_globalref!(edge, latest_bpart, nothing, new_max_world) + else + invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) end - return true end - b = convert(Core.Binding, gr) - if isdefined(b, :backedges) - for edge in b.backedges - if isa(edge, CodeInstance) - ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world) - else - invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) - end + end + if (b.flags & BINDING_FLAG_EXPORTP) != 0 + # This binding was exported - we need to check all modules that `using` us to see if they + # have an implicit binding to us. + usings_backedges = ccall(:jl_get_module_usings_backedges, Any, (Any,), gr.mod) + if usings_backedges !== nothing + for user in usings_backedges::Vector{Any} + user_binding = ccall(:jl_get_module_binding_or_nothing, Any, (Any, Any), user, gr.name) + user_binding === nothing && continue + isdefined(user_binding, :partitions) || continue + latest_bpart = user_binding.partitions + latest_bpart.max_world == typemax(UInt) || continue + is_some_imported(binding_kind(latest_bpart)) || continue + partition_restriction(latest_bpart) === b || continue + invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, nothing, new_max_world) end end - catch err - bt = catch_backtrace() - invokelatest(Base.println, "Internal Error during invalidation:") - invokelatest(Base.display_error, err, bt) end end +invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) = + invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world) gr_needs_backedge_in_module(gr::GlobalRef, mod::Module) = gr.mod !== mod diff --git a/base/reflection.jl b/base/reflection.jl index 78e701692a2a7..c98f6244cc89f 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -46,92 +46,6 @@ function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool= return ret end -# high-level, more convenient method lookup functions - -function visit(f, mt::Core.MethodTable) - mt.defs !== nothing && visit(f, mt.defs) - nothing -end -function visit(f, mc::Core.TypeMapLevel) - function avisit(f, e::Memory{Any}) - for i in 2:2:length(e) - isassigned(e, i) || continue - ei = e[i] - if ei isa Memory{Any} - for j in 2:2:length(ei) - isassigned(ei, j) || continue - visit(f, ei[j]) - end - else - visit(f, ei) - end - end - end - if mc.targ !== nothing - avisit(f, mc.targ::Memory{Any}) - end - if mc.arg1 !== nothing - avisit(f, mc.arg1::Memory{Any}) - end - if mc.tname !== nothing - avisit(f, mc.tname::Memory{Any}) - end - if mc.name1 !== nothing - avisit(f, mc.name1::Memory{Any}) - end - mc.list !== nothing && visit(f, mc.list) - mc.any !== nothing && visit(f, mc.any) - nothing -end -function visit(f, d::Core.TypeMapEntry) - while d !== nothing - f(d.func) - d = d.next - end - nothing -end -struct MethodSpecializations - specializations::Union{Nothing, Core.MethodInstance, Core.SimpleVector} -end -""" - specializations(m::Method) → itr - -Return an iterator `itr` of all compiler-generated specializations of `m`. -""" -specializations(m::Method) = MethodSpecializations(isdefined(m, :specializations) ? m.specializations : nothing) -function iterate(specs::MethodSpecializations) - s = specs.specializations - s === nothing && return nothing - isa(s, Core.MethodInstance) && return (s, nothing) - return iterate(specs, 0) -end -iterate(specs::MethodSpecializations, ::Nothing) = nothing -function iterate(specs::MethodSpecializations, i::Int) - s = specs.specializations::Core.SimpleVector - n = length(s) - i >= n && return nothing - item = nothing - while i < n && item === nothing - item = s[i+=1] - end - item === nothing && return nothing - return (item, i) -end -length(specs::MethodSpecializations) = count(Returns(true), specs) - -function length(mt::Core.MethodTable) - n = 0 - visit(mt) do m - n += 1 - end - return n::Int -end -isempty(mt::Core.MethodTable) = (mt.defs === nothing) - -uncompressed_ir(m::Method) = isdefined(m, :source) ? _uncompressed_ir(m) : - isdefined(m, :generator) ? error("Method is @generated; try `code_lowered` instead.") : - error("Code for this Method is not available.") - # for backwards compat const uncompressed_ast = uncompressed_ir const _uncompressed_ast = _uncompressed_ir diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index b61e24c11f3f9..36961f58c5c3f 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -177,28 +177,6 @@ ispublic(m::Module, s::Symbol) = ccall(:jl_module_public_p, Cint, (Any, Any), m, # `Base.deprecate`, not the @deprecated macro: isdeprecated(m::Module, s::Symbol) = ccall(:jl_is_binding_deprecated, Cint, (Any, Any), m, s) != 0 -""" - isbindingresolved(m::Module, s::Symbol) -> Bool - -Returns whether the binding of a symbol in a module is resolved. - -See also: [`isexported`](@ref), [`ispublic`](@ref), [`isdeprecated`](@ref) - -```jldoctest -julia> module Mod - foo() = 17 - end -Mod - -julia> Base.isbindingresolved(Mod, :foo) -true - -julia> Base.isbindingresolved(Mod, :bar) -false -``` -""" -isbindingresolved(m::Module, var::Symbol) = ccall(:jl_binding_resolved_p, Cint, (Any, Any), m, var) != 0 - function binding_module(m::Module, s::Symbol) p = ccall(:jl_get_module_of_binding, Ptr{Cvoid}, (Any, Any), m, s) p == C_NULL && return m @@ -234,7 +212,7 @@ const BINDING_KIND_BACKDATED_CONST = 0xa is_defined_const_binding(kind::UInt8) = (kind == BINDING_KIND_CONST || kind == BINDING_KIND_CONST_IMPORT || kind == BINDING_KIND_BACKDATED_CONST) is_some_const_binding(kind::UInt8) = (is_defined_const_binding(kind) || kind == BINDING_KIND_UNDEF_CONST) is_some_imported(kind::UInt8) = (kind == BINDING_KIND_IMPLICIT || kind == BINDING_KIND_EXPLICIT || kind == BINDING_KIND_IMPORTED) -is_some_guard(kind::UInt8) = (kind == BINDING_KIND_GUARD || kind == BINDING_KIND_DECLARED || kind == BINDING_KIND_FAILED || kind == BINDING_KIND_UNDEF_CONST) +is_some_guard(kind::UInt8) = (kind == BINDING_KIND_GUARD || kind == BINDING_KIND_FAILED || kind == BINDING_KIND_UNDEF_CONST) function lookup_binding_partition(world::UInt, b::Core.Binding) ccall(:jl_get_binding_partition, Ref{Core.BindingPartition}, (Any, UInt), b, world) @@ -412,8 +390,13 @@ parentmodule(t::UnionAll) = parentmodule(unwrap_unionall(t)) """ isconst(m::Module, s::Symbol) -> Bool + isconst(g::GlobalRef) -Determine whether a global is declared `const` in a given module `m`. +Determine whether a global is `const` in a given module `m`, either +because it was declared constant or because it was imported from a +constant binding. Note that constant-ness is specific to a particular +world age, so the result of this function may not be assumed to hold +after a world age update. """ isconst(m::Module, s::Symbol) = ccall(:jl_is_const, Cint, (Any, Any), m, s) != 0 @@ -1584,3 +1567,90 @@ hasintersect(@nospecialize(a), @nospecialize(b)) = typeintersect(a, b) !== Botto ########### _topmod(m::Module) = ccall(:jl_base_relative_to, Any, (Any,), m)::Module + + +# high-level, more convenient method lookup functions + +function visit(f, mt::Core.MethodTable) + mt.defs !== nothing && visit(f, mt.defs) + nothing +end +function visit(f, mc::Core.TypeMapLevel) + function avisit(f, e::Memory{Any}) + for i in 2:2:length(e) + isassigned(e, i) || continue + ei = e[i] + if ei isa Memory{Any} + for j in 2:2:length(ei) + isassigned(ei, j) || continue + visit(f, ei[j]) + end + else + visit(f, ei) + end + end + end + if mc.targ !== nothing + avisit(f, mc.targ::Memory{Any}) + end + if mc.arg1 !== nothing + avisit(f, mc.arg1::Memory{Any}) + end + if mc.tname !== nothing + avisit(f, mc.tname::Memory{Any}) + end + if mc.name1 !== nothing + avisit(f, mc.name1::Memory{Any}) + end + mc.list !== nothing && visit(f, mc.list) + mc.any !== nothing && visit(f, mc.any) + nothing +end +function visit(f, d::Core.TypeMapEntry) + while d !== nothing + f(d.func) + d = d.next + end + nothing +end +struct MethodSpecializations + specializations::Union{Nothing, Core.MethodInstance, Core.SimpleVector} +end +""" + specializations(m::Method) → itr + +Return an iterator `itr` of all compiler-generated specializations of `m`. +""" +specializations(m::Method) = MethodSpecializations(isdefined(m, :specializations) ? m.specializations : nothing) +function iterate(specs::MethodSpecializations) + s = specs.specializations + s === nothing && return nothing + isa(s, Core.MethodInstance) && return (s, nothing) + return iterate(specs, 0) +end +iterate(specs::MethodSpecializations, ::Nothing) = nothing +function iterate(specs::MethodSpecializations, i::Int) + s = specs.specializations::Core.SimpleVector + n = length(s) + i >= n && return nothing + item = nothing + while i < n && item === nothing + item = s[i+=1] + end + item === nothing && return nothing + return (item, i) +end +length(specs::MethodSpecializations) = count(Returns(true), specs) + +function length(mt::Core.MethodTable) + n = 0 + visit(mt) do m + n += 1 + end + return n::Int +end +isempty(mt::Core.MethodTable) = (mt.defs === nothing) + +uncompressed_ir(m::Method) = isdefined(m, :source) ? _uncompressed_ir(m) : + isdefined(m, :generator) ? error("Method is @generated; try `code_lowered` instead.") : + error("Code for this Method is not available.") diff --git a/base/show.jl b/base/show.jl index 42788f05eceb5..6b27fe838c89b 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1021,18 +1021,24 @@ end # If an object with this name exists in 'from', we need to check that it's the same binding # and that it's not deprecated. function isvisible(sym::Symbol, parent::Module, from::Module) - owner = ccall(:jl_binding_owner, Ptr{Cvoid}, (Any, Any), parent, sym) - from_owner = ccall(:jl_binding_owner, Ptr{Cvoid}, (Any, Any), from, sym) - return owner !== C_NULL && from_owner === owner && - !isdeprecated(parent, sym) && - isdefinedglobal(from, sym) # if we're going to return true, force binding resolution + isdeprecated(parent, sym) && return false + isdefinedglobal(from, sym) || return false + parent_binding = convert(Core.Binding, GlobalRef(parent, sym)) + from_binding = convert(Core.Binding, GlobalRef(from, sym)) + while true + from_binding === parent_binding && return true + partition = lookup_binding_partition(tls_world_age(), from_binding) + is_some_imported(binding_kind(partition)) || break + from_binding = partition_restriction(partition)::Core.Binding + end + return false end function is_global_function(tn::Core.TypeName, globname::Union{Symbol,Nothing}) if globname !== nothing globname_str = string(globname::Symbol) if ('#' ∉ globname_str && '@' ∉ globname_str && isdefined(tn, :module) && - isbindingresolved(tn.module, globname) && isdefinedglobal(tn.module, globname) && + isdefinedglobal(tn.module, globname) && isconcretetype(tn.wrapper) && isa(getglobal(tn.module, globname), tn.wrapper)) return true end @@ -3377,16 +3383,16 @@ function print_partition(io::IO, partition::Core.BindingPartition) elseif kind == BINDING_KIND_FAILED print(io, "ambiguous binding - guard entry") elseif kind == BINDING_KIND_DECLARED - print(io, "undefined, but declared using `global` - guard entry") + print(io, "weak global binding declared using `global` (implicit type Any)") elseif kind == BINDING_KIND_IMPLICIT print(io, "implicit `using` from ") - print(io, partition_restriction(partition)) + print(io, partition_restriction(partition).globalref) elseif kind == BINDING_KIND_EXPLICIT print(io, "explicit `using` from ") - print(io, partition_restriction(partition)) + print(io, partition_restriction(partition).globalref) elseif kind == BINDING_KIND_IMPORTED print(io, "explicit `import` from ") - print(io, partition_restriction(partition)) + print(io, partition_restriction(partition).globalref) else @assert kind == BINDING_KIND_GLOBAL print(io, "global variable with type ") diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index b075223d9c7e4..f12e94e22fca5 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -107,6 +107,9 @@ precompile(Base.CoreLogging.env_override_minlevel, (Symbol, Module)) precompile(Base.StackTraces.lookup, (Ptr{Nothing},)) precompile(Tuple{typeof(Base.run_module_init), Module, Int}) +# Presence tested in the tests +precompile(Tuple{typeof(Base.print), Base.IOStream, String}) + # precompilepkgs precompile(Tuple{typeof(Base.get), Type{Array{String, 1}}, Base.Dict{String, Any}, String}) precompile(Tuple{typeof(Base.get), Type{Base.Dict{String, Any}}, Base.Dict{String, Any}, String}) @@ -352,10 +355,10 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe PrecompileStagingArea = Module() for (_pkgid, _mod) in Base.loaded_modules if !(_pkgid.name in ("Main", "Core", "Base")) - eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod)) + Core.eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod)) end end - eval(PrecompileStagingArea, :(const Compiler = Base.Compiler)) + Core.eval(PrecompileStagingArea, :(const Compiler = Base.Compiler)) n_succeeded = 0 # Make statements unique diff --git a/doc/src/manual/variables-and-scoping.md b/doc/src/manual/variables-and-scoping.md index 99f7ba088311d..ab2d969dd9b0e 100644 --- a/doc/src/manual/variables-and-scoping.md +++ b/doc/src/manual/variables-and-scoping.md @@ -730,65 +730,28 @@ Note that `const` only affects the variable binding; the variable may be bound t object (such as an array), and that object may still be modified. Additionally when one tries to assign a value to a variable that is declared constant the following scenarios are possible: -* if a new value has a different type than the type of the constant then an error is thrown: +* Attempting to replace a constant without the const `keyword` is disallowed: ```jldoctest julia> const x = 1.0 1.0 julia> x = 1 -ERROR: invalid redefinition of constant x +ERROR: invalid assignment to constant x. This redefinition may be permitted using the `const` keyword. ``` -* if a new value has the same type as the constant then a warning is printed: +* All other defefinitions of constants are permitted, but may cause significant re-compilation: ```jldoctest julia> const y = 1.0 1.0 julia> const y = 2.0 -WARNING: redefinition of constant y. This may fail, cause incorrect answers, or produce other errors. 2.0 ``` -* if an assignment would not result in the change of variable value no message is given: -```jldoctest -julia> const z = 100 -100 - -julia> z = 100 -100 -``` -* if an assignment would change the mutable object to which the variable points (regardless of whether those two objects are deeply equal), a warning is printed: -```jldoctest -julia> const a = [1] -1-element Vector{Int64}: - 1 - -julia> const a = [1] -WARNING: redefinition of constant a. This may fail, cause incorrect answers, or produce other errors. -1-element Vector{Int64}: - 1 -``` -Note that although sometimes possible, changing the value of a `const` variable is strongly -discouraged, and is intended only for convenience during interactive use. Changing constants can -cause various problems or unexpected behaviors. For instance, if a method references a constant and -is already compiled before the constant is changed, then it might keep using the old value: - -```jldoctest -julia> const x = 1 -1 - -julia> f() = x -f (generic function with 1 method) - -julia> f() -1 - -julia> const x = 2 -WARNING: redefinition of constant x. This may fail, cause incorrect answers, or produce other errors. -2 - -julia> f() -1 -``` +!!! compat "Julia 1.12" + Prior to julia 1.12, redefinition of constants was poorly supported. It was restricted to + redefinition of constants of the same type and could lead to observably incorrect behavior + or crashes. Constant redefinition is highly discouraged in versions of julia prior to 1.12. + See the manual for prior julia versions for further information. ## [Typed Globals](@id man-typed-globals) diff --git a/doc/src/manual/variables.md b/doc/src/manual/variables.md index ad2c60a029032..4c3e98ca57281 100644 --- a/doc/src/manual/variables.md +++ b/doc/src/manual/variables.md @@ -79,10 +79,12 @@ julia> Base.length length (generic function with 79 methods) ``` -However, if you try to redefine a built-in constant or function already in use, Julia will give -you an error: +However, if you try to redefine a built-in constant or function that you +have explicitly imported, Julia will give you an error: ```jldoctest +julia> using Base: pi, sqrt + julia> pi π = 3.1415926535897... @@ -96,6 +98,10 @@ julia> sqrt = 4 ERROR: cannot assign a value to imported variable Base.sqrt from module Main ``` +!!! compat "Julia 1.12" + Note that in versions prior to Julia 1.12, these errors depended on *use* rather than definition of + the conflicting binding. + ## [Allowed Variable Names](@id man-allowed-variable-names) Variable names must begin with a letter (A-Z or a-z), underscore, or a subset of Unicode code diff --git a/src/builtins.c b/src/builtins.c index f67ef65d35356..f3d2dfad42819 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -2197,11 +2197,13 @@ static int references_name(jl_value_t *p, jl_typename_t *name, int affects_layou JL_CALLABLE(jl_f__typebody) { - JL_NARGS(_typebody!, 1, 2); - jl_datatype_t *dt = (jl_datatype_t*)jl_unwrap_unionall(args[0]); + JL_NARGS(_typebody!, 2, 3); + jl_value_t *prev = args[0]; + jl_value_t *tret = args[1]; + jl_datatype_t *dt = (jl_datatype_t*)jl_unwrap_unionall(args[1]); JL_TYPECHK(_typebody!, datatype, (jl_value_t*)dt); - if (nargs == 2) { - jl_value_t *ft = args[1]; + if (nargs == 3) { + jl_value_t *ft = args[2]; JL_TYPECHK(_typebody!, simplevector, ft); size_t nf = jl_svec_len(ft); for (size_t i = 0; i < nf; i++) { @@ -2212,30 +2214,53 @@ JL_CALLABLE(jl_f__typebody) (jl_value_t*)jl_type_type, elt); } } - if (dt->types != NULL) { - if (!equiv_field_types((jl_value_t*)dt->types, ft)) - jl_errorf("invalid redefinition of type %s", jl_symbol_name(dt->name->name)); - } - else { - dt->types = (jl_svec_t*)ft; - jl_gc_wb(dt, ft); - // If a supertype can reference the same type, then we may not be - // able to compute the layout of the object before needing to - // publish it, so we must assume it cannot be inlined, if that - // check passes, then we also still need to check the fields too. - if (!dt->name->mutabl && (nf == 0 || !references_name((jl_value_t*)dt->super, dt->name, 0, 1))) { - int mayinlinealloc = 1; - size_t i; - for (i = 0; i < nf; i++) { - jl_value_t *fld = jl_svecref(ft, i); - if (references_name(fld, dt->name, 1, 1)) { - mayinlinealloc = 0; - break; + // Optimization: To avoid lots of unnecessary churning, lowering contains an optimization + // that re-uses the typevars of an existing definition (if any exists) for compute the field + // types. If such a previous type exists, there are two possibilities: + // 1. The field types are identical, we don't need to do anything and can proceed with the + // old type as if it was the new one. + // 2. The field types are not identical, in which case we need to rename the typevars + // back to their equivalents in the new type before proceeding. + if (prev == jl_false) { + if (dt->types != NULL) + jl_errorf("Internal Error: Expected type fields to be unset"); + } else { + jl_datatype_t *prev_dt = (jl_datatype_t*)jl_unwrap_unionall(prev); + JL_TYPECHK(_typebody!, datatype, (jl_value_t*)prev_dt); + if (equiv_field_types((jl_value_t*)prev_dt->types, ft)) { + tret = prev; + goto have_type; + } else { + if (jl_svec_len(prev_dt->parameters) != jl_svec_len(dt->parameters)) + jl_errorf("Internal Error: Types should not have been considered equivalent"); + for (size_t i = 0; i < nf; i++) { + jl_value_t *elt = jl_svecref(ft, i); + for (int j = 0; j < jl_svec_len(prev_dt->parameters); ++j) { + // Only the last svecset matters for semantics, but we re-use the GC root + elt = jl_substitute_var(elt, (jl_tvar_t *)jl_svecref(prev_dt->parameters, j), jl_svecref(dt->parameters, j)); + jl_svecset(ft, i, elt); } } - dt->name->mayinlinealloc = mayinlinealloc; } } + dt->types = (jl_svec_t*)ft; + jl_gc_wb(dt, ft); + // If a supertype can reference the same type, then we may not be + // able to compute the layout of the object before needing to + // publish it, so we must assume it cannot be inlined, if that + // check passes, then we also still need to check the fields too. + if (!dt->name->mutabl && (nf == 0 || !references_name((jl_value_t*)dt->super, dt->name, 0, 1))) { + int mayinlinealloc = 1; + size_t i; + for (i = 0; i < nf; i++) { + jl_value_t *fld = jl_svecref(ft, i); + if (references_name(fld, dt->name, 1, 1)) { + mayinlinealloc = 0; + break; + } + } + dt->name->mayinlinealloc = mayinlinealloc; + } } JL_TRY { @@ -2248,7 +2273,8 @@ JL_CALLABLE(jl_f__typebody) if (jl_is_structtype(dt)) jl_compute_field_offsets(dt); - return jl_nothing; +have_type: + return tret; } // this is a heuristic for allowing "redefining" a type to something identical diff --git a/src/codegen.cpp b/src/codegen.cpp index 7787b684fbd12..5507bd7bad801 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -918,7 +918,7 @@ static const auto jldeclareglobal_func = new JuliaFunction<>{ auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); auto T_prjlvalue = JuliaType::get_prjlvalue_ty(C); return FunctionType::get(getVoidTy(C), - {T_pjlvalue, T_pjlvalue, T_prjlvalue}, false); }, + {T_pjlvalue, T_pjlvalue, T_prjlvalue, getInt32Ty(C)}, false); }, nullptr, }; static const auto jlgetbindingorerror_func = new JuliaFunction<>{ @@ -6943,16 +6943,21 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ } } else if (head == jl_globaldecl_sym) { - assert(nargs == 2); + assert(nargs <= 2 && nargs >= 1); jl_sym_t *sym = (jl_sym_t*)args[0]; jl_module_t *mod = ctx.module; if (jl_is_globalref(sym)) { mod = jl_globalref_mod(sym); sym = jl_globalref_name(sym); } - jl_cgval_t typ = emit_expr(ctx, args[1]); - ctx.builder.CreateCall(prepare_call(jldeclareglobal_func), - { literal_pointer_val(ctx, (jl_value_t*)mod), literal_pointer_val(ctx, (jl_value_t*)sym), boxed(ctx, typ) }); + if (nargs == 2) { + jl_cgval_t typ = emit_expr(ctx, args[1]); + ctx.builder.CreateCall(prepare_call(jldeclareglobal_func), + { literal_pointer_val(ctx, (jl_value_t*)mod), literal_pointer_val(ctx, (jl_value_t*)sym), boxed(ctx, typ), ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 1) }); + } else { + ctx.builder.CreateCall(prepare_call(jldeclareglobal_func), + { literal_pointer_val(ctx, (jl_value_t*)mod), literal_pointer_val(ctx, (jl_value_t*)sym), ConstantPointerNull::get(cast(ctx.types().T_prjlvalue)), ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 1) }); + } } else if (head == jl_new_sym) { bool is_promotable = false; diff --git a/src/gc-stock.c b/src/gc-stock.c index 8118b3c5629ae..72479d14e67a5 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -2144,6 +2144,9 @@ STATIC_INLINE void gc_mark_module_binding(jl_ptls_t ptls, jl_module_t *mb_parent gc_heap_snapshot_record_module_to_binding(mb_parent, bindings, bindingkeyset); gc_assert_parent_validity((jl_value_t *)mb_parent, (jl_value_t *)mb_parent->parent); gc_try_claim_and_push(mq, (jl_value_t *)mb_parent->parent, &nptr); + gc_assert_parent_validity((jl_value_t *)mb_parent, (jl_value_t *)mb_parent->usings_backedges); + gc_try_claim_and_push(mq, (jl_value_t *)mb_parent->usings_backedges, &nptr); + gc_heap_snapshot_record_binding_partition_edge((jl_value_t*)mb_parent, mb_parent->usings_backedges); size_t nusings = module_usings_length(mb_parent); if (nusings > 0) { // this is only necessary because bindings for "using" modules diff --git a/src/interpreter.c b/src/interpreter.c index 35c70a9ead2f1..513fe58f7b5cc 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -634,9 +634,12 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, s->locals[jl_source_nslots(s->src) + s->ip] = res; } else if (head == jl_globaldecl_sym) { - jl_value_t *val = eval_value(jl_exprarg(stmt, 1), s); - s->locals[jl_source_nslots(s->src) + s->ip] = val; // temporarily root - jl_declare_global(s->module, jl_exprarg(stmt, 0), val); + jl_value_t *val = NULL; + if (jl_expr_nargs(stmt) >= 2) { + val = eval_value(jl_exprarg(stmt, 1), s); + s->locals[jl_source_nslots(s->src) + s->ip] = val; // temporarily root + } + jl_declare_global(s->module, jl_exprarg(stmt, 0), val, 1); s->locals[jl_source_nslots(s->src) + s->ip] = jl_nothing; } else if (head == jl_const_sym) { diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 21b5afa74c5f4..664e1270c7381 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -42,7 +42,6 @@ XX(jl_atomic_swap_bits) \ XX(jl_backtrace_from_here) \ XX(jl_base_relative_to) \ - XX(jl_binding_resolved_p) \ XX(jl_bitcast) \ XX(jl_boundp) \ XX(jl_bounds_error) \ @@ -72,6 +71,7 @@ XX(jl_call1) \ XX(jl_call2) \ XX(jl_call3) \ + XX(jl_call4) \ XX(jl_calloc) \ XX(jl_call_in_typeinf_world) \ XX(jl_capture_interp_frame) \ @@ -213,6 +213,8 @@ XX(jl_get_module_infer) \ XX(jl_get_module_of_binding) \ XX(jl_get_module_optlevel) \ + XX(jl_get_module_usings_backedges) \ + XX(jl_get_module_binding_or_nothing) \ XX(jl_get_next_task) \ XX(jl_get_nth_field) \ XX(jl_get_nth_field_checked) \ diff --git a/src/jlapi.c b/src/jlapi.c index b8fbda801f43b..e205a4a4dc723 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -437,6 +437,46 @@ JL_DLLEXPORT jl_value_t *jl_call3(jl_function_t *f, jl_value_t *a, return v; } +/** + * @brief Call a Julia function with three arguments. + * + * A specialized case of `jl_call` for simpler scenarios. + * + * @param f A pointer to `jl_function_t` representing the Julia function to call. + * @param a A pointer to `jl_value_t` representing the first argument. + * @param b A pointer to `jl_value_t` representing the second argument. + * @param c A pointer to `jl_value_t` representing the third argument. + * @param d A pointer to `jl_value_t` representing the fourth argument. + * @return A pointer to `jl_value_t` representing the result of the function call. + */ +JL_DLLEXPORT jl_value_t *jl_call4(jl_function_t *f, jl_value_t *a, + jl_value_t *b, jl_value_t *c, + jl_value_t *d) +{ + jl_value_t *v; + jl_task_t *ct = jl_current_task; + JL_TRY { + jl_value_t **argv; + JL_GC_PUSHARGS(argv, 5); + argv[0] = f; + argv[1] = a; + argv[2] = b; + argv[3] = c; + argv[4] = d; + size_t last_age = ct->world_age; + ct->world_age = jl_get_world_counter(); + v = jl_apply(argv, 5); + ct->world_age = last_age; + JL_GC_POP(); + _jl_exception_clear(ct); + } + JL_CATCH { + ct->ptls->previous_exception = jl_current_exception(ct); + v = NULL; + } + return v; +} + /** * @brief Get a field from a Julia object. * diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 97d76e7762a9e..726312a13d10f 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1002,7 +1002,9 @@ (default-inner-ctors name field-names field-types params bounds locs) defs)) (min-initialized (min (ctors-min-initialized defs) (length fields))) - (prev (make-ssavalue))) + (hasprev (make-ssavalue)) + (prev (make-ssavalue)) + (newdef (make-ssavalue))) (let ((dups (has-dups field-names))) (if dups (error (string "duplicate field name: \"" (car dups) "\" is not unique")))) (for-each (lambda (v) @@ -1023,22 +1025,21 @@ (call (core svec) ,@attrs) ,mut ,min-initialized)) (call (core _setsuper!) ,name ,super) - (if (call (core isdefinedglobal) (thismodule) (inert ,name) (false)) - (block - (= ,prev (globalref (thismodule) ,name)) - (if (call (core _equiv_typedef) ,prev ,name) - ;; if this is compatible with an old definition, use the existing type object - ;; and its parameters - (block (= ,name ,prev) - ,@(if (pair? params) - `((= (tuple ,@params) (|.| - ,(foldl (lambda (_ x) `(|.| ,x (quote body))) - prev - params) - (quote parameters)))) - '()))))) - (call (core _typebody!) ,name (call (core svec) ,@(insert-struct-shim field-types name))) - (const (globalref (thismodule) ,name) ,name) + (= ,hasprev (&& (call (core isdefinedglobal) (thismodule) (inert ,name) (false)) (call (core _equiv_typedef) (globalref (thismodule) ,name) ,name))) + (= ,prev (if ,hasprev (globalref (thismodule) ,name) (false))) + (if ,hasprev + ;; if this is compatible with an old definition, use the old parameters, but the + ;; new object. This will fail to capture recursive cases, but the call to typebody! + ;; below is permitted to choose either type definition to put into the binding table + (block ,@(if (pair? params) + `((= (tuple ,@params) (|.| + ,(foldl (lambda (_ x) `(|.| ,x (quote body))) + prev + params) + (quote parameters)))) + '()))) + (= ,newdef (call (core _typebody!) ,prev ,name (call (core svec) ,@(insert-struct-shim field-types name)))) + (const (globalref (thismodule) ,name) ,newdef) (latestworld) (null))) ;; "inner" constructors @@ -1084,7 +1085,7 @@ (toplevel-only abstract_type) (= ,name (call (core _abstracttype) (thismodule) (inert ,name) (call (core svec) ,@params))) (call (core _setsuper!) ,name ,super) - (call (core _typebody!) ,name) + (call (core _typebody!) (false) ,name) (if (&& (call (core isdefinedglobal) (thismodule) (inert ,name) (false)) (call (core _equiv_typedef) (globalref (thismodule) ,name) ,name)) (null) @@ -1105,7 +1106,7 @@ (toplevel-only primitive_type) (= ,name (call (core _primitivetype) (thismodule) (inert ,name) (call (core svec) ,@params) ,n)) (call (core _setsuper!) ,name ,super) - (call (core _typebody!) ,name) + (call (core _typebody!) (false) ,name) (if (&& (call (core isdefinedglobal) (thismodule) (inert ,name) (false)) (call (core _equiv_typedef) (globalref (thismodule) ,name) ,name)) (null) @@ -3525,7 +3526,7 @@ f(x) = yt(x) (false) ,(length fields))) (call (core _setsuper!) ,s ,super) (const (globalref (thismodule) ,name) ,s) - (call (core _typebody!) ,s (call (core svec) ,@types)) + (call (core _typebody!) (false) ,s (call (core svec) ,@types)) (return (null))))))))) (define (type-for-closure name fields super) @@ -3539,7 +3540,7 @@ f(x) = yt(x) (false) ,(length fields))) (call (core _setsuper!) ,s ,super) (const (globalref (thismodule) ,name) ,s) - (call (core _typebody!) ,s + (call (core _typebody!) (false) ,s (call (core svec) ,@(map (lambda (v) '(core Box)) fields))) (return (null))))))))) @@ -3641,8 +3642,8 @@ f(x) = yt(x) rhs1)) (ex `(= ,var ,rhs))) (if (eq? rhs1 rhs0) - `(block ,ex ,rhs0) - `(block (= ,rhs1 ,rhs0) + `(block (globaldecl ,var) ,ex ,rhs0) + `(block (globaldecl ,var) (= ,rhs1 ,rhs0) ,ex ,rhs1)))) @@ -4624,11 +4625,7 @@ f(x) = yt(x) tests)) (define (emit-assignment-or-setglobal lhs rhs) (if (globalref? lhs) - (begin - (emit `(global ,lhs)) - (if (null? (cadr lam)) - (emit `(latestworld))) - (emit `(call (top setglobal!) ,(cadr lhs) (inert ,(caddr lhs)) ,rhs))) + (emit `(call (top setglobal!) ,(cadr lhs) (inert ,(caddr lhs)) ,rhs)) (emit `(= ,lhs ,rhs)))) (define (emit-assignment lhs rhs) (if rhs @@ -4951,11 +4948,12 @@ f(x) = yt(x) (emit `(latestworld)))) ((globaldecl) (if value (error "misplaced \"global\" declaration")) - (if (atom? (caddr e)) (begin (emit e) (emit `(latestworld))) + (if (or (length= e 2) (atom? (caddr e))) (emit e) (let ((rr (make-ssavalue))) (emit `(= ,rr ,(caddr e))) - (emit `(globaldecl ,(cadr e) ,rr)) - (emit `(latestworld))))) + (emit `(globaldecl ,(cadr e) ,rr)))) + (if (null? (cadr lam)) + (emit `(latestworld)))) ((local-def) #f) ((local) #f) ((moved-local) diff --git a/src/julia.h b/src/julia.h index a80a69049ccb2..dd3bc713a517f 100644 --- a/src/julia.h +++ b/src/julia.h @@ -616,6 +616,52 @@ typedef struct _jl_weakref_t { } jl_weakref_t; // N.B: Needs to be synced with runtime_internals.jl +// We track essentially three levels of binding strength: +// +// 1. Implicit Bindings (Weakest) +// These binding kinds depend solely on the set of using'd packages and are not explicitly +// declared: +// +// BINDING_KIND_IMPLICIT +// BINDING_KIND_GUARD +// BINDING_KIND_FAILED +// +// 2. Weakly Declared Bindings (Weak) +// The binding was declared using `global`. It is treated as a mutable, `Any` type global +// for almost all purposes, except that it receives slightly worse optimizations, since it +// may be replaced. +// +// BINDING_KIND_DECLARED +// +// 3. Strong Declared Bindings (Weak) +// All other bindings are explicitly declared using a keyword or global assignment. +// These are considered strongest: +// +// BINDING_KIND_CONST +// BINDING_KIND_CONST_IMPORT +// BINDING_KIND_EXPLICIT +// BINDING_KIND_IMPORTED +// BINDING_KIND_GLOBAL +// BINDING_KIND_UNDEF_CONST +// +// The runtime supports syntactic invalidation (by raising the world age and changing the partition type +// in the new world age) from any partition kind to any other. +// +// However, not all transitions are allowed syntactically. We have the following rules for SYNTACTIC invalidation: +// 1. It is always syntactically permissable to replace a weaker binding by a stronger binding +// 2. Implicit bindings can be syntactically changed to other implicit bindings by changing the `using` set. +// 3. Finally, we syntactically permit replacing one BINDING_KIND_CONST(_IMPORT) by another of a different value. +// +// We may make this list more permissive in the future. +// +// Finally, BINDING_KIND_BACKDATED_CONST is a special case, and the only case where we may replace an +// existing partition by a different partition kind in the same world age. As such, it needs special +// support in inference. Any partition kind that may be replaced by a BINDING_KIND_BACKDATED_CONST +// must be inferred accordingly. BINDING_KIND_BACKDATED_CONST is intended as a temporary compatibility +// measure. The following kinds may be replaced by BINDING_KIND_BACKDATED_CONST: +// - BINDING_KIND_GUARD +// - BINDING_KIND_FAILED +// - BINDING_KIND_DECLARED enum jl_partition_kind { // Constant: This binding partition is a constant declared using `const _ = ...` // ->restriction holds the constant value @@ -623,7 +669,8 @@ enum jl_partition_kind { // Import Constant: This binding partition is a constant declared using `import A` // ->restriction holds the constant value BINDING_KIND_CONST_IMPORT = 0x1, - // Global: This binding partition is a global variable. + // Global: This binding partition is a global variable. It was declared either using + // `global x::T` to implicitly through a syntactic global assignment. // -> restriction holds the type restriction BINDING_KIND_GLOBAL = 0x2, // Implicit: The binding was implicitly imported from a `using`'d module. @@ -638,7 +685,9 @@ enum jl_partition_kind { // Failed: We attempted to import the binding, but the import was ambiguous // ->restriction is NULL. BINDING_KIND_FAILED = 0x6, - // Declared: The binding was declared using `global` or similar + // Declared: The binding was declared using `global` or similar. This acts in most ways like + // BINDING_KIND_GLOBAL with an `Any` restriction, except that it may be redefined to a stronger + // binding like `const` or an explicit import. // ->restriction is NULL. BINDING_KIND_DECLARED = 0x7, // Guard: The binding was looked at, but no global or import was resolved at the time @@ -651,6 +700,10 @@ enum jl_partition_kind { // Backated constant. A constant that was backdated for compatibility. In all other // ways equivalent to BINDING_KIND_CONST, but prints a warning on access BINDING_KIND_BACKDATED_CONST = 0xa, + + // This is not a real binding kind, but can be used to ask for a re-resolution + // of the implicit binding kind + BINDING_KIND_IMPLICIT_RECOMPUTE = 0xb }; #ifdef _P64 @@ -672,17 +725,7 @@ typedef struct __attribute__((aligned(8))) _jl_binding_partition_t { * * Currently: Low 3 bits hold ->kind on _P64 to avoid needing >8 byte atomics * - * This field is updated atomically with both kind and restriction. The following - * transitions are allowed and modeled by the system: - * - * GUARD -> any - * (DECLARED, FAILED) -> any non-GUARD - * IMPLICIT -> {EXPLICIT, IMPORTED} (->restriction unchanged only) - * - * In addition, we permit (with warning about undefined behavior) changing the restriction - * pointer for CONST(_IMPORT). - * - * All other kind or restriction transitions are disallowed. + * This field is updated atomically with both kind and restriction */ _Atomic(jl_ptr_kind_union_t) restriction; size_t min_world; @@ -717,6 +760,7 @@ typedef struct _jl_module_t { _Atomic(jl_genericmemory_t*) bindingkeyset; // index lookup by name into bindings jl_sym_t *file; int32_t line; + jl_value_t *usings_backedges; // hidden fields: arraylist_t usings; /* arraylist of struct jl_module_using */ // modules with all bindings potentially imported jl_uuid_t build_id; @@ -1996,6 +2040,9 @@ JL_DLLEXPORT void jl_set_module_infer(jl_module_t *self, int value); JL_DLLEXPORT int jl_get_module_infer(jl_module_t *m); JL_DLLEXPORT void jl_set_module_max_methods(jl_module_t *self, int value); JL_DLLEXPORT int jl_get_module_max_methods(jl_module_t *m); +JL_DLLEXPORT jl_value_t *jl_get_module_usings_backedges(jl_module_t *m); +JL_DLLEXPORT jl_value_t *jl_get_module_binding_or_nothing(jl_module_t *m, jl_sym_t *s); + // get binding for reading JL_DLLEXPORT jl_binding_t *jl_get_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); JL_DLLEXPORT jl_binding_t *jl_get_binding_or_error(jl_module_t *m, jl_sym_t *var); @@ -2007,7 +2054,6 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import); JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var); -JL_DLLEXPORT int jl_binding_resolved_p(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr); JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr); @@ -2253,6 +2299,9 @@ JL_DLLEXPORT jl_value_t *jl_call1(jl_function_t *f JL_MAYBE_UNROOTED, jl_value_t JL_DLLEXPORT jl_value_t *jl_call2(jl_function_t *f JL_MAYBE_UNROOTED, jl_value_t *a JL_MAYBE_UNROOTED, jl_value_t *b JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_value_t *jl_call3(jl_function_t *f JL_MAYBE_UNROOTED, jl_value_t *a JL_MAYBE_UNROOTED, jl_value_t *b JL_MAYBE_UNROOTED, jl_value_t *c JL_MAYBE_UNROOTED); +JL_DLLEXPORT jl_value_t *jl_call4(jl_function_t *f JL_MAYBE_UNROOTED, jl_value_t *a JL_MAYBE_UNROOTED, + jl_value_t *b JL_MAYBE_UNROOTED, jl_value_t *c JL_MAYBE_UNROOTED, + jl_value_t *d JL_MAYBE_UNROOTED); // async signal handling ------------------------------------------------------ diff --git a/src/julia_internal.h b/src/julia_internal.h index 41a650969ae01..5fe6cad0d096c 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -843,12 +843,19 @@ int jl_type_equality_is_identity(jl_value_t *t1, jl_value_t *t2) JL_NOTSAFEPOINT JL_DLLEXPORT void jl_eval_const_decl(jl_module_t *m, jl_value_t *arg, jl_value_t *val); void jl_binding_set_type(jl_binding_t *b, jl_module_t *mod, jl_sym_t *sym, jl_value_t *ty); void jl_eval_global_expr(jl_module_t *m, jl_expr_t *ex, int set_type); -JL_DLLEXPORT void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type); +JL_DLLEXPORT void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type, int strong); JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, enum jl_partition_kind, size_t new_world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int expanded, const char **toplevel_filename, int *toplevel_lineno); STATIC_INLINE struct _jl_module_using *module_usings_getidx(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; STATIC_INLINE jl_module_t *module_usings_getmod(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; +void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to); +typedef struct _modstack_t { + jl_binding_t *b; + struct _modstack_t *prev; +} modstack_t; +void jl_check_new_binding_implicit( + jl_binding_partition_t *new_bpart JL_MAYBE_UNROOTED, jl_binding_t *b, modstack_t *st, size_t world); #ifndef __clang_gcanalyzer__ // The analyzer doesn't like looking through the arraylist, so just model the @@ -905,6 +912,8 @@ void jl_compute_field_offsets(jl_datatype_t *st); void jl_module_run_initializer(jl_module_t *m); JL_DLLEXPORT jl_binding_t *jl_get_module_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var, int alloc); JL_DLLEXPORT void jl_binding_deprecation_warning(jl_module_t *m, jl_sym_t *sym, jl_binding_t *b); +JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked(jl_binding_t *b JL_PROPAGATES_ROOT, + jl_binding_partition_t *old_bpart, jl_value_t *restriction_val, enum jl_partition_kind kind, size_t new_world) JL_GLOBALLY_ROOTED; extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; @@ -991,7 +1000,7 @@ STATIC_INLINE int jl_bkind_is_defined_constant(enum jl_partition_kind kind) JL_N } STATIC_INLINE int jl_bkind_is_some_guard(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == BINDING_KIND_FAILED || kind == BINDING_KIND_GUARD || kind == BINDING_KIND_DECLARED; + return kind == BINDING_KIND_FAILED || kind == BINDING_KIND_GUARD; } JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world) JL_GLOBALLY_ROOTED; diff --git a/src/method.c b/src/method.c index 8a14eb00182b1..1b38a16649d8a 100644 --- a/src/method.c +++ b/src/method.c @@ -62,6 +62,11 @@ static jl_value_t *resolve_definition_effects(jl_value_t *expr, jl_module_t *mod jl_eval_global_expr(module, e, 1); return jl_nothing; } + if (e->head == jl_globaldecl_sym && binding_effects) { + assert(jl_expr_nargs(e) == 1); + jl_declare_global(module, jl_exprarg(e, 0), NULL, 1); + return jl_nothing; + } // These exprs are not fully linearized if (e->head == jl_assign_sym) { jl_exprargset(e, 1, resolve_definition_effects(jl_exprarg(e, 1), module, sparam_vals, binding_edge, binding_effects, eager_resolve)); @@ -180,26 +185,24 @@ static jl_value_t *resolve_definition_effects(jl_value_t *expr, jl_module_t *mod jl_value_t *fe = jl_exprarg(e, 0); jl_module_t *fe_mod = jl_globalref_mod(fe); jl_sym_t *fe_sym = jl_globalref_name(fe); - if (jl_binding_resolved_p(fe_mod, fe_sym)) { - // look at some known called functions - jl_binding_t *b = jl_get_binding(fe_mod, fe_sym); - if (jl_get_binding_value_if_const(b) == jl_builtin_tuple) { - size_t j; - for (j = 1; j < nargs; j++) { - if (!jl_is_quotenode(jl_exprarg(e, j))) - break; + // look at some known called functions + jl_binding_t *b = jl_get_binding(fe_mod, fe_sym); + if (jl_get_binding_value_if_const(b) == jl_builtin_tuple) { + size_t j; + for (j = 1; j < nargs; j++) { + if (!jl_is_quotenode(jl_exprarg(e, j))) + break; + } + if (j == nargs) { + jl_value_t *val = NULL; + JL_TRY { + val = jl_interpret_toplevel_expr_in(module, (jl_value_t*)e, NULL, sparam_vals); } - if (j == nargs) { - jl_value_t *val = NULL; - JL_TRY { - val = jl_interpret_toplevel_expr_in(module, (jl_value_t*)e, NULL, sparam_vals); - } - JL_CATCH { - val = NULL; // To make the analyzer happy see #define JL_TRY - } - if (val) - return val; + JL_CATCH { + val = NULL; // To make the analyzer happy see #define JL_TRY } + if (val) + return val; } } } @@ -1052,9 +1055,11 @@ JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_binding_t *b, jl_module_t *mod, JL_LOCK(&world_counter_lock); size_t new_world = jl_atomic_load_relaxed(&jl_world_counter) + 1; jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); - jl_ptr_kind_union_t pku = jl_walk_binding_inplace(&b, &bpart, new_world); + jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); jl_value_t *gf = NULL; - if (!jl_bkind_is_some_guard(decode_restriction_kind(pku))) { + enum jl_partition_kind kind = decode_restriction_kind(pku); + if (!jl_bkind_is_some_guard(kind) && kind != BINDING_KIND_DECLARED && kind != BINDING_KIND_IMPLICIT) { + pku = jl_walk_binding_inplace(&b, &bpart, new_world); if (jl_bkind_is_some_constant(decode_restriction_kind(pku))) { gf = decode_restriction_value(pku); JL_GC_PROMISE_ROOTED(gf); diff --git a/src/module.c b/src/module.c index b2a4018519fca..1b6b37e49949e 100644 --- a/src/module.c +++ b/src/module.c @@ -29,7 +29,130 @@ static jl_binding_partition_t *new_binding_partition(void) return bpart; } -jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) { + +static jl_binding_partition_t *jl_get_binding_partition2(jl_binding_t *b, size_t world, modstack_t *st); + +static int eq_bindings(jl_binding_partition_t *owner, jl_binding_t *alias, size_t world) +{ + jl_binding_t *ownerb = NULL; + jl_binding_partition_t *alias_bpart = jl_get_binding_partition(alias, world); + if (owner == alias_bpart) + return 1; + jl_ptr_kind_union_t owner_pku = jl_walk_binding_inplace(&ownerb, &owner, world); + jl_ptr_kind_union_t alias_pku = jl_walk_binding_inplace(&alias, &alias_bpart, world); + if (jl_bkind_is_some_constant(decode_restriction_kind(owner_pku)) && + jl_bkind_is_some_constant(decode_restriction_kind(alias_pku)) && + decode_restriction_value(owner_pku) && + decode_restriction_value(alias_pku) == decode_restriction_value(owner_pku)) + return 1; + return owner == alias_bpart; +} + +// find a binding from a module's `usings` list +void jl_check_new_binding_implicit( + jl_binding_partition_t *new_bpart JL_MAYBE_UNROOTED, jl_binding_t *b, modstack_t *st, size_t world) +{ + modstack_t top = { b, st }; + modstack_t *tmp = st; + for (; tmp != NULL; tmp = tmp->prev) { + if (tmp->b == b) { + jl_atomic_store_relaxed(&new_bpart->restriction, encode_restriction(NULL, BINDING_KIND_FAILED /* BINDING_KIND_CYCLE */)); + return; + } + } + + JL_GC_PUSH1(&new_bpart); + jl_module_t *m = b->globalref->mod; + jl_sym_t *var = b->globalref->name; + + jl_binding_t *deprecated_impb = NULL; + jl_binding_t *impb = NULL; + + size_t min_world = new_bpart->min_world; + size_t max_world = jl_atomic_load_relaxed(&new_bpart->max_world); + + JL_LOCK(&m->lock); + int i = (int)module_usings_length(m) - 1; + JL_UNLOCK(&m->lock); + enum jl_partition_kind guard_kind = BINDING_KIND_GUARD; + for (; i >= 0; --i) { + JL_LOCK(&m->lock); + struct _jl_module_using data = *module_usings_getidx(m, i); + JL_UNLOCK(&m->lock); + if (data.min_world > world) { + if (max_world > data.min_world) + max_world = data.min_world - 1; + continue; + } + if (data.max_world < world) { + if (min_world < data.max_world) + min_world = data.max_world + 1; + continue; + } + jl_module_t *imp = data.mod; + JL_GC_PROMISE_ROOTED(imp); + jl_binding_t *tempb = jl_get_module_binding(imp, var, 0); + if (tempb != NULL && tempb->exportp) { + if (data.min_world > min_world) + min_world = data.min_world; + if (data.max_world < min_world) + max_world = data.max_world; + + jl_binding_partition_t *tempbpart = jl_get_binding_partition2(tempb, world, &top); + JL_GC_PROMISE_ROOTED(tempbpart); + + size_t tempbmax_world = jl_atomic_load_relaxed(&tempbpart->max_world); + if (tempbpart->min_world > min_world) + min_world = tempbpart->min_world; + if (tempbmax_world < max_world) + max_world = tempbmax_world; + + if (impb) { + if (tempb->deprecated) + continue; + if (eq_bindings(tempbpart, impb, world)) + continue; + // Binding is ambiguous + // TODO: Even for eq bindings, this may need to further constrain the world age. + deprecated_impb = impb = NULL; + guard_kind = BINDING_KIND_FAILED; + break; + } + else if (tempb->deprecated) { + if (deprecated_impb) { + if (!eq_bindings(tempbpart, deprecated_impb, world)) { + guard_kind = BINDING_KIND_FAILED; + deprecated_impb = NULL; + } + } + else if (guard_kind == BINDING_KIND_GUARD) { + deprecated_impb = tempb; + } + } + else { + impb = tempb; + } + } + } + + if (deprecated_impb && !impb) + impb = deprecated_impb; + + assert(min_world <= max_world); + new_bpart->min_world = min_world; + jl_atomic_store_relaxed(&new_bpart->max_world, max_world); + if (impb) { + jl_atomic_store_relaxed(&new_bpart->restriction, encode_restriction((jl_value_t*)impb, BINDING_KIND_IMPLICIT)); + // TODO: World age constraints? + } else { + jl_atomic_store_relaxed(&new_bpart->restriction, encode_restriction(NULL, guard_kind)); + } + JL_GC_POP(); + return; +} + +STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED +{ if (!b) return NULL; assert(jl_is_binding(b)); @@ -53,6 +176,8 @@ jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) jl_gc_wb_fresh(new_bpart, bpart); new_bpart->min_world = bpart ? jl_atomic_load_relaxed(&bpart->max_world) + 1 : 0; jl_atomic_store_relaxed(&new_bpart->max_world, max_world); + JL_GC_PROMISE_ROOTED(new_bpart); // TODO: Analyzer doesn't understand MAYBE_UNROOTED properly + jl_check_new_binding_implicit(new_bpart, b, st, world); if (jl_atomic_cmpswap(insert, &bpart, new_bpart)) { jl_gc_wb(parent, new_bpart); return new_bpart; @@ -60,6 +185,15 @@ jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) } } +jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) { + // Duplicate the code for the entry frame for branch prediction + return jl_get_binding_partition_(b, world, NULL); +} + +jl_binding_partition_t *jl_get_binding_partition2(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED { + return jl_get_binding_partition_(b, world, st); +} + jl_binding_partition_t *jl_get_binding_partition_all(jl_binding_t *b, size_t min_world, size_t max_world) { if (!b) return NULL; @@ -89,6 +223,7 @@ JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, ui m->build_id.lo++; // build id 0 is invalid m->build_id.hi = ~(uint64_t)0; jl_atomic_store_relaxed(&m->counter, 1); + m->usings_backedges = jl_nothing; m->nospecialize = 0; m->optlevel = -1; m->compile = -1; @@ -255,7 +390,7 @@ extern jl_mutex_t jl_modules_mutex; extern void check_safe_newbinding(jl_module_t *m, jl_sym_t *var) { if (jl_current_task->ptls->in_pure_callback) - jl_errorf("new globals cannot be created in a generated function"); + jl_errorf("new strong globals cannot be created in a generated function. Declare them outside using `global x::Any`."); if (jl_options.incremental && jl_generating_output()) { JL_LOCK(&jl_modules_mutex); int open = ptrhash_has(&jl_current_modules, (void*)m); @@ -285,20 +420,14 @@ JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module { jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); -retry: - if (decode_restriction_kind(pku) != BINDING_KIND_GLOBAL && !jl_bkind_is_some_constant(decode_restriction_kind(pku))) { - if (jl_bkind_is_some_guard(decode_restriction_kind(pku))) { - if (decode_restriction_kind(pku) != BINDING_KIND_DECLARED) { - jl_errorf("Global %s.%s does not exist and cannot be assigned.\n" - "Note: Julia 1.9 and 1.10 inadvertently omitted this error check (#56933).\n" - "Hint: Declare it using `global %s` inside `%s` before attempting assignment.", - jl_symbol_name(m->name), jl_symbol_name(s), - jl_symbol_name(s), jl_symbol_name(m->name)); - } - jl_ptr_kind_union_t new_pku = encode_restriction((jl_value_t*)jl_any_type, BINDING_KIND_GLOBAL); - if (!jl_atomic_cmpswap(&bpart->restriction, &pku, new_pku)) - goto retry; - jl_gc_wb_knownold(bpart, jl_any_type); + enum jl_partition_kind kind = decode_restriction_kind(pku); + if (kind != BINDING_KIND_GLOBAL && kind != BINDING_KIND_DECLARED && !jl_bkind_is_some_constant(kind)) { + if (jl_bkind_is_some_guard(kind)) { + jl_errorf("Global %s.%s does not exist and cannot be assigned.\n" + "Note: Julia 1.9 and 1.10 inadvertently omitted this error check (#56933).\n" + "Hint: Declare it using `global %s` inside `%s` before attempting assignment.", + jl_symbol_name(m->name), jl_symbol_name(s), + jl_symbol_name(s), jl_symbol_name(m->name)); } else { jl_module_t *from = jl_binding_dbgmodule(b, m, s); if (from == m) @@ -321,10 +450,10 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, // return module of binding JL_DLLEXPORT jl_module_t *jl_get_module_of_binding(jl_module_t *m, jl_sym_t *var) { - jl_binding_t *b = jl_get_binding(m, var); - if (b == NULL) - return NULL; - return b->globalref->mod; // TODO: deprecate this? + jl_binding_t *b = jl_get_module_binding(m, var, 1); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + return b ? b->globalref->mod : m; } static NOINLINE void print_backdate_admonition(jl_binding_t *b) JL_NOTSAFEPOINT @@ -358,6 +487,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value(jl_binding_t *b) check_backdated_binding(b, kind); return decode_restriction_value(pku); } + assert(!jl_bkind_is_some_import(kind)); return jl_atomic_load_relaxed(&b->value); } @@ -372,6 +502,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_seqcst(jl_binding_t *b) check_backdated_binding(b, kind); return decode_restriction_value(pku); } + assert(!jl_bkind_is_some_import(kind)); return jl_atomic_load(&b->value); } @@ -444,19 +575,14 @@ JL_DLLEXPORT jl_value_t *jl_bpart_get_restriction_value(jl_binding_partition_t * return v; } -typedef struct _modstack_t { - jl_module_t *m; - jl_sym_t *var; - struct _modstack_t *prev; -} modstack_t; -static jl_binding_t *jl_resolve_owner(jl_binding_t *b/*optional*/, jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var, modstack_t *st, size_t world); - JL_DLLEXPORT jl_value_t *jl_reresolve_binding_value_seqcst(jl_binding_t *b) { + /* jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); if (jl_bkind_is_some_guard(decode_restriction_kind(jl_atomic_load_relaxed(&bpart->restriction)))) { jl_resolve_owner(b, b->globalref->mod, b->globalref->name, NULL, jl_current_task->world_age); } + */ return jl_get_binding_value_seqcst(b); } @@ -467,91 +593,48 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ jl_binding_t *b = jl_get_module_binding(m, var, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); - if (decode_restriction_kind(pku) != BINDING_KIND_GLOBAL && !jl_bkind_is_some_constant(decode_restriction_kind(pku))) { - if (jl_bkind_is_some_guard(decode_restriction_kind(pku))) { - if (decode_restriction_kind(pku) != BINDING_KIND_DECLARED) { - check_safe_newbinding(m, var); - } + enum jl_partition_kind kind = decode_restriction_kind(pku); + if (kind == BINDING_KIND_GLOBAL || kind == BINDING_KIND_DECLARED || jl_bkind_is_some_constant(decode_restriction_kind(pku))) + return b; + if (jl_bkind_is_some_guard(kind)) { + check_safe_newbinding(m, var); + return b; + } + jl_binding_t *ownerb = b; + pku = jl_walk_binding_inplace(&ownerb, &bpart, jl_current_task->world_age); + jl_value_t *f = NULL; + if (jl_bkind_is_some_constant(decode_restriction_kind(pku))) + f = decode_restriction_value(pku); + if (f == NULL) { + if (kind == BINDING_KIND_IMPLICIT) { + check_safe_newbinding(m, var); return b; } - jl_value_t *f = jl_get_binding_value_if_const(b); - if (f == NULL) { - jl_module_t *from = jl_binding_dbgmodule(b, m, var); - // we must have implicitly imported this with using, so call jl_binding_dbgmodule to try to get the name of the module we got this from - jl_errorf("invalid method definition in %s: exported function %s.%s does not exist", - jl_symbol_name(m->name), jl_symbol_name(from->name), jl_symbol_name(var)); + jl_module_t *from = jl_binding_dbgmodule(b, m, var); + // we must have implicitly imported this with using, so call jl_binding_dbgmodule to try to get the name of the module we got this from + jl_errorf("invalid method definition in %s: exported function %s.%s does not exist", + jl_symbol_name(m->name), from ? jl_symbol_name(from->name) : "", jl_symbol_name(var)); + } + int istype = f && jl_is_type(f); + if (!istype) { + if (kind == BINDING_KIND_IMPLICIT) { + check_safe_newbinding(m, var); + return b; } - // TODO: we might want to require explicitly importing types to add constructors - // or we might want to drop this error entirely - if (decode_restriction_kind(pku) != BINDING_KIND_IMPORTED && !(f && jl_is_type(f) && strcmp(jl_symbol_name(var), "=>") != 0)) { + else if (kind != BINDING_KIND_IMPORTED) { + // TODO: we might want to require explicitly importing types to add constructors + // or we might want to drop this error entirely jl_module_t *from = jl_binding_dbgmodule(b, m, var); jl_errorf("invalid method definition in %s: function %s.%s must be explicitly imported to be extended", - jl_symbol_name(m->name), jl_symbol_name(from->name), jl_symbol_name(var)); + jl_symbol_name(m->name), from ? jl_symbol_name(from->name) : "", jl_symbol_name(var)); } - return b; } - return b; -} - -static int eq_bindings(jl_binding_partition_t *owner, jl_binding_t *alias, size_t world) -{ - jl_ptr_kind_union_t owner_pku = jl_atomic_load_relaxed(&owner->restriction); - assert(decode_restriction_kind(owner_pku) == BINDING_KIND_GLOBAL || decode_restriction_kind(owner_pku) == BINDING_KIND_DECLARED || - jl_bkind_is_some_constant(decode_restriction_kind(owner_pku))); - jl_binding_partition_t *alias_bpart = jl_get_binding_partition(alias, world); - if (owner == alias_bpart) - return 1; - jl_ptr_kind_union_t alias_pku = jl_walk_binding_inplace(&alias, &alias_bpart, world); - if (jl_bkind_is_some_constant(decode_restriction_kind(owner_pku)) && - jl_bkind_is_some_constant(decode_restriction_kind(alias_pku)) && - decode_restriction_value(owner_pku) && - decode_restriction_value(alias_pku) == decode_restriction_value(owner_pku)) - return 1; - return owner == alias_bpart; -} - -// find a binding from a module's `usings` list -static jl_binding_t *using_resolve_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var, jl_module_t **from, modstack_t *st, int warn, size_t world) -{ - jl_binding_t *b = NULL; - jl_binding_partition_t *bpart = NULL; - jl_module_t *owner = NULL; - JL_LOCK(&m->lock); - int i = (int)module_usings_length(m) - 1; - JL_UNLOCK(&m->lock); - for (; i >= 0; --i) { - JL_LOCK(&m->lock); - jl_module_t *imp = module_usings_getmod(m, i); - JL_UNLOCK(&m->lock); - jl_binding_t *tempb = jl_get_module_binding(imp, var, 0); - if (tempb != NULL && tempb->exportp) { - tempb = jl_resolve_owner(NULL, imp, var, st, world); // find the owner for tempb - if (tempb == NULL) - // couldn't resolve; try next using (see issue #6105) - continue; - jl_binding_partition_t *tempbpart = jl_get_binding_partition(tempb, world); - jl_ptr_kind_union_t tempb_pku = jl_atomic_load_relaxed(&tempbpart->restriction); - assert(jl_bkind_is_some_guard(decode_restriction_kind(tempb_pku)) || decode_restriction_kind(tempb_pku) == BINDING_KIND_GLOBAL || decode_restriction_kind(tempb_pku) == BINDING_KIND_DECLARED || jl_bkind_is_some_constant(decode_restriction_kind(tempb_pku))); - (void)tempb_pku; - if (bpart != NULL && !tempb->deprecated && !b->deprecated && !eq_bindings(tempbpart, b, world)) { - if (warn) { - // set usingfailed=1 to avoid repeating this warning - // the owner will still be NULL, so it can be later imported or defined - tempb = jl_get_module_binding(m, var, 1); - tempbpart = jl_get_binding_partition(tempb, world); - jl_atomic_store_release(&tempbpart->restriction, encode_restriction(NULL, BINDING_KIND_FAILED)); - } - return NULL; - } - if (owner == NULL || !tempb->deprecated) { - owner = imp; - b = tempb; - bpart = tempbpart; - } - } + else if (strcmp(jl_symbol_name(var), "=>") == 0 && (kind == BINDING_KIND_IMPLICIT || kind == BINDING_KIND_EXPLICIT)) { + jl_module_t *from = jl_binding_dbgmodule(b, m, var); + jl_errorf("invalid method definition in %s: function %s.%s must be explicitly imported to be extended", + jl_symbol_name(m->name), from ? jl_symbol_name(from->name) : "", jl_symbol_name(var)); } - *from = owner; - return b; + return ownerb; } // for error message printing: look up the module that exported a binding to m as var @@ -559,100 +642,15 @@ static jl_binding_t *using_resolve_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b, jl_module_t *m, jl_sym_t *var) { jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (decode_restriction_kind(jl_atomic_load_relaxed(&bpart->restriction)) != BINDING_KIND_GLOBAL) { - // for implicitly imported globals, try to re-resolve it to find the module we got it from most directly - jl_module_t *from = NULL; - jl_binding_t *b2 = using_resolve_binding(m, var, &from, NULL, 0, jl_current_task->world_age); - if (b2) { - jl_binding_partition_t *b2part = jl_get_binding_partition(b2, jl_current_task->world_age); - if (eq_bindings(b2part, b, jl_current_task->world_age)) - return from; - // if we did not find it (or accidentally found a different one), ignore this - } + jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); + if (jl_bkind_is_some_import(decode_restriction_kind(pku))) { + return ((jl_binding_t*)decode_restriction_value(pku))->globalref->mod; } return m; } static void jl_binding_dep_message(jl_module_t *m, jl_sym_t *name, jl_binding_t *b); -// get binding for reading. might return NULL for unbound. -static jl_binding_t *jl_resolve_owner(jl_binding_t *b/*optional*/, jl_module_t *m, jl_sym_t *var, modstack_t *st, size_t world) -{ - if (b == NULL) - b = jl_get_module_binding(m, var, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); - jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); -retry: - if (decode_restriction_kind(pku) == BINDING_KIND_FAILED) - return NULL; - if (decode_restriction_kind(pku) == BINDING_KIND_DECLARED) { - return b; - } - if (decode_restriction_kind(pku) == BINDING_KIND_GUARD) { - jl_binding_t *b2 = NULL; - modstack_t top = { m, var, st }; - modstack_t *tmp = st; - for (; tmp != NULL; tmp = tmp->prev) { - if (tmp->m == m && tmp->var == var) { - // import cycle without finding actual location - return NULL; - } - } - jl_module_t *from = NULL; // for error message printing - b2 = using_resolve_binding(m, var, &from, &top, 1, world); - if (b2 == NULL) - return NULL; - assert(from); - JL_GC_PROMISE_ROOTED(from); // gc-analysis does not understand output parameters - JL_GC_PROMISE_ROOTED(b2); - if (b2->deprecated) { - if (jl_get_binding_value(b2) == jl_nothing) { - // silently skip importing deprecated values assigned to nothing (to allow later mutation) - return NULL; - } - } - // do a full import to prevent the result of this lookup from - // changing, for example if this var is assigned to later. - if (!jl_atomic_cmpswap(&bpart->restriction, &pku, encode_restriction((jl_value_t*)b2, BINDING_KIND_IMPLICIT))) - goto retry; - jl_gc_wb(bpart, b2); - if (b2->deprecated) { - b->deprecated = 1; // we will warn about this below, but we might want to warn at the use sites too - if (m != jl_main_module && m != jl_base_module && - jl_options.depwarn != JL_OPTIONS_DEPWARN_OFF) { - /* with #22763, external packages wanting to replace - deprecated Base bindings should simply export the new - binding */ - jl_printf(JL_STDERR, - "WARNING: using deprecated binding %s.%s in %s.\n", - jl_symbol_name(from->name), jl_symbol_name(var), - jl_symbol_name(m->name)); - jl_binding_dep_message(from, var, b2); - } - } - return b2; - } - jl_walk_binding_inplace(&b, &bpart, world); - return b; -} - -// get the current likely owner of binding when accessing m.var, without resolving the binding (it may change later) -JL_DLLEXPORT jl_binding_t *jl_binding_owner(jl_module_t *m, jl_sym_t *var) -{ - jl_binding_t *b = jl_get_module_binding(m, var, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_module_t *from = m; - jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); - if (decode_restriction_kind(pku) == BINDING_KIND_GUARD) { - b = using_resolve_binding(m, var, &from, NULL, 0, jl_current_task->world_age); - bpart = jl_get_binding_partition(b, jl_current_task->world_age); - } - pku = jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); - if (decode_restriction_kind(pku) != BINDING_KIND_GLOBAL && !jl_bkind_is_some_constant(decode_restriction_kind(pku))) - return NULL; - return b; -} - // get type of binding m.var, without resolving the binding JL_DLLEXPORT jl_value_t *jl_get_binding_type(jl_module_t *m, jl_sym_t *var) { @@ -675,7 +673,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_type(jl_module_t *m, jl_sym_t *var) JL_DLLEXPORT jl_binding_t *jl_get_binding(jl_module_t *m, jl_sym_t *var) { - return jl_resolve_owner(NULL, m, var, NULL, jl_current_task->world_age); + return jl_get_module_binding(m, var, 1); } JL_DLLEXPORT jl_binding_t *jl_get_binding_or_error(jl_module_t *m, jl_sym_t *var) @@ -762,81 +760,82 @@ static void jl_binding_dep_message(jl_module_t *m, jl_sym_t *name, jl_binding_t static void module_import_(jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici) { jl_binding_t *b = jl_get_binding(from, s); - if (b == NULL) { + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); + (void)pku; + if (b->deprecated) { + if (jl_get_binding_value(b) == jl_nothing) { + // silently skip importing deprecated values assigned to nothing (to allow later mutation) + return; + } + else if (to != jl_main_module && to != jl_base_module && + jl_options.depwarn != JL_OPTIONS_DEPWARN_OFF) { + /* with #22763, external packages wanting to replace + deprecated Base bindings should simply export the new + binding */ + jl_printf(JL_STDERR, + "WARNING: importing deprecated binding %s.%s into %s%s%s.\n", + jl_symbol_name(from->name), jl_symbol_name(s), + jl_symbol_name(to->name), + asname == s ? "" : " as ", + asname == s ? "" : jl_symbol_name(asname)); + jl_binding_dep_message(from, s, b); + } + } + + jl_binding_t *ownerb = b; + jl_binding_partition_t *ownerbpart = bpart; + jl_ptr_kind_union_t owner_pku = jl_walk_binding_inplace(&ownerb, &ownerbpart, jl_current_task->world_age); + + if (jl_bkind_is_some_guard(decode_restriction_kind(owner_pku))) { jl_printf(JL_STDERR, - "WARNING: could not import %s.%s into %s\n", + "WARNING: Imported binding %s.%s was undeclared at import time during import to %s.\n", jl_symbol_name(from->name), jl_symbol_name(s), jl_symbol_name(to->name)); } + + jl_binding_t *bto = jl_get_module_binding(to, asname, 1); + if (bto == b) { + // importing a binding on top of itself. harmless. + return; + } + JL_LOCK(&world_counter_lock); + size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; + jl_binding_partition_t *btopart = jl_get_binding_partition(bto, new_world); + jl_ptr_kind_union_t bto_pku = jl_atomic_load_relaxed(&btopart->restriction); + if (decode_restriction_kind(bto_pku) == BINDING_KIND_GUARD || + decode_restriction_kind(bto_pku) == BINDING_KIND_IMPLICIT || + decode_restriction_kind(bto_pku) == BINDING_KIND_FAILED) { + + jl_binding_partition_t *new_bpart = jl_replace_binding_locked(bto, btopart, (jl_value_t*)b, (explici != 0) ? BINDING_KIND_IMPORTED : BINDING_KIND_EXPLICIT, new_world); + if (jl_atomic_load_relaxed(&new_bpart->max_world) == ~(size_t)0) + jl_add_binding_backedge(b, (jl_value_t*)bto); + jl_atomic_store_release(&jl_world_counter, new_world); + } else { - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); - assert(decode_restriction_kind(pku) == BINDING_KIND_GLOBAL || decode_restriction_kind(pku) == BINDING_KIND_DECLARED || jl_bkind_is_some_constant(decode_restriction_kind(pku))); - (void)pku; - if (b->deprecated) { - if (jl_get_binding_value(b) == jl_nothing) { - // silently skip importing deprecated values assigned to nothing (to allow later mutation) - return; - } - else if (to != jl_main_module && to != jl_base_module && - jl_options.depwarn != JL_OPTIONS_DEPWARN_OFF) { - /* with #22763, external packages wanting to replace - deprecated Base bindings should simply export the new - binding */ - jl_printf(JL_STDERR, - "WARNING: importing deprecated binding %s.%s into %s%s%s.\n", - jl_symbol_name(from->name), jl_symbol_name(s), - jl_symbol_name(to->name), - asname == s ? "" : " as ", - asname == s ? "" : jl_symbol_name(asname)); - jl_binding_dep_message(from, s, b); + if (eq_bindings(bpart, bto, new_world)) { + // already imported - potentially upgrade _EXPLICIT to _IMPORTED + if (decode_restriction_kind(bto_pku) == BINDING_KIND_EXPLICIT && explici != 0) { + jl_replace_binding_locked(bto, btopart, (jl_value_t*)b, BINDING_KIND_IMPORTED, new_world); + jl_atomic_store_release(&jl_world_counter, new_world); } } - - jl_binding_t *bto = jl_get_module_binding(to, asname, 1); - if (bto == b) { - // importing a binding on top of itself. harmless. - return; - } - jl_binding_partition_t *btopart = jl_get_binding_partition(bto, jl_current_task->world_age); - jl_ptr_kind_union_t bto_pku = jl_atomic_load_relaxed(&btopart->restriction); -retry: - if (decode_restriction_kind(bto_pku) == BINDING_KIND_GUARD || - decode_restriction_kind(bto_pku) == BINDING_KIND_IMPLICIT || - decode_restriction_kind(bto_pku) == BINDING_KIND_FAILED) { - - jl_ptr_kind_union_t new_pku = encode_restriction((jl_value_t*)b, (explici != 0) ? BINDING_KIND_IMPORTED : BINDING_KIND_EXPLICIT); - if (!jl_atomic_cmpswap(&btopart->restriction, &bto_pku, new_pku)) - goto retry; - jl_gc_wb(btopart, b); - bto->deprecated |= b->deprecated; // we already warned about this above, but we might want to warn at the use sites too + else if (jl_bkind_is_some_import(decode_restriction_kind(bto_pku))) { + // already imported from somewhere else + jl_printf(JL_STDERR, + "WARNING: ignoring conflicting import of %s.%s into %s\n", + jl_symbol_name(from->name), jl_symbol_name(s), + jl_symbol_name(to->name)); } else { - if (eq_bindings(bpart, bto, jl_current_task->world_age)) { - // already imported - potentially upgrade to _IMPORTED or _EXPLICIT - if (jl_bkind_is_some_import(decode_restriction_kind(bto_pku))) { - jl_ptr_kind_union_t new_pku = encode_restriction(decode_restriction_value(bto_pku), (explici != 0) ? BINDING_KIND_IMPORTED : BINDING_KIND_EXPLICIT); - if (!jl_atomic_cmpswap(&btopart->restriction, &bto_pku, new_pku)) - goto retry; - // No wb, because the value is unchanged - } - } - else if (jl_bkind_is_some_import(decode_restriction_kind(bto_pku))) { - // already imported from somewhere else - jl_printf(JL_STDERR, - "WARNING: ignoring conflicting import of %s.%s into %s\n", - jl_symbol_name(from->name), jl_symbol_name(s), - jl_symbol_name(to->name)); - } - else { - // conflict with name owned by destination module - jl_printf(JL_STDERR, - "WARNING: import of %s.%s into %s conflicts with an existing identifier; ignored.\n", - jl_symbol_name(from->name), jl_symbol_name(s), - jl_symbol_name(to->name)); - } + // conflict with name owned by destination module + jl_printf(JL_STDERR, + "WARNING: import of %s.%s into %s conflicts with an existing identifier; ignored.\n", + jl_symbol_name(from->name), jl_symbol_name(s), + jl_symbol_name(to->name)); } } + JL_UNLOCK(&world_counter_lock); } JL_DLLEXPORT void jl_module_import(jl_module_t *to, jl_module_t *from, jl_sym_t *s) @@ -859,57 +858,85 @@ JL_DLLEXPORT void jl_module_use_as(jl_module_t *to, jl_module_t *from, jl_sym_t module_import_(to, from, asname, s, 0); } +void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to) +{ + JL_LOCK(&from->lock); + if (from->usings_backedges == jl_nothing) { + from->usings_backedges = (jl_value_t*)jl_alloc_vec_any(0); + jl_gc_wb(from, from->usings_backedges); + } + jl_array_ptr_1d_push((jl_array_t*)from->usings_backedges, (jl_value_t*)to); + JL_UNLOCK(&from->lock); +} + JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from) { if (to == from) return; + JL_LOCK(&world_counter_lock); JL_LOCK(&to->lock); for (size_t i = 0; i < module_usings_length(to); i++) { if (from == module_usings_getmod(to, i)) { JL_UNLOCK(&to->lock); + JL_UNLOCK(&world_counter_lock); return; } } + + size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; struct _jl_module_using new_item = { .mod = from, - .min_world = 0, - .max_world = (size_t)-1 + .min_world = new_world, + .max_world = ~(size_t)0 }; arraylist_grow(&to->usings, sizeof(struct _jl_module_using)/sizeof(void*)); memcpy(&to->usings.items[to->usings.len-3], &new_item, sizeof(struct _jl_module_using)); jl_gc_wb(to, from); + JL_UNLOCK(&to->lock); - // print a warning if something visible via this "using" conflicts with - // an existing identifier. note that an identifier added later may still - // silently override a "using" name. see issue #2054. + // Go through all exported bindings. If we have a binding for this in the + // importing module and it is some import or guard, we need to recompute + // it. jl_svec_t *table = jl_atomic_load_relaxed(&from->bindings); for (size_t i = 0; i < jl_svec_len(table); i++) { jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i); if ((void*)b == jl_nothing) break; - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); - if (b->exportp && (decode_restriction_kind(pku) == BINDING_KIND_GLOBAL || decode_restriction_kind(pku) == BINDING_KIND_IMPORTED)) { + if (b->exportp) { jl_sym_t *var = b->globalref->name; jl_binding_t *tob = jl_get_module_binding(to, var, 0); if (tob) { - jl_binding_partition_t *tobpart = jl_get_binding_partition(tob, jl_current_task->world_age); - jl_ptr_kind_union_t tobpku = jl_walk_binding_inplace(&tob, &tobpart, jl_current_task->world_age); - if (tob && decode_restriction_kind(tobpku) != BINDING_KIND_GUARD && - // don't warn for conflicts with the module name itself. - // see issue #4715 - var != to->name && - !eq_bindings(tobpart, b, jl_current_task->world_age)) { - jl_printf(JL_STDERR, - "WARNING: using %s.%s in module %s conflicts with an existing identifier.\n", - jl_symbol_name(from->name), jl_symbol_name(var), - jl_symbol_name(to->name)); + jl_binding_partition_t *tobpart = jl_get_binding_partition(tob, new_world); + jl_ptr_kind_union_t tobpku = jl_atomic_load_relaxed(&tobpart->restriction); + enum jl_partition_kind kind = decode_restriction_kind(tobpku); + if (kind == BINDING_KIND_IMPLICIT || jl_bkind_is_some_guard(kind)) { + jl_replace_binding_locked(tob, tobpart, NULL, BINDING_KIND_IMPLICIT_RECOMPUTE, new_world); } } } table = jl_atomic_load_relaxed(&from->bindings); } + + jl_add_usings_backedge(from, to); + + jl_atomic_store_release(&jl_world_counter, new_world); + JL_UNLOCK(&world_counter_lock); +} + +JL_DLLEXPORT jl_value_t *jl_get_module_usings_backedges(jl_module_t *m) +{ + // We assume the caller holds the world_counter_lock, which is the only place we set this + // TODO: We may want to make this more precise with the module lock + return m->usings_backedges; +} + +JL_DLLEXPORT jl_value_t *jl_get_module_binding_or_nothing(jl_module_t *m, jl_sym_t *s) +{ + jl_binding_t *b = jl_get_module_binding(m, s, 0); + if (!b) + return jl_nothing; + return (jl_value_t*)b; } JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported) @@ -941,9 +968,6 @@ JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import) // u if (!bpart || jl_bkind_is_some_import(decode_restriction_kind(pku))) return 0; } else { - if (jl_bkind_is_some_guard(decode_restriction_kind(pku))) { - jl_resolve_owner(b, b->globalref->mod, b->globalref->name, NULL, jl_current_task->world_age); - } pku = jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); } if (jl_bkind_is_some_guard(decode_restriction_kind(pku))) @@ -974,16 +998,6 @@ JL_DLLEXPORT int jl_module_public_p(jl_module_t *m, jl_sym_t *var) return b && b->publicp; } -JL_DLLEXPORT int jl_binding_resolved_p(jl_module_t *m, jl_sym_t *var) -{ - jl_binding_t *b = jl_get_module_binding(m, var, 0); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (!bpart) - return 0; - enum jl_partition_kind kind = decode_restriction_kind(jl_atomic_load_relaxed(&bpart->restriction)); - return kind == BINDING_KIND_DECLARED || !jl_bkind_is_some_guard(kind); -} - uint_t bindingkey_hash(size_t idx, jl_value_t *data) { jl_binding_t *b = (jl_binding_t*)jl_svecref(data, idx); // This must always happen inside the lock @@ -1052,7 +1066,8 @@ JL_DLLEXPORT jl_binding_t *jl_get_module_binding(jl_module_t *m, jl_sym_t *var, JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr) { jl_binding_t *b = gr->binding; - b = jl_resolve_owner(b, gr->mod, gr->name, NULL, jl_current_task->world_age); + if (!b) + b = jl_get_module_binding(gr->mod, gr->name, 1); // ignores b->deprecated return b == NULL ? NULL : jl_get_binding_value(b); } @@ -1079,23 +1094,27 @@ JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var // this function is mostly only used during initialization, so the data races here are not too important to us jl_binding_t *bp = jl_get_module_binding(m, var, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(bp, jl_current_task->world_age); - assert(jl_bkind_is_some_guard(decode_restriction_kind(jl_atomic_load_relaxed(&bpart->restriction)))); + bpart->min_world = 0; + jl_atomic_store_release(&bpart->max_world, ~(size_t)0); jl_atomic_store_release(&bpart->restriction, encode_restriction(val, BINDING_KIND_CONST)); jl_gc_wb(bpart, val); } -void jl_invalidate_binding_refs(jl_globalref_t *ref, jl_binding_partition_t *invalidated_bpart, size_t new_world) +void jl_invalidate_binding_refs(jl_globalref_t *ref, jl_binding_partition_t *invalidated_bpart, jl_binding_partition_t *new_bpart, size_t new_world) { static jl_value_t *invalidate_code_for_globalref = NULL; if (invalidate_code_for_globalref == NULL && jl_base_module != NULL) invalidate_code_for_globalref = jl_get_global(jl_base_module, jl_symbol("invalidate_code_for_globalref!")); if (!invalidate_code_for_globalref) jl_error("Binding invalidation is not permitted during bootstrap."); - if (jl_generating_output()) - jl_error("Binding invalidation is not permitted during image generation."); - jl_value_t *boxed_world = jl_box_ulong(new_world); - JL_GC_PUSH1(&boxed_world); - jl_call3((jl_function_t*)invalidate_code_for_globalref, (jl_value_t*)ref, (jl_value_t*)invalidated_bpart, boxed_world); + jl_value_t **fargs; + JL_GC_PUSHARGS(fargs, 5); + fargs[0] = (jl_function_t*)invalidate_code_for_globalref; + fargs[1] = (jl_value_t*)ref; + fargs[2] = (jl_value_t*)invalidated_bpart; + fargs[3] = (jl_value_t*)new_bpart; + fargs[4] = jl_box_ulong(new_world); + jl_apply(fargs, 5); JL_GC_POP(); } @@ -1131,50 +1150,90 @@ JL_DLLEXPORT void jl_maybe_add_binding_backedge(jl_globalref_t *gr, jl_module_t jl_add_binding_backedge(b, edge); } -JL_DLLEXPORT void jl_disable_binding(jl_globalref_t *gr) +JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked(jl_binding_t *b, + jl_binding_partition_t *old_bpart, jl_value_t *restriction_val, enum jl_partition_kind kind, size_t new_world) { - jl_binding_t *b = gr->binding; - b = jl_resolve_owner(b, gr->mod, gr->name, NULL, jl_current_task->world_age); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + assert(jl_atomic_load_relaxed(&b->partitions) == old_bpart); + jl_atomic_store_release(&old_bpart->max_world, new_world-1); + jl_binding_partition_t *new_bpart = new_binding_partition(); + new_bpart->min_world = new_world; + if (kind == BINDING_KIND_IMPLICIT_RECOMPUTE) { + assert(!restriction_val); + jl_check_new_binding_implicit(new_bpart, b, NULL, new_world); + } + else + jl_atomic_store_relaxed(&new_bpart->restriction, encode_restriction(restriction_val, kind)); + jl_atomic_store_relaxed(&new_bpart->next, old_bpart); + jl_gc_wb(new_bpart, old_bpart); - if (decode_restriction_kind(jl_atomic_load_relaxed(&bpart->restriction)) == BINDING_KIND_GUARD) { - // Already guard - return; + jl_atomic_store_release(&b->partitions, new_bpart); + jl_gc_wb(b, new_bpart); + + + if (jl_typeinf_world != 1) { + jl_task_t *ct = jl_current_task; + size_t last_world = ct->world_age; + ct->world_age = jl_typeinf_world; + jl_invalidate_binding_refs(b->globalref, old_bpart, new_bpart, new_world-1); + ct->world_age = last_world; } + return new_bpart; +} + +JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding(jl_binding_t *b, + jl_binding_partition_t *old_bpart, jl_value_t *restriction_val, enum jl_partition_kind kind) { + JL_LOCK(&world_counter_lock); - jl_task_t *ct = jl_current_task; - size_t last_world = ct->world_age; - size_t new_max_world = jl_atomic_load_acquire(&jl_world_counter); - jl_atomic_store_release(&bpart->max_world, new_max_world); - ct->world_age = jl_typeinf_world; - jl_invalidate_binding_refs(gr, bpart, new_max_world); - ct->world_age = last_world; - jl_atomic_store_release(&jl_world_counter, new_max_world + 1); + + if (jl_atomic_load_relaxed(&b->partitions) != old_bpart) { + JL_UNLOCK(&world_counter_lock); + return NULL; + } + + size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; + jl_binding_partition_t *bpart = jl_replace_binding_locked(b, old_bpart, restriction_val, kind, new_world); + if (bpart && bpart->min_world == new_world) + jl_atomic_store_release(&jl_world_counter, new_world); + JL_UNLOCK(&world_counter_lock); + return bpart; } JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr) { jl_binding_t *b = gr->binding; - b = jl_resolve_owner(b, gr->mod, gr->name, NULL, jl_current_task->world_age); + if (!b) + b = jl_get_module_binding(gr->mod, gr->name, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); if (!bpart) return 0; return jl_bkind_is_some_constant(decode_restriction_kind(jl_atomic_load_relaxed(&bpart->restriction))); } -JL_DLLEXPORT void jl_force_binding_resolution(jl_globalref_t *gr, size_t world) +JL_DLLEXPORT void jl_disable_binding(jl_globalref_t *gr) { jl_binding_t *b = gr->binding; - jl_resolve_owner(b, gr->mod, gr->name, NULL, world); + if (!b) + b = jl_get_module_binding(gr->mod, gr->name, 1); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + + if (decode_restriction_kind(jl_atomic_load_relaxed(&bpart->restriction)) == BINDING_KIND_GUARD) { + // Already guard + return; + } + + for (;;) + if (jl_replace_binding(b, bpart, NULL, BINDING_KIND_GUARD)) + break; } JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_binding(m, var); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - return b && jl_bkind_is_some_constant(decode_restriction_kind(jl_atomic_load_relaxed(&bpart->restriction))); + jl_ptr_kind_union_t pku = jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + return b && jl_bkind_is_some_constant(decode_restriction_kind(pku)); } // set the deprecated flag for a binding: @@ -1188,12 +1247,9 @@ JL_DLLEXPORT void jl_deprecate_binding(jl_module_t *m, jl_sym_t *var, int flag) JL_DLLEXPORT int jl_is_binding_deprecated(jl_module_t *m, jl_sym_t *var) { - if (jl_binding_resolved_p(m, var)) { - // XXX: this only considers if the original is deprecated, not this precise binding - jl_binding_t *b = jl_get_binding(m, var); - return b && b->deprecated; - } - return 0; + // XXX: this only considers if the original is deprecated, not this precise binding + jl_binding_t *b = jl_get_binding(m, var); + return b && b->deprecated; } void jl_binding_deprecation_warning(jl_module_t *m, jl_sym_t *s, jl_binding_t *b) @@ -1229,22 +1285,19 @@ jl_value_t *jl_check_binding_assign_value(jl_binding_t *b JL_PROPAGATES_ROOT, jl JL_GC_PUSH1(&rhs); // callee-rooted jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); - assert(!jl_bkind_is_some_guard(decode_restriction_kind(pku)) && !jl_bkind_is_some_import(decode_restriction_kind(pku))); - if (jl_bkind_is_some_constant(decode_restriction_kind(pku))) { + enum jl_partition_kind kind = decode_restriction_kind(pku); + if (jl_bkind_is_some_constant(kind)) { jl_value_t *old = decode_restriction_value(pku); JL_GC_PROMISE_ROOTED(old); if (jl_egal(rhs, old)) { JL_GC_POP(); return NULL; } - if (jl_typeof(rhs) == jl_typeof(old)) - jl_errorf("invalid redefinition of constant %s.%s. This redefinition may be permitted using the `const` keyword.", - jl_symbol_name(mod->name), jl_symbol_name(var)); - else - jl_errorf("invalid redefinition of constant %s.%s.", - jl_symbol_name(mod->name), jl_symbol_name(var)); + jl_errorf("invalid assignment to constant %s.%s. This redefinition may be permitted using the `const` keyword.", + jl_symbol_name(mod->name), jl_symbol_name(var)); } - jl_value_t *old_ty = decode_restriction_value(pku); + assert(kind == BINDING_KIND_DECLARED || kind == BINDING_KIND_GLOBAL); + jl_value_t *old_ty = kind == BINDING_KIND_DECLARED ? (jl_value_t*)jl_any_type : decode_restriction_value(pku); JL_GC_PROMISE_ROOTED(old_ty); if (old_ty != (jl_value_t*)jl_any_type && jl_typeof(rhs) != old_ty) { if (!jl_isa(rhs, old_ty)) @@ -1285,7 +1338,7 @@ JL_DLLEXPORT jl_value_t *jl_checked_modify(jl_binding_t *b, jl_module_t *mod, jl jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); assert(!jl_bkind_is_some_guard(decode_restriction_kind(pku)) && !jl_bkind_is_some_import(decode_restriction_kind(pku))); if (jl_bkind_is_some_constant(decode_restriction_kind(pku))) - jl_errorf("invalid redefinition of constant %s.%s", + jl_errorf("invalid assignment to constant %s.%s", jl_symbol_name(mod->name), jl_symbol_name(var)); jl_value_t *ty = decode_restriction_value(pku); JL_GC_PROMISE_ROOTED(ty); @@ -1422,7 +1475,7 @@ JL_DLLEXPORT void jl_clear_implicit_imports(jl_module_t *m) break; jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); if (decode_restriction_kind(jl_atomic_load_relaxed(&bpart->restriction)) == BINDING_KIND_IMPLICIT) { - jl_atomic_store_relaxed(&bpart->restriction, encode_restriction(NULL, BINDING_KIND_GUARD)); + jl_atomic_store_relaxed(&b->partitions, NULL); } } JL_UNLOCK(&m->lock); diff --git a/src/staticdata.c b/src/staticdata.c index ec77c771bd880..9eb2922380c63 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -827,6 +827,8 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ for (size_t i = 0; i < module_usings_length(m); i++) { jl_queue_for_serialization(s, module_usings_getmod(m, i)); } + + jl_queue_for_serialization(s, m->usings_backedges); } // Anything that requires uniquing or fixing during deserialization needs to be "toplevel" @@ -1299,10 +1301,14 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t newm->file = NULL; arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, file))); arraylist_push(&s->relocs_list, (void*)backref_id(s, m->file, s->link_ids_relocs)); + newm->usings_backedges = NULL; + arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings_backedges))); + arraylist_push(&s->relocs_list, (void*)backref_id(s, m->usings_backedges, s->link_ids_relocs)); // write out the usings list memset(&newm->usings._space, 0, sizeof(newm->usings._space)); if (m->usings.items == &m->usings._space[0]) { + newm->usings.items = &newm->usings._space[0]; // Push these relocations here, to keep them in order. This pairs with the `newm->usings.items = ` below. arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings.items))); arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item)); @@ -1314,9 +1320,9 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t newm_data->min_world = data->min_world; newm_data->max_world = data->max_world; if (s->incremental) { - if (data->max_world != (size_t)-1) + if (data->max_world != ~(size_t)0) newm_data->max_world = 0; - newm_data->min_world = 0; + newm_data->min_world = jl_require_world; } arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings._space[3*i]))); arraylist_push(&s->relocs_list, (void*)backref_id(s, data->mod, s->link_ids_relocs)); @@ -1331,8 +1337,14 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t for (i = 0; i < module_usings_length(m); i++) { struct _jl_module_using *data = module_usings_getidx(m, i); write_pointerfield(s, (jl_value_t*)data->mod); - write_uint(s->s, data->min_world); - write_uint(s->s, data->max_world); + if (s->incremental) { + // TODO: Drop dead ones entirely? + write_uint(s->s, jl_require_world); + write_uint(s->s, data->max_world == ~(size_t)0 ? ~(size_t)0 : 1); + } else { + write_uint(s->s, data->min_world); + write_uint(s->s, data->max_world); + } static_assert(sizeof(struct _jl_module_using) == 3*sizeof(void*), "_jl_module_using mismatch"); tot += sizeof(struct _jl_module_using); } @@ -3492,6 +3504,73 @@ extern void export_jl_small_typeof(void); // into the native code of the image. See https://github.com/JuliaLang/julia/pull/52123#issuecomment-1959965395. int IMAGE_NATIVE_CODE_TAINTED = 0; +// TODO: This should possibly be in Julia +static void jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t *bpart, size_t mod_idx) +{ + + if (jl_atomic_load_relaxed(&bpart->max_world) != ~(size_t)0) + return; + jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); + enum jl_partition_kind kind = decode_restriction_kind(pku); + if (!jl_bkind_is_some_import(kind)) + return; + jl_binding_t *imported_binding = (jl_binding_t*)decode_restriction_value(pku); + jl_binding_partition_t *latest_imported_bpart = jl_atomic_load_relaxed(&imported_binding->partitions); + if (!latest_imported_bpart) + return; + if (kind == BINDING_KIND_IMPLICIT || kind == BINDING_KIND_FAILED) { + jl_check_new_binding_implicit(bpart, b, NULL, jl_atomic_load_relaxed(&jl_world_counter)); + if (bpart->min_world > jl_require_world) + goto invalidated; + } + if (latest_imported_bpart->min_world <= bpart->min_world) { + // Imported binding is still valid + if ((kind == BINDING_KIND_EXPLICIT || kind == BINDING_KIND_IMPORTED) && + external_blob_index((jl_value_t*)imported_binding) != mod_idx) { + jl_add_binding_backedge(imported_binding, (jl_value_t*)b); + } + return; + } + else { + // Binding partition was invalidated + assert(bpart->min_world == jl_require_world); + bpart->min_world = latest_imported_bpart->min_world; + } +invalidated: + // We need to go through and re-validate any bindings in the same image that + // may have imported us. + if (b->backedges) { + for (size_t i = 0; i < jl_array_len(b->backedges); i++) { + jl_value_t *edge = jl_array_ptr_ref(b->backedges, i); + if (!jl_is_binding(edge)) + continue; + jl_binding_t *bedge = (jl_binding_t*)edge; + if (!jl_atomic_load_relaxed(&bedge->partitions)) + continue; + jl_validate_binding_partition(bedge, jl_atomic_load_relaxed(&bedge->partitions), mod_idx); + } + } + if (b->exportp) { + jl_module_t *mod = b->globalref->mod; + jl_sym_t *name = b->globalref->name; + JL_LOCK(&mod->lock); + if (mod->usings_backedges) { + for (size_t i = 0; i < jl_array_len(mod->usings_backedges); i++) { + jl_module_t *edge = (jl_module_t*)jl_array_ptr_ref(mod->usings_backedges, i); + jl_binding_t *importee = jl_get_module_binding(edge, name, 0); + if (!importee) + continue; + if (!jl_atomic_load_relaxed(&importee->partitions)) + continue; + JL_UNLOCK(&mod->lock); + jl_validate_binding_partition(importee, jl_atomic_load_relaxed(&importee->partitions), mod_idx); + JL_LOCK(&mod->lock); + } + } + JL_UNLOCK(&mod->lock); + } +} + static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_array_t *depmods, uint64_t checksum, /* outputs */ jl_array_t **restored, jl_array_t **init_order, jl_array_t **extext_methods, jl_array_t **internal_methods, @@ -3926,6 +4005,16 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl memcpy(newitems, mod->usings.items, mod->usings.len * sizeof(void*)); mod->usings.items = newitems; } + size_t mod_idx = external_blob_index((jl_value_t*)mod); + if (s.incremental) { + // Rebuild cross-image usings backedges + for (size_t i = 0; i < module_usings_length(mod); ++i) { + struct _jl_module_using *data = module_usings_getidx(mod, i); + if (external_blob_index((jl_value_t*)data->mod) != mod_idx) { + jl_add_usings_backedge(data->mod, mod); + } + } + } // Move the binding bits back to their correct place #ifdef _P64 jl_svec_t *table = jl_atomic_load_relaxed(&mod->bindings); @@ -3935,18 +4024,41 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl continue; jl_binding_partition_t *bpart = jl_atomic_load_relaxed(&b->partitions); while (bpart) { - jl_atomic_store_relaxed(&bpart->restriction, - encode_restriction((jl_value_t*)jl_atomic_load_relaxed(&bpart->restriction), bpart->reserved)); + jl_ptr_kind_union_t pku = encode_restriction( + (jl_value_t*)jl_atomic_load_relaxed(&bpart->restriction), + (enum jl_partition_kind)bpart->reserved); + jl_atomic_store_relaxed(&bpart->restriction, pku); bpart->reserved = 0; bpart = jl_atomic_load_relaxed(&bpart->next); } } + #endif } else { abort(); } } + if (s.incremental) { + // This needs to be done in a second pass after the binding partitions + // have the proper ABI again. + for (size_t i = 0; i < s.fixup_objs.len; i++) { + uintptr_t item = (uintptr_t)s.fixup_objs.items[i]; + jl_value_t *obj = (jl_value_t*)(image_base + item); + if (jl_is_module(obj)) { + jl_module_t *mod = (jl_module_t*)obj; + size_t mod_idx = external_blob_index((jl_value_t*)mod); + jl_svec_t *table = jl_atomic_load_relaxed(&mod->bindings); + for (size_t i = 0; i < jl_svec_len(table); i++) { + jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i); + if ((jl_value_t*)b == jl_nothing) + continue; + jl_binding_partition_t *bpart = jl_atomic_load_relaxed(&b->partitions); + jl_validate_binding_partition(b, bpart, mod_idx); + } + } + } + } arraylist_free(&s.fixup_types); arraylist_free(&s.fixup_objs); diff --git a/src/toplevel.c b/src/toplevel.c index cdd390b9b49ed..321ef8c79dac0 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -303,7 +303,7 @@ static jl_value_t *jl_eval_dot_expr(jl_module_t *m, jl_value_t *x, jl_value_t *f } extern void check_safe_newbinding(jl_module_t *m, jl_sym_t *var); -void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type) { +void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type, int strong) { // create uninitialized mutable binding for "global x" decl sometimes or probably jl_module_t *gm; jl_sym_t *gs; @@ -321,42 +321,58 @@ void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type) { size_t new_world = jl_atomic_load_relaxed(&jl_world_counter) + 1; jl_binding_t *b = jl_get_module_binding(gm, gs, 1); jl_binding_partition_t *bpart = NULL; - jl_ptr_kind_union_t new_pku = encode_restriction(set_type, set_type == NULL ? BINDING_KIND_DECLARED : BINDING_KIND_GLOBAL); + if (!strong && set_type) + jl_error("Weak global definitions cannot have types"); + enum jl_partition_kind new_kind = strong ? BINDING_KIND_GLOBAL : BINDING_KIND_DECLARED; + jl_value_t *global_type = set_type; + if (strong && !global_type) + global_type = (jl_value_t*)jl_any_type; while (1) { bpart = jl_get_binding_partition(b, new_world); jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); - if (decode_restriction_kind(pku) != BINDING_KIND_GLOBAL) { - if (jl_bkind_is_some_guard(decode_restriction_kind(pku))) { - if (decode_restriction_kind(pku) == BINDING_KIND_DECLARED && !set_type) - goto done; + enum jl_partition_kind kind = decode_restriction_kind(pku); + if (kind != BINDING_KIND_GLOBAL) { + if (jl_bkind_is_some_guard(kind) || kind == BINDING_KIND_DECLARED || kind == BINDING_KIND_IMPLICIT) { + if (decode_restriction_kind(pku) == new_kind) { + if (!set_type) + goto done; + goto check_type; + } check_safe_newbinding(gm, gs); - if (jl_atomic_cmpswap(&bpart->restriction, &pku, new_pku)) { - break; + if (bpart->min_world == new_world) { + if (jl_atomic_cmpswap(&bpart->restriction, &pku, encode_restriction(global_type, new_kind))) { + break; + } + if (set_type) + jl_gc_wb(bpart, set_type); + continue; + } else { + jl_replace_binding_locked(b, bpart, global_type, new_kind, new_world); } - continue; + break; } else if (set_type) { if (jl_bkind_is_some_constant(decode_restriction_kind(pku))) { - jl_errorf("cannot set type for imported constant %s.%s.", + jl_errorf("cannot set type for constant %s.%s.", jl_symbol_name(gm->name), jl_symbol_name(gs)); } else { - jl_errorf("cannot set type for imported global %s.%s.", + jl_errorf("cannot set type for imported binding %s.%s.", jl_symbol_name(gm->name), jl_symbol_name(gs)); } } } - if (!set_type) - goto done; - jl_value_t *old_ty = decode_restriction_value(pku); - JL_GC_PROMISE_ROOTED(old_ty); - if (!jl_types_equal(set_type, old_ty)) { - jl_errorf("cannot set type for global %s.%s. It already has a value or is already set to a different type.", - jl_symbol_name(gm->name), jl_symbol_name(gs)); + if (set_type) + { +check_type: ; + jl_value_t *old_ty = decode_restriction_value(pku); + JL_GC_PROMISE_ROOTED(old_ty); + if (!jl_types_equal(set_type, old_ty)) { + jl_errorf("cannot set type for global %s.%s. It already has a value or is already set to a different type.", + jl_symbol_name(gm->name), jl_symbol_name(gs)); + } + } goto done; } - if (set_type) - jl_gc_wb(bpart, set_type); - bpart->min_world = new_world; jl_atomic_store_release(&jl_world_counter, new_world); done: JL_UNLOCK(&world_counter_lock); @@ -367,7 +383,7 @@ void jl_eval_global_expr(jl_module_t *m, jl_expr_t *ex, int set_type) size_t i, l = jl_array_nrows(ex->args); for (i = 0; i < l; i++) { jl_value_t *arg = jl_exprarg(ex, i); - jl_declare_global(m, arg, NULL); + jl_declare_global(m, arg, NULL, 0); } } @@ -431,10 +447,8 @@ static void expr_attributes(jl_value_t *v, jl_array_t *body, int *has_ccall, int if (jl_is_globalref(f)) { jl_module_t *mod = jl_globalref_mod(f); jl_sym_t *name = jl_globalref_name(f); - if (jl_binding_resolved_p(mod, name)) { - jl_binding_t *b = jl_get_binding(mod, name); - called = jl_get_binding_value_if_const(b); - } + jl_binding_t *b = jl_get_binding(mod, name); + called = jl_get_binding_value_if_const(b); } else if (jl_is_quotenode(f)) { called = jl_quotenode_value(f); @@ -666,7 +680,8 @@ static void import_module(jl_module_t *JL_NONNULL m, jl_module_t *import, jl_sym jl_binding_t *b = jl_get_module_binding(m, name, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); - if (decode_restriction_kind(pku) != BINDING_KIND_GUARD && decode_restriction_kind(pku) != BINDING_KIND_FAILED) { + enum jl_partition_kind kind = decode_restriction_kind(pku); + if (kind != BINDING_KIND_GUARD && kind != BINDING_KIND_FAILED && kind != BINDING_KIND_DECLARED && kind != BINDING_KIND_IMPLICIT) { // Unlike regular constant declaration, we allow this as long as we eventually end up at a constant. pku = jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); if (decode_restriction_kind(pku) == BINDING_KIND_CONST || decode_restriction_kind(pku) == BINDING_KIND_BACKDATED_CONST || decode_restriction_kind(pku) == BINDING_KIND_CONST_IMPORT) { @@ -747,72 +762,72 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( if (!b) { b = jl_get_module_binding(mod, var, 1); } + jl_binding_partition_t *new_bpart = NULL; jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); - int did_warn = 0; - while (1) { + while (!new_bpart) { enum jl_partition_kind kind = decode_restriction_kind(pku); if (jl_bkind_is_some_constant(kind)) { if (!val) { + new_bpart = bpart; break; } jl_value_t *old = decode_restriction_value(pku); JL_GC_PROMISE_ROOTED(old); - if (jl_egal(val, old)) + if (jl_egal(val, old)) { + new_bpart = bpart; break; - if (!did_warn) { - if (jl_typeof(val) != jl_typeof(old) || jl_is_type(val) || jl_is_module(val)) - jl_errorf("invalid redefinition of constant %s.%s", - jl_symbol_name(mod->name), - jl_symbol_name(var)); - else - jl_safe_printf("WARNING: redefinition of constant %s.%s. This may fail, cause incorrect answers, or produce other errors.\n", - jl_symbol_name(mod->name), - jl_symbol_name(var)); - did_warn = 1; - } - if (new_world > bpart->min_world) { - // TODO: Invoke invalidation logic here - jl_atomic_store_relaxed(&bpart->max_world, new_world - 1); - bpart = jl_get_binding_partition(b, new_world); - pku = jl_atomic_load_relaxed(&bpart->restriction); - } - } else if (!jl_bkind_is_some_guard(decode_restriction_kind(pku))) { - if (jl_bkind_is_some_import(decode_restriction_kind(pku))) { - jl_errorf("cannot declare %s.%s constant; it was already declared as an import", - jl_symbol_name(mod->name), jl_symbol_name(var)); - } else { - jl_errorf("cannot declare %s.%s constant; it was already declared global", - jl_symbol_name(mod->name), jl_symbol_name(var)); } + } else if (jl_bkind_is_some_import(kind) && kind != BINDING_KIND_IMPLICIT) { + jl_errorf("cannot declare %s.%s constant; it was already declared as an import", + jl_symbol_name(mod->name), jl_symbol_name(var)); + } else if (kind == BINDING_KIND_GLOBAL) { + jl_errorf("cannot declare %s.%s constant; it was already declared global", + jl_symbol_name(mod->name), jl_symbol_name(var)); } - if (!jl_atomic_cmpswap(&bpart->restriction, &pku, encode_restriction(val, constant_kind))) { - continue; + if (bpart->min_world == new_world) { + if (!jl_atomic_cmpswap(&bpart->restriction, &pku, encode_restriction(val, constant_kind))) { + continue; + } + jl_gc_wb(bpart, val); + new_bpart = bpart; + } else { + new_bpart = jl_replace_binding_locked(b, bpart, val, constant_kind, new_world); } - jl_gc_wb(bpart, val); - size_t prev_min_world = bpart->min_world; - bpart->min_world = new_world; - int need_backdate = 0; - if (new_world && val) { - if (prev_min_world == 0) { - need_backdate = 1; - } else if (kind == BINDING_KIND_DECLARED) { - jl_binding_partition_t *prev_bpart = jl_get_binding_partition(b, prev_min_world-1); + int need_backdate = new_world && val; + if (need_backdate) { + // We will backdate as long as this partition was never explicitly + // declared const, global, or imported. + jl_binding_partition_t *prev_bpart = bpart; + for (;;) { jl_ptr_kind_union_t prev_pku = jl_atomic_load_relaxed(&prev_bpart->restriction); - if (prev_bpart->min_world == 0 && decode_restriction_kind(prev_pku) == BINDING_KIND_GUARD) { - // Just keep it simple and use one backdated const entry for both previous guard partition - // ranges. - jl_atomic_store_relaxed(&prev_bpart->max_world, new_world-1); - need_backdate = 1; + enum jl_partition_kind prev_kind = decode_restriction_kind(prev_pku); + if (jl_bkind_is_some_constant(prev_kind) || prev_kind == BINDING_KIND_GLOBAL || + (jl_bkind_is_some_import(prev_kind))) { + need_backdate = 0; + break; } + if (prev_bpart->min_world == 0) + break; + prev_bpart = jl_get_binding_partition(b, prev_bpart->min_world - 1); } } + // If backdate is required, rewrite all previous binding partitions to + // backdated const if (need_backdate) { - jl_declare_constant_val3(b, mod, var, val, BINDING_KIND_BACKDATED_CONST, 0); + // We will backdate as long as this partition was never explicitly + // declared const, global, or *explicitly* imported. + jl_binding_partition_t *prev_bpart = bpart; + for (;;) { + jl_atomic_store_relaxed(&prev_bpart->restriction, encode_restriction(val, BINDING_KIND_BACKDATED_CONST)); + if (prev_bpart->min_world == 0) + break; + prev_bpart = jl_get_binding_partition(b, prev_bpart->min_world - 1); + } } } JL_GC_POP(); - return bpart; + return new_bpart; } JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val2( @@ -1035,7 +1050,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val size_t i, l = jl_array_nrows(ex->args); for (i = 0; i < l; i++) { jl_value_t *arg = jl_exprarg(ex, i); - jl_declare_global(m, arg, NULL); + jl_declare_global(m, arg, NULL, 0); } JL_GC_POP(); return jl_nothing; diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index 4a320282610cd..6b75a228b2761 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -17,7 +17,7 @@ export apropos, edit, less, code_warntype, code_llvm, code_native, methodswith, import Base.Docs.apropos using Base: unwrap_unionall, rewrap_unionall, isdeprecated, Bottom, summarysize, - signature_type, format_bytes, isbindingresolved + signature_type, format_bytes using Base.Libc using Markdown @@ -264,7 +264,7 @@ function _subtypes_in!(mods::Array, x::Type) m = pop!(mods) xt = xt::DataType for s in names(m, all = true) - if isbindingresolved(m, s) && !isdeprecated(m, s) && isdefined(m, s) + if !isdeprecated(m, s) && isdefined(m, s) t = getfield(m, s) dt = isa(t, UnionAll) ? unwrap_unionall(t) : t if isa(dt, DataType) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index cc4f4f00cf8f6..561e4bcd4feb2 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -72,7 +72,6 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) end function _UndefVarError_warnfor(io::IO, m::Module, var::Symbol) - Base.isbindingresolved(m, var) || return false (Base.isexported(m, var) || Base.ispublic(m, var)) || return false active_mod = Base.active_module() print(io, "\nHint: ") diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index eadc2672dd29b..e70eb8dd97927 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -240,8 +240,7 @@ function complete_symbol!(suggestions::Vector{Completion}, return suggestions end -completes_module(mod::Module, x::Symbol) = - Base.isbindingresolved(mod, x) && isdefined(mod, x) && isa(getglobal(mod, x), Module) +completes_module(mod::Module, x::Symbol) = isdefined(mod, x) && isa(getglobal(mod, x), Module) function add_field_completions!(suggestions::Vector{Completion}, name::String, @nospecialize(t)) if isa(t, Union) diff --git a/stdlib/REPL/src/docview.jl b/stdlib/REPL/src/docview.jl index 313994505b3ee..6a73a9631a49a 100644 --- a/stdlib/REPL/src/docview.jl +++ b/stdlib/REPL/src/docview.jl @@ -312,10 +312,11 @@ function summarize(binding::Binding, sig) else println(io, "No documentation found.\n") quot = any(isspace, sprint(print, binding)) ? "'" : "" - if Base.isbindingresolved(binding.mod, binding.var) - println(io, "Binding ", quot, "`", binding, "`", quot, " exists, but has not been assigned a value.") - else + bpart = Base.lookup_binding_partition(Base.tls_world_age(), convert(Core.Binding, GlobalRef(binding.mod, binding.var))) + if Base.binding_kind(bpart) === Base.BINDING_KIND_GUARD println(io, "Binding ", quot, "`", binding, "`", quot, " does not exist.") + else + println(io, "Binding ", quot, "`", binding, "`", quot, " exists, but has not been assigned a value.") end end md = Markdown.parse(seekstart(io)) @@ -567,8 +568,7 @@ function repl(io::IO, s::Symbol; brief::Bool=true, mod::Module=Main, internal_ac quote repl_latex($io, $str) repl_search($io, $str, $mod) - $(if !isdefined(mod, s) && !Base.isbindingresolved(mod, s) && !haskey(keywords, s) && !Base.isoperator(s) - # n.b. we call isdefined for the side-effect of resolving the binding, if possible + $(if !isdefined(mod, s) && !haskey(keywords, s) && !Base.isoperator(s) :(repl_corrections($io, $str, $mod)) end) $(_repl(s, brief, mod, internal_accesses)) diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 77d056b63655d..59e994f88945b 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -214,8 +214,6 @@ end let s = "using REP" c, r = test_complete_32377(s) @test count(isequal("REPL"), c) == 1 - # issue #30234 - @test !Base.isbindingresolved(M32377, :tanh) # check what happens if REPL is already imported M32377.eval(:(using REPL)) c, r = test_complete_32377(s) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 88fd055f7bd58..e8d670b3d7d00 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -2158,7 +2158,7 @@ function detect_ambiguities(mods::Module...; while !isempty(work) mod = pop!(work) for n in names(mod, all = true) - (!Base.isbindingresolved(mod, n) || Base.isdeprecated(mod, n)) && continue + Base.isdeprecated(mod, n) && continue if !isdefined(mod, n) if is_in_mods(mod, recursive, mods) if allowed_undefineds === nothing || GlobalRef(mod, n) ∉ allowed_undefineds @@ -2229,7 +2229,7 @@ function detect_unbound_args(mods...; while !isempty(work) mod = pop!(work) for n in names(mod, all = true) - (!Base.isbindingresolved(mod, n) || Base.isdeprecated(mod, n)) && continue + Base.isdeprecated(mod, n) && continue if !isdefined(mod, n) if is_in_mods(mod, recursive, mods) if allowed_undefineds === nothing || GlobalRef(mod, n) ∉ allowed_undefineds diff --git a/test/ambiguous.jl b/test/ambiguous.jl index 5f859e773f5d2..0f29817e74dd5 100644 --- a/test/ambiguous.jl +++ b/test/ambiguous.jl @@ -171,12 +171,8 @@ module UnboundAmbig55868 using .B export C, D end -@test !Base.isbindingresolved(UnboundAmbig55868, :C) -@test !Base.isbindingresolved(UnboundAmbig55868, :D) @test isempty(detect_unbound_args(UnboundAmbig55868)) @test isempty(detect_ambiguities(UnboundAmbig55868)) -@test !Base.isbindingresolved(UnboundAmbig55868, :C) -@test !Base.isbindingresolved(UnboundAmbig55868, :D) # Test that Core and Base are free of ambiguities # not using isempty so this prints more information when it fails diff --git a/test/atomics.jl b/test/atomics.jl index adfe4c87138cd..7e9f29c23ca10 100644 --- a/test/atomics.jl +++ b/test/atomics.jl @@ -23,22 +23,32 @@ mutable struct Refxy{T} end modname = String(nameof(@__MODULE__)) -@test_throws ErrorException("invalid redefinition of constant $modname.ARefxy") @eval mutable struct ARefxy{T} +const orig_Refxy = Refxy +const orig_ARefxy = ARefxy +mutable struct ARefxy{T} @atomic x::T @atomic y::T end -@test_throws ErrorException("invalid redefinition of constant $modname.ARefxy") @eval mutable struct ARefxy{T} +@test orig_ARefxy !== ARefxy +const ARefxy = orig_ARefxy +mutable struct ARefxy{T} x::T y::T end -@test_throws ErrorException("invalid redefinition of constant $modname.ARefxy") @eval mutable struct ARefxy{T} +@test orig_ARefxy !== ARefxy +const ARefxy = orig_ARefxy +mutable struct ARefxy{T} x::T @atomic y::T end -@test_throws ErrorException("invalid redefinition of constant $modname.Refxy") @eval mutable struct Refxy{T} +@test orig_ARefxy !== ARefxy +const ARefxy = orig_ARefxy +mutable struct Refxy{T} x::T @atomic y::T end +@test orig_Refxy !== Refxy +const Refxy = orig_Refxy copy(r::Union{Refxy,ARefxy}) = typeof(r)(r.x, r.y) function add(x::T, y)::T where {T}; x + y; end diff --git a/test/core.jl b/test/core.jl index 5e2677f0f075f..ee47eba0d2c7d 100644 --- a/test/core.jl +++ b/test/core.jl @@ -63,20 +63,7 @@ mutable struct ABCDconst c const d::Union{Int,Nothing} end -@test_throws(ErrorException("invalid redefinition of constant $(nameof(curmod)).ABCDconst"), - mutable struct ABCDconst - const a - const b::Int - c - d::Union{Int,Nothing} - end) -@test_throws(ErrorException("invalid redefinition of constant $(nameof(curmod)).ABCDconst"), - mutable struct ABCDconst - a - b::Int - c - d::Union{Int,Nothing} - end) + let abcd = ABCDconst(1, 2, 3, 4) @test (1, 2, 3, 4) === (abcd.a, abcd.b, abcd.c, abcd.d) @test_throws(ErrorException("setfield!: const field .a of type ABCDconst cannot be changed"), @@ -113,6 +100,21 @@ let abcd = ABCDconst(1, 2, 3, 4) abcd.d = nothing) @test (1, 2, "not constant", 4) === (abcd.a, abcd.b, abcd.c, abcd.d) end +const orig_ABCDconst = ABCDconst +mutable struct ABCDconst + const a + const b::Int + c + d::Union{Int,Nothing} +end +@test ABCDconst !== orig_ABCDconst +mutable struct ABCDconst + a + b::Int + c + d::Union{Int,Nothing} +end +@test ABCDconst !== orig_ABCDconst # Issue #52686 struct A52686{T} end struct B52686{T, S} @@ -1210,15 +1212,11 @@ let A = [1] @test x == 1 end -# Make sure that `Module` is not resolved to `Core.Module` during sysimg generation -# so that users can define their own binding named `Module` in Main. -@test success(`$(Base.julia_cmd()) -e '@assert !Base.isbindingresolved(Main, :Module)'`) - # Module() constructor @test names(Module(:anonymous), all = true, imported = true) == [:anonymous] @test names(Module(:anonymous, false), all = true, imported = true) == [:anonymous] -@test Module(:anonymous, false, true).Core == Core -@test_throws UndefVarError Module(:anonymous, false, false).Core +@test invokelatest(getfield, Module(:anonymous, false, true), :Core) == Core +@test_throws UndefVarError invokelatest(getfield, Module(:anonymous, false, false), :Core) # exception from __init__() let didthrow = @@ -3885,11 +3883,13 @@ end struct NInitializedTestType a end +const orig_NInitializedTestType = NInitializedTestType -@test_throws ErrorException @eval struct NInitializedTestType +struct NInitializedTestType a NInitializedTestType() = new() end +@test orig_NInitializedTestType !== NInitializedTestType # issue #12394 mutable struct Empty12394 end @@ -5578,76 +5578,94 @@ struct A16424 x y end +const orig_A16424 = A16424 struct A16424 # allowed x y end +@test A16424 === orig_A16424 -@test_throws ErrorException @eval struct A16424 +struct A16424 x z end +@test A16424 !== orig_A16424 +const A16424 = orig_A16424 -@test_throws ErrorException @eval struct A16424 +struct A16424 x y::Real end +@test A16424 !== orig_A16424 +const A16424 = orig_A16424 struct B16424{T} a end +const orig_B16424 = B16424 struct B16424{T} a end +@test B16424 === orig_B16424 -@test_throws ErrorException @eval struct B16424{S} +struct B16424{S} a end +@test B16424 !== orig_B16424 struct C16424{T,S} x::T y::S end +const orig_C16424 = C16424 struct C16424{T,S} x::T y::S end +@test C16424 === orig_C16424 -@test_throws ErrorException @eval struct C16424{T,S} +struct C16424{T,S} x::S y::T end +@test C16424 !== orig_C16424 struct D16424{T<:Real,S<:T} x::Vector{S} y::Vector{T} end +const orig_D16424 = D16424 struct D16424{T<:Real,S<:T} x::Vector{S} y::Vector{T} end +@test D16424 === orig_D16424 -@test_throws ErrorException struct D16424{T<:Real,S<:Real} +struct D16424{T<:Real,S<:Real} x::Vector{S} y::Vector{T} end +@test D16424 !== orig_D16424 # issue #20999, allow more type redefinitions struct T20999 x::Array{T} where T<:Real end +const orig_T20999 = T20999 struct T20999 x::Array{T} where T<:Real end +@test T20999 === orig_T20999 -@test_throws ErrorException struct T20999 +struct T20999 x::Array{T} where T<:Integer end +@test T20999 !== orig_T20999 # issue #54757, type redefinitions with recursive reference in supertype struct T54757{A>:Int,N} <: AbstractArray{Tuple{X,Tuple{Vararg},Union{T54757{Union{X,Integer}},T54757{A,N}},Vararg{Y,N}} where {X,Y<:T54757}, N} @@ -5655,20 +5673,40 @@ struct T54757{A>:Int,N} <: AbstractArray{Tuple{X,Tuple{Vararg},Union{T54757{Unio y::Union{A,T54757{A,N}} z::T54757{A} end +const orig_T54757 = T54757 struct T54757{A>:Int,N} <: AbstractArray{Tuple{X,Tuple{Vararg},Union{T54757{Union{X,Integer}},T54757{A,N}},Vararg{Y,N}} where {X,Y<:T54757}, N} x::A y::Union{A,T54757{A,N}} z::T54757{A} end +# The type is identical - either answer is semantically allowed here +# However, knowing that the type is identical would require reasoning about the purity of the +# field definitions exprs, which we do not do. Thus, simply check that this doesn't error and +# then reset to the original for the next test. +const T54757 = orig_T54757 -@test_throws ErrorException struct T54757{A>:Int,N} <: AbstractArray{Tuple{X,Tuple{Vararg},Union{T54757{Union{X,Integer}},T54757{A}},Vararg{Y,N}} where {X,Y<:T54757}, N} +struct T54757{A>:Int,N} <: AbstractArray{Tuple{X,Tuple{Vararg},Union{T54757{Union{X,Integer}},T54757{A}},Vararg{Y,N}} where {X,Y<:T54757}, N} x::A y::Union{A,T54757{A,N}} z::T54757{A} end +@test orig_T54757 !== T54757 +# Type redefinition with multiple tvars and reference in the field types +struct DictLike{K, V} <: AbstractDict{K, V} + self::DictLike{K, V} +end +const orig_DictLike = DictLike + +struct DictLike{K, V} <: AbstractDict{K, V} + self::DictLike{K, V} +end +# It is semantically allowable to re-use the old type, but we need to +# make sure in either case that the field type matches the definition +@test fieldtype(DictLike, 1) === DictLike +# initialization of Vector{Core.TypeofBottom} let a = Vector{Core.TypeofBottom}(undef, 2) @test a[1] == Union{} @test a == [Union{}, Union{}] @@ -7676,29 +7714,35 @@ struct S36104{K,V} # check that redefining it works S36104{K,V}() where {K,V} = new() S36104{K,V}(x::S36104) where {K,V} = new(x) end -# with a gensymmed unionall -struct Symmetric{T,S<:AbstractMatrix{<:T}} <: AbstractMatrix{T} + +# with a gensymmed unionall (#39778) +struct Symmetric39778{T,S<:AbstractMatrix{<:T}} <: AbstractMatrix{T} data::S uplo::Char end -struct Symmetric{T,S<:AbstractMatrix{<:T}} <: AbstractMatrix{T} +const orig_Symmetric39778 = Symmetric39778 +struct Symmetric39778{T,S<:AbstractMatrix{<:T}} <: AbstractMatrix{T} data::S uplo::Char end -@test_throws ErrorException begin - struct Symmetric{T,S<:AbstractMatrix{T}} <: AbstractMatrix{T} - data::S - uplo::Char - end -end +@test Symmetric39778 === orig_Symmetric39778 +struct Symmetric39778{T,S<:AbstractMatrix{T}} <: AbstractMatrix{T} + data::S + uplo::Char end +@test Symmetric39778 !== orig_Symmetric39778 + +end # module M36104 + @test fieldtypes(M36104.T36104) == (Vector{M36104.T36104},) @test_throws ErrorException("expected") @eval(struct X36104; x::error("expected"); end) @test !@isdefined(X36104) struct X36104; x::Int; end @test fieldtypes(X36104) == (Int,) primitive type P36104 8 end -@test_throws ErrorException("invalid redefinition of constant $(nameof(curmod)).P36104") @eval(primitive type P36104 16 end) +const orig_P36104 = P36104 +primitive type P36104 16 end +@test P36104 !== orig_P36104 # Malformed invoke f_bad_invoke(x::Int) = invoke(x, (Any,), x) diff --git a/test/docs.jl b/test/docs.jl index 0fff85e90cb59..4404e4454849a 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -119,7 +119,7 @@ end # issue #38819 module NoDocStrings end -@test meta(NoDocStrings) === getfield(NoDocStrings, Base.Docs.META) +@test meta(NoDocStrings) === invokelatest(getfield, NoDocStrings, Base.Docs.META) # General tests for docstrings. diff --git a/test/errorshow.jl b/test/errorshow.jl index f83bbe31b7cc4..8f7482ce3235e 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -216,13 +216,14 @@ Base.show_method_candidates(buf, try bad_vararg_decl("hello", 3) catch e e end) macro except_str(expr, err_type) source_info = __source__ + errmsg = "expected failure, but no exception thrown for $expr" return quote let err = nothing try $(esc(expr)) catch err end - err === nothing && error("expected failure, but no exception thrown") + err === nothing && error($errmsg) @testset let expr=$(repr(expr)) $(Expr(:macrocall, Symbol("@test"), source_info, :(typeof(err) === $(esc(err_type))))) end @@ -255,6 +256,7 @@ end macro except_stackframe(expr, err_type) source_info = __source__ + errmsg = "expected failure, but no exception thrown for $expr" return quote let err = nothing local st @@ -263,7 +265,7 @@ macro except_stackframe(expr, err_type) catch err st = stacktrace(catch_backtrace()) end - err === nothing && error("expected failure, but no exception thrown") + err === nothing && error($errmsg) @testset let expr=$(repr(expr)) $(Expr(:macrocall, Symbol("@test"), source_info, :(typeof(err) === $(esc(err_type))))) end @@ -297,6 +299,7 @@ err_str = @except_str 1 + 2 MethodError err_str = @except_str Float64[](1) MethodError @test !occursin("import Base.Array", err_str) +global Array Array() = 1 err_str = @except_str Array([1]) MethodError @test occursin("import Base.Array", err_str) diff --git a/test/misc.jl b/test/misc.jl index 070952db89032..fef573e9fc747 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1218,8 +1218,8 @@ end @test readlines(`$(Base.julia_cmd()) --startup-file=no -e 'foreach(println, names(Main))'`) == ["Base","Core","Main"] # issue #26310 -@test_warn "could not import" Core.eval(@__MODULE__, :(import .notdefined_26310__)) -@test_warn "could not import" Core.eval(Main, :(import ........notdefined_26310__)) +@test_warn "undeclared at import time" Core.eval(@__MODULE__, :(import .notdefined_26310__)) +@test_warn "undeclared at import time" Core.eval(Main, :(import ........notdefined_26310__)) @test_nowarn Core.eval(Main, :(import .Main)) @test_nowarn Core.eval(Main, :(import ....Main)) diff --git a/test/precompile.jl b/test/precompile.jl index 194f88719642e..e6c987326e44c 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -657,7 +657,7 @@ precompile_test_harness(false) do dir @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc @eval using FooBar - fb_uuid = Base.module_build_id(FooBar) + fb_uuid = invokelatest(()->Base.module_build_id(FooBar)) sleep(2); touch(FooBar_file) insert!(DEPOT_PATH, 1, dir2) @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc @@ -667,9 +667,11 @@ precompile_test_harness(false) do dir @test isfile(joinpath(cachedir2, "FooBar1.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc @test Base.stale_cachefile(FooBar1_file, joinpath(cachedir2, "FooBar1.ji")) isa Tsc - @test fb_uuid == Base.module_build_id(FooBar) - fb_uuid1 = Base.module_build_id(FooBar1) - @test fb_uuid != fb_uuid1 + invokelatest() do + @test fb_uuid == Base.module_build_id(FooBar) + fb_uuid1 = Base.module_build_id(FooBar1) + @test fb_uuid != fb_uuid1 + end # test checksum open(joinpath(cachedir2, "FooBar1.ji"), "a") do f @@ -778,45 +780,48 @@ precompile_test_harness("code caching") do dir Base.compilecache(pkgid) @test Base.isprecompiled(pkgid) @eval using $Cache_module - M = getfield(@__MODULE__, Cache_module) - # Test that this cache file "owns" all the roots + M = invokelatest(getfield, @__MODULE__, Cache_module) Mid = rootid(M) - for name in (:f, :fpush, :callboth) - func = getfield(M, name) - m = only(collect(methods(func))) - @test all(i -> root_provenance(m, i) == Mid, 1:length(m.roots)) - end - # Check that we can cache external CodeInstances: - # length(::Vector) has an inferred specialization for `Vector{X}` - msize = which(length, (Vector{<:Any},)) - hasspec = false - for mi in Base.specializations(msize) - if mi.specTypes == Tuple{typeof(length),Vector{Cacheb8321416e8a3e2f1.X}} - if (isdefined(mi, :cache) && isa(mi.cache, Core.CodeInstance) && - mi.cache.max_world == typemax(UInt) && mi.cache.inferred !== nothing) - hasspec = true - break + invokelatest() do + # Test that this cache file "owns" all the roots + for name in (:f, :fpush, :callboth) + func = getfield(M, name) + m = only(collect(methods(func))) + @test all(i -> root_provenance(m, i) == Mid, 1:length(m.roots)) + end + # Check that we can cache external CodeInstances: + # length(::Vector) has an inferred specialization for `Vector{X}` + msize = which(length, (Vector{<:Any},)) + hasspec = false + for mi in Base.specializations(msize) + if mi.specTypes == Tuple{typeof(length),Vector{Cacheb8321416e8a3e2f1.X}} + if (isdefined(mi, :cache) && isa(mi.cache, Core.CodeInstance) && + mi.cache.max_world == typemax(UInt) && mi.cache.inferred !== nothing) + hasspec = true + break + end end end + @test hasspec + + # Check that internal methods and their roots are accounted appropriately + minternal = which(M.getelsize, (Vector,)) + mi = minternal.specializations::Core.MethodInstance + @test mi.specTypes == Tuple{typeof(M.getelsize),Vector{Int32}} + ci = mi.cache + @test (codeunits(ci.inferred::String)[end]) === 0x01 + @test ci.inferred !== nothing + # ...and that we can add "untracked" roots & non-relocatable CodeInstances to them too + Base.invokelatest() do + M.getelsize(M.X2[]) + end + mispecs = minternal.specializations::Core.SimpleVector + @test mispecs[1] === mi + mi = mispecs[2]::Core.MethodInstance + mi.specTypes == Tuple{typeof(M.getelsize),Vector{M.X2}} + ci = mi.cache + @test (codeunits(ci.inferred::String)[end]) == 0x00 end - @test hasspec - # Check that internal methods and their roots are accounted appropriately - minternal = which(M.getelsize, (Vector,)) - mi = minternal.specializations::Core.MethodInstance - @test mi.specTypes == Tuple{typeof(M.getelsize),Vector{Int32}} - ci = mi.cache - @test (codeunits(ci.inferred::String)[end]) === 0x01 - @test ci.inferred !== nothing - # ...and that we can add "untracked" roots & non-relocatable CodeInstances to them too - Base.invokelatest() do - M.getelsize(M.X2[]) - end - mispecs = minternal.specializations::Core.SimpleVector - @test mispecs[1] === mi - mi = mispecs[2]::Core.MethodInstance - mi.specTypes == Tuple{typeof(M.getelsize),Vector{M.X2}} - ci = mi.cache - @test (codeunits(ci.inferred::String)[end]) == 0x00 # PkgA loads PkgB, and both add roots to the same `push!` method (both before and after loading B) Cache_module2 = :Cachea1544c83560f0c99 write(joinpath(dir, "$Cache_module2.jl"), @@ -835,24 +840,26 @@ precompile_test_harness("code caching") do dir """) Base.compilecache(Base.PkgId(string(Cache_module2))) @eval using $Cache_module2 - M2 = getfield(@__MODULE__, Cache_module2) - M2id = rootid(M2) - dest = [] - Base.invokelatest() do # use invokelatest to see the results of loading the compile - M2.f(dest) - M.fpush(dest) - M2.g(dest) - @test dest == [M2.Y(), M.X(), M2.Z()] - @test M2.callf() == [M2.Y()] - @test M2.callg() == [M2.Z()] - @test M.fpush(M.X[]) == [M.X()] + invokelatest() do + M2 = getfield(@__MODULE__, Cache_module2) + M2id = rootid(M2) + dest = [] + Base.invokelatest() do # use invokelatest to see the results of loading the compile + M2.f(dest) + M.fpush(dest) + M2.g(dest) + @test dest == [M2.Y(), M.X(), M2.Z()] + @test M2.callf() == [M2.Y()] + @test M2.callg() == [M2.Z()] + @test M.fpush(M.X[]) == [M.X()] + end + mT = which(push!, (Vector{T} where T, Any)) + groups = group_roots(mT) + @test Memory{M2.Y} ∈ groups[M2id] + @test Memory{M2.Z} ∈ groups[M2id] + @test Memory{M.X} ∈ groups[Mid] + @test Memory{M.X} ∉ groups[M2id] end - mT = which(push!, (Vector{T} where T, Any)) - groups = group_roots(mT) - @test Memory{M2.Y} ∈ groups[M2id] - @test Memory{M2.Z} ∈ groups[M2id] - @test Memory{M.X} ∈ groups[Mid] - @test Memory{M.X} ∉ groups[M2id] # backedges of external MethodInstances # Root gets used by RootA and RootB, and both consumers end up inferring the same MethodInstance from Root # Do both callers get listed as backedges? @@ -895,31 +902,33 @@ precompile_test_harness("code caching") do dir Base.compilecache(Base.PkgId(string(RootB))) @eval using $RootA @eval using $RootB - MA = getfield(@__MODULE__, RootA) - MB = getfield(@__MODULE__, RootB) - M = getfield(MA, RootModule) - m = which(M.f, (Any,)) - for mi in Base.specializations(m) - mi === nothing && continue - mi = mi::Core.MethodInstance - if mi.specTypes.parameters[2] === Int8 - # external callers - mods = Module[] - for be in mi.backedges - push!(mods, ((be.def::Core.MethodInstance).def::Method).module) # XXX - end - @test MA ∈ mods - @test MB ∈ mods - @test length(mods) == 2 - elseif mi.specTypes.parameters[2] === Int16 - # internal callers - meths = Method[] - for be in mi.backedges - push!(meths, (be.def::Method).def) # XXX + invokelatest() do + MA = getfield(@__MODULE__, RootA) + MB = getfield(@__MODULE__, RootB) + M = getfield(MA, RootModule) + m = which(M.f, (Any,)) + for mi in Base.specializations(m) + mi === nothing && continue + mi = mi::Core.MethodInstance + if mi.specTypes.parameters[2] === Int8 + # external callers + mods = Module[] + for be in mi.backedges + push!(mods, ((be.def::Core.MethodInstance).def::Method).module) # XXX + end + @test MA ∈ mods + @test MB ∈ mods + @test length(mods) == 2 + elseif mi.specTypes.parameters[2] === Int16 + # internal callers + meths = Method[] + for be in mi.backedges + push!(meths, (be.def::Method).def) # XXX + end + @test which(M.g1, ()) ∈ meths + @test which(M.g2, ()) ∈ meths + @test length(meths) == 2 end - @test which(M.g1, ()) ∈ meths - @test which(M.g2, ()) ∈ meths - @test length(meths) == 2 end end @@ -1005,65 +1014,67 @@ precompile_test_harness("code caching") do dir Base.compilecache(Base.PkgId(string(pkg))) end @eval using $StaleA - MA = getfield(@__MODULE__, StaleA) + MA = invokelatest(getfield, @__MODULE__, StaleA) Base.eval(MA, :(nbits(::UInt8) = 8)) @eval using $StaleC invalidations = Base.StaticData.debug_method_invalidation(true) @eval using $StaleB Base.StaticData.debug_method_invalidation(false) - MB = getfield(@__MODULE__, StaleB) - MC = getfield(@__MODULE__, StaleC) - world = Base.get_world_counter() - m = only(methods(MA.use_stale)) - mi = m.specializations::Core.MethodInstance - @test hasvalid(mi, world) # it was re-inferred by StaleC - m = only(methods(MA.build_stale)) - mis = filter(!isnothing, collect(m.specializations::Core.SimpleVector)) - @test length(mis) == 2 - for mi in mis - mi = mi::Core.MethodInstance - if mi.specTypes.parameters[2] == Int - @test mi.cache.max_world < world - else - # The variant for String got "healed" by recompilation in StaleC - @test mi.specTypes.parameters[2] == String - @test mi.cache.max_world == typemax(UInt) + invokelatest() do + MB = getfield(@__MODULE__, StaleB) + MC = getfield(@__MODULE__, StaleC) + world = Base.get_world_counter() + m = only(methods(MA.use_stale)) + mi = m.specializations::Core.MethodInstance + @test hasvalid(mi, world) # it was re-inferred by StaleC + m = only(methods(MA.build_stale)) + mis = filter(!isnothing, collect(m.specializations::Core.SimpleVector)) + @test length(mis) == 2 + for mi in mis + mi = mi::Core.MethodInstance + if mi.specTypes.parameters[2] == Int + @test mi.cache.max_world < world + else + # The variant for String got "healed" by recompilation in StaleC + @test mi.specTypes.parameters[2] == String + @test mi.cache.max_world == typemax(UInt) + end + end + m = only(methods(MB.useA)) + mi = m.specializations::Core.MethodInstance + @test !hasvalid(mi, world) # invalidated by the stale(x::String) method in StaleC + m = only(methods(MC.call_buildstale)) + mi = m.specializations::Core.MethodInstance + @test hasvalid(mi, world) # was compiled with the new method + + # Reporting test (ensure SnoopCompile works) + @test all(i -> isassigned(invalidations, i), eachindex(invalidations)) + m = only(methods(MB.call_nbits)) + for mi in Base.specializations(m) + hv = hasvalid(mi, world) + @test mi.specTypes.parameters[end] === Integer ? !hv : hv end - end - m = only(methods(MB.useA)) - mi = m.specializations::Core.MethodInstance - @test !hasvalid(mi, world) # invalidated by the stale(x::String) method in StaleC - m = only(methods(MC.call_buildstale)) - mi = m.specializations::Core.MethodInstance - @test hasvalid(mi, world) # was compiled with the new method - - # Reporting test (ensure SnoopCompile works) - @test all(i -> isassigned(invalidations, i), eachindex(invalidations)) - m = only(methods(MB.call_nbits)) - for mi in Base.specializations(m) - hv = hasvalid(mi, world) - @test mi.specTypes.parameters[end] === Integer ? !hv : hv - end - idxs = findall(==("verify_methods"), invalidations) - idxsbits = filter(idxs) do i - mi = invalidations[i-1] - mi.def.def === m + idxs = findall(==("verify_methods"), invalidations) + idxsbits = filter(idxs) do i + mi = invalidations[i-1] + mi.def.def === m + end + idx = only(idxsbits) + tagbad = invalidations[idx+1] + @test isa(tagbad, Core.CodeInstance) + j = findfirst(==(tagbad), invalidations) + @test invalidations[j-1] == "insert_backedges_callee" + @test isa(invalidations[j-2], Type) + @test isa(invalidations[j+1], Vector{Any}) # [nbits(::UInt8)] + m = only(methods(MB.useA2)) + mi = only(Base.specializations(m)) + @test !hasvalid(mi, world) + @test any(x -> x isa Core.CodeInstance && x.def === mi, invalidations) + + m = only(methods(MB.map_nbits)) + @test !hasvalid(m.specializations::Core.MethodInstance, world+1) # insert_backedges invalidations also trigger their backedges end - idx = only(idxsbits) - tagbad = invalidations[idx+1] - @test isa(tagbad, Core.CodeInstance) - j = findfirst(==(tagbad), invalidations) - @test invalidations[j-1] == "insert_backedges_callee" - @test isa(invalidations[j-2], Type) - @test isa(invalidations[j+1], Vector{Any}) # [nbits(::UInt8)] - m = only(methods(MB.useA2)) - mi = only(Base.specializations(m)) - @test !hasvalid(mi, world) - @test any(x -> x isa Core.CodeInstance && x.def === mi, invalidations) - - m = only(methods(MB.map_nbits)) - @test !hasvalid(m.specializations::Core.MethodInstance, world+1) # insert_backedges invalidations also trigger their backedges end precompile_test_harness("invoke") do dir @@ -1181,88 +1192,90 @@ precompile_test_harness("invoke") do dir """) Base.compilecache(Base.PkgId(string(CallerModule))) @eval using $InvokeModule: $InvokeModule - MI = getfield(@__MODULE__, InvokeModule) + MI = invokelatest(getfield, @__MODULE__, InvokeModule) @eval $MI.getlast(a::UnitRange) = a.stop @eval using $CallerModule - M = getfield(@__MODULE__, CallerModule) + invokelatest() do + M = getfield(@__MODULE__, CallerModule) + + get_method_for_type(func, @nospecialize(T)) = which(func, (T,)) # return the method func(::T) + function nvalid(mi::Core.MethodInstance) + isdefined(mi, :cache) || return 0 + ci = mi.cache + n = Int(ci.max_world == typemax(UInt)) + while isdefined(ci, :next) + ci = ci.next + n += ci.max_world == typemax(UInt) + end + return n + end - get_method_for_type(func, @nospecialize(T)) = which(func, (T,)) # return the method func(::T) - function nvalid(mi::Core.MethodInstance) - isdefined(mi, :cache) || return 0 - ci = mi.cache - n = Int(ci.max_world == typemax(UInt)) - while isdefined(ci, :next) - ci = ci.next - n += ci.max_world == typemax(UInt) + for func in (M.f, M.g, M.internal, M.fnc, M.gnc, M.internalnc) + m = get_method_for_type(func, Real) + mi = m.specializations::Core.MethodInstance + @test length(mi.backedges) == 2 || length(mi.backedges) == 4 # internalnc might have a constprop edge + @test mi.backedges[1] === Tuple{typeof(func), Real} + @test isa(mi.backedges[2], Core.CodeInstance) + if length(mi.backedges) == 4 + @test mi.backedges[3] === Tuple{typeof(func), Real} + @test isa(mi.backedges[4], Core.CodeInstance) + @test mi.backedges[2] !== mi.backedges[4] + @test mi.backedges[2].def === mi.backedges[4].def + end + @test mi.cache.max_world == typemax(mi.cache.max_world) + end + for func in (M.q, M.qnc) + m = get_method_for_type(func, Integer) + mi = m.specializations::Core.MethodInstance + @test length(mi.backedges) == 2 + @test mi.backedges[1] === Tuple{typeof(func), Integer} + @test isa(mi.backedges[2], Core.CodeInstance) + @test mi.cache.max_world == typemax(mi.cache.max_world) end - return n - end - for func in (M.f, M.g, M.internal, M.fnc, M.gnc, M.internalnc) - m = get_method_for_type(func, Real) - mi = m.specializations::Core.MethodInstance - @test length(mi.backedges) == 2 || length(mi.backedges) == 4 # internalnc might have a constprop edge - @test mi.backedges[1] === Tuple{typeof(func), Real} - @test isa(mi.backedges[2], Core.CodeInstance) - if length(mi.backedges) == 4 - @test mi.backedges[3] === Tuple{typeof(func), Real} - @test isa(mi.backedges[4], Core.CodeInstance) - @test mi.backedges[2] !== mi.backedges[4] - @test mi.backedges[2].def === mi.backedges[4].def - end - @test mi.cache.max_world == typemax(mi.cache.max_world) - end - for func in (M.q, M.qnc) - m = get_method_for_type(func, Integer) + m = get_method_for_type(M.h, Real) + @test nvalid(m.specializations::Core.MethodInstance) == 0 + m = get_method_for_type(M.hnc, Real) + @test nvalid(m.specializations::Core.MethodInstance) == 0 + m = only(methods(M.callq)) + @test nvalid(m.specializations::Core.MethodInstance) == 0 + m = only(methods(M.callqnc)) + @test nvalid(m.specializations::Core.MethodInstance) == 1 + m = only(methods(M.callqi)) + @test (m.specializations::Core.MethodInstance).specTypes == Tuple{typeof(M.callqi), Int} + m = only(methods(M.callqnci)) + @test (m.specializations::Core.MethodInstance).specTypes == Tuple{typeof(M.callqnci), Int} + + m = only(methods(M.g44320)) + @test (m.specializations::Core.MethodInstance).cache.max_world == typemax(UInt) + + m = only(methods(M.g57115)) mi = m.specializations::Core.MethodInstance - @test length(mi.backedges) == 2 - @test mi.backedges[1] === Tuple{typeof(func), Integer} - @test isa(mi.backedges[2], Core.CodeInstance) - @test mi.cache.max_world == typemax(mi.cache.max_world) - end - m = get_method_for_type(M.h, Real) - @test nvalid(m.specializations::Core.MethodInstance) == 0 - m = get_method_for_type(M.hnc, Real) - @test nvalid(m.specializations::Core.MethodInstance) == 0 - m = only(methods(M.callq)) - @test nvalid(m.specializations::Core.MethodInstance) == 0 - m = only(methods(M.callqnc)) - @test nvalid(m.specializations::Core.MethodInstance) == 1 - m = only(methods(M.callqi)) - @test (m.specializations::Core.MethodInstance).specTypes == Tuple{typeof(M.callqi), Int} - m = only(methods(M.callqnci)) - @test (m.specializations::Core.MethodInstance).specTypes == Tuple{typeof(M.callqnci), Int} - - m = only(methods(M.g44320)) - @test (m.specializations::Core.MethodInstance).cache.max_world == typemax(UInt) - - m = only(methods(M.g57115)) - mi = m.specializations::Core.MethodInstance - - f_m = get_method_for_type(M.f57115, Any) - f_mi = f_m.specializations::Core.MethodInstance - - # Make sure that f57115(::Any) has a 'call' backedge to 'g57115' - has_f_call_backedge = false - i = 1 - while i ≤ length(f_mi.backedges) - if f_mi.backedges[i] isa DataType - # invoke edge - skip - i += 2 - else - caller = f_mi.backedges[i]::Core.CodeInstance - if caller.def === mi - has_f_call_backedge = true - break + f_m = get_method_for_type(M.f57115, Any) + f_mi = f_m.specializations::Core.MethodInstance + + # Make sure that f57115(::Any) has a 'call' backedge to 'g57115' + has_f_call_backedge = false + i = 1 + while i ≤ length(f_mi.backedges) + if f_mi.backedges[i] isa DataType + # invoke edge - skip + i += 2 + else + caller = f_mi.backedges[i]::Core.CodeInstance + if caller.def === mi + has_f_call_backedge = true + break + end + i += 1 end - i += 1 end - end - @test has_f_call_backedge + @test has_f_call_backedge - m = which(MI.getlast, (Any,)) - @test (m.specializations::Core.MethodInstance).cache.max_world == typemax(UInt) + m = which(MI.getlast, (Any,)) + @test (m.specializations::Core.MethodInstance).cache.max_world == typemax(UInt) + end # Precompile specific methods for arbitrary arg types invokeme(x) = 1 @@ -1359,7 +1372,6 @@ precompile_test_harness("package_callbacks") do dir """) Base.compilecache(Base.PkgId("$(Test2_module)")) - @test !Base.isbindingresolved(Main, Test2_module) Base.require(Main, Test2_module) @test take!(loaded_modules) == Test1_module @test take!(loaded_modules) == Test2_module @@ -1454,13 +1466,15 @@ end end try @eval using $ModuleB - uuid = Base.module_build_id(Base.root_module(Main, ModuleB)) - for wid in test_workers - @test Distributed.remotecall_eval(Main, wid, quote - Base.module_build_id(Base.root_module(Main, $(QuoteNode(ModuleB)))) - end) == uuid - if wid != myid() # avoid world-age errors on the local proc - @test remotecall_fetch(g, wid) == wid + invokelatest() do + uuid = Base.module_build_id(Base.root_module(Main, ModuleB)) + for wid in test_workers + @test Distributed.remotecall_eval(Main, wid, quote + Base.module_build_id(Base.root_module(Main, $(QuoteNode(ModuleB)))) + end) == uuid + if wid != myid() # avoid world-age errors on the local proc + @test remotecall_fetch(g, wid) == wid + end end end finally @@ -1612,7 +1626,9 @@ precompile_test_harness("Issue #29936") do load_path end """) @eval using Foo29936 - @test [("Plan", Foo29936.m), ("Plan", Foo29936.h),] isa Vector{Tuple{String,Val}} + invokelatest() do + @test [("Plan", Foo29936.m), ("Plan", Foo29936.h),] isa Vector{Tuple{String,Val}} + end end precompile_test_harness("Issue #25971") do load_path @@ -1795,8 +1811,10 @@ precompile_test_harness("Recursive types") do load_path """) Base.compilecache(Base.PkgId("RecursiveTypeDef")) (@eval (using RecursiveTypeDef)) - a = Base.invokelatest(RecursiveTypeDef.A{Float64,2,String}, (3, 3)) - @test isa(a, AbstractArray) + invokelatest() do + a = Base.invokelatest(RecursiveTypeDef.A{Float64,2,String}, (3, 3)) + @test isa(a, AbstractArray) + end end @testset "issue 46778" begin @@ -1820,7 +1838,9 @@ precompile_test_harness("Module tparams") do load_path """) Base.compilecache(Base.PkgId("ModuleTparams")) (@eval (using ModuleTparams)) - @test ModuleTparams.the_struct === Base.invokelatest(ModuleTparams.ParamStruct{ModuleTparams.TheTParam}) + invokelatest() do + @test ModuleTparams.the_struct === Base.invokelatest(ModuleTparams.ParamStruct{ModuleTparams.TheTParam}) + end end precompile_test_harness("PkgCacheInspector") do load_path @@ -1894,7 +1914,9 @@ precompile_test_harness("DynamicExpressions") do load_path """) Base.compilecache(Base.PkgId("Float16MWE")) @eval using Float16MWE - @test @invokelatest(Float16MWE.doconvert(Float16MWE.Node{Float16}, -1.2)) === Float16(-1.2) + invokelatest() do + @test Float16MWE.doconvert(Float16MWE.Node{Float16}, -1.2) === Float16(-1.2) + end end precompile_test_harness("BadInvalidations") do load_path @@ -1909,7 +1931,9 @@ precompile_test_harness("BadInvalidations") do load_path Base.compilecache(Base.PkgId("BadInvalidations")) @eval Base a_method_to_overwrite_in_test() = inferencebarrier(2) @eval using BadInvalidations - @test Base.invokelatest(BadInvalidations.getval) === 2 + invokelatest() do + @test BadInvalidations.getval() === 2 + end end # https://github.com/JuliaLang/julia/issues/48074 @@ -1945,7 +1969,7 @@ precompile_test_harness("Issue #48391") do load_path """) ji, ofile = Base.compilecache(Base.PkgId("I48391")) @eval using I48391 - x = Base.invokelatest(I48391.SurrealFinite) + x = invokelatest(()->I48391.SurrealFinite()) @test Base.invokelatest(isless, x, x) === "good" @test_throws ErrorException isless(x, x) end @@ -1985,13 +2009,17 @@ precompile_test_harness("Issue #50538") do load_path """) ji, ofile = Base.compilecache(Base.PkgId("I50538")) @eval using I50538 - @test I50538.newglobal.msg == "Creating a new global in closed module `Base` (`newglobal`) breaks incremental compilation because the side effects will not be permanent." - @test I50538.newtype.msg == "Evaluation into the closed module `Base` breaks incremental compilation because the side effects will not be permanent. This is likely due to some other module mutating `Base` with `eval` during precompilation - don't do this." - @test_throws(ErrorException("cannot set type for global I50538.undefglobal. It already has a value or is already set to a different type."), - Core.eval(I50538, :(global undefglobal::Int))) - Core.eval(I50538, :(global undefglobal::Any)) - @test Core.get_binding_type(I50538, :undefglobal) === Any - @test !isdefined(I50538, :undefglobal) + invokelatest() do + @test I50538.newglobal.msg == "Creating a new global in closed module `Base` (`newglobal`) breaks incremental compilation because the side effects will not be permanent." + @test I50538.newtype.msg == "Evaluation into the closed module `Base` breaks incremental compilation because the side effects will not be permanent. This is likely due to some other module mutating `Base` with `eval` during precompilation - don't do this." + @test_throws(ErrorException("cannot set type for global I50538.undefglobal. It already has a value or is already set to a different type."), + Core.eval(I50538, :(global undefglobal::Int))) + Core.eval(I50538, :(global undefglobal::Any)) + invokelatest() do + @test Core.get_binding_type(I50538, :undefglobal) === Any + @test !isdefined(I50538, :undefglobal) + end + end end precompile_test_harness("Test flags") do load_path @@ -2040,14 +2068,15 @@ precompile_test_harness("No backedge precompile") do load_path write(joinpath(load_path, "NoBackEdges.jl"), """ module NoBackEdges - using Core.Intrinsics: add_int - f(a::Int, b::Int) = add_int(a, b) + @eval f(a::Int, b::Int) = \$(Core.Intrinsics.add_int)(a, b) precompile(f, (Int, Int)) end """) ji, ofile = Base.compilecache(Base.PkgId("NoBackEdges")) @eval using NoBackEdges - @test first(methods(NoBackEdges.f)).specializations.cache.max_world === typemax(UInt) + invokelatest() do + @test first(methods(NoBackEdges.f)).specializations.cache.max_world === typemax(UInt) + end end # Test precompilation of generated functions that return opaque closures @@ -2078,7 +2107,7 @@ precompile_test_harness("Generated Opaque") do load_path """) Base.compilecache(Base.PkgId("GeneratedOpaque")) @eval using GeneratedOpaque - let oc = invokelatest(GeneratedOpaque.oc_re_generated_no_partial) + let oc = invokelatest(()->GeneratedOpaque.oc_re_generated_no_partial()) @test oc.source.specializations.cache.max_world === typemax(UInt) @test oc() === 1 end @@ -2143,9 +2172,10 @@ precompile_test_harness("Binding Unique") do load_path @eval using UniqueBinding1 @eval using UniqueBinding2 - - @test UniqueBinding2.thebinding === ccall(:jl_get_module_binding, Ref{Core.Binding}, (Any, Any, Cint), UniqueBinding1, :x, true) - @test UniqueBinding2.thebinding2 === ccall(:jl_get_module_binding, Ref{Core.Binding}, (Any, Any, Cint), UniqueBinding2, :thebinding, true) + invokelatest() do + @test UniqueBinding2.thebinding === ccall(:jl_get_module_binding, Ref{Core.Binding}, (Any, Any, Cint), UniqueBinding1, :x, true) + @test UniqueBinding2.thebinding2 === ccall(:jl_get_module_binding, Ref{Core.Binding}, (Any, Any, Cint), UniqueBinding2, :thebinding, true) + end end precompile_test_harness("Detecting importing outside of a package module") do load_path diff --git a/test/rebinding.jl b/test/rebinding.jl index aee866facaf02..23feb8ded0e56 100644 --- a/test/rebinding.jl +++ b/test/rebinding.jl @@ -62,6 +62,26 @@ module Rebinding @test f_generated_return_delete_me() == 4 Base.delete_binding(@__MODULE__, :delete_me) @test_throws UndefVarError f_generated_return_delete_me() + + module DeleteMeModule + export delete_me_implicit + const delete_me_explicit = 5 + const delete_me_implicit = 6 + end + + # + via import + using .DeleteMeModule: delete_me_explicit + f_return_delete_me_explicit() = delete_me_explicit + @test f_return_delete_me_explicit() == 5 + Base.delete_binding(DeleteMeModule, :delete_me_explicit) + @test_throws UndefVarError f_return_delete_me_explicit() + + # + via using + using .DeleteMeModule + f_return_delete_me_implicit() = delete_me_implicit + @test f_return_delete_me_implicit() == 6 + Base.delete_binding(DeleteMeModule, :delete_me_implicit) + @test_throws UndefVarError f_return_delete_me_implicit() end module RebindingPrecompile @@ -77,6 +97,9 @@ module RebindingPrecompile const delete_me_2 = 2 const delete_me_3 = 3 const delete_me_4 = 4 + export delete_me_5 + const delete_me_5 = 5 + const delete_me_6 = 6 end """) Base.compilecache(Base.PkgId("LotsOfBindingsToDelete")) @@ -88,16 +111,22 @@ module RebindingPrecompile @eval f_use_bindings2() = \$(GlobalRef(LotsOfBindingsToDelete, :delete_me_2)) f_use_bindings3() = LotsOfBindingsToDelete.delete_me_3 f_use_bindings4() = LotsOfBindingsToDelete.delete_me_4 + f_use_bindings5() = delete_me_5 + import LotsOfBindingsToDelete: delete_me_6 + f_use_bindings6() = delete_me_6 # Code Instances for each of these - @assert (f_use_bindings1(), f_use_bindings2(), f_use_bindings3(), f_use_bindings4()) == - (1, 2, 3, 4) + @assert (f_use_bindings1(), f_use_bindings2(), f_use_bindings3(), + f_use_bindings4(), f_use_bindings5(), f_use_bindings6()) == + (1, 2, 3, 4, 5, 6) end """) Base.compilecache(Base.PkgId("UseTheBindings")) @eval using LotsOfBindingsToDelete - # Delete some bindings before loading the dependent package - Base.delete_binding(LotsOfBindingsToDelete, :delete_me_1) - Base.delete_binding(LotsOfBindingsToDelete, :delete_me_3) + invokelatest() do + # Delete some bindings before loading the dependent package + Base.delete_binding(LotsOfBindingsToDelete, :delete_me_1) + Base.delete_binding(LotsOfBindingsToDelete, :delete_me_3) + end # Load the dependent package @eval using UseTheBindings invokelatest() do @@ -105,13 +134,67 @@ module RebindingPrecompile @test UseTheBindings.f_use_bindings2() == 2 @test_throws UndefVarError UseTheBindings.f_use_bindings3() @test UseTheBindings.f_use_bindings4() == 4 + @test UseTheBindings.f_use_bindings5() == 5 + @test UseTheBindings.f_use_bindings6() == 6 # Delete remaining bindings Base.delete_binding(LotsOfBindingsToDelete, :delete_me_2) Base.delete_binding(LotsOfBindingsToDelete, :delete_me_4) + Base.delete_binding(LotsOfBindingsToDelete, :delete_me_5) + Base.delete_binding(LotsOfBindingsToDelete, :delete_me_6) invokelatest() do @test_throws UndefVarError UseTheBindings.f_use_bindings2() @test_throws UndefVarError UseTheBindings.f_use_bindings4() + @test_throws UndefVarError UseTheBindings.f_use_bindings5() + @test_throws UndefVarError UseTheBindings.f_use_bindings6() + end + end + end + + precompile_test_harness("export change") do load_path + write(joinpath(load_path, "Export1.jl"), + """ + module Export1 + export import_me1 + const import_me1 = 11 + export import_me2 + const import_me2 = 12 + end + """) + write(joinpath(load_path, "Export2.jl"), + """ + module Export2 + end + """) + write(joinpath(load_path, "ImportTest.jl"), + """ + module ImportTest + using Export1, Export2 + f_use_binding1() = import_me1 + f_use_binding2() = import_me2 + @assert f_use_binding1() == 11 + @assert f_use_binding2() == 12 + end + """) + @eval using Export1 + @eval using Export2 + # Change the import resolution for ImportTest + invokelatest() do + Core.eval(Export2, :(export import_me1)) + Core.eval(Export2, :(const import_me1 = 21)) + end + @eval using ImportTest + invokelatest() do + @test_throws UndefVarError ImportTest.f_use_binding1() + @test ImportTest.f_use_binding2() == 12 + end + invokelatest() do + Core.eval(Export2, :(export import_me2)) + Core.eval(Export2, :(const import_me2 = 22)) + end + invokelatest() do + # Currently broken + # @test_throws UndefVarError ImportTest.f_use_binding2() end end diff --git a/test/reflection.jl b/test/reflection.jl index 9aa8fe512cd7c..57c32d19de629 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -969,10 +969,6 @@ f20872(::Val, ::Val) = false @test_throws ErrorException which(f20872, Tuple{Any,Val{N}} where N) @test which(Tuple{typeof(f20872), Val{1}, Val{2}}).sig == Tuple{typeof(f20872), Val, Val} -module M29962 end -# make sure checking if a binding is deprecated does not resolve it -@test !Base.isdeprecated(M29962, :sin) && !Base.isbindingresolved(M29962, :sin) - # @locals using Base: @locals let diff --git a/test/show.jl b/test/show.jl index 75f04c1e02096..99bb14df15b23 100644 --- a/test/show.jl +++ b/test/show.jl @@ -1903,15 +1903,6 @@ end b = IOBuffer() show(IOContext(b, :module => @__MODULE__), TypeA) @test String(take!(b)) == "TypeA" - - # issue #26354; make sure testing for symbol visibility doesn't cause - # spurious binding resolutions - show(IOContext(b, :module => TestShowType), Base.Pair) - @test !Base.isbindingresolved(TestShowType, :Pair) - @test String(take!(b)) == "Core.Pair" - show(IOContext(b, :module => TestShowType), Base.Complex) - @test Base.isbindingresolved(TestShowType, :Complex) - @test String(take!(b)) == "Complex" end @testset "typeinfo" begin diff --git a/test/staged.jl b/test/staged.jl index f3dbdcd73d811..6811bc05a9b68 100644 --- a/test/staged.jl +++ b/test/staged.jl @@ -312,7 +312,7 @@ end :(global x33243 = 2) end @test_throws ErrorException f33243() -global x33243 +global x33243::Any @test f33243() === 2 @test x33243 === 2 diff --git a/test/syntax.jl b/test/syntax.jl index f29dd4978d309..61672a6786762 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -2652,10 +2652,10 @@ using ..Mod end @test Mod3.f(10) == 21 @test !isdefined(Mod3, :func) -@test_throws ErrorException("invalid method definition in Mod3: function Mod3.f must be explicitly imported to be extended") Core.eval(Mod3, :(f(x::Int) = x)) +@test_throws ErrorException("invalid method definition in Mod3: function Mod.f must be explicitly imported to be extended") Core.eval(Mod3, :(f(x::Int) = x)) @test !isdefined(Mod3, :always_undef) # resolve this binding now in Mod3 -@test_throws ErrorException("invalid method definition in Mod3: exported function Mod.always_undef does not exist") Core.eval(Mod3, :(always_undef(x::Int) = x)) -@test_throws ErrorException("cannot declare Mod3.always_undef constant; it was already declared as an import") Core.eval(Mod3, :(const always_undef = 3)) +@test Core.eval(Mod3, :(always_undef(x::Int) = x)) == invokelatest(getglobal, Mod3, :always_undef) +@test Core.eval(Mod3, :(const always_undef = 3)) == invokelatest(getglobal, Mod3, :always_undef) @test_throws ErrorException("cannot declare Mod3.f constant; it was already declared as an import") Core.eval(Mod3, :(const f = 3)) @test_throws ErrorException("cannot declare Mod.maybe_undef constant; it was already declared global") Core.eval(Mod, :(const maybe_undef = 3)) @@ -4046,3 +4046,9 @@ function fs56711() return f end @test !@isdefined(x_should_not_be_defined) + +# Test that importing twice is allowed without warning +@test_nowarn @eval baremodule ImportTwice + import ..Base + using .Base: zero, zero +end diff --git a/test/worlds.jl b/test/worlds.jl index 48fb6593d3a37..f4558327c744b 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -107,7 +107,7 @@ end g265() = [f265(x) for x in 1:3.] wc265 = get_world_counter() wc265_41332a = Task(tls_world_age) -@test tls_world_age() == wc265 + 2 +@test tls_world_age() == wc265 + 1 (function () global wc265_41332b = Task(tls_world_age) @eval f265(::Any) = 1.0 @@ -115,15 +115,15 @@ wc265_41332a = Task(tls_world_age) global wc265_41332d = Task(tls_world_age) nothing end)() -@test wc265 + 10 == get_world_counter() == tls_world_age() +@test wc265 + 12 == get_world_counter() == tls_world_age() schedule(wc265_41332a) schedule(wc265_41332b) schedule(wc265_41332c) schedule(wc265_41332d) @test wc265 + 1 == fetch(wc265_41332a) -@test wc265 + 8 == fetch(wc265_41332b) -@test wc265 + 10 == fetch(wc265_41332c) -@test wc265 + 8 == fetch(wc265_41332d) +@test wc265 + 10 == fetch(wc265_41332b) +@test wc265 + 12 == fetch(wc265_41332c) +@test wc265 + 10 == fetch(wc265_41332d) chnls, tasks = Base.channeled_tasks(2, wfunc) t265 = tasks[1] @@ -509,3 +509,18 @@ struct FooBackdated FooBackdated() = new(FooBackdated[]) end @test Base.invoke_in_world(before_backdate_age, isdefined, @__MODULE__, :FooBackdated) + +# Test that ambiguous binding intersect the using'd binding's world ranges +module AmbigWorldTest + using Test + module M1; export x; end + module M2; export x; end + using .M1, .M2 + Core.eval(M1, :(x=1)) + Core.eval(M2, :(x=2)) + @test_throws UndefVarError x + @test convert(Core.Binding, GlobalRef(@__MODULE__, :x)).partitions.min_world == max( + convert(Core.Binding, GlobalRef(M1, :x)).partitions.min_world, + convert(Core.Binding, GlobalRef(M2, :x)).partitions.min_world + ) +end From 603d517c41c2d969f0c8f35cedf9a116e41aea5d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 6 Feb 2025 13:03:27 +0100 Subject: [PATCH 08/31] fix "Right arrow autocompletes at line end" implementation (#57273) Fixes https://github.com/JuliaLang/julia/issues/56864. Ref https://github.com/JuliaLang/julia/blob/99fd5d9a92190e826bc462d5739e7be948a3bf44/stdlib/REPL/src/LineEdit.jl#L504 --------- Co-authored-by: Ian Butterworth (cherry picked from commit f0446c624b2af960c597e22a39131ced9cfce6fb) --- stdlib/REPL/src/LineEdit.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 1075aa648b926..288a9cb1ea91f 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -835,7 +835,7 @@ function edit_move_right(m::MIState) # Replace word by completion prev_pos = position(s) push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1]) + edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion) refresh_line(state(s)) return true else @@ -2260,7 +2260,7 @@ function complete_line(s::SearchState, repeats, mod::Module; hint::Bool=false) if length(completions) == 1 prev_pos = position(s) push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1]) + edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion) return true end return false From 78253642d24589f8c020a94866af9969c602491f Mon Sep 17 00:00:00 2001 From: Chengyu Han Date: Thu, 6 Feb 2025 23:57:55 +0800 Subject: [PATCH 09/31] dep: Update JuliaSyntax (#57280) - Update `JuliaSyntax.jl` to v1.0.1 - Revert workaround changes, use original test case `n37134` Fix #57223 (cherry picked from commit 97c920dc4e12f7c6c65e681adb3f21d6f7abebdf) --- deps/JuliaSyntax.version | 2 +- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + test/syntax.jl | 4 +--- 6 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 deps/checksums/JuliaSyntax-2e965a159dd9f87d216d2d50ecbd2ed4f9af2c5a.tar.gz/md5 delete mode 100644 deps/checksums/JuliaSyntax-2e965a159dd9f87d216d2d50ecbd2ed4f9af2c5a.tar.gz/sha512 create mode 100644 deps/checksums/JuliaSyntax-86bc4331eaa08e08bf2af1ba7b50bbbf4af70cdb.tar.gz/md5 create mode 100644 deps/checksums/JuliaSyntax-86bc4331eaa08e08bf2af1ba7b50bbbf4af70cdb.tar.gz/sha512 diff --git a/deps/JuliaSyntax.version b/deps/JuliaSyntax.version index a7d31b7c16403..70dac3231f626 100644 --- a/deps/JuliaSyntax.version +++ b/deps/JuliaSyntax.version @@ -1,4 +1,4 @@ JULIASYNTAX_BRANCH = main -JULIASYNTAX_SHA1 = 2e965a159dd9f87d216d2d50ecbd2ed4f9af2c5a +JULIASYNTAX_SHA1 = 86bc4331eaa08e08bf2af1ba7b50bbbf4af70cdb JULIASYNTAX_GIT_URL := https://github.com/JuliaLang/JuliaSyntax.jl.git JULIASYNTAX_TAR_URL = https://api.github.com/repos/JuliaLang/JuliaSyntax.jl/tarball/$1 diff --git a/deps/checksums/JuliaSyntax-2e965a159dd9f87d216d2d50ecbd2ed4f9af2c5a.tar.gz/md5 b/deps/checksums/JuliaSyntax-2e965a159dd9f87d216d2d50ecbd2ed4f9af2c5a.tar.gz/md5 deleted file mode 100644 index 96f356f3faaec..0000000000000 --- a/deps/checksums/JuliaSyntax-2e965a159dd9f87d216d2d50ecbd2ed4f9af2c5a.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -40d7bcc6e5741d50a457ace2ca8b2c0c diff --git a/deps/checksums/JuliaSyntax-2e965a159dd9f87d216d2d50ecbd2ed4f9af2c5a.tar.gz/sha512 b/deps/checksums/JuliaSyntax-2e965a159dd9f87d216d2d50ecbd2ed4f9af2c5a.tar.gz/sha512 deleted file mode 100644 index fd7770cdeaa75..0000000000000 --- a/deps/checksums/JuliaSyntax-2e965a159dd9f87d216d2d50ecbd2ed4f9af2c5a.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -b9429b90a28460ef0272cd42a5c221629c6d60221ed088ae3e591cc3d8dbdec32788074397419e58b611bda7df32c7379ec7fafeead7056ed9665591474cec5d diff --git a/deps/checksums/JuliaSyntax-86bc4331eaa08e08bf2af1ba7b50bbbf4af70cdb.tar.gz/md5 b/deps/checksums/JuliaSyntax-86bc4331eaa08e08bf2af1ba7b50bbbf4af70cdb.tar.gz/md5 new file mode 100644 index 0000000000000..c5d3c249ca1e0 --- /dev/null +++ b/deps/checksums/JuliaSyntax-86bc4331eaa08e08bf2af1ba7b50bbbf4af70cdb.tar.gz/md5 @@ -0,0 +1 @@ +a514fe65096a489bd4d3c06b675573c7 diff --git a/deps/checksums/JuliaSyntax-86bc4331eaa08e08bf2af1ba7b50bbbf4af70cdb.tar.gz/sha512 b/deps/checksums/JuliaSyntax-86bc4331eaa08e08bf2af1ba7b50bbbf4af70cdb.tar.gz/sha512 new file mode 100644 index 0000000000000..bdb48e7ef87eb --- /dev/null +++ b/deps/checksums/JuliaSyntax-86bc4331eaa08e08bf2af1ba7b50bbbf4af70cdb.tar.gz/sha512 @@ -0,0 +1 @@ +95d45a27e427f2553da4d4e821edaee6896121977ce6572212c4234013c6f85bc69fc78d237b4dae5d4ed3451f3ba9e1a7172668025ef7bf8aad024293aa2865 diff --git a/test/syntax.jl b/test/syntax.jl index 61672a6786762..c9a5d5dd52a41 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -2454,9 +2454,7 @@ end @test_throws MethodError @m37134()(1.0) == 62 macro n37134() - quote - ((x...,)) -> (x) - end |> esc + :($(esc(Expr(:tuple, Expr(:..., :x))))->$(esc(:x))) end @test @n37134()(2,1) === (2,1) From f8ef8df5bfe2951a944c6a4466932437866ece59 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 6 Feb 2025 14:31:05 -0500 Subject: [PATCH 10/31] staticdata: Close data race after backedge insertion (#57229) Addresses review comment in https://github.com/JuliaLang/julia/pull/57212#discussion_r1937694448. The key is that the hand-off of responsibility for verification between the loading code and the ordinary backedge mechanism happens under the world counter lock to ensure synchronization. (cherry picked from commit 34aceb528d26c2a3529d1afe5aed3bc039fdb181) --- base/staticdata.jl | 28 ++++++++++++++++------------ src/staticdata.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/base/staticdata.jl b/base/staticdata.jl index a4a40b9af0a7c..7283a93dd8b3b 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -37,9 +37,16 @@ end function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, external::Bool=false) for i = 1:length(edges) codeinst = edges[i]::CodeInstance - verify_method_graph(codeinst, stack, visiting, mwis) + validation_world = get_world_counter() + verify_method_graph(codeinst, stack, visiting, mwis, validation_world) + # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies + # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining + # validity. + @ccall jl_promote_ci_to_current(codeinst::Any, validation_world::UInt)::Cvoid minvalid = codeinst.min_world maxvalid = codeinst.max_world + # Finally, if this CI is still valid in some world age and and belongs to an external method(specialization), + # poke it that mi's cache if maxvalid ≥ minvalid && external caller = get_ci_mi(codeinst) @assert isdefined(codeinst, :inferred) # See #53586, #53109 @@ -55,9 +62,9 @@ function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visi end end -function verify_method_graph(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}) +function verify_method_graph(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, validation_world::UInt) @assert isempty(stack); @assert isempty(visiting); - child_cycle, minworld, maxworld = verify_method(codeinst, stack, visiting, mwis) + child_cycle, minworld, maxworld = verify_method(codeinst, stack, visiting, mwis, validation_world) @assert child_cycle == 0 @assert isempty(stack); @assert isempty(visiting); nothing @@ -67,15 +74,14 @@ end # - Visit the entire call graph, starting from edges[idx] to determine if that method is valid # - Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable # and slightly modified with an early termination option once the computation reaches its minimum -function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}) +function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, validation_world::UInt) world = codeinst.min_world let max_valid2 = codeinst.max_world if max_valid2 ≠ WORLD_AGE_REVALIDATION_SENTINEL return 0, world, max_valid2 end end - current_world = get_world_counter() - local minworld::UInt, maxworld::UInt = 1, current_world + local minworld::UInt, maxworld::UInt = 1, validation_world def = get_ci_mi(codeinst).def @assert def isa Method if haskey(visiting, codeinst) @@ -177,7 +183,7 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi end callee = edge local min_valid2::UInt, max_valid2::UInt - child_cycle, min_valid2, max_valid2 = verify_method(callee, stack, visiting, mwis) + child_cycle, min_valid2, max_valid2 = verify_method(callee, stack, visiting, mwis, validation_world) if minworld < min_valid2 minworld = min_valid2 end @@ -209,16 +215,14 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi if maxworld ≠ 0 @atomic :monotonic child.min_world = minworld end - if maxworld == current_world + @atomic :monotonic child.max_world = maxworld + if maxworld == validation_world && validation_world == get_world_counter() Base.Compiler.store_backedges(child, child.edges) - @atomic :monotonic child.max_world = typemax(UInt) - else - @atomic :monotonic child.max_world = maxworld end @assert visiting[child] == length(stack) + 1 delete!(visiting, child) invalidations = _jl_debug_method_invalidation[] - if invalidations !== nothing && maxworld < current_world + if invalidations !== nothing && maxworld < validation_world push!(invalidations, child, "verify_methods", cause) end end diff --git a/src/staticdata.c b/src/staticdata.c index 9eb2922380c63..3b646bed4739e 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -4380,6 +4380,34 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j return mod; } +JL_DLLEXPORT void _jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) JL_NOTSAFEPOINT +{ + if (jl_atomic_load_relaxed(&ci->max_world) != validated_world) + return; + jl_atomic_store_relaxed(&ci->max_world, ~(size_t)0); + jl_svec_t *edges = jl_atomic_load_relaxed(&ci->edges); + for (size_t i = 0; i < jl_svec_len(edges); i++) { + jl_value_t *edge = jl_svecref(edges, i); + if (!jl_is_code_instance(edge)) + continue; + _jl_promote_ci_to_current(ci, validated_world); + } +} + +JL_DLLEXPORT void jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) +{ + size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); + // No need to acquire the lock if we've been invalidated anyway + if (current_world > validated_world) + return; + JL_LOCK(&world_counter_lock); + current_world = jl_atomic_load_relaxed(&jl_world_counter); + if (current_world == validated_world) { + _jl_promote_ci_to_current(ci, validated_world); + } + JL_UNLOCK(&world_counter_lock); +} + #ifdef __cplusplus } #endif From 24ad17beea58142f03da256172e77697ecc5da8b Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Sat, 8 Feb 2025 12:28:49 +1100 Subject: [PATCH 11/31] Updating binding version to fix MMTk CI (#57298) Updating mmtk-julia version to include https://github.com/mmtk/mmtk-julia/pull/228 and fix the MMTk CI. I've also changed the allocation profiler tests to skip all tests instead of just a few since I've seen some spurious errors - they should all be related though, we need to make sure the profiler accounts for fastpath allocation (see https://github.com/JuliaLang/julia/issues/57103) This should fix https://github.com/JuliaLang/julia/issues/57306. (cherry picked from commit 72f8a106c357de6819e07352e88e49148c8c9546) --- Makefile | 18 ++++++++++++++++++ deps/checksums/mmtk_julia | 4 ++++ deps/mmtk_julia.mk | 19 +++++++++++++++++++ deps/mmtk_julia.version | 6 +++--- stdlib/Profile/test/allocs.jl | 19 ++++++++----------- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index b193b3849c6aa..0f1e8c45edf40 100644 --- a/Makefile +++ b/Makefile @@ -282,8 +282,26 @@ endif endif ifneq (${MMTK_PLAN},None) +# Make sure we use the right version of $MMTK_PLAN, $MMTK_MOVING and $MMTK_BUILD +# if we use the BinaryBuilder version of mmtk-julia +ifeq ($(USE_BINARYBUILDER_MMTK_JULIA),1) +ifeq (${MMTK_PLAN},Immix) +LIB_PATH_PLAN = immix +else ifeq (${MMTK_PLAN},StickyImmix) +LIB_PATH_PLAN = sticky +endif + +ifeq ($(MMTK_MOVING), 0) +LIB_PATH_MOVING := non_moving +else +LIB_PATH_MOVING := moving +endif + +JL_PRIVATE_LIBS-0 += $(LIB_PATH_PLAN)/$(LIB_PATH_MOVING)/$(MMTK_BUILD)/libmmtk_julia +else JL_PRIVATE_LIBS-0 += libmmtk_julia endif +endif # Note that we disable MSYS2's path munging here, as otherwise # it replaces our `:`-separated list as a `;`-separated one. diff --git a/deps/checksums/mmtk_julia b/deps/checksums/mmtk_julia index 098937aea1991..4ccc7b407cb60 100644 --- a/deps/checksums/mmtk_julia +++ b/deps/checksums/mmtk_julia @@ -8,3 +8,7 @@ mmtk_julia-c9e046baf3a0d52fe75d6c8b28f6afd69b045d95.tar.gz/md5/73a8fbea71edce30a mmtk_julia-c9e046baf3a0d52fe75d6c8b28f6afd69b045d95.tar.gz/sha512/374848b7696b565dea66daa208830581f92c1fcb0138e7a7ab88564402e94bc79c54b6ed370ec68473e31e2bd411bf82c97793796c31d39aafbbfffea9c05588 mmtk_julia.v0.30.4+0.x86_64-linux-gnu.tar.gz/md5/8cdeb14fd69945f64308be49f6912f9c mmtk_julia.v0.30.4+0.x86_64-linux-gnu.tar.gz/sha512/3692502f65dec8c0971b56b9bf8178641892b390d520cbcd69880d75b7500e6341534d87882246e68998f590f824ec54c18f4b8fb4aa09b8f313de065c48450e +mmtk_julia-10ad6638b69b31a97a844f2f4e651e5ccea4e298.tar.gz/md5/59ed2c0e0b48673988a40527907f13ae +mmtk_julia-10ad6638b69b31a97a844f2f4e651e5ccea4e298.tar.gz/sha512/d0988c37e82b8d481753f4ce83f38ba11276af3dafa8f65ee2c51122fce0dab056a65b3029cb255732226cc28d1a02e607bdaac91a02c0fd6a9fcfae834fee8c +mmtk_julia.v0.30.5+1.x86_64-linux-gnu.tar.gz/md5/4d12d64754bb5c61e86e97e88bcf7912 +mmtk_julia.v0.30.5+1.x86_64-linux-gnu.tar.gz/sha512/0d619f00fd644338ca1ca2582b20e41db702dff8e0c338c093b2759b54379ba26ae7e0181c64931a45ebd5c3995540e535c248df9b986e73b18b65a39c5d78d2 diff --git a/deps/mmtk_julia.mk b/deps/mmtk_julia.mk index 424113fd4164c..1dc59749a00b5 100644 --- a/deps/mmtk_julia.mk +++ b/deps/mmtk_julia.mk @@ -75,6 +75,25 @@ endif # MMTK_JULIA_DIR else # We are building using the BinaryBuilder version of the binding +# This will download all the versions of the binding that are available in the BinaryBuilder $(eval $(call bb-install,mmtk_julia,MMTK_JULIA,false)) +# Make sure we use the right version of $MMTK_PLAN, $MMTK_MOVING and $MMTK_BUILD +ifeq (${MMTK_PLAN},Immix) +LIB_PATH_PLAN = immix +else ifeq (${MMTK_PLAN},StickyImmix) +LIB_PATH_PLAN = sticky +endif + +ifeq ($(MMTK_MOVING), 0) +LIB_PATH_MOVING := non_moving +else +LIB_PATH_MOVING := moving +endif + +version-check-mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so + +$(BUILDROOT)/usr/lib/libmmtk_julia.so: get-mmtk_julia + @ln -sf $(BUILDROOT)/usr/lib/$(LIB_PATH_PLAN)/$(LIB_PATH_MOVING)/$(MMTK_BUILD)/libmmtk_julia.so $@ + endif # USE_BINARYBUILDER_MMTK_JULIA diff --git a/deps/mmtk_julia.version b/deps/mmtk_julia.version index 684197bbe3e4e..530b6d9ed81e1 100644 --- a/deps/mmtk_julia.version +++ b/deps/mmtk_julia.version @@ -1,6 +1,6 @@ MMTK_JULIA_BRANCH = master -MMTK_JULIA_SHA1 = c9e046baf3a0d52fe75d6c8b28f6afd69b045d95 +MMTK_JULIA_SHA1 = 10ad6638b69b31a97a844f2f4e651e5ccea4e298 MMTK_JULIA_GIT_URL := https://github.com/mmtk/mmtk-julia.git -MMTK_JULIA_TAR_URL = https://github.com/mmtk/mmtk-julia/archive/refs/tags/v0.30.4.tar.gz -MMTK_JULIA_JLL_VER := 0.30.4+0 +MMTK_JULIA_TAR_URL = https://github.com/mmtk/mmtk-julia/archive/refs/tags/v0.30.5.tar.gz +MMTK_JULIA_JLL_VER := 0.30.5+1 MMTK_JULIA_JLL_NAME := mmtk_julia diff --git a/stdlib/Profile/test/allocs.jl b/stdlib/Profile/test/allocs.jl index 5607783c782f9..8f6539e0baed6 100644 --- a/stdlib/Profile/test/allocs.jl +++ b/stdlib/Profile/test/allocs.jl @@ -8,6 +8,11 @@ let iobuf = IOBuffer() end end +# Issue #57103: This test does not work with MMTk because of fastpath +# allocation which never calls the allocation profiler. +# TODO: We should port these observability tools (e.g. allocation +# profiler and heap snapshot) to MMTk +@static if Base.USING_STOCK_GC @testset "alloc profiler doesn't segfault" begin res = Allocs.@profile sample_rate=1.0 begin # test the allocations during compilation @@ -73,14 +78,8 @@ end @test length(first_alloc.stacktrace) > 0 @test length(string(first_alloc.type)) > 0 - # Issue #57103: This test does not work with MMTk because of fastpath - # allocation which never calls the allocation profiler. - # TODO: We should port these observability tools (e.g. allocation - # profiler and heap snapshot) to MMTk - @static if Base.USING_STOCK_GC - @testset for type in (Task, Vector{Float64},) - @test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS - end + @testset for type in (Task, Vector{Float64},) + @test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS end # TODO: it would be nice to assert that these tasks @@ -149,8 +148,6 @@ end @test length([a for a in prof.allocs if a.type == String]) >= 1 end -# FIXME: Issue #57103 disabling test for MMTk. -@static if Base.USING_STOCK_GC @testset "alloc profiler catches allocs from codegen" begin @eval begin struct MyType x::Int; y::Int end @@ -170,7 +167,6 @@ end @test length(prof.allocs) >= 1 @test length([a for a in prof.allocs if a.type == MyType]) >= 1 end -end @testset "alloc profiler catches allocs from buffer resize" begin f(a) = for _ in 1:100; push!(a, 1); end @@ -187,3 +183,4 @@ end @test length([a for a in prof.allocs if a.type === Allocs.BufferType]) == 1 @test length([a for a in prof.allocs if a.type === Memory{Int}]) >= 2 end +end From f1572ac0e251390ca65d29f28bcaa29ed0a9a377 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sat, 8 Feb 2025 13:14:56 +0900 Subject: [PATCH 12/31] improve concurrency safety for `Compiler.finish!` (#57248) Similar to JuliaLang/julia#57229, this commit ensures that `Compiler.finish!` properly synchronizes the operations to set `max_world` for cached `CodeInstance`s by holding the world counter lock. Previously, `Compiler.finish!` relied on a narrow timing window to avoid race conditions, which was not a robust approach in a concurrent execution environment. This change ensures that `Compiler.finish!` holds the appropriate lock (via `jl_promote_ci_to_current`). (cherry picked from commit 4ebb50b6a0445b6ecdd74374c3999da5b3f33b9d) --- Compiler/src/typeinfer.jl | 29 ++++++++++++++++++++--------- src/staticdata.c | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index f1da2e8a75a67..427f485f9118b 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -92,7 +92,7 @@ If set to `true`, record per-method-instance timings within type inference in th __set_measure_typeinf(onoff::Bool) = __measure_typeinf__[] = onoff const __measure_typeinf__ = RefValue{Bool}(false) -function finish!(interp::AbstractInterpreter, caller::InferenceState) +function finish!(interp::AbstractInterpreter, caller::InferenceState, validation_world::UInt) result = caller.result opt = result.src if opt isa OptimizationState @@ -108,12 +108,7 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState) ci = result.ci # if we aren't cached, we don't need this edge # but our caller might, so let's just make it anyways - if last(result.valid_worlds) >= get_world_counter() - # TODO: this should probably come after all store_backedges (after optimizations) for the entire graph in finish_cycle - # since we should be requiring that all edges first get their backedges set, as a batch - result.valid_worlds = WorldRange(first(result.valid_worlds), typemax(UInt)) - end - if last(result.valid_worlds) == typemax(UInt) + if last(result.valid_worlds) >= validation_world # if we can record all of the backedges in the global reverse-cache, # we can now widen our applicability in the global cache too store_backedges(ci, edges) @@ -202,7 +197,14 @@ function finish_nocycle(::AbstractInterpreter, frame::InferenceState) if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` optimize(frame.interp, opt, frame.result) end - finish!(frame.interp, frame) + validation_world = get_world_counter() + finish!(frame.interp, frame, validation_world) + if isdefined(frame.result, :ci) + # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies + # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining + # validity. + ccall(:jl_promote_ci_to_current, Cvoid, (Any, UInt), frame.result.ci, validation_world) + end if frame.cycleid != 0 frames = frame.callstack::Vector{AbsIntState} @assert frames[end] === frame @@ -236,10 +238,19 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei optimize(caller.interp, opt, caller.result) end end + validation_world = get_world_counter() + cis = CodeInstance[] for frameid = cycleid:length(frames) caller = frames[frameid]::InferenceState - finish!(caller.interp, caller) + finish!(caller.interp, caller, validation_world) + if isdefined(caller.result, :ci) + push!(cis, caller.result.ci) + end end + # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies + # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining + # validity. + ccall(:jl_promote_cis_to_current, Cvoid, (Ptr{CodeInstance}, Csize_t, UInt), cis, length(cis), validation_world) resize!(frames, cycleid - 1) return nothing end diff --git a/src/staticdata.c b/src/staticdata.c index 3b646bed4739e..c0fa8931e27c8 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -4408,6 +4408,22 @@ JL_DLLEXPORT void jl_promote_ci_to_current(jl_code_instance_t *ci, size_t valida JL_UNLOCK(&world_counter_lock); } +JL_DLLEXPORT void jl_promote_cis_to_current(jl_code_instance_t **cis, size_t n, size_t validated_world) +{ + size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); + // No need to acquire the lock if we've been invalidated anyway + if (current_world > validated_world) + return; + JL_LOCK(&world_counter_lock); + current_world = jl_atomic_load_relaxed(&jl_world_counter); + if (current_world == validated_world) { + for (size_t i = 0; i < n; i++) { + _jl_promote_ci_to_current(cis[i], validated_world); + } + } + JL_UNLOCK(&world_counter_lock); +} + #ifdef __cplusplus } #endif From 3a650d71937e9674b61fcc907ce68e3751cb7b24 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 8 Feb 2025 10:29:25 -0500 Subject: [PATCH 13/31] Profile.print: de-focus sleeping frames as gray (#57312) (cherry picked from commit 132057c378af992f768e5b2a9b69543c0017c0d9) --- stdlib/Profile/src/Profile.jl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index f59b49d8a4a36..2e4092fb22c24 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -980,13 +980,14 @@ mutable struct StackFrameTree{T} # where T <: Union{UInt64, StackFrame} flat_count::Int # number of times this frame was in the flattened representation (unlike count, this'll sum to 100% of parent) max_recur::Int # maximum number of times this frame was the *top* of the recursion in the stack count_recur::Int # sum of the number of times this frame was the *top* of the recursion in a stack (divide by count to get an average) + sleeping::Bool # whether this frame was in a sleeping state down::Dict{T, StackFrameTree{T}} # construction workers: recur::Int builder_key::Vector{UInt64} builder_value::Vector{StackFrameTree{T}} up::StackFrameTree{T} - StackFrameTree{T}() where {T} = new(UNKNOWN, 0, 0, 0, 0, 0, Dict{T, StackFrameTree{T}}(), 0, UInt64[], StackFrameTree{T}[]) + StackFrameTree{T}() where {T} = new(UNKNOWN, 0, 0, 0, 0, 0, true, Dict{T, StackFrameTree{T}}(), 0, UInt64[], StackFrameTree{T}[]) end @@ -1027,6 +1028,10 @@ function tree_format(frames::Vector{<:StackFrameTree}, level::Int, cols::Int, ma base = string(base, "+", nextra, " ") end strcount = rpad(string(frame.count), ndigcounts, " ") + if frame.sleeping + stroverhead = styled"{gray:$(stroverhead)}" + strcount = styled"{gray:$(strcount)}" + end if li != UNKNOWN if li.line == li.pointer strs[i] = string(stroverhead, "╎", base, strcount, " ", @@ -1039,6 +1044,7 @@ function tree_format(frames::Vector{<:StackFrameTree}, level::Int, cols::Int, ma else fname = string(li.func) end + frame.sleeping && (fname = styled"{gray:$(fname)}") path, pkgname, filename = short_path(li.file, filenamemap) if showpointer fname = string( @@ -1082,15 +1088,15 @@ function tree!(root::StackFrameTree{T}, all::Vector{UInt64}, lidict::Union{LineI skip = false nsleeping = 0 is_task_profile = false + is_sleeping = true for i in startframe:-1:1 (startframe - 1) >= i >= (startframe - (nmeta + 1)) && continue # skip metadata (it's read ahead below) and extra block end NULL IP ip = all[i] if is_block_end(all, i) # read metadata thread_sleeping_state = all[i - META_OFFSET_SLEEPSTATE] - 1 # subtract 1 as state is incremented to avoid being equal to 0 - if thread_sleeping_state == 2 - is_task_profile = true - end + is_sleeping = thread_sleeping_state == 1 + is_task_profile = thread_sleeping_state == 2 # cpu_cycle_clock = all[i - META_OFFSET_CPUCYCLECLOCK] taskid = all[i - META_OFFSET_TASKID] threadid = all[i - META_OFFSET_THREADID] @@ -1145,6 +1151,7 @@ function tree!(root::StackFrameTree{T}, all::Vector{UInt64}, lidict::Union{LineI parent = build[j] parent.recur += 1 parent.count_recur += 1 + parent.sleeping &= is_sleeping found = true break end @@ -1164,6 +1171,7 @@ function tree!(root::StackFrameTree{T}, all::Vector{UInt64}, lidict::Union{LineI while this !== parent && (recur === :off || this.recur == 0) this.count += 1 this.recur = 1 + this.sleeping &= is_sleeping this = this.up end end @@ -1185,6 +1193,7 @@ function tree!(root::StackFrameTree{T}, all::Vector{UInt64}, lidict::Union{LineI this.up = parent this.count += 1 this.recur = 1 + this.sleeping &= is_sleeping end parent = this end From 730507ad7f1de7060bea556205c1b252b6876fd2 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Sat, 8 Feb 2025 14:46:24 -0400 Subject: [PATCH 14/31] Make `OncePerX` subtype `Function` (#57289) (cherry picked from commit dbd528025166f92bf0041471b2ddea3027e106e6) --- base/lock.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/lock.jl b/base/lock.jl index 59e554c01c24a..17a08888bf711 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -693,7 +693,7 @@ julia> procstate === fetch(@async global_state()) true ``` """ -mutable struct OncePerProcess{T, F} +mutable struct OncePerProcess{T, F} <: Function value::Union{Nothing,T} @atomic state::UInt8 # 0=initial, 1=hasrun, 2=error @atomic allow_compile_time::Bool @@ -801,7 +801,7 @@ julia> threadvec === thread_state[Threads.threadid()] true ``` """ -mutable struct OncePerThread{T, F} +mutable struct OncePerThread{T, F} <: Function @atomic xs::AtomicMemory{T} # values @atomic ss::AtomicMemory{UInt8} # states: 0=initial, 1=hasrun, 2=error, 3==concurrent const initializer::F @@ -926,7 +926,7 @@ Making lazy task value...done. false ``` """ -mutable struct OncePerTask{T, F} +mutable struct OncePerTask{T, F} <: Function const initializer::F OncePerTask{T}(initializer::F) where {T, F} = new{T,F}(initializer) From 7f2462f940a43fcf1f1ca676bf4a3047baf5f86e Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Sat, 8 Feb 2025 21:36:34 -0300 Subject: [PATCH 15/31] Make ptls allocations at least 128 byte aligned (#57310) Fixes https://github.com/JuliaLang/julia/issues/54560#issuecomment-2439710448 (cherry picked from commit 79e98e3b89da5e57f829ccde21b576be9fc8ea1b) --- src/threading.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/threading.c b/src/threading.c index a51916cdcd8d8..690c5fafb5792 100644 --- a/src/threading.c +++ b/src/threading.c @@ -336,7 +336,17 @@ jl_ptls_t jl_init_threadtls(int16_t tid) #endif if (jl_get_pgcstack() != NULL) abort(); - jl_ptls_t ptls = (jl_ptls_t)calloc(1, sizeof(jl_tls_states_t)); + jl_ptls_t ptls; +#if defined(_OS_WINDOWS_) + ptls = _aligned_malloc(sizeof(jl_tls_states_t), alignof(jl_tls_states_t)); + if (ptls == NULL) + abort(); +#else + if (posix_memalign((void**)&ptls, alignof(jl_tls_states_t), sizeof(jl_tls_states_t))) + abort(); +#endif + memset(ptls, 0, sizeof(jl_tls_states_t)); + #ifndef _OS_WINDOWS_ pthread_setspecific(jl_task_exit_key, (void*)ptls); #endif From da6ee025994b8d72f9fa3c6cab71177b9d039584 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 10 Feb 2025 08:31:20 -0500 Subject: [PATCH 16/31] Add a warning for auto-import of types (#57311) This adds a warning for the auto-import of types cases (#25744) that we have long considered a bit of a bug, but didn't want to change because it is too breaking. The reason to do it now is that the binding rework has made this case more problematic (see #57290). To summarize, the question is what happens when the compiler sees `f(x) = ...` and `f` is currently and implicitly imported binding. There are two options: 1. We add a method to the generic function referred to by `f`, or 2. We create a new generic function `f` in the current module. Historically, case 1 has the additional complication that this error'd unless `f` is a type. It is my impression that a lot of existing code did not have a particularly good understanding of the resolved-ness dependence of this behavior. However, because case 1 errors for generic functions, it appears that existing code generally expects case 2. On the other hand, for types, there is existing code in both directions (#57290 is an example of case 2; see #57302 for examples of case 1). That said, case 1 is more common (because types tend to be resolved because they're used in signatures at toplevel). Thus, to retain compatibility, the current behavior on master (where resolvedness is no longer available) is that we always choose case 2 for functions and case 1 for types. This inconsistency is unfortunate, but I tried resolving this in either way (making all situations case 1 or all case 2) and the result was too breaking. Nevertheless, it is problematic that there is existing code that expects case 2 beavior for types and we should help users to know what the correct way to fix it is. The proposed resolution is thus: 1. Retain case 1 behavior for types 2. Make it a warning to use, encouraging people to explicitly import, since we generally consider the #25744 case a bug. Example: ``` julia> module Foo String(i::Int) = i end WARNING: Type Core.String was auto-`import`ed in `Foo`. NOTE: This behavior is deprecated and may change in future Julia versions. NOTE: This behavior may have differed in Julia versions prior to 1.12 depending on binding resolution. Hint: To retain the current behavior, add an explicit `import Core: String` in Foo. Hint: To create a new generic function of the same name use `function String end`. Main.Foo ``` (cherry picked from commit 8c62f42947ee01fb17d1baf8c5e953fb3516b31f) --- src/codegen.cpp | 25 ++++------------------ src/interpreter.c | 3 +-- src/julia.h | 11 ++++++++-- src/julia_internal.h | 1 + src/method.c | 3 ++- src/module.c | 28 ++++++++++++++++++------- stdlib/SharedArrays/src/SharedArrays.jl | 1 + test/arrayops.jl | 2 +- test/errorshow.jl | 2 +- test/intrinsics.jl | 2 +- 10 files changed, 41 insertions(+), 37 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 5507bd7bad801..da34d8720d359 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1019,7 +1019,7 @@ static const auto jlgenericfunction_func = new JuliaFunction<>{ auto T_jlvalue = JuliaType::get_jlvalue_ty(C); auto T_pjlvalue = PointerType::get(T_jlvalue, 0); auto T_prjlvalue = PointerType::get(T_jlvalue, AddressSpace::Tracked); - return FunctionType::get(T_prjlvalue, {T_pjlvalue, T_pjlvalue, T_pjlvalue}, false); + return FunctionType::get(T_prjlvalue, {T_pjlvalue, T_pjlvalue}, false); }, nullptr, }; @@ -6868,8 +6868,6 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ jl_value_t *mn = args[0]; assert(jl_is_symbol(mn) || jl_is_slotnumber(mn) || jl_is_globalref(mn)); - Value *bp = NULL, *name; - jl_binding_t *bnd = NULL; bool issym = jl_is_symbol(mn); bool isglobalref = !issym && jl_is_globalref(mn); jl_module_t *mod = ctx.module; @@ -6878,26 +6876,11 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ mod = jl_globalref_mod(mn); mn = (jl_value_t*)jl_globalref_name(mn); } - JL_TRY { - if (jl_symbol_name((jl_sym_t*)mn)[0] == '@') - jl_errorf("macro definition not allowed inside a local scope"); - name = literal_pointer_val(ctx, mn); - bnd = jl_get_binding_for_method_def(mod, (jl_sym_t*)mn); - } - JL_CATCH { - jl_value_t *e = jl_current_exception(jl_current_task); - // errors. boo. :( - JL_GC_PUSH1(&e); - e = jl_as_global_root(e, 1); - JL_GC_POP(); - raise_exception(ctx, literal_pointer_val(ctx, e)); - return ghostValue(ctx, jl_nothing_type); - } - bp = julia_binding_gv(ctx, bnd); jl_cgval_t gf = mark_julia_type( ctx, - ctx.builder.CreateCall(prepare_call(jlgenericfunction_func), { bp, - literal_pointer_val(ctx, (jl_value_t*)mod), name + ctx.builder.CreateCall(prepare_call(jlgenericfunction_func), { + literal_pointer_val(ctx, (jl_value_t*)mod), + literal_pointer_val(ctx, (jl_value_t*)mn) }), true, jl_function_type); diff --git a/src/interpreter.c b/src/interpreter.c index 513fe58f7b5cc..7ab284df78dff 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -93,8 +93,7 @@ static jl_value_t *eval_methoddef(jl_expr_t *ex, interpreter_state *s) if (!jl_is_symbol(fname)) { jl_error("method: invalid declaration"); } - jl_binding_t *b = jl_get_binding_for_method_def(modu, fname); - return jl_declare_const_gf(b, modu, fname); + return jl_declare_const_gf(modu, fname); } jl_value_t *atypes = NULL, *meth = NULL, *fname = NULL; diff --git a/src/julia.h b/src/julia.h index dd3bc713a517f..1fd709f42ee31 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1475,6 +1475,13 @@ STATIC_INLINE char *jl_symbol_name_(jl_sym_t *s) JL_NOTSAFEPOINT } #define jl_symbol_name(s) jl_symbol_name_(s) +STATIC_INLINE const char *jl_module_debug_name(jl_module_t *mod) JL_NOTSAFEPOINT +{ + if (!mod) + return ""; + return jl_symbol_name(mod->name); +} + static inline uint32_t jl_fielddesc_size(int8_t fielddesc_type) JL_NOTSAFEPOINT { assert(fielddesc_type >= 0 && fielddesc_type <= 2); @@ -1901,7 +1908,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value(jl_binding_t *b JL_PROPAGATES_ROOT JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT); JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_and_const(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_binding_t *b, jl_module_t *mod, jl_sym_t *name); +JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name); JL_DLLEXPORT jl_method_t *jl_method_def(jl_svec_t *argdata, jl_methtable_t *mt, jl_code_info_t *f, jl_module_t *module); JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo, size_t world, jl_code_instance_t **cache); JL_DLLEXPORT jl_code_info_t *jl_copy_code_info(jl_code_info_t *src); @@ -2051,7 +2058,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_type(jl_module_t *m, jl_sym_t *var); // get binding for assignment JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module_t *m, jl_sym_t *s); JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); -JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); +JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var, size_t new_world); JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import); JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var); diff --git a/src/julia_internal.h b/src/julia_internal.h index 5fe6cad0d096c..6d83000184880 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -876,6 +876,7 @@ STATIC_INLINE size_t module_usings_max(jl_module_t *m) JL_NOTSAFEPOINT { return m->usings.max/3; } +JL_DLLEXPORT jl_sym_t *jl_module_name(jl_module_t *m) JL_NOTSAFEPOINT; jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e); jl_value_t *jl_interpret_opaque_closure(jl_opaque_closure_t *clos, jl_value_t **args, size_t nargs); jl_value_t *jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t *src); diff --git a/src/method.c b/src/method.c index 1b38a16649d8a..68542fdacabb6 100644 --- a/src/method.c +++ b/src/method.c @@ -1050,10 +1050,11 @@ JL_DLLEXPORT void jl_check_gf(jl_value_t *gf, jl_sym_t *name) jl_errorf("cannot define function %s; it already has a value", jl_symbol_name(name)); } -JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_binding_t *b, jl_module_t *mod, jl_sym_t *name) +JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name) { JL_LOCK(&world_counter_lock); size_t new_world = jl_atomic_load_relaxed(&jl_world_counter) + 1; + jl_binding_t *b = jl_get_binding_for_method_def(mod, name, new_world); jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); jl_value_t *gf = NULL; diff --git a/src/module.c b/src/module.c index 1b6b37e49949e..402a86dfd4aef 100644 --- a/src/module.c +++ b/src/module.c @@ -588,10 +588,10 @@ JL_DLLEXPORT jl_value_t *jl_reresolve_binding_value_seqcst(jl_binding_t *b) // get binding for adding a method // like jl_get_binding_wr, but has different error paths and messages -JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_t *var) +JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_t *var, size_t new_world) { jl_binding_t *b = jl_get_module_binding(m, var, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); enum jl_partition_kind kind = decode_restriction_kind(pku); if (kind == BINDING_KIND_GLOBAL || kind == BINDING_KIND_DECLARED || jl_bkind_is_some_constant(decode_restriction_kind(pku))) @@ -601,7 +601,7 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ return b; } jl_binding_t *ownerb = b; - pku = jl_walk_binding_inplace(&ownerb, &bpart, jl_current_task->world_age); + pku = jl_walk_binding_inplace(&ownerb, &bpart, new_world); jl_value_t *f = NULL; if (jl_bkind_is_some_constant(decode_restriction_kind(pku))) f = decode_restriction_value(pku); @@ -613,7 +613,7 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ jl_module_t *from = jl_binding_dbgmodule(b, m, var); // we must have implicitly imported this with using, so call jl_binding_dbgmodule to try to get the name of the module we got this from jl_errorf("invalid method definition in %s: exported function %s.%s does not exist", - jl_symbol_name(m->name), from ? jl_symbol_name(from->name) : "", jl_symbol_name(var)); + jl_module_debug_name(m), jl_module_debug_name(from), jl_symbol_name(var)); } int istype = f && jl_is_type(f); if (!istype) { @@ -626,13 +626,25 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ // or we might want to drop this error entirely jl_module_t *from = jl_binding_dbgmodule(b, m, var); jl_errorf("invalid method definition in %s: function %s.%s must be explicitly imported to be extended", - jl_symbol_name(m->name), from ? jl_symbol_name(from->name) : "", jl_symbol_name(var)); + jl_module_debug_name(m), jl_module_debug_name(from), jl_symbol_name(var)); } } - else if (strcmp(jl_symbol_name(var), "=>") == 0 && (kind == BINDING_KIND_IMPLICIT || kind == BINDING_KIND_EXPLICIT)) { + else if (kind != BINDING_KIND_IMPORTED) { + int should_error = strcmp(jl_symbol_name(var), "=>") == 0; jl_module_t *from = jl_binding_dbgmodule(b, m, var); - jl_errorf("invalid method definition in %s: function %s.%s must be explicitly imported to be extended", - jl_symbol_name(m->name), from ? jl_symbol_name(from->name) : "", jl_symbol_name(var)); + if (should_error) + jl_errorf("invalid method definition in %s: function %s.%s must be explicitly imported to be extended", + jl_module_debug_name(m), jl_module_debug_name(from), jl_symbol_name(var)); + else + jl_printf(JL_STDERR, "WARNING: Constructor for type \"%s\" was extended in `%s` without explicit qualification or import.\n" + " NOTE: Assumed \"%s\" refers to `%s.%s`. This behavior is deprecated and may differ in future versions.`\n" + " NOTE: This behavior may have differed in Julia versions prior to 1.12.\n" + " Hint: If you intended to create a new generic function of the same name, use `function %s end`.\n" + " Hint: To silence the warning, qualify `%s` as `%s.%s` or explicitly `import %s: %s`\n", + jl_symbol_name(var), jl_module_debug_name(m), + jl_symbol_name(var), jl_module_debug_name(from), jl_symbol_name(var), + jl_symbol_name(var), jl_symbol_name(var), jl_module_debug_name(from), jl_symbol_name(var), + jl_module_debug_name(from), jl_symbol_name(var)); } return ownerb; } diff --git a/stdlib/SharedArrays/src/SharedArrays.jl b/stdlib/SharedArrays/src/SharedArrays.jl index 93ce396277af7..6106bc9c3c81a 100644 --- a/stdlib/SharedArrays/src/SharedArrays.jl +++ b/stdlib/SharedArrays/src/SharedArrays.jl @@ -9,6 +9,7 @@ using Mmap, Distributed, Random import Base: length, size, elsize, ndims, IndexStyle, reshape, convert, deepcopy_internal, show, getindex, setindex!, fill!, similar, reduce, map!, copyto!, cconvert +import Base: Array import Random using Serialization using Serialization: serialize_cycle_header, serialize_type, writetag, UNDEFREF_TAG, serialize, deserialize diff --git a/test/arrayops.jl b/test/arrayops.jl index 655e14675bfb4..b2da3eac6386b 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -2948,7 +2948,7 @@ end Base.ArithmeticStyle(::Type{F21666{T}}) where {T} = T() Base.:+(x::F, y::F) where {F <: F21666} = F(x.x + y.x) -Float64(x::F21666) = Float64(x.x) +Base.Float64(x::F21666) = Float64(x.x) @testset "Exactness of cumsum # 21666" begin # test that cumsum uses more stable algorithm # for types with unknown/rounding arithmetic diff --git a/test/errorshow.jl b/test/errorshow.jl index 8f7482ce3235e..8e13d0242ae35 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -438,7 +438,7 @@ let err_str @test occursin("For element-wise subtraction, use broadcasting with dot syntax: array .- scalar", err_str) end - +import Core: String method_defs_lineno = @__LINE__() + 1 String() = throw(ErrorException("1")) (::String)() = throw(ErrorException("2")) diff --git a/test/intrinsics.jl b/test/intrinsics.jl index c3e9bb1680d48..12867908bf5a4 100644 --- a/test/intrinsics.jl +++ b/test/intrinsics.jl @@ -66,7 +66,7 @@ end # test functionality of non-power-of-2 primitive type constants primitive type Int24 24 end Int24(x::Int) = Core.Intrinsics.trunc_int(Int24, x) -Int(x::Int24) = Core.Intrinsics.zext_int(Int, x) +Base.Int(x::Int24) = Core.Intrinsics.zext_int(Int, x) let x, y, f x = Int24(Int(0x12345678)) # create something (via truncation) @test Int(0x345678) === Int(x) From 145a85c0581885ccd1bc0588154a434d49a8dc07 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 10 Feb 2025 21:51:02 +0100 Subject: [PATCH 17/31] fix typo in Float32 random number generation (#57338) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit introduced in https://github.com/JuliaLang/julia/pull/56144 --------- Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> (cherry picked from commit dd1387854540977a77725821c2ce0249bde1e443) --- stdlib/Random/src/XoshiroSimd.jl | 2 +- stdlib/Random/test/runtests.jl | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/stdlib/Random/src/XoshiroSimd.jl b/stdlib/Random/src/XoshiroSimd.jl index f77be4a347111..a23f5d7590b40 100644 --- a/stdlib/Random/src/XoshiroSimd.jl +++ b/stdlib/Random/src/XoshiroSimd.jl @@ -46,7 +46,7 @@ simdThreshold(::Type{Bool}) = 640 ui = (x>>>32) % UInt32 li = x % UInt32 u = _uint2float(ui, Float32) - l = _uint2float(ui, Float32) + l = _uint2float(li, Float32) (UInt64(reinterpret(UInt32, u)) << 32) | UInt64(reinterpret(UInt32, l)) end @inline function _bits2float(x::UInt64, ::Type{Float16}) diff --git a/stdlib/Random/test/runtests.jl b/stdlib/Random/test/runtests.jl index 13edf2e6553ec..a28197bdfe833 100644 --- a/stdlib/Random/test/runtests.jl +++ b/stdlib/Random/test/runtests.jl @@ -1250,6 +1250,16 @@ end @test length(xs) == 3 end +@testset "Float32 RNG typo" begin + for T in (Float16, Float32, Float64) + # Make sure generated numbers are sufficiently diverse + # for both SIMD and non-SIMD RNG code paths for all types. + @test length(unique!(rand(T, 7))) > 3 + @test length(unique!(rand(T, 14))) > 10 + @test length(unique!(rand(T, 34))) > 20 + end +end + @testset "Docstrings" begin @test isempty(Docs.undocumented_names(Random)) end From f8f65baa9ce99a5a1034efa21a0ad3c4e726562d Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Mon, 10 Feb 2025 14:20:43 -0800 Subject: [PATCH 18/31] Fix getfield_tfunc when order or boundscheck is Vararg (#57293) Even if T has no intersection with the type we want, we don't know that we will throw because the arguments are optional. Fixes #57292. (cherry picked from commit 53431309e20eb057709a4f0d4de5663f4a233afd) --- Compiler/src/tfuncs.jl | 12 +++++------- Compiler/test/inference.jl | 6 ++++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index 50b88bb0222ce..cacee7a7d29ac 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -1073,17 +1073,15 @@ end end @nospecs function getfield_tfunc(𝕃::AbstractLattice, s00, name, boundscheck_or_order) - t = isvarargtype(boundscheck_or_order) ? unwrapva(boundscheck_or_order) : - widenconst(boundscheck_or_order) - hasintersect(t, Symbol) || hasintersect(t, Bool) || return Bottom + if !isvarargtype(boundscheck_or_order) + t = widenconst(boundscheck_or_order) + hasintersect(t, Symbol) || hasintersect(t, Bool) || return Bottom + end return getfield_tfunc(𝕃, s00, name) end @nospecs function getfield_tfunc(𝕃::AbstractLattice, s00, name, order, boundscheck) hasintersect(widenconst(order), Symbol) || return Bottom - if isvarargtype(boundscheck) - t = unwrapva(boundscheck) - hasintersect(t, Symbol) || hasintersect(t, Bool) || return Bottom - else + if !isvarargtype(boundscheck) hasintersect(widenconst(boundscheck), Bool) || return Bottom end return getfield_tfunc(𝕃, s00, name) diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index 563828ac77296..b77c99513a8b6 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -6188,3 +6188,9 @@ end == Union{Float64,DomainError} @test Compiler.argtypes_to_type(Any[ Int, UnitRange{Int}, Vararg{Pair{Any, Union{}}}, Float64 ]) === Tuple{Int, UnitRange{Int}, Float64} @test Compiler.argtypes_to_type(Any[ Int, UnitRange{Int}, Vararg{Pair{Any, Union{}}}, Float64, Memory{2} ]) === Union{} @test Base.return_types(Tuple{Tuple{Int, Vararg{Pair{Any, Union{}}}}},) do x; Returns(true)(x...); end |> only === Bool + +# issue #57292 +f57292(xs::Union{Tuple{String}, Int}...) = getfield(xs...) +g57292(xs::String...) = getfield(("abc",), 1, :not_atomic, xs...) +@test Base.infer_return_type(f57292) == String +@test Base.infer_return_type(g57292) == String From 339db6b5a445dfbceba0120e756db8a4bd75a5f0 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Mon, 10 Feb 2025 19:57:55 -0500 Subject: [PATCH 19/31] docs: fix-up world-age handling for META access (#57349) This should be enough to get rid of the 3 lingering warnings in our main test suite. (cherry picked from commit 751a0d728e32932a3f6f57065e9e792ce9325a8c) --- base/docs/Docs.jl | 8 ++++---- test/testhelpers/OffsetDenseArrays.jl | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 061a94bffd9cf..ae3891e218824 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -75,8 +75,8 @@ const META = gensym(:meta) const METAType = IdDict{Any,Any} function meta(m::Module; autoinit::Bool=true) - if !isdefinedglobal(m, META) - return autoinit ? invokelatest(initmeta, m) : nothing + if !invokelatest(isdefinedglobal, m, META) + return autoinit ? initmeta(m) : nothing end # TODO: This `invokelatest` is not technically required, but because # of the automatic constant backdating is currently required to avoid @@ -85,13 +85,13 @@ function meta(m::Module; autoinit::Bool=true) end function initmeta(m::Module) - if !isdefinedglobal(m, META) + if !invokelatest(isdefinedglobal, m, META) val = METAType() Core.eval(m, :(const $META = $val)) push!(modules, m) return val end - return getglobal(m, META) + return invokelatest(getglobal, m, META) end function signature!(tv::Vector{Any}, expr::Expr) diff --git a/test/testhelpers/OffsetDenseArrays.jl b/test/testhelpers/OffsetDenseArrays.jl index 44a1b8d627800..fb256234e2099 100644 --- a/test/testhelpers/OffsetDenseArrays.jl +++ b/test/testhelpers/OffsetDenseArrays.jl @@ -24,7 +24,7 @@ function Base.setindex(x::OffsetDenseArray, v, i::Integer) x.x[i - x.offset] = v end -IndexStyle(::Type{<:OffsetDenseArray}) = Base.IndexLinear() +Base.IndexStyle(::Type{<:OffsetDenseArray}) = Base.IndexLinear() Base.axes(x::OffsetDenseArray) = (x.offset + 1 : x.offset + length(x.x),) Base.keys(x::OffsetDenseArray) = only(axes(x)) From 547c8a4bad24f0b02205addb42ab622bed181d69 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Mon, 10 Feb 2025 23:32:33 -0300 Subject: [PATCH 20/31] Add missing type asserts when taking the queue out of the task struct (#57344) This was causing some dynamic dispatches (caught by the juliac test) (cherry picked from commit 90662b669e6b0a25f8b276fe60efba2a8018711c) --- base/experimental.jl | 2 +- base/lock.jl | 2 +- stdlib/REPL/test/repl.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/base/experimental.jl b/base/experimental.jl index e35e920298c3d..f56f8d8234282 100644 --- a/base/experimental.jl +++ b/base/experimental.jl @@ -658,7 +658,7 @@ function wait_with_timeout(c::GenericCondition; first::Bool=false, timeout::Real # Confirm that the waiting task is still in the wait queue and remove it. If # the task is not in the wait queue, it must have been notified already so we # don't do anything here. - if !waiter_left[] && ct.queue == c.waitq + if !waiter_left[] && ct.queue === c.waitq dosched = true Base.list_deletefirst!(c.waitq, ct) end diff --git a/base/lock.jl b/base/lock.jl index 17a08888bf711..869e61599b443 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -252,7 +252,7 @@ function wait_no_relock(c::GenericCondition) try return wait() catch - ct.queue === nothing || list_deletefirst!(ct.queue, ct) + ct.queue === nothing || list_deletefirst!(ct.queue::IntrusiveLinkedList{Task}, ct) rethrow() end end diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 018cf9c36430e..c1c5c7844bc96 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -32,7 +32,7 @@ function kill_timer(delay) # **DON'T COPY ME.** # The correct way to handle timeouts is to close the handle: # e.g. `close(stdout_read); close(stdin_write)` - test_task.queue === nothing || Base.list_deletefirst!(test_task.queue, test_task) + test_task.queue === nothing || Base.list_deletefirst!(test_task.queue::IntrusiveLinkedList{Task}, test_task) schedule(test_task, "hard kill repl test"; error=true) print(stderr, "WARNING: attempting hard kill of repl test after exceeding timeout\n") end From 8b64d3752b4b32db31d8dd7416cb48c5b4bdea40 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Mon, 10 Feb 2025 21:54:40 -0500 Subject: [PATCH 21/31] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20Sp?= =?UTF-8?q?arseArrays=20stdlib=20from=20212981b=20to=2072c7cac=20(#57348)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: SparseArrays URL: https://github.com/JuliaSparse/SparseArrays.jl.git Stdlib branch: main Julia branch: master Old commit: 212981b New commit: 72c7cac Julia version: 1.13.0-DEV SparseArrays version: 1.12.0(Does not match) Bump invoked by: @ViralBShah Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaSparse/SparseArrays.jl/compare/212981bf29b03ba460d3251ee9aa4399931b3f2d...72c7cac6bbf21367a3c2fbc5c50e908aea5984bb ``` $ git log --oneline 212981b..72c7cac 72c7cac Explicitly declare type constructor imports (#598) ff083ce README: update github action badge (#596) ``` Co-authored-by: ViralBShah <744411+ViralBShah@users.noreply.github.com> (cherry picked from commit ac79b9f413784be185db0487e785c526c57d7daa) --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/SparseArrays.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/SparseArrays-212981bf29b03ba460d3251ee9aa4399931b3f2d.tar.gz/md5 delete mode 100644 deps/checksums/SparseArrays-212981bf29b03ba460d3251ee9aa4399931b3f2d.tar.gz/sha512 create mode 100644 deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/md5 create mode 100644 deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/sha512 diff --git a/deps/checksums/SparseArrays-212981bf29b03ba460d3251ee9aa4399931b3f2d.tar.gz/md5 b/deps/checksums/SparseArrays-212981bf29b03ba460d3251ee9aa4399931b3f2d.tar.gz/md5 deleted file mode 100644 index 3fddcf07235f8..0000000000000 --- a/deps/checksums/SparseArrays-212981bf29b03ba460d3251ee9aa4399931b3f2d.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -621e67dc98707b587fb0f6e319dadbb2 diff --git a/deps/checksums/SparseArrays-212981bf29b03ba460d3251ee9aa4399931b3f2d.tar.gz/sha512 b/deps/checksums/SparseArrays-212981bf29b03ba460d3251ee9aa4399931b3f2d.tar.gz/sha512 deleted file mode 100644 index 68885439a1213..0000000000000 --- a/deps/checksums/SparseArrays-212981bf29b03ba460d3251ee9aa4399931b3f2d.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -5608adf92eaf7479eacf5ed75b3139438d0d4acf53d55a38c73a553c7fd899f553e1648fa657d35b9a0289e69fc461025dae5f8d15ec891eafcab3a663a8413a diff --git a/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/md5 b/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/md5 new file mode 100644 index 0000000000000..12c4f2ff97697 --- /dev/null +++ b/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/md5 @@ -0,0 +1 @@ +3f25f8a47a7945b55c9cc53ef489a55f diff --git a/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/sha512 b/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/sha512 new file mode 100644 index 0000000000000..5daf7514ff4ed --- /dev/null +++ b/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/sha512 @@ -0,0 +1 @@ +5fd827602430e79846d974661b039902a5ab6495f94af8292a3d66c3c3a07a0c59858bfa5bfa941bf8bf6418af98d1dd41b88aad4cc7c7355a8e56cad7f1f3ac diff --git a/stdlib/SparseArrays.version b/stdlib/SparseArrays.version index 0234c754191f8..f690a34eb5ae4 100644 --- a/stdlib/SparseArrays.version +++ b/stdlib/SparseArrays.version @@ -1,4 +1,4 @@ SPARSEARRAYS_BRANCH = main -SPARSEARRAYS_SHA1 = 212981bf29b03ba460d3251ee9aa4399931b3f2d +SPARSEARRAYS_SHA1 = 72c7cac6bbf21367a3c2fbc5c50e908aea5984bb SPARSEARRAYS_GIT_URL := https://github.com/JuliaSparse/SparseArrays.jl.git SPARSEARRAYS_TAR_URL = https://api.github.com/repos/JuliaSparse/SparseArrays.jl/tarball/$1 From 71cd5563ca66ffe09d269109130e46a93276101c Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 7 Feb 2025 02:38:24 -0500 Subject: [PATCH 22/31] Add missing latestworld after parameterized type alias (#57299) Fixes #57267 (cherry picked from commit e553fc8c6f794118a5dfcb0506fe3a0a0cd34ee4) --- src/julia-syntax.scm | 1 + test/syntax.jl | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 726312a13d10f..035d2a84e729d 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1480,6 +1480,7 @@ `(block (= ,rr (where ,type-ex ,@params)) (,(if allow-local 'assign-const-if-global 'const) ,name ,rr) + (latestworld-if-toplevel) ,rr))) (expand-forms `(const (= ,name ,type-ex))))) diff --git a/test/syntax.jl b/test/syntax.jl index c9a5d5dd52a41..298614a3f9247 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -4050,3 +4050,10 @@ end import ..Base using .Base: zero, zero end + +# #57267 - Missing `latestworld` after typealias +abstract type A57267{S, T} end +@test_nowarn @eval begin + B57267{S} = A57267{S, 1} + const C57267 = B57267 +end From 9335d1c2e6b0cab128263a691e712227ee8a97e6 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 7 Feb 2025 00:08:34 -0500 Subject: [PATCH 23/31] Allow macrocall as function sig (#55040) @KristofferC requested that `function @main(args) end` should work. This is currently a parse error. This PR makes it work as expected by allowing macrocall as a valid signature in function (needs to exapand to a call expr). Note that this is only the flisp changes. If this PR is accepted, an equivalent change would need to be made in JuliaSyntax. (cherry picked from commit b65f0040dd367f2da76f6ed4b01924a368fe50b8) --- NEWS.md | 2 ++ base/client.jl | 15 ++++++++------- src/julia-parser.scm | 6 +++--- test/syntax.jl | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/NEWS.md b/NEWS.md index 403d832ee6f68..3a29ba1a10eac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -50,6 +50,8 @@ Language changes * It is now an error to mark a binding as both `public` and `export`ed ([#53664]). * Errors during `getfield` now raise a new `FieldError` exception type instead of the generic `ErrorException` ([#54504]). +* Macros in function-signature-position no longer require parentheses. E.g. `function @main(args) ... end` is now permitted, whereas `function (@main)(args) ... end` was required in prior Julia versions. + Compiler/Runtime improvements ----------------------------- diff --git a/base/client.jl b/base/client.jl index 3449e2fbf2e62..df0cf9043996e 100644 --- a/base/client.jl +++ b/base/client.jl @@ -605,7 +605,7 @@ The `@main` macro may be used standalone or as part of the function definition, case, parentheses are required. In particular, the following are equivalent: ``` -function (@main)(args) +function @main(args) println("Hello World") end ``` @@ -624,7 +624,7 @@ imported into `Main`, it will be treated as an entrypoint in `Main`: ``` module MyApp export main - (@main)(args) = println("Hello World") + @main(args) = println("Hello World") end using .MyApp # `julia` Will execute MyApp.main at the conclusion of script execution @@ -634,7 +634,7 @@ Note that in particular, the semantics do not attach to the method or the name: ``` module MyApp - (@main)(args) = println("Hello World") + @main(args) = println("Hello World") end const main = MyApp.main # `julia` Will *NOT* execute MyApp.main unless there is a separate `@main` annotation in `Main` @@ -644,9 +644,6 @@ const main = MyApp.main This macro is new in Julia 1.11. At present, the precise semantics of `@main` are still subject to change. """ macro main(args...) - if !isempty(args) - error("`@main` is expected to be used as `(@main)` without macro arguments.") - end if isdefined(__module__, :main) if Base.binding_module(__module__, :main) !== __module__ error("Symbol `main` is already a resolved import in module $(__module__). `@main` must be used in the defining module.") @@ -657,5 +654,9 @@ macro main(args...) global main global var"#__main_is_entrypoint__#"::Bool = true end) - esc(:main) + if !isempty(args) + Expr(:call, esc(:main), map(esc, args)...) + else + esc(:main) + end end diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 891a26bb0ea49..4415dc8686065 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -1329,13 +1329,13 @@ (define (valid-func-sig? paren sig) (and (pair? sig) - (or (eq? (car sig) 'call) - (eq? (car sig) 'tuple) + (or (memq (car sig) '(call tuple)) + (and (not paren) (eq? (car sig) 'macrocall)) (and paren (eq? (car sig) 'block)) (and paren (eq? (car sig) '...)) (and (eq? (car sig) '|::|) (pair? (cadr sig)) - (eq? (car (cadr sig)) 'call)) + (memq (car (cadr sig)) '(call macrocall))) (and (eq? (car sig) 'where) (valid-func-sig? paren (cadr sig)))))) diff --git a/test/syntax.jl b/test/syntax.jl index 298614a3f9247..ebf6ed2e0b837 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -4051,6 +4051,38 @@ end using .Base: zero, zero end + +# PR# 55040 - Macrocall as function sig +@test :(function @f()() end) == :(function (@f)() end) + +function callme end +macro callmemacro(args...) + Expr(:call, esc(:callme), map(esc, args)...) +end +function @callmemacro(a::Int) + return 1 +end +@callmemacro(b::Float64) = 2 +function @callmemacro(a::T, b::T) where T <: Int + return 3 +end +function @callmemacro(a::Int, b::Int, c::Int)::Float64 + return 4 +end +function @callmemacro(d::String) + (a, b, c) + # ^ Should not be accidentally parsed as an argument list + return 4 +end + +@test callme(1) === 1 +@test callme(2.0) === 2 +@test callme(3, 3) === 3 +@test callme(4, 4, 4) === 4.0 + +# Ambiguous 1-arg anymous vs macrosig +@test_parseerror "function (@foo(a)) end" + # #57267 - Missing `latestworld` after typealias abstract type A57267{S, T} end @test_nowarn @eval begin From 8bcd2efd16d8eda27290de937978c861248c6dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Tue, 11 Feb 2025 23:36:11 +0100 Subject: [PATCH 24/31] [OpenSSL_jll] Update to v3.0.16 (#57363) https://github.com/openssl/openssl/releases/tag/openssl-3.0.16 (cherry picked from commit 7ec984a738630d11ea976da5409553eafc4e75bc) --- deps/checksums/openssl | 76 ++++++++++++++--------------- deps/openssl.version | 2 +- stdlib/OpenSSL_jll/Project.toml | 2 +- stdlib/OpenSSL_jll/test/runtests.jl | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/deps/checksums/openssl b/deps/checksums/openssl index c973f592861f3..134ad867cbd3f 100644 --- a/deps/checksums/openssl +++ b/deps/checksums/openssl @@ -1,38 +1,38 @@ -OpenSSL.v3.0.15+2.aarch64-apple-darwin.tar.gz/md5/d11d92e6530705e3d93925bbb4dfccff -OpenSSL.v3.0.15+2.aarch64-apple-darwin.tar.gz/sha512/e30d763d956f930c3dab961ef1b382385b78cbb2324ae7f5e943420b9178bc2b086d9877c2d2b41b30a92ca109d7832a2ae50f70547fcc9788e25889d8252ffc -OpenSSL.v3.0.15+2.aarch64-linux-gnu.tar.gz/md5/d29f0d3a35d592488ba3a8bbb0dc8d0e -OpenSSL.v3.0.15+2.aarch64-linux-gnu.tar.gz/sha512/67c527c1930b903d2fbb55df1bd3fc1b8394bc4fadd15dd8fb84e776bae8c448487c117492e22b9b014f823cc7fe709695f4064639066b10427b06355540e997 -OpenSSL.v3.0.15+2.aarch64-linux-musl.tar.gz/md5/4f5313f1f18e29585951e95372a7a0fe -OpenSSL.v3.0.15+2.aarch64-linux-musl.tar.gz/sha512/48007a1f6667d6aeb87cc7287723ed00e39fe2bc9c353ff33348442516f1a28961985cc4a29a2a8f76b3a7049bd955973562d7c6c4af43af884596def636f7f8 -OpenSSL.v3.0.15+2.aarch64-unknown-freebsd.tar.gz/md5/5b6041353197bb8f75b39ed8f58cf4e9 -OpenSSL.v3.0.15+2.aarch64-unknown-freebsd.tar.gz/sha512/9be617d51fdc167085887380e720e6baf8e1e180f455b297f44d0bc0862fd490f015b5292d952d4ad095750cde796cc7dac4f901389b73135cb399b3a9d378c1 -OpenSSL.v3.0.15+2.armv6l-linux-gnueabihf.tar.gz/md5/858f548a28e289153842226473138a3e -OpenSSL.v3.0.15+2.armv6l-linux-gnueabihf.tar.gz/sha512/f9385678fca65d1fb8d96756442518b16607a57a9b6d76991414b37dfc4e30a7e1eebe5f3977b088b491216af4a34f958b64fe95062ee9ae23a9212f46c4e923 -OpenSSL.v3.0.15+2.armv6l-linux-musleabihf.tar.gz/md5/c4e52ecb4f9e24d948724424f1070071 -OpenSSL.v3.0.15+2.armv6l-linux-musleabihf.tar.gz/sha512/12f9276c68049026f2741c7d97e62d24525e5e832911546e1ea3868362034e6384304d749730122edf828b8c5573084055d59cc0bd75bda32f000ce630837c2b -OpenSSL.v3.0.15+2.armv7l-linux-gnueabihf.tar.gz/md5/767d3f3047366ccd6e2aa275f80d9f6c -OpenSSL.v3.0.15+2.armv7l-linux-gnueabihf.tar.gz/sha512/17700fd33c221070a7dd2db79d045e102591b85e16b3d4099356fb6a8635aea297b5fcef91740f75c55344a12ed356772b3b85c0cc68627856093ceb53ea8eb3 -OpenSSL.v3.0.15+2.armv7l-linux-musleabihf.tar.gz/md5/3ef2385cb1fec9e2d3af2ba9385ac733 -OpenSSL.v3.0.15+2.armv7l-linux-musleabihf.tar.gz/sha512/6156e9431fa8269b8d037149271be6cca0b119be67be01cfd958dabf59cdd468ef2a5ebf885e5835585006efdedd29afc308076283d070d4ae743146b57cd2b1 -OpenSSL.v3.0.15+2.i686-linux-gnu.tar.gz/md5/e62992d214cec6b1970f9fbd04cb8ecd -OpenSSL.v3.0.15+2.i686-linux-gnu.tar.gz/sha512/dfdb3d2d1d5fed7bf1c322899d6138c81f0653350f4b918858dd51bf7bcc86d2d04de824533925fa5f8d366a5c18ee33ade883f50a538b657717f8a428be8c60 -OpenSSL.v3.0.15+2.i686-linux-musl.tar.gz/md5/186a6bb8055ce089ac0c9897bd2cd697 -OpenSSL.v3.0.15+2.i686-linux-musl.tar.gz/sha512/f3c8d608113e9b0e91dd6af697172a46892d4a66572e35e13ad394397291dded3042667c1ec4fafe051778e71ff56a876dc3e848a2b85cef9f925ef3969ab950 -OpenSSL.v3.0.15+2.i686-w64-mingw32.tar.gz/md5/b72b8e4883337e4bc90094dce86c8b8b -OpenSSL.v3.0.15+2.i686-w64-mingw32.tar.gz/sha512/3b5ddef15ca1463ab92ef5b88df36f8418c8c44ffb123a0922e55718ab317b5fe379994aba9a5e8ca112475043d5cf99b1574702cdb30de438f458ee06ac80ea -OpenSSL.v3.0.15+2.powerpc64le-linux-gnu.tar.gz/md5/da194ce6f37f34cc19cc78d25c9af5e2 -OpenSSL.v3.0.15+2.powerpc64le-linux-gnu.tar.gz/sha512/e256a9d9a0af8764de730419281aa4d3ee9f6146692ec9105a318d8301d8fda5cca82c6ef4d0d7b70d721992361771724b237ce26ef81f92c295f6056d5a7cdd -OpenSSL.v3.0.15+2.riscv64-linux-gnu.tar.gz/md5/86825ee5f83ec0c827d5c051fe1a3d41 -OpenSSL.v3.0.15+2.riscv64-linux-gnu.tar.gz/sha512/7db4ae2f0a9491ae484da5b8b0c3698d970ada91c83f9783c9e5bd92006f52dffa1a4c7fb282b63e34760199a97c52793040dc306ad0986970cfa233e29cb195 -OpenSSL.v3.0.15+2.x86_64-apple-darwin.tar.gz/md5/271cc359f5bc4718659044ad5ac7631d -OpenSSL.v3.0.15+2.x86_64-apple-darwin.tar.gz/sha512/10e7575dc4cce6c617c96e6f94dbfe3058aad696292d3fac4bde7c92623f2a849b7d10e35b156b7582294b3cf103d61b3ea73605f958ee4c9f8ff05b647939a7 -OpenSSL.v3.0.15+2.x86_64-linux-gnu.tar.gz/md5/5d045d93d632af9914bff551f67eed9b -OpenSSL.v3.0.15+2.x86_64-linux-gnu.tar.gz/sha512/240791382d9549be029e2d404bc0e962f9876ab0597bf20cf34c87fcfafc3d75ba9f223641287895f9aee8519a5a33293910ed6d67bc1424ff3513eedaa8b699 -OpenSSL.v3.0.15+2.x86_64-linux-musl.tar.gz/md5/bb2637babf3730ed1117f89cb8aab34a -OpenSSL.v3.0.15+2.x86_64-linux-musl.tar.gz/sha512/b847539acc00870f77b242eeccfcf16f590493b7deb0089fa3654026f4016d40f9595d3bbb21ab981e9decfde4321da71f162beb1837a158fd3a884375a86fee -OpenSSL.v3.0.15+2.x86_64-unknown-freebsd.tar.gz/md5/23b69e0256e6c86e026be3ade20aed5c -OpenSSL.v3.0.15+2.x86_64-unknown-freebsd.tar.gz/sha512/1b7da1e13d325c7776b8e1a63aaa334bd633bb10604f8bed5f5f6a81955268b3d11ad221a5dd181dbdc7ad27c35d5754e6875d36226003c2fd7da6cd91854de1 -OpenSSL.v3.0.15+2.x86_64-w64-mingw32.tar.gz/md5/73cf4138ab403b7c9f91368a030590f9 -OpenSSL.v3.0.15+2.x86_64-w64-mingw32.tar.gz/sha512/052bb52837c29b4b18a97df71a80ad77486bd6ccef6e2e57dfa68a02754180976dc0302a158886393ef13fe91904f963119b17429a4ecc6f8b6c80ff878df05d -openssl-3.0.15.tar.gz/md5/08f458c00fff496a52ef931c481045cd -openssl-3.0.15.tar.gz/sha512/acd80f2f7924d90c1416946a5c61eff461926ad60f4821bb6b08845ea18f8452fd5e88a2c2c5bd0d7590a792cb8341a3f3be042fd0a5b6c9c1b84a497c347bbf +OpenSSL.v3.0.16+0.aarch64-apple-darwin.tar.gz/md5/72c29fd0048b0fee44410cfb82ed6235 +OpenSSL.v3.0.16+0.aarch64-apple-darwin.tar.gz/sha512/a1d23c15c16d577e7f350004c3e3e8c9f14375ca31256f4e42dcd828b610eeea269eecac58e1c3449c99dcc80b7a8885bbbd39beb5d5ff85167d953c26c10d00 +OpenSSL.v3.0.16+0.aarch64-linux-gnu.tar.gz/md5/29bf1097dbe8ac0c42ffb0c1ff9234cf +OpenSSL.v3.0.16+0.aarch64-linux-gnu.tar.gz/sha512/b388a13fbfd416feb95bac4f2f4f47605ac8ad971551ec218a8822618cd6e127ad538782524fed5c5f75ab3caf5b86107598967993ae03516706efa6a3af1010 +OpenSSL.v3.0.16+0.aarch64-linux-musl.tar.gz/md5/971ba30a9e0be025433afe3b0aae6260 +OpenSSL.v3.0.16+0.aarch64-linux-musl.tar.gz/sha512/43b6bea8d3e0ab783ed2ec1140fb9054ef0cdd0ddd34e5e95fb36c7b1b72d7e988b2bb17c878c57b0721c44b7783b2db9d4fc614a63bb557b1c32088dd01d506 +OpenSSL.v3.0.16+0.aarch64-unknown-freebsd.tar.gz/md5/3a3d963e16c7efbacdaea9754db640e3 +OpenSSL.v3.0.16+0.aarch64-unknown-freebsd.tar.gz/sha512/7c3d0ed3a7f37e879e3b8f4c5c67cf2766b5e421fab273806a0412ba12ab4de421bce094713aeb3f6c3915260cb8d7fcb35e214344131e9a5b0081cb7bf0d5dc +OpenSSL.v3.0.16+0.armv6l-linux-gnueabihf.tar.gz/md5/c94d4882e57cb9d3688127f5f82331a5 +OpenSSL.v3.0.16+0.armv6l-linux-gnueabihf.tar.gz/sha512/b8133e873a960b125d0ab8ddd5b4071a6ab269e2b2f5b36e0d723e759875d21f734ac44f4baa4122bc6b94e23f0d82d401f16c88d9c70dac4031f07dcd20597a +OpenSSL.v3.0.16+0.armv6l-linux-musleabihf.tar.gz/md5/149aacf601e86ed15cd4305d035fb7b2 +OpenSSL.v3.0.16+0.armv6l-linux-musleabihf.tar.gz/sha512/dad7bade8fdf62c642ba1a8553f8a02218dbddd15040388b0a0eaac5391fb3c606860378c5dc40b16978c2f8f3837a1698788788d83006a4b312b1c6e73b2a53 +OpenSSL.v3.0.16+0.armv7l-linux-gnueabihf.tar.gz/md5/a1d45f34e463df42c2ea77d9084a4360 +OpenSSL.v3.0.16+0.armv7l-linux-gnueabihf.tar.gz/sha512/5f5ba285564b1ff1c5db3a2fe4d2051b9b17a77e6c6da37bc739f64051d64a5bff3968d5326760c102f9ddbc3509bf3eeb3ae267acfecbd0a55ff86ec90b5cb0 +OpenSSL.v3.0.16+0.armv7l-linux-musleabihf.tar.gz/md5/81ed6b1188a7ccda1768b67fdfc6e094 +OpenSSL.v3.0.16+0.armv7l-linux-musleabihf.tar.gz/sha512/2424620b462b5596e317e77ebc294377811ec1210c65baf4cd072b62f8d303aa77b2b4f799611ca91202dc371ea4575c2c13cb222e6edd809a036b639c1595c0 +OpenSSL.v3.0.16+0.i686-linux-gnu.tar.gz/md5/714bab6e53849d0bf6d9922be51871e2 +OpenSSL.v3.0.16+0.i686-linux-gnu.tar.gz/sha512/141102d20810986b75ba3b2b4446e4a260f4ae38583b7f8dd8a59b8e0ec8b2bac03ee83127b8f4cdb67bd8fd5ebbb102cb581ed182735283109ff85d049cc55b +OpenSSL.v3.0.16+0.i686-linux-musl.tar.gz/md5/e4f5f918c011d87626a0f830243324f9 +OpenSSL.v3.0.16+0.i686-linux-musl.tar.gz/sha512/128e6c29f0537818a68cbbd262a3735ee99ccca2377874c22978782abd43c3d0f9bd49d68c05d09f37293df52c238aa33d87244276eef080959aefe42fe8c92f +OpenSSL.v3.0.16+0.i686-w64-mingw32.tar.gz/md5/a68b80b7725887ef33ea36e7d19f7cd1 +OpenSSL.v3.0.16+0.i686-w64-mingw32.tar.gz/sha512/91aeb66c49f73eaa00114a61fc2b644e278bc39948f408806c66f61529ad98d3bc1f885152e1cc275a8374dde2d5ace3695e6f70eec718bacb0269560bce83b0 +OpenSSL.v3.0.16+0.powerpc64le-linux-gnu.tar.gz/md5/ec36f9b42c64ab4b1ce862229bc06924 +OpenSSL.v3.0.16+0.powerpc64le-linux-gnu.tar.gz/sha512/71470814b096ca9127fc2055b4ecc6e6acb30f03fe16754010c5c860f8d82cb37c2e723dca3f92f2aa2e9604fd1f4d141eca333d2c7524274e33d98da34326df +OpenSSL.v3.0.16+0.riscv64-linux-gnu.tar.gz/md5/5aecf142b6849b8c2cca957830d49f4e +OpenSSL.v3.0.16+0.riscv64-linux-gnu.tar.gz/sha512/d25879a4d6b8cc76c8fc4ed49b38d48938c80ba163e7ffe276bb86fdc4bb76cd596f150564bfa3bd88fa90451916a086ca04d31ba884bd978f40f57f0e4332f7 +OpenSSL.v3.0.16+0.x86_64-apple-darwin.tar.gz/md5/41c847ee490a5935fac8c4b663d8e325 +OpenSSL.v3.0.16+0.x86_64-apple-darwin.tar.gz/sha512/04979622fae5b24b0f0d79b0853ea311e872a7ca465c6e6700e713a34fa90a182ad78db8babacf322fb288cb62caed1a73f8d47e55fd2b074238191649e0141f +OpenSSL.v3.0.16+0.x86_64-linux-gnu.tar.gz/md5/b3c143deff5a311740ccc592a1a433e9 +OpenSSL.v3.0.16+0.x86_64-linux-gnu.tar.gz/sha512/31819e2e78dbd7aeb95b84664eac9ee29b0ec4e1b0bc9e06a4ea8f7cb65c21929414d9e4e3fa6ce15944bfd7fc68d699d4016cbe5ace16e98394a60fe369541f +OpenSSL.v3.0.16+0.x86_64-linux-musl.tar.gz/md5/bed866803232e6ada4e22eadfd2b98d1 +OpenSSL.v3.0.16+0.x86_64-linux-musl.tar.gz/sha512/ee6f26d150c81e30c93f08de5428d2f92e02e12565e6dcef09bbd1029ff5477bb2176b6cc7616a7cc898dfec8c5d8263108c3558460397c71ca90af8b230e0c1 +OpenSSL.v3.0.16+0.x86_64-unknown-freebsd.tar.gz/md5/ee6f11020b0ce6eac8016877d1635a04 +OpenSSL.v3.0.16+0.x86_64-unknown-freebsd.tar.gz/sha512/dd154fe1e8c537d42919c0889036ac79e181b765c87130edfeffd2ef8414da902995469167812664e6f77f3c89379c799d4311336f4233b05796b6823838e01c +OpenSSL.v3.0.16+0.x86_64-w64-mingw32.tar.gz/md5/d08f7d27c775eec9c7e4709d0898d60e +OpenSSL.v3.0.16+0.x86_64-w64-mingw32.tar.gz/sha512/4806c67ff8f629ee6b3a7c358146675310f0812772773aad7e30d0ccee0cc085741c06d252ed16bf8f874fe794d1b8291e0cbcd65c8eacb9e87c0a5f65742c5f +openssl-3.0.16.tar.gz/md5/7b6a9cded21b9fa51877444f5defebd4 +openssl-3.0.16.tar.gz/sha512/5eea2b0c60d870549fc2b8755f1220a57f870d95fbc8d5cc5abb9589f212d10945f355c3e88ff48540a7ee1c4db774b936023ca33d7c799ea82d91eef9c1c16d diff --git a/deps/openssl.version b/deps/openssl.version index 7253e063167db..48831039a313e 100644 --- a/deps/openssl.version +++ b/deps/openssl.version @@ -3,4 +3,4 @@ OPENSSL_JLL_NAME := OpenSSL ## source build -OPENSSL_VER := 3.0.15 +OPENSSL_VER := 3.0.16 diff --git a/stdlib/OpenSSL_jll/Project.toml b/stdlib/OpenSSL_jll/Project.toml index 0773311e11043..7c8067261c253 100644 --- a/stdlib/OpenSSL_jll/Project.toml +++ b/stdlib/OpenSSL_jll/Project.toml @@ -1,6 +1,6 @@ name = "OpenSSL_jll" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.15+2" +version = "3.0.16+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/stdlib/OpenSSL_jll/test/runtests.jl b/stdlib/OpenSSL_jll/test/runtests.jl index 35431d04bfcac..8bf67288834c9 100644 --- a/stdlib/OpenSSL_jll/test/runtests.jl +++ b/stdlib/OpenSSL_jll/test/runtests.jl @@ -6,5 +6,5 @@ using Test, Libdl, OpenSSL_jll major = ccall((:OPENSSL_version_major, libcrypto), Cuint, ()) minor = ccall((:OPENSSL_version_minor, libcrypto), Cuint, ()) patch = ccall((:OPENSSL_version_patch, libcrypto), Cuint, ()) - @test VersionNumber(major, minor, patch) == v"3.0.15" + @test VersionNumber(major, minor, patch) == v"3.0.16" end From 1e962acfc96da2fc61c222bc44599f762329fa3c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 11 Feb 2025 08:08:44 -0500 Subject: [PATCH 25/31] replace `Linking.__init__` with `OncePerProcess` (#57351) This will cause these values to be populated at slightly different times than now, but I don't think it will matter. (cherry picked from commit 55d5a4be999cab0ee377a94698ebaa5d4929f53e) --- base/linking.jl | 55 ++++++++++++++++++------------------------------- 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/base/linking.jl b/base/linking.jl index 953d80c82cc42..f3dbe6abba3ec 100644 --- a/base/linking.jl +++ b/base/linking.jl @@ -3,15 +3,8 @@ module Linking import Base.Libc: Libdl -# inlined LLD_jll -# These get calculated in __init__() -const PATH = Ref("") -const LIBPATH = Ref("") -const PATH_list = String[] -const LIBPATH_list = String[] -const lld_path = Ref{String}() +# from LLD_jll const lld_exe = Sys.iswindows() ? "lld.exe" : "lld" -const dsymutil_path = Ref{String}() const dsymutil_exe = Sys.iswindows() ? "dsymutil.exe" : "dsymutil" if Sys.iswindows() @@ -47,61 +40,51 @@ function adjust_ENV!(env::Dict, PATH::String, LIBPATH::String, adjust_PATH::Bool return env end -function __init_lld_path() +const lld_path = OncePerProcess{String}() do # Prefer our own bundled lld, but if we don't have one, pick it up off of the PATH # If this is an in-tree build, `lld` will live in `tools`. Otherwise, it'll be in `private_libexecdir` for bundled_lld_path in (joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, lld_exe), joinpath(Sys.BINDIR, "..", "tools", lld_exe), joinpath(Sys.BINDIR, lld_exe)) if isfile(bundled_lld_path) - lld_path[] = abspath(bundled_lld_path) - return + return abspath(bundled_lld_path) end end - lld_path[] = something(Sys.which(lld_exe), lld_exe) - return + return something(Sys.which(lld_exe), lld_exe) end -function __init_dsymutil_path() - #Same as with lld but for dsymutil +const dsymutil_path = OncePerProcess{String}() do + # Same as with lld but for dsymutil for bundled_dsymutil_path in (joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, dsymutil_exe), joinpath(Sys.BINDIR, "..", "tools", dsymutil_exe), joinpath(Sys.BINDIR, dsymutil_exe)) if isfile(bundled_dsymutil_path) - dsymutil_path[] = abspath(bundled_dsymutil_path) - return + return abspath(bundled_dsymutil_path) end end - dsymutil_path[] = something(Sys.which(dsymutil_exe), dsymutil_exe) - return + return something(Sys.which(dsymutil_exe), dsymutil_exe) end -const VERBOSE = Ref{Bool}(false) +PATH() = dirname(lld_path()) -function __init__() - VERBOSE[] = something(Base.get_bool_env("JULIA_VERBOSE_LINKING", false), false) - - __init_lld_path() - __init_dsymutil_path() - PATH[] = dirname(lld_path[]) +const LIBPATH = OncePerProcess{String}() do if Sys.iswindows() # On windows, the dynamic libraries (.dll) are in Sys.BINDIR ("usr\\bin") - append!(LIBPATH_list, [abspath(Sys.BINDIR, Base.LIBDIR, "julia"), Sys.BINDIR]) + LIBPATH_list = [abspath(Sys.BINDIR, Base.LIBDIR, "julia"), Sys.BINDIR] else - append!(LIBPATH_list, [abspath(Sys.BINDIR, Base.LIBDIR, "julia"), abspath(Sys.BINDIR, Base.LIBDIR)]) + LIBPATH_list = [abspath(Sys.BINDIR, Base.LIBDIR, "julia"), abspath(Sys.BINDIR, Base.LIBDIR)] end - LIBPATH[] = join(LIBPATH_list, pathsep) - return + return join(LIBPATH_list, pathsep) end function lld(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) - env = adjust_ENV!(copy(ENV), PATH[], LIBPATH[], adjust_PATH, adjust_LIBPATH) - return Cmd(Cmd([lld_path[]]); env) + env = adjust_ENV!(copy(ENV), PATH(), LIBPATH(), adjust_PATH, adjust_LIBPATH) + return Cmd(Cmd([lld_path()]); env) end function dsymutil(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) - env = adjust_ENV!(copy(ENV), PATH[], LIBPATH[], adjust_PATH, adjust_LIBPATH) - return Cmd(Cmd([dsymutil_path[]]); env) + env = adjust_ENV!(copy(ENV), PATH(), LIBPATH(), adjust_PATH, adjust_LIBPATH) + return Cmd(Cmd([dsymutil_path()]); env) end function ld() @@ -149,6 +132,8 @@ else shlibdir() = libdir() end +verbose_linking() = something(Base.get_bool_env("JULIA_VERBOSE_LINKING", false), false) + function link_image_cmd(path, out) PRIVATE_LIBDIR = "-L$(private_libdir())" SHLIBDIR = "-L$(shlibdir())" @@ -158,7 +143,7 @@ function link_image_cmd(path, out) LIBS = (LIBS..., "-lopenlibm", "-lssp", "-lgcc_s", "-lgcc", "-lmsvcrt") end - V = VERBOSE[] ? "--verbose" : "" + V = verbose_linking() ? "--verbose" : "" `$(ld()) $V $SHARED -o $out $WHOLE_ARCHIVE $path $NO_WHOLE_ARCHIVE $PRIVATE_LIBDIR $SHLIBDIR $LIBS` end From a1233eee2f7e0e92f50acda973c469354c4ffb80 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 11 Feb 2025 08:10:01 -0500 Subject: [PATCH 26/31] trimming: make sure to fail / warn on `Expr(:call, ...)` (#57342) This dispatch should not be treated as resolved just because arg0 is constant. With the previous code, that meant it was eligible for last-minute call resolution, but call-resolution in codegen is now forbidden so this needs to fail unilaterally. (cherry picked from commit 0b74d1752e4a02f9078e95bc9893619b2e52a0b6) --- src/codegen.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/codegen.cpp b/src/codegen.cpp index da34d8720d359..4a836b4f2a4c6 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5923,6 +5923,7 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo if (ctx.params->trim != JL_TRIM_NO) { // TODO: Implement the last-minute call resolution that used to be here // in inference instead. + failed_dispatch = 1; } if (failed_dispatch && trim_may_error(ctx.params->trim)) { From 102337bba366367a04e7c5c7bdfa1b0cbca7d664 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 11 Feb 2025 17:46:20 +0100 Subject: [PATCH 27/31] add a missing type assert to `OncePerTask` callable (#57356) also test that the difference `OncePerX` infer (cherry picked from commit 73506ed5e7f6f40028d4851c99ec45b5dc452adc) --- base/lock.jl | 4 +++- test/threads.jl | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/base/lock.jl b/base/lock.jl index 869e61599b443..79a2d1491bc00 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -933,4 +933,6 @@ mutable struct OncePerTask{T, F} <: Function OncePerTask{T,F}(initializer::F) where {T, F} = new{T,F}(initializer) OncePerTask(initializer) = new{Base.promote_op(initializer), typeof(initializer)}(initializer) end -@inline (once::OncePerTask)() = get!(once.initializer, task_local_storage(), once) +@inline function (once::OncePerTask{T})() where {T} + get!(once.initializer, task_local_storage(), once)::T +end diff --git a/test/threads.jl b/test/threads.jl index a1da408252fc4..52d0546f0e31b 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -439,7 +439,7 @@ end let once = OncePerProcess(() -> return [nothing]) @test typeof(once) <: OncePerProcess{Vector{Nothing}} - x = once() + x = @inferred once() @test x === once() @atomic once.state = 0xff @test_throws ErrorException("invalid state for OncePerProcess") once() @@ -456,7 +456,7 @@ let e = Base.Event(true), started = Channel{Int16}(Inf), finish = Channel{Nothing}(Inf), exiting = Channel{Nothing}(Inf), - starttest2 = Event(), + starttest2 = Base.Event(), once = OncePerThread() do push!(started, threadid()) take!(finish) @@ -468,7 +468,7 @@ let e = Base.Event(true), @test typeof(once) <: OncePerThread{Vector{Nothing}} push!(finish, nothing) @test_throws ArgumentError once[0] - x = once() + x = @inferred once() @test_throws ArgumentError once[0] @test x === once() === fetch(@async once()) === once[threadid()] @test take!(started) == threadid() @@ -558,7 +558,7 @@ end let once = OncePerTask(() -> return [nothing]) @test typeof(once) <: OncePerTask{Vector{Nothing}} - x = once() + x = @inferred once() @test x === once() !== fetch(@async once()) delete!(task_local_storage(), once) @test x !== once() === once() From 8315340fbb8edacadc32df57832a9abeef9df2ff Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 11 Feb 2025 23:47:09 +0100 Subject: [PATCH 28/31] use an empty symbol instead of module name as placeholder for module `file` metadata (#57359) Based on the discussion in https://github.com/JuliaLang/julia/pull/55963#discussion_r1786421289 (cherry picked from commit c46b5334547293fdc93fb9b5116d061bb84e9d0c) --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 402a86dfd4aef..7fb2af14ef271 100644 --- a/src/module.c +++ b/src/module.c @@ -229,7 +229,7 @@ JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, ui m->compile = -1; m->infer = -1; m->max_methods = -1; - m->file = name; // Using the name as a placeholder is better than nothing + m->file = jl_empty_sym; m->line = 0; m->hash = parent == NULL ? bitmix(name->hash, jl_module_type->hash) : bitmix(name->hash, parent->hash); From 86158263a6fd9732f6dd18b2ce5b127f0709b8ab Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 11 Feb 2025 21:14:04 -0500 Subject: [PATCH 29/31] Bump Documenter to v1.8.1 (#57362) This should get fix the binding world-age warnings in our `doctest` CI (cherry picked from commit 796a849d11e5a5b72df327e005dac6ff84307b65) --- doc/Manifest.toml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/Manifest.toml b/doc/Manifest.toml index 481b7240c0caf..05dd8edebe5a6 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -45,9 +45,9 @@ version = "0.9.3" [[deps.Documenter]] deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] -git-tree-sha1 = "d0ea2c044963ed6f37703cead7e29f70cba13d7e" +git-tree-sha1 = "182a9a3fe886587ba230a417f1651a4cbc2b92d4" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.8.0" +version = "1.8.1" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] @@ -117,7 +117,7 @@ version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.9.1+0" +version = "8.11.1+1" [[deps.LibGit2]] deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] @@ -127,12 +127,12 @@ version = "1.11.0" [[deps.LibGit2_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.8.4+0" +version = "1.9.0+0" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "OpenSSL_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.3+0" +version = "1.11.3+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -165,21 +165,21 @@ version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2024.3.11" +version = "2024.12.31" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.2.0" +version = "1.3.0" [[deps.OpenSSL_jll]] -deps = ["Artifacts", "Libdl", "NetworkOptions"] +deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.15+1" +version = "3.0.15+2" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.43.0+1" +version = "10.44.0+1" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] @@ -277,14 +277,14 @@ version = "1.11.0" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.3.1+1" +version = "1.3.1+2" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.63.0+1" +version = "1.64.0+1" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.5.0+1" +version = "17.5.0+2" From ba4290bf24d477e36873455053a35491a521f674 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 12 Feb 2025 13:11:04 +0100 Subject: [PATCH 30/31] update path to Compiler stdlib to a valid one similar to how it is done for other stdlibs (#57274) Fixes #56865. The Compiler "stdlib" resides in a different location so the current logic didn't handle updating it. (cherry picked from commit 9339dea0426c5416e97b9a4bf2a2b52bb678ad91) --- base/methodshow.jl | 14 +++++++++----- stdlib/InteractiveUtils/test/runtests.jl | 4 ++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/base/methodshow.jl b/base/methodshow.jl index a2158cb9180e4..7fdefc9b7311f 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -131,13 +131,17 @@ function fixup_stdlib_path(path::String) # The file defining Base.Sys gets included after this file is included so make sure # this function is valid even in this intermediary state if isdefined(@__MODULE__, :Sys) - BUILD_STDLIB_PATH = Sys.BUILD_STDLIB_PATH::String - STDLIB = Sys.STDLIB::String - if BUILD_STDLIB_PATH != STDLIB + if Sys.BUILD_STDLIB_PATH != Sys.STDLIB # BUILD_STDLIB_PATH gets defined in sysinfo.jl npath = normpath(path) - npath′ = replace(npath, normpath(BUILD_STDLIB_PATH) => normpath(STDLIB)) - return npath == npath′ ? path : npath′ + npath′ = replace(npath, normpath(Sys.BUILD_STDLIB_PATH) => normpath(Sys.STDLIB)) + path = npath == npath′ ? path : npath′ + end + if isdefined(@__MODULE__, :Core) && isdefined(Core, :Compiler) + compiler_folder = dirname(String(Base.moduleloc(Core.Compiler).file)) + if dirname(path) == compiler_folder + return abspath(Sys.STDLIB, "..", "..", "Compiler", "src", basename(path)) + end end end return path diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index 0de67fea69dea..739ed5fac9ef2 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -658,6 +658,10 @@ file, ln = functionloc(versioninfo, Tuple{}) @test isfile(pathof(InteractiveUtils)) @test isdir(pkgdir(InteractiveUtils)) +# compiler stdlib path updating +file, ln = functionloc(Core.Compiler.tmeet, Tuple{Int, Float64}) +@test isfile(file) + @testset "buildbot path updating" begin file, ln = functionloc(versioninfo, Tuple{}) @test isfile(file) From ebef6108017755475dc6321c0fc7f58b7b9c7545 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Wed, 12 Feb 2025 07:12:00 -0500 Subject: [PATCH 31/31] cconvert(Ref{BigFloat}, x) should return BigFloatData (#57367) #55906 changed `cconvert(Ref{BigFloat}, x::BigFloat)` to return `x.d`, but neglected to do so for other types of `x`, where it still returns a `Ref{BigFloat}` and hence is now returning the wrong type for `ccall`. Not only does this break backwards compatibility (https://github.com/JuliaMath/SpecialFunctions.jl/issues/485), but it also seems simply wrong: the *whole* job of `cconvert` is to convert objects to the correct type for use with `ccall`. This PR does so (at least for `Number` and `Ref{BigFloat}`). (cherry picked from commit 6dca4f4eda08df9cff9626dba8cb9539818b28f8) --- base/mpfr.jl | 2 ++ test/mpfr.jl | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/base/mpfr.jl b/base/mpfr.jl index 1e39f52b9d1a3..933e8eb46fb27 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -211,6 +211,8 @@ end Base.unsafe_convert(::Type{Ref{BigFloat}}, x::Ptr{BigFloat}) = error("not compatible with mpfr") Base.unsafe_convert(::Type{Ref{BigFloat}}, x::Ref{BigFloat}) = error("not compatible with mpfr") Base.cconvert(::Type{Ref{BigFloat}}, x::BigFloat) = x.d # BigFloatData is the Ref type for BigFloat +Base.cconvert(::Type{Ref{BigFloat}}, x::Number) = convert(BigFloat, x).d # avoid default conversion to Ref(BigFloat(x)) +Base.cconvert(::Type{Ref{BigFloat}}, x::Ref{BigFloat}) = x[].d function Base.unsafe_convert(::Type{Ref{BigFloat}}, x::BigFloatData) d = getfield(x, :d) p = Base.unsafe_convert(Ptr{Limb}, d) diff --git a/test/mpfr.jl b/test/mpfr.jl index c212bdfc92821..3bb8768b280d5 100644 --- a/test/mpfr.jl +++ b/test/mpfr.jl @@ -1097,3 +1097,10 @@ end end end end + +# BigFloatData is the Ref type for BigFloat in ccall: +@testset "cconvert(Ref{BigFloat}, x)" begin + for x in (1.0, big"1.0", Ref(big"1.0")) + @test Base.cconvert(Ref{BigFloat}, x) isa Base.MPFR.BigFloatData + end +end