Backports for upcoming 1.12.0(-beta1) (#57258)

Backported PRs:
- [x] #57142 <!-- Add reference to time_ns in time -->
- [x] #57241 <!-- Handle `waitpid` race condition when `SIGCHLD` is set
to `SIG_IGN` -->
- [x] #57249 <!-- restore non-freebsd-unix fix for profiling -->
- [x] #57211 <!-- Ensure read/readavailable for BufferStream are
threadsafe -->
- [x] #57262 <!-- edit NEWS for v1.12 -->
- [x] #57226 <!-- cfunction: reimplement, as originally planned, for
reliable performance -->
- [x] #57253 <!-- bpart: Fully switch to partitioned semantics -->
- [x] #57273 <!-- fix "Right arrow autocompletes at line end"
implementation -->
- [x] #57280 <!-- dep: Update JuliaSyntax -->
- [x] #57229 <!-- staticdata: Close data race after backedge insertion
-->
- [x] #57298 <!-- Updating binding version to fix MMTk CI -->
- [x] #57248 <!-- improve concurrency safety for `Compiler.finish!` -->
- [x] #57312 <!-- Profile.print: de-focus sleeping frames as gray -->
- [x] #57289 <!-- Make `OncePerX` subtype `Function` -->
- [x] #57310 <!-- Make ptls allocations at least 128 byte aligned -->
- [x] #57311 <!-- Add a warning for auto-import of types -->
- [x] #57338 <!-- fix typo in Float32 random number generation -->
- [x] #57293 <!-- Fix getfield_tfunc when order or boundscheck is Vararg
-->
- [x] #57349 <!-- docs: fix-up world-age handling for META access -->
- [x] #57344 <!-- Add missing type asserts when taking the queue out of
the task struct -->
- [x] #57348 <!-- 🤖 [master] Bump the SparseArrays stdlib from 212981b
to 72c7cac -->
- [x] #55040 <!-- Allow macrocall as function sig -->
- [x] #57299 <!-- Add missing latestworld after parameterized type alias
-->

Author: KristofferC

Compiler/src/Compiler.jl

@@ -49,7 +49,7 @@ using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstanc
 
 using Base
 using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospecializeinfer,
-    BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST,
+    BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST, BINDING_KIND_DECLARED,
     Base, BitVector, Bottom, Callable, DataTypeFieldDesc,
     EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES,
     OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME,

Compiler/src/abstractinterpretation.jl

@@ -3464,14 +3464,7 @@ world_range(ci::CodeInfo) = WorldRange(ci.min_world, ci.max_world)
 world_range(ci::CodeInstance) = WorldRange(ci.min_world, ci.max_world)
 world_range(compact::IncrementalCompact) = world_range(compact.ir)
 
-function force_binding_resolution!(g::GlobalRef, world::UInt)
-    # Force resolution of the binding
-    # TODO: This will go away once we switch over to fully partitioned semantics
-    ccall(:jl_force_binding_resolution, Cvoid, (Any, Csize_t), g, world)
-    return nothing
-end
-
-function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}, retry_after_resolve::Bool=true)
+function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact})
     worlds = world_range(src)
     partition = lookup_binding_partition(min_world(worlds), g)
     partition.max_world < max_world(worlds) && return Any
@@ -3480,25 +3473,18 @@ function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode,
         partition = lookup_binding_partition(min_world(worlds), imported_binding)
         partition.max_world < max_world(worlds) && return Any
     end
-    if is_some_guard(binding_kind(partition))
-        if retry_after_resolve
-            # This method is surprisingly hot. For performance, don't ask the runtime to resolve
-            # the binding unless necessary - doing so triggers an additional lookup, which though
-            # not super expensive is hot enough to show up in benchmarks.
-            force_binding_resolution!(g, min_world(worlds))
-            return abstract_eval_globalref_type(g, src, false)
-        end
+    kind = binding_kind(partition)
+    if is_some_guard(kind)
         # return Union{}
         return Any
     end
-    if is_some_const_binding(binding_kind(partition))
+    if is_some_const_binding(kind)
         return Const(partition_restriction(partition))
     end
-    return partition_restriction(partition)
+    return kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition)
 end
 
 function lookup_binding_partition!(interp::AbstractInterpreter, g::GlobalRef, sv::AbsIntState)
-    force_binding_resolution!(g, get_inference_world(interp))
     partition = lookup_binding_partition(get_inference_world(interp), g)
     update_valid_age!(sv, WorldRange(partition.min_world, partition.max_world))
     partition
@@ -3541,7 +3527,11 @@ function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Co
         return RTEffects(rt, Union{}, Effects(EFFECTS_TOTAL, inaccessiblememonly=is_mutation_free_argtype(rt) ? ALWAYS_TRUE : ALWAYS_FALSE))
     end
 
-    rt = partition_restriction(partition)
+    if kind == BINDING_KIND_DECLARED
+        rt = Any
+    else
+        rt = partition_restriction(partition)
+    end
     return RTEffects(rt, UndefVarError, generic_getglobal_effects)
 end
 
@@ -3580,7 +3570,7 @@ function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partitio
     elseif is_some_const_binding(kind)
         return Pair{Any,Any}(Bottom, ErrorException)
     end
-    ty = partition_restriction(partition)
+    ty = kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition)
     wnewty = widenconst(newty)
     if !hasintersect(wnewty, ty)
         return Pair{Any,Any}(Bottom, TypeError)

Compiler/src/ssair/ir.jl

@@ -581,7 +581,7 @@ function is_relevant_expr(e::Expr)
                       :foreigncall, :isdefined, :copyast,
                       :throw_undef_if_not,
                       :cfunction, :method, :pop_exception,
-                      :leave,
+                      :leave, :const, :globaldecl,
                       :new_opaque_closure)
 end
 

Compiler/src/ssair/verify.jl

@@ -61,7 +61,6 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int,
             raise_error()
         end
     elseif isa(op, GlobalRef)
-        force_binding_resolution!(op, min_world(ir.valid_worlds))
         bpart = lookup_binding_partition(min_world(ir.valid_worlds), op)
         while is_some_imported(binding_kind(bpart)) && max_world(ir.valid_worlds) <= bpart.max_world
             imported_binding = partition_restriction(bpart)::Core.Binding

Compiler/src/tfuncs.jl

@@ -1073,17 +1073,15 @@ end
 end
 
 @nospecs function getfield_tfunc(𝕃::AbstractLattice, s00, name, boundscheck_or_order)
-    t = isvarargtype(boundscheck_or_order) ? unwrapva(boundscheck_or_order) :
-        widenconst(boundscheck_or_order)
-    hasintersect(t, Symbol) || hasintersect(t, Bool) || return Bottom
+    if !isvarargtype(boundscheck_or_order)
+        t = widenconst(boundscheck_or_order)
+        hasintersect(t, Symbol) || hasintersect(t, Bool) || return Bottom
+    end
     return getfield_tfunc(𝕃, s00, name)
 end
 @nospecs function getfield_tfunc(𝕃::AbstractLattice, s00, name, order, boundscheck)
     hasintersect(widenconst(order), Symbol) || return Bottom
-    if isvarargtype(boundscheck)
-        t = unwrapva(boundscheck)
-        hasintersect(t, Symbol) || hasintersect(t, Bool) || return Bottom
-    else
+    if !isvarargtype(boundscheck)
         hasintersect(widenconst(boundscheck), Bool) || return Bottom
     end
     return getfield_tfunc(𝕃, s00, name)

Compiler/src/typeinfer.jl

@@ -92,7 +92,7 @@ If set to `true`, record per-method-instance timings within type inference in th
 __set_measure_typeinf(onoff::Bool) = __measure_typeinf__[] = onoff
 const __measure_typeinf__ = RefValue{Bool}(false)
 
-function finish!(interp::AbstractInterpreter, caller::InferenceState)
+function finish!(interp::AbstractInterpreter, caller::InferenceState, validation_world::UInt)
     result = caller.result
     opt = result.src
     if opt isa OptimizationState
@@ -108,12 +108,7 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState)
         ci = result.ci
         # if we aren't cached, we don't need this edge
         # but our caller might, so let's just make it anyways
-        if last(result.valid_worlds) >= get_world_counter()
-            # TODO: this should probably come after all store_backedges (after optimizations) for the entire graph in finish_cycle
-            # since we should be requiring that all edges first get their backedges set, as a batch
-            result.valid_worlds = WorldRange(first(result.valid_worlds), typemax(UInt))
-        end
-        if last(result.valid_worlds) == typemax(UInt)
+        if last(result.valid_worlds) >= validation_world
             # if we can record all of the backedges in the global reverse-cache,
             # we can now widen our applicability in the global cache too
             store_backedges(ci, edges)
@@ -202,7 +197,14 @@ function finish_nocycle(::AbstractInterpreter, frame::InferenceState)
     if opt isa OptimizationState # implies `may_optimize(caller.interp) === true`
         optimize(frame.interp, opt, frame.result)
     end
-    finish!(frame.interp, frame)
+    validation_world = get_world_counter()
+    finish!(frame.interp, frame, validation_world)
+    if isdefined(frame.result, :ci)
+        # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies
+        # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining
+        # validity.
+        ccall(:jl_promote_ci_to_current, Cvoid, (Any, UInt), frame.result.ci, validation_world)
+    end
     if frame.cycleid != 0
         frames = frame.callstack::Vector{AbsIntState}
         @assert frames[end] === frame
@@ -236,10 +238,19 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei
             optimize(caller.interp, opt, caller.result)
         end
     end
+    validation_world = get_world_counter()
+    cis = CodeInstance[]
     for frameid = cycleid:length(frames)
         caller = frames[frameid]::InferenceState
-        finish!(caller.interp, caller)
+        finish!(caller.interp, caller, validation_world)
+        if isdefined(caller.result, :ci)
+            push!(cis, caller.result.ci)
+        end
     end
+    # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies
+    # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining
+    # validity.
+    ccall(:jl_promote_cis_to_current, Cvoid, (Ptr{CodeInstance}, Csize_t, UInt), cis, length(cis), validation_world)
     resize!(frames, cycleid - 1)
     return nothing
 end
@@ -1266,6 +1277,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim::
     tocompile = Vector{CodeInstance}()
     codeinfos = []
     # first compute the ABIs of everything
+    latest = true # whether this_world == world_counter()
     for this_world in reverse(sort!(worlds))
         interp = NativeInterpreter(this_world)
         for i = 1:length(methods)
@@ -1278,18 +1290,18 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim::
                 # then we want to compile and emit this
                 if item.def.primary_world <= this_world <= item.def.deleted_world
                     ci = typeinf_ext(interp, item, SOURCE_MODE_NOT_REQUIRED)
-                    ci isa CodeInstance && !use_const_api(ci) && push!(tocompile, ci)
+                    ci isa CodeInstance && push!(tocompile, ci)
                 end
-            elseif item isa SimpleVector
+            elseif item isa SimpleVector && latest
                 (rt::Type, sig::Type) = item
                 # make a best-effort attempt to enqueue the relevant code for the ccallable
                 ptr = ccall(:jl_get_specialization1,
                             #= MethodInstance =# Ptr{Cvoid}, (Any, Csize_t, Cint),
                             sig, this_world, #= mt_cache =# 0)
                 if ptr !== C_NULL
-                    mi = unsafe_pointer_to_objref(ptr)
+                    mi = unsafe_pointer_to_objref(ptr)::MethodInstance
                     ci = typeinf_ext(interp, mi, SOURCE_MODE_NOT_REQUIRED)
-                    ci isa CodeInstance && !use_const_api(ci) && push!(tocompile, ci)
+                    ci isa CodeInstance && push!(tocompile, ci)
                 end
                 # additionally enqueue the ccallable entrypoint / adapter, which implicitly
                 # invokes the above ci
@@ -1305,7 +1317,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim::
             mi = get_ci_mi(callee)
             def = mi.def
             if use_const_api(callee)
-                src = codeinfo_for_const(interp, mi, code.rettype_const)
+                src = codeinfo_for_const(interp, mi, callee.rettype_const)
             elseif haskey(interp.codegen, callee)
                 src = interp.codegen[callee]
             elseif isa(def, Method) && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 && !trim
@@ -1327,6 +1339,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim::
                 println("warning: failed to get code for ", mi)
             end
         end
+        latest = false
     end
     return codeinfos
 end

Compiler/src/validation.jl

@@ -22,7 +22,7 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}(
     :copyast => 1:1,
     :meta => 0:typemax(Int),
     :global => 1:1,
-    :globaldecl => 2:2,
+    :globaldecl => 1:2,
     :foreigncall => 5:typemax(Int), # name, RT, AT, nreq, (cconv, effects), args..., roots...
     :cfunction => 5:5,
     :isdefined => 1:2,

Compiler/test/effects.jl

@@ -378,32 +378,27 @@ let effects = Base.infer_effects(; optimize=false) do
 end
 
 # we should taint `nothrow` if the binding doesn't exist and isn't fixed yet,
-# as the cached effects can be easily wrong otherwise
-# since the inference currently doesn't track "world-age" of global variables
-@eval global_assignment_undefinedyet() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET)) = 42
 setglobal!_nothrow_undefinedyet() = setglobal!(@__MODULE__, :UNDEFINEDYET, 42)
-let effects = Base.infer_effects() do
-        global_assignment_undefinedyet()
-    end
+let effects = Base.infer_effects(setglobal!_nothrow_undefinedyet)
     @test !Compiler.is_nothrow(effects)
 end
-let effects = Base.infer_effects() do
-        setglobal!_nothrow_undefinedyet()
-    end
-    @test !Compiler.is_nothrow(effects)
+@test_throws ErrorException setglobal!_nothrow_undefinedyet()
+# This declares the binding as ::Any
+@eval global_assignment_undefinedyet() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET)) = 42
+let effects = Base.infer_effects(global_assignment_undefinedyet)
+    @test Compiler.is_nothrow(effects)
 end
-global UNDEFINEDYET::String = "0"
-let effects = Base.infer_effects() do
-        global_assignment_undefinedyet()
-    end
+# Again with type mismatch
+global UNDEFINEDYET2::String = "0"
+setglobal!_nothrow_undefinedyet2() = setglobal!(@__MODULE__, :UNDEFINEDYET2, 42)
+@eval global_assignment_undefinedyet2() = $(GlobalRef(@__MODULE__, :UNDEFINEDYET2)) = 42
+let effects = Base.infer_effects(global_assignment_undefinedyet2)
     @test !Compiler.is_nothrow(effects)
 end
-let effects = Base.infer_effects() do
-        setglobal!_nothrow_undefinedyet()
-    end
+let effects = Base.infer_effects(setglobal!_nothrow_undefinedyet2)
     @test !Compiler.is_nothrow(effects)
 end
-@test_throws Union{ErrorException,TypeError} setglobal!_nothrow_undefinedyet() # TODO: what kind of error should this be?
+@test_throws TypeError setglobal!_nothrow_undefinedyet2()
 
 # Nothrow for setfield!
 mutable struct SetfieldNothrow

Compiler/test/inference.jl

@@ -6160,7 +6160,7 @@ end === Int
     swapglobal!(@__MODULE__, :swapglobal!_xxx, x)
 end === Union{}
 
-global swapglobal!_must_throw
+eval(Expr(:const, :swapglobal!_must_throw))
 @newinterp SwapGlobalInterp
 Compiler.InferenceParams(::SwapGlobalInterp) = Compiler.InferenceParams(; assume_bindings_static=true)
 function func_swapglobal!_must_throw(x)
@@ -6188,3 +6188,9 @@ end == Union{Float64,DomainError}
 @test Compiler.argtypes_to_type(Any[ Int, UnitRange{Int}, Vararg{Pair{Any, Union{}}}, Float64 ]) === Tuple{Int, UnitRange{Int}, Float64}
 @test Compiler.argtypes_to_type(Any[ Int, UnitRange{Int}, Vararg{Pair{Any, Union{}}}, Float64, Memory{2} ]) === Union{}
 @test Base.return_types(Tuple{Tuple{Int, Vararg{Pair{Any, Union{}}}}},) do x; Returns(true)(x...); end |> only === Bool
+
+# issue #57292
+f57292(xs::Union{Tuple{String}, Int}...) = getfield(xs...)
+g57292(xs::String...) = getfield(("abc",), 1, :not_atomic, xs...)
+@test Base.infer_return_type(f57292) == String
+@test Base.infer_return_type(g57292) == String

Makefile

@@ -282,8 +282,26 @@ endif
 endif
 
 ifneq (${MMTK_PLAN},None)
+# Make sure we use the right version of $MMTK_PLAN, $MMTK_MOVING and $MMTK_BUILD
+# if we use the BinaryBuilder version of mmtk-julia
+ifeq ($(USE_BINARYBUILDER_MMTK_JULIA),1)
+ifeq (${MMTK_PLAN},Immix)
+LIB_PATH_PLAN = immix
+else ifeq (${MMTK_PLAN},StickyImmix)
+LIB_PATH_PLAN = sticky
+endif
+
+ifeq ($(MMTK_MOVING), 0)
+LIB_PATH_MOVING := non_moving
+else
+LIB_PATH_MOVING := moving
+endif
+
+JL_PRIVATE_LIBS-0 += $(LIB_PATH_PLAN)/$(LIB_PATH_MOVING)/$(MMTK_BUILD)/libmmtk_julia
+else
 JL_PRIVATE_LIBS-0 += libmmtk_julia
 endif
+endif
 
 # Note that we disable MSYS2's path munging here, as otherwise
 # it replaces our `:`-separated list as a `;`-separated one.