Fix struct/const revision

docs/src/debugging.md

@@ -23,7 +23,7 @@
 
 Currently, the best way to turn on logging is within a running Julia session:
 
 ```jldoctest; setup=(using Revise)
 julia> rlogger = Revise.debug_logger()
 Revise.ReviseLogger(Revise.LogRecord[], Debug)
 ```
@@ -104,7 +104,7 @@
 ### The structure of the logs
 
 For those who want to do a little investigating on their own, it may be helpful to
-know that Revise's core decisions are captured in the group called "Action," and they come in three
+know that Revise's core changes are captured in the group called "Action," and they come in three
 flavors:
 
 - log entries with message `"Eval"` signify a call to `eval`; for these events,

src/loading.jl

@@ -110,3 +110,19 @@ function modulefiles(mod::Module)
     included_files = filter(mf->mf.id == id, includes)
     return keypath(parentfile), [keypath(mf.filename) for mf in included_files]
 end
+
+function modulefiles_basestlibs(id)
+    ret = Revise.pkg_fileinfo(id)
+    cachefile, includes = ret === nothing ? (nothing, nothing) : ret[1:2]
+    # `cachefile` will be nothing for Base and stdlibs that *haven't* been moved out
+    cachefile === nothing && return Iterators.drop(Base._included_files, 1)  # stepping through sysimg.jl rebuilds Base, omit it
+    # stdlibs that are packages
+    mod = Base.loaded_modules[id]
+    return map(includes) do inc
+        submod = mod
+        for sm in inc.modpath
+            submod = getfield(submod, Symbol(sm))
+        end
+        return (submod, inc.filename)
+    end
+end

src/logging.jl

@@ -44,6 +44,7 @@ CoreLogging.catch_exceptions(::ReviseLogger) = false
 
 function Base.show(io::IO, l::LogRecord; kwargs...)
     verbose = get(io, :verbose, false)::Bool
+    tmin = get(io, :time_min, nothing)::Union{Float64, Nothing}
     if !isempty(kwargs)
         Base.depwarn("Supplying keyword arguments to `show(io, l::Revise.LogRecord; verbose)` is deprecated, use `IOContext` instead.", :show)
         for (kw, val) in kwargs
@@ -57,6 +58,9 @@ function Base.show(io::IO, l::LogRecord; kwargs...)
         print(io, '(', l.level, ", ", l.message, ", ", l.group, ", ", l.id, ", \"", l.file, "\", ", l.line)
     else
         print(io, "Revise ", l.message)
+        if tmin !== nothing
+            print(io, ", time=", l.kwargs[:time] - tmin)
+        end
     end
     exc = nothing
     if !isempty(l.kwargs)
@@ -69,11 +73,11 @@ function Base.show(io::IO, l::LogRecord; kwargs...)
             elseif kw === :deltainfo
                 keepitem = nothing
                 for item in val
-                    if isa(item, DataType) || isa(item, MethodSummary) || (keepitem === nothing && isa(item, Union{RelocatableExpr,Expr}))
+                    if isa(item, DataType) || isa(item, Union{MethodSummary,Vector{MethodSummary}}) || (keepitem === nothing && isa(item, Union{RelocatableExpr,Expr}))
                         keepitem = item
                     end
                 end
-                if isa(keepitem, MethodSummary)
+                if isa(keepitem, Union{MethodSummary,Vector{MethodSummary}})
                     print(io, ": ", keepitem)
                 elseif isa(keepitem, Union{RelocatableExpr,Expr})
                     print(io, ": ", firstline(keepitem))
@@ -95,6 +99,15 @@ function Base.show(io::IO, l::LogRecord; kwargs...)
     end
 end
 
+function Base.show(io::IO, ::MIME"text/plain", rlogger::ReviseLogger)
+    print(io, "ReviseLogger with min_level=", rlogger.min_level)
+    if !isempty(rlogger.logs)
+        println(io, ":")
+        ioctx = IOContext(io, :time_min => first(rlogger.logs).kwargs[:time], :compact => true)
+        show(ioctx, MIME("text/plain"), rlogger.logs)
+    end
+end
+
 const _debug_logger = ReviseLogger()
 
 """
@@ -113,6 +126,8 @@ with the following relevant fields:
     examined for possible code changes. This is typically done on the basis of `mtime`,
     the modification time of the file, and does not necessarily indicate that there were
     any changes.
+  + "Bindings": "propagating" consequences of rebinding event(s), where dependent types
+    or methods need to be re-evaluated.
 - `message`: a string containing more information. Some examples:
   + For entries in the "Action" group, `message` can be `"Eval"` when modifying
     old methods or defining new ones, "DeleteMethod" when deleting a method,

src/lowered.jl

@@ -448,10 +448,31 @@ function _methods_by_execution!(interp::Interpreter, methodinfo, frame::Frame, i
                     pc = step_expr!(interp, frame, stmt, true)
                 end
             elseif head === :call
-                f = lookup(interp, frame, stmt.args[1])
-                if isdefined(Core, :_defaultctors) && f === Core._defaultctors && length(stmt.args) == 3
-                    T = lookup(interp, frame, stmt.args[2])
-                    lnn = lookup(interp, frame, stmt.args[3])
+                f = lookup(frame, stmt.args[1])
+                if __bpart__ && f === Core._typebody!
+                    # Handle type redefinition
+                    newtype = Base.unwrap_unionall(lookup(frame, stmt.args[3]))
+                    newtypename = newtype.name
+                    oldtype = isdefinedglobal(newtypename.module, newtypename.name) ? getglobal(newtypename.module, newtypename.name) : nothing
+                    if oldtype !== nothing
+                        nfts = lookup(frame, stmt.args[4])
+                        oldtype = Base.unwrap_unionall(oldtype)
+                        ofts = fieldtypes(oldtype)
+                        if !Core._equiv_typedef(oldtype, newtype) || !all(ab -> recursive_egal(ab..., oldtype), zip(nfts, ofts))
+                            isrequired[pc:end] .= true   # ensure we evaluate all remaining statements (probably not needed, but just in case)
+                            # Find all methods restricted to `oldtype`
+                            meths = methods_with(oldtype)
+                            # For any modules that have not yet been parsed and had their signatures extracted,
+                            # we need to do this now, before the binding changes to the new type
+                            maybe_extract_sigs_for_meths(meths)
+                            union!(reeval_methods, meths)
+                        end
+                    end
+                    pc = step_expr!(interp, frame, stmt, true)
+                elseif isdefined(Core, :_defaultctors) && f === Core._defaultctors && length(stmt.args) == 3
+                    # Create the constructors for a type (i.e., a method definition)
+                    T = lookup(frame, stmt.args[2])
+                    lnn = lookup(frame, stmt.args[3])
                     if T isa Type && lnn isa LineNumberNode
                         empty!(signatures)
                         uT = Base.unwrap_unionall(T)::DataType

src/packagedef.jl

@@ -6,6 +6,10 @@ using Base: PkgId
 using Base.Meta: isexpr
 using Core: CodeInfo, MethodTable
 
+if !isdefined(Core, :isdefinedglobal)
+    isdefinedglobal(m::Module, s::Symbol) = isdefined(m, s)
+end
+
 export revise, includet, entr, MethodSummary
 
 ## BEGIN abstract Distributed API
@@ -101,6 +105,7 @@ include("utils.jl")
 include("parsing.jl")
 include("lowered.jl")
 include("loading.jl")
+include("visit.jl")
 include("pkgs.jl")
 include("git.jl")
 include("recipes.jl")
@@ -147,6 +152,15 @@ Global variable, maps `(pkgdata, filename)` pairs that errored upon last revisio
 """
 const queue_errors = Dict{Tuple{PkgData,String},Tuple{Exception, Any}}() # locking is covered by revision_queue_lock
 
+# Can we revise types?
+const __bpart__ = Base.VERSION >= v"1.12.0-DEV.2047"
+# Julia 1.12+: when bindings switch to a new type, we need to re-evaluate method
+# definitions using the new binding resolution.
+const reeval_methods = Set{Method}()
+# Track (signature, module, method) tuples for methods that failed to re-evaluate
+# due to type incompatibility and need retry when types become compatible again
+const pending_eval_sigs = Set{Tuple{MethodInfoKey, Module, Method}}()
+
 """
     Revise.NOPACKAGE
 
@@ -265,6 +279,37 @@ const silence_pkgs = Set{Symbol}()
 const depsdir = joinpath(dirname(@__DIR__), "deps")
 const silencefile = Ref(joinpath(depsdir, "silence.txt"))  # Ref so that tests don't clobber
 
+function collect_mis(sigs)
+    mis = Core.MethodInstance[]
+    world = Base.get_world_counter()
+    for tt in sigs
+        matches = Base._methods_by_ftype(tt, 10, world)::Vector
+        for mm in matches
+            m = mm.method
+            for mi in Base.specializations(m)
+                if mi.specTypes <: tt
+                    push!(mis, mi)
+                end
+            end
+        end
+    end
+    return mis
+end
+
+# from Compiler/bootstrap.jl
+const compiler_mis = if __bpart__
+    collect_mis(Any[
+        Tuple{typeof(Core.Compiler.compact!), Vararg{Any}},
+        Tuple{typeof(Core.Compiler.ssa_inlining_pass!), Core.Compiler.IRCode, Core.Compiler.InliningState{Core.Compiler.NativeInterpreter}, Bool},
+        Tuple{typeof(Core.Compiler.optimize), Core.Compiler.NativeInterpreter, Core.Compiler.OptimizationState{Core.Compiler.NativeInterpreter}, Core.Compiler.InferenceResult},
+        Tuple{typeof(Core.Compiler.typeinf_ext), Core.Compiler.NativeInterpreter, Core.MethodInstance, UInt8},
+        Tuple{typeof(Core.Compiler.typeinf), Core.Compiler.NativeInterpreter, Core.Compiler.InferenceState},
+        Tuple{typeof(Core.Compiler.typeinf_edge), Core.Compiler.NativeInterpreter, Method, Any, Core.SimpleVector, Core.Compiler.InferenceState, Bool, Bool},
+    ])
+else
+    Core.MethodInstance[]
+end
+
 ##
 ## The inputs are sets of expressions found in each file.
 ## Some of those expressions will generate methods which are identified via their signatures.
@@ -472,7 +517,7 @@ struct CodeTrackingMethodInfo
     allsigs::Vector{SigInfo}
     includes::Vector{Pair{Module,String}}
 end
-CodeTrackingMethodInfo(ex::Expr) = CodeTrackingMethodInfo([ex], SigInfo[], Pair{Module,String}[])
+CodeTrackingMethodInfo(ex::Expr) = CodeTrackingMethodInfo(Expr[ex], SigInfo[], Pair{Module,String}[])
 
 function add_signature!(methodinfo::CodeTrackingMethodInfo, mt_sig::MethodInfoKey, ln::LineNumberNode)
     locdefs = CodeTracking.invoked_get!(Vector{Tuple{LineNumberNode,Expr}}, CodeTracking.method_info, mt_sig)
@@ -712,7 +757,7 @@ function revise_file_now(pkgdata::PkgData, file)
         error(file, " is not currently being tracked.")
     end
     mexsnew, mexsold = handle_deletions(pkgdata, file)
-    if mexsnew != nothing
+    if mexsnew !== nothing
         _, includes = eval_new!(mexsnew, mexsold)
         fi = fileinfo(pkgdata, i)
         pkgdata.fileinfos[i] = FileInfo(mexsnew, fi)
@@ -768,8 +813,16 @@ otherwise these are only logged.
 function revise(; throw::Bool=false)
     active[] || return nothing
     sleep(0.01)  # in case the file system isn't quite done writing out the new files
+    # To ensure we don't just call `Core.Compiler.bootstrap!()` for reasons of invalidation rather than redefinition,
+    # record the pre-revision max world ages of the compiler methods.
+    # This means that if those methods get invalidated by loading packages, automatic revision of the compiler won't
+    # work anymore---Revise's changes won't lower the world age to anything less than it already is.
+    # But people testing the compiler often do so in a fresh & slim session, so there's still some value in
+    # automatic revision. One can always call `Compiler.bootstrap!()` manually to reinitialize the compiler.
+    cmaxworlds = Dict(mi => mi.cache.max_world for mi in compiler_mis)
     lock(revision_queue_lock) do
         have_queue_errors = !isempty(queue_errors)
+        empty!(reeval_methods)
 
         # Do all the deletion first. This ensures that a method that moved from one file to another
         # won't get redefined first and deleted second.
@@ -839,6 +892,82 @@ function revise(; throw::Bool=false)
                 queue_errors[(pkgdata, file)] = (err, catch_backtrace())
             end
         end
+        # Retry methods that previously failed to re-evaluate
+        if !isempty(pending_eval_sigs)
+            for (mt_sig, mod, m) in collect(pending_eval_sigs)
+                methinfo = get(CodeTracking.method_info, mt_sig, nothing)
+                if methinfo !== nothing && length(methinfo) == 1
+                    _, ex = methinfo[1]
+                    try
+                        invokelatest(eval_with_signatures, mod, ex; mode=:eval)
+                        # If successful, remove from both queues
+                        delete!(CodeTracking.method_info, mt_sig)
+                        delete!(pending_eval_sigs, (mt_sig, mod, m))
+                        with_logger(_debug_logger) do
+                            @debug "RetrySucceeded" _group="Action" time=time() deltainfo=(mod, ex)
+                        end
+                    catch
+                        # Keep in queue for later
+                    end
+                end
+            end
+        end
+        # Handle binding invalidations
+        if !isempty(reeval_methods)
+            handled = Base.IdSet{Type}()
+            while !isempty(reeval_methods)
+                list = collect(reeval_methods)
+                with_logger(_debug_logger) do
+                    @debug "OldTypeMethods" _group="Bindings" time=time() deltainfo=(MethodSummary.(list),)
+                end
+                empty!(reeval_methods)
+                for m in list
+                    mt_sig = MethodInfoKey(nothing, m.sig)
+                    methinfo = get(CodeTracking.method_info, mt_sig, missing)
+                    if methinfo === missing
+                        push!(handled, m.sig)
+                        continue
+                    end
+                    if length(methinfo) != 1 && Base.unwrap_unionall(m.sig).parameters[1] !== typeof(Core.kwcall)
+                        with_logger(_debug_logger) do
+                            @debug "FailedDeletion" _group="Action" time=time() deltainfo=(m.sig, methinfo)
+                        end
+                        continue
+                    end
+                    if isdefinedglobal(m.module, m.name)
+                        f = getglobal(m.module, m.name)
+                        if isa(f, DataType)
+                            newmeths = methods_with(f)
+                            filter!(m -> m.sig ∉ handled, newmeths)
+                            maybe_extract_sigs_for_meths(newmeths)
+                            union!(reeval_methods, newmeths)
+                        end
+                    end
+                    !iszero(m.dispatch_status) && with_logger(_debug_logger) do
+                        @debug "DeleteMethod" _group="Action" time=time() deltainfo=(m.sig, MethodSummary(m))
+                        Base.delete_method(m)  # ensure that "old data" doesn't get run with "old methods"
+                        _, ex = methinfo[1]
+                        @debug "Eval" _group="Action" time=time() deltainfo=(m.module, ex)
+                        try
+                            invokelatest(eval_with_signatures, m.module, ex; mode=:eval)
+                            delete!(CodeTracking.method_info, mt_sig)
+                        catch err
+                            # Re-evaluation failed, likely due to type incompatibility
+                            # Store in pending_eval_sigs for retry when types become compatible
+                            push!(pending_eval_sigs, (mt_sig, m.module, m))
+                            @debug "EvalFailed" _group="Action" time=time() deltainfo=(m.module, ex, err)
+                        end
+                    end
+                    push!(handled, m.sig)
+                end
+            end
+        end
+        # If needed, reinitialize the compiler
+        init_compiler = false
+        for mi in compiler_mis
+            init_compiler |= mi.cache.max_world < cmaxworlds[mi]
+        end
+        init_compiler && Core.Compiler.bootstrap!()
         if interrupt
             for pkgfile in finished
                 haskey(queue_errors, pkgfile) || delete!(revision_queue, pkgfile)
@@ -947,7 +1076,7 @@ function track(mod::Module, file::AbstractString; mode=:sigs, kwargs...)
         if mode === :includet
             mode = :sigs   # we already handled evaluation in `parse_source`
         end
-        instantiate_sigs!(mod_exs_sigs; mode, kwargs...)
+        invokelatest(instantiate_sigs!, mod_exs_sigs; mode, kwargs...)
         if !haskey(pkgdatas, id)
             # Wait a bit to see if `mod` gets initialized
             sleep(0.1)

src/pkgs.jl

@@ -129,6 +129,23 @@ function maybe_extract_sigs!(fi::FileInfo)
 end
 maybe_extract_sigs!(pkgdata::PkgData, file::AbstractString) = maybe_extract_sigs!(fileinfo(pkgdata, file))
 
+function maybe_extract_sigs_for_meths(meths)
+    for m in meths
+        methinfo = get(CodeTracking.method_info, MethodInfoKey(m), false)
+        if methinfo === false
+            pkgdata = get(pkgdatas, PkgId(m.module), nothing)
+            pkgdata === nothing && continue
+            for file in srcfiles(pkgdata)
+                fi = fileinfo(pkgdata, file)
+                if (isempty(fi.modexsigs) && !fi.parsed[]) && (!isempty(fi.cachefile) || !isempty(fi.cacheexprs))
+                    fi = maybe_parse_from_cache!(pkgdata, file)
+                    instantiate_sigs!(fi.modexsigs)
+                end
+            end
+        end
+    end
+end
+
 function maybe_add_includes_to_pkgdata!(pkgdata::PkgData, file::AbstractString, includes; eval_now::Bool=false)
     for (mod, inc) in includes
         inc = joinpath(splitdir(file)[1], inc)

src/precompile.jl

@@ -62,7 +62,7 @@ function _precompile_()
     mbody = bodymethod(mex)
     # use `typeof(pairs(NamedTuple()))` here since it actually differs between Julia versions
     @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, typeof(pairs(NamedTuple())), typeof(methods_by_execution!), Compiled, MI, Module, Expr})
-    @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, Iterators.Pairs{Symbol,Bool,Tuple{Symbol},NamedTuple{(:skip_include,),Tuple{Bool}}}, typeof(methods_by_execution!), Compiled, MI, Module, Expr})
+    @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, Iterators.Pairs{Symbol,Bool,Nothing,NamedTuple{(:skip_include,),Tuple{Bool}}}, typeof(methods_by_execution!), Compiled, MI, Module, Expr})
     mfr = which(_methods_by_execution!, (Compiled, MI, Frame, Vector{Bool}))
     mbody = bodymethod(mfr)
     @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, typeof(_methods_by_execution!), Compiled, MI, Frame, Vector{Bool}})