diff --git a/docs/src/debugging.md b/docs/src/debugging.md index 6d849884..2a1df0c9 100644 --- a/docs/src/debugging.md +++ b/docs/src/debugging.md @@ -104,7 +104,7 @@ on to `rlogger`.) ### The structure of the logs For those who want to do a little investigating on their own, it may be helpful to -know that Revise's core decisions are captured in the group called "Action," and they come in three +know that Revise's core changes are captured in the group called "Action," and they come in three flavors: - log entries with message `"Eval"` signify a call to `eval`; for these events, diff --git a/src/loading.jl b/src/loading.jl index c5c0af25..bf9644a1 100644 --- a/src/loading.jl +++ b/src/loading.jl @@ -110,3 +110,19 @@ function modulefiles(mod::Module) included_files = filter(mf->mf.id == id, includes) return keypath(parentfile), [keypath(mf.filename) for mf in included_files] end + +function modulefiles_basestlibs(id) + ret = Revise.pkg_fileinfo(id) + cachefile, includes = ret === nothing ? (nothing, nothing) : ret[1:2] + # `cachefile` will be nothing for Base and stdlibs that *haven't* been moved out + cachefile === nothing && return Iterators.drop(Base._included_files, 1) # stepping through sysimg.jl rebuilds Base, omit it + # stdlibs that are packages + mod = Base.loaded_modules[id] + return map(includes) do inc + submod = mod + for sm in inc.modpath + submod = getfield(submod, Symbol(sm)) + end + return (submod, inc.filename) + end +end diff --git a/src/logging.jl b/src/logging.jl index 7bf6c615..6ab35c2e 100644 --- a/src/logging.jl +++ b/src/logging.jl @@ -44,6 +44,7 @@ CoreLogging.catch_exceptions(::ReviseLogger) = false function Base.show(io::IO, l::LogRecord; kwargs...) verbose = get(io, :verbose, false)::Bool + tmin = get(io, :time_min, nothing)::Union{Float64, Nothing} if !isempty(kwargs) Base.depwarn("Supplying keyword arguments to `show(io, l::Revise.LogRecord; verbose)` is deprecated, use `IOContext` instead.", :show) for (kw, val) in kwargs @@ -57,6 +58,9 @@ function Base.show(io::IO, l::LogRecord; kwargs...) print(io, '(', l.level, ", ", l.message, ", ", l.group, ", ", l.id, ", \"", l.file, "\", ", l.line) else print(io, "Revise ", l.message) + if tmin !== nothing + print(io, ", time=", l.kwargs[:time] - tmin) + end end exc = nothing if !isempty(l.kwargs) @@ -69,11 +73,11 @@ function Base.show(io::IO, l::LogRecord; kwargs...) elseif kw === :deltainfo keepitem = nothing for item in val - if isa(item, DataType) || isa(item, MethodSummary) || (keepitem === nothing && isa(item, Union{RelocatableExpr,Expr})) + if isa(item, DataType) || isa(item, Union{MethodSummary,Vector{MethodSummary}}) || (keepitem === nothing && isa(item, Union{RelocatableExpr,Expr})) keepitem = item end end - if isa(keepitem, MethodSummary) + if isa(keepitem, Union{MethodSummary,Vector{MethodSummary}}) print(io, ": ", keepitem) elseif isa(keepitem, Union{RelocatableExpr,Expr}) print(io, ": ", firstline(keepitem)) @@ -95,6 +99,15 @@ function Base.show(io::IO, l::LogRecord; kwargs...) end end +function Base.show(io::IO, ::MIME"text/plain", rlogger::ReviseLogger) + print(io, "ReviseLogger with min_level=", rlogger.min_level) + if !isempty(rlogger.logs) + println(io, ":") + ioctx = IOContext(io, :time_min => first(rlogger.logs).kwargs[:time], :compact => true) + show(ioctx, MIME("text/plain"), rlogger.logs) + end +end + const _debug_logger = ReviseLogger() """ @@ -113,6 +126,8 @@ with the following relevant fields: examined for possible code changes. This is typically done on the basis of `mtime`, the modification time of the file, and does not necessarily indicate that there were any changes. + + "Bindings": "propagating" consequences of rebinding event(s), where dependent types + or methods need to be re-evaluated. - `message`: a string containing more information. Some examples: + For entries in the "Action" group, `message` can be `"Eval"` when modifying old methods or defining new ones, "DeleteMethod" when deleting a method, diff --git a/src/lowered.jl b/src/lowered.jl index 9eb3129c..6b665e40 100644 --- a/src/lowered.jl +++ b/src/lowered.jl @@ -448,10 +448,31 @@ function _methods_by_execution!(interp::Interpreter, methodinfo, frame::Frame, i pc = step_expr!(interp, frame, stmt, true) end elseif head === :call - f = lookup(interp, frame, stmt.args[1]) - if isdefined(Core, :_defaultctors) && f === Core._defaultctors && length(stmt.args) == 3 - T = lookup(interp, frame, stmt.args[2]) - lnn = lookup(interp, frame, stmt.args[3]) + f = lookup(frame, stmt.args[1]) + if __bpart__ && f === Core._typebody! + # Handle type redefinition + newtype = Base.unwrap_unionall(lookup(frame, stmt.args[3])) + newtypename = newtype.name + oldtype = isdefinedglobal(newtypename.module, newtypename.name) ? getglobal(newtypename.module, newtypename.name) : nothing + if oldtype !== nothing + nfts = lookup(frame, stmt.args[4]) + oldtype = Base.unwrap_unionall(oldtype) + ofts = fieldtypes(oldtype) + if !Core._equiv_typedef(oldtype, newtype) || !all(ab -> recursive_egal(ab..., oldtype), zip(nfts, ofts)) + isrequired[pc:end] .= true # ensure we evaluate all remaining statements (probably not needed, but just in case) + # Find all methods restricted to `oldtype` + meths = methods_with(oldtype) + # For any modules that have not yet been parsed and had their signatures extracted, + # we need to do this now, before the binding changes to the new type + maybe_extract_sigs_for_meths(meths) + union!(reeval_methods, meths) + end + end + pc = step_expr!(interp, frame, stmt, true) + elseif isdefined(Core, :_defaultctors) && f === Core._defaultctors && length(stmt.args) == 3 + # Create the constructors for a type (i.e., a method definition) + T = lookup(frame, stmt.args[2]) + lnn = lookup(frame, stmt.args[3]) if T isa Type && lnn isa LineNumberNode empty!(signatures) uT = Base.unwrap_unionall(T)::DataType diff --git a/src/packagedef.jl b/src/packagedef.jl index 41d85e51..3a17335a 100644 --- a/src/packagedef.jl +++ b/src/packagedef.jl @@ -6,6 +6,10 @@ using Base: PkgId using Base.Meta: isexpr using Core: CodeInfo, MethodTable +if !isdefined(Core, :isdefinedglobal) + isdefinedglobal(m::Module, s::Symbol) = isdefined(m, s) +end + export revise, includet, entr, MethodSummary ## BEGIN abstract Distributed API @@ -101,6 +105,7 @@ include("utils.jl") include("parsing.jl") include("lowered.jl") include("loading.jl") +include("visit.jl") include("pkgs.jl") include("git.jl") include("recipes.jl") @@ -147,6 +152,15 @@ Global variable, maps `(pkgdata, filename)` pairs that errored upon last revisio """ const queue_errors = Dict{Tuple{PkgData,String},Tuple{Exception, Any}}() # locking is covered by revision_queue_lock +# Can we revise types? +const __bpart__ = Base.VERSION >= v"1.12.0-DEV.2047" +# Julia 1.12+: when bindings switch to a new type, we need to re-evaluate method +# definitions using the new binding resolution. +const reeval_methods = Set{Method}() +# Track (signature, module, method) tuples for methods that failed to re-evaluate +# due to type incompatibility and need retry when types become compatible again +const pending_eval_sigs = Set{Tuple{MethodInfoKey, Module, Method}}() + """ Revise.NOPACKAGE @@ -265,6 +279,37 @@ const silence_pkgs = Set{Symbol}() const depsdir = joinpath(dirname(@__DIR__), "deps") const silencefile = Ref(joinpath(depsdir, "silence.txt")) # Ref so that tests don't clobber +function collect_mis(sigs) + mis = Core.MethodInstance[] + world = Base.get_world_counter() + for tt in sigs + matches = Base._methods_by_ftype(tt, 10, world)::Vector + for mm in matches + m = mm.method + for mi in Base.specializations(m) + if mi.specTypes <: tt + push!(mis, mi) + end + end + end + end + return mis +end + +# from Compiler/bootstrap.jl +const compiler_mis = if __bpart__ + collect_mis(Any[ + Tuple{typeof(Core.Compiler.compact!), Vararg{Any}}, + Tuple{typeof(Core.Compiler.ssa_inlining_pass!), Core.Compiler.IRCode, Core.Compiler.InliningState{Core.Compiler.NativeInterpreter}, Bool}, + Tuple{typeof(Core.Compiler.optimize), Core.Compiler.NativeInterpreter, Core.Compiler.OptimizationState{Core.Compiler.NativeInterpreter}, Core.Compiler.InferenceResult}, + Tuple{typeof(Core.Compiler.typeinf_ext), Core.Compiler.NativeInterpreter, Core.MethodInstance, UInt8}, + Tuple{typeof(Core.Compiler.typeinf), Core.Compiler.NativeInterpreter, Core.Compiler.InferenceState}, + Tuple{typeof(Core.Compiler.typeinf_edge), Core.Compiler.NativeInterpreter, Method, Any, Core.SimpleVector, Core.Compiler.InferenceState, Bool, Bool}, + ]) +else + Core.MethodInstance[] +end + ## ## The inputs are sets of expressions found in each file. ## Some of those expressions will generate methods which are identified via their signatures. @@ -472,7 +517,7 @@ struct CodeTrackingMethodInfo allsigs::Vector{SigInfo} includes::Vector{Pair{Module,String}} end -CodeTrackingMethodInfo(ex::Expr) = CodeTrackingMethodInfo([ex], SigInfo[], Pair{Module,String}[]) +CodeTrackingMethodInfo(ex::Expr) = CodeTrackingMethodInfo(Expr[ex], SigInfo[], Pair{Module,String}[]) function add_signature!(methodinfo::CodeTrackingMethodInfo, mt_sig::MethodInfoKey, ln::LineNumberNode) locdefs = CodeTracking.invoked_get!(Vector{Tuple{LineNumberNode,Expr}}, CodeTracking.method_info, mt_sig) @@ -712,7 +757,7 @@ function revise_file_now(pkgdata::PkgData, file) error(file, " is not currently being tracked.") end mexsnew, mexsold = handle_deletions(pkgdata, file) - if mexsnew != nothing + if mexsnew !== nothing _, includes = eval_new!(mexsnew, mexsold) fi = fileinfo(pkgdata, i) pkgdata.fileinfos[i] = FileInfo(mexsnew, fi) @@ -768,8 +813,16 @@ otherwise these are only logged. function revise(; throw::Bool=false) active[] || return nothing sleep(0.01) # in case the file system isn't quite done writing out the new files + # To ensure we don't just call `Core.Compiler.bootstrap!()` for reasons of invalidation rather than redefinition, + # record the pre-revision max world ages of the compiler methods. + # This means that if those methods get invalidated by loading packages, automatic revision of the compiler won't + # work anymore---Revise's changes won't lower the world age to anything less than it already is. + # But people testing the compiler often do so in a fresh & slim session, so there's still some value in + # automatic revision. One can always call `Compiler.bootstrap!()` manually to reinitialize the compiler. + cmaxworlds = Dict(mi => mi.cache.max_world for mi in compiler_mis) lock(revision_queue_lock) do have_queue_errors = !isempty(queue_errors) + empty!(reeval_methods) # Do all the deletion first. This ensures that a method that moved from one file to another # won't get redefined first and deleted second. @@ -839,6 +892,82 @@ function revise(; throw::Bool=false) queue_errors[(pkgdata, file)] = (err, catch_backtrace()) end end + # Retry methods that previously failed to re-evaluate + if !isempty(pending_eval_sigs) + for (mt_sig, mod, m) in collect(pending_eval_sigs) + methinfo = get(CodeTracking.method_info, mt_sig, nothing) + if methinfo !== nothing && length(methinfo) == 1 + _, ex = methinfo[1] + try + invokelatest(eval_with_signatures, mod, ex; mode=:eval) + # If successful, remove from both queues + delete!(CodeTracking.method_info, mt_sig) + delete!(pending_eval_sigs, (mt_sig, mod, m)) + with_logger(_debug_logger) do + @debug "RetrySucceeded" _group="Action" time=time() deltainfo=(mod, ex) + end + catch + # Keep in queue for later + end + end + end + end + # Handle binding invalidations + if !isempty(reeval_methods) + handled = Base.IdSet{Type}() + while !isempty(reeval_methods) + list = collect(reeval_methods) + with_logger(_debug_logger) do + @debug "OldTypeMethods" _group="Bindings" time=time() deltainfo=(MethodSummary.(list),) + end + empty!(reeval_methods) + for m in list + mt_sig = MethodInfoKey(nothing, m.sig) + methinfo = get(CodeTracking.method_info, mt_sig, missing) + if methinfo === missing + push!(handled, m.sig) + continue + end + if length(methinfo) != 1 && Base.unwrap_unionall(m.sig).parameters[1] !== typeof(Core.kwcall) + with_logger(_debug_logger) do + @debug "FailedDeletion" _group="Action" time=time() deltainfo=(m.sig, methinfo) + end + continue + end + if isdefinedglobal(m.module, m.name) + f = getglobal(m.module, m.name) + if isa(f, DataType) + newmeths = methods_with(f) + filter!(m -> m.sig ∉ handled, newmeths) + maybe_extract_sigs_for_meths(newmeths) + union!(reeval_methods, newmeths) + end + end + !iszero(m.dispatch_status) && with_logger(_debug_logger) do + @debug "DeleteMethod" _group="Action" time=time() deltainfo=(m.sig, MethodSummary(m)) + Base.delete_method(m) # ensure that "old data" doesn't get run with "old methods" + _, ex = methinfo[1] + @debug "Eval" _group="Action" time=time() deltainfo=(m.module, ex) + try + invokelatest(eval_with_signatures, m.module, ex; mode=:eval) + delete!(CodeTracking.method_info, mt_sig) + catch err + # Re-evaluation failed, likely due to type incompatibility + # Store in pending_eval_sigs for retry when types become compatible + push!(pending_eval_sigs, (mt_sig, m.module, m)) + @debug "EvalFailed" _group="Action" time=time() deltainfo=(m.module, ex, err) + end + end + push!(handled, m.sig) + end + end + end + # If needed, reinitialize the compiler + init_compiler = false + for mi in compiler_mis + init_compiler |= mi.cache.max_world < cmaxworlds[mi] + end + init_compiler && Core.Compiler.bootstrap!() if interrupt for pkgfile in finished haskey(queue_errors, pkgfile) || delete!(revision_queue, pkgfile) @@ -947,7 +1076,7 @@ function track(mod::Module, file::AbstractString; mode=:sigs, kwargs...) if mode === :includet mode = :sigs # we already handled evaluation in `parse_source` end - instantiate_sigs!(mod_exs_sigs; mode, kwargs...) + invokelatest(instantiate_sigs!, mod_exs_sigs; mode, kwargs...) if !haskey(pkgdatas, id) # Wait a bit to see if `mod` gets initialized sleep(0.1) diff --git a/src/pkgs.jl b/src/pkgs.jl index ec30b7e3..cab32b29 100644 --- a/src/pkgs.jl +++ b/src/pkgs.jl @@ -129,6 +129,23 @@ function maybe_extract_sigs!(fi::FileInfo) end maybe_extract_sigs!(pkgdata::PkgData, file::AbstractString) = maybe_extract_sigs!(fileinfo(pkgdata, file)) +function maybe_extract_sigs_for_meths(meths) + for m in meths + methinfo = get(CodeTracking.method_info, MethodInfoKey(m), false) + if methinfo === false + pkgdata = get(pkgdatas, PkgId(m.module), nothing) + pkgdata === nothing && continue + for file in srcfiles(pkgdata) + fi = fileinfo(pkgdata, file) + if (isempty(fi.modexsigs) && !fi.parsed[]) && (!isempty(fi.cachefile) || !isempty(fi.cacheexprs)) + fi = maybe_parse_from_cache!(pkgdata, file) + instantiate_sigs!(fi.modexsigs) + end + end + end + end +end + function maybe_add_includes_to_pkgdata!(pkgdata::PkgData, file::AbstractString, includes; eval_now::Bool=false) for (mod, inc) in includes inc = joinpath(splitdir(file)[1], inc) diff --git a/src/precompile.jl b/src/precompile.jl index 9160f9d2..3bd1b412 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -62,7 +62,7 @@ function _precompile_() mbody = bodymethod(mex) # use `typeof(pairs(NamedTuple()))` here since it actually differs between Julia versions @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, typeof(pairs(NamedTuple())), typeof(methods_by_execution!), Compiled, MI, Module, Expr}) - @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, Iterators.Pairs{Symbol,Bool,Tuple{Symbol},NamedTuple{(:skip_include,),Tuple{Bool}}}, typeof(methods_by_execution!), Compiled, MI, Module, Expr}) + @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, Iterators.Pairs{Symbol,Bool,Nothing,NamedTuple{(:skip_include,),Tuple{Bool}}}, typeof(methods_by_execution!), Compiled, MI, Module, Expr}) mfr = which(_methods_by_execution!, (Compiled, MI, Frame, Vector{Bool})) mbody = bodymethod(mfr) @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, typeof(_methods_by_execution!), Compiled, MI, Frame, Vector{Bool}}) diff --git a/src/recipes.jl b/src/recipes.jl index 727446c0..fbe572ca 100644 --- a/src/recipes.jl +++ b/src/recipes.jl @@ -67,8 +67,17 @@ function _track(id::PkgId, modname::Symbol; modified_files=revision_queue) if pkgdata === nothing pkgdata = PkgData(id, srcdir) end + ret = Revise.pkg_fileinfo(id) + if ret !== nothing + cachefile, _ = ret + if cachefile === nothing + @error "unable to find cache file for $id, tracking is not possible" + end + else + cachefile = basesrccache + end lock(revision_queue_lock) do - for (submod, filename) in Iterators.drop(Base._included_files, 1) # stepping through sysimg.jl rebuilds Base, omit it + for (submod, filename) in modulefiles_basestlibs(id) ffilename = fixpath(filename) inpath(ffilename, dirs) || continue keypath = ffilename[1:last(findfirst(dirs[end], ffilename))] @@ -78,7 +87,7 @@ function _track(id::PkgId, modname::Symbol; modified_files=revision_queue) cache_file_key[fullpath] = filename src_file_key[filename] = fullpath end - push!(pkgdata, rpath=>FileInfo(submod, basesrccache)) + push!(pkgdata, rpath=>FileInfo(submod, cachefile)) if mtime(ffilename) > mtcache with_logger(_debug_logger) do @debug "Recipe for Base/StdLib" _group="Watching" filename=filename mtime=mtime(filename) mtimeref=mtcache @@ -209,7 +218,8 @@ const stdlib_names = Set([ # This replacement is needed because the path written during compilation differs from # the git source path -const stdlib_rep = joinpath("usr", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)") => "stdlib" +const stdpath_rep = (joinpath("usr", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)") => "stdlib", + joinpath("usr", "share", "julia", "Compiler") => "Compiler") -const juliaf2m = Dict(normpath(replace(file, stdlib_rep))=>mod +const juliaf2m = Dict(normpath(replace(file, stdpath_rep...))=>mod for (mod,file) in Base._included_files) diff --git a/src/utils.jl b/src/utils.jl index eab6b0c0..6eff1007 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -96,6 +96,23 @@ function unwrap_where(ex::Expr) return ex::Expr end +function recursive_egal(@nospecialize(a), @nospecialize(b), @nospecialize(bskip)) + # Like ===, except for recursive structs this unpacks all the parameters + (isa(a, Type) && isa(b, Type)) || return a === b + b === bskip && return true + typeof(a) === typeof(b) || return false + isa(a, Core.TypeofBottom) && return a === b + isa(a, Union) && return (recursive_egal(a.a, b.a, bskip) && recursive_egal(a.b, b.b, bskip)) + a = Base.unwrap_unionall(a) + b = Base.unwrap_unionall(b) + length(a.parameters) === length(b.parameters) || return false + length(a.parameters) == 0 && return a === b + for (ap, bp) in zip(a.parameters, b.parameters) + recursive_egal(ap, bp, bskip) || return false + end + return true +end + function pushex!(exsigs::ExprsSigs, ex::Expr) uex = unwrap(ex) if is_doc_expr(uex) diff --git a/src/visit.jl b/src/visit.jl new file mode 100644 index 00000000..387ecfe0 --- /dev/null +++ b/src/visit.jl @@ -0,0 +1,72 @@ +function methods_with(@nospecialize(T::Type)) + meths = Set{Method}() + T = Base.unwrap_unionall(T) + Tname = T.name + methodtable = @static isdefinedglobal(Core, :methodtable) ? Core.methodtable : Core.GlobalMethods + Base.visit(methodtable) do method + # condition commented out due to https://github.com/timholy/Revise.jl/pull/894#issuecomment-3274102493 + # see the "MoreConstructors" test case in test/runtests.jl + # if method.module !== Tname.module || method.name !== Tname.name # skip constructor + if hastype(method.sig, T) || hastype_by_name(method.sig, Tname) + push!(meths, method) + end + # end + end + return meths +end + +function hastype(@nospecialize(S), @nospecialize(T)) + isa(S, TypeVar) && return hassubtype(S.ub, T) + isa(S, Type) || return false + S = Base.unwrap_unionall(S) + isa(S, Core.TypeofBottom) && return false + if isa(S, Union) + return hastype(S.a, T) | hastype(S.b, T) + end + Base.isvarargtype(S) && return hastype(S.T, T) + S === T && return true + for P in S.parameters + hastype(P, T) && return true + end + return false +end + +function hassubtype(@nospecialize(S), @nospecialize(T)) + isa(S, TypeVar) && return hassubtype(S.ub, T) + isa(S, Type) || return false + S = Base.unwrap_unionall(S) + isa(S, Core.TypeofBottom) && return false + if isa(S, Union) + return hassubtype(S.a, T) | hassubtype(S.b, T) + end + Base.isvarargtype(S) && return hassubtype(S.T, T) + S <: T && return true + for P in S.parameters + hassubtype(P, T) && return true + end + return false +end + +# Check if a type signature S contains a reference to a type with the given TypeName +# This is useful for finding methods that reference old world-age versions of a type +function hastype_by_name(@nospecialize(S), Tname::Core.TypeName) + isa(S, TypeVar) && return hastype_by_name(S.ub, Tname) + isa(S, Type) || return false + S_unwrapped = Base.unwrap_unionall(S) + isa(S_unwrapped, Core.TypeofBottom) && return false + if isa(S_unwrapped, Union) + return hastype_by_name(S_unwrapped.a, Tname) | hastype_by_name(S_unwrapped.b, Tname) + end + Base.isvarargtype(S_unwrapped) && return hastype_by_name(S_unwrapped.T, Tname) + if isa(S_unwrapped, DataType) + # Compare TypeNames by their module and name, not by identity (===) + # This is necessary because different world-age versions of a type have different TypeName objects + if S_unwrapped.name.module === Tname.module && S_unwrapped.name.name === Tname.name + return true + end + for P in S_unwrapped.parameters + hastype_by_name(P, Tname) && return true + end + end + return false +end diff --git a/test/common.jl b/test/common.jl index 6ad8ed05..5688139e 100644 --- a/test/common.jl +++ b/test/common.jl @@ -19,6 +19,8 @@ end @static if Sys.isapple() const mtimedelay = 3.1 # so the defining files are old enough not to trigger mtime criterion +elseif Sys.islinux() && isfile("/etc/wsl.conf") # WSL + const mtimedelay = 3.0 else const mtimedelay = 0.1 end diff --git a/test/runtests.jl b/test/runtests.jl index 2efefd5b..9e289c1a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -18,6 +18,10 @@ using Base.CoreLogging: Debug,Info # *.ji files for the package. using EponymTuples +# # Also ensure packages that we'll `@require` are precompiled, as otherwise Pkg `@info` +# # may contaminate the log and cause test failures. +# Pkg.precompile(["EndpointRanges", "CatIndices", "IndirectArrays", "RoundingIntegers", "UnsafeArrays"]) + include("common.jl") throwing_function(bt) = bt[2] @@ -2434,6 +2438,275 @@ const issue639report = [] pop!(LOAD_PATH) end + if Revise.__bpart__ && do_test("struct/const revision") # can we revise types and constants? + @testset "struct/const revision" begin + testdir = newtestdir() + dn = joinpath(testdir, "StructConst", "src") + mkpath(dn) + write(joinpath(dn, "StructConst.jl"), """ + module StructConst + const __hash__ = 0x71716e828e2d6093 + struct Fixed + x::Int + end + Base.hash(f::Fixed, h::UInt) = hash(__hash__, hash(f.x, h)) + struct Point + x::Float64 + end + # Three methods that won't need to be explicitly redefined (but will need re-evaluation for new type) + firstval(p::Point) = p.x + firstvalP(p::P) where P<:Point = p.x + returnsconst(::Point) = 1 + # Method that will need to be explicitly redefined + mynorm(p::Point) = sqrt(p.x^2) + # Method that uses `Point` without it being in the signature + hiddenconstructor(x) = Point(ntuple(_ -> x, length(fieldnames(Point)))...) + # Change of field that has no parameters (https://github.com/timholy/Revise.jl/pull/894#issuecomment-3271461024) + struct ChangePrimitiveType + x::Int + end + useprimitivetype(::ChangePrimitiveType) = 1 + # Additional constructors (https://github.com/timholy/Revise.jl/pull/894#issuecomment-3274102493) + abstract type AbstractMoreConstructors end + struct MoreConstructors + x::Int + end + MoreConstructors() = MoreConstructors(1) + end + """) + # Also create another package that uses it + dn2 = joinpath(testdir, "StructConstUser", "src") + mkpath(dn2) + write(joinpath(dn2, "StructConstUser.jl"), """ + module StructConstUser + using StructConst: StructConst, Point + struct PointWrapper + p::Point + end + scuf(f::StructConst.Fixed) = 33 * f.x + scup(p::Point) = 44 * p.x + scup(pw::PointWrapper) = 55 * pw.p.x + end + """) + # ...and one that uses that. This is to check whether the propagation of + # signature extraction works correctly. + dn3 = joinpath(testdir, "StructConstUserUser", "src") + mkpath(dn3) + write(joinpath(dn3, "StructConstUserUser.jl"), """ + module StructConstUserUser + using StructConstUser + struct PointWrapperWrapper + pw::StructConstUser.PointWrapper + end + StructConstUser.scup(pw::PointWrapperWrapper) = 2 * StructConstUser.scup(pw.pw) + end + """) + sleep(mtimedelay) + @eval using StructConst + @eval using StructConstUser + @eval using StructConstUserUser + sleep(mtimedelay) + w1 = Base.get_world_counter() + f = StructConst.Fixed(5) + v1 = hash(f) + p = StructConst.Point(5.0) + hp = StructConst.hiddenconstructor(5) + @test isa(hp, StructConst.Point) && hp.x === 5.0 + pw = StructConstUser.PointWrapper(p) + pww = StructConstUserUser.PointWrapperWrapper(pw) + @test StructConst.firstval(p) == StructConst.firstvalP(p) === 5.0 + @test StructConst.returnsconst(p) === 1 + @test StructConst.mynorm(p) == 5.0 + @test StructConstUser.scuf(f) == 33 * 5.0 + @test StructConstUser.scup(p) == 44 * 5.0 + @test StructConstUser.scup(pw) == 55 * 5.0 + @test StructConstUser.scup(pww) == 2 * 55 * 5.0 + spt = StructConst.ChangePrimitiveType(3) + @test StructConst.useprimitivetype(spt) === 1 + mc = StructConst.MoreConstructors() + @test mc.x == 1 && supertype(typeof(mc)) === Any + write(joinpath(dn, "StructConst.jl"), """ + module StructConst + const __hash__ = 0xddaab158621d200c + struct Fixed + x::Int + end + Base.hash(f::Fixed, h::UInt) = hash(__hash__, hash(f.x, h)) + struct Point + x::Float64 + y::Float64 + end + firstval(p::Point) = p.x + firstvalP(p::P) where P<:Point = p.x + returnsconst(::Point) = 1 + mynorm(p::Point) = sqrt(p.x^2 + p.y^2) + hiddenconstructor(x) = Point(ntuple(_ -> x, length(fieldnames(Point)))...) + struct ChangePrimitiveType + x::Float64 + end + useprimitivetype(::ChangePrimitiveType) = 1 + abstract type AbstractMoreConstructors end + struct MoreConstructors <: AbstractMoreConstructors + x::Int + end + MoreConstructors() = MoreConstructors(1) + end + """) + @yry() + @test StructConst.__hash__ == 0xddaab158621d200c + v2 = hash(f) + @test v1 != v2 + # Call with old objects---ensure we deleted all the outdated methods to reduce user confusion + @test_throws MethodError @invokelatest(StructConst.firstval(p)) + @test_throws MethodError @invokelatest(StructConst.firstvalP(p)) + @test_throws MethodError @invokelatest(StructConst.returnsconst(p)) + @test_throws MethodError @invokelatest(StructConst.mynorm(p)) + @test @invokelatest(StructConstUser.scuf(f)) == 33 * 5.0 + @test_throws MethodError @invokelatest(StructConstUser.scup(p)) + @test_throws MethodError @invokelatest(StructConstUser.scup(pw)) + @test_throws MethodError @invokelatest(StructConstUser.scup(pww)) + @test_throws MethodError StructConst.useprimitivetype(spt) + # Call with new objects + p2 = @invokelatest(StructConst.Point(3.0, 4.0)) + hp = @invokelatest(StructConst.hiddenconstructor(5)) + @test isa(hp, StructConst.Point) && hp.x === 5.0 && hp.y === 5.0 + pw2 = @invokelatest(StructConstUser.PointWrapper(p2)) + pww2 = @invokelatest(StructConstUserUser.PointWrapperWrapper(pw2)) + @test @invokelatest(StructConst.firstval(p2)) == @invokelatest(StructConst.firstvalP(p2)) === 3.0 + @test StructConst.returnsconst(p2) === 1 + @test @invokelatest(StructConst.mynorm(p2)) == 5.0 + @test @invokelatest(StructConstUser.scup(p2)) == 44 * 3.0 + @test @invokelatest(StructConstUser.scup(pw2)) == 55 * 3.0 + @test @invokelatest(StructConstUser.scup(pww2)) == 2 * 55 * 3.0 + spt2 = StructConst.ChangePrimitiveType(3.0) + @test StructConst.useprimitivetype(spt2) === 1 + mc = StructConst.MoreConstructors() + @test mc.x == 1 && supertype(typeof(mc)) === StructConst.AbstractMoreConstructors + write(joinpath(dn, "StructConst.jl"), """ + module StructConst + const __hash__ = 0x71716e828e2d6093 + struct Fixed + x::Int + end + Base.hash(f::Fixed, h::UInt) = hash(__hash__, hash(f.x, h)) + struct Point{T<:Real} <: AbstractVector{T} + x::T + y::T + end + firstval(p::Point) = p.x + firstvalP(p::P) where P<:Point = p.x + returnsconst(::Point) = 1 + mynorm(p::Point) = sqrt(p.x^2 + p.y^2) + hiddenconstructor(x) = Point(ntuple(_ -> x, length(fieldnames(Point)))...) + end + """) + @yry() + @test StructConst.__hash__ == 0x71716e828e2d6093 + v3 = hash(f) + @test v1 == v3 + p3 = @invokelatest(StructConst.Point(3.0, 4.0)) + hp = @invokelatest(StructConst.hiddenconstructor(5)) + @test isa(hp, StructConst.Point) && hp.x === 5 && hp.y === 5 + pw3 = @invokelatest(StructConstUser.PointWrapper(p3)) + pww3 = @invokelatest(StructConstUserUser.PointWrapperWrapper(pw3)) + @test @invokelatest(StructConst.firstval(p3)) == @invokelatest(StructConst.firstvalP(p3)) === 3.0 + @test @invokelatest(StructConst.mynorm(p3)) == 5.0 + @test @invokelatest(StructConstUser.scup(p3)) == 44 * 3.0 + @test @invokelatest(StructConstUser.scup(pw3)) == 55 * 3.0 + @test @invokelatest(StructConstUser.scup(pww3)) == 2 * 55 * 3.0 + + rm_precompile("StructConst") + rm_precompile("StructConstUser") + rm_precompile("StructConstUserUser") + + # Example from https://github.com/timholy/Revise.jl/pull/894#issuecomment-2824111764 + dn = joinpath(testdir, "StructExample", "src") + mkpath(dn) + write(joinpath(dn, "StructExample.jl"), + """ + module StructExample + export hello, Hello + struct Hello + who::String + end + hello(x::Hello) = hello(x.who) + hello(who::String) = "Hello, \$who" + end + """) + sleep(mtimedelay) + @eval using StructExample + sleep(mtimedelay) + @test StructExample.hello(StructExample.Hello("World")) == "Hello, World" + write(joinpath(dn, "StructExample.jl"), + """ + module StructExample + export hello, Hello + struct Hello + who2::String + end + hello(x::Hello) = hello(x.who2 * " (changed)") + hello(who::String) = "Hello, \$who" + end + """) + @yry() + @test StructExample.hello(StructExample.Hello("World")) == "Hello, World (changed)" + + # Full-circle parametric removal and restoration + # Start with a parametric struct and a method depending on its type parameter. + # Switch the struct to non-parametric (with its own method), then switch back + # to the original parametric definition and ensure calls work again. + dn4 = joinpath(testdir, "StructParamFullCircle", "src") + mkpath(dn4) + fn4 = joinpath(dn4, "StructParamFullCircle.jl") + write(fn4, raw""" + module StructParamFullCircle + export Foo, bar + struct Foo{T} + x::T + end + bar(::Foo{T}) where {T} = "parametric with $T" + end + """) + sleep(mtimedelay) + @eval using StructParamFullCircle + sleep(mtimedelay) + foo1 = StructParamFullCircle.Foo(1) + @test StructParamFullCircle.bar(foo1) == "parametric with $Int" + + # Change Foo to be non-parametric + write(fn4, raw""" + module StructParamFullCircle + export Foo, bar + struct Foo + x::Int + end + bar(::Foo{T}) where {T} = "parametric with $T" + end + """) + @yry() + foo2 = @invokelatest(StructParamFullCircle.Foo(1)) + @test_throws MethodError @invokelatest(StructParamFullCircle.bar(foo2)) + + # Now change Foo back to its original parametric definition + write(fn4, raw""" + module StructParamFullCircle + export Foo, bar + struct Foo{T} + x::T + end + bar(::Foo{T}) where {T} = "parametric with $T" + end + """) + @yry() + foo3 = @invokelatest(StructParamFullCircle.Foo(1)) + @test @invokelatest(StructParamFullCircle.bar(foo3)) == "parametric with $Int" + + rm_precompile("StructParamFullCircle") + + pop!(LOAD_PATH) + end + end + do_test("get_def") && @testset "get_def" begin testdir = newtestdir() dn = joinpath(testdir, "GetDef", "src") @@ -2939,6 +3212,8 @@ const issue639report = [] end do_test("Recipes") && @testset "Recipes" begin + @test !isempty(methods(Core.Compiler.NativeInterpreter)) + # https://github.com/JunoLab/Juno.jl/issues/257#issuecomment-473856452 meth = @which gcd(10, 20) signatures_at(Base.find_source_file(String(meth.file)), meth.line) # this should track Base @@ -2956,6 +3231,8 @@ const issue639report = [] m = @which redirect_stdout() @test definition(m).head ∈ (:function, :(=)) + @test !isempty(methods(Core.Compiler.NativeInterpreter)) + # Tracking stdlibs Revise.track(Unicode) id = Base.PkgId(Unicode) @@ -2965,20 +3242,28 @@ const issue639report = [] @test definition(m) isa Expr @test isfile(whereis(m)[1]) + @test !isempty(methods(Core.Compiler.NativeInterpreter)) + # Submodule of Pkg (note that package is developed outside the # Julia repo, this tests new cases) id = Revise.get_tracked_id(Pkg.Types) pkgdata = Revise.pkgdatas[id] @test definition(first(methods(Pkg.API.add))) isa Expr + @test !isempty(methods(Core.Compiler.NativeInterpreter)) + # Test that we skip over files that don't end in ".jl" logs, _ = Test.collect_test_logs() do Revise.track(REPL) end @test isempty(logs) + @test !isempty(methods(Core.Compiler.NativeInterpreter)) + Revise.get_tracked_id(Core) # just test that this doesn't error + @test !isempty(methods(Core.Compiler.NativeInterpreter)) + if !haskey(ENV, "BUILDKITE") # disable on buildkite, see discussion in https://github.com/JuliaCI/julia-buildkite/pull/372#issuecomment-2262840304 # Determine whether a git repo is available. Travis & Appveyor do not have this. repo, path = Revise.git_repo(Revise.juliadir) @@ -2995,6 +3280,8 @@ const issue639report = [] @warn "skipping Core.Compiler tests due to lack of git repo" end end + + @test !isempty(methods(Core.Compiler.NativeInterpreter)) end do_test("CodeTracking #48") && @testset "CodeTracking #48" begin @@ -4004,9 +4291,9 @@ do_test("includet with mod arg (issue #689)") && @testset "includet with mod arg @test Driver.Codes.Common.foo == 2 end -do_test("misc - coverage") && @testset "misc - coverage" begin +do_test("misc - coverage") && !isinteractive() && @testset "misc - coverage" begin @test Revise.ReviseEvalException("undef", UndefVarError(:foo)).loc isa String - @test !Revise.throwto_repl(UndefVarError(:foo)) + @test !Revise.throwto_repl(UndefVarError(:foo)) # this causes an error in interactive @test endswith(Revise.fallback_juliadir(), "julia")