diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl index 345cd99..0f4ec0d 100644 --- a/src/ReTestItems.jl +++ b/src/ReTestItems.jl @@ -9,6 +9,7 @@ using TestEnv using Logging: current_logger, with_logger export runtests, runtestitem +export print_tests export @testsetup, @testitem export TestSetup, TestItem, TestItemResult @@ -231,15 +232,18 @@ will be run. """ function runtests end -runtests(; kw...) = runtests(Returns(true), dirname(Base.active_project()); kw...) -runtests(shouldrun; kw...) = runtests(shouldrun, dirname(Base.active_project()); kw...) -runtests(paths::AbstractString...; kw...) = runtests(Returns(true), paths...; kw...) - -runtests(pkg::Module; kw...) = runtests(Returns(true), pkg; kw...) -function runtests(shouldrun, pkg::Module; kw...) - dir = pkgdir(pkg) - isnothing(dir) && error("Could not find directory for `$pkg`") - return runtests(shouldrun, dir; kw...) +for func in (:runtests, :print_tests) + @eval begin + $func(; kw...) = $func(Returns(true), dirname(Base.active_project()); kw...) + $func(shouldrun; kw...) = $func(shouldrun, dirname(Base.active_project()); kw...) + $func(paths::AbstractString...; kw...) = $func(Returns(true), paths...; kw...) + $func(pkg::Module; kw...) = $func(Returns(true), pkg; kw...) + function $func(shouldrun, pkg::Module; kw...) + dir = pkgdir(pkg) + isnothing(dir) && error("Could not find directory for `$pkg`") + return $func(shouldrun, dir; kw...) + end + end end @kwdef struct _Config @@ -257,8 +261,10 @@ end timeout_profile_wait::Int memory_threshold::Float64 gc_between_testitems::Bool + dryrun::Bool end +default_validate_paths() = parse(Bool, get(ENV, "RETESTITEMS_VALIDATE_PATHS", "false")) function runtests( shouldrun, @@ -276,11 +282,12 @@ function runtests( logs::Symbol=Symbol(get(ENV, "RETESTITEMS_LOGS", default_log_display_mode(report, nworkers))), verbose_results::Bool=(logs !== :issues && isinteractive()), test_end_expr::Expr=Expr(:block), - validate_paths::Bool=parse(Bool, get(ENV, "RETESTITEMS_VALIDATE_PATHS", "false")), + validate_paths::Bool=default_validate_paths(), timeout_profile_wait::Real=parse(Int, get(ENV, "RETESTITEMS_TIMEOUT_PROFILE_WAIT", "0")), gc_between_testitems::Bool=parse(Bool, get(ENV, "RETESTITEMS_GC_BETWEEN_TESTITEMS", string(nworkers > 1))), failfast::Bool=parse(Bool, get(ENV, "RETESTITEMS_FAILFAST", "false")), testitem_failfast::Bool=parse(Bool, get(ENV, "RETESTITEMS_TESTITEM_FAILFAST", string(failfast))), + dryrun::Bool=parse(Bool, get(ENV, "RETESTITEMS_DRYRUN", "false")), ) nworker_threads = _validated_nworker_threads(nworker_threads) paths′ = _validated_paths(paths, validate_paths) @@ -301,7 +308,7 @@ function runtests( (timeout_profile_wait > 0 && Sys.iswindows()) && @warn "CPU profiles on timeout is not supported on Windows, ignoring `timeout_profile_wait`" mkpath(RETESTITEMS_TEMP_FOLDER[]) # ensure our folder wasn't removed save_current_stdio() - cfg = _Config(; nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, testitem_failfast, failfast, retries, logs, report, verbose_results, timeout_profile_wait, memory_threshold, gc_between_testitems) + cfg = _Config(; nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, testitem_failfast, failfast, retries, logs, report, verbose_results, timeout_profile_wait, memory_threshold, gc_between_testitems, dryrun) debuglvl = Int(debug) if debuglvl > 0 withdebug(debuglvl) do @@ -312,6 +319,21 @@ function runtests( end end +""" + ReTestItems.print_tests() + ReTestItems.print_tests(mod::Module) + ReTestItems.print_tests(paths::AbstractString...) + +Print the tests that would be run by `runtests`. +Also available via `runtests(args...; dryrun=true, kw...)`. +""" +function print_tests( + shouldrun, paths::AbstractString...; + name=nothing, tags=nothing, validate_paths=default_validate_paths(), +) + runtests(shouldrun, paths...; dryrun=true, name, tags, validate_paths) +end + # keep track of temporary test environments we create in case we can reuse them # on repeated runs of `runtests` in the same Project # in https://relationalai.atlassian.net/browse/RAI-11599, it was noted that when @@ -339,8 +361,9 @@ function _runtests(ti_filter, paths, cfg::_Config) # Wrapping with the logger that was set before eval'ing any user code to # avoid world age issues when logging https://github.com/JuliaLang/julia/issues/33865 with_logger(current_logger()) do - if is_running_test_runtests_jl(proj_file) + if is_running_test_runtests_jl(proj_file) || cfg.dryrun # Assume this is `Pkg.test`, so test env already active. + # Or if `dryrun`, we don't need the env as we won't run the tests. @debugv 2 "Running in current environment `$(Base.active_project())`" return _runtests_in_current_env(ti_filter, paths, proj_file, cfg) else @@ -376,8 +399,12 @@ function _runtests_in_current_env( nworker_threads = cfg.nworker_threads ntestitems = length(testitems.testitems) @debugv 1 "Done including tests in $paths" - @info "Finished scanning for test items in $(round(time() - inc_time, digits=2)) seconds." * - " Scheduling $ntestitems tests on pid $(Libc.getpid())" * + @info "Finished scanning for test items in $(round(time() - inc_time, digits=2)) seconds." + if cfg.dryrun + _print_testitems(testitems) + return nothing + end + @info "Scheduling $ntestitems tests on pid $(Libc.getpid())" * (nworkers == 0 ? "" : " with $nworkers worker processes and $nworker_threads threads per worker.") try if nworkers == 0 diff --git a/src/testcontext.jl b/src/testcontext.jl index 8035aab..c462659 100644 --- a/src/testcontext.jl +++ b/src/testcontext.jl @@ -103,6 +103,47 @@ cancel!(t::TestItems) = @atomicswap t.cancelled = true # `cancel` and check the return value to know if they had already been cancelled. is_cancelled(t::TestItems) = @atomic t.cancelled +function _print_testitems(tis::TestItems) + ntestitems = length(tis.testitems) + if ntestitems == 0 + printstyled("No test items found\n"; bold=true) + return nothing + end + plural = ntestitems == 1 ? "" : "s" + printstyled("$ntestitems test item$plural found:\n"; bold=true) + _print_testitems(tis.graph) + @info "Found $ntestitems test item$plural." +end +function _print_testitems(dir::DirNode, indent::Int=0) + println(" "^indent, dir.path) + for child in dir.children + _print_testitems(child, indent + 1) + end +end +function _print_testitems(file::FileNode, indent::Int=0) + println(" "^indent, file.path) + for ti in file.testitems + printstyled(" "^(indent+1), ti.name; bold=true) + if !isempty(ti.tags) + printstyled(sprint(_print_tags, ti.tags); color=:light_black) + end + print('\n') + end +end +function _print_tags(io::IO, tags::AbstractVector{Symbol}) + print(io, " [") + isfirst = true + for tag in tags + if !isfirst + print(io, ", ") + end + show(io, tag) + isfirst = false + end + print(io, "]") +end + + ### ### record results ###