diff --git a/Project.toml b/Project.toml index 5c391d8..b07cd9c 100644 --- a/Project.toml +++ b/Project.toml @@ -3,12 +3,14 @@ uuid = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" version = "1.34.0" [deps] +CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" StringViews = "354b36f9-a18e-4713-926e-db85100087ba" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestEnv = "1e6cf692-eddd-4d53-88a5-2d735e33781b" @@ -21,6 +23,7 @@ Random = "1" Serialization = "1" Sockets = "1" StringViews = "1" +TOML = "1" Test = "1" TestEnv = "1.8" julia = "1.8" diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl index 638ed64..5c6256f 100644 --- a/src/ReTestItems.jl +++ b/src/ReTestItems.jl @@ -12,6 +12,8 @@ export runtests, runtestitem export @testsetup, @testitem export TestSetup, TestItem, TestItemResult +global TIS = nothing + const RETESTITEMS_TEMP_FOLDER = Ref{String}() const DEFAULT_TESTITEM_TIMEOUT = 30*60 const DEFAULT_RETRIES = 0 @@ -25,9 +27,12 @@ end # Used by failures_first to sort failures before unseen before passes. @enum _TEST_STATUS::UInt8 begin - _FAILED = 0 - _UNSEEN = 1 + _UNSEEN = 0 + _STARTED = 1 _PASSED = 2 + _FAILED = 3 + _CRASHED = 4 + _TIMEDOUT = 5 end const GLOBAL_TEST_STATUS = Dict{String,_TEST_STATUS}() reset_test_status!() = (empty!(GLOBAL_TEST_STATUS); nothing) @@ -94,6 +99,24 @@ function softscope_all!(@nospecialize ex) end end +@kwdef struct _Config + nworkers::Int + nworker_threads::String + worker_init_expr::Expr + test_end_expr::Expr + testitem_timeout::Int + testitem_failfast::Bool + failfast::Bool + retries::Int + logs::Symbol + report::Bool + verbose_results::Bool + timeout_profile_wait::Int + memory_threshold::Float64 + gc_between_testitems::Bool + failures_first::Bool +end + include("debug.jl") include("workers.jl") using .Workers @@ -103,6 +126,7 @@ include("testcontext.jl") include("log_capture.jl") include("filtering.jl") include("include_test_file.jl") +include("run_state.jl") function __init__() if ccall(:jl_generating_output, Cint, ()) == 0 # not precompiling @@ -146,22 +170,42 @@ function _validated_nworker_threads(str) return replace(str, "auto" => string(Sys.CPU_THREADS)) end +struct ValidatedPaths{D,F} + by_dir::D + by_file::F +end + + function _validated_paths(paths, should_throw::Bool) - return filter(paths) do p - if !ispath(p) + dirs = String[] + files = String[] + + for p in paths + s = stat(p) + if !ispath(s) msg = "No such path $(repr(p))" should_throw ? throw(NoTestException(msg)) : @warn msg - return false - elseif !(is_test_file(p) || is_testsetup_file(p)) && isfile(p) + continue + elseif !(is_test_file(s) || is_testsetup_file(s)) && isfile(s) msg = "$(repr(p)) is not a test file" should_throw ? throw(NoTestException(msg)) : @warn msg - return false + continue + elseif isdir(s) + push!(dirs, abspath(p)) else - return true + push!(files, abspath(p)) end end + + return ValidatedPaths( + isempty(dirs) ? nothing : Tuple(dirs), + isempty(files) ? nothing : Set(files), + ) end +Base.first(vp::ValidatedPaths) = vp.by_dir === nothing ? first(vp.by_file) : first(vp.by_dir) +Base.isempty(vp::ValidatedPaths) = vp.by_dir === nothing && vp.by_file === nothing + """ ReTestItems.runtests() ReTestItems.runtests(mod::Module) @@ -277,25 +321,6 @@ function runtests(shouldrun, pkg::Module; kw...) return runtests(shouldrun, dir; kw...) end -@kwdef struct _Config - nworkers::Int - nworker_threads::String - worker_init_expr::Expr - test_end_expr::Expr - testitem_timeout::Int - testitem_failfast::Bool - failfast::Bool - retries::Int - logs::Symbol - report::Bool - verbose_results::Bool - timeout_profile_wait::Int - memory_threshold::Float64 - gc_between_testitems::Bool - failures_first::Bool -end - - function runtests( shouldrun, paths::AbstractString...; @@ -409,6 +434,8 @@ function _runtests_in_current_env( inc_time = time() @debugv 1 "Including tests in $paths" testitems, _ = include_testfiles!(proj_name, projectfile, paths, ti_filter, cfg.verbose_results, cfg.report) + global TIS = testitems + statefile = init_run_state(joinpath(dirname(projectfile), "testrun"), testitems, projectfile, dirname(projectfile), cfg) @debugv 1 "Done including tests in $paths" nworkers = cfg.nworkers nworker_threads = cfg.nworker_threads @@ -444,6 +471,7 @@ function _runtests_in_current_env( max_runs = 1 + max(cfg.retries, testitem.retries) is_non_pass = false while run_number ≤ max_runs + write_status(statefile, testitem, run_number, nothing, _STARTED) res = runtestitem(testitem, ctx; cfg.test_end_expr, cfg.verbose_results, cfg.logs, failfast=cfg.testitem_failfast) ts = res.testset print_errors_and_captured_logs(testitem, run_number; cfg.logs) @@ -454,9 +482,11 @@ function _runtests_in_current_env( end testitem.is_non_pass[] = is_non_pass = any_non_pass(ts) if is_non_pass && run_number != max_runs + write_status(statefile, testitem, run_number, nothing, _FAILED) run_number += 1 @info "Retrying $(repr(testitem.name)). Run=$run_number." else + write_status(statefile, testitem, run_number, nothing, _PASSED) break end end @@ -492,7 +522,7 @@ function _runtests_in_current_env( ti = starting[i] @spawn begin with_logger(original_logger) do - manage_worker($w, $proj_name, $testitems, $ti, $cfg; worker_num=$i) + manage_worker($w, $proj_name, $testitems, $ti, $cfg, $statefile; worker_num=$i) end end end @@ -631,12 +661,13 @@ end # The provided `worker_num` is only for logging purposes, and not persisted as part of the worker. function manage_worker( - worker::Worker, proj_name::AbstractString, testitems::TestItems, testitem::Union{TestItem,Nothing}, cfg::_Config; + worker::Worker, proj_name::AbstractString, testitems::TestItems, testitem::Union{TestItem,Nothing}, cfg::_Config, statefile; worker_num::Int ) ntestitems = length(testitems.testitems) run_number = 1 memory_threshold_percent = 100 * cfg.memory_threshold + timeout = nothing while testitem !== nothing ch = Channel{TestItemResult}(1) if memory_percent() > memory_threshold_percent @@ -648,6 +679,7 @@ function manage_worker( testitem.workerid[] = worker.pid timeout = something(testitem.timeout, cfg.testitem_timeout) fut = remote_eval(worker, :(ReTestItems.runtestitem($testitem, GLOBAL_TEST_CONTEXT; test_end_expr=$(QuoteNode(cfg.test_end_expr)), verbose_results=$(cfg.verbose_results), logs=$(QuoteNode(cfg.logs)), failfast=$(cfg.testitem_failfast)))) + write_status_locked(statefile, testitem, run_number, timeout, _STARTED) max_runs = 1 + max(cfg.retries, testitem.retries) try timer = Timer(timeout) do tm @@ -682,13 +714,16 @@ function manage_worker( end testitem.is_non_pass[] = is_non_pass = any_non_pass(ts) if is_non_pass && run_number != max_runs + write_status_locked(statefile, testitem, run_number, timeout, _FAILED) run_number += 1 @info "Retrying $(repr(testitem.name)) on $worker. Run=$run_number." else if cfg.failfast && is_non_pass + write_status_locked(statefile, testitem, _FAILED) already_cancelled = cancel!(testitems) already_cancelled || print_failfast_cancellation(testitem) end + write_status_locked(statefile, testitem, run_number, timeout, _PASSED) testitem = next_testitem(testitems, testitem.number[]) run_number = 1 end @@ -699,6 +734,7 @@ function manage_worker( @debugv 2 "Error: $e" # Handle the exception if e isa TimeoutException + write_status_locked(statefile, testitem, run_number, timeout, _TIMEDOUT) if cfg.timeout_profile_wait > 0 @warn "$worker timed out running test item $(repr(testitem.name)) after $timeout seconds. \ A CPU profile will be triggered on the worker and then it will be terminated." @@ -716,6 +752,7 @@ function manage_worker( Recording test error." record_timeout!(testitem, run_number, timeout) elseif e isa WorkerTerminatedException + write_status_locked(statefile, testitem, run_number, timeout, _CRASHED) println(DEFAULT_STDOUT[]) _print_captured_logs(DEFAULT_STDOUT[], testitem, run_number) @error "$worker died running test item $(repr(testitem.name)). \ @@ -792,14 +829,15 @@ end # is `dir` the root of a subproject inside the current project? # all three paths are assumed to be absolute paths let test_project = joinpath("test", "Project.toml") - global function _is_subproject(dir, current_projectfile, current_project_dir) + global function _is_subproject(dir, current_project_dir) projectfile = _project_file(dir) isnothing(projectfile) && return false + dir == current_project_dir && return false - projectfile == current_projectfile && return false # a `test/Project.toml` is special and doesn't indicate a subproject rel_projectfile = nestedrelpath(projectfile, current_project_dir) rel_projectfile == test_project && return false + @debugv 1 "Skipping files under subproject `$projectfile`" return true end end @@ -814,7 +852,6 @@ function walkdir_task(walkdir_channel::Channel{Tuple{String,FileNode}}, project_ @assert isabspath(project_root) @assert isabspath(projectfile) dir_nodes = Dict{String, DirNode}() - subproject_root = nothing # don't recurse into directories with their own Project.toml. abspaths = map(abspath, paths) try # Since test items don't store paths to their test setups, we need to traverse the @@ -822,13 +859,8 @@ function walkdir_task(walkdir_channel::Channel{Tuple{String,FileNode}}, project_ stack = [project_root] while !isempty(stack) root = pop!(stack) - if subproject_root !== nothing && startswith(root, subproject_root) - @debugv 1 "Skipping files in `$root` in subproject `$subproject_root`" - continue - elseif _is_subproject(root, projectfile, project_root) - subproject_root = root - continue - end + # Don't recurse into directories with their own Project.toml. + _is_subproject(root, project_root) && continue rel_root = nestedrelpath(root, project_root) dir_node = DirNode(rel_root; report, verbose=verbose_results) dir_nodes[rel_root] = dir_node @@ -966,12 +998,16 @@ function _throw_duplicate_ids(testitems) end # Is filepath one of the paths the user requested? -is_requested(filepath, paths::Tuple{}) = true # no paths means no restrictions -function is_requested(filepath, abspaths::Tuple) +is_requested_by_dir(filepath, paths::Nothing) = true # no paths means no restrictions +is_requested_by_file(filepath, paths::Nothing) = true # no paths means no restrictions +function is_requested_by_dir(filepath::String, abspaths::Tuple) return any(abspaths) do p startswith(filepath, p) end end +function is_requested_by_file(filepath::String, abspaths::Set{String}) + return filepath in abspaths +end function is_running_test_runtests_jl(projectfile::String) file_running = get(task_local_storage(), :SOURCE_PATH, nothing) diff --git a/src/debug.jl b/src/debug.jl index af9a1c3..493dd05 100644 --- a/src/debug.jl +++ b/src/debug.jl @@ -30,7 +30,16 @@ macro debugv(level::Int, messsage) _file = $last($splitdir(_full_file)) _line = $(QuoteNode(__source__.line)) msg = $(esc(messsage)) - $print("DEBUG @ $(_file):$(_line) | $msg\n") + io = IOBuffer(sizehint=34 + ncodeunits(msg)) + $print(io, "DEBUG @ ") + $print(io, _file) + $print(io, ":") + $print(io, _line) + $print(io, " | ") + $print(io, msg) + $print(io, "\n") + $write(stderr, seekstart(io)) + # $print("DEBUG @ $file:$line | $msg\n") end end end diff --git a/src/filtering.jl b/src/filtering.jl index 0202e0b..4ed07a3 100644 --- a/src/filtering.jl +++ b/src/filtering.jl @@ -8,7 +8,7 @@ end struct TestItemFilter{ F<:Function, T<:Union{Nothing,Symbol,AbstractVector{Symbol}}, - N<:Union{Nothing,AbstractString,Regex} + N<:Union{Nothing,AbstractString,Regex,Set{<:AbstractString}} } <: Function shouldrun::F tags::T @@ -21,6 +21,7 @@ end _shouldrun(name::AbstractString, ti) = name == ti.name _shouldrun(pattern::Regex, ti) = contains(ti.name, pattern) +_shouldrun(names::Set{<:AbstractString}, ti) = ti.name in names _shouldrun(tags::AbstractVector{Symbol}, ti) = issubset(tags, ti.tags) _shouldrun(tag::Symbol, ti) = tag in ti.tags _shouldrun(::Nothing, ti) = true diff --git a/src/run_state.jl b/src/run_state.jl new file mode 100644 index 0000000..8767f9a --- /dev/null +++ b/src/run_state.jl @@ -0,0 +1,245 @@ +using TOML: TOML +using CRC32c: CRC32c +using Dates: Dates + + +const MAGIC = 0x78ab8eb8 # hash("ReTestItems.jl") % UInt32 +const CURR_VERSION = UInt32(0) + +@enum PROJECT_ID_KIND::UInt8 begin + UUID = 1 + NAME = 2 + CRC32C = 3 +end + +function _parse_project_for_header(io) + proj = TOML.parse(io) + id = get(proj, "uuid", nothing) + id_kind = UUID + if id === nothing + id = get(proj, "name", nothing) + id_kind = NAME + end + if id === nothing + id = get(proj, "crc32c", CRC32c.crc32c(seekstart(io))) + id_kind = CRC32C + end + manifest = get(proj, "manifest", "") + if manifest != "" + manifest = dirname(manifest) + else + manifest = "" + end + return id, id_kind, manifest +end + +struct RunStateFile + io::IO + path::String + offset_to_statuses::UInt32 +end + +function init_run_state(path, testitems, projectfile, project_root, cfg::_Config) + io = open(path, "w") + + ## Metadata + write(io, MAGIC) # 4 bytes + write(io, CURR_VERSION) # 4 bytes + write(io, UInt32(0)) # offset_to_statuses, will be filled later + + ## Run info + write(io, UInt32(length(testitems.testitems))) # number of test items + write(io, Dates.datetime2unix(Dates.now(Dates.UTC))) # 8 bytes, start time + let pkgversion = string(pkgversion(ReTestItems)) + write(io, UInt32(ncodeunits(pkgversion))) + write(io, pkgversion) + end + let juliaversion = string(VERSION) + write(io, UInt32(ncodeunits(juliaversion))) + write(io, juliaversion) + end + # commit_hash = chomp(read(`git rev-parse HEAD`, String)) # or use LibGit2 + # commit_hash = LibGit2.GitHash(LibGit2.peel(LibGit2.head(LibGit2.GitRepo(".")))) + # write(io, UInt32(ncodeunits(commit_hash))) + # write(io, commit_hash) + + # Config + write(io, UInt32(cfg.nworkers)) + write(io, UInt32(ncodeunits(cfg.nworker_threads))) + write(io, cfg.nworker_threads) + let init_str = string(cfg.worker_init_expr) # compress? + write(io, UInt32(ncodeunits(init_str))) + write(io, init_str) + end + let end_str = string(cfg.test_end_expr) # compress? + write(io, UInt32(ncodeunits(end_str))) + write(io, end_str) + end + write(io, UInt32(cfg.testitem_timeout)) + write(io, cfg.testitem_failfast) # bitflags for bools? + write(io, cfg.failfast) + write(io, UInt32(cfg.retries)) + let logs = string(cfg.logs) # unroll the three options? + write(io, UInt32(ncodeunits(logs))) + write(io, logs) + end + write(io, cfg.report) + write(io, cfg.verbose_results) + write(io, UInt32(cfg.timeout_profile_wait)) + write(io, Float32(cfg.memory_threshold)) + write(io, cfg.gc_between_testitems) + write(io, cfg.failures_first) + + ## Project info + protect_id, project_id_kind, manifest_field = open(_parse_project_for_header, projectfile) + write(io, UInt32(project_id_kind) << 24 | UInt32(ncodeunits(protect_id))) # 4 bytes: length of id and kind in high byte + write(io, protect_id) + write(io, UInt32(ncodeunits(manifest_field))) # manifest or nothing + write(io, manifest_field) + # TODO: Manifest hash? + + ## Test items + prev_path = "" + for ti in testitems.testitems + if ti.file != prev_path + let file = nestedrelpath(ti.file, project_root) + write(io, UInt32(0)) # sentinel for new path + write(io, UInt32(ncodeunits(file))) + write(io, file) + end + prev_path = ti.file + end + write(io, UInt32(ncodeunits(ti.name))) + write(io, ti.name) + end + + offset_to_statuses = position(io) + seek(io, 8) + write(io, UInt32(offset_to_statuses)) # offset_to_statuses + seek(io, offset_to_statuses) + + ## Statuses + for _ in 1:length(testitems.testitems) + write(io, ((UInt32(_UNSEEN) << 28) | (0x0f000000 & (UInt32(0) << 24)) | UInt32(0))) # status 4bits, nretries 4bits, number 24bits + write(io, Float32(0.0)) # running time + write(io, Float32(0.0)) # compilation time + end + flush(io) + seek(io, offset_to_statuses) + + return RunStateFile(io, path, offset_to_statuses) +end + +const _STATE_LOCK = Threads.SpinLock() + +function write_status(rsf::RunStateFile, ti::TestItem, run_number::Int, timeout::Union{Nothing, Int}, status::_TEST_STATUS) + index = UInt32(ti.number[]) + seek(rsf.io, rsf.offset_to_statuses + (index - one(UInt32)) * (sizeof(UInt32) + 2*sizeof(Float32))) + write(rsf.io, (((UInt32(status) << 28) | (0x0f000000 & (UInt32(run_number) << 24))) | index)) + if status in (_PASSED, _FAILED) + stats = ti.stats[run_number] + write(rsf.io, Float32(stats.elapsedtime/1e9)) + write(rsf.io, Float32(stats.compile_time/1e9)) + elseif status === _TIMEDOUT + write(rsf.io, Float32(something(timeout, 0))) + write(rsf.io, Float32(0.0)) + else + write(rsf.io, Float32(0.0)) + write(rsf.io, Float32(0.0)) + end + flush(rsf.io) + return nothing +end + +function write_status_locked(rsf::RunStateFile, ti::TestItem, run_number::Int, timeout::Union{Nothing, Int}, status::_TEST_STATUS) + @lock _STATE_LOCK @inline write_status(rsf, ti, run_number, timeout, status) + return nothing +end + +function read_string_view(io, bytes) + len = read(io, UInt32) + pos = UInt32(position(io) + 1) + skip(io, len) # move the cursor + return StringView(@view(bytes[pos:pos+len-one(UInt32)])) +end +function read_string(io) + len = read(io, UInt32) + return String(read(io, len)) +end + + +function read_string_view(io, bytes, len) + pos = UInt32(position(io) + 1) + skip(io, len) # move the cursor + return StringView(@view(bytes[pos:pos+len-one(UInt32)])) +end + +function read_run_state(path) + bytes = read(path) # mmap? needs to be GC rooted for all the string views + io = IOBuffer(bytes) + + magic = read(io, UInt32) + magic != MAGIC && error("File $path is not a valid ReTestItems run state file") + format_version = read(io, UInt32) + format_version != CURR_VERSION && error("File $path has version $format_version, but this version of ReTestItems only supports version $CURR_VERSION") + offset_to_statuses = read(io, UInt32) + + ntestitems = read(io, UInt32) + start_time = Dates.unix2datetime(read(io, Float64)) + pkgversion = read_string_view(io, bytes) + juliaversion = read_string_view(io, bytes) + + nworkers = read(io, UInt32) + nworker_threads = read_string(io) + worker_init_expr = Meta.parse(read_string(io)) + test_end_expr = Meta.parse(read_string(io)) + testitem_timeout = read(io, UInt32) + testitem_failfast = read(io, Bool) + failfast = read(io, Bool) + retries = read(io, UInt32) + logs = Symbol(read_string(io)) + report = read(io, Bool) + verbose_results = read(io, Bool) + timeout_profile_wait = read(io, UInt32) + memory_threshold = Float64(read(io, Float32)) + gc_between_testitems = read(io, Bool) + failures_first = read(io, Bool) + + cfg = _Config(nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, testitem_failfast, failfast, retries, logs, report, verbose_results, timeout_profile_wait, memory_threshold, gc_between_testitems, failures_first) + + project_id_info = read(io, UInt32) + project_id_kind = Base.bitcast(PROJECT_ID_KIND, UInt8((project_id_info >> 24) & 0xff)) + project_id = read_string_view(io, bytes, project_id_info & 0x00ffffff) + manifest_field = read_string_view(io, bytes) + + + ntestitem_paths = Int(ntestitems) + testitems = Vector{@NamedTuple{name::typeof(pkgversion), file::typeof(pkgversion)}}(undef, ntestitems) + @assert read(io, UInt32) == 0 + path = read_string_view(io, bytes) + while ntestitem_paths > 0 + len_or_sentinel = read(io, UInt32) + if len_or_sentinel == 0 + path = read_string_view(io, bytes) + else + name = read_string_view(io, bytes, len_or_sentinel) + ntestitem_paths -= 1 + testitems[ntestitems - ntestitem_paths] = (name=name, file=path) + end + end + + statuses = Vector{@NamedTuple{status::_TEST_STATUS, run::Int8, number::Int, elapsed::Float32, compile::Float32}}(undef, ntestitems) + if ntestitems > 0 + # seek(io, offset_to_statuses) + for i in 1:ntestitems + status_info = read(io, UInt32) + status = Base.bitcast(_TEST_STATUS, UInt8((status_info >> 28) & 0x0f)) + run = Int8((status_info >> 24) & 0x0f) + number = Int(status_info & 0x00ffffff) + elapsed = read(io, Float32) + compile = read(io, Float32) + statuses[i] = (status=status, run=run, number=number, elapsed=elapsed, compile=compile) + end + end + return testitems, statuses, cfg, bytes +end diff --git a/test/internals.jl b/test/internals.jl index 1b215c9..f855bc6 100644 --- a/test/internals.jl +++ b/test/internals.jl @@ -75,11 +75,11 @@ end @assert isfile(monorepo_proj) for pkg in ("B", "C", "D") path = joinpath(monorepo, "monorepo_packages", pkg) - @test _is_subproject(path, monorepo_proj, monorepo) + @test _is_subproject(path, monorepo) end for dir in ("src", "test") path = joinpath(monorepo, dir) - @test !_is_subproject(path, monorepo_proj, monorepo) + @test !_is_subproject(path, monorepo) end # Test "test/Project.toml" does cause "test/" to be subproject tpf = joinpath(test_pkg_dir, "TestProjectFile.jl") @@ -88,7 +88,7 @@ end @assert isfile(joinpath(tpf, "test", "Project.toml")) for dir in ("src", "test") path = joinpath(tpf, dir) - @test !_is_subproject(path, tpf_proj, tpf) + @test !_is_subproject(path, tpf) end end