diff --git a/src/NarrativeTest.jl b/src/NarrativeTest.jl index 83f4bd1..a65e8c2 100644 --- a/src/NarrativeTest.jl +++ b/src/NarrativeTest.jl @@ -125,6 +125,7 @@ struct TestCase <: AbstractTestCase code::TextBlock pre::Union{TextBlock,Nothing} expect::Union{TextBlock,Nothing} + repl::Bool end location(test::TestCase) = test.loc @@ -537,11 +538,14 @@ function parsemd!(stack::Vector{TextBlock}) line = pop!(stack) if isfence(line) # Extract a fenced block. + isrepl = false fenceloc = line.loc lang = strip(line.val[4:end]) jlstack = TextBlock[] while !isempty(stack) && !isfence(stack[end]) - push!(jlstack, pop!(stack)) + block = pop!(stack) + isrepl = isrepl || startswith(block.val, "julia>") + push!(jlstack, block) end if isempty(stack) push!(suite, BrokenTestCase(fenceloc, "incomplete fenced code block")) @@ -549,17 +553,21 @@ function parsemd!(stack::Vector{TextBlock}) pop!(stack) if isempty(lang) reverse!(jlstack) - append!(suite, parsejl!(jlstack)) + append!(suite, isrepl ? parsejlrepl!(jlstack) : parsejl!(jlstack)) end end elseif isindent(line) && !isblank(line) # Extract an indented block. - jlstack = TextBlock[unindent(line)] + block = unindent(line) + isrepl = startswith(block.val, "julia>") + jlstack = TextBlock[block] while !isempty(stack) && (isindent(stack[end]) || isblank(stack[end])) - push!(jlstack, unindent(pop!(stack))) + block = unindent(pop!(stack)) + isrepl = isrepl || startswith(block.val, "julia>") + push!(jlstack, block) end reverse!(jlstack) - append!(suite, parsejl!(jlstack)) + append!(suite, isrepl ? parsejlrepl!(jlstack) : parsejl!(jlstack)) elseif isadmonition(line) # Skip an indented admonition block. while !isempty(stack) && (isindent(stack[end]) || isblank(stack[end])) @@ -588,6 +596,54 @@ function parsejl!(stack::Vector{TextBlock}) return suite end +const PROMPT_REGEX = r"^julia>(?: (.*))?$" +const SOURCE_REGEX = r"^ (.*)$" + +function parsejlrepl!(stack::Vector{TextBlock}) + reverse!(stack) + suite = AbstractTestCase[] + code, buf = nothing, IOBuffer() + function addcase!(expect) + case = !isempty(code.val) ? + TestCase(code.loc, code, nothing, expect, true) : + BrokenTestCase(code.loc, "missing test code") + push!(suite, case) + code = nothing + end + while true + line = popfirst!(stack) + prompt = match(PROMPT_REGEX, line.val) + if prompt !== nothing + prompt[1] !== nothing && println(buf, prompt[1]) + while !isempty(stack) + source = match(SOURCE_REGEX, stack[1].val) + source !== nothing || break + println(buf, source[1]) + popfirst!(stack) + end + code !== nothing && addcase!(TextBlock(code.loc, "")) + code = TextBlock(line.loc, consumebuf!(buf)) + else + println(buf, rstrip(line.val)) + while !isempty(stack) && !occursin(PROMPT_REGEX, stack[1].val) + println(buf, rstrip(popfirst!(stack).val)) + end + expect = TextBlock(line.loc, rstrip(consumebuf!(buf))) + code !== nothing && addcase!(expect) + end + if isempty(stack) + code !== nothing && addcase!(TextBlock(code.loc, "")) + break + end + end + suite +end + +function consumebuf!(buf) + n = bytesavailable(seekstart(buf)) + n > 0 ? String(take!(buf)) : "" +end + # Extract a test case from Julia source. function parsecase!(stack::Vector{TextBlock}) @@ -638,7 +694,7 @@ function parsecase!(stack::Vector{TextBlock}) end end !isempty(code) || return BrokenTestCase(loc, "missing test code") - return TestCase(loc, collapse(code), collapse(pre), collapse(expect)) + return TestCase(loc, collapse(code), collapse(pre), collapse(expect), false) end # Run a single test case. @@ -703,7 +759,11 @@ function runtest(test::TestCase; subs=common_subs(), mod=nothing) body = asexpr(test.code) ans = Core.eval(mod, body) if ans !== nothing && !no_output - Base.invokelatest(show, io, ans) + if test.repl + Base.invokelatest(show, io, "text/plain", ans) + else + Base.invokelatest(show, io, ans) + end end end catch exc @@ -746,10 +806,11 @@ end runtest(test::BrokenTestCase; subs=common_subs(), mod=nothing) = Error(test) -runtest(loc, code; pre=nothing, expect=nothing, subs=common_subs(), mod=nothing) = +runtest(loc, code; pre=nothing, expect=nothing, subs=common_subs(), mod=nothing, repl=false) = runtest(TestCase(loc, TextBlock(loc, code), pre !== nothing ? TextBlock(loc, pre) : nothing, - expect !== nothing ? TextBlock(loc, expect) : nothing), + expect !== nothing ? TextBlock(loc, expect) : nothing, + repl), subs=subs, mod=mod) # Convert expected output block to a regex. diff --git a/test/index.md b/test/index.md index 72dedc8..0898872 100644 --- a/test/index.md +++ b/test/index.md @@ -15,7 +15,7 @@ extract and run the embedded test suite. ans = runtests([joinpath(@__DIR__, "sample_good.md_")]); #=> - Tests passed: 3 + Tests passed: 4 TESTING SUCCESSFUL! =# @@ -50,9 +50,19 @@ reports the problem and returns `false`. Error at …/sample_bad.md_:17 missing test code ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Test failed at …/sample_bad.md_:21 + 42 + Expected output: + 43 + Actual output: + 42 + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Error at …/sample_bad.md_:26 + missing test code + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Tests passed: 1 - Tests failed: 2 - Errors: 1 + Tests failed: 3 + Errors: 2 TESTING UNSUCCESSFUL! =# @@ -77,6 +87,16 @@ To suppress any output except for error reports, specify parameter Error at …/sample_bad.md_:17 missing test code ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Test failed at …/sample_bad.md_:21 + 42 + Expected output: + 43 + Actual output: + 42 + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Error at …/sample_bad.md_:26 + missing test code + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ false =# @@ -144,7 +164,7 @@ library. joinpath(@__DIR__, "sample_bad.md_")]) #=> ⋮ - ERROR: Some tests did not pass: 4 passed, 2 failed, 1 errored, 0 broken. + ERROR: Some tests did not pass: 5 passed, 3 failed, 2 errored, 0 broken. =# Normally, it should be invoked from a test set. @@ -157,7 +177,7 @@ Normally, it should be invoked from a test set. end #=> ⋮ - ERROR: Some tests did not pass: 4 passed, 4 failed, 1 errored, 0 broken. + ERROR: Some tests did not pass: 5 passed, 5 failed, 2 errored, 0 broken. =# @@ -190,6 +210,12 @@ extracted test cases. 0.0 + 1.0im Error at …/sample_bad.md_:17 missing test code + Test case at …/sample_bad.md_:21 + 42 + Expected output: + 43 + Error at …/sample_bad.md_:26 + missing test code =# suite = parsemd("sample_missing.md_") @@ -384,6 +410,190 @@ It is also an error if a multi-line output block is not closed. incomplete multiline comment block =# +It is possible to define tests using syntax simulating REPL session (in fact +REPL session can be copy pasted as-is to become test cases), just prefix a line +in a code block with `julia> `: + + suite = parsemd( + @__FILE__, + IOBuffer(""" + These test cases are embedded in an indented code block. + + julia> (3+4)*6 + 42 + + julia> 2+2 + 5 + + The following test cases are embedded in a fenced code block. + ``` + julia> print(2^16) + 65526 + + julia> sqrt(-1) + 0.0 + 1.0im + ``` + """)) + foreach(display, suite) + #=> + Test case at …/index.md:3 + (3+4)*6 + Expected output: + 42 + Test case at …/index.md:6 + 2+2 + Expected output: + 5 + Test case at …/index.md:11 + print(2^16) + Expected output: + 65526 + Test case at …/index.md:14 + sqrt(-1) + Expected output: + 0.0 + 1.0im + =# + +Expected output can be missing: + + suite = parsemd( + @__FILE__, + IOBuffer(""" + The indented code block below is missing few expected outputs: + + julia> 1 + + julia> 2 + 2 + + julia> 3 + + Now fenced code block with missing few expected outputs: + + ``` + julia> 1 + + julia> 2 + 2 + + julia> 3 + ``` + """)) + foreach(display, suite) + #=> + Test case at …/index.md:3 + 1 + Expected output: + + Test case at …/index.md:5 + 2 + Expected output: + 2 + Test case at …/index.md:8 + 3 + Expected output: + + Test case at …/index.md:13 + 1 + Expected output: + + Test case at …/index.md:15 + 2 + Expected output: + 2 + Test case at …/index.md:18 + 3 + Expected output: + + =# + +REPL-like test cases can span multiple lines as in real REPL if we keep +indentation: + + suite = parsemd( + @__FILE__, + IOBuffer(""" + Multiline code blocks: + + julia> [1 2; + 3 4] + 2×2 Matrix{Int64}: + 1 2 + 3 4 + + julia> 1 + 2 + 2 + + julia> "no + output?" + + """)) + foreach(display, suite) + #=> + Test case at …/index.md:3 + [1 2; + 3 4] + Expected output: + 2×2 Matrix{Int64}: + 1 2 + 3 4 + Test case at …/index.md:9 + 1 + 2 + Expected output: + 2 + Test case at …/index.md:13 + "no + output?" + Expected output: + =# + +Empty prompt with REPL-like test cases is considered as invalid: + + suite = parsemd( + @__FILE__, + IOBuffer(""" + Some test cases are invalid below as they are missing code + to execute: + + julia> 1 + + julia> + + julia> 3 + 3 + + julia> 4; + julia> 5 + 5 + + julia> + + """)) + foreach(display, suite) + #=> + Test case at …/index.md:4 + 1 + Expected output: + + Error at …/index.md:6 + missing test code + Test case at …/index.md:8 + 3 + Expected output: + 3 + Test case at …/index.md:11 + 4; + Expected output: + + Test case at …/index.md:12 + 5 + Expected output: + 5 + Error at …/index.md:15 + missing test code + =# ## Running one test diff --git a/test/sample_bad.md_ b/test/sample_bad.md_ index 46db808..c66d883 100644 --- a/test/sample_bad.md_ +++ b/test/sample_bad.md_ @@ -12,6 +12,15 @@ This test should raise an exception: sqrt(-1) #-> 0.0 + 1.0im -Finally, the last test is ill-formed: +This one is ill-formed: #-> test code is missing + +Another test which should fail: + + julia> 42 + 43 + +And yet another ill-formed: + + julia> diff --git a/test/sample_good.md_ b/test/sample_good.md_ index 7756030..74eaa70 100644 --- a/test/sample_good.md_ +++ b/test/sample_good.md_ @@ -11,3 +11,8 @@ We could also check if some code produces expected output: We could use ellipsis to match a sequence of characters: collect(1:10) #-> [1, 2, …, 10] + +We could use specify tests using syntax simulating REPL session: + + julia> (3+4)*6 + 42