Merge pull request #621 from JuliaGPU/ksh/testset_nightly

Try using the new with-testset for nightly

Author: kshyatt

test/runtests.jl

@@ -276,75 +276,149 @@ t1 = now()
 elapsed = canonicalize(Dates.CompoundPeriod(t1-t0))
 println("Testing finished in $elapsed")
 
-# construct a testset to render the test results
-o_ts = Test.DefaultTestSet("Overall")
-Test.push_testset(o_ts)
-completed_tests = Set{String}()
-for (testname, (resp,)) in results
-    push!(completed_tests, testname)
-    if isa(resp, Test.DefaultTestSet)
-        Test.push_testset(resp)
-        Test.record(o_ts, resp)
-        Test.pop_testset()
-    elseif isa(resp, Tuple{Int,Int})
-        fake = Test.DefaultTestSet(testname)
-        for i in 1:resp[1]
-            Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
-        end
-        for i in 1:resp[2]
-            Test.record(fake, Test.Broken(:test, nothing))
+@static if VERSION < v"1.13.0-DEV.1044"
+    # construct a testset to render the test results
+    o_ts = Test.DefaultTestSet("Overall")
+    Test.push_testset(o_ts)
+    completed_tests = Set{String}()
+    for (testname, (resp,)) in results
+        push!(completed_tests, testname)
+        if isa(resp, Test.DefaultTestSet)
+            Test.push_testset(resp)
+            Test.record(o_ts, resp)
+            Test.pop_testset()
+        elseif isa(resp, Tuple{Int,Int})
+            fake = Test.DefaultTestSet(testname)
+            for i in 1:resp[1]
+                Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
+            end
+            for i in 1:resp[2]
+                Test.record(fake, Test.Broken(:test, nothing))
+            end
+            Test.push_testset(fake)
+            Test.record(o_ts, fake)
+            Test.pop_testset()
+        elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException)
+            println("Worker $(resp.pid) failed running test $(testname):")
+            Base.showerror(stdout, resp.captured)
+            println()
+            fake = Test.DefaultTestSet(testname)
+            for i in 1:resp.captured.ex.pass
+                Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
+            end
+            for i in 1:resp.captured.ex.broken
+                Test.record(fake, Test.Broken(:test, nothing))
+            end
+            for t in resp.captured.ex.errors_and_fails
+                Test.record(fake, t)
+            end
+            Test.push_testset(fake)
+            Test.record(o_ts, fake)
+            Test.pop_testset()
+        else
+            if !isa(resp, Exception)
+                resp = ErrorException(string("Unknown result type : ", typeof(resp)))
+            end
+            # If this test raised an exception that is not a remote testset exception,
+            # i.e. not a RemoteException capturing a TestSetException that means
+            # the test runner itself had some problem, so we may have hit a segfault,
+            # deserialization errors or something similar.  Record this testset as Errored.
+            fake = Test.DefaultTestSet(testname)
+            Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1)))
+            Test.push_testset(fake)
+            Test.record(o_ts, fake)
+            Test.pop_testset()
         end
+    end
+    for test in all_tests
+        (test in completed_tests) && continue
+        fake = Test.DefaultTestSet(test)
+        Test.record(fake, Test.Error(:test_interrupted, test, nothing,
+                                        [("skipped", [])], LineNumberNode(1)))
         Test.push_testset(fake)
         Test.record(o_ts, fake)
         Test.pop_testset()
-    elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException)
-        println("Worker $(resp.pid) failed running test $(testname):")
-        Base.showerror(stdout, resp.captured)
-        println()
-        fake = Test.DefaultTestSet(testname)
-        for i in 1:resp.captured.ex.pass
-            Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
-        end
-        for i in 1:resp.captured.ex.broken
-            Test.record(fake, Test.Broken(:test, nothing))
+    end
+    println()
+    Test.print_test_results(o_ts, 1)
+    if !o_ts.anynonpass
+        println("    \033[32;1mSUCCESS\033[0m")
+    else
+        println("    \033[31;1mFAILURE\033[0m\n")
+        Test.print_test_errors(o_ts)
+        throw(Test.FallbackTestSetException("Test run finished with errors"))
+    end
+else
+    # construct a testset to render the test results
+    o_ts = Test.DefaultTestSet("Overall")
+    Test.@with_testset o_ts begin
+        completed_tests = Set{String}()
+        for (testname, (resp,)) in results
+            push!(completed_tests, testname)
+            if isa(resp, Test.DefaultTestSet)
+                Test.@with_testset resp begin
+                    Test.record(o_ts, resp)
+                end
+            elseif isa(resp, Tuple{Int,Int})
+                fake = Test.DefaultTestSet(testname)
+                for i in 1:resp[1]
+                    Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
+                end
+                for i in 1:resp[2]
+                    Test.record(fake, Test.Broken(:test, nothing))
+                end
+                Test.@with_testset fake begin
+                    Test.record(o_ts, fake)
+                end
+            elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException)
+                println("Worker $(resp.pid) failed running test $(testname):")
+                Base.showerror(stdout, resp.captured)
+                println()
+                fake = Test.DefaultTestSet(testname)
+                for i in 1:resp.captured.ex.pass
+                    Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
+                end
+                for i in 1:resp.captured.ex.broken
+                    Test.record(fake, Test.Broken(:test, nothing))
+                end
+                for t in resp.captured.ex.errors_and_fails
+                    Test.record(fake, t)
+                end
+                Test.@with_testset fake begin
+                    Test.record(o_ts, fake)
+                end
+            else
+                if !isa(resp, Exception)
+                    resp = ErrorException(string("Unknown result type : ", typeof(resp)))
+                end
+                # If this test raised an exception that is not a remote testset exception,
+                # i.e. not a RemoteException capturing a TestSetException that means
+                # the test runner itself had some problem, so we may have hit a segfault,
+                # deserialization errors or something similar.  Record this testset as Errored.
+                fake = Test.DefaultTestSet(testname)
+                Test.record(fake, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack([(exception=resp,backtrace=[])]), LineNumberNode(1)))
+                Test.@with_testset fake begin
+                    Test.record(o_ts, fake)
+                end
+            end
         end
-        for t in resp.captured.ex.errors_and_fails
-            Test.record(fake, t)
+    end
+    for test in all_tests
+        (test in completed_tests) && continue
+        fake = Test.DefaultTestSet(test)
+        Test.record(fake, Test.Error(:test_interrupted, test, nothing, Base.ExceptionStack([(exception="skipped",backtrace=[])]), LineNumberNode(1)))
+        Test.@with_testset fake begin
+            Test.record(o_ts, fake)
         end
-        Test.push_testset(fake)
-        Test.record(o_ts, fake)
-        Test.pop_testset()
+    end
+    println()
+    Test.print_test_results(o_ts, 1)
+    if !o_ts.anynonpass
+        println("    \033[32;1mSUCCESS\033[0m")
     else
-        if !isa(resp, Exception)
-            resp = ErrorException(string("Unknown result type : ", typeof(resp)))
-        end
-        # If this test raised an exception that is not a remote testset exception,
-        # i.e. not a RemoteException capturing a TestSetException that means
-        # the test runner itself had some problem, so we may have hit a segfault,
-        # deserialization errors or something similar.  Record this testset as Errored.
-        fake = Test.DefaultTestSet(testname)
-        Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1)))
-        Test.push_testset(fake)
-        Test.record(o_ts, fake)
-        Test.pop_testset()
+        println("    \033[31;1mFAILURE\033[0m\n")
+        Test.print_test_errors(o_ts)
+        throw(Test.FallbackTestSetException("Test run finished with errors"))
     end
 end
-for test in all_tests
-    (test in completed_tests) && continue
-    fake = Test.DefaultTestSet(test)
-    Test.record(fake, Test.Error(:test_interrupted, test, nothing,
-                                    [("skipped", [])], LineNumberNode(1)))
-    Test.push_testset(fake)
-    Test.record(o_ts, fake)
-    Test.pop_testset()
-end
-println()
-Test.print_test_results(o_ts, 1)
-if !o_ts.anynonpass
-    println("    \033[32;1mSUCCESS\033[0m")
-else
-    println("    \033[31;1mFAILURE\033[0m\n")
-    Test.print_test_errors(o_ts)
-    throw(Test.FallbackTestSetException("Test run finished with errors"))
-end
 

test/setup.jl

@@ -9,7 +9,11 @@ using Random
 
 function runtests(f, name)
     old_print_setting = Test.TESTSET_PRINT_ENABLE[]
-    Test.TESTSET_PRINT_ENABLE[] = false
+    @static if VERSION < v"1.13.0-DEV.1044"
+        Test.TESTSET_PRINT_ENABLE[] = false
+    else
+        Test.TESTSET_PRINT_ENABLE[] => false
+    end
 
     try
         # generate a temporary module to execute the tests in
@@ -56,7 +60,11 @@ function runtests(f, name)
         GC.gc(true)
         res
     finally
-        Test.TESTSET_PRINT_ENABLE[] = old_print_setting
+        @static if VERSION < v"1.13.0-DEV.1044"
+            Test.TESTSET_PRINT_ENABLE[] = old_print_setting
+        else
+            Test.TESTSET_PRINT_ENABLE[] => old_print_setting
+        end
     end
 end