`replaying a built-in returned a different result than expected` when a process is restarted

[thalesmg]

Describe the bug
While trying to model an algorithm where a named process is restarted, I get a Concuerror crashed error that says replaying a built-in returned a different result than expected.

To Reproduce
Steps to reproduce the behavior:

Module with code and test at `/tmp/test.erl`

-module(test).

%% rm concuerror_report.txt; clear; (bin/concuerror -f /tmp/test.erl -t spin_down_test --treat-as-normal spindown --treat_as_normal shutdown -x error_handler -x logger || {cat concuerror_report.txt; exit 1})

-compile([export_all]).

start_monitor() ->
    gen_server:start_monitor({local, ?MODULE}, ?MODULE, [], []).

start_manager() ->
    gen_server:start_link({local, manager}, ?MODULE, manager, []).

ensure_started() ->
    ok = gen_server:call(manager, spinup, infinity).

stop() ->
    catch gen_server:stop(?MODULE),
    ok.

request(X) ->
    case whereis(?MODULE) of
        undefined ->
            ensure_started(),
            request(X);
        Pid ->
            Ref = monitor(process, Pid),
            Pid ! {req, Ref, self(), X},
            receive
                {Ref, Resp} ->
                    Resp;
                {'DOWN', Ref, process, _Pid, spindown} ->
                    ensure_started(),
                    request(X);
                {'DOWN', Ref, process, _Pid, noproc} ->
                    ensure_started(),
                    request(X)
            end
    end.

init(manager) ->
    State = #{pending => #{}, worker => undefined, will_ensure => false},
    {ok, State};
init(_) ->
    {ok, #{}}.

handle_call(spinup, From, State0 = #{pending := Pending0}) ->
    case maps:get(will_ensure, State0, false) of
        false ->
            self() ! ensure,
            State = State0#{ pending := Pending0#{From => true}
                           , will_ensure => true
                           };
        true ->
            State = State0#{ pending := Pending0#{From => true}}
    end,
    {noreply, State}.

handle_cast(spindown, State) ->
    {stop, spindown, State}.

handle_info({req, Ref, From, _X}, State) ->
    From ! {Ref, ok},
    {noreply, State};
handle_info(ensure, State0 = #{pending := Pending}) ->
    case maps:get(worker, State0, undefined) of
        undefined ->
            {ok, Ret} = start_monitor(),
            maps:foreach(fun(From, _) -> gen_server:reply(From, ok) end, Pending),
            State = State0#{worker => Ret, pending => #{}, will_ensure => false};
        {Pid, Ref} ->
            case is_process_alive(Pid) of
                true ->
                    maps:foreach(fun(From, _) -> gen_server:reply(From, ok) end, Pending),
                    State = State0#{pending => #{}, will_ensure => false};
                false ->
                    demonitor(Ref, [flush]),
                    {ok, Ret} = start_monitor(),
                    maps:foreach(fun(From, _) -> gen_server:reply(From, ok) end, Pending),
                    State = State0#{worker => Ret, pending => #{}, will_ensure => false}
            end
    end,
    {noreply, State};
handle_info({'DOWN', Ref, process, Pid, spindown}, State0 = #{worker := {Pid, Ref}}) ->
    State = State0#{worker := undefined},
    {noreply, State};
handle_info(_, State) ->
    {noreply, State}.

spin_down_test() ->
    start_manager(),
    Me = self(),
    spawn(fun() ->
                  Resp = request(go),
                  Me ! {ok, Resp}
          end),
    spawn(fun() ->
                  Resp = request(go),
                  Me ! {ok, Resp}
          end),
    spawn(fun() ->
                  gen_server:cast(?MODULE, spindown)
          end),
    receive
        {ok, _} -> ok
    end,
    receive
        {ok, _} -> ok
    end,
    stop(),
    gen_server:stop(manager),
    ok.

Concuerror command:

rm concuerror_report.txt; clear; (bin/concuerror -f /tmp/test.erl -t spin_down_test --treat-as-normal spindown --treat_as_normal shutdown -x error_handler -x logger || {cat concuerror_report.txt; exit 1})

Output

Concuerror 0.21.0+build.2381.refedc846d started at 22 Mar 2024 19:40:38

* Info: Showing progress ('-h progress', for details)
* Info: Writing results in concuerror_report.txt
* Info: Only logging errors ('--log_all false')
* Info: Automatically instrumented module io_lib
* Info: Showing PIDs as "<symbolic name(/last registered name)>" ('-h symbolic_names').
* Info: "--treat-as-normal" converted to "--treat_as_normal"
* Warning: Not instrumenting module error_handler
* Warning: Not instrumenting module logger
* Info: Instrumented & loaded module test
* Info: Automatically instrumented module gen_server
* Info: Automatically instrumented module gen
* Tip: Running without a scheduling_bound corresponds to verification and may take a long time.
* Info: Automatically instrumented module proc_lib
* Info: Automatically instrumented module erlang
* Info: Automatically instrumented module maps
* Info: Automatically instrumented module sys
* Warning: Concuerror may let exiting processes emit 'DOWN' messages for cancelled monitors. Any such messages are discarded upon delivery and can never be received.
* Info: You can see pairs of racing instructions (in the report and '--graph') with '--show_races true'
* Info: Automatically instrumented module lists
* Info: Automatically instrumented module error_logger
* Info: Automatically instrumented module application
* Info: Automatically instrumented module application_controller
* Warning: Concuerror does not properly support erlang:process_info(_, messages), returning an empty list instead.
* Warning: Concuerror does not properly support erlang:process_info(_, status), returning always 'running' instead.
* Warning: Concuerror does not properly support erlang:process_info(_, heap_size), returning 42 instead.
* Warning: Concuerror does not properly support erlang:process_info(_, stack_size), returning 42 instead.
* Warning: Concuerror does not properly support erlang:process_info(_, reductions), returning 42 instead.
* Warning: Some abnormal exit reasons were treated as normal ('--treat_as_normal').
* Tip: Increase '--print_depth' if output/graph contains "...".
* Error: Stop testing on first error. (Check '-h keep_going').
* Error: On step 56, replaying a built-in returned a different result than expected:
  original:
    <0.137.0>: {dictionary,[{'$ancestors',[manager,<0.122.0>]},{rand_seed,{#{max => 288230376151711743,type => exsplus,next => #Fun<rand.5.65977474>,jump => #Fun<rand.3.65977474>},[62625312397571194|161708621020023473]}},{'$initial_call',{test,init,1}}]} = erlang:process_info(<0.137.0>, dictionary)
    in proc_lib.erl line {812,5}
  new:
    <0.137.0>: {dictionary,[{'$ancestors',[manager,<0.122.0>]},{'$initial_call',{test,init,1}}]} = erlang:process_info(<0.137.0>, dictionary)
    in proc_lib.erl line {812,5}
 Please notify the developers, as this is a bug of Concuerror.
Done at 22 Mar 2024 19:40:44 (Exit status: fail)
  Summary: 1 errors, 6/12 interleavings explored
Concuerror 0.21.0+build.2381.refedc846d started at 22 Mar 2024 19:40:38
 Options:
  [{after_timeout,infinity},
   {assertions_only,false},
   {assume_racing,true},
   {depth_bound,500},
   {disable_sleep_sets,false},
   {dpor,optimal},
   {entry_point,{test,spin_down_test,[]}},
   {exclude_module,[error_handler,logger]},
   {files,["/tmp/test.erl"]},
   {first_process_errors_only,false},
   {ignore_error,[]},
   {instant_delivery,true},
   {interleaving_bound,infinity},
   {keep_going,false},
   {log_all,false},
   {non_racing_system,[]},
   {print_depth,20},
   {scheduling,round_robin},
   {scheduling_bound_type,none},
   {show_races,false},
   {strict_scheduling,false},
   {symbolic_names,true},
   {timeout,5000},
   {treat_as_normal,[shutdown,spindown]},
   {use_receive_patterns,true}]
################################################################################
Interleaving #6
--------------------------------------------------------------------------------
Errors found:
* Concuerror crashed
--------------------------------------------------------------------------------
Event trace:
   1: <P>: undefined = erlang:whereis(manager)
    in gen.erl line {617,26}
   2: <P>: [] = erlang:process_info(<P>, registered_name)
    in proc_lib.erl line {812,5}
   3: <P>: {<P.1/manager>,#Ref<0.3730236886.1714946049.81187>} = erlang:spawn_opt(proc_lib, init_p, [<P>,[],gen,init_it,[gen_server,<P>,<P>,{local,manager},test,manager,[]]], [link,monitor])
    in proc_lib.erl line {192,5}
   4: <P.1/manager>: true = erlang:register(manager, <P.1/manager>)
    in gen.erl line {622,9}
   5: <P.1/manager>: {ack,<P.1/manager>,{ok,<P.1/manager>}} = <P> ! {ack,<P.1/manager>,{ok,<P.1/manager>}}
    in proc_lib.erl line {449,12}
   6: <P>: receives message ({ack,<P.1/manager>,{ok,<P.1/manager>}})
    in proc_lib.erl line {310,5}
   7: <P>: true = erlang:demonitor(#Ref<0.3730236886.1714946049.81187>, [flush])
    in proc_lib.erl line {312,6}
   8: <P>: <P.2> = erlang:spawn(erlang, apply, [#Fun<test.'-spin_down_test/0-fun-0-'.0>,[]])
    in erlang.erl line {3220,5}
   9: <P>: <P.3> = erlang:spawn(erlang, apply, [#Fun<test.'-spin_down_test/0-fun-1-'.0>,[]])
    in erlang.erl line {3220,5}
  10: <P>: <P.4> = erlang:spawn(erlang, apply, [#Fun<test.'-spin_down_test/0-fun-2-'.0>,[]])
    in erlang.erl line {3220,5}
  11: <P.2>: undefined = erlang:whereis(test)
    in test.erl line {21,10}
  12: <P.2>: <P.1/manager> = erlang:whereis(manager)
    in gen.erl line {575,10}
  13: <P.2>: #Ref<0.3730236886.1714946049.81282> = erlang:monitor(process, <P.1/manager>)
    in gen.erl line {236,12}
  14: <P.2>: {'$gen_call',{<P.2>,#Ref<0.3730236886.1714946049.81282>},spinup} = <P.1/manager> ! {'$gen_call',{<P.2>,#Ref<0.3730236886.1714946049.81282>},spinup}
    in gen.erl line {240,13}
  15: <P.3>: undefined = erlang:whereis(test)
    in test.erl line {21,10}
  16: <P.3>: <P.1/manager> = erlang:whereis(manager)
    in gen.erl line {575,10}
  17: <P.3>: #Ref<0.3730236886.1714946049.81289> = erlang:monitor(process, <P.1/manager>)
    in gen.erl line {236,12}
  18: <P.3>: {'$gen_call',{<P.3>,#Ref<0.3730236886.1714946049.81289>},spinup} = <P.1/manager> ! {'$gen_call',{<P.3>,#Ref<0.3730236886.1714946049.81289>},spinup}
    in gen.erl line {240,13}
  19: <P.1/manager>: receives message ({'$gen_call',{<P.2>,#Ref<0.3730236886.1714946049.81282>},spinup})
    in gen_server.erl line {1011,5}
  20: <P.1/manager>: ensure = <P.1/manager> ! ensure
    in test.erl line {49,20}
  21: <P.1/manager>: receives message ({'$gen_call',{<P.3>,#Ref<0.3730236886.1714946049.81289>},spinup})
    in gen_server.erl line {1011,5}
  22: <P.1/manager>: receives message (ensure)
    in gen_server.erl line {1011,5}
  23: <P.1/manager>: undefined = erlang:whereis(test)
    in gen.erl line {617,26}
  24: <P.1/manager>: {registered_name,manager} = erlang:process_info(<P.1/manager>, registered_name)
    in proc_lib.erl line {812,5}
  25: <P.1/manager>: {<P.1.1/test>,#Ref<0.3730236886.1714946049.81336>} = erlang:spawn_opt(proc_lib, init_p, [manager,[<P>],gen,init_it,[gen_server,<P.1/manager>,<P.1/manager>,{local,test},test,[],[]]], [monitor])
    in proc_lib.erl line {192,5}
  26: <P.1.1/test>: true = erlang:register(test, <P.1.1/test>)
    in gen.erl line {622,9}
  27: <P.1.1/test>: {ack,<P.1.1/test>,{ok,<P.1.1/test>}} = <P.1/manager> ! {ack,<P.1.1/test>,{ok,<P.1.1/test>}}
    in proc_lib.erl line {449,12}
  28: <P.1/manager>: receives message ({ack,<P.1.1/test>,{ok,<P.1.1/test>}})
    in proc_lib.erl line {399,5}
  29: <P.1/manager>: {#Ref<0.3730236886.1714946049.81282>,ok} = <P.2> ! {#Ref<0.3730236886.1714946049.81282>,ok}
    in gen.erl line {551,12}
  30: <P.1/manager>: {#Ref<0.3730236886.1714946049.81289>,ok} = <P.3> ! {#Ref<0.3730236886.1714946049.81289>,ok}
    in gen.erl line {551,12}
  31: <P.2>: receives message ({#Ref<0.3730236886.1714946049.81282>,ok})
    in gen.erl line {241,5}
  32: <P.2>: true = erlang:demonitor(#Ref<0.3730236886.1714946049.81282>, [flush])
    in gen.erl line {243,13}
  33: <P.2>: <P.1.1/test> = erlang:whereis(test)
    in test.erl line {21,10}
  34: <P.2>: #Ref<0.3730236886.1714946049.81353> = erlang:monitor(process, <P.1.1/test>)
    in test.erl line {26,19}
  35: <P.2>: {req,#Ref<0.3730236886.1714946049.81353>,<P.2>,go} = <P.1.1/test> ! {req,#Ref<0.3730236886.1714946049.81353>,<P.2>,go}
    in test.erl line {27,17}
  36: <P.3>: receives message ({#Ref<0.3730236886.1714946049.81289>,ok})
    in gen.erl line {241,5}
  37: <P.3>: true = erlang:demonitor(#Ref<0.3730236886.1714946049.81289>, [flush])
    in gen.erl line {243,13}
  38: <P.3>: <P.1.1/test> = erlang:whereis(test)
    in test.erl line {21,10}
  39: <P.3>: #Ref<0.3730236886.1714946049.81361> = erlang:monitor(process, <P.1.1/test>)
    in test.erl line {26,19}
  40: <P.3>: {req,#Ref<0.3730236886.1714946049.81361>,<P.3>,go} = <P.1.1/test> ! {req,#Ref<0.3730236886.1714946049.81361>,<P.3>,go}
    in test.erl line {27,17}
  41: <P.4>: {'$gen_cast',spindown} = erlang:send(test, {'$gen_cast',spindown})
    in gen_server.erl line {1062,9}
  42: <P.4>: exits normally
  43: <P.1.1/test>: receives message ({req,#Ref<0.3730236886.1714946049.81353>,<P.2>,go})
    in gen_server.erl line {1011,5}
  44: <P.1.1/test>: {#Ref<0.3730236886.1714946049.81353>,ok} = <P.2> ! {#Ref<0.3730236886.1714946049.81353>,ok}
    in test.erl line {62,10}
  45: <P.1.1/test>: receives message ({req,#Ref<0.3730236886.1714946049.81361>,<P.3>,go})
    in gen_server.erl line {1011,5}
  46: <P.1.1/test>: {#Ref<0.3730236886.1714946049.81361>,ok} = <P.3> ! {#Ref<0.3730236886.1714946049.81361>,ok}
    in test.erl line {62,10}
  47: <P.1.1/test>: receives message ({'$gen_cast',spindown})
    in gen_server.erl line {1011,5}
  48: <P.1.1/test>: {current_stacktrace,[{gen_server,handle_common_reply,8,[{file,[103,101,110,95,115,101,114,118,101|...]},{line,1226}]},{proc_lib,init_p_do_apply,3,[{file,[112,114,111,99,95,108,105,98|...]},{line,241}]}]} = erlang:process_info(<P.1.1/test>, current_stacktrace)
    in gen_server.erl line {1226,25}
  49: <P.1.1/test>: {registered_name,test} = erlang:process_info(<P.1.1/test>, registered_name)
    in proc_lib.erl line {812,5}
  50: <P.1.1/test>: {dictionary,[{'$ancestors',[manager,<P>]},{'$initial_call',{test,init,1}}]} = erlang:process_info(<P.1.1/test>, dictionary)
    in proc_lib.erl line {812,5}
  51: <P.1.1/test>: {message_queue_len,0} = erlang:process_info(<P.1.1/test>, message_queue_len)
    in proc_lib.erl line {812,5}
  52: <P.1.1/test>: unlimited = ets:lookup_element(ac_tab, {env,kernel,error_logger_format_depth}, 2, unlimited)
    in application_controller.erl line {357,5}
  53: <P.1.1/test>: {messages,[]} = erlang:process_info(<P.1.1/test>, messages)
    in proc_lib.erl line {812,5}
  54: <P.1.1/test>: unlimited = ets:lookup_element(ac_tab, {env,kernel,error_logger_format_depth}, 2, unlimited)
    in application_controller.erl line {357,5}
  55: <P.1.1/test>: {links,[]} = erlang:process_info(<P.1.1/test>, links)
    in proc_lib.erl line {812,5}
  56: <P.1.1/test>: {dictionary,[{'$ancestors',[manager,<P>]},{rand_seed,{#{max => 288230376151711743,type => exsplus,next => #Fun<rand.exsp_next.1>,jump => #Fun<rand.exsplus_jump.1>},[62625312397571194|161708621020023473]}},{'$initial_call',{test,init,1}}]} = erlang:process_info(<P.1.1/test>, dictionary)
    in proc_lib.erl line {812,5}
  57: <P.1.1/test>: unlimited = ets:lookup_element(ac_tab, {env,kernel,error_logger_format_depth}, 2, unlimited)
    in application_controller.erl line {357,5}
  58: <P.1.1/test>: {trap_exit,false} = erlang:process_info(<P.1.1/test>, trap_exit)
    in proc_lib.erl line {812,5}
  59: <P.1.1/test>: {status,running} = erlang:process_info(<P.1.1/test>, status)
    in proc_lib.erl line {812,5}
  60: <P.1.1/test>: {heap_size,42} = erlang:process_info(<P.1.1/test>, heap_size)
    in proc_lib.erl line {812,5}
  61: <P.1.1/test>: {stack_size,42} = erlang:process_info(<P.1.1/test>, stack_size)
    in proc_lib.erl line {812,5}
  62: <P.1.1/test>: {reductions,42} = erlang:process_info(<P.1.1/test>, reductions)
    in proc_lib.erl line {812,5}
  63: <P.1.1/test>: {links,[]} = erlang:process_info(<P.1.1/test>, links)
    in proc_lib.erl line {812,5}
  64: <P.1.1/test>: exits abnormally (spindown)
  65: <P.1.1/test>: {'DOWN',#Ref<0.3730236886.1714946049.81336>,process,<P.1.1/test>,spindown} = erlang:send(<P.1/manager>, {'DOWN',#Ref<0.3730236886.1714946049.81336>,process,<P.1.1/test>,spindown})
    (while exiting)
  66: <P.1.1/test>: {'DOWN',#Ref<0.3730236886.1714946049.81353>,process,<P.1.1/test>,spindown} = erlang:send(<P.2>, {'DOWN',#Ref<0.3730236886.1714946049.81353>,process,<P.1.1/test>,spindown})
    (while exiting)
  67: <P.1.1/test>: {'DOWN',#Ref<0.3730236886.1714946049.81361>,process,<P.1.1/test>,spindown} = erlang:send(<P.3>, {'DOWN',#Ref<0.3730236886.1714946049.81361>,process,<P.1.1/test>,spindown})
    (while exiting)
  68: <P.1/manager>: receives message ({'DOWN',#Ref<0.3730236886.1714946049.81336>,process,<P.1.1/test>,spindown})
    in gen_server.erl line {1011,5}
  69: <P.2>: receives message ({#Ref<0.3730236886.1714946049.81353>,ok})
    in test.erl line {28,13}
################################################################################
Exploration completed!
################################################################################
Errors:
--------------------------------------------------------------------------------
* Stop testing on first error. (Check '-h keep_going').
* On step 56, replaying a built-in returned a different result than expected:
  original:
    <0.137.0>: {dictionary,[{'$ancestors',[manager,<0.122.0>]},{rand_seed,{#{max => 288230376151711743,type => exsplus,next => #Fun<rand.5.65977474>,jump => #Fun<rand.3.65977474>},[62625312397571194|161708621020023473]}},{'$initial_call',{test,init,1}}]} = erlang:process_info(<0.137.0>, dictionary)
    in proc_lib.erl line {812,5}
  new:
    <0.137.0>: {dictionary,[{'$ancestors',[manager,<0.122.0>]},{'$initial_call',{test,init,1}}]} = erlang:process_info(<0.137.0>, dictionary)
    in proc_lib.erl line {812,5}
 Please notify the developers, as this is a bug of Concuerror.

################################################################################
Warnings:
--------------------------------------------------------------------------------
* Not instrumenting module error_handler
* Not instrumenting module logger
* Concuerror may let exiting processes emit 'DOWN' messages for cancelled monitors. Any such messages are discarded upon delivery and can never be received.
* Concuerror does not properly support erlang:process_info(_, messages), returning an empty list instead.
* Concuerror does not properly support erlang:process_info(_, status), returning always 'running' instead.
* Concuerror does not properly support erlang:process_info(_, heap_size), returning 42 instead.
* Concuerror does not properly support erlang:process_info(_, stack_size), returning 42 instead.
* Concuerror does not properly support erlang:process_info(_, reductions), returning 42 instead.
* Some abnormal exit reasons were treated as normal ('--treat_as_normal').

################################################################################
Tips:
--------------------------------------------------------------------------------
* Running without a scheduling_bound corresponds to verification and may take a long time.
* Increase '--print_depth' if output/graph contains "...".

################################################################################
Info:
--------------------------------------------------------------------------------
* Showing progress ('-h progress', for details)
* Writing results in concuerror_report.txt
* Only logging errors ('--log_all false')
* Automatically instrumented module io_lib
* Showing PIDs as "<symbolic name(/last registered name)>" ('-h symbolic_names').
* "--treat-as-normal" converted to "--treat_as_normal"
* Instrumented & loaded module test
* Automatically instrumented module gen_server
* Automatically instrumented module gen
* Automatically instrumented module proc_lib
* Automatically instrumented module erlang
* Automatically instrumented module maps
* Automatically instrumented module sys
* You can see pairs of racing instructions (in the report and '--graph') with '--show_races true'
* Automatically instrumented module lists
* Automatically instrumented module error_logger
* Automatically instrumented module application
* Automatically instrumented module application_controller

################################################################################
Done at 22 Mar 2024 19:40:44 (Exit status: fail)
  Summary: 1 errors, 6/12 interleavings explored

concuerror_report.txt

Highlighting the main error:

Interleaving #6
--------------------------------------------------------------------------------
Errors found:
* Concuerror crashed

...

Errors:
--------------------------------------------------------------------------------
* Stop testing on first error. (Check '-h keep_going').
* On step 56, replaying a built-in returned a different result than expected:
  original:
    <0.137.0>: {dictionary,[{'$ancestors',[manager,<0.122.0>]},{rand_seed,{#{max => 288230376151711743,type => exsplus,next => #Fun<rand.5.65977474>,jump => #Fun<rand.3.65977474>},[62625312397571194|161708621020023473]}},{'$initial_call',{test,init,1}}]} = erlang:process_info(<0.137.0>, dictionary)
    in proc_lib.erl line {812,5}
  new:
    <0.137.0>: {dictionary,[{'$ancestors',[manager,<0.122.0>]},{'$initial_call',{test,init,1}}]} = erlang:process_info(<0.137.0>, dictionary)
    in proc_lib.erl line {812,5}
 Please notify the developers, as this is a bug of Concuerror.

Expected behavior

There should be no replay mismatch mentioning rand_seed, since it's not used. I've tried setting it explicitly to a fixed value when initializing the gen_server process, but I still get a replay mismatch.

Environment (please complete the following information):

• OS: Archlinux
• Concuerror Version: Concuerror 0.21.0+build.2381.refedc846d

Additional context

This apparently is related to restarting a named process, and also to some internal process in Concuerror that injects rand_seed into the process dictionary. After the process is restarted during exploration, it lacks rand_seed and the replay_mismatch error is hit.

Using the following patch, the model is verified successfully. Of course, this is not a proper fix. But I don't expect that the correctness in this case relies on the process dictionary.

diff --git a/src/concuerror_scheduler.erl b/src/concuerror_scheduler.erl
index efbd16f..3dd29f3 100644
--- a/src/concuerror_scheduler.erl
+++ b/src/concuerror_scheduler.erl
@@ -420,7 +420,7 @@ get_next_event(Event, MaybeNeedsReplayState) ->
         case Label =/= undefined of
           true ->
             NewEvent = get_next_event_backend(Event, State),
-            try {ok, Event} = NewEvent
+            try {ok, _Event} = NewEvent
             catch
               _:_ ->
                 New =
@@ -1727,7 +1727,7 @@ replay_prefix_aux([#trace_state{done = [Event|_], index = I}|Rest], State) ->
   ?debug(_Logger, "~s~n", [?pretty_s(I, Event)]),
   {ok, #event{actor = Actor} = NewEvent} = get_next_event_backend(Event, State),
   try
-    true = Event =:= NewEvent
+    true
   catch
     _:_ ->
       #scheduler_state{print_depth = PrintDepth} = State,

Output with patch

Concuerror 0.21.0+build.2381.refedc846d started at 22 Mar 2024 19:53:16

* Info: Showing progress ('-h progress', for details)
* Info: Writing results in concuerror_report.txt
* Info: Only logging errors ('--log_all false')
* Info: Automatically instrumented module io_lib
* Info: Showing PIDs as "<symbolic name(/last registered name)>" ('-h symbolic_names').
* Info: "--treat-as-normal" converted to "--treat_as_normal"
* Warning: Not instrumenting module error_handler
* Warning: Not instrumenting module logger
* Info: Instrumented & loaded module test
* Info: Automatically instrumented module gen_server
* Info: Automatically instrumented module gen
* Tip: Running without a scheduling_bound corresponds to verification and may take a long time.
* Info: Automatically instrumented module proc_lib
* Info: Automatically instrumented module erlang
* Info: Automatically instrumented module maps
* Info: Automatically instrumented module sys
* Warning: Concuerror may let exiting processes emit 'DOWN' messages for cancelled monitors. Any such messages are discarded upon delivery and can never be received.
* Info: You can see pairs of racing instructions (in the report and '--graph') with '--show_races true'
* Info: Automatically instrumented module lists
* Info: Automatically instrumented module error_logger
* Info: Automatically instrumented module application
* Info: Automatically instrumented module application_controller
* Warning: Concuerror does not properly support erlang:process_info(_, messages), returning an empty list instead.
* Warning: Concuerror does not properly support erlang:process_info(_, status), returning always 'running' instead.
* Warning: Concuerror does not properly support erlang:process_info(_, heap_size), returning 42 instead.
* Warning: Concuerror does not properly support erlang:process_info(_, stack_size), returning 42 instead.
* Warning: Concuerror does not properly support erlang:process_info(_, reductions), returning 42 instead.
* Warning: Some abnormal exit reasons were treated as normal ('--treat_as_normal').
Done at 22 Mar 2024 19:54:08 (Exit status: ok)
  Summary: 0 errors, 9040/9040 interleavings explored

[aronisstav]

Hi! Definitely a bug, but the suggested fix breaks the fundamental assumption that "functions that can return different results shouldn't do so on replays" (otherwise we can't be sure we are "back" at the same state), so it cannot be used.

The right fix requires to find the builtin op that has been missed (or identify why concuerror injected the rand_seed) and will be more involved... I may be able to suggest workarounds if you can share some relatively minimal example that breaks, but I can't promise a lot these days.

[thalesmg]

Hi!

I didn't suggest the patch above as a fix at all, it's not a proper fix. 😅
It was merely a workaround so I could let the check continue and see if I got any other errors like dead locks (which it did find in previous versions of my example code above after the workaround) .

I already shared an example in the issue that breaks without the patch at the current HEAD of the repo. Would you need for it to be even smaller?

[thalesmg]

I shrank the example module a bit more, hope it helps. 🤞
The command to run is the same as above.

/tmp/test.erl

-module(test).

%% rm concuerror_report.txt; clear; (bin/concuerror -f /tmp/test.erl -t spin_down_test --treat-as-normal spindown --treat_as_normal shutdown -x error_handler -x logger || {cat concuerror_report.txt; exit 1})

-compile([export_all]).

start_monitor() ->
    gen_server:start_monitor({local, ?MODULE}, ?MODULE, [], []).

ensure_started() ->
    case start_monitor() of
        {ok, _} ->
            ok;
        {error, {already_started, _}} ->
            ok
    end.

stop() ->
    catch gen_server:stop(?MODULE),
    ok.

request(X) ->
    case whereis(?MODULE) of
        undefined ->
            ok = ensure_started(),
            request(X);
        Pid ->
            Ref = monitor(process, Pid),
            Pid ! {req, Ref, self(), X},
            receive
                {Ref, Resp} ->
                    Resp;
                {'DOWN', Ref, process, _Pid, spindown} ->
                    ok = ensure_started(),
                    request(X);
                {'DOWN', Ref, process, _Pid, noproc} ->
                    ok = ensure_started(),
                    request(X)
            end
    end.

init(_) ->
    {ok, #{}}.

handle_cast(spindown, State) ->
    {stop, spindown, State}.

handle_info({req, Ref, From, _X}, State) ->
    From ! {Ref, ok},
    {noreply, State};
handle_info(_, State) ->
    {noreply, State}.

spin_down_test() ->
    Me = self(),
    spawn(fun() ->
                  Resp = request(go),
                  Me ! {ok, Resp}
          end),
    spawn(fun() ->
                  Resp = request(go),
                  Me ! {ok, Resp}
          end),
    spawn(fun() ->
                  gen_server:cast(?MODULE, spindown)
          end),
    receive
        {ok, _} -> ok
    end,
    receive
        {ok, _} -> ok
    end,
    stop(),
    ok.