Add option enable_multi_node to edb:reverse_attach()

Summary:
# Context
As part of multi-node debugging support, we need to issue a startDebugging reverse-request for each new node that attempts to attach.

# Problem
We need to make this an optional part of reverse attaching, since we want the experience of debugging and reverse attaching onto one node to remain the same.

# This diff
Adds `multi_node_enabled` and `add_flags_necessary` as `boolean()` options to `edb:reverse_attach/1`. The `node_attached` event is now only sent when `multi_node_enabled` is set to `true`. It is by default `false`. `add_flags_necessary` is true by default and specifies if the user requires edb to setup ERL_AFLAGS to inject reverse attach code, or if they have already done so.

Reviewed By: jcpetruzza

Differential Revision: D80168956

fbshipit-source-id: 003f868f9ddb1f38d89cf5a6414aa919bd2a3810

Author: agastyabahl

edb_core/src/edb.erl

@@ -271,16 +271,33 @@ Subscribers will receive a single event of the form `{edb_event, Subscription, E
 
 The events include a `Ref` that should match the reference that was returned by this call.
 
+When the `enable_multi_node` options is set to `true`, additional nodes may attempt to reverse attach after the
+initial reverse-attach event. In that case, those nodes will be paused and subscribers will receive a new
+ `{reverse_attach, Ref, {node_attached, AdditionalNode}}` event. In that case, `edb` will remain attached
+ to the initial node.
+
+By default, it is the caller's responsibility to ensure that additional nodes execute the code returned by this
+function, so that these nodes try to reverse-attach as well. As a convenience, the `instrument_erl_aflags_on_attach` can
+be set to `true` and the `ERL_AFLAGS` variable of the first node attached will be instrumented to contain a suitable
+`-eval` instruction.
+
 This call may start distribution and set the node name.
 
 Options:
 
 - `name_domain`: whether we expect to be attached by a node using longnames or shortnames;
-- `timeout`: how long to wait for the node to be up; defaults to infinity.
+- `timeout`: how long to wait for the node to be up; defaults to infinity;
+- `multi_node_enabled`: whether to allow multiple nodes to attach; defaults to `false`;
+- `instrument_erl_aflags_on_attach`: the ERL_AFLAGS will be updated with an `-eval` flag on the first attachment; defaults to `false`.
 
 """.
 -spec reverse_attach(Opts) -> {ok, Info} | {error, Reason} when
-    Opts :: #{name_domain := longnames | shortnames, timeout => timeout()},
+    Opts :: #{
+        name_domain := longnames | shortnames,
+        timeout => timeout(),
+        multi_node_enabled => boolean(),
+        instrument_erl_aflags_on_attach => boolean()
+    },
     Info :: #{erl_code_to_inject := binary(), reverse_attach_ref := reference()},
     Reason :: attachment_in_progress.
 reverse_attach(AttachOpts0) ->
@@ -289,19 +306,43 @@ reverse_attach(AttachOpts0) ->
         default => infinity,
         parse => fun parse_timeout/1
     }),
+    {MultiNodeEnabled, AttachOpts3} = take_arg(multi_node_enabled, AttachOpts2, #{
+        default => false,
+        parse => fun parse_boolean/1
+    }),
+    {InstrumentErlAflagsOnAttach, AttachOpts4} = take_arg(instrument_erl_aflags_on_attach, AttachOpts3, #{
+        default => false,
+        parse => fun parse_boolean/1
+    }),
 
-    ok = no_more_args(AttachOpts2),
+    ok = no_more_args(AttachOpts4),
 
     case net_kernel:get_state() of
         #{started := no} -> ok = start_distribution(NameDomain);
         #{started := _, name_domain := NameDomain} -> ok;
         #{started := _, name_domain := _} -> error({invalid_name_domain, NameDomain})
     end,
 
-    {ok, GatekeeperId, ReverseAttachCode} = edb_gatekeeper:new(),
+    {ok, GatekeeperId, ReverseAttachCode} = edb_gatekeeper:new(#{multi_node_enabled => MultiNodeEnabled}),
+    ReverseAttachCodeInFlags =
+        case {MultiNodeEnabled, InstrumentErlAflagsOnAttach} of
+            {true, true} ->
+                ReverseAttachCode;
+            {false, true} ->
+                error({badarg, {instrument_erl_aflags_on_attach, requires, multi_node_enabled}});
+            _ ->
+                none
+        end,
+
     ReverseAttachRef = erlang:make_ref(),
     case
-        edb_node_monitor:expect_reverse_attach(GatekeeperId, ReverseAttachRef, ReverseAttachTimeout, ReverseAttachCode)
+        edb_node_monitor:expect_reverse_attach(
+            GatekeeperId, ReverseAttachRef, #{
+                timeout => ReverseAttachTimeout,
+                reverse_attach_code => ReverseAttachCodeInFlags,
+                multi_node_enabled => MultiNodeEnabled
+            }
+        )
     of
         ok ->
             {ok, #{
@@ -865,6 +906,10 @@ parse_timeout(Timeout) -> parse_non_neg_integer(Timeout).
 parse_name_domain(longnames) -> longnames;
 parse_name_domain(shortnames) -> shortnames.
 
+-spec parse_boolean(term()) -> boolean().
+parse_boolean(true) -> true;
+parse_boolean(false) -> false.
+
 -spec parse_pair(L, R) -> fun((term()) -> {A, B}) when
     L :: fun((term()) -> A), R :: fun((term()) -> B).
 parse_pair(L, R) ->

edb_core/src/edb_node_monitor.erl

@@ -26,7 +26,7 @@
 %% Public API
 -export([start_link/0]).
 -export([attach/2, detach/0, attached_node/0]).
--export([expect_reverse_attach/4, reverse_attach_notification/2]).
+-export([expect_reverse_attach/3, reverse_attach_notification/2]).
 
 -export([subscribe/0, unsubscribe/1]).
 -export([safe_sname_hostname/0]).
@@ -138,20 +138,22 @@ attached_node() ->
             error(not_attached)
     end.
 
--spec expect_reverse_attach(Id, ReverseAttachRef, Timeout, ReverseAttachCode) -> ok | {error, Reason} when
+-spec expect_reverse_attach(Id, ReverseAttachRef, Opts) ->
+    ok | {error, Reason}
+when
     Id :: edb_gatekeeper:id(),
     ReverseAttachRef :: reference(),
-    Timeout :: timeout(),
-    ReverseAttachCode :: none | binary(),
+    Opts :: #{
+        timeout := timeout(),
+        reverse_attach_code := none | binary(),
+        multi_node_enabled := boolean()
+    },
     Reason :: attachment_in_progress.
-expect_reverse_attach(Id, ReverseAttachRef, Timeout, ReverseAttachCode) ->
+expect_reverse_attach(Id, ReverseAttachRef, Opts) ->
     call(
-        {expect_reverse_attach, #{
+        {expect_reverse_attach, Opts#{
             gatekeeper_id => Id,
-            reverse_attach_ref => ReverseAttachRef,
-            timeout => Timeout,
-            reverse_attach_code => ReverseAttachCode,
-            multi_node_enabled => false
+            reverse_attach_ref => ReverseAttachRef
         }}
     ).
 
@@ -393,17 +395,12 @@ expect_reverse_attach_impl(
     Data0
 ) ->
     {_, _, Data1} = detach_impl_1(State0, Data0),
-    ReverseAttachCodeToUse =
-        case MultiNodeEnabled of
-            true -> ReverseAttachCode;
-            false -> none
-        end,
     State1 = #{
         state => attachment_in_progress,
         type => reverse_attach,
         gatekeeper => GatekeeperId,
         reverse_attach_ref => ReverseAttachRef,
-        multi_node_reverse_attach_code => ReverseAttachCodeToUse,
+        multi_node_reverse_attach_code => ReverseAttachCode,
         multi_node_enabled => MultiNodeEnabled
     },
     {next_state, State1, Data1, [