solcap: v2

Author: nipatel-jump

contrib/test/run_solcap_tests.sh

@@ -76,8 +76,5 @@ $OBJDIR/bin/firedancer-dev configure init all --config $DUMP/$LEDGER/devnet-3987
 $OBJDIR/bin/firedancer-dev backtest --config $DUMP/$LEDGER/devnet-398736132_current.toml
 $OBJDIR/bin/firedancer-dev configure fini all --config $DUMP/$LEDGER/devnet-398736132_current.toml &> /dev/null
 
-$OBJDIR/bin/fd_solcap_import $DUMP/$LEDGER/bank_hash_details/ $DUMP/$LEDGER/solana.solcap
-$OBJDIR/bin/fd_solcap_diff $DUMP/$LEDGER/solana.solcap $DUMP/$LEDGER/fd.solcap -v 4
-
 # check that the ledger is not corrupted after a run
 check_ledger_checksum

src/app/firedancer-dev/commands/backtest.c

@@ -27,7 +27,7 @@
 #include "../../../discof/tower/fd_tower_tile.h"
 #include "../../../discof/replay/fd_exec.h"
 #include "../../../ballet/lthash/fd_lthash.h"
-#include "../../../flamenco/runtime/context/fd_capture_ctx.h"
+#include "../../../discof/capture/fd_capture_ctx.h"
 #include "../../../disco/pack/fd_pack_cost.h"
 #include "../../../flamenco/progcache/fd_progcache_admin.h"
 
@@ -100,6 +100,14 @@ backtest_topo( config_t * config ) {
   #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
   FOR(exec_tile_cnt) fd_topob_tile( topo, "exec", "exec", "metric_in", cpu_idx++, 0, 0 );
 
+  /**********************************************************************/
+  /* Add the capture tile to topo                                       */
+  /**********************************************************************/
+  if (solcap_enabled) {
+    fd_topob_wksp( topo, "captur" );
+    fd_topob_tile( topo, "captur", "captur", "metric_in", cpu_idx++, 0, 0 );
+  }
+
   /**********************************************************************/
   /* Add the snapshot tiles to topo                                       */
   /**********************************************************************/
@@ -236,15 +244,7 @@ backtest_topo( config_t * config ) {
   /**********************************************************************/
   fd_topob_wksp( topo, "exec_replay" );
 
-  /* If solcap is enabled, we need to overload this link to also send
-     solcap account updates to the replay tile. We can't use a separate
-     link for this without introducing a race. This will get removed with solcap V2. */
-  if( FD_UNLIKELY( solcap_enabled ) ) {
-    /* TODO: remove this with solcap V2 */
-    FOR(exec_tile_cnt) fd_topob_link( topo, "exec_replay", "exec_replay", 1024UL, FD_CAPTURE_CTX_ACCOUNT_UPDATE_MSG_FOOTPRINT, 1UL );
-  } else {
-    FOR(exec_tile_cnt) fd_topob_link( topo, "exec_replay", "exec_replay", 16384UL, sizeof(fd_exec_task_done_msg_t), 1UL );
-  }
+  FOR(exec_tile_cnt) fd_topob_link( topo, "exec_replay", "exec_replay", 16384UL, sizeof(fd_exec_task_done_msg_t), 1UL );
 
   FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "exec_replay", i );
   FOR(exec_tile_cnt) fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "exec_replay", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
@@ -253,6 +253,18 @@ backtest_topo( config_t * config ) {
   /* Setup the shared objs used by replay and exec tiles                */
   /**********************************************************************/
 
+  if ( FD_UNLIKELY( solcap_enabled ) ) {
+    /* 32 sections of SOLCAP_WRITE_ACCOUNT_DATA_MTU bytes each ≈ 4MB.
+       This is done to ideally avoid cache thrashing and allow for all
+       the links to sit on L3 cache. */
+    fd_topob_link( topo, "cap_repl", "captur", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    fd_topob_tile_out( topo, "replay", 0UL, "cap_repl", 0UL );
+    fd_topob_tile_in( topo, "captur", 0UL, "metric_in", "cap_repl", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+    FOR(exec_tile_cnt) fd_topob_link( topo, "cap_exec", "captur",  32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "cap_exec", i );
+    FOR(exec_tile_cnt) fd_topob_tile_in( topo, "captur", 0UL, "metric_in", "cap_exec", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+  }
+
   fd_topob_wksp( topo, "store" );
   fd_topo_obj_t * store_obj = setup_topo_store( topo, "store", config->firedancer.store.max_completed_shred_sets, 1 );
   fd_topob_tile_uses( topo, backt_tile, store_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );

src/app/firedancer-dev/main.c

@@ -106,6 +106,7 @@ extern fd_topo_run_tile_t fd_tile_archiver_feeder;
 extern fd_topo_run_tile_t fd_tile_archiver_writer;
 extern fd_topo_run_tile_t fd_tile_archiver_playback;
 extern fd_topo_run_tile_t fd_tile_shredcap;
+extern fd_topo_run_tile_t fd_tile_capture;
 
 extern fd_topo_run_tile_t fd_tile_snapct;
 extern fd_topo_run_tile_t fd_tile_snapld;
@@ -161,6 +162,7 @@ fd_topo_run_tile_t * TILES[] = {
   &fd_tile_snapwr,
   &fd_tile_genesi,
   &fd_tile_ipecho,
+  &fd_tile_capture,
   NULL,
 };
 

src/app/firedancer/topology.c

@@ -20,6 +20,7 @@
 #include "../../util/tile/fd_tile_private.h"
 #include "../../discof/restore/utils/fd_ssctrl.h"
 #include "../../discof/restore/utils/fd_ssmsg.h"
+#include "../../flamenco/capture/fd_solcap_writer.h"
 #include "../../flamenco/progcache/fd_progcache_admin.h"
 #include "../../vinyl/meta/fd_vinyl_meta.h"
 
@@ -270,6 +271,8 @@ fd_topo_initialize( config_t * config ) {
   topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
   topo->gigantic_page_threshold = config->hugetlbfs.gigantic_page_threshold_mib << 20;
 
+  int solcap_enabled = strlen( config->capture.solcap_capture ) > 0;
+
   /*             topo, name */
   fd_topob_wksp( topo, "metric"       );
   fd_topob_wksp( topo, "genesi"       );
@@ -508,6 +511,11 @@ fd_topo_initialize( config_t * config ) {
   /**/                 fd_topob_tile( topo, "poh",     "poh",     "metric_in",  tile_to_cpu[ topo->tile_cnt ], 0,        1 );
   FOR(sign_tile_cnt)   fd_topob_tile( topo, "sign",    "sign",    "metric_in",  tile_to_cpu[ topo->tile_cnt ], 0,        1 );
 
+  if( FD_UNLIKELY( solcap_enabled ) ) {
+    fd_topob_wksp( topo, "captur"      );
+    fd_topob_tile( topo, "captur", "captur", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
+  }
+
   /*                                        topo, tile_name, tile_kind_id, fseq_wksp,   link_name,      link_kind_id, reliable,            polled */
   FOR(gossvf_tile_cnt) for( ulong j=0UL; j<net_tile_cnt; j++ )
                       fd_topob_tile_in(     topo, "gossvf",  i,            "metric_in", "net_gossvf",   j,            FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
@@ -728,6 +736,15 @@ fd_topo_initialize( config_t * config ) {
   FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec",      i,                               "exec_replay", i );
   FOR(exec_tile_cnt) fd_topob_tile_in(  topo, "replay",    0UL,    "metric_in", "exec_replay", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
 
+  if( FD_UNLIKELY( solcap_enabled ) ) {
+    fd_topob_link( topo, "cap_repl", "captur", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    fd_topob_tile_out( topo, "replay", 0UL, "cap_repl", 0UL );
+    fd_topob_tile_in( topo, "captur", 0UL, "metric_in", "cap_repl", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+    FOR(exec_tile_cnt) fd_topob_link( topo, "cap_exec", "captur", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "cap_exec", i );
+    FOR(exec_tile_cnt) fd_topob_tile_in( topo, "captur", 0UL, "metric_in", "cap_exec", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+  }
+
   if( FD_LIKELY( !is_auto_affinity ) ) {
     if( FD_UNLIKELY( affinity_tile_cnt<topo->tile_cnt ) )
       FD_LOG_ERR(( "The topology you are using has %lu tiles, but the CPU affinity specified in the config tile as [layout.affinity] only provides for %lu cores. "
@@ -1246,6 +1263,11 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
     tile->shredcap.enable_publish_stake_weights = 0; /* this is not part of the config */
     strncpy( tile->shredcap.manifest_path, "", PATH_MAX ); /* this is not part of the config */
 
+  } else if( FD_UNLIKELY( !strcmp( tile->name, "captur" ) ) ) {
+
+    tile->capctx.capture_start_slot = config->capture.capture_start_slot;
+    strncpy( tile->capctx.solcap_capture, config->capture.solcap_capture, sizeof(tile->capctx.solcap_capture) );
+
   } else {
     FD_LOG_ERR(( "unknown tile name `%s`", tile->name ));
   }

src/app/ledger/main.c

@@ -1,6 +1,6 @@
 #include "../../flamenco/types/fd_types.h"
 #include "../../flamenco/runtime/fd_rocksdb.h"
-#include "../../flamenco/runtime/context/fd_capture_ctx.h"
+#include "../../discof/capture/fd_capture_ctx.h"
 #include <unistd.h>
 #include <sys/stat.h>
 

src/app/shared/fd_config.h

@@ -156,6 +156,10 @@ struct fd_configf {
   struct {
     ulong max_completed_shred_sets;
   } store;
+
+  struct {
+    char path[ PATH_MAX ];
+  } capctx;
 };
 
 typedef struct fd_configf fd_configf_t;

src/disco/stem/fd_stem.c

@@ -37,6 +37,14 @@
    or monitoring tools.  The ctx is a user-provided context object from
    when the stem tile was initialized.
 
+    CUSTOM_INPUT_SELECTION
+   Is called to determine if the stem should shuffle the input
+   selection.  The ctx is a user-provided context object from when the
+   stem tile was initialized.  The stem should return 1 if the input
+   selection should be shuffled, 0 otherwise. Used alongside
+   STEM_CUSTOM_INPUT_ADVANCE_FLAG to determine if the input selection
+   should be advanced.
+
      BEFORE_CREDIT
    Is called every iteration of the stem run loop, whether there is a
    new frag ready to receive or not.  This callback is also still
@@ -484,14 +492,19 @@ STEM_(run1)( ulong                        in_cnt,
         /* We also do the same with the ins to prevent there being a
            correlated order frag origins from different inputs
            downstream at extreme fan in and extreme in load. */
-
-        if( FD_LIKELY( in_cnt>1UL ) ) {
+#ifdef STEM_CUSTOM_INPUT_SELECTION
+        int shuffle_flag = STEM_CUSTOM_INPUT_ADVANCE_FLAG(ctx);
+#else
+        int shuffle_flag = 1;
+#endif
+        if( FD_LIKELY( in_cnt>1UL && shuffle_flag ) ) {
           swap_idx = (ulong)fd_rng_uint_roll( rng, (uint)in_cnt );
           fd_stem_tile_in_t in_tmp;
           in_tmp         = in[ swap_idx ];
           in[ swap_idx ] = in[ 0        ];
           in[ 0        ] = in_tmp;
         }
+
       }
 
       /* Reload housekeeping timer */
@@ -579,10 +592,12 @@ STEM_(run1)( ulong                        in_cnt,
     }
 #endif
 
-    fd_stem_tile_in_t * this_in = &in[ in_seq ];
+fd_stem_tile_in_t * this_in = &in[ in_seq ];
+#ifdef STEM_CUSTOM_INPUT_SELECTION
+#else
     in_seq++;
     if( in_seq>=in_cnt ) in_seq = 0UL; /* cmov */
-
+#endif
     /* Check if this in has any new fragments to mux */
 
     ulong                  this_in_seq   = this_in->seq;
@@ -714,6 +729,15 @@ STEM_(run1)( ulong                        in_cnt,
     this_in->accum[ FD_METRICS_COUNTER_LINK_CONSUMED_COUNT_OFF ]++;
     this_in->accum[ FD_METRICS_COUNTER_LINK_CONSUMED_SIZE_BYTES_OFF ] += (uint)sz;
 
+    /* Custom input selection: advance to next input based on flag */
+#ifdef STEM_CUSTOM_INPUT_SELECTION
+    int should_advance = STEM_CUSTOM_INPUT_ADVANCE_FLAG(ctx);
+    if( FD_LIKELY( should_advance ) ) {
+      in_seq++;
+      if( in_seq>=in_cnt ) in_seq = 0UL; /* cmov */
+    }
+#endif
+
     metric_regime_ticks[1] += housekeeping_ticks;
     metric_regime_ticks[4] += prefrag_ticks;
     long next = fd_tickcount();
@@ -815,3 +839,5 @@ STEM_(run)( fd_topo_t *      topo,
 #undef STEM_CALLBACK_RETURNABLE_FRAG
 #undef STEM_CALLBACK_AFTER_FRAG
 #undef STEM_CALLBACK_AFTER_POLL_OVERRUN
+#undef STEM_CUSTOM_INPUT_SELECTION
+#undef STEM_CUSTOM_INPUT_ADVANCE_FLAG

src/disco/topo/fd_topo.h

@@ -584,6 +584,11 @@ struct fd_topo_tile {
       uint target_gid;
       uint target_uid;
     } genesi;
+
+    struct {
+      ulong capture_start_slot;
+      char solcap_capture[ PATH_MAX ];
+    } capctx;
   };
 };
 

src/discof/capture/Local.mk

@@ -0,0 +1,6 @@
+ifdef FD_HAS_INT128
+ifdef FD_HAS_ALLOCA
+$(call add-hdrs,fd_capture_ctx.h)
+$(call add-objs,fd_capture_ctx fd_capture_tile,fd_discof)
+endif
+endif