solcap: v2 (#7337)

* solcap: v2

* solcap: adding stake hooks

* solcap: add integration test

* solcap: integration test fix

---------

Co-authored-by: Kunal Bhargava <[email protected]>

Author: nipatel-jump

.github/workflows/tests.yml

@@ -147,3 +147,10 @@ jobs:
           sudo prlimit --pid $$ --memlock=-1:-1
           source /opt/${{ matrix.compiler }}/${{ matrix.compiler }}-${{ matrix.compiler-version }}/activate
           make run-integration-test
+
+      - name: run solcap tests
+        if: ${{ matrix.run-unit-tests && matrix.test-case == 'native' }}
+        run: |
+          sudo prlimit --pid $$ --memlock=-1:-1
+          source /opt/${{ matrix.compiler }}/${{ matrix.compiler }}-${{ matrix.compiler-version }}/activate
+          DUMP=../dump make run-solcap-tests

contrib/test/run_solcap_tests.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
-set -xeou pipefail
+set -eou pipefail
 
 source contrib/test/ledger_common.sh
 
 DUMP=${DUMP:="./dump"}
 OBJDIR=${OBJDIR:-build/native/gcc}
-echo $OBJDIR
+SKIP_INGEST=${SKIP_INGEST:-0}
 
-LEDGER="devnet-398736132-solcap"
+LEDGER="mainnet-376969880-solcap"
 REDOWNLOAD=1
 
 while [[ $# -gt 0 ]]; do
@@ -49,41 +49,86 @@ else
   check_ledger_checksum_and_redownload
 fi
 
-rm -rf $DUMP/$LEDGER/devnet-398736132_current.toml
-rm -rf $DUMP/$LEDGER/fd.solcap
-
-cp $DUMP/$LEDGER/devnet-398736132.toml $DUMP/$LEDGER/devnet-398736132_current.toml
+# Clone and build solcap-tools
+ORIG_DIR=$(pwd)
+cd $DUMP
+if [ ! -d "solcap-tools" ]; then
+  git clone https://github.com/firedancer-io/solcap-tools.git > /dev/null 2>&1
+fi
+cd solcap-tools
+git fetch origin > /dev/null 2>&1
+git checkout nishk/bp-fixes > /dev/null 2>&1
+git reset --hard origin/nishk/bp-fixes > /dev/null 2>&1
+cargo build --release > /dev/null 2>&1
+cd "$ORIG_DIR"
 
 export ledger_dir=$(realpath $DUMP/$LEDGER)
-sed -i "s#{ledger_dir}#${ledger_dir}#g" "$DUMP/$LEDGER/devnet-398736132_current.toml"
-sed -i "s/max_total_banks = [0-9]*/max_total_banks = 32/g" "$DUMP/$LEDGER/devnet-398736132_current.toml"
-sed -i -z "s/\[snapshots\].*\[layout\]/[layout]/" "$DUMP/$LEDGER/devnet-398736132_current.toml"
-sed -i "/writer_tile_count/d" "$DUMP/$LEDGER/devnet-398736132_current.toml"
-sed -i "/lock_pages/d" "$DUMP/$LEDGER/devnet-398736132_current.toml"
-sed -i "/heap_size_gib/d" "$DUMP/$LEDGER/devnet-398736132_current.toml"
-sed -i "/max_total_banks/d" "$DUMP/$LEDGER/devnet-398736132_current.toml"
-sed -i "/max_fork_width/d" "$DUMP/$LEDGER/devnet-398736132_current.toml"
-sed -i "/cluster_version/d" "$DUMP/$LEDGER/devnet-398736132_current.toml"
-
-echo "
-[gossip]
-  entrypoints = [ \"0.0.0.0:1\" ]" >> "$DUMP/$LEDGER/devnet-398736132_current.toml"
 
-echo "
+cat > "$DUMP/$LEDGER/mainnet-376969880_current.toml" << EOF
+
 [snapshots]
     incremental_snapshots = false
     [snapshots.sources]
         servers = []
         [snapshots.sources.gossip]
             allow_any = false
-            allow_list = []" >> "$DUMP/$LEDGER/devnet-398736132_current.toml"
+            allow_list = []
+[layout]
+    shred_tile_count = 4
+    snapla_tile_count = 1
+    verify_tile_count = 2
+    exec_tile_count = 6
+[tiles]
+    [tiles.archiver]
+        enabled = true
+        end_slot = 376969900
+        rocksdb_path = "${ledger_dir}/rocksdb"
+        shredcap_path = "${ledger_dir}/shreds.pcapng.zst"
+        ingest_mode = "shredcap"
+    [tiles.replay]
+        enable_features = [  ]
+    [tiles.gui]
+        enabled = false
+    [tiles.rpc]
+        enabled = false
+[store]
+    max_completed_shred_sets = 32768
+[funk]
+    heap_size_gib = 1
+    max_account_records = 2000000
+    max_database_transactions = 64
+[runtime]
+    max_live_slots = 32
+    max_fork_width = 4
+[log]
+    level_stderr = "NOTICE"
+    path = "/tmp/ledger_log_solcap"
+
+[paths]
+    snapshots = "${ledger_dir}"
 
-$OBJDIR/bin/firedancer-dev configure init all --config $DUMP/$LEDGER/devnet-398736132_current.toml
-$OBJDIR/bin/firedancer-dev backtest --config $DUMP/$LEDGER/devnet-398736132_current.toml
-$OBJDIR/bin/firedancer-dev configure fini all --config $DUMP/$LEDGER/devnet-398736132_current.toml
+[capture]
+    solcap_capture = "${ledger_dir}/mainnet-376969880-solcap.solcap"
+[development]
+    [development.snapshots]
+        disable_lthash_verification = true
+[gossip]
+    entrypoints = [ "0.0.0.0:1" ]
+EOF
+
+$OBJDIR/bin/firedancer-dev backtest --config $DUMP/$LEDGER/mainnet-376969880_current.toml
+
+# Run solcap-tools diff and check the summary for zero differences
+DIFF_OUTPUT=$($DUMP/solcap-tools/target/release/solcap-tools diff $DUMP/$LEDGER/mainnet-376969880-solcap.solcap $DUMP/$LEDGER/ledger_tool/bank_hash_details/ -v 5)
+echo "$DIFF_OUTPUT"
 
-$OBJDIR/bin/fd_solcap_import $DUMP/$LEDGER/bank_hash_details/ $DUMP/$LEDGER/solana.solcap
-$OBJDIR/bin/fd_solcap_diff $DUMP/$LEDGER/solana.solcap $DUMP/$LEDGER/fd.solcap -v 4
+SUMMARY=$(echo "$DIFF_OUTPUT" | tail -3) > /dev/null 2>&1
+DIFFERING_SLOTS=$(echo "$SUMMARY" | grep -ioP 'Differing Slots: \K\d+' || echo "") > /dev/null 2>&1
+DIFFERING_ACCOUNTS=$(echo "$SUMMARY" | grep -ioP 'Differing Accounts: \K\d+' || echo "") > /dev/null 2>&1
+
+if [[ "$DIFFERING_SLOTS" != "0" || "$DIFFERING_ACCOUNTS" != "0" ]]; then
+  echo -e "\033[0;31mFAIL\033[0m Solcap diff found mismatches! Differing Slots: $DIFFERING_SLOTS, Differing Accounts: $DIFFERING_ACCOUNTS" >&2
+  exit 1
+fi
 
-# check that the ledger is not corrupted after a run
-check_ledger_checksum
+exit 0

src/app/firedancer-dev/commands/backtest.c

@@ -27,7 +27,7 @@
 #include "../../../discof/tower/fd_tower_tile.h"
 #include "../../../discof/replay/fd_exec.h"
 #include "../../../ballet/lthash/fd_lthash.h"
-#include "../../../flamenco/runtime/context/fd_capture_ctx.h"
+#include "../../../flamenco/capture/fd_capture_ctx.h"
 #include "../../../disco/pack/fd_pack_cost.h"
 #include "../../../flamenco/progcache/fd_progcache_admin.h"
 
@@ -102,6 +102,14 @@ backtest_topo( config_t * config ) {
   #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
   FOR(exec_tile_cnt) fd_topob_tile( topo, "exec", "exec", "metric_in", cpu_idx++, 0, 0 );
 
+  /**********************************************************************/
+  /* Add the capture tile to topo                                       */
+  /**********************************************************************/
+  if( solcap_enabled ) {
+    fd_topob_wksp( topo, "solcap" );
+    fd_topob_tile( topo, "solcap", "solcap", "metric_in", cpu_idx++, 0, 0 );
+  }
+
   /**********************************************************************/
   /* Add the snapshot tiles to topo                                       */
   /**********************************************************************/
@@ -313,15 +321,7 @@ backtest_topo( config_t * config ) {
   /**********************************************************************/
   fd_topob_wksp( topo, "exec_replay" );
 
-  /* If solcap is enabled, we need to overload this link to also send
-     solcap account updates to the replay tile. We can't use a separate
-     link for this without introducing a race. This will get removed with solcap V2. */
-  if( FD_UNLIKELY( solcap_enabled ) ) {
-    /* TODO: remove this with solcap V2 */
-    FOR(exec_tile_cnt) fd_topob_link( topo, "exec_replay", "exec_replay", 1024UL, FD_CAPTURE_CTX_ACCOUNT_UPDATE_MSG_FOOTPRINT, 1UL );
-  } else {
-    FOR(exec_tile_cnt) fd_topob_link( topo, "exec_replay", "exec_replay", 16384UL, sizeof(fd_exec_task_done_msg_t), 1UL );
-  }
+  FOR(exec_tile_cnt) fd_topob_link( topo, "exec_replay", "exec_replay", 16384UL, sizeof(fd_exec_task_done_msg_t), 1UL );
 
   FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "exec_replay", i );
   FOR(exec_tile_cnt) fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "exec_replay", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
@@ -330,6 +330,18 @@ backtest_topo( config_t * config ) {
   /* Setup the shared objs used by replay and exec tiles                */
   /**********************************************************************/
 
+  if( FD_UNLIKELY( solcap_enabled ) ) {
+    /* 32 sections of SOLCAP_WRITE_ACCOUNT_DATA_MTU bytes each ≈ 4MB.
+       This is done to ideally avoid cache thrashing and allow for all
+       the links to sit on L3 cache. */
+    fd_topob_link( topo, "cap_repl", "solcap", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    fd_topob_tile_out( topo, "replay", 0UL, "cap_repl", 0UL );
+    fd_topob_tile_in( topo, "solcap", 0UL, "metric_in", "cap_repl", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+    FOR(exec_tile_cnt) fd_topob_link( topo, "cap_exec", "solcap", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "cap_exec", i );
+    FOR(exec_tile_cnt) fd_topob_tile_in( topo, "solcap", 0UL, "metric_in", "cap_exec", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+  }
+
   fd_topob_wksp( topo, "store" );
   fd_topo_obj_t * store_obj = setup_topo_store( topo, "store", config->firedancer.store.max_completed_shred_sets, 1 );
   fd_topob_tile_uses( topo, backt_tile, store_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );

src/app/firedancer-dev/main.c

@@ -108,6 +108,7 @@ extern fd_topo_run_tile_t fd_tile_archiver_writer;
 extern fd_topo_run_tile_t fd_tile_archiver_playback;
 extern fd_topo_run_tile_t fd_tile_shredcap;
 extern fd_topo_run_tile_t fd_tile_vinyl;
+extern fd_topo_run_tile_t fd_tile_solcap;
 
 extern fd_topo_run_tile_t fd_tile_snapct;
 extern fd_topo_run_tile_t fd_tile_snapld;
@@ -168,6 +169,7 @@ fd_topo_run_tile_t * TILES[] = {
   &fd_tile_genesi,
   &fd_tile_ipecho,
   &fd_tile_vinyl,
+  &fd_tile_solcap,
   NULL,
 };
 

src/app/firedancer/topology.c

@@ -22,6 +22,7 @@
 #include "../../util/tile/fd_tile_private.h"
 #include "../../discof/restore/utils/fd_ssctrl.h"
 #include "../../discof/restore/utils/fd_ssmsg.h"
+#include "../../flamenco/capture/fd_solcap_writer.h"
 #include "../../flamenco/progcache/fd_progcache_admin.h"
 #include "../../vinyl/meta/fd_vinyl_meta.h"
 #include "../../vinyl/io/fd_vinyl_io.h" /* FD_VINYL_IO_TYPE_* */
@@ -344,6 +345,8 @@ fd_topo_initialize( config_t * config ) {
   topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
   topo->gigantic_page_threshold = config->hugetlbfs.gigantic_page_threshold_mib << 20;
 
+  int solcap_enabled = strlen( config->capture.solcap_capture ) > 0;
+
   /*             topo, name */
   fd_topob_wksp( topo, "metric"       );
   fd_topob_wksp( topo, "genesi"       );
@@ -619,6 +622,11 @@ fd_topo_initialize( config_t * config ) {
   /**/                 fd_topob_tile( topo, "poh",     "poh",     "metric_in",  tile_to_cpu[ topo->tile_cnt ], 0,        1 );
   FOR(sign_tile_cnt)   fd_topob_tile( topo, "sign",    "sign",    "metric_in",  tile_to_cpu[ topo->tile_cnt ], 0,        1 );
 
+  if( FD_UNLIKELY( solcap_enabled ) ) {
+    fd_topob_wksp( topo, "solcap" );
+    fd_topob_tile( topo, "solcap", "solcap", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
+  }
+
   /*                                        topo, tile_name, tile_kind_id, fseq_wksp,   link_name,      link_kind_id, reliable,            polled */
   FOR(gossvf_tile_cnt) for( ulong j=0UL; j<net_tile_cnt; j++ )
                       fd_topob_tile_in(     topo, "gossvf",  i,            "metric_in", "net_gossvf",   j,            FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
@@ -898,6 +906,15 @@ fd_topo_initialize( config_t * config ) {
     fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "rpc_replay", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
   }
 
+  if( FD_UNLIKELY( solcap_enabled ) ) {
+    fd_topob_link( topo, "cap_repl", "solcap", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    fd_topob_tile_out( topo, "replay", 0UL, "cap_repl", 0UL );
+    fd_topob_tile_in( topo, "solcap", 0UL, "metric_in", "cap_repl", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+    FOR(exec_tile_cnt) fd_topob_link( topo, "cap_exec", "solcap", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "cap_exec", i );
+    FOR(exec_tile_cnt) fd_topob_tile_in( topo, "solcap", 0UL, "metric_in", "cap_exec", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+  }
+
   if( FD_LIKELY( !is_auto_affinity ) ) {
     if( FD_UNLIKELY( affinity_tile_cnt<topo->tile_cnt ) )
       FD_LOG_ERR(( "The topology you are using has %lu tiles, but the CPU affinity specified in the config tile as [layout.affinity] only provides for %lu cores. "
@@ -1480,6 +1497,13 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
         FD_VINYL_IO_TYPE_UR : FD_VINYL_IO_TYPE_BD;
     tile->vinyl.uring_depth = config->firedancer.vinyl.io_uring.queue_depth;
 
+  } else if( FD_UNLIKELY( !strcmp( tile->name, "solcap" ) ) ) {
+
+    tile->capctx.capture_start_slot = config->capture.capture_start_slot;
+    strncpy( tile->capctx.solcap_capture, config->capture.solcap_capture, sizeof(tile->capctx.solcap_capture) );
+    tile->capctx.recent_only = config->capture.recent_only;
+    tile->capctx.recent_slots_per_file = config->capture.recent_slots_per_file;
+
   } else {
     FD_LOG_ERR(( "unknown tile name `%s`", tile->name ));
   }

src/app/shared/fd_config.h

@@ -167,6 +167,10 @@ struct fd_configf {
   struct {
     ulong max_completed_shred_sets;
   } store;
+
+  struct {
+    char path[ PATH_MAX ];
+  } capctx;
 };
 
 typedef struct fd_configf fd_configf_t;
@@ -496,6 +500,8 @@ struct fd_config {
     ulong capture_start_slot;
     char  dump_proto_dir[ PATH_MAX ];
     char  solcap_capture[ PATH_MAX ];
+    int   recent_only;
+    ulong recent_slots_per_file;
     int   dump_elf_to_pb;
     int   dump_syscall_to_pb;
     int   dump_instr_to_pb;

src/app/shared/fd_config_parse.c

@@ -247,6 +247,8 @@ fd_config_extract_pod( uchar *       pod,
 
   CFG_POP      ( ulong,  capture.capture_start_slot                       );
   CFG_POP      ( cstr,   capture.solcap_capture                           );
+  CFG_POP      ( bool,   capture.recent_only                              );
+  CFG_POP      ( ulong,  capture.recent_slots_per_file                    );
   CFG_POP      ( cstr,   capture.dump_proto_dir                           );
   CFG_POP      ( bool,   capture.dump_elf_to_pb                           );
   CFG_POP      ( bool,   capture.dump_syscall_to_pb                       );

src/disco/topo/fd_topo.h

@@ -628,6 +628,13 @@ struct fd_topo_tile {
       int  io_type; /* FD_VINYL_IO_TYPE_* */
       uint uring_depth;
     } vinyl;
+
+    struct {
+      ulong capture_start_slot;
+      char  solcap_capture[ PATH_MAX ];
+      int   recent_only;
+      ulong recent_slots_per_file;
+    } capctx;
   };
 };
 

src/discof/capture/Local.mk

@@ -0,0 +1,5 @@
+ifdef FD_HAS_INT128
+ifdef FD_HAS_ALLOCA
+$(call add-objs,fd_solcap_tile,fd_discof)
+endif
+endif