Add txset construction mode to apply load

docs/apply-load-benchmark-sac.cfg

@@ -8,10 +8,22 @@ APPLY_LOAD_MODE="benchmark"
 APPLY_LOAD_MODEL_TX="sac"
 
 # Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
+# "txset-validation-and-apply" also times tx-set construction from an
+# overfilled candidate backlog and validation before application.
 APPLY_LOAD_TIMING_PHASES = "apply"
 
+# Purposefully overfill the candidate backlog to simulate a busy production
+# validator's queue. In "txset-validation-and-apply", these multipliers default
+# to 2 and must be at least 2 (the classic multiplier is checked only when
+# APPLY_LOAD_CLASSIC_TXS_PER_LEDGER > 0). Raise them, e.g. to 4, to increase
+# the backlog without raising ledger limits. More candidates need more
+# accounts; increase GENESIS_TEST_ACCOUNT_COUNT if needed.
+# These flags scale candidate counts. The Soroban multiplier applies after
+# SAC batching, i.e. to
+# APPLY_LOAD_MAX_SOROBAN_TX_COUNT / APPLY_LOAD_BATCH_SAC_COUNT.
+SOROBAN_TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING = 2
+TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING = 2
+
 # Whether to time the write part of the apply stage. This can be
 # disabled to get less noisy results for non-write related changes,
 # but should be enabled to get more comprehensive e2e numbers.

docs/apply-load-benchmark-token.cfg

@@ -8,10 +8,21 @@ APPLY_LOAD_MODE="benchmark"
 APPLY_LOAD_MODEL_TX="custom_token"
 
 # Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
+# "txset-validation-and-apply" also times tx-set construction from an
+# overfilled candidate backlog and validation before application.
 APPLY_LOAD_TIMING_PHASES = "apply"
 
+# Purposefully overfill the candidate backlog to simulate a busy production
+# validator's queue. In "txset-validation-and-apply", these multipliers default
+# to 2 and must be at least 2 (the classic multiplier is checked only when
+# APPLY_LOAD_CLASSIC_TXS_PER_LEDGER > 0). Raise them, e.g. to 4, to increase
+# the backlog without raising ledger limits. More candidates need more
+# accounts; increase GENESIS_TEST_ACCOUNT_COUNT if needed.
+# These flags scale candidate counts. Token transfers need two accounts
+# per candidate.
+SOROBAN_TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING = 2
+TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING = 2
+
 # Whether to time the write part of the apply stage. This can be
 # disabled to get less noisy results for non-write related changes,
 # but should be enabled to get more comprehensive e2e numbers.
@@ -61,4 +72,4 @@ NODE_SEED="SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2 self"
 
 [QUORUM_SET]
 THRESHOLD_PERCENT=100
-VALIDATORS=["$self"]
+VALIDATORS=["$self"]

docs/apply-load-for-meta.cfg

@@ -10,10 +10,22 @@
 APPLY_LOAD_MODE="ledger-limits"
 
 # Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
+# "txset-validation-and-apply" also times tx-set construction from an
+# overfilled candidate backlog and validation before application.
 APPLY_LOAD_TIMING_PHASES = "apply"
 
+# Purposefully overfill the candidate backlog to simulate a busy production
+# validator's queue. In "txset-validation-and-apply", these multipliers default
+# to 2 and must be at least 2 (the classic multiplier is checked only when
+# APPLY_LOAD_CLASSIC_TXS_PER_LEDGER > 0). Raise them, e.g. to 4, to increase
+# the backlog without raising ledger limits. More candidates need more
+# accounts; increase GENESIS_TEST_ACCOUNT_COUNT if needed.
+# The classic multiplier scales the candidate count in the tx-set timing path.
+# The Soroban multiplier scales the candidate resource budget in both timing
+# paths, so 2 means a budget of 2x ledger capacity.
+SOROBAN_TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING = 2
+TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING = 2
+
 # Custom meta path - if not set it will be written to a temp directory and
 # cleaned up after running the benchmark
 METADATA_OUTPUT_STREAM='meta.xdr'

docs/apply-load-ledger-limits.cfg

@@ -8,10 +8,22 @@
 APPLY_LOAD_MODE="ledger-limits"
 
 # Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
+# "txset-validation-and-apply" also times tx-set construction from an
+# overfilled candidate backlog and validation before application.
 APPLY_LOAD_TIMING_PHASES = "apply"
 
+# Purposefully overfill the candidate backlog to simulate a busy production
+# validator's queue. In "txset-validation-and-apply", these multipliers default
+# to 2 and must be at least 2 (the classic multiplier is checked only when
+# APPLY_LOAD_CLASSIC_TXS_PER_LEDGER > 0). Raise them, e.g. to 4, to increase
+# the backlog without raising ledger limits. More candidates need more
+# accounts; increase GENESIS_TEST_ACCOUNT_COUNT if needed.
+# The classic multiplier scales the candidate count in the tx-set timing path.
+# The Soroban multiplier scales the candidate resource budget in both timing
+# paths, so 2 means a budget of 2x ledger capacity.
+SOROBAN_TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING = 2
+TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING = 2
+
 # Medida metrics (histograms in particular) in apply path cause severe and
 # non-deterministic performance degradation. While this has to be addressed
 # eventually, it is useful to disable these when optimizing anything besides

docs/metrics.md

@@ -62,6 +62,7 @@ crypto.verify.miss                        | meter     | number of signature cach
 crypto.verify.total                       | meter     | sum of both hits and misses
 crypto.verify.tx-valid-hit                | meter     | signature cache hits that occurred while validating transactions (outside of background signature validation)
 crypto.verify.tx-valid-total              | meter     | sum of both hits and misses during transaction validation (outside of background signature validation)
+herder.txset.build                        | timer     | time spent building the tx set proposed at nomination from the transaction queues
 herder.txset.validate                     | timer     | time spent turning a received tx set into an applicable tx set and validating it on a validity-cache miss
 herder.pending[-soroban]-txs.age0         | counter   | number of gen0 pending transactions
 herder.pending[-soroban]-txs.age1         | counter   | number of gen1 pending transactions

docs/software/commands.md

@@ -23,8 +23,8 @@ Command options can only by placed after command.
     synthetic transactions. By default the benchmark is isolated to mostly just
     executing the transactions and thus it omits a lot of the supporting
     mechanisms (such as overlay, SCP, mempool etc). It may also measure tx-set
-    validation and consensus processing via `APPLY_LOAD_TIMING_PHASES` (see
-    below). This command will generate enough
+    construction, validation and consensus processing via
+    `APPLY_LOAD_TIMING_PHASES` (see below). This command will generate enough
     transactions to fill up a synthetic transaction queue (it's just a list of
     transactions with the same limits as the real queue), and then create a
     transaction set off of that to apply. This can also be used to record the
@@ -42,15 +42,22 @@ Command options can only by placed after command.
     - `"apply"`: the default apply-only benchmark. Its close helper still calls
       `checkValid`, but that happens before the recorded ledger-close timer and
       leaves the caches warm, as consensus validation would on a live node.
-    - `"txset-validation-and-apply"`: simulates a non-leader receiving the tx
-      set over the wire, validating it through local consensus (with the node
-      as its own single-validator quorum), and then applying it. It reports
-      validation, ledger close, and end-to-end time in addition to the
-      apply-only output. It does not simulate network transport, peer fetching,
-      or multi-node timing. Leader-side tx-set creation and signing happen
-      before the measured span. The signature verification cache is cleared
-      before validation, then retained so apply sees the warm cache produced by
-      validation.
+    - `"txset-validation-and-apply"`: times tx-set construction, decoding,
+      validation and application through local consensus with a
+      single-validator quorum. Construction
+      uses a synthetic candidate backlog of at least 2x ledger capacity to
+      exercise trimming, surge pricing and parallel partitioning as a busy
+      validator would. The queue size multipliers
+      (`SOROBAN_TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING` and
+      `TRANSACTION_QUEUE_SIZE_MULTIPLIER_FOR_TESTING`) control this backlog;
+      both default to 2 and must be at least 2 for enabled transaction types.
+      Raise them to increase the backlog without raising ledger limits. They
+      scale model and classic transaction counts; in `ledger-limits` mode the
+      Soroban multiplier scales the candidate resource budget in both timing
+      paths. Construction measures the builder itself,
+      the output reports construction, validation, and ledger close
+      timings. Network transport, peer fetching, tx queue submission, and
+      multi-node timing are not simulated.
     `"txset-validation-and-apply"` is not supported with
     `APPLY_LOAD_MODE="max-sac-tps"`; that search targets apply-only close time.
   * Load generation is configured in the Core config file. The relevant settings

src/herder/HerderImpl.cpp

@@ -47,6 +47,7 @@
 
 #include "util/GlobalChecks.h"
 #include <algorithm>
+#include <chrono>
 #include <ctime>
 #include <fmt/format.h>
 
@@ -82,6 +83,7 @@ HerderImpl::SCPMetrics::SCPMetrics(Application& app)
           {"scp", "envelope", "invalidsig"}, "envelope"))
     , mTriggerPrepareStartFallback(app.getMetrics().NewMeter(
           {"scp", "trigger", "prepare-start-fallback"}, "trigger"))
+    , mTxSetBuild(app.getMetrics().NewTimer({"herder", "txset", "build"}))
 {
 }
 
@@ -1664,6 +1666,7 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
     // Since we are not currently applying, it is safe to use read-only LCL, as
     // it's guaranteed to be up-to-date
     auto lcl = mLedgerManager.getLastClosedLedgerHeader();
+    auto const txSetBuildStart = std::chrono::steady_clock::now();
     PerPhaseTransactionList txPhases;
     txPhases.emplace_back(mTransactionQueue.getTransactions(lcl.header));
 
@@ -1752,6 +1755,12 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
     mTransactionQueue.ban(
         invalidTxPhases[static_cast<size_t>(TxSetPhase::CLASSIC)]);
 
+    // Stop before addTxSet below: its side effects can include SCP callbacks
+    // and even externalizing a ledger, which are not tx set building.
+    mSCPMetrics.mTxSetBuild.Update(
+        std::chrono::duration_cast<std::chrono::nanoseconds>(
+            std::chrono::steady_clock::now() - txSetBuildStart));
+
     auto txSetHash = proposedSet->getContentsHash();
 
     // Inform the item fetcher so queries from other peers about his txSet

src/herder/HerderImpl.h

@@ -362,6 +362,9 @@ class HerderImpl : public Herder
         // network-close-time anchor to the local prepare-start anchor.
         medida::Meter& mTriggerPrepareStartFallback;
 
+        // Time spent building the tx set proposed at nomination.
+        medida::Timer& mTxSetBuild;
+
         SCPMetrics(Application& app);
     };