diff --git a/src/ledger/InMemorySorobanState.cpp b/src/ledger/InMemorySorobanState.cpp index cf0e1e690e..630fc02cfc 100644 --- a/src/ledger/InMemorySorobanState.cpp +++ b/src/ledger/InMemorySorobanState.cpp @@ -488,7 +488,7 @@ InMemorySorobanState::updateState( std::vector const& liveEntries, std::vector const& deadEntries, LedgerHeader const& lh, std::optional const& sorobanConfig, - SorobanMetrics& metrics) + SorobanMetricsRegistry& metrics) { // After initialization, we must apply every ledger in order to the // in-memory state with no gaps. @@ -575,7 +575,7 @@ InMemorySorobanState::getSize() const } void -InMemorySorobanState::reportMetrics(SorobanMetrics& metrics) const +InMemorySorobanState::reportMetrics(SorobanMetricsRegistry& metrics) const { metrics.mContractCodeStateSize.set_count(mContractCodeStateSize); metrics.mContractDataStateSize.set_count(mContractDataStateSize); diff --git a/src/ledger/InMemorySorobanState.h b/src/ledger/InMemorySorobanState.h index 17a67a3454..cab5cb2317 100644 --- a/src/ledger/InMemorySorobanState.h +++ b/src/ledger/InMemorySorobanState.h @@ -20,7 +20,7 @@ namespace stellar class ApplyLedgerView; class InvariantManagerImpl; -class SorobanMetrics; +class SorobanMetricsRegistry; // TTLData stores both liveUntilLedgerSeq and lastModifiedLedgerSeq for TTL // entries. This allows us to construct a LedgerEntry for TTLs without having to @@ -396,7 +396,7 @@ class InMemorySorobanState // CONTRACT_CODE. void deleteContractCode(LedgerKey const& ledgerKey); - void reportMetrics(SorobanMetrics& metrics) const; + void reportMetrics(SorobanMetricsRegistry& metrics) const; public: InMemorySorobanState() = default; @@ -449,7 +449,7 @@ class InMemorySorobanState std::vector const& deadEntries, LedgerHeader const& lh, std::optional const& sorobanConfig, - SorobanMetrics& metrics); + SorobanMetricsRegistry& metrics); // Should only be called in manual ledger close paths. void manuallyAdvanceLedgerHeader(LedgerHeader const& lh); diff --git a/src/ledger/LedgerManager.h b/src/ledger/LedgerManager.h index a5127e8c61..d15a9a8b53 100644 --- a/src/ledger/LedgerManager.h +++ b/src/ledger/LedgerManager.h @@ -19,7 +19,8 @@ namespace stellar class LedgerCloseData; class Database; -class SorobanMetrics; +class SorobanMetricsRegistry; +struct SorobanApplyMetrics; class InMemorySorobanState; // This diagram provides a schematic of the flow of (logical) ledgers coming in @@ -373,11 +374,11 @@ class LedgerManager // upgradeApplied should be true if a protocol or network config setting // upgrade occurred during the ledger close. If inMemorySnapshotForInvariant // is not null, this will kick off a snapshot invariant check. - virtual void completeLedgerClose(uint32_t ledgerSeq, - bool calledViaExternalize, - LedgerCloseData const& ledgerData, - ImmutableLedgerDataPtr appliedLedgerState, - bool upgradeApplied) = 0; + virtual void completeLedgerClose( + uint32_t ledgerSeq, bool calledViaExternalize, + LedgerCloseData const& ledgerData, + ImmutableLedgerDataPtr appliedLedgerState, bool upgradeApplied, + std::vector&& sorobanApplyMetrics) = 0; virtual void assertSetupPhase() const = 0; #ifdef BUILD_TESTS @@ -398,7 +399,7 @@ class LedgerManager virtual void manuallyAdvanceLedgerHeader(LedgerHeader const& header) = 0; - virtual SorobanMetrics& getSorobanMetrics() = 0; + virtual SorobanMetricsRegistry& getSorobanMetrics() = 0; virtual ::rust::Box getModuleCache() = 0; virtual ~LedgerManager() diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index ac976caf79..d3583fa210 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -958,7 +958,7 @@ LedgerManagerImpl::getSorobanInMemoryStateSizeForTesting() } #endif -SorobanMetrics& +SorobanMetricsRegistry& LedgerManagerImpl::getSorobanMetrics() { return mApplyState.getMetrics().mSorobanMetrics; @@ -1198,7 +1198,8 @@ LedgerManagerImpl::ApplyState::maybeRebuildModuleCache( } void -LedgerManagerImpl::publishSorobanMetrics() +LedgerManagerImpl::publishSorobanMetrics( + std::vector& sorobanApplyMetricsPerThread) { if (!hasLastClosedSorobanNetworkConfig()) { @@ -1235,8 +1236,13 @@ LedgerManagerImpl::publishSorobanMetrics() conf.sorobanStateTargetSizeBytes()); m.mConfigFeeWrite1KB.set_count(conf.feeRent1KB()); - // then publish the actual ledger usage - m.publishAndResetLedgerWideMetrics(); + // then publish the actual ledger usage, merged into a single instance + auto& totalSorobanMetrics = sorobanApplyMetricsPerThread[0]; + for (size_t i = 1; i < sorobanApplyMetricsPerThread.size(); ++i) + { + totalSorobanMetrics.merge(std::move(sorobanApplyMetricsPerThread[i])); + } + m.recordApplyMetrics(totalSorobanMetrics); } // called by txherder @@ -1575,7 +1581,8 @@ void LedgerManagerImpl::completeLedgerClose( uint32_t ledgerSeq, bool calledViaExternalize, LedgerCloseData const& ledgerData, - ImmutableLedgerDataPtr appliedLedgerState, bool upgradeApplied) + ImmutableLedgerDataPtr appliedLedgerState, bool upgradeApplied, + std::vector&& sorobanApplyMetricsPerThread) { #ifdef BUILD_TESTS if (mCompleteLedgerCloseOverride) @@ -1598,7 +1605,7 @@ LedgerManagerImpl::completeLedgerClose( // We can publish Soroban metrics at any point after advancing the LCL // state. - publishSorobanMetrics(); + publishSorobanMetrics(sorobanApplyMetricsPerThread); // Maybe kick off publishing on complete checkpoint files auto& hm = mApp.getHistoryManager(); @@ -1813,6 +1820,15 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, #endif TransactionResultSet txResultSet; + // Soroban apply metrics are collected by different worker threads, so + // in order to avoid synchronization we give every thread its own metrics + // container. At least one thread (the apply thread) is going to exist, so + // this starts at 1, and is extended on-demand before spawning more workers. + // The metrics set in each thread's slot should be considered to be + // arbitrary by the consumers - the metrics must commute, and in the end + // we merge all the metric containers together without worrying about their + // order or origin. + std::vector sorobanApplyMetricsPerThread(1); #ifdef BUILD_TESTS if (mApp.getRunInOverlayOnlyMode()) { @@ -1854,8 +1870,9 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, // to use auto const mutableTxResults = processFeesSeqNums( *applicableTxSet, ltx, ledgerCloseMeta, ledgerData); - txResultSet = applyTransactions(*applicableTxSet, mutableTxResults, ltx, - ledgerCloseMeta); + txResultSet = + applyTransactions(*applicableTxSet, mutableTxResults, ltx, + ledgerCloseMeta, sorobanApplyMetricsPerThread); } auto ledgerSeq = ltx.loadHeader().current().ledgerSeq; @@ -2081,15 +2098,19 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, if (threadIsMain()) { completeLedgerClose(ledgerSeq, calledViaExternalize, ledgerData, - std::move(appliedLedgerState), upgradeApplied); + std::move(appliedLedgerState), upgradeApplied, + std::move(sorobanApplyMetricsPerThread)); } else { auto cb = [this, ledgerSeq, calledViaExternalize, ledgerData, appliedLedgerState = std::move(appliedLedgerState), - upgradeApplied]() mutable { + upgradeApplied, + sorobanApplyMetricsPerThread = + std::move(sorobanApplyMetricsPerThread)]() mutable { completeLedgerClose(ledgerSeq, calledViaExternalize, ledgerData, - std::move(appliedLedgerState), upgradeApplied); + std::move(appliedLedgerState), upgradeApplied, + std::move(sorobanApplyMetricsPerThread)); }; mApp.postOnMainThread(std::move(cb), "completeLedgerClose"); } @@ -2604,26 +2625,18 @@ LedgerManagerImpl::applyThread( AppConnector& app, std::unique_ptr threadState, Cluster const& cluster, Config const& config, ParallelLedgerInfo ledgerInfo, - Hash sorobanBasePrngSeed) + Hash sorobanBasePrngSeed, SorobanApplyMetrics& sorobanMetrics) { for (auto const& txBundle : cluster) { - // Apply timer; samples go into the thread's metrics batch and are - // published at ledger close. - std::optional txTime; - if (!mApp.getConfig().DISABLE_SOROBAN_METRICS_FOR_TESTING) - { - txTime.emplace(getSorobanMetrics(), - &SorobanMetrics::ApplyMetricsBatch::mTxApplyNsecs); - } - + auto applyStart = std::chrono::steady_clock::now(); Hash txSubSeed = subSha256(sorobanBasePrngSeed, txBundle.getTxNum()); threadState->flushRoTTLBumpsInTxWriteFootprint(txBundle); auto res = txBundle.getTx()->parallelApply( app, *threadState, config, ledgerInfo, txBundle.getResPayload(), - getSorobanMetrics(), txSubSeed, txBundle.getEffects()); + sorobanMetrics, txSubSeed, txBundle.getEffects()); if (res) { @@ -2633,6 +2646,10 @@ LedgerManagerImpl::applyThread( { releaseAssert(!txBundle.getResPayload().isSuccess()); } + sorobanMetrics.mTxApplyNsecs.push_back( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - applyStart) + .count()); } threadState->flushRemainingRoTTLBumps(); @@ -2652,11 +2669,18 @@ LedgerManagerImpl::applySorobanStageClustersInParallel( AppConnector& app, ApplyStage const& stage, GlobalParallelApplyLedgerState const& globalState, Hash const& sorobanBasePrngSeed, Config const& config, - ParallelLedgerInfo const& ledgerInfo) + ParallelLedgerInfo const& ledgerInfo, + std::vector& sorobanApplyMetricsPerThread) { ZoneScoped; DeactivateScopeGuard globalStateDeactivateGuard(globalState); + // Stages may contain a different number of clusters, so we ensure that + // there is a corresponding metrics entry for each cluster. + if (sorobanApplyMetricsPerThread.size() < stage.numClusters()) + { + sorobanApplyMetricsPerThread.resize(stage.numClusters()); + } std::vector< std::function()>> @@ -2665,13 +2689,16 @@ LedgerManagerImpl::applySorobanStageClustersInParallel( for (size_t i = 0; i < stage.numClusters(); ++i) { tasks.emplace_back([this, &app, &globalState, &stage, i, &config, - &ledgerInfo, &sorobanBasePrngSeed]() { + &ledgerInfo, &sorobanBasePrngSeed, + &sorobanApplyMetricsPerThread]() { auto const& cluster = stage.getCluster(i); auto threadStatePtr = std::make_unique( app, globalState, cluster, i); + // Give every thread its own metrics entry to write to. return applyThread(app, std::move(threadStatePtr), cluster, config, - ledgerInfo, sorobanBasePrngSeed); + ledgerInfo, sorobanBasePrngSeed, + sorobanApplyMetricsPerThread[i]); }); } @@ -2728,14 +2755,16 @@ void LedgerManagerImpl::applySorobanStage( AppConnector& app, LedgerHeader const& header, GlobalParallelApplyLedgerState& globalParState, ApplyStage const& stage, - Hash const& sorobanBasePrngSeed) + Hash const& sorobanBasePrngSeed, + std::vector& sorobanApplyMetricsPerThread) { ZoneScoped; auto const& config = app.getConfig(); auto ledgerInfo = getParallelLedgerInfo(app, header); auto threadStates = applySorobanStageClustersInParallel( - app, stage, globalParState, sorobanBasePrngSeed, config, ledgerInfo); + app, stage, globalParState, sorobanBasePrngSeed, config, ledgerInfo, + sorobanApplyMetricsPerThread); if (config.invariantsEnabled()) { @@ -2754,22 +2783,24 @@ LedgerManagerImpl::applySorobanStage( } void -LedgerManagerImpl::applySorobanStages(AppConnector& app, AbstractLedgerTxn& ltx, - std::vector const& stages, - SorobanNetworkConfig const& sorobanConfig, - Hash const& sorobanBasePrngSeed) +LedgerManagerImpl::applySorobanStages( + AppConnector& app, AbstractLedgerTxn& ltx, + std::vector const& stages, + SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, + std::vector& sorobanApplyMetricsPerThread) { ZoneScoped; GlobalParallelApplyLedgerState globalParState( app, mApplyState.copyApplyLedgerView(), ltx, stages, - mApplyState.getInMemorySorobanState(), sorobanConfig); + mApplyState.getInMemorySorobanState(), sorobanConfig, + sorobanApplyMetricsPerThread[0]); // LedgerTxn is not passed into applySorobanStage, so there's no risk // of the header being updated while we apply the stages. auto const& header = ltx.loadHeader().current(); for (auto const& stage : stages) { applySorobanStage(app, header, globalParState, stage, - sorobanBasePrngSeed); + sorobanBasePrngSeed, sorobanApplyMetricsPerThread); } globalParState.commitChangesToLedgerTxn(ltx); } @@ -2837,7 +2868,8 @@ LedgerManagerImpl::applyTransactions( ApplicableTxSetFrame const& txSet, std::vector const& mutableTxResults, AbstractLedgerTxn& ltx, - std::unique_ptr const& ledgerCloseMeta) + std::unique_ptr const& ledgerCloseMeta, + std::vector& sorobanApplyMetricsPerThread) { ZoneNamedN(txsZone, "applyTransactions", true); size_t numTxs = txSet.sizeTxTotal(); @@ -2896,7 +2928,8 @@ LedgerManagerImpl::applyTransactions( releaseAssert(sorobanConfig.has_value()); applyParallelPhase(phase, applyStages, mutableTxResults, index, ltx, enableTxMeta, *sorobanConfig, - sorobanBasePrngSeed); + sorobanBasePrngSeed, + sorobanApplyMetricsPerThread); } catch (std::exception const& e) { @@ -2914,7 +2947,7 @@ LedgerManagerImpl::applyTransactions( applySequentialPhase(phase, mutableTxResults, index, ltx, enableTxMeta, sorobanConfig, sorobanBasePrngSeed, ledgerCloseMeta, - txResultSet); + txResultSet, sorobanApplyMetricsPerThread[0]); } } @@ -2942,7 +2975,8 @@ LedgerManagerImpl::applyParallelPhase( TxSetPhaseFrame const& phase, std::vector& applyStages, std::vector const& mutableTxResults, uint32_t& index, stellar::AbstractLedgerTxn& ltx, bool enableTxMeta, - SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed) + SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, + std::vector& sorobanApplyMetricsPerThread) { ZoneScoped; @@ -2991,7 +3025,7 @@ LedgerManagerImpl::applyParallelPhase( } applySorobanStages(mApp.getAppConnector(), ltx, applyStages, sorobanConfig, - sorobanBasePrngSeed); + sorobanBasePrngSeed, sorobanApplyMetricsPerThread); // meta will be processed in processPostTxSetApply } @@ -3004,7 +3038,7 @@ LedgerManagerImpl::applySequentialPhase( std::optional const& sorobanConfig, Hash const& sorobanBasePrngSeed, std::unique_ptr const& ledgerCloseMeta, - TransactionResultSet& txResultSet) + TransactionResultSet& txResultSet, SorobanApplyMetrics& sorobanMetrics) { for (auto const& tx : phase) { @@ -3040,7 +3074,7 @@ LedgerManagerImpl::applySequentialPhase( } tx->apply(mApp.getAppConnector(), ltx, tm, mutableTxResult, - sorobanConfig, subSeed); + sorobanConfig, subSeed, sorobanMetrics); tx->processPostApply(mApp.getAppConnector(), ltx, tm, mutableTxResult); tm.maybeSetRefundableFeeMeta(mutableTxResult.getRefundableFeeTracker()); diff --git a/src/ledger/LedgerManagerImpl.h b/src/ledger/LedgerManagerImpl.h index 197c81f4a6..9a5b535935 100644 --- a/src/ledger/LedgerManagerImpl.h +++ b/src/ledger/LedgerManagerImpl.h @@ -68,7 +68,7 @@ class LedgerManagerImpl : public LedgerManager private: struct LedgerApplyMetrics { - SorobanMetrics mSorobanMetrics; + SorobanMetricsRegistry mSorobanMetrics; medida::Timer& mTransactionApply; medida::Timer& mTotalTxApply; medida::Histogram& mTransactionCount; @@ -358,14 +358,16 @@ class LedgerManagerImpl : public LedgerManager ApplicableTxSetFrame const& txSet, std::vector const& mutableTxResults, AbstractLedgerTxn& ltx, - std::unique_ptr const& ledgerCloseMeta); + std::unique_ptr const& ledgerCloseMeta, + std::vector& sorobanApplyMetricsPerThread); void applyParallelPhase( TxSetPhaseFrame const& phase, std::vector& applyStages, std::vector const& mutableTxResults, uint32_t& index, AbstractLedgerTxn& ltx, bool enableTxMeta, SorobanNetworkConfig const& sorobanConfig, - Hash const& sorobanBasePrngSeed); + Hash const& sorobanBasePrngSeed, + std::vector& sorobanApplyMetricsPerThread); void applySequentialPhase( TxSetPhaseFrame const& phase, @@ -374,7 +376,7 @@ class LedgerManagerImpl : public LedgerManager std::optional const& sorobanConfig, Hash const& sorobanBasePrngSeed, std::unique_ptr const& ledgerCloseMeta, - TransactionResultSet& txResultSet); + TransactionResultSet& txResultSet, SorobanApplyMetrics& sorobanMetrics); void processPostTxSetApply( std::vector const& phases, @@ -386,29 +388,34 @@ class LedgerManagerImpl : public LedgerManager applyThread(AppConnector& app, std::unique_ptr threadState, Cluster const& cluster, Config const& config, - ParallelLedgerInfo ledgerInfo, Hash sorobanBasePrngSeed); + ParallelLedgerInfo ledgerInfo, Hash sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics); std::vector> applySorobanStageClustersInParallel( AppConnector& app, ApplyStage const& stage, GlobalParallelApplyLedgerState const& globalState, Hash const& sorobanBasePrngSeed, Config const& config, - ParallelLedgerInfo const& ledgerInfo); + ParallelLedgerInfo const& ledgerInfo, + std::vector& sorobanApplyMetricsPerThread); void checkAllTxBundleInvariants(AppConnector& app, ApplyStage const& stage, Config const& config, ParallelLedgerInfo const& ledgerInfo, LedgerHeader const& header); - void applySorobanStage(AppConnector& app, LedgerHeader const& header, - GlobalParallelApplyLedgerState& globalParState, - ApplyStage const& stage, - Hash const& sorobanBasePrngSeed); + void applySorobanStage( + AppConnector& app, LedgerHeader const& header, + GlobalParallelApplyLedgerState& globalParState, ApplyStage const& stage, + Hash const& sorobanBasePrngSeed, + std::vector& sorobanApplyMetricsPerThread); - void applySorobanStages(AppConnector& app, AbstractLedgerTxn& ltx, - std::vector const& stages, - SorobanNetworkConfig const& sorobanConfig, - Hash const& sorobanBasePrngSeed); + void applySorobanStages( + AppConnector& app, AbstractLedgerTxn& ltx, + std::vector const& stages, + SorobanNetworkConfig const& sorobanConfig, + Hash const& sorobanBasePrngSeed, + std::vector& sorobanApplyMetricsPerThread); // initialLedgerVers must be the ledger version at the start of the ledger. // On the ledger in which a protocol upgrade from vN to vN + 1 occurs, @@ -478,9 +485,11 @@ class LedgerManagerImpl : public LedgerManager void emitNextMeta(); - // Publishes soroban metrics, including select network config limits as well - // as the actual ledger usage. - void publishSorobanMetrics(); + // Publishes soroban metrics, including select network config limits as + // well as the actual ledger usage accumulated in `sorobanApplyMetrics` + // (which is consumed by this call). + void publishSorobanMetrics( + std::vector& sorobanApplyMetricsPerThread); // Update cached last closed ledger state values managed by this class. void @@ -600,7 +609,9 @@ class LedgerManagerImpl : public LedgerManager void completeLedgerClose(uint32_t ledgerSeq, bool calledViaExternalize, LedgerCloseData const& ledgerData, ImmutableLedgerDataPtr appliedLedgerState, - bool upgradeApplied) override; + bool upgradeApplied, + std::vector&& + sorobanApplyMetricsPerThread) override; void notifyLedgerCloseComplete(uint32_t lcl, bool calledViaExternalize, LedgerCloseData const& ledgerData, bool upgradeApplied); @@ -612,7 +623,7 @@ class LedgerManagerImpl : public LedgerManager void setupLedgerCloseMetaStream(); void maybeResetLedgerCloseMetaDebugStream(uint32_t ledgerSeq); - SorobanMetrics& getSorobanMetrics() override; + SorobanMetricsRegistry& getSorobanMetrics() override; ImmutableLedgerView copyImmutableLedgerView() const override; ApplyLedgerView copyApplyLedgerView() const override; void maybeUpdateImmutableLedgerView( diff --git a/src/ledger/SorobanMetrics.cpp b/src/ledger/SorobanMetrics.cpp index 5db27cd2ec..515d081ec1 100644 --- a/src/ledger/SorobanMetrics.cpp +++ b/src/ledger/SorobanMetrics.cpp @@ -5,11 +5,77 @@ #include #include #include -#include namespace stellar { -SorobanMetrics::SorobanMetrics(MetricsRegistry& metrics) +void +SorobanApplyMetrics::merge(SorobanApplyMetrics&& other) +{ + auto drain = [](std::vector& dst, std::vector&& src) { + if (dst.empty()) + { + dst = std::move(src); + } + else + { + dst.insert(dst.end(), src.begin(), src.end()); + } + }; + + mHostFnOpReadEntry += other.mHostFnOpReadEntry; + mHostFnOpWriteEntry += other.mHostFnOpWriteEntry; + mHostFnOpReadKeyByte += other.mHostFnOpReadKeyByte; + mHostFnOpWriteKeyByte += other.mHostFnOpWriteKeyByte; + mHostFnOpReadLedgerByte += other.mHostFnOpReadLedgerByte; + mHostFnOpReadDataByte += other.mHostFnOpReadDataByte; + mHostFnOpReadCodeByte += other.mHostFnOpReadCodeByte; + mHostFnOpWriteLedgerByte += other.mHostFnOpWriteLedgerByte; + mHostFnOpWriteDataByte += other.mHostFnOpWriteDataByte; + mHostFnOpWriteCodeByte += other.mHostFnOpWriteCodeByte; + mHostFnOpEmitEvent += other.mHostFnOpEmitEvent; + mHostFnOpEmitEventByte += other.mHostFnOpEmitEventByte; + mHostFnOpCpuInsn += other.mHostFnOpCpuInsn; + mHostFnOpMemByte += other.mHostFnOpMemByte; + mHostFnOpCpuInsnExclVm += other.mHostFnOpCpuInsnExclVm; + mHostFnOpMaxRwKeyByte += other.mHostFnOpMaxRwKeyByte; + mHostFnOpMaxRwDataByte += other.mHostFnOpMaxRwDataByte; + mHostFnOpMaxRwCodeByte += other.mHostFnOpMaxRwCodeByte; + mHostFnOpMaxEmitEventByte += other.mHostFnOpMaxEmitEventByte; + mHostFnOpSuccess += other.mHostFnOpSuccess; + mHostFnOpFailure += other.mHostFnOpFailure; + mExtFpTtlOpReadLedgerByte += other.mExtFpTtlOpReadLedgerByte; + mRestoreFpOpReadLedgerByte += other.mRestoreFpOpReadLedgerByte; + mRestoreFpOpWriteLedgerByte += other.mRestoreFpOpWriteLedgerByte; + + mLedgerTxCount += other.mLedgerTxCount; + mLedgerCpuInsn += other.mLedgerCpuInsn; + mLedgerTxsSizeByte += other.mLedgerTxsSizeByte; + mLedgerReadEntry += other.mLedgerReadEntry; + mLedgerReadByte += other.mLedgerReadByte; + mLedgerWriteEntry += other.mLedgerWriteEntry; + mLedgerWriteByte += other.mLedgerWriteByte; + mLedgerInsnsCount += other.mLedgerInsnsCount; + mLedgerInsnsExclVmCount += other.mLedgerInsnsExclVmCount; + mLedgerHostFnExecTimeNsecs += other.mLedgerHostFnExecTimeNsecs; + + drain(mHostFnOpInvokeTimeNsecs, std::move(other.mHostFnOpInvokeTimeNsecs)); + drain(mHostFnOpInvokeTimeNsecsExclVm, + std::move(other.mHostFnOpInvokeTimeNsecsExclVm)); + drain(mHostFnOpInvokeTimeFsecsCpuInsnRatio, + std::move(other.mHostFnOpInvokeTimeFsecsCpuInsnRatio)); + drain(mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm, + std::move(other.mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm)); + drain(mHostFnOpDeclaredInsnsUsageRatio, + std::move(other.mHostFnOpDeclaredInsnsUsageRatio)); + drain(mHostFnOpExecNsecs, std::move(other.mHostFnOpExecNsecs)); + drain(mExtFpTtlOpExecNsecs, std::move(other.mExtFpTtlOpExecNsecs)); + drain(mRestoreFpOpExecNsecs, std::move(other.mRestoreFpOpExecNsecs)); + drain(mTxSizeByte, std::move(other.mTxSizeByte)); + drain(mTxApplyNsecs, std::move(other.mTxApplyNsecs)); + drain(mOpApplyNsecs, std::move(other.mOpApplyNsecs)); +} + +SorobanMetricsRegistry::SorobanMetricsRegistry(MetricsRegistry& metrics) : /* ledger-wide metrics */ mLedgerTxCount(metrics.NewHistogram({"soroban", "ledger", "tx-count"})) , mLedgerCpuInsn(metrics.NewHistogram({"soroban", "ledger", "cpu-insn"})) @@ -163,216 +229,70 @@ SorobanMetrics::SorobanMetrics(MetricsRegistry& metrics) } void -SorobanMetrics::accumulateModelledCpuInsns(uint64_t insnsCount, - uint64_t insnsExclVmCount, - uint64_t hostFnExecTimeNsecs) -{ - mLedgerInsnsCount += insnsCount; - mLedgerInsnsExclVmCount += insnsExclVmCount; - mLedgerHostFnExecTimeNsecs += hostFnExecTimeNsecs; -} - -void -SorobanMetrics::accumulateLedgerTxCount(uint64_t txCount) -{ - mCounterLedgerTxCount += txCount; -} -void -SorobanMetrics::accumulateLedgerCpuInsn(uint64_t cpuInsn) -{ - mCounterLedgerCpuInsn += cpuInsn; -} -void -SorobanMetrics::accumulateLedgerTxsSizeByte(uint64_t txsSizeByte) -{ - mCounterLedgerTxsSizeByte += txsSizeByte; -} -void -SorobanMetrics::accumulateLedgerReadEntry(uint64_t readEntry) -{ - mCounterLedgerReadEntry += readEntry; -} -void -SorobanMetrics::accumulateLedgerReadByte(uint64_t readByte) -{ - mCounterLedgerReadByte += readByte; -} -void -SorobanMetrics::accumulateLedgerWriteEntry(uint64_t writeEntry) -{ - mCounterLedgerWriteEntry += writeEntry; -} -void -SorobanMetrics::accumulateLedgerWriteByte(uint64_t writeByte) -{ - mCounterLedgerWriteByte += writeByte; -} - -SorobanMetrics::ApplyMetricsBatch& -SorobanMetrics::getApplyThreadBatch() -{ - std::lock_guard lock(mApplyBatchesMutex); - auto [it, inserted] = mApplyBatches.try_emplace(std::this_thread::get_id()); - if (inserted) - { - it->second = std::make_unique(); - } - return *it->second; -} - -void -SorobanMetrics::flushApplyMetricsBatches() +SorobanMetricsRegistry::recordApplyMetrics(SorobanApplyMetrics const& metrics) { - std::vector> batches; - { - std::lock_guard lock(mApplyBatchesMutex); - if (mApplyBatches.empty()) - { - return; - } - batches.reserve(mApplyBatches.size()); - for (auto& [_, v] : mApplyBatches) - { - batches.emplace_back(std::move(v)); - } - mApplyBatches.clear(); - } - - ApplyMetricsBatch total; - auto take = [](uint64_t& v) { - auto res = v; - v = 0; - return res; - }; - auto drain = [](std::vector& dst, std::vector& src) { - dst.insert(dst.end(), src.begin(), src.end()); - src.clear(); - }; - for (auto const& b : batches) - { - total.mHostFnOpReadEntry += take(b->mHostFnOpReadEntry); - total.mHostFnOpWriteEntry += take(b->mHostFnOpWriteEntry); - total.mHostFnOpReadKeyByte += take(b->mHostFnOpReadKeyByte); - total.mHostFnOpWriteKeyByte += take(b->mHostFnOpWriteKeyByte); - total.mHostFnOpReadLedgerByte += take(b->mHostFnOpReadLedgerByte); - total.mHostFnOpReadDataByte += take(b->mHostFnOpReadDataByte); - total.mHostFnOpReadCodeByte += take(b->mHostFnOpReadCodeByte); - total.mHostFnOpWriteLedgerByte += take(b->mHostFnOpWriteLedgerByte); - total.mHostFnOpWriteDataByte += take(b->mHostFnOpWriteDataByte); - total.mHostFnOpWriteCodeByte += take(b->mHostFnOpWriteCodeByte); - total.mHostFnOpEmitEvent += take(b->mHostFnOpEmitEvent); - total.mHostFnOpEmitEventByte += take(b->mHostFnOpEmitEventByte); - total.mHostFnOpCpuInsn += take(b->mHostFnOpCpuInsn); - total.mHostFnOpMemByte += take(b->mHostFnOpMemByte); - total.mHostFnOpCpuInsnExclVm += take(b->mHostFnOpCpuInsnExclVm); - total.mHostFnOpMaxRwKeyByte += take(b->mHostFnOpMaxRwKeyByte); - total.mHostFnOpMaxRwDataByte += take(b->mHostFnOpMaxRwDataByte); - total.mHostFnOpMaxRwCodeByte += take(b->mHostFnOpMaxRwCodeByte); - total.mHostFnOpMaxEmitEventByte += take(b->mHostFnOpMaxEmitEventByte); - total.mHostFnOpSuccess += take(b->mHostFnOpSuccess); - total.mHostFnOpFailure += take(b->mHostFnOpFailure); - total.mExtFpTtlOpReadLedgerByte += take(b->mExtFpTtlOpReadLedgerByte); - total.mRestoreFpOpReadLedgerByte += take(b->mRestoreFpOpReadLedgerByte); - total.mRestoreFpOpWriteLedgerByte += - take(b->mRestoreFpOpWriteLedgerByte); - - drain(total.mHostFnOpInvokeTimeNsecs, b->mHostFnOpInvokeTimeNsecs); - drain(total.mHostFnOpInvokeTimeNsecsExclVm, - b->mHostFnOpInvokeTimeNsecsExclVm); - drain(total.mHostFnOpInvokeTimeFsecsCpuInsnRatio, - b->mHostFnOpInvokeTimeFsecsCpuInsnRatio); - drain(total.mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm, - b->mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm); - drain(total.mHostFnOpDeclaredInsnsUsageRatio, - b->mHostFnOpDeclaredInsnsUsageRatio); - drain(total.mHostFnOpExecNsecs, b->mHostFnOpExecNsecs); - drain(total.mExtFpTtlOpExecNsecs, b->mExtFpTtlOpExecNsecs); - drain(total.mRestoreFpOpExecNsecs, b->mRestoreFpOpExecNsecs); - drain(total.mTxSizeByte, b->mTxSizeByte); - drain(total.mTxApplyNsecs, b->mTxApplyNsecs); - drain(total.mOpApplyNsecs, b->mOpApplyNsecs); - } - // Publish into the underlying medida metrics, one bulk call per metric. // Zero meter increments are skipped (a Mark(0) does not change any - // observable value); empty sample batches are no-ops in UpdateMany. + // observable value); empty sample metric vectors are no-ops in UpdateMany. auto markIf = [](medida::Meter& meter, uint64_t value) { if (value != 0) { meter.Mark(value); } }; - markIf(mHostFnOpReadEntry, total.mHostFnOpReadEntry); - markIf(mHostFnOpWriteEntry, total.mHostFnOpWriteEntry); - markIf(mHostFnOpReadKeyByte, total.mHostFnOpReadKeyByte); - markIf(mHostFnOpWriteKeyByte, total.mHostFnOpWriteKeyByte); - markIf(mHostFnOpReadLedgerByte, total.mHostFnOpReadLedgerByte); - markIf(mHostFnOpReadDataByte, total.mHostFnOpReadDataByte); - markIf(mHostFnOpReadCodeByte, total.mHostFnOpReadCodeByte); - markIf(mHostFnOpWriteLedgerByte, total.mHostFnOpWriteLedgerByte); - markIf(mHostFnOpWriteDataByte, total.mHostFnOpWriteDataByte); - markIf(mHostFnOpWriteCodeByte, total.mHostFnOpWriteCodeByte); - markIf(mHostFnOpEmitEvent, total.mHostFnOpEmitEvent); - markIf(mHostFnOpEmitEventByte, total.mHostFnOpEmitEventByte); - markIf(mHostFnOpCpuInsn, total.mHostFnOpCpuInsn); - markIf(mHostFnOpMemByte, total.mHostFnOpMemByte); - markIf(mHostFnOpCpuInsnExclVm, total.mHostFnOpCpuInsnExclVm); - markIf(mHostFnOpMaxRwKeyByte, total.mHostFnOpMaxRwKeyByte); - markIf(mHostFnOpMaxRwDataByte, total.mHostFnOpMaxRwDataByte); - markIf(mHostFnOpMaxRwCodeByte, total.mHostFnOpMaxRwCodeByte); - markIf(mHostFnOpMaxEmitEventByte, total.mHostFnOpMaxEmitEventByte); - markIf(mHostFnOpSuccess, total.mHostFnOpSuccess); - markIf(mHostFnOpFailure, total.mHostFnOpFailure); - markIf(mExtFpTtlOpReadLedgerByte, total.mExtFpTtlOpReadLedgerByte); - markIf(mRestoreFpOpReadLedgerByte, total.mRestoreFpOpReadLedgerByte); - markIf(mRestoreFpOpWriteLedgerByte, total.mRestoreFpOpWriteLedgerByte); + markIf(mHostFnOpReadEntry, metrics.mHostFnOpReadEntry); + markIf(mHostFnOpWriteEntry, metrics.mHostFnOpWriteEntry); + markIf(mHostFnOpReadKeyByte, metrics.mHostFnOpReadKeyByte); + markIf(mHostFnOpWriteKeyByte, metrics.mHostFnOpWriteKeyByte); + markIf(mHostFnOpReadLedgerByte, metrics.mHostFnOpReadLedgerByte); + markIf(mHostFnOpReadDataByte, metrics.mHostFnOpReadDataByte); + markIf(mHostFnOpReadCodeByte, metrics.mHostFnOpReadCodeByte); + markIf(mHostFnOpWriteLedgerByte, metrics.mHostFnOpWriteLedgerByte); + markIf(mHostFnOpWriteDataByte, metrics.mHostFnOpWriteDataByte); + markIf(mHostFnOpWriteCodeByte, metrics.mHostFnOpWriteCodeByte); + markIf(mHostFnOpEmitEvent, metrics.mHostFnOpEmitEvent); + markIf(mHostFnOpEmitEventByte, metrics.mHostFnOpEmitEventByte); + markIf(mHostFnOpCpuInsn, metrics.mHostFnOpCpuInsn); + markIf(mHostFnOpMemByte, metrics.mHostFnOpMemByte); + markIf(mHostFnOpCpuInsnExclVm, metrics.mHostFnOpCpuInsnExclVm); + markIf(mHostFnOpMaxRwKeyByte, metrics.mHostFnOpMaxRwKeyByte); + markIf(mHostFnOpMaxRwDataByte, metrics.mHostFnOpMaxRwDataByte); + markIf(mHostFnOpMaxRwCodeByte, metrics.mHostFnOpMaxRwCodeByte); + markIf(mHostFnOpMaxEmitEventByte, metrics.mHostFnOpMaxEmitEventByte); + markIf(mHostFnOpSuccess, metrics.mHostFnOpSuccess); + markIf(mHostFnOpFailure, metrics.mHostFnOpFailure); + markIf(mExtFpTtlOpReadLedgerByte, metrics.mExtFpTtlOpReadLedgerByte); + markIf(mRestoreFpOpReadLedgerByte, metrics.mRestoreFpOpReadLedgerByte); + markIf(mRestoreFpOpWriteLedgerByte, metrics.mRestoreFpOpWriteLedgerByte); - mHostFnOpInvokeTimeNsecs.UpdateMany(total.mHostFnOpInvokeTimeNsecs); + mHostFnOpInvokeTimeNsecs.UpdateMany(metrics.mHostFnOpInvokeTimeNsecs); mHostFnOpInvokeTimeNsecsExclVm.UpdateMany( - total.mHostFnOpInvokeTimeNsecsExclVm); + metrics.mHostFnOpInvokeTimeNsecsExclVm); mHostFnOpInvokeTimeFsecsCpuInsnRatio.UpdateMany( - total.mHostFnOpInvokeTimeFsecsCpuInsnRatio); + metrics.mHostFnOpInvokeTimeFsecsCpuInsnRatio); mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm.UpdateMany( - total.mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm); + metrics.mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm); mHostFnOpDeclaredInsnsUsageRatio.UpdateMany( - total.mHostFnOpDeclaredInsnsUsageRatio); - mHostFnOpExec.UpdateMany(total.mHostFnOpExecNsecs); - mExtFpTtlOpExec.UpdateMany(total.mExtFpTtlOpExecNsecs); - mRestoreFpOpExec.UpdateMany(total.mRestoreFpOpExecNsecs); - mTxSizeByte.UpdateMany(total.mTxSizeByte); - mTransactionApply.UpdateMany(total.mTxApplyNsecs); - mOperationApply.UpdateMany(total.mOpApplyNsecs); -} + metrics.mHostFnOpDeclaredInsnsUsageRatio); + mHostFnOpExec.UpdateMany(metrics.mHostFnOpExecNsecs); + mExtFpTtlOpExec.UpdateMany(metrics.mExtFpTtlOpExecNsecs); + mRestoreFpOpExec.UpdateMany(metrics.mRestoreFpOpExecNsecs); + mTxSizeByte.UpdateMany(metrics.mTxSizeByte); + mTransactionApply.UpdateMany(metrics.mTxApplyNsecs); + mOperationApply.UpdateMany(metrics.mOpApplyNsecs); -void -SorobanMetrics::publishAndResetLedgerWideMetrics() -{ - flushApplyMetricsBatches(); - - mLedgerTxCount.Update(mCounterLedgerTxCount); - mLedgerCpuInsn.Update(mCounterLedgerCpuInsn); - mLedgerTxsSizeByte.Update(mCounterLedgerTxsSizeByte); - mLedgerReadEntry.Update(mCounterLedgerReadEntry); - mLedgerReadLedgerByte.Update(mCounterLedgerReadByte); - mLedgerWriteEntry.Update(mCounterLedgerWriteEntry); - mLedgerWriteLedgerByte.Update(mCounterLedgerWriteByte); + mLedgerTxCount.Update(metrics.mLedgerTxCount); + mLedgerCpuInsn.Update(metrics.mLedgerCpuInsn); + mLedgerTxsSizeByte.Update(metrics.mLedgerTxsSizeByte); + mLedgerReadEntry.Update(metrics.mLedgerReadEntry); + mLedgerReadLedgerByte.Update(metrics.mLedgerReadByte); + mLedgerWriteEntry.Update(metrics.mLedgerWriteEntry); + mLedgerWriteLedgerByte.Update(metrics.mLedgerWriteByte); mLedgerHostFnCpuInsnsRatio.Update( - mLedgerHostFnExecTimeNsecs * 1000000 / - std::max(mLedgerInsnsCount.load(), uint64_t(1))); - + metrics.mLedgerHostFnExecTimeNsecs * 1000000 / + std::max(metrics.mLedgerInsnsCount, uint64_t(1))); mLedgerHostFnCpuInsnsRatioExclVm.Update( - mLedgerHostFnExecTimeNsecs * 1000000 / - std::max(mLedgerInsnsExclVmCount.load(), uint64_t(1))); - - mCounterLedgerTxCount = 0; - mCounterLedgerCpuInsn = 0; - mCounterLedgerTxsSizeByte = 0; - mCounterLedgerReadEntry = 0; - mCounterLedgerReadByte = 0; - mCounterLedgerWriteEntry = 0; - mCounterLedgerWriteByte = 0; - mLedgerHostFnExecTimeNsecs = 0; - mLedgerInsnsCount = 0; - mLedgerInsnsExclVmCount = 0; + metrics.mLedgerHostFnExecTimeNsecs * 1000000 / + std::max(metrics.mLedgerInsnsExclVmCount, uint64_t(1))); } } diff --git a/src/ledger/SorobanMetrics.h b/src/ledger/SorobanMetrics.h index 8c0499564c..5936f22303 100644 --- a/src/ledger/SorobanMetrics.h +++ b/src/ledger/SorobanMetrics.h @@ -7,13 +7,8 @@ // This class exists to cache soroban metrics: resource usage and network config // limits. It also performs aggregation of ledger-wide resource usage across // different operations. -#include #include #include -#include -#include -#include -#include #include namespace medida @@ -28,85 +23,82 @@ namespace stellar { class MetricsRegistry; -class SorobanMetrics +// Collection of counters and sample metric streams for Soroban. +// +// For the sake of optimization, the counters here are simple accumulators and +// vectors, which then can be merged and flushed into Medida for the actual +// publish. +// +// Note: this struct is _not_ threadsafe, and meant to be accumulated +// per-thread. +struct SorobanApplyMetrics { - public: - // Accumulates apply-path metric updates from a single thread. Hot apply - // code records into its own thread's batch, and the batches are drained - // into the underlying process-wide medida metrics once per ledger on the - // main thread via publishAndResetLedgerWideMetrics(). This both avoids fine - // grained locking and leverages batch update interfaces on medida. - // - // Note: this class is _not_ threadsafe. An instance is owned by each thread - // ( in a map keyed by thread ID) and each thread should only access its - // own. - struct ApplyMetricsBatch - { - - // Pending Meter increments (Marks summed since the last publish). - uint64_t mHostFnOpReadEntry{0}; - uint64_t mHostFnOpWriteEntry{0}; - uint64_t mHostFnOpReadKeyByte{0}; - uint64_t mHostFnOpWriteKeyByte{0}; - uint64_t mHostFnOpReadLedgerByte{0}; - uint64_t mHostFnOpReadDataByte{0}; - uint64_t mHostFnOpReadCodeByte{0}; - uint64_t mHostFnOpWriteLedgerByte{0}; - uint64_t mHostFnOpWriteDataByte{0}; - uint64_t mHostFnOpWriteCodeByte{0}; - uint64_t mHostFnOpEmitEvent{0}; - uint64_t mHostFnOpEmitEventByte{0}; - uint64_t mHostFnOpCpuInsn{0}; - uint64_t mHostFnOpMemByte{0}; - uint64_t mHostFnOpCpuInsnExclVm{0}; - uint64_t mHostFnOpMaxRwKeyByte{0}; - uint64_t mHostFnOpMaxRwDataByte{0}; - uint64_t mHostFnOpMaxRwCodeByte{0}; - uint64_t mHostFnOpMaxEmitEventByte{0}; - uint64_t mHostFnOpSuccess{0}; - uint64_t mHostFnOpFailure{0}; - uint64_t mExtFpTtlOpReadLedgerByte{0}; - uint64_t mRestoreFpOpReadLedgerByte{0}; - uint64_t mRestoreFpOpWriteLedgerByte{0}; - - // Pending sample streams for percentile-bearing histograms/timers - // (timer samples are in nanoseconds). - std::vector mHostFnOpInvokeTimeNsecs; - std::vector mHostFnOpInvokeTimeNsecsExclVm; - std::vector mHostFnOpInvokeTimeFsecsCpuInsnRatio; - std::vector mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm; - std::vector mHostFnOpDeclaredInsnsUsageRatio; - std::vector mHostFnOpExecNsecs; - std::vector mExtFpTtlOpExecNsecs; - std::vector mRestoreFpOpExecNsecs; - std::vector mTxSizeByte; - std::vector mTxApplyNsecs; - std::vector mOpApplyNsecs; - }; - - private: - std::atomic mCounterLedgerTxCount{0}; - std::atomic mCounterLedgerCpuInsn{0}; - std::atomic mCounterLedgerTxsSizeByte{0}; - std::atomic mCounterLedgerReadEntry{0}; - std::atomic mCounterLedgerReadByte{0}; - std::atomic mCounterLedgerWriteEntry{0}; - std::atomic mCounterLedgerWriteByte{0}; - - // These are modified within InvokeHostFunctionOp - std::atomic mLedgerInsnsCount{0}; - std::atomic mLedgerInsnsExclVmCount{0}; - std::atomic mLedgerHostFnExecTimeNsecs{0}; - - // All per-thread batches handed out by getApplyThreadBatch(), drained on - // each publishAndResetLedgerWideMetrics() call. - std::mutex mApplyBatchesMutex; - std::unordered_map> - mApplyBatches; - - void flushApplyMetricsBatches(); + // Pending Meter increments (Marks summed since the last publish). + uint64_t mHostFnOpReadEntry{0}; + uint64_t mHostFnOpWriteEntry{0}; + uint64_t mHostFnOpReadKeyByte{0}; + uint64_t mHostFnOpWriteKeyByte{0}; + uint64_t mHostFnOpReadLedgerByte{0}; + uint64_t mHostFnOpReadDataByte{0}; + uint64_t mHostFnOpReadCodeByte{0}; + uint64_t mHostFnOpWriteLedgerByte{0}; + uint64_t mHostFnOpWriteDataByte{0}; + uint64_t mHostFnOpWriteCodeByte{0}; + uint64_t mHostFnOpEmitEvent{0}; + uint64_t mHostFnOpEmitEventByte{0}; + uint64_t mHostFnOpCpuInsn{0}; + uint64_t mHostFnOpMemByte{0}; + uint64_t mHostFnOpCpuInsnExclVm{0}; + uint64_t mHostFnOpMaxRwKeyByte{0}; + uint64_t mHostFnOpMaxRwDataByte{0}; + uint64_t mHostFnOpMaxRwCodeByte{0}; + uint64_t mHostFnOpMaxEmitEventByte{0}; + uint64_t mHostFnOpSuccess{0}; + uint64_t mHostFnOpFailure{0}; + uint64_t mExtFpTtlOpReadLedgerByte{0}; + uint64_t mRestoreFpOpReadLedgerByte{0}; + uint64_t mRestoreFpOpWriteLedgerByte{0}; + + // Pending ledger-wide accumulator values, published as single per-ledger + // samples into the corresponding histograms. + uint64_t mLedgerTxCount{0}; + uint64_t mLedgerCpuInsn{0}; + uint64_t mLedgerTxsSizeByte{0}; + uint64_t mLedgerReadEntry{0}; + uint64_t mLedgerReadByte{0}; + uint64_t mLedgerWriteEntry{0}; + uint64_t mLedgerWriteByte{0}; + uint64_t mLedgerInsnsCount{0}; + uint64_t mLedgerInsnsExclVmCount{0}; + uint64_t mLedgerHostFnExecTimeNsecs{0}; + + // Pending sample streams for percentile-bearing histograms/timers + // (timer samples are in nanoseconds). + std::vector mHostFnOpInvokeTimeNsecs; + std::vector mHostFnOpInvokeTimeNsecsExclVm; + std::vector mHostFnOpInvokeTimeFsecsCpuInsnRatio; + std::vector mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm; + std::vector mHostFnOpDeclaredInsnsUsageRatio; + std::vector mHostFnOpExecNsecs; + std::vector mExtFpTtlOpExecNsecs; + std::vector mRestoreFpOpExecNsecs; + std::vector mTxSizeByte; + std::vector mTxApplyNsecs; + std::vector mOpApplyNsecs; + + // Adds all the accumulated values and samples of `other` into this + // instance. + void merge(SorobanApplyMetrics&& other); +}; +// Collection of the Medida metrics related to Soroban. +class SorobanMetricsRegistry +{ +#ifdef BUILD_TESTS public: +#else + private: +#endif // ledger-wide metrics medida::Histogram& mLedgerTxCount; medida::Histogram& mLedgerCpuInsn; @@ -122,10 +114,10 @@ class SorobanMetrics medida::Histogram& mTxSizeByte; // Cached references to the (op-kind-agnostic) "ledger.transaction.apply" - // and "ledger.operation.apply" timers: the parallel apply path records - // per-tx/per-op samples into its ApplyMetricsBatch and they are published - // into these at ledger close. These are the same timer instances the - // sequential apply path updates directly via registry lookups. + // and "ledger.operation.apply" timers. Unlike the rest of the metrics + // here these are shared with the Classic apply path, the class name + // ambiguity is a trade-off here (SorobanAndClassicMetricsRegistry doesn't + // read too well). medida::Timer& mTransactionApply; medida::Timer& mOperationApply; @@ -167,6 +159,7 @@ class SorobanMetrics medida::Meter& mRestoreFpOpWriteLedgerByte; medida::Timer& mRestoreFpOpExec; + public: // `NetworkConfig` metrics medida::Counter& mConfigContractDataKeySizeBytes; medida::Counter& mConfigMaxContractDataEntrySizeBytes; @@ -201,24 +194,10 @@ class SorobanMetrics medida::Counter& mContractCodeEntryCount; medida::Counter& mContractDataEntryCount; - SorobanMetrics(MetricsRegistry& metrics); - - // Returns the calling thread's metrics batch for this SorobanMetrics - // instance, creating and registering it on first use. - ApplyMetricsBatch& getApplyThreadBatch(); - - void accumulateModelledCpuInsns(uint64_t insnsCount, - uint64_t insnsExclVmCount, - uint64_t execTimeNsecs); - void accumulateLedgerTxCount(uint64_t txCount); - void accumulateLedgerCpuInsn(uint64_t cpuInsn); - void accumulateLedgerTxsSizeByte(uint64_t txsSizeByte); - void accumulateLedgerReadEntry(uint64_t readEntry); - void accumulateLedgerReadByte(uint64_t readByte); - void accumulateLedgerWriteEntry(uint64_t writeEntry); - void accumulateLedgerWriteByte(uint64_t writeByte); + SorobanMetricsRegistry(MetricsRegistry& metrics); - void publishAndResetLedgerWideMetrics(); + // Records the provided apply metrics into the underlying medida metrics. + void recordApplyMetrics(SorobanApplyMetrics const& metrics); }; // Adds the wall-clock duration of its lifetime (in nanoseconds) to a @@ -245,37 +224,4 @@ class ScopedNsecsTimer uint64_t& mTarget; std::chrono::steady_clock::time_point mStart; }; - -// Times its lifetime and records the elapsed nanoseconds as a sample in the -// given vector of the calling thread's ApplyMetricsBatch. -class BatchedTimerScope -{ - public: - using SampleField = - std::vector SorobanMetrics::ApplyMetricsBatch::*; - - BatchedTimerScope(SorobanMetrics& metrics, SampleField field) - : mMetrics(metrics) - , mField(field) - , mStart(std::chrono::steady_clock::now()) - { - } - - BatchedTimerScope(BatchedTimerScope const&) = delete; - BatchedTimerScope& operator=(BatchedTimerScope const&) = delete; - - ~BatchedTimerScope() - { - auto elapsed = std::chrono::duration_cast( - std::chrono::steady_clock::now() - mStart) - .count(); - auto& batch = mMetrics.getApplyThreadBatch(); - (batch.*mField).push_back(elapsed); - } - - private: - SorobanMetrics& mMetrics; - SampleField mField; - std::chrono::steady_clock::time_point mStart; -}; } diff --git a/src/main/AppConnector.cpp b/src/main/AppConnector.cpp index b03c4283bd..1ef19daa10 100644 --- a/src/main/AppConnector.cpp +++ b/src/main/AppConnector.cpp @@ -67,7 +67,7 @@ AppConnector::isStopping() const return mApp.isStopping(); } -SorobanMetrics& +SorobanMetricsRegistry& AppConnector::getSorobanMetrics() const { return mApp.getLedgerManager().getSorobanMetrics(); diff --git a/src/main/AppConnector.h b/src/main/AppConnector.h index cf3bb124de..49146dc522 100644 --- a/src/main/AppConnector.h +++ b/src/main/AppConnector.h @@ -18,7 +18,7 @@ class Herder; class BanManager; struct OverlayMetrics; class SorobanNetworkConfig; -class SorobanMetrics; +class SorobanMetricsRegistry; class SearchableHotArchiveBucketListSnapshot; struct LedgerTxnDelta; class CapacityTrackedMessage; @@ -49,7 +49,7 @@ class AppConnector Hash const& getNetworkID() const; // Thread-safe methods - SorobanMetrics& getSorobanMetrics() const; + SorobanMetricsRegistry& getSorobanMetrics() const; void postOnMainThread( std::function&& f, std::string&& message, Scheduler::ActionType type = Scheduler::ActionType::NORMAL_ACTION); diff --git a/src/test/fuzz/targets/TxFuzzTarget.cpp b/src/test/fuzz/targets/TxFuzzTarget.cpp index c43b3c879f..6fb109b3dc 100644 --- a/src/test/fuzz/targets/TxFuzzTarget.cpp +++ b/src/test/fuzz/targets/TxFuzzTarget.cpp @@ -7,6 +7,7 @@ #include "crypto/SHA.h" #include "invariant/OrderBookIsNotCrossed.h" #include "ledger/LedgerTxn.h" +#include "ledger/SorobanMetrics.h" #include "ledger/TrustLineWrapper.h" #include "main/Application.h" #include "test/Catch2.h" @@ -160,8 +161,12 @@ FuzzTransactionFrame::attemptApplication(Application& app, app.getAppConnector()); std::optional sorobanNetworkConfig; Hash sorobanRngSeed; + // Fuzzed applies run outside of a ledger close, so the apply metrics + // recorded here are simply dropped. + SorobanApplyMetrics sorobanMetrics; applyOperations(signatureChecker, app.getAppConnector(), ltx, tm, - *mTxResult, sorobanNetworkConfig, sorobanRngSeed); + *mTxResult, sorobanNetworkConfig, sorobanRngSeed, + sorobanMetrics); if (mTxResult->getResultCode() == txINTERNAL_ERROR) { throw std::runtime_error("Internal error while fuzzing"); diff --git a/src/transactions/ExtendFootprintTTLOpFrame.cpp b/src/transactions/ExtendFootprintTTLOpFrame.cpp index 1a87a760a4..1c9a08415c 100644 --- a/src/transactions/ExtendFootprintTTLOpFrame.cpp +++ b/src/transactions/ExtendFootprintTTLOpFrame.cpp @@ -26,7 +26,7 @@ innerResult(OperationResult& res) struct ExtendFootprintTTLMetrics { - SorobanMetrics& mMetrics; + SorobanApplyMetrics& mMetrics; uint32 mLedgerReadByte{0}; @@ -34,20 +34,17 @@ struct ExtendFootprintTTLMetrics uint64_t mExecTimeNsecs{0}; bool mExecTimed{false}; - ExtendFootprintTTLMetrics(SorobanMetrics& metrics) : mMetrics(metrics) + explicit ExtendFootprintTTLMetrics(SorobanApplyMetrics& metrics) + : mMetrics(metrics) { } ~ExtendFootprintTTLMetrics() { - // Record into the calling thread's batch (published once per ledger) - // rather than updating process-wide metrics from every (possibly - // concurrent) operation. - auto& batch = mMetrics.getApplyThreadBatch(); - batch.mExtFpTtlOpReadLedgerByte += mLedgerReadByte; + mMetrics.mExtFpTtlOpReadLedgerByte += mLedgerReadByte; if (mExecTimed) { - batch.mExtFpTtlOpExecNsecs.push_back( + mMetrics.mExtFpTtlOpExecNsecs.push_back( static_cast(mExecTimeNsecs)); } } @@ -95,7 +92,8 @@ class ExtendFootprintTTLApplyHelper : virtual public LedgerAccessHelper AppConnector& app, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta, ExtendFootprintTTLOpFrame const& opFrame, - SorobanNetworkConfig const& sorobanConfig) + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) : mApp(app) , mRes(res) , mRefundableFeeTracker(refundableFeeTracker) @@ -104,7 +102,7 @@ class ExtendFootprintTTLApplyHelper : virtual public LedgerAccessHelper , mResources(mOpFrame.mParentTx.sorobanResources()) , mSorobanConfig(sorobanConfig) , mAppConfig(app.getConfig()) - , mMetrics(app.getSorobanMetrics()) + , mMetrics(sorobanMetrics) , mDiagnosticEvents(mOpMeta.getDiagnosticEventManager()) { } @@ -220,9 +218,10 @@ class ExtendFootprintTTLPreV23ApplyHelper AppConnector& app, AbstractLedgerTxn& ltx, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta, ExtendFootprintTTLOpFrame const& opFrame, - SorobanNetworkConfig const& sorobanConfig) + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) : ExtendFootprintTTLApplyHelper(app, res, refundableFeeTracker, opMeta, - opFrame, sorobanConfig) + opFrame, sorobanConfig, sorobanMetrics) , PreV23LedgerAccessHelper(ltx) { } @@ -255,9 +254,11 @@ class ExtendFootprintTTLParallelApplyHelper AppConnector& app, ThreadParallelApplyLedgerState const& threadState, ParallelLedgerInfo const& ledgerInfo, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta, ExtendFootprintTTLOpFrame const& opFrame) + OperationMetaBuilder& opMeta, ExtendFootprintTTLOpFrame const& opFrame, + SorobanApplyMetrics& sorobanMetrics) : ExtendFootprintTTLApplyHelper(app, res, refundableFeeTracker, opMeta, - opFrame, threadState.getSorobanConfig()) + opFrame, threadState.getSorobanConfig(), + sorobanMetrics) , ParallelLedgerAccessHelper(threadState, ledgerInfo) { } @@ -278,7 +279,7 @@ std::optional ExtendFootprintTTLOpFrame::doParallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& appConfig, Hash const& _txPrngSeed, - ParallelLedgerInfo const& ledgerInfo, SorobanMetrics& sorobanMetrics, + ParallelLedgerInfo const& ledgerInfo, SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta) const @@ -287,8 +288,9 @@ ExtendFootprintTTLOpFrame::doParallelApply( releaseAssertOrThrow( protocolVersionStartsFrom(ledgerInfo.getLedgerVersion(), PARALLEL_SOROBAN_PHASE_PROTOCOL_VERSION)); - ExtendFootprintTTLParallelApplyHelper helper( - app, threadState, ledgerInfo, res, refundableFeeTracker, opMeta, *this); + ExtendFootprintTTLParallelApplyHelper helper(app, threadState, ledgerInfo, + res, refundableFeeTracker, + opMeta, *this, sorobanMetrics); return helper.takeResult(helper.apply()); } @@ -298,14 +300,15 @@ ExtendFootprintTTLOpFrame::doApplyForSoroban( SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const + OperationMetaBuilder& opMeta, SorobanApplyMetrics& sorobanMetrics) const { ZoneNamedN(applyZone, "ExtendFootprintTTLOpFrame apply", true); releaseAssertOrThrow( protocolVersionIsBefore(ltx.loadHeader().current().ledgerVersion, PARALLEL_SOROBAN_PHASE_PROTOCOL_VERSION)); ExtendFootprintTTLPreV23ApplyHelper helper( - app, ltx, res, refundableFeeTracker, opMeta, *this, sorobanConfig); + app, ltx, res, refundableFeeTracker, opMeta, *this, sorobanConfig, + sorobanMetrics); return helper.apply(); } diff --git a/src/transactions/ExtendFootprintTTLOpFrame.h b/src/transactions/ExtendFootprintTTLOpFrame.h index 72138d2837..ed0b115e0a 100644 --- a/src/transactions/ExtendFootprintTTLOpFrame.h +++ b/src/transactions/ExtendFootprintTTLOpFrame.h @@ -28,7 +28,8 @@ class ExtendFootprintTTLOpFrame : public OperationFrame SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const override; + OperationMetaBuilder& opMeta, + SorobanApplyMetrics& sorobanMetrics) const override; bool doApply(AppConnector& app, AbstractLedgerTxn& ltx, OperationResult& res, OperationMetaBuilder& opMeta) const override; @@ -45,7 +46,7 @@ class ExtendFootprintTTLOpFrame : public OperationFrame ThreadParallelApplyLedgerState const& threadState, Config const& appConfig, Hash const& txPrngSeed, ParallelLedgerInfo const& ledgerInfo, - SorobanMetrics& sorobanMetrics, OperationResult& res, + SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta) const override; diff --git a/src/transactions/FeeBumpTransactionFrame.cpp b/src/transactions/FeeBumpTransactionFrame.cpp index c400ebf8c6..e63e2b25c6 100644 --- a/src/transactions/FeeBumpTransactionFrame.cpp +++ b/src/transactions/FeeBumpTransactionFrame.cpp @@ -86,7 +86,8 @@ void FeeBumpTransactionFrame::preParallelApply( AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, - SorobanNetworkConfig const& sorobanConfig) const + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) const { try { @@ -107,7 +108,8 @@ FeeBumpTransactionFrame::preParallelApply( try { mInnerTx->preParallelApply(/*chargeFee=*/false, app, ltx, meta, - txResult, sorobanConfig, getContentsHash()); + txResult, sorobanConfig, getContentsHash(), + sorobanMetrics); } catch (std::exception& e) { @@ -123,7 +125,7 @@ std::optional FeeBumpTransactionFrame::parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, - MutableTransactionResultBase& txResult, SorobanMetrics& sorobanMetrics, + MutableTransactionResultBase& txResult, SorobanApplyMetrics& sorobanMetrics, Hash const& txPrngSeed, TxEffects& effects) const { try @@ -154,7 +156,7 @@ FeeBumpTransactionFrame::apply( AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, - Hash const& sorobanBasePrngSeed) const + Hash const& sorobanBasePrngSeed, SorobanApplyMetrics& sorobanMetrics) const { try { @@ -180,7 +182,8 @@ FeeBumpTransactionFrame::apply( // If this throws, then we may not have the correct TransactionResult so // we must crash. return mInnerTx->apply(false, app, ltx, meta, txResult, sorobanConfig, - sorobanBasePrngSeed, getContentsHash()); + sorobanBasePrngSeed, getContentsHash(), + sorobanMetrics); } catch (std::exception& e) { diff --git a/src/transactions/FeeBumpTransactionFrame.h b/src/transactions/FeeBumpTransactionFrame.h index d3dd0efa73..0f1de59afe 100644 --- a/src/transactions/FeeBumpTransactionFrame.h +++ b/src/transactions/FeeBumpTransactionFrame.h @@ -90,24 +90,25 @@ class FeeBumpTransactionFrame : public TransactionFrameBase ~FeeBumpTransactionFrame() override = default; - void - preParallelApply(AppConnector& app, AbstractLedgerTxn& ltx, - TransactionMetaBuilder& meta, - MutableTransactionResultBase& txResult, - SorobanNetworkConfig const& sorobanConfig) const override; + void preParallelApply(AppConnector& app, AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) const override; std::optional parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, MutableTransactionResultBase& resPayload, - SorobanMetrics& sorobanMetrics, Hash const& sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics, Hash const& sorobanBasePrngSeed, TxEffects& effects) const override; bool apply(AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, - Hash const& sorobanBasePrngSeed) const override; + Hash const& sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics) const override; void processPostApply(AppConnector& app, AbstractLedgerTxn& ltx, diff --git a/src/transactions/InvokeHostFunctionOpFrame.cpp b/src/transactions/InvokeHostFunctionOpFrame.cpp index 5fc3cf8d34..92c243890c 100644 --- a/src/transactions/InvokeHostFunctionOpFrame.cpp +++ b/src/transactions/InvokeHostFunctionOpFrame.cpp @@ -106,7 +106,7 @@ maybePopulateOutputDiagnosticEvents(Config const& cfg, // Metrics for host function execution struct HostFunctionMetrics { - SorobanMetrics& mMetrics; + SorobanApplyMetrics& mMetrics; bool const mDisableMetrics; uint32_t mReadEntry{0}; @@ -147,7 +147,7 @@ struct HostFunctionMetrics bool mSuccess{false}; - HostFunctionMetrics(SorobanMetrics& metrics, bool disableMetrics) + HostFunctionMetrics(SorobanApplyMetrics& metrics, bool disableMetrics) : mMetrics(metrics), mDisableMetrics(disableMetrics) { } @@ -159,65 +159,62 @@ struct HostFunctionMetrics return; } - mMetrics.accumulateModelledCpuInsns(mCpuInsn, mCpuInsnExclVm, - mInvokeTimeNsecs); + mMetrics.mLedgerInsnsCount += mCpuInsn; + mMetrics.mLedgerInsnsExclVmCount += mCpuInsnExclVm; + mMetrics.mLedgerHostFnExecTimeNsecs += mInvokeTimeNsecs; - // Record everything into the calling thread's batch (published once - // per ledger) instead of updating ~25 process-wide metrics from - // every (possibly concurrent) operation. - auto& batch = mMetrics.getApplyThreadBatch(); + mMetrics.mHostFnOpReadEntry += mReadEntry; + mMetrics.mHostFnOpWriteEntry += mWriteEntry; - batch.mHostFnOpReadEntry += mReadEntry; - batch.mHostFnOpWriteEntry += mWriteEntry; + mMetrics.mHostFnOpReadKeyByte += mReadKeyByte; + mMetrics.mHostFnOpWriteKeyByte += mWriteKeyByte; - batch.mHostFnOpReadKeyByte += mReadKeyByte; - batch.mHostFnOpWriteKeyByte += mWriteKeyByte; + mMetrics.mHostFnOpReadLedgerByte += mLedgerReadByte; + mMetrics.mHostFnOpReadDataByte += mReadDataByte; + mMetrics.mHostFnOpReadCodeByte += mReadCodeByte; - batch.mHostFnOpReadLedgerByte += mLedgerReadByte; - batch.mHostFnOpReadDataByte += mReadDataByte; - batch.mHostFnOpReadCodeByte += mReadCodeByte; + mMetrics.mHostFnOpWriteLedgerByte += mLedgerWriteByte; + mMetrics.mHostFnOpWriteDataByte += mWriteDataByte; + mMetrics.mHostFnOpWriteCodeByte += mWriteCodeByte; - batch.mHostFnOpWriteLedgerByte += mLedgerWriteByte; - batch.mHostFnOpWriteDataByte += mWriteDataByte; - batch.mHostFnOpWriteCodeByte += mWriteCodeByte; + mMetrics.mHostFnOpEmitEvent += mEmitEvent; + mMetrics.mHostFnOpEmitEventByte += mEmitEventByte; - batch.mHostFnOpEmitEvent += mEmitEvent; - batch.mHostFnOpEmitEventByte += mEmitEventByte; - - batch.mHostFnOpCpuInsn += mCpuInsn; - batch.mHostFnOpMemByte += mMemByte; - batch.mHostFnOpInvokeTimeNsecs.push_back( + mMetrics.mHostFnOpCpuInsn += mCpuInsn; + mMetrics.mHostFnOpMemByte += mMemByte; + mMetrics.mHostFnOpInvokeTimeNsecs.push_back( static_cast(mInvokeTimeNsecs)); - batch.mHostFnOpCpuInsnExclVm += mCpuInsnExclVm; - batch.mHostFnOpInvokeTimeNsecsExclVm.push_back( + mMetrics.mHostFnOpCpuInsnExclVm += mCpuInsnExclVm; + mMetrics.mHostFnOpInvokeTimeNsecsExclVm.push_back( static_cast(mInvokeTimeNsecsExclVm)); - batch.mHostFnOpInvokeTimeFsecsCpuInsnRatio.push_back( + mMetrics.mHostFnOpInvokeTimeFsecsCpuInsnRatio.push_back( static_cast(mInvokeTimeNsecs * 1000000 / std::max(mCpuInsn, uint64_t(1)))); - batch.mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm.push_back( + mMetrics.mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm.push_back( static_cast(mInvokeTimeNsecsExclVm * 1000000 / std::max(mCpuInsnExclVm, uint64_t(1)))); - batch.mHostFnOpDeclaredInsnsUsageRatio.push_back(static_cast( - mCpuInsn * 1000000 / std::max(mDeclaredCpuInsn, uint64_t(1)))); + mMetrics.mHostFnOpDeclaredInsnsUsageRatio.push_back( + static_cast(mCpuInsn * 1000000 / + std::max(mDeclaredCpuInsn, uint64_t(1)))); - batch.mHostFnOpMaxRwKeyByte += mMaxReadWriteKeyByte; - batch.mHostFnOpMaxRwDataByte += mMaxReadWriteDataByte; - batch.mHostFnOpMaxRwCodeByte += mMaxReadWriteCodeByte; - batch.mHostFnOpMaxEmitEventByte += mMaxEmitEventByte; + mMetrics.mHostFnOpMaxRwKeyByte += mMaxReadWriteKeyByte; + mMetrics.mHostFnOpMaxRwDataByte += mMaxReadWriteDataByte; + mMetrics.mHostFnOpMaxRwCodeByte += mMaxReadWriteCodeByte; + mMetrics.mHostFnOpMaxEmitEventByte += mMaxEmitEventByte; if (mExecTimed) { - batch.mHostFnOpExecNsecs.push_back( + mMetrics.mHostFnOpExecNsecs.push_back( static_cast(mExecTimeNsecs)); } if (mSuccess) { - ++batch.mHostFnOpSuccess; + ++mMetrics.mHostFnOpSuccess; } else { - ++batch.mHostFnOpFailure; + ++mMetrics.mHostFnOpFailure; } } @@ -315,7 +312,7 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper OperationMetaBuilder& opMeta, InvokeHostFunctionOpFrame const& opFrame, SorobanNetworkConfig const& sorobanConfig, ApplyLedgerView applyView, rust::Box const& moduleCache, - uint32_t protocolVersion) + uint32_t protocolVersion, SorobanApplyMetrics& sorobanMetrics) : mApp(app) , mRes(res) , mRefundableFeeTracker(refundableFeeTracker) @@ -325,7 +322,7 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper , mResources(mOpFrame.mParentTx.sorobanResources()) , mSorobanConfig(sorobanConfig) , mAppConfig(app.getConfig()) - , mMetrics(app.getSorobanMetrics(), + , mMetrics(sorobanMetrics, app.getConfig().DISABLE_SOROBAN_METRICS_FOR_TESTING) , mApplyLedgerView(std::move(applyView)) , mModuleCache(moduleCache) @@ -1148,11 +1145,12 @@ class InvokeHostFunctionPreV23ApplyHelper std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta, InvokeHostFunctionOpFrame const& opFrame, SorobanNetworkConfig const& sorobanConfig, - rust::Box const& moduleCache) + rust::Box const& moduleCache, + SorobanApplyMetrics& sorobanMetrics) : InvokeHostFunctionApplyHelper( app, sorobanBasePrngSeed, res, refundableFeeTracker, opMeta, opFrame, sorobanConfig, app.copyApplyLedgerView(), moduleCache, - ltx.loadHeader().current().ledgerVersion) + ltx.loadHeader().current().ledgerVersion, sorobanMetrics) , PreV23LedgerAccessHelper(ltx) { } @@ -1331,12 +1329,13 @@ class InvokeHostFunctionParallelApplyHelper ParallelLedgerInfo const& ledgerInfo, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta, InvokeHostFunctionOpFrame const& opFrame) + OperationMetaBuilder& opMeta, InvokeHostFunctionOpFrame const& opFrame, + SorobanApplyMetrics& sorobanMetrics) : InvokeHostFunctionApplyHelper( app, sorobanBasePrngSeed, res, refundableFeeTracker, opMeta, opFrame, threadState.getSorobanConfig(), threadState.getSnapshot(), threadState.getModuleCache(), - ledgerInfo.getLedgerVersion()) + ledgerInfo.getLedgerVersion(), sorobanMetrics) , ParallelLedgerAccessHelper(threadState, ledgerInfo) { ZoneScoped; @@ -1390,7 +1389,7 @@ InvokeHostFunctionOpFrame::doApplyForSoroban( SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const + OperationMetaBuilder& opMeta, SorobanApplyMetrics& sorobanMetrics) const { ZoneNamedN(applyZone, "InvokeHostFunctionOpFrame apply", true); releaseAssertOrThrow(refundableFeeTracker); @@ -1402,7 +1401,7 @@ InvokeHostFunctionOpFrame::doApplyForSoroban( auto moduleCache = app.getModuleCache(); InvokeHostFunctionPreV23ApplyHelper helper( app, ltx, sorobanBasePrngSeed, res, refundableFeeTracker, opMeta, *this, - sorobanConfig, moduleCache); + sorobanConfig, moduleCache, sorobanMetrics); return helper.apply(); } @@ -1419,7 +1418,7 @@ std::optional InvokeHostFunctionOpFrame::doParallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& appConfig, Hash const& txPrngSeed, - ParallelLedgerInfo const& ledgerInfo, SorobanMetrics& sorobanMetrics, + ParallelLedgerInfo const& ledgerInfo, SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta) const @@ -1432,7 +1431,7 @@ InvokeHostFunctionOpFrame::doParallelApply( InvokeHostFunctionParallelApplyHelper helper( app, threadState, ledgerInfo, txPrngSeed, res, refundableFeeTracker, - opMeta, *this); + opMeta, *this, sorobanMetrics); return helper.takeResult(helper.apply()); } diff --git a/src/transactions/InvokeHostFunctionOpFrame.h b/src/transactions/InvokeHostFunctionOpFrame.h index c1538f4661..146acff063 100644 --- a/src/transactions/InvokeHostFunctionOpFrame.h +++ b/src/transactions/InvokeHostFunctionOpFrame.h @@ -42,7 +42,8 @@ class InvokeHostFunctionOpFrame : public OperationFrame SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const override; + OperationMetaBuilder& opMeta, + SorobanApplyMetrics& sorobanMetrics) const override; bool doApply(AppConnector& app, AbstractLedgerTxn& ltx, OperationResult& res, @@ -60,7 +61,7 @@ class InvokeHostFunctionOpFrame : public OperationFrame ThreadParallelApplyLedgerState const& threadState, Config const& appConfig, Hash const& txPrngSeed, ParallelLedgerInfo const& ledgerInfo, - SorobanMetrics& sorobanMetrics, OperationResult& res, + SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta) const override; diff --git a/src/transactions/OperationFrame.cpp b/src/transactions/OperationFrame.cpp index d70bcea9ed..f53414d8aa 100644 --- a/src/transactions/OperationFrame.cpp +++ b/src/transactions/OperationFrame.cpp @@ -143,7 +143,7 @@ OperationFrame::apply( std::optional const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const + OperationMetaBuilder& opMeta, SorobanApplyMetrics& sorobanMetrics) const { ZoneScoped; CLOG_TRACE(Tx, "{}", xdrToCerealString(mOperation, "Operation")); @@ -157,9 +157,9 @@ OperationFrame::apply( if (isSoroban()) { releaseAssertOrThrow(sorobanConfig); - applyRes = - doApplyForSoroban(app, ltx, *sorobanConfig, sorobanBasePrngSeed, - res, refundableFeeTracker, opMeta); + applyRes = doApplyForSoroban( + app, ltx, *sorobanConfig, sorobanBasePrngSeed, res, + refundableFeeTracker, opMeta, sorobanMetrics); } else { @@ -176,7 +176,7 @@ std::optional OperationFrame::parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, - SorobanMetrics& sorobanMetrics, OperationResult& res, + SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta, Hash const& txPrngSeed) const { @@ -192,7 +192,7 @@ std::optional OperationFrame::doParallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& appConfig, Hash const& txPrngSeed, - ParallelLedgerInfo const& ledgerInfo, SorobanMetrics& sorobanMetrics, + ParallelLedgerInfo const& ledgerInfo, SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta) const @@ -377,7 +377,7 @@ OperationFrame::doApplyForSoroban( SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const + OperationMetaBuilder& opMeta, SorobanApplyMetrics& sorobanMetrics) const { // This implementation is just a stub for classic operations, it's not // supposed to be called by them. diff --git a/src/transactions/OperationFrame.h b/src/transactions/OperationFrame.h index a4028de87c..c05f6069d3 100644 --- a/src/transactions/OperationFrame.h +++ b/src/transactions/OperationFrame.h @@ -27,6 +27,7 @@ class DiagnosticEventManager; class RefundableFeeTracker; class OperationMetaBuilder; class ThreadParallelApplyLedgerState; +struct SorobanApplyMetrics; enum class ThresholdLevel { @@ -53,7 +54,8 @@ class OperationFrame SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const; + OperationMetaBuilder& opMeta, + SorobanApplyMetrics& sorobanMetrics) const; virtual bool doApply(AppConnector& app, AbstractLedgerTxn& ltx, std::optional const& sorobanConfig, @@ -67,7 +69,7 @@ class OperationFrame ThreadParallelApplyLedgerState const& threadState, Config const& config, Hash const& txPrngSeed, ParallelLedgerInfo const& ledgerInfo, - SorobanMetrics& sorobanMetrics, OperationResult& res, + SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta) const; @@ -114,13 +116,14 @@ class OperationFrame std::optional const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const; + OperationMetaBuilder& opMeta, + SorobanApplyMetrics& sorobanMetrics) const; // Returns std::nullopt if operation fails. std::optional parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, - SorobanMetrics& sorobanMetrics, OperationResult& res, + SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta, Hash const& sorobanBasePrngSeed) const; diff --git a/src/transactions/ParallelApplyUtils.cpp b/src/transactions/ParallelApplyUtils.cpp index 618901d499..e9c333b777 100644 --- a/src/transactions/ParallelApplyUtils.cpp +++ b/src/transactions/ParallelApplyUtils.cpp @@ -298,7 +298,8 @@ GlobalParallelApplyLedgerState::GlobalParallelApplyLedgerState( AppConnector& app, ApplyLedgerView applyView, AbstractLedgerTxn& ltx, std::vector const& stages, InMemorySorobanState const& inMemoryState, - SorobanNetworkConfig const& sorobanConfig) + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) : LedgerEntryScope(ScopeIdT(0, ltx.getHeader().ledgerSeq)) , mLCLApplyView(std::move(applyView)) , mInMemorySorobanState(inMemoryState) @@ -319,14 +320,16 @@ GlobalParallelApplyLedgerState::GlobalParallelApplyLedgerState( // had their sequence numbers bumped and fees charged. preParallelApply will // update sequence numbers so it needs to be called before we check // LedgerTxn. - preParallelApplyAndCollectModifiedClassicEntries(app, ltx, stages); + preParallelApplyAndCollectModifiedClassicEntries(app, ltx, stages, + sorobanMetrics); } void GlobalParallelApplyLedgerState:: preParallelApplyAndCollectModifiedClassicEntries( AppConnector& app, AbstractLedgerTxn& ltx, - std::vector const& stages) + std::vector const& stages, + SorobanApplyMetrics& sorobanMetrics) { auto fetchInMemoryClassicEntries = [&](xdr::xvector const& keys) { @@ -366,7 +369,7 @@ GlobalParallelApplyLedgerState:: // modify the fee source accounts sequence numbers. txBundle.getTx()->preParallelApply( app, ltx, txBundle.getEffects().getMeta(), - txBundle.getResPayload(), mSorobanConfig); + txBundle.getResPayload(), mSorobanConfig, sorobanMetrics); } } diff --git a/src/transactions/ParallelApplyUtils.h b/src/transactions/ParallelApplyUtils.h index 7e7fd8b743..7bd7e0684f 100644 --- a/src/transactions/ParallelApplyUtils.h +++ b/src/transactions/ParallelApplyUtils.h @@ -19,6 +19,7 @@ namespace stellar class InMemorySorobanState; class GlobalParallelApplyLedgerState; +struct SorobanApplyMetrics; class ParallelLedgerInfo { @@ -221,7 +222,8 @@ class GlobalParallelApplyLedgerState void preParallelApplyAndCollectModifiedClassicEntries( AppConnector& app, AbstractLedgerTxn& ltx, - std::vector const& stages); + std::vector const& stages, + SorobanApplyMetrics& sorobanMetrics); bool maybeMergeRoTTLBumps(LedgerKey const& key, @@ -245,7 +247,8 @@ class GlobalParallelApplyLedgerState AbstractLedgerTxn& ltx, std::vector const& stages, InMemorySorobanState const& inMemoryState, - SorobanNetworkConfig const& sorobanConfig); + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics); ParallelApplyEntryMap const& getGlobalEntryMap() const; RestoredEntries const& getRestoredEntries() const; diff --git a/src/transactions/RestoreFootprintOpFrame.cpp b/src/transactions/RestoreFootprintOpFrame.cpp index 8ec877c3b0..acdd42399e 100644 --- a/src/transactions/RestoreFootprintOpFrame.cpp +++ b/src/transactions/RestoreFootprintOpFrame.cpp @@ -27,7 +27,7 @@ innerResult(OperationResult& res) struct RestoreFootprintMetrics { - SorobanMetrics& mMetrics; + SorobanApplyMetrics& mMetrics; uint32_t mLedgerReadByte{0}; uint32_t mLedgerWriteByte{0}; @@ -36,21 +36,17 @@ struct RestoreFootprintMetrics uint64_t mExecTimeNsecs{0}; bool mExecTimed{false}; - RestoreFootprintMetrics(SorobanMetrics& metrics) : mMetrics(metrics) + RestoreFootprintMetrics(SorobanApplyMetrics& metrics) : mMetrics(metrics) { } ~RestoreFootprintMetrics() { - // Record into the calling thread's batch (published once per ledger) - // rather than updating process-wide metrics from every (possibly - // concurrent) operation. - auto& batch = mMetrics.getApplyThreadBatch(); - batch.mRestoreFpOpReadLedgerByte += mLedgerReadByte; - batch.mRestoreFpOpWriteLedgerByte += mLedgerWriteByte; + mMetrics.mRestoreFpOpReadLedgerByte += mLedgerReadByte; + mMetrics.mRestoreFpOpWriteLedgerByte += mLedgerWriteByte; if (mExecTimed) { - batch.mRestoreFpOpExecNsecs.push_back( + mMetrics.mRestoreFpOpExecNsecs.push_back( static_cast(mExecTimeNsecs)); } } @@ -91,7 +87,8 @@ class RestoreFootprintApplyHelper : virtual public LedgerAccessHelper AppConnector& app, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta, RestoreFootprintOpFrame const& opFrame, - SorobanNetworkConfig const& sorobanConfig) + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) : mApp(app) , mRes(res) , mRefundableFeeTracker(refundableFeeTracker) @@ -100,7 +97,7 @@ class RestoreFootprintApplyHelper : virtual public LedgerAccessHelper , mResources(mOpFrame.mParentTx.sorobanResources()) , mSorobanConfig(sorobanConfig) , mAppConfig(app.getConfig()) - , mMetrics(app.getSorobanMetrics()) + , mMetrics(sorobanMetrics) , mDiagnosticEvents(mOpMeta.getDiagnosticEventManager()) { } @@ -268,9 +265,10 @@ class RestoreFootprintPreV23ApplyHelper AppConnector& app, AbstractLedgerTxn& ltx, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta, RestoreFootprintOpFrame const& opFrame, - SorobanNetworkConfig const& sorobanConfig) + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) : RestoreFootprintApplyHelper(app, res, refundableFeeTracker, opMeta, - opFrame, sorobanConfig) + opFrame, sorobanConfig, sorobanMetrics) , PreV23LedgerAccessHelper(ltx) { } @@ -309,9 +307,11 @@ class RestoreFootprintParallelApplyHelper AppConnector& app, ThreadParallelApplyLedgerState const& threadState, ParallelLedgerInfo const& ledgerInfo, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta, RestoreFootprintOpFrame const& opFrame) + OperationMetaBuilder& opMeta, RestoreFootprintOpFrame const& opFrame, + SorobanApplyMetrics& sorobanMetrics) : RestoreFootprintApplyHelper(app, res, refundableFeeTracker, opMeta, - opFrame, threadState.getSorobanConfig()) + opFrame, threadState.getSorobanConfig(), + sorobanMetrics) , ParallelLedgerAccessHelper(threadState, ledgerInfo) , mApplyLedgerView(threadState.getSnapshot()) { @@ -384,7 +384,7 @@ std::optional RestoreFootprintOpFrame::doParallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& appConfig, Hash const& txPrngSeed, - ParallelLedgerInfo const& ledgerInfo, SorobanMetrics& sorobanMetrics, + ParallelLedgerInfo const& ledgerInfo, SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta) const @@ -394,8 +394,9 @@ RestoreFootprintOpFrame::doParallelApply( protocolVersionStartsFrom(ledgerInfo.getLedgerVersion(), PARALLEL_SOROBAN_PHASE_PROTOCOL_VERSION)); releaseAssertOrThrow(refundableFeeTracker); - RestoreFootprintParallelApplyHelper helper( - app, threadState, ledgerInfo, res, refundableFeeTracker, opMeta, *this); + RestoreFootprintParallelApplyHelper helper(app, threadState, ledgerInfo, + res, refundableFeeTracker, + opMeta, *this, sorobanMetrics); return helper.takeResult(helper.apply()); } @@ -405,14 +406,15 @@ RestoreFootprintOpFrame::doApplyForSoroban( SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const + OperationMetaBuilder& opMeta, SorobanApplyMetrics& sorobanMetrics) const { releaseAssertOrThrow( protocolVersionIsBefore(ltx.loadHeader().current().ledgerVersion, PARALLEL_SOROBAN_PHASE_PROTOCOL_VERSION)); releaseAssertOrThrow(refundableFeeTracker); RestoreFootprintPreV23ApplyHelper helper( - app, ltx, res, refundableFeeTracker, opMeta, *this, sorobanConfig); + app, ltx, res, refundableFeeTracker, opMeta, *this, sorobanConfig, + sorobanMetrics); return helper.apply(); } diff --git a/src/transactions/RestoreFootprintOpFrame.h b/src/transactions/RestoreFootprintOpFrame.h index 0db0b25e19..fff8662a00 100644 --- a/src/transactions/RestoreFootprintOpFrame.h +++ b/src/transactions/RestoreFootprintOpFrame.h @@ -27,7 +27,8 @@ class RestoreFootprintOpFrame : public OperationFrame SorobanNetworkConfig const& sorobanConfig, Hash const& sorobanBasePrngSeed, OperationResult& res, std::optional& refundableFeeTracker, - OperationMetaBuilder& opMeta) const override; + OperationMetaBuilder& opMeta, + SorobanApplyMetrics& sorobanMetrics) const override; bool doApply(AppConnector& app, AbstractLedgerTxn& ltx, OperationResult& res, OperationMetaBuilder& opMeta) const override; @@ -44,7 +45,7 @@ class RestoreFootprintOpFrame : public OperationFrame ThreadParallelApplyLedgerState const& threadState, Config const& appConfig, Hash const& txPrngSeed, ParallelLedgerInfo const& ledgerInfo, - SorobanMetrics& sorobanMetrics, OperationResult& res, + SorobanApplyMetrics& sorobanMetrics, OperationResult& res, std::optional& refundableFeeTracker, OperationMetaBuilder& opMeta) const override; diff --git a/src/transactions/TransactionFrame.cpp b/src/transactions/TransactionFrame.cpp index a21dc5afe7..c5211c5024 100644 --- a/src/transactions/TransactionFrame.cpp +++ b/src/transactions/TransactionFrame.cpp @@ -992,7 +992,8 @@ TransactionFrame::refundSorobanFee(AbstractLedgerTxn& ltxOuter, } void -TransactionFrame::updateSorobanMetrics(AppConnector& app) const +TransactionFrame::updateSorobanMetrics( + AppConnector& app, SorobanApplyMetrics& sorobanMetrics) const { releaseAssertOrThrow(isSoroban()); if (app.getConfig().DISABLE_SOROBAN_METRICS_FOR_TESTING) @@ -1000,26 +1001,17 @@ TransactionFrame::updateSorobanMetrics(AppConnector& app) const return; } - SorobanMetrics& metrics = app.getSorobanMetrics(); auto txSize = static_cast(this->getSize()); auto const& r = sorobanResources(); - // record the tx metrics into the per-thread batch (published once per - // ledger) - { - auto& batch = metrics.getApplyThreadBatch(); - batch.mTxSizeByte.push_back(txSize); - } - // accumulate the ledger-wide metrics, which will get emitted at the ledger - // close - metrics.accumulateLedgerTxCount(getNumOperations()); - metrics.accumulateLedgerCpuInsn(r.instructions); - metrics.accumulateLedgerTxsSizeByte(txSize); - metrics.accumulateLedgerReadEntry(static_cast( - r.footprint.readOnly.size() + r.footprint.readWrite.size())); - metrics.accumulateLedgerReadByte(r.diskReadBytes); - metrics.accumulateLedgerWriteEntry( - static_cast(r.footprint.readWrite.size())); - metrics.accumulateLedgerWriteByte(r.writeBytes); + sorobanMetrics.mTxSizeByte.push_back(txSize); + sorobanMetrics.mLedgerTxCount += getNumOperations(); + sorobanMetrics.mLedgerCpuInsn += r.instructions; + sorobanMetrics.mLedgerTxsSizeByte += txSize; + sorobanMetrics.mLedgerReadEntry += + r.footprint.readOnly.size() + r.footprint.readWrite.size(); + sorobanMetrics.mLedgerReadByte += r.diskReadBytes; + sorobanMetrics.mLedgerWriteEntry += r.footprint.readWrite.size(); + sorobanMetrics.mLedgerWriteByte += r.writeBytes; } bool @@ -1978,7 +1970,11 @@ TransactionFrame::apply( { TransactionMetaBuilder tm(true, *this, ltx.loadHeader().current().ledgerVersion, app); - return apply(app, ltx, tm, txResult, sorobanConfig, sorobanBasePrngSeed); + // Direct test applies run outside of a ledger close, so the apply metrics + // recorded here are simply dropped. + SorobanApplyMetrics sorobanMetrics; + return apply(app, ltx, tm, txResult, sorobanConfig, sorobanBasePrngSeed, + sorobanMetrics); } #endif @@ -2073,13 +2069,14 @@ TransactionFrame::commonPreApply(bool chargeFee, AppConnector& app, } void -TransactionFrame::preParallelApply( - AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, - MutableTransactionResultBase& resPayload, - SorobanNetworkConfig const& sorobanConfig) const +TransactionFrame::preParallelApply(AppConnector& app, AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& resPayload, + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) const { preParallelApply(true, app, ltx, meta, resPayload, sorobanConfig, - getContentsHash()); + getContentsHash(), sorobanMetrics); } void @@ -2088,7 +2085,8 @@ TransactionFrame::preParallelApply(bool chargeFee, AppConnector& app, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, SorobanNetworkConfig const& sorobanConfig, - Hash const& envelopeContentsHash) const + Hash const& envelopeContentsHash, + SorobanApplyMetrics& sorobanMetrics) const { ZoneScoped; releaseAssert(threadIsMain() || @@ -2103,7 +2101,7 @@ TransactionFrame::preParallelApply(bool chargeFee, AppConnector& app, bool ok = signatureChecker != nullptr; if (ok) { - updateSorobanMetrics(app); + updateSorobanMetrics(app, sorobanMetrics); auto& opResult = txResult.getOpResultAt(0); @@ -2141,7 +2139,7 @@ std::optional TransactionFrame::parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, - MutableTransactionResultBase& txResult, SorobanMetrics& sorobanMetrics, + MutableTransactionResultBase& txResult, SorobanApplyMetrics& sorobanMetrics, Hash const& txPrngSeed, TxEffects& effects) const { ZoneScoped; @@ -2166,12 +2164,7 @@ TransactionFrame::parallelApply( ledgerInfo.getLedgerVersion() >= config.LEDGER_PROTOCOL_MIN_VERSION_INTERNAL_ERROR_REPORT; - std::optional opTimer; - if (!config.DISABLE_SOROBAN_METRICS_FOR_TESTING) - { - opTimer.emplace(sorobanMetrics, - &SorobanMetrics::ApplyMetricsBatch::mOpApplyNsecs); - } + auto applyStart = std::chrono::steady_clock::now(); releaseAssertOrThrow(mOperations.size() == 1); @@ -2202,6 +2195,10 @@ TransactionFrame::parallelApply( txResult.setInnermostError(txFAILED); } + sorobanMetrics.mOpApplyNsecs.push_back( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - applyStart) + .count()); return res; } catch (std::bad_alloc& e) @@ -2253,7 +2250,7 @@ TransactionFrame::applyOperations( AbstractLedgerTxn& ltx, TransactionMetaBuilder& outerMeta, MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, - Hash const& sorobanBasePrngSeed) const + Hash const& sorobanBasePrngSeed, SorobanApplyMetrics& sorobanMetrics) const { ZoneScoped; if (!maybeAdoptFailedReplayResult(txResult)) @@ -2304,9 +2301,9 @@ TransactionFrame::applyOperations( } ++opNum; auto& opMeta = outerMeta.getOperationMetaBuilderAt(i); - bool txRes = - op->apply(app, signatureChecker, ltxOp, sorobanConfig, subSeed, - opResult, txResult.getRefundableFeeTracker(), opMeta); + bool txRes = op->apply( + app, signatureChecker, ltxOp, sorobanConfig, subSeed, opResult, + txResult.getRefundableFeeTracker(), opMeta, sorobanMetrics); #ifdef BUILD_TESTS maybeTriggerTestInternalError(mEnvelope); #endif @@ -2449,7 +2446,8 @@ TransactionFrame::apply( bool chargeFee, AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, - Hash const& sorobanBasePrngSeed, Hash const& envelopeContentsHash) const + Hash const& sorobanBasePrngSeed, Hash const& envelopeContentsHash, + SorobanApplyMetrics& sorobanMetrics) const { ZoneScoped; try @@ -2468,12 +2466,12 @@ TransactionFrame::apply( { if (isSoroban()) { - updateSorobanMetrics(app); + updateSorobanMetrics(app, sorobanMetrics); } - ok = - applyOperations(*signatureChecker, app, ltx, meta, txResult, - sorobanConfig, sorobanBasePrngSeed); + ok = applyOperations(*signatureChecker, app, ltx, meta, + txResult, sorobanConfig, + sorobanBasePrngSeed, sorobanMetrics); } return ok; } @@ -2505,10 +2503,10 @@ TransactionFrame::apply( AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, - Hash const& sorobanBasePrngSeed) const + Hash const& sorobanBasePrngSeed, SorobanApplyMetrics& sorobanMetrics) const { return apply(true, app, ltx, meta, txResult, sorobanConfig, - sorobanBasePrngSeed, getContentsHash()); + sorobanBasePrngSeed, getContentsHash(), sorobanMetrics); } void diff --git a/src/transactions/TransactionFrame.h b/src/transactions/TransactionFrame.h index 02eb5badfb..8286153269 100644 --- a/src/transactions/TransactionFrame.h +++ b/src/transactions/TransactionFrame.h @@ -143,7 +143,8 @@ class TransactionFrame : public TransactionFrameBase SignatureChecker& checker, AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, - Hash const& sorobanBasePrngSeed) const; + Hash const& sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics) const; void processSeqNum(AbstractLedgerTxn& ltx) const; @@ -159,7 +160,8 @@ class TransactionFrame : public TransactionFrameBase bool validateSorobanOpsConsistency() const; int64_t refundSorobanFee(AbstractLedgerTxn& ltx, AccountID const& feeSource, MutableTransactionResultBase& txResult) const; - void updateSorobanMetrics(AppConnector& app) const; + void updateSorobanMetrics(AppConnector& app, + SorobanApplyMetrics& sorobanMetrics) const; bool accessesFrozenKey(SorobanNetworkConfig const& cfg) const; #ifdef BUILD_TESTS @@ -311,19 +313,20 @@ class TransactionFrame : public TransactionFrameBase AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, SorobanNetworkConfig const& sorobanConfig, - Hash const& envelopeContentsHash) const; + Hash const& envelopeContentsHash, + SorobanApplyMetrics& sorobanMetrics) const; - void - preParallelApply(AppConnector& app, AbstractLedgerTxn& ltx, - TransactionMetaBuilder& meta, - MutableTransactionResultBase& txResult, - SorobanNetworkConfig const& sorobanConfig) const override; + void preParallelApply(AppConnector& app, AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) const override; std::optional parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, MutableTransactionResultBase& resPayload, - SorobanMetrics& sorobanMetrics, Hash const& sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics, Hash const& sorobanBasePrngSeed, TxEffects& effects) const override; // apply this transaction to the current ledger @@ -333,12 +336,14 @@ class TransactionFrame : public TransactionFrameBase MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, Hash const& sorobanBasePrngSeed, - Hash const& envelopeContentsHash) const; + Hash const& envelopeContentsHash, + SorobanApplyMetrics& sorobanMetrics) const; bool apply(AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, - Hash const& sorobanBasePrngSeed) const override; + Hash const& sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics) const override; // Performs the necessary post-apply transaction processing. // This has to be called after both `processFeeSeqNum` and diff --git a/src/transactions/TransactionFrameBase.h b/src/transactions/TransactionFrameBase.h index cfc112e87c..38b7ae137b 100644 --- a/src/transactions/TransactionFrameBase.h +++ b/src/transactions/TransactionFrameBase.h @@ -32,6 +32,7 @@ class SignatureChecker; class ParallelLedgerInfo; class TxEffects; class ThreadParallelApplyLedgerState; +struct SorobanApplyMetrics; class MutableTransactionResultBase; using MutableTxResultPtr = std::unique_ptr; @@ -154,13 +155,15 @@ class TransactionFrameBase apply(AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, - Hash const& sorobanBasePrngSeed) const = 0; + Hash const& sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics) const = 0; virtual void preParallelApply(AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, - SorobanNetworkConfig const& sorobanConfig) const = 0; + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) const = 0; // If the transaction fails during parallel apply, returns std::nullopt. // Otherwise returns a ParallelTxSuccessVal containing the modified entries @@ -169,7 +172,7 @@ class TransactionFrameBase AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, MutableTransactionResultBase& resPayload, - SorobanMetrics& sorobanMetrics, Hash const& sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics, Hash const& sorobanBasePrngSeed, TxEffects& effects) const = 0; // When validationLedgerSeq is set, ledger sequence precondition diff --git a/src/transactions/test/TransactionTestFrame.cpp b/src/transactions/test/TransactionTestFrame.cpp index 640d8d0d17..649c7524b9 100644 --- a/src/transactions/test/TransactionTestFrame.cpp +++ b/src/transactions/test/TransactionTestFrame.cpp @@ -3,6 +3,7 @@ // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 #include "transactions/test/TransactionTestFrame.h" +#include "ledger/SorobanMetrics.h" #include "transactions/EventManager.h" #include "transactions/MutableTransactionResult.h" #include "transactions/SignatureUtils.h" @@ -36,8 +37,8 @@ TransactionTestFrame::apply( std::optional const& sorobanConfig, Hash const& sorobanBasePrngSeed) { - return mTransactionFrame->apply(app, ltx, meta, *mTransactionTxResult, - sorobanConfig, sorobanBasePrngSeed); + return apply(app, ltx, meta, *mTransactionTxResult, sorobanConfig, + sorobanBasePrngSeed); } void @@ -72,9 +73,23 @@ TransactionTestFrame::apply( MutableTransactionResultBase& txResult, std::optional const& sorobanConfig, Hash const& sorobanBasePrngSeed) const +{ + // Direct test applies run outside of a ledger close, so the apply metrics + // recorded here are simply dropped. + SorobanApplyMetrics sorobanMetrics; + return apply(app, ltx, meta, txResult, sorobanConfig, sorobanBasePrngSeed, + sorobanMetrics); +} + +bool +TransactionTestFrame::apply( + AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + std::optional const& sorobanConfig, + Hash const& sorobanBasePrngSeed, SorobanApplyMetrics& sorobanMetrics) const { auto ret = mTransactionFrame->apply(app, ltx, meta, txResult, sorobanConfig, - sorobanBasePrngSeed); + sorobanBasePrngSeed, sorobanMetrics); mTransactionTxResult = txResult.clone(); return ret; } @@ -367,18 +382,20 @@ void TransactionTestFrame::preParallelApply( AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& resPayload, - SorobanNetworkConfig const& sorobanConfig) const + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) const { mTransactionFrame->preParallelApply(app, ltx, meta, resPayload, - sorobanConfig); + sorobanConfig, sorobanMetrics); } std::optional TransactionTestFrame::parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, - MutableTransactionResultBase& resPayload, SorobanMetrics& sorobanMetrics, - Hash const& txPrngSeed, TxEffects& effects) const + MutableTransactionResultBase& resPayload, + SorobanApplyMetrics& sorobanMetrics, Hash const& txPrngSeed, + TxEffects& effects) const { return mTransactionFrame->parallelApply( app, threadState, config, ledgerInfo, resPayload, sorobanMetrics, diff --git a/src/transactions/test/TransactionTestFrame.h b/src/transactions/test/TransactionTestFrame.h index d04eeee440..0e42d0f018 100644 --- a/src/transactions/test/TransactionTestFrame.h +++ b/src/transactions/test/TransactionTestFrame.h @@ -55,13 +55,22 @@ class TransactionTestFrame : public TransactionFrameBase TransactionFrame const& getRawTransactionFrame() const; TransactionFrameBasePtr getTxFramePtr() const; - // Redefinitions of TransactionFrameBase functions + // Test-only overload that records the apply metrics into an instance that + // is simply dropped. bool apply(AppConnector& app, AbstractLedgerTxn& ltx, TransactionMetaBuilder& meta, MutableTransactionResultBase& txResult, std::optional const& sorobanConfig = std::nullopt, - Hash const& sorobanBasePrngSeed = Hash{}) const override; + Hash const& sorobanBasePrngSeed = Hash{}) const; + + // Redefinitions of TransactionFrameBase functions + bool apply(AppConnector& app, AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& txResult, + std::optional const& sorobanConfig, + Hash const& sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics) const override; MutableTxResultPtr checkValid(AppConnector& app, AbstractLedgerTxn& ltxOuter, @@ -155,17 +164,17 @@ class TransactionTestFrame : public TransactionFrameBase insertKeysForFeeProcessing(UnorderedSet& keys) const override; void insertKeysForTxApply(UnorderedSet& keys) const override; - void - preParallelApply(AppConnector& app, AbstractLedgerTxn& ltx, - TransactionMetaBuilder& meta, - MutableTransactionResultBase& resPayload, - SorobanNetworkConfig const& sorobanConfig) const override; + void preParallelApply(AppConnector& app, AbstractLedgerTxn& ltx, + TransactionMetaBuilder& meta, + MutableTransactionResultBase& resPayload, + SorobanNetworkConfig const& sorobanConfig, + SorobanApplyMetrics& sorobanMetrics) const override; std::optional parallelApply( AppConnector& app, ThreadParallelApplyLedgerState const& threadState, Config const& config, ParallelLedgerInfo const& ledgerInfo, MutableTransactionResultBase& resPayload, - SorobanMetrics& sorobanMetrics, Hash const& sorobanBasePrngSeed, + SorobanApplyMetrics& sorobanMetrics, Hash const& sorobanBasePrngSeed, TxEffects& effects) const override; MutableTxResultPtr