Tcmalloc fixes

Builds/VisualStudio/stellar-core.vcxproj

@@ -908,6 +908,7 @@ exit /b 0
     <ClCompile Include="..\..\src\util\test\Uint128Tests.cpp" />
     <ClCompile Include="..\..\src\util\test\XDRStreamTests.cpp" />
     <ClCompile Include="..\..\src\util\TarjanSCCCalculator.cpp" />
+    <ClCompile Include="..\..\src\util\TcmallocConfig.cpp" />
     <ClCompile Include="..\..\src\util\Thread.cpp" />
     <ClCompile Include="..\..\src\util\TmpDir.cpp" />
     <ClCompile Include="..\..\src\util\Timer.cpp" />
@@ -1280,6 +1281,7 @@ exit /b 0
     <ClInclude Include="..\..\src\util\SociNoWarnings.h" />
     <ClInclude Include="..\..\src\util\StatusManager.h" />
     <ClInclude Include="..\..\src\util\TarjanSCCCalculator.h" />
+    <ClInclude Include="..\..\src\util\TcmallocConfig.h" />
     <ClInclude Include="..\..\src\util\Thread.h" />
     <ClInclude Include="..\..\src\util\TmpDir.h" />
     <ClInclude Include="..\..\src\util\Timer.h" />

Builds/VisualStudio/stellar-core.vcxproj.filters

@@ -489,6 +489,9 @@
     <ClCompile Include="..\..\src\util\TarjanSCCCalculator.cpp">
       <Filter>util</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\util\TcmallocConfig.cpp">
+      <Filter>util</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\util\Thread.cpp">
       <Filter>util</Filter>
     </ClCompile>
@@ -1737,6 +1740,9 @@
     <ClInclude Include="..\..\src\util\TarjanSCCCalculator.h">
       <Filter>util</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\util\TcmallocConfig.h">
+      <Filter>util</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\src\util\Thread.h">
       <Filter>util</Filter>
     </ClInclude>

INSTALL.md

@@ -268,3 +268,15 @@ The GUI depends on the `capstone`, `freetype` and `glfw` libraries and their hea
 
     # On MacOS
     $ brew install capstone freetype2 glfw
+
+On Windows, follow the [build and install instructions](https://github.com/wolfpld/tracy) from the main Tracy site.
+
+At a high level you need to install the required prerequisites to build clients, run in a shell:
+
+    vcpkg.exe integrate install
+    vcpkg.exe install --triplet x64-windows-static capstone freetype glfw3
+
+Then build one of the servers. Solutions for servers compatible with the version of stellar-core can be found under:
+
+    * lib/tracy/profiler/build/win32 (GUI)
+    * lib/tracy/capture/build/win32

common.mk

@@ -31,6 +31,10 @@ if USE_TRACY
 AM_CPPFLAGS += -DUSE_TRACY $(tracy_CFLAGS)
 endif # USE_TRACY
 
+if USE_TRACY_MEMORY_TRACKING
+AM_CPPFLAGS += -DUSE_TRACY_MEMORY_TRACKING
+endif # USE_TRACY_MEMORY_TRACKING
+
 if BUILD_TESTS
 AM_CPPFLAGS += -DBUILD_TESTS=1
 endif # BUILD_TESTS

configure.ac

@@ -470,8 +470,13 @@ AM_CONDITIONAL(USE_POSTGRES, [test -n "$have_postgres"])
 # tcmalloc_minimal - enabled by default on Linux for improved memory allocation performance
 # Disabled when using sanitizers (asan, memcheck, tsan) as they replace malloc
 # Uses vendored gperftools from lib/gperftools
+AC_ARG_ENABLE(tcmalloc,
+    AS_HELP_STRING([--disable-tcmalloc],
+        [Disable tcmalloc support]))
 unset have_tcmalloc
-if test "x$enable_asan" = "xyes" -o "x$enable_memcheck" = "xyes" -o "x$enable_threadsanitizer" = "xyes"; then
+if test x"$enable_tcmalloc" = xno; then
+    AC_MSG_NOTICE([tcmalloc disabled by user])
+elif test "x$enable_asan" = "xyes" -o "x$enable_memcheck" = "xyes" -o "x$enable_threadsanitizer" = "xyes"; then
     AC_MSG_NOTICE([tcmalloc disabled due to sanitizer being enabled])
 else
     case "${host_os}" in
@@ -529,7 +534,12 @@ AC_ARG_ENABLE(tracy-memory-tracking,
     AS_HELP_STRING([--enable-tracy-memory-tracking],
         [Enable 'tracy' profiler/tracer memory tracking code (slow)]))
 AM_CONDITIONAL(USE_TRACY_MEMORY_TRACKING, [test x$enable_tracy_memory_tracking = xyes])
-
+if test x"$enable_tracy_memory_tracking" = xyes -a x"$have_tcmalloc" = x1; then
+  AC_MSG_ERROR([--enable-tracy-memory-tracking requires --disable-tcmalloc])
+fi
+if test x"$enable_tracy_memory_tracking" = xyes -a x"$enable_tracy" != xyes; then
+  AC_MSG_ERROR([--enable-tracy-memory-tracking requires --enable-tracy])
+fi
 if test x"$enable_tracy" = xyes -a x"$enable_asan" = xyes; then
        AC_MSG_ERROR([--enable-asan is not compatible with --enable-tracy])
 fi

docs/metrics.md

@@ -284,7 +284,8 @@ soroban.config.bucket-list-target-size-byte  | counter   | soroban config settin
 soroban.module-cache.num-entries             | counter   | current number of entries in module cache
 soroban.module-cache.compilation-time        | timer     | times each contract compilation when adding to module cache
 soroban.module-cache.rebuild-time            | timer     | times each rebuild of module cache (including all compilations)
-soroban.module-cache.rebuild-bytes           | counter   | bytes of WASM bytecode compiled in last rebuild of module cache
+soroban.module-cache.rebuild-wasm-bytes      | counter   | bytes of WASM bytecode compiled in last rebuild of module cache
+soroban.module-cache.rebuild-heap-bytes      | counter   | bytes of heap memory allocated in last rebuild of module cache
 soroban.in-memory-state.contract-code-size   | counter   | size in bytes of non-evicted ContractCode entries according to memory cost model
 soroban.in-memory-state.contract-data-size   | counter   | size in bytes of ContractData entries in memory
 soroban.in-memory-state.contract-code-entries   | counter   | number of ContractCode entries in memory

performance-eval/performance-eval.md

@@ -294,28 +294,84 @@ solution is, as root to run
 
 Stellar-core has built-in support for Tracy traces.
 
-To install the visualizer, follow the [build and install instructions](https://github.com/wolfpld/tracy) from the main Tracy site.
+To install the visualizer, follow the directions in [INSTALL.md](../INSTALL.md).
 
-At a high level you need to
+### General Visual Studio profiler
 
-install the required pre-requesites to build clients, run in a shell:
+The main page for the profiler built into Visual Studio Community Edition is located there:  https://docs.microsoft.com/en-us/visualstudio/profiling/index
 
-    vcpkg.exe integrate install
-    vcpkg.exe install --triplet x64-windows-static capstone freetype glfw3
+## All platforms
 
-Then build one of the servers.
+Intel V-Tune (free, unlimited license 90 days renewal) https://software.intel.com/en-us/system-studio/choose-download
 
-Solutions for servers compatible with the version of stellar-core can be found under:
+# Memory-use profiling
 
-    * lib/tracy/profiler/build/win32 (GUI)
-    * lib/tracy/capture/build/win32
+Tools for memory use profiling are less well-developed than CPU profiling, but there are some options available.
 
-Note: when connecting, use `localhost` instead of `127.0.0.1` as Tracy binds by default to IPV6 addresses.
+## Tracy
 
-### General Visual Studio profiler
+Tracy has some built-in support for memory profiling, but turning it on will
+slow down core significantly and will use memory in the tracy client _very
+quickly_, easily overwhelming your workstation if you're not careful. So you can
+usually only turn it on for a brief period of time.
 
-The main page for the profiler built into Visual Studio Community Edition is located there:  https://docs.microsoft.com/en-us/visualstudio/profiling/index
+It is most useful for examining a small part of the code for a short period of
+time, where you already have a fairly good idea of there being memory allocation
+issues that you want to see a precise accounting of. Allocations get linked to
+zones (as a list in each zone detail view) and are available for inspection in
+the "memory" window, along with a total map of memory and a list of all
+allocations.
 
-## All platforms
+Stellar-core has support for this mode separate from normal tracy tracing,
+because it is so performance intensive and memory hungry. You need to configure
+with --enable-tracy-memory-tracking and --disable-tcmalloc.
+
+## Heaptrack
+
+A better option for a high level "profile" of memory is the "heaptrack" tool,
+which is available on Linux.
+
+    $ sudo apt install heaptrack heaptrack-gui
+
+To use it you will also need to configure with --disable-tcmalloc, because it
+works by intercepting malloc/free calls underlying the default operator
+new/delete, and tcmalloc's operator new and delete will bypass that
+interception.
+
+Heaptrack should also be run only for a moderate amount of time, otherwise the
+recording will be huge. But it at least writes its recording to disk, and the
+recording is much more compact than tracy's in-memory structure, so it can run
+much longer than tracy in memory-recording mode without issue.
+
+Heaptrack can run a program as a subprocess or attach remotely. The remote
+attach mode allows you to avoid starting it until the program is close to the
+period you want to measure, so is recommended. You will need to enable ptrace
+permissions.
+
+Heaptrack's default recording mode is very slow as it symbolicates all the
+stacks while it runs. A better way is to record a _raw_ profile and then
+symbolicate the data after the fact.
+
+Combining these facts, the best execution we've found is like the following:
+
+    # in one terminal...
+    $ stellar-core ...
+
+    # in another terminal...
+    $ echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+    $ heaptrack --raw $(pidof stellar-core)
+
+    # switch back core and stop it with Ctrl-C when done
+
+    # heaptrack will exit and write a file like
+    # heaptrack.stellar-core.12345.raw.zst along with, hopefully, instructions
+    # to run something like this to post-process the raw file into a more
+    # compact and symbolicated form. This will run a long time:
+
+    $ zstd -dc < ".../heaptrack.stellar-core.12345.raw.zst" \
+        | /usr/lib/heaptrack/libexec/heaptrack_interpret \
+        | zstd -c > ".../heaptrack.stellar-core.12345.zst"
+
+    # finally load the profile into the visualization tool
+    $ heaptrack_gui ".../heaptrack.stellar-core.12345.zst"
 
-Intel V-Tune (free, unlimited license 90 days renewal) https://software.intel.com/en-us/system-studio/choose-download

src/ledger/LedgerManagerImpl.cpp

@@ -1028,8 +1028,10 @@ LedgerManagerImpl::ApplyState::finishPendingCompilation()
     releaseAssert(mPhase == Phase::SETTING_UP_STATE);
     releaseAssert(mCompiler);
     auto newCache = mCompiler->wait();
-    getMetrics().mSorobanMetrics.mModuleCacheRebuildBytes.set_count(
+    getMetrics().mSorobanMetrics.mModuleCacheRebuildWasmBytes.set_count(
         (int64)mCompiler->getBytesCompiled());
+    getMetrics().mSorobanMetrics.mModuleCacheRebuildHeapBytes.set_count(
+        mCompiler->getBytesAllocatedDuringCompilation());
     getMetrics().mSorobanMetrics.mModuleCacheNumEntries.set_count(
         (int64)mCompiler->getContractsCompiled());
     getMetrics().mSorobanMetrics.mModuleCacheRebuildTime.Update(
@@ -1175,7 +1177,7 @@ LedgerManagerImpl::ApplyState::maybeRebuildModuleCache(
     // contract-set in the live BL as an event that warrants a rebuild.
 
     int64_t lastCompiledWasmBytesCount =
-        getMetrics().mSorobanMetrics.mModuleCacheRebuildBytes.count();
+        getMetrics().mSorobanMetrics.mModuleCacheRebuildWasmBytes.count();
     uint64_t lastCompiledWasmBytes =
         lastCompiledWasmBytesCount < 0
             ? 0

src/ledger/SharedModuleCacheCompiler.cpp

@@ -7,6 +7,7 @@
 #include "crypto/SHA.h"
 #include "rust/RustBridge.h"
 #include "util/Logging.h"
+#include "util/TcmallocConfig.h"
 #include "xdr/Stellar-ledger-entries.h"
 #include <chrono>
 #include <cstddef>
@@ -139,6 +140,8 @@ SharedModuleCacheCompiler::start()
 {
     mStarted = std::chrono::steady_clock::now();
 
+    mHeapSizeAtStart = getMallocBytesInUse();
+
     LOG_INFO(DEFAULT_LOG,
              "Launching 1 loading and {} compiling background threads",
              mNumThreads - 1);
@@ -204,13 +207,20 @@ SharedModuleCacheCompiler::wait()
     auto end = std::chrono::steady_clock::now();
     LOG_INFO(
         DEFAULT_LOG,
-        "Compiled {} contracts ({} bytes of Wasm) in {}ms real time, {}ms "
+        "Compiled {} contracts ({} of Wasm) in {}ms real time, {}ms "
         "CPU time",
-        mContractsCompiled, mBytesCompiled,
+        mContractsCompiled, formatSize(mBytesCompiled),
         std::chrono::duration_cast<std::chrono::milliseconds>(end - mStarted)
             .count(),
         std::chrono::duration_cast<std::chrono::milliseconds>(mTotalCompileTime)
             .count());
+    mHeapSizeAtEnd = getMallocBytesInUse();
+    int64_t heapDiff = static_cast<int64_t>(mHeapSizeAtEnd) -
+                       static_cast<int64_t>(mHeapSizeAtStart);
+    LOG_INFO(DEFAULT_LOG,
+             "Heap changed from {} to {} during compilation ({} difference)",
+             formatSize(mHeapSizeAtStart), formatSize(mHeapSizeAtEnd),
+             formatSize(heapDiff));
     return mModuleCache->shallow_clone();
 }
 
@@ -221,6 +231,14 @@ SharedModuleCacheCompiler::getBytesCompiled()
     return mBytesCompiled * mLedgerVersions.size();
 }
 
+int64_t
+SharedModuleCacheCompiler::getBytesAllocatedDuringCompilation()
+{
+    std::unique_lock lock(mMutex);
+    return static_cast<int64_t>(mHeapSizeAtEnd) -
+           static_cast<int64_t>(mHeapSizeAtStart);
+}
+
 std::chrono::nanoseconds
 SharedModuleCacheCompiler::getCompileTime()
 {

src/ledger/SharedModuleCacheCompiler.h

@@ -44,6 +44,8 @@ class SharedModuleCacheCompiler : NonMovableOrCopyable
     std::condition_variable mHaveContracts;
 
     std::chrono::steady_clock::time_point mStarted;
+    size_t mHeapSizeAtStart{0};
+    size_t mHeapSizeAtEnd{0};
     std::chrono::nanoseconds mTotalCompileTime{0};
 
     void setFinishedLoading(size_t nContracts);
@@ -61,6 +63,7 @@ class SharedModuleCacheCompiler : NonMovableOrCopyable
     void start();
     ::rust::Box<stellar::rust_bridge::SorobanModuleCache> wait();
     size_t getBytesCompiled();
+    int64_t getBytesAllocatedDuringCompilation();
     std::chrono::nanoseconds getCompileTime();
     size_t getContractsCompiled();
 };

src/ledger/SorobanMetrics.cpp

@@ -148,8 +148,10 @@ SorobanMetrics::SorobanMetrics(MetricsRegistry& metrics)
           metrics.NewTimer({"soroban", "module-cache", "compilation-time"}))
     , mModuleCacheRebuildTime(
           metrics.NewTimer({"soroban", "module-cache", "rebuild-time"}))
-    , mModuleCacheRebuildBytes(
-          metrics.NewCounter({"soroban", "module-cache", "rebuild-bytes"}))
+    , mModuleCacheRebuildWasmBytes(
+          metrics.NewCounter({"soroban", "module-cache", "rebuild-wasm-bytes"}))
+    , mModuleCacheRebuildHeapBytes(
+          metrics.NewCounter({"soroban", "module-cache", "rebuild-heap-bytes"}))
     , mContractCodeStateSize(metrics.NewCounter(
           {"soroban", "in-memory-state", "contract-code-size"}))
     , mContractDataStateSize(metrics.NewCounter(

src/ledger/SorobanMetrics.h

@@ -193,7 +193,8 @@ class SorobanMetrics
     medida::Counter& mModuleCacheNumEntries;
     medida::Timer& mModuleCompilationTime;
     medida::Timer& mModuleCacheRebuildTime;
-    medida::Counter& mModuleCacheRebuildBytes;
+    medida::Counter& mModuleCacheRebuildWasmBytes;
+    medida::Counter& mModuleCacheRebuildHeapBytes;
 
     // In-memory state metrics
     medida::Counter& mContractCodeStateSize;

src/main/main.cpp

@@ -345,6 +345,10 @@ checkStellarCoreMajorVersionProtocolIdentity()
 
 #ifdef USE_TRACY_MEMORY_TRACKING
 
+#ifndef USE_TRACY
+#error "USE_TRACY_MEMORY_TRACKING requires USE_TRACY"
+#endif
+
 #ifdef __has_feature
 #if __has_feature(address_sanitizer)
 #define ASAN_ENABLED

src/transactions/test/InvokeHostFunctionTests.cpp

@@ -7615,7 +7615,7 @@ TEST_CASE("module cache rebuild on incremental wasm uploads",
     REQUIRE(wasmsAreCached(*app, initialHashes));
 
     auto& metrics = app->getLedgerManager().getSorobanMetrics();
-    auto rebuildBytesAtStartup = metrics.mModuleCacheRebuildBytes.count();
+    auto rebuildBytesAtStartup = metrics.mModuleCacheRebuildWasmBytes.count();
     REQUIRE(rebuildBytesAtStartup > 0);
 
     auto uploader = app->getRoot();
@@ -7644,7 +7644,8 @@ TEST_CASE("module cache rebuild on incremental wasm uploads",
         // ledger close at apply start.
         closeLedger(*app);
 
-        if (metrics.mModuleCacheRebuildBytes.count() != rebuildBytesAtStartup)
+        if (metrics.mModuleCacheRebuildWasmBytes.count() !=
+            rebuildBytesAtStartup)
         {
             rebuilt = true;
             uploadedRawAtTrigger = uploadedRawBytes;

src/util/TcmallocConfig.cpp

@@ -2,11 +2,27 @@
 // under the Apache License, Version 2.0. See the COPYING file at the root
 // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0
 
+#include "util/TcmallocConfig.h"
 #include "config.h"
 
 #ifdef USE_TCMALLOC
 #include <gperftools/malloc_extension.h>
 
+namespace stellar
+{
+size_t
+getMallocBytesInUse()
+{
+    size_t out;
+    MallocExtension* ext = MallocExtension::instance();
+    if (ext && ext->GetNumericProperty("generic.current_allocated_bytes", &out))
+    {
+        return out;
+    }
+    return 0;
+}
+}
+
 namespace
 {
 // Configure tcmalloc parameters at startup using a constructor attribute.
@@ -44,4 +60,13 @@ initTcmallocConfig()
     }
 }
 } // namespace
+#else
+namespace stellar
+{
+size_t
+getMallocBytesInUse()
+{
+    return 0;
+}
+}
 #endif // USE_TCMALLOC