Restore priming

ddprof-lib/src/main/cpp/context_api.cpp

@@ -19,7 +19,7 @@
 #include "guards.h"
 #include "otel_context.h"
 #include "profiler.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
 #include <cstring>
 
 /**

ddprof-lib/src/main/cpp/counters.h

@@ -134,6 +134,7 @@
   X(SAFECOPY_FAILED, "safecopy_failed")                                       \
   X(SAFEFETCH_FAILED, "safefetch_failed")                                     \
   X(STACKWALK_LONGJMP_RECOVERED, "stackwalk_longjmp_recovered")               \
+  X(SAMPLES_DROPPED_TLS_POOL_EXHAUSTED, "thread_local_pool_exhausted")        \
   DD_COUNTER_TABLE_FAULT_INJECTION(X)                                          \
   DD_COUNTER_TABLE_FI_DEBUG(X)                                                 \
   DD_COUNTER_TABLE_DEBUG(X)

ddprof-lib/src/main/cpp/ctimer_linux.cpp

@@ -27,6 +27,7 @@
 #include "log.h"
 #include "profiler.h"
 #include "signalCookie.h"
+#include "threadLocalData.inline.h"
 #include "threadState.inline.h"
 #include <assert.h>
 #include <errno.h>

ddprof-lib/src/main/cpp/faultInjection.cpp

@@ -22,7 +22,7 @@
 
 #include "counters.h"          // Counters::increment (FAULTS_INJECTED)
 #include "os.h"                // OS::page_size
-#include "threadLocalData.h"   // ProfiledThread::current / nextFiRandom
+#include "threadLocalData.inline.h"   // ProfiledThread::current / nextFiRandom
 #include <atomic>
 #include <sys/mman.h>
 

ddprof-lib/src/main/cpp/faultInjection.h

@@ -34,8 +34,9 @@
 //
 //   return INJECT_FAULT_BOOL_LIKELY(dlopen(name, flags) != nullptr);
 //
-// The three tiers name their firing frequency: RARE 0.01%, UNLIKELY 0.1%,
-// LIKELY 1%.  See faultInjection.cpp for the poison-address and PRNG details.
+// The four tiers name their firing frequency: RARE 0.01%, UNLIKELY 0.1%,
+// LIKELY 1%, HIGH 10%.  See faultInjection.cpp for the poison-address and PRNG
+// details.
 
 #ifndef _FAULT_INJECTION_H
 #define _FAULT_INJECTION_H
@@ -56,6 +57,7 @@ namespace faultinj {
 constexpr u64 PROB_RARE     = 1844674407370955ULL;    // 1e-4  (0.01%)
 constexpr u64 PROB_UNLIKELY = 18446744073709552ULL;   // 1e-3  (0.1%)
 constexpr u64 PROB_LIKELY   = 184467440737095520ULL;  // 1e-2  (1%)
+constexpr u64 PROB_HIGH     = 1844674407370955162ULL; // 1e-1  (10%)
 
 // Called once at profiler startup (off the signal path) to mmap the PROT_NONE
 // guard region used by poisonAddress().  Safe to call before any injection.
@@ -106,31 +108,39 @@ inline T injectValue(T orig, T faulty, u64 threshold, const char* fn) {
     ::faultinj::injectAddress((ptr), ::faultinj::PROB_UNLIKELY, __func__)
 #define INJECT_FAULT_ADDRESS_LIKELY(ptr) \
     ::faultinj::injectAddress((ptr), ::faultinj::PROB_LIKELY, __func__)
+#define INJECT_FAULT_ADDRESS_HIGH(ptr) \
+    ::faultinj::injectAddress((ptr), ::faultinj::PROB_HIGH, __func__)
 
 #define INJECT_FAULT_BOOL_RARE(v) \
     ::faultinj::injectValue((v), false, ::faultinj::PROB_RARE, __func__)
 #define INJECT_FAULT_BOOL_UNLIKELY(v) \
     ::faultinj::injectValue((v), false, ::faultinj::PROB_UNLIKELY, __func__)
 #define INJECT_FAULT_BOOL_LIKELY(v) \
     ::faultinj::injectValue((v), false, ::faultinj::PROB_LIKELY, __func__)
+#define INJECT_FAULT_BOOL_HIGH(v) \
+    ::faultinj::injectValue((v), false, ::faultinj::PROB_HIGH, __func__)
 
 #else  // __FAULT_INJECTION__ not defined — strict identity, zero cost.
 
 #define INJECT_FAULT_ADDRESS_RARE(ptr)     (ptr)
 #define INJECT_FAULT_ADDRESS_UNLIKELY(ptr) (ptr)
 #define INJECT_FAULT_ADDRESS_LIKELY(ptr)   (ptr)
+#define INJECT_FAULT_ADDRESS_HIGH(ptr)     (ptr)
 
 #define INJECT_FAULT_INT_RARE(v)     (v)
 #define INJECT_FAULT_INT_UNLIKELY(v) (v)
 #define INJECT_FAULT_INT_LIKELY(v)   (v)
+#define INJECT_FAULT_INT_HIGH(v)     (v)
 
 #define INJECT_FAULT_LONG_RARE(v)     (v)
 #define INJECT_FAULT_LONG_UNLIKELY(v) (v)
 #define INJECT_FAULT_LONG_LIKELY(v)   (v)
+#define INJECT_FAULT_LONG_HIGH(v)     (v)
 
 #define INJECT_FAULT_BOOL_RARE(v)     (v)
 #define INJECT_FAULT_BOOL_UNLIKELY(v) (v)
 #define INJECT_FAULT_BOOL_LIKELY(v)   (v)
+#define INJECT_FAULT_BOOL_HIGH(v)     (v)
 
 #define NO_INJECTION_ASSERT(a) (assert(a))
 

ddprof-lib/src/main/cpp/flightRecorder.cpp

@@ -28,6 +28,7 @@
 #include "unwindStats.h"
 #include "symbols.h"
 #include "threadFilter.h"
+#include "threadLocalData.inline.h"
 #include "threadState.h"
 #include "tsc.h"
 #include "hotspot/vmStructs.h"

ddprof-lib/src/main/cpp/guards.cpp

@@ -17,7 +17,7 @@
 #include "guards.h"
 #include "common.h"
 #include "os.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
 
 // Signal-context tracking — backed by ProfiledThread::_signal_depth; see
 // the comment block in guards.h for the rationale (initial-exec TLS was

ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp

@@ -18,6 +18,7 @@
 #include "profiler.h"
 #include "stackWalker.inline.h"
 #include "threadLocal.h"
+#include "threadLocalData.inline.h"
 
 using StackWalkValidation::inDeadZone;
 using StackWalkValidation::aligned;
@@ -240,7 +241,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
     // VMStructs is only available for hotspot JVM 
     assert(VM::isHotspot());
 
-    ProfiledThread* prof_thread = ProfiledThread::current();
+    ProfiledThread* prof_thread = ProfiledThread::acquireCurrent();
     if (prof_thread == nullptr) {
         Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL);
         return 0;
@@ -1210,12 +1211,16 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) {
   // AsyncGetCallTrace. Install a jmp ctx here too, so a SIGSEGV anywhere in
   // walkJavaStack, except HotSpot's AsyncGetCallTrace call, is caught by
   // Profiler::checkFault() and siglongjmp'd back here instead of crashing the process.
-  ProfiledThread* prof_thread = ProfiledThread::current();
-  const bool prev_unwinding_java = prof_thread != nullptr ? prof_thread->is_unwinding_Java() : false;
+  ProfiledThread* prof_thread = ProfiledThread::acquireCurrent();
+  if (prof_thread == nullptr) {
+    Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL);
+    return 0;
+  }
+  const bool prev_unwinding_java = prof_thread->is_unwinding_Java();
   sigjmp_buf crash_protection_ctx;
-  sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr;
+  sigjmp_buf* prev_jmp_buf = prof_thread->getJmpCtx();
 
-  if (prof_thread != nullptr && sigsetjmp(crash_protection_ctx, 1) != 0) {
+  if (sigsetjmp(crash_protection_ctx, 1) != 0) {
     // checkFault() does a siglongjmp from inside segvHandler, bypassing
     // segvHandler's SignalHandlerScope destructor. Compensate.
     SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP();
@@ -1228,9 +1233,7 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) {
     }
     return java_frames;
   }
-  if (prof_thread != nullptr) {
-    prof_thread->setJmpCtx(&crash_protection_ctx);
-  }
+  prof_thread->setJmpCtx(&crash_protection_ctx);
 
   if (features.mixed) {
     java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated);
@@ -1284,9 +1287,7 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) {
     }
   }
 
-  if (prof_thread != nullptr) {
-    prof_thread->setJmpCtx(prev_jmp_buf);
-  }
+  prof_thread->setJmpCtx(prev_jmp_buf);
   return java_frames;
 }
 

ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp

@@ -15,7 +15,7 @@
 #include "jvmThread.h"
 #include "safeAccess.h"
 #include "spinLock.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
 #include "threadState.h"
 
 CodeCache* VMStructs::_libjvm = nullptr;

ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h

@@ -11,7 +11,7 @@
 #include "hotspot/vmStructs.h"
 #include "jvmThread.h"
 #include "safeAccess.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
 
 inline bool crashProtectionActive() {
     ProfiledThread* pt = ProfiledThread::current();

ddprof-lib/src/main/cpp/itimer.cpp

@@ -22,7 +22,7 @@
 #include "profiler.h"
 #include "signalInflight.h"
 #include "stackWalker.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
 #include "threadState.inline.h"
 #include "guards.h"
 #include <sys/time.h>

ddprof-lib/src/main/cpp/javaApi.cpp

@@ -31,7 +31,7 @@
 #include "os.h"
 #include "otel_process_ctx.h"
 #include "profiler.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
 #include "tsc.h"
 #include "vmEntry.h"
 #include <errno.h>

ddprof-lib/src/main/cpp/jvmSupport.cpp

@@ -6,10 +6,12 @@
 #include "jvmSupport.inline.h"
 
 #include "asyncSampleMutex.h"
+#include "common.h"
 #include "frames.h"
 #include "os.h"
 #include "profiler.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
+#include "threadLocalDataPool.h"
 #include "vmEntry.h"
 
 #include "hotspot/hotspotSupport.h"
@@ -38,6 +40,12 @@ bool JVMSupport::initialize() {
         return false;
     }
 
+    if (ProfiledThread::supportPriming()) {
+        ThreadLocalDataPool::initialize();
+    } else {
+        LOG_WARN("Thread priming is not supported");
+    }
+
     // Check ProfiledThread key, it is critical for storing per-thread metadata
     return ProfiledThread::isThreadKeyValid();
 }

ddprof-lib/src/main/cpp/perfEvents_linux.cpp

@@ -35,7 +35,7 @@
 #include "stackFrame.h"
 #include "stackWalker.h"
 #include "symbols.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
 #include "threadState.inline.h"
 #include <dlfcn.h>
 #include <errno.h>

ddprof-lib/src/main/cpp/profiler.cpp

@@ -35,6 +35,7 @@
 #include "stackFrame.h"
 #include "stackWalker.h"
 #include "symbols.h"
+#include "threadLocalData.inline.h"
 #include "tsc.h"
 #include "utils.h"
 #include "wallClock.h"
@@ -865,10 +866,10 @@ bool Profiler::prewarmUnwinder() {
   // libgcc_s.so.1 has been the stable SONAME since 2002; a bump would
   // constitute a glibc/GCC C++ ABI break and is treated as a fixed contract.
   //
-  // INJECT_FAULT_BOOL_LIKELY lets fault-injection builds force this to
+  // INJECT_FAULT_BOOL_HIGH lets fault-injection builds force this to
   // report failure without the library actually being absent, so
   // checkState()'s "Missing libgcc_s.so" path can be exercised in CI.
-  return INJECT_FAULT_BOOL_LIKELY(dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL) != nullptr);
+  return INJECT_FAULT_BOOL_HIGH(dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL) != nullptr);
 #else
   return true;
 #endif

ddprof-lib/src/main/cpp/refCountGuard.cpp

@@ -10,7 +10,7 @@
 #include "log.h"
 #include "os.h"
 #include "primeProbing.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
 #include <atomic>
 #include <time.h>
 

ddprof-lib/src/main/cpp/safeAccess.cpp

@@ -23,7 +23,7 @@
 #include <signal.h>
 #include <ucontext.h>
 #ifdef DEBUG
-#include "threadLocalData.h"  // ProfiledThread::current / isProtected
+#include "threadLocalData.inline.h"  // ProfiledThread::current / isProtected
 #endif
 
 extern "C" int safefetch32_cont(int* adr, int errValue);

ddprof-lib/src/main/cpp/signalSafety.h

@@ -18,7 +18,7 @@
 #define _SIGNAL_SAFETY_H
 
 #include "guards.h"   // isInSignalContext, SIGNAL_HANDLER_GUARD, ...
-#include "threadLocalData.h"   // ProfiledThread::current
+#include "threadLocalData.inline.h"   // ProfiledThread::current
 
 // Detect ASAN using compiler-provided macros so the ASAN_ENABLED guard below
 // works in every TU that includes this header, independent of include order.

ddprof-lib/src/main/cpp/stackWalker.cpp

@@ -13,7 +13,7 @@
 #include "symbols.h"
 #include "jvmSupport.inline.h"
 #include "jvmThread.h"
-#include "threadLocalData.h"
+#include "threadLocalData.inline.h"
 
 // Use validation helpers from header (shared with tests)
 using StackWalkValidation::inDeadZone;
@@ -45,11 +45,15 @@ int StackWalker::walkFP(void* ucontext, const void** callchain, int max_depth, S
     // falls inside this library while a jmp ctx is installed gets caught by
     // Profiler::checkFault() from the SEGV handler and siglongjmp'd back here,
     // instead of crashing the process.
-    ProfiledThread* prof_thread = ProfiledThread::current();
+    ProfiledThread* prof_thread = ProfiledThread::acquireCurrent();
+    if (prof_thread == nullptr) {
+        Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL);
+        return 0;
+    }
     sigjmp_buf crash_protection_ctx;
-    sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr;
+    sigjmp_buf* prev_jmp_buf = prof_thread->getJmpCtx();
 
-    if (prof_thread != nullptr && sigsetjmp(crash_protection_ctx, 1) != 0) {
+    if (sigsetjmp(crash_protection_ctx, 1) != 0) {
         // checkFault() does a siglongjmp from inside segvHandler, bypassing
         // segvHandler's SignalHandlerScope destructor. Compensate.
         SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP();
@@ -62,9 +66,7 @@ int StackWalker::walkFP(void* ucontext, const void** callchain, int max_depth, S
         }
         return depth;
     }
-    if (prof_thread != nullptr) {
-        prof_thread->setJmpCtx(&crash_protection_ctx);
-    }
+    prof_thread->setJmpCtx(&crash_protection_ctx);
 
     // Walk until the bottom of the stack or until the first Java frame
     while (depth < actual_max_depth) {
@@ -132,11 +134,16 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth
     // falls inside this library while a jmp ctx is installed gets caught by
     // Profiler::checkFault() from the SEGV handler and siglongjmp'd back here,
     // instead of crashing the process.
-    ProfiledThread* prof_thread = ProfiledThread::current();
+    ProfiledThread* prof_thread = ProfiledThread::acquireCurrent();
+    if (prof_thread == nullptr) {
+        Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL);
+        return 0;
+    }
+
     sigjmp_buf crash_protection_ctx;
-    sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr;
+    sigjmp_buf* prev_jmp_buf = prof_thread->getJmpCtx();
 
-    if (prof_thread != nullptr && sigsetjmp(crash_protection_ctx, 1) != 0) {
+    if (sigsetjmp(crash_protection_ctx, 1) != 0) {
         // checkFault() does a siglongjmp from inside segvHandler, bypassing
         // segvHandler's SignalHandlerScope destructor. Compensate.
         SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP();
@@ -149,9 +156,7 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth
         }
         return depth;
     }
-    if (prof_thread != nullptr) {
-        prof_thread->setJmpCtx(&crash_protection_ctx);
-    }
+    prof_thread->setJmpCtx(&crash_protection_ctx);
 
     // Walk until the bottom of the stack or until the first Java frame
     while (depth < actual_max_depth) {
@@ -229,9 +234,7 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth
         }
     }
 
-    if (prof_thread != nullptr) {
-        prof_thread->setJmpCtx(prev_jmp_buf);
-    }
+    prof_thread->setJmpCtx(prev_jmp_buf);
 
     if (truncated && depth > max_depth) {
         *truncated = true;

ddprof-lib/src/main/cpp/threadLocal.h

@@ -86,6 +86,10 @@ class ThreadLocal {
         return _key != INVALID_KEY;
     }
 
+    pthread_key_t key() const {
+        return _key;
+    }
+
     /**
      * set(nullptr) will result in the value being recreated when get() is called
      * when CREATE_FUNC is not nullptr.