diff --git a/AGENTS.md b/AGENTS.md index c69f02df4..201993abe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -288,6 +288,11 @@ The profiler uses a sophisticated double-buffered storage system for call traces - **Atomic Operations**: Instance ID management and counter updates use atomics - **Memory Allocation**: Minimize malloc() in hot paths, use pre-allocated containers +### Sampler Safety +- **Stack walker**: `HotspotSupport::walkVM()`, `StackWalker::walkDwarf()`, and `StackWalker::walkFP()` must be protected by `sigsetjmp()`/`siglongjmp()`. +- **Samplers**: Every sampler must set up the `ProfiledThread` thread-local before sampling, and skip the sample if it isn't available. Signal-based samplers use `ProfiledThread::acquireCurrent()`; non-signal-based samplers use `ProfiledThread::initCurrentThreadSignalSafe()`. +- **JNI/JVMTI callbacks**: Use `ProfiledThread::initCurrentThreadSignalSafe()` to set up `ProfiledThread` for the thread. + ### Atomic Memory Ordering (Critical for arm64) arm64 has a weakly-ordered memory model (unlike x86 TSO). Incorrect ordering causes real lockups on arm64 that never reproduce on x86. - **Cross-thread reads**: Always use `__ATOMIC_ACQUIRE` for loads that must see stores from another thread. Never use `__ATOMIC_RELAXED` for cross-thread visibility unless you can prove no ordering dependency exists. diff --git a/ddprof-lib/src/main/cpp/context_api.cpp b/ddprof-lib/src/main/cpp/context_api.cpp index 082131a92..8a4f556f5 100644 --- a/ddprof-lib/src/main/cpp/context_api.cpp +++ b/ddprof-lib/src/main/cpp/context_api.cpp @@ -19,7 +19,7 @@ #include "guards.h" #include "otel_context.h" #include "profiler.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include /** diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h index a3b3ea34f..6903e9532 100644 --- a/ddprof-lib/src/main/cpp/counters.h +++ b/ddprof-lib/src/main/cpp/counters.h @@ -134,6 +134,7 @@ X(SAFECOPY_FAILED, "safecopy_failed") \ X(SAFEFETCH_FAILED, "safefetch_failed") \ X(STACKWALK_LONGJMP_RECOVERED, "stackwalk_longjmp_recovered") \ + X(SAMPLES_DROPPED_TLS_POOL_EXHAUSTED, "thread_local_pool_exhausted") \ /* writeElement() guards against a corrupted/dangling JfrMetadata tree. \ * Root cause is still unconfirmed, so these counters are the durable \ * signal for spotting a recurrence. */ \ diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 11502c488..e3fd527f1 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -27,6 +27,7 @@ #include "log.h" #include "profiler.h" #include "signalCookie.h" +#include "threadLocalData.inline.h" #include "threadState.inline.h" #include #include @@ -225,19 +226,19 @@ void CTimerJvmti::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { return; } int tid = 0; - ProfiledThread *current = ProfiledThread::current(); - assert(current == nullptr || !current->isDeepCrashHandler()); - if (current != nullptr && JVMThread::current() == nullptr + ProfiledThread *current = SIGNAL_HANDLER_CURRENT_THREAD(); + assert(current != nullptr); + + if (JVMThread::current() == nullptr && current->inInitWindow()) { current->tickInitWindow(); errno = saved_errno; return; } + if (current != NULL) { current->noteCPUSample(Profiler::instance()->recordingEpoch()); tid = current->tid(); - } else { - tid = OS::threadId(); } Shims::instance().setSighandlerTid(tid); @@ -267,6 +268,8 @@ void CTimer::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { Counters::increment(CTIMER_SIGNAL_OWN); InflightGuard inflight; + ProfiledThread* current = SIGNAL_HANDLER_CURRENT_THREAD(); + assert(current != nullptr); // Atomically try to enter critical section - prevents all reentrancy races CriticalSection cs; @@ -280,25 +283,18 @@ void CTimer::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { if (!__atomic_load_n(&_enabled, __ATOMIC_ACQUIRE)) { return; } - int tid = 0; - ProfiledThread *current = ProfiledThread::current(); - assert(current == nullptr || !current->isDeepCrashHandler()); + assert(!current->isDeepCrashHandler()); // Guard against the race window between Profiler::registerThread() and // thread_native_entry setting JVM TLS (PROF-13072): skip at most one signal // per thread. Pure native threads (where JVMThread::current() is always null) // are allowed through once the one-shot window expires. - if (current != nullptr && JVMThread::current() == nullptr - && current->inInitWindow()) { + if (JVMThread::current() == nullptr && current->inInitWindow()) { current->tickInitWindow(); errno = saved_errno; return; } - if (current != NULL) { - current->noteCPUSample(Profiler::instance()->recordingEpoch()); - tid = current->tid(); - } else { - tid = OS::threadId(); - } + current->noteCPUSample(Profiler::instance()->recordingEpoch()); + int tid = current->tid(); Shims::instance().setSighandlerTid(tid); ExecutionEvent event; diff --git a/ddprof-lib/src/main/cpp/faultInjection.cpp b/ddprof-lib/src/main/cpp/faultInjection.cpp index d61be77b6..d1389cfb4 100644 --- a/ddprof-lib/src/main/cpp/faultInjection.cpp +++ b/ddprof-lib/src/main/cpp/faultInjection.cpp @@ -22,7 +22,7 @@ #include "counters.h" // Counters::increment (FAULTS_INJECTED) #include "os.h" // OS::page_size -#include "threadLocalData.h" // ProfiledThread::current / nextFiRandom +#include "threadLocalData.inline.h" // ProfiledThread::current / nextFiRandom #include #include diff --git a/ddprof-lib/src/main/cpp/faultInjection.h b/ddprof-lib/src/main/cpp/faultInjection.h index 5ac8ead3b..9729668e9 100644 --- a/ddprof-lib/src/main/cpp/faultInjection.h +++ b/ddprof-lib/src/main/cpp/faultInjection.h @@ -34,8 +34,9 @@ // // return INJECT_FAULT_BOOL_LIKELY(dlopen(name, flags) != nullptr); // -// The three tiers name their firing frequency: RARE 0.01%, UNLIKELY 0.1%, -// LIKELY 1%. See faultInjection.cpp for the poison-address and PRNG details. +// The four tiers name their firing frequency: RARE 0.01%, UNLIKELY 0.1%, +// LIKELY 1%, HIGH 10%. See faultInjection.cpp for the poison-address and PRNG +// details. #ifndef _FAULT_INJECTION_H #define _FAULT_INJECTION_H @@ -56,6 +57,7 @@ namespace faultinj { constexpr u64 PROB_RARE = 1844674407370955ULL; // 1e-4 (0.01%) constexpr u64 PROB_UNLIKELY = 18446744073709552ULL; // 1e-3 (0.1%) constexpr u64 PROB_LIKELY = 184467440737095520ULL; // 1e-2 (1%) +constexpr u64 PROB_HIGH = 1844674407370955162ULL; // 1e-1 (10%) // Called once at profiler startup (off the signal path) to mmap the PROT_NONE // guard region used by poisonAddress(). Safe to call before any injection. @@ -106,6 +108,8 @@ inline T injectValue(T orig, T faulty, u64 threshold, const char* fn) { ::faultinj::injectAddress((ptr), ::faultinj::PROB_UNLIKELY, __func__) #define INJECT_FAULT_ADDRESS_LIKELY(ptr) \ ::faultinj::injectAddress((ptr), ::faultinj::PROB_LIKELY, __func__) +#define INJECT_FAULT_ADDRESS_HIGH(ptr) \ + ::faultinj::injectAddress((ptr), ::faultinj::PROB_HIGH, __func__) #define INJECT_FAULT_BOOL_RARE(v) \ ::faultinj::injectValue((v), false, ::faultinj::PROB_RARE, __func__) @@ -113,24 +117,30 @@ inline T injectValue(T orig, T faulty, u64 threshold, const char* fn) { ::faultinj::injectValue((v), false, ::faultinj::PROB_UNLIKELY, __func__) #define INJECT_FAULT_BOOL_LIKELY(v) \ ::faultinj::injectValue((v), false, ::faultinj::PROB_LIKELY, __func__) +#define INJECT_FAULT_BOOL_HIGH(v) \ + ::faultinj::injectValue((v), false, ::faultinj::PROB_HIGH, __func__) #else // __FAULT_INJECTION__ not defined — strict identity, zero cost. #define INJECT_FAULT_ADDRESS_RARE(ptr) (ptr) #define INJECT_FAULT_ADDRESS_UNLIKELY(ptr) (ptr) #define INJECT_FAULT_ADDRESS_LIKELY(ptr) (ptr) +#define INJECT_FAULT_ADDRESS_HIGH(ptr) (ptr) #define INJECT_FAULT_INT_RARE(v) (v) #define INJECT_FAULT_INT_UNLIKELY(v) (v) #define INJECT_FAULT_INT_LIKELY(v) (v) +#define INJECT_FAULT_INT_HIGH(v) (v) #define INJECT_FAULT_LONG_RARE(v) (v) #define INJECT_FAULT_LONG_UNLIKELY(v) (v) #define INJECT_FAULT_LONG_LIKELY(v) (v) +#define INJECT_FAULT_LONG_HIGH(v) (v) #define INJECT_FAULT_BOOL_RARE(v) (v) #define INJECT_FAULT_BOOL_UNLIKELY(v) (v) #define INJECT_FAULT_BOOL_LIKELY(v) (v) +#define INJECT_FAULT_BOOL_HIGH(v) (v) #define NO_INJECTION_ASSERT(a) (assert(a)) diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index 7d096dc6c..7badb2c6d 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -28,6 +28,7 @@ #include "unwindStats.h" #include "symbols.h" #include "threadFilter.h" +#include "threadLocalData.inline.h" #include "threadState.h" #include "tsc.h" #include "hotspot/vmStructs.h" diff --git a/ddprof-lib/src/main/cpp/guards.cpp b/ddprof-lib/src/main/cpp/guards.cpp index 9905182e9..4d365c749 100644 --- a/ddprof-lib/src/main/cpp/guards.cpp +++ b/ddprof-lib/src/main/cpp/guards.cpp @@ -17,7 +17,9 @@ #include "guards.h" #include "common.h" #include "os.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" + +#include // Signal-context tracking — backed by ProfiledThread::_signal_depth; see // the comment block in guards.h for the rationale (initial-exec TLS was @@ -30,15 +32,17 @@ int getInSignalDepth() { bool isInTrackedSignalContext() { ProfiledThread *pt = ProfiledThread::current(); - // null ProfiledThread = no thread context; the SignalHandlerScope - // never ran, so we have no positive evidence of a signal frame. + // null ProfiledThread = no thread context; + // the SignalHandlerScope never ran, so we have no positive evidence + // of a signal frame. // See header comment for the rationale of returning false here. return pt != nullptr && pt->signalDepth() != 0; } -SignalHandlerScope::SignalHandlerScope() : _active(true) { - ProfiledThread *pt = ProfiledThread::current(); +SignalHandlerScope::SignalHandlerScope() : _active(true), _current(nullptr) { + ProfiledThread *pt = ProfiledThread::acquireCurrent(); if (pt != nullptr) { + _current = pt; pt->enterSignalScope(); } else { // No thread context: nothing to update; mark inactive so destructor @@ -49,9 +53,8 @@ SignalHandlerScope::SignalHandlerScope() : _active(true) { SignalHandlerScope::~SignalHandlerScope() { if (!_active) return; - ProfiledThread *pt = ProfiledThread::current(); - if (pt != nullptr) { - pt->exitSignalScope(); + if (_current != nullptr) { + _current->exitSignalScope(); } } @@ -71,46 +74,20 @@ void signalHandlerUnwindAfterLongjmp() { } } -// Static bitmap storage for fallback cases -uint64_t CriticalSection::_fallback_bitmap[CriticalSection::FALLBACK_BITMAP_WORDS] = {}; -CriticalSection::CriticalSection() : _entered(false), _using_fallback(false), _word_index(0), _bit_mask(0), _thread_ptr(nullptr) { +CriticalSection::CriticalSection() : _entered(false), _thread_ptr(nullptr) { +#ifdef UNIT_TEST + _thread_ptr = ProfiledThread::initCurrentThreadSignalSafe(); +#else _thread_ptr = ProfiledThread::current(); - if (_thread_ptr != nullptr) { - // Primary path: Use ProfiledThread storage (fast and memory-efficient) - _entered = _thread_ptr->tryEnterCriticalSection(); - } else { - // Fallback path: Use hash-based bitmap for stress tests and edge cases - _using_fallback = true; - int tid = OS::threadId(); - - // Hash TID to distribute across bitmap words, reducing clustering - // We are OK with false collision for the fallback - it should be used only for testing when we don't have full profiler initialized - _word_index = hash_tid(tid) % FALLBACK_BITMAP_WORDS; - uint32_t bit_index = tid % 64; - _bit_mask = 1ULL << bit_index; - - // Use ACQUIRE ordering to ensure visibility of protected data after acquiring critical section - uint64_t old_word = __atomic_fetch_or(&_fallback_bitmap[_word_index], _bit_mask, __ATOMIC_ACQUIRE); - _entered = !(old_word & _bit_mask); // Success if bit was previously 0 - } +#endif + assert(_thread_ptr != nullptr); + _entered = _thread_ptr->tryEnterCriticalSection(); } CriticalSection::~CriticalSection() { + assert(_thread_ptr != nullptr); if (_entered) { - if (_using_fallback) { - // Clear the bit atomically for fallback bitmap - // Use RELEASE ordering to ensure protected data writes are visible before releasing - __atomic_fetch_and(&_fallback_bitmap[_word_index], ~_bit_mask, __ATOMIC_RELEASE); - } else { - // Release ProfiledThread flag using the pointer captured at construction - if (_thread_ptr != nullptr) { - _thread_ptr->exitCriticalSection(); - } - } + _thread_ptr->exitCriticalSection(); } } - -uint32_t CriticalSection::hash_tid(int tid) { - return static_cast(tid * KNUTH_MULTIPLICATIVE_CONSTANT); -} diff --git a/ddprof-lib/src/main/cpp/guards.h b/ddprof-lib/src/main/cpp/guards.h index 18bc4fbed..8edf50803 100644 --- a/ddprof-lib/src/main/cpp/guards.h +++ b/ddprof-lib/src/main/cpp/guards.h @@ -22,6 +22,8 @@ #include #include +#include "counters.h" + class ProfiledThread; // --------------------------------------------------------------------------- @@ -42,8 +44,8 @@ class ProfiledThread; // pthread_getspecific (POSIX guarantees it does not allocate; returns // nullptr when unset). // -// When ProfiledThread is null on a thread we don't yet have a thread -// context — uninstrumented JVM-internal threads (VM Thread, JIT, GC) fall +// When ProfiledThread is null or via thread priming on a thread +// — uninstrumented JVM-internal threads (VM Thread, JIT, GC) fall // into this bucket too, and they can receive signals. The // SignalHandlerScope guard is a no-op on those threads (nothing to // update), so isInTrackedSignalContext() returns false: production code @@ -80,14 +82,26 @@ class SignalHandlerScope { void release(); SignalHandlerScope(const SignalHandlerScope&) = delete; SignalHandlerScope& operator=(const SignalHandlerScope&) = delete; + + bool isActive() const { return _active; } + ProfiledThread* current() const { return _current; } private: + ProfiledThread* _current; bool _active; }; // Declare a scope guard local that increments the depth on entry and // decrements on scope exit. Use as the very first statement in every // installed signal handler. -#define SIGNAL_HANDLER_GUARD() SignalHandlerScope _signal_handler_scope +#define SIGNAL_HANDLER_GUARD() \ + SignalHandlerScope _signal_handler_scope; \ + if (!_signal_handler_scope.isActive()) { \ + Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); \ + return; \ + } + +// Cheaper way to retrieve current ProfiledThread inside the scope +#define SIGNAL_HANDLER_CURRENT_THREAD() _signal_handler_scope.current() // Manually release the most recent SIGNAL_HANDLER_GUARD() before chaining to // another handler that may siglongjmp through us (e.g. J9's SIGSEGV null-pointer @@ -136,17 +150,7 @@ void signalHandlerUnwindAfterLongjmp(); */ class CriticalSection { private: - static constexpr size_t FALLBACK_BITMAP_WORDS = 1024; // 8KB for 64K bits - // Atomic bitmap for thread-safe critical section tracking without TLS - // Must be atomic because multiple signal handlers can run concurrently across - // different threads and attempt to set/clear bits simultaneously. Compare-and-swap - // operations ensure race-free bit manipulation even during signal interruption. - static uint64_t _fallback_bitmap[FALLBACK_BITMAP_WORDS]; - bool _entered; // Track if this instance successfully entered - bool _using_fallback; // Track which storage mechanism we're using - uint32_t _word_index; // For fallback bitmap cleanup - uint64_t _bit_mask; // For fallback bitmap cleanup ProfiledThread* _thread_ptr; // ProfiledThread captured at construction public: @@ -161,10 +165,6 @@ class CriticalSection { // Check if this instance successfully entered the critical section bool entered() const { return _entered; } - -private: - // Hash function to distribute thread IDs across bitmap words - static uint32_t hash_tid(int tid); }; /** diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp index 66057d141..140442533 100644 --- a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp +++ b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp @@ -18,6 +18,7 @@ #include "profiler.h" #include "stackWalker.inline.h" #include "threadLocal.h" +#include "threadLocalData.inline.h" using StackWalkValidation::inDeadZone; using StackWalkValidation::aligned; @@ -241,10 +242,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex assert(VM::isHotspot()); ProfiledThread* prof_thread = ProfiledThread::current(); - if (prof_thread == nullptr) { - Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); - return 0; - } + assert(prof_thread != nullptr && "Should have been setup at signal handler entery"); HotspotStackFrame frame(ucontext); uintptr_t bottom = (uintptr_t)&frame + MAX_WALK_SIZE; @@ -1210,12 +1208,16 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) { // AsyncGetCallTrace. Install a jmp ctx here too, so a SIGSEGV anywhere in // walkJavaStack, except HotSpot's AsyncGetCallTrace call, is caught by // Profiler::checkFault() and siglongjmp'd back here instead of crashing the process. - ProfiledThread* prof_thread = ProfiledThread::current(); - const bool prev_unwinding_java = prof_thread != nullptr ? prof_thread->is_unwinding_Java() : false; + ProfiledThread* prof_thread = ProfiledThread::acquireCurrent(); + if (prof_thread == nullptr) { + Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); + return 0; + } + const bool prev_unwinding_java = prof_thread->is_unwinding_Java(); sigjmp_buf crash_protection_ctx; - sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr; + sigjmp_buf* prev_jmp_buf = prof_thread->getJmpCtx(); - if (prof_thread != nullptr && sigsetjmp(crash_protection_ctx, 1) != 0) { + if (sigsetjmp(crash_protection_ctx, 1) != 0) { // checkFault() does a siglongjmp from inside segvHandler, bypassing // segvHandler's SignalHandlerScope destructor. Compensate. SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP(); @@ -1228,9 +1230,7 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) { } return java_frames; } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(&crash_protection_ctx); - } + prof_thread->setJmpCtx(&crash_protection_ctx); if (features.mixed) { java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated); @@ -1284,9 +1284,7 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) { } } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(prev_jmp_buf); - } + prof_thread->setJmpCtx(prev_jmp_buf); return java_frames; } diff --git a/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp b/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp index 73d7e2aec..c1a1c59ec 100644 --- a/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp +++ b/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp @@ -15,7 +15,7 @@ #include "jvmThread.h" #include "safeAccess.h" #include "spinLock.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "threadState.h" CodeCache* VMStructs::_libjvm = nullptr; diff --git a/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h b/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h index 5de4c94ca..100da7fb4 100644 --- a/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h +++ b/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h @@ -11,7 +11,7 @@ #include "hotspot/vmStructs.h" #include "jvmThread.h" #include "safeAccess.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" inline bool crashProtectionActive() { ProfiledThread* pt = ProfiledThread::current(); diff --git a/ddprof-lib/src/main/cpp/itimer.cpp b/ddprof-lib/src/main/cpp/itimer.cpp index 0c1a134df..eeff35c2e 100644 --- a/ddprof-lib/src/main/cpp/itimer.cpp +++ b/ddprof-lib/src/main/cpp/itimer.cpp @@ -16,13 +16,14 @@ */ #include "itimer.h" +#include "counters.h" #include "debugSupport.h" #include "jvmThread.h" #include "os.h" #include "profiler.h" #include "signalInflight.h" #include "stackWalker.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "threadState.inline.h" #include "guards.h" #include @@ -42,20 +43,16 @@ void ITimer::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { InflightGuard inflight; if (!__atomic_load_n(&_enabled, __ATOMIC_ACQUIRE)) return; - + + ProfiledThread *current = SIGNAL_HANDLER_CURRENT_THREAD(); + // Atomically try to enter critical section - prevents all reentrancy races CriticalSection cs; if (!cs.entered()) { return; // Another critical section is active, defer profiling } - int tid = 0; - ProfiledThread *current = ProfiledThread::current(); - if (current != NULL) { - current->noteCPUSample(Profiler::instance()->recordingEpoch()); - tid = current->tid(); - } else { - tid = OS::threadId(); - } + current->noteCPUSample(Profiler::instance()->recordingEpoch()); + int tid = current->tid(); Shims::instance().setSighandlerTid(tid); ExecutionEvent event; @@ -106,6 +103,9 @@ long ITimerJvmti::_interval = 0; void ITimerJvmti::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { SIGNAL_HANDLER_GUARD(); + ProfiledThread *current = SIGNAL_HANDLER_CURRENT_THREAD(); + assert(current != nullptr); + InflightGuard inflight; CriticalSection cs; if (!cs.entered()) { @@ -116,17 +116,14 @@ void ITimerJvmti::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { errno = saved_errno; return; } - ProfiledThread *current = ProfiledThread::current(); - if (current != nullptr && JVMThread::current() == nullptr + if (JVMThread::current() == nullptr && current->inInitWindow()) { current->tickInitWindow(); errno = saved_errno; return; } - int tid = current ? current->tid() : OS::threadId(); - if (current) { - current->noteCPUSample(Profiler::instance()->recordingEpoch()); - } + int tid = current->tid(); + current->noteCPUSample(Profiler::instance()->recordingEpoch()); Shims::instance().setSighandlerTid(tid); ExecutionEvent event; diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 3f71a1b74..6654b84b7 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -31,7 +31,7 @@ #include "os.h" #include "otel_process_ctx.h" #include "profiler.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "tsc.h" #include "vmEntry.h" #include @@ -75,8 +75,17 @@ Java_com_datadoghq_profiler_JavaProfiler_init0(JNIEnv *env, jclass unused) { return JNI_FALSE; } + // JavaVM* has already been stored when the native library was loaded so we can pass nullptr here - return VM::initProfilerBridge(nullptr, true); + if (VM::initProfilerBridge(nullptr, true)) { + // Attach ProfiledThread + ProfiledThread* current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + + return JNI_TRUE; + } else { + return JNI_FALSE; + } } extern "C" DLLEXPORT void JNICALL @@ -90,6 +99,10 @@ Java_com_datadoghq_profiler_JavaProfiler_stop0(JNIEnv *env, jobject unused) { extern "C" DLLEXPORT jint JNICALL Java_com_datadoghq_profiler_JavaProfiler_getTid0(JNIEnv *env, jclass unused) { + // Attach ProfiledThread + ProfiledThread* current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + return OS::threadId(); } @@ -108,6 +121,11 @@ Java_com_datadoghq_profiler_JavaProfiler_execute0(JNIEnv *env, jobject unused, Log::open(args); std::ostringstream out; + + // Attach ProfiledThread + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + error = Profiler::instance()->runInternal(args, out); if (!error) { if (out.tellp() >= 0x3fffffff) { @@ -126,6 +144,10 @@ extern "C" DLLEXPORT jstring JNICALL Java_com_datadoghq_profiler_JavaProfiler_getStatus0(JNIEnv* env, jclass unused) { char msg[2048]; + // Attach ProfiledThread + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + Profiler::instance()->status((char*)msg, sizeof(msg) - 1); return env->NewStringUTF(msg); } @@ -133,6 +155,10 @@ Java_com_datadoghq_profiler_JavaProfiler_getStatus0(JNIEnv* env, extern "C" DLLEXPORT jlong JNICALL Java_com_datadoghq_profiler_JavaProfiler_getSamples(JNIEnv *env, jclass unused) { + // Attach ProfiledThread + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + return (jlong)Profiler::instance()->total_samples(); } @@ -146,9 +172,7 @@ extern "C" DLLEXPORT void JNICALL JavaCritical_com_datadoghq_profiler_JavaProfiler_filterThreadAdd0() { // Initialize thread TLS if it has not yet done ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); - if(current == nullptr) { - return; - } + assert(current != nullptr && "Out of order initialization"); int tid = current->tid(); if (unlikely(tid < 0)) { @@ -180,10 +204,7 @@ extern "C" DLLEXPORT void JNICALL JavaCritical_com_datadoghq_profiler_JavaProfiler_filterThreadRemove0() { // Initialize thread TLS if it has not yet done ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); - if(current == nullptr) { - return; - } - + assert(current != nullptr && "Out of order initialization"); int tid = current->tid(); if (unlikely(tid < 0)) { return; @@ -221,7 +242,8 @@ Java_com_datadoghq_profiler_JavaProfiler_recordTrace0( JniString endpoint_str(env, endpoint); // Initialize thread TLS if it has not yet done - ProfiledThread::initCurrentThreadSignalSafe(); + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); u32 endpointLabel = Profiler::instance()->stringLabelMap()->bounded_lookup( endpoint_str.c_str(), endpoint_str.length(), sizeLimit); @@ -244,6 +266,10 @@ Java_com_datadoghq_profiler_JavaProfiler_recordTrace0( extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_dump0(JNIEnv *env, jclass unused, jstring path) { + // Initialize thread TLS if it has not yet done + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + JniString path_str(env, path); Profiler::instance()->dump(path_str.c_str(), path_str.length()); } @@ -263,6 +289,10 @@ extern "C" DLLEXPORT jobjectArray JNICALL Java_com_datadoghq_profiler_JavaProfiler_describeDebugCounters0( JNIEnv *env, jclass unused) { #ifdef COUNTERS + // Initialize thread TLS if it has not yet done + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + std::vector counter_names = Counters::describeCounters(); jobjectArray array = (jobjectArray)env->NewObjectArray( counter_names.size(), env->FindClass("java/lang/String"), @@ -281,7 +311,8 @@ extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_recordSettingEvent0( JNIEnv *env, jclass unused, jstring name, jstring value, jstring unit) { // Initialize thread TLS if it has not yet done - ProfiledThread::initCurrentThreadSignalSafe(); + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); int tid = ProfiledThread::currentTid(); if (tid < 0) { @@ -310,6 +341,8 @@ Java_com_datadoghq_profiler_JavaProfiler_recordQueueEnd0( // Initialize thread TLS if it has not yet done ProfiledThread::initCurrentThreadSignalSafe(); + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); int tid = ProfiledThread::currentTid(); if (tid < 0) { @@ -346,9 +379,8 @@ Java_com_datadoghq_profiler_JavaProfiler_recordQueueEnd0( extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_parkEnter0(JNIEnv *env, jclass unused) { ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); - if (current == nullptr) { - return; - } + assert(current != nullptr && "Out of order initialization"); + bool first_park = current->parkEnter(); ThreadFilter *tf = Profiler::instance()->threadFilter(); if (first_park && tf->enabled()) { @@ -364,9 +396,7 @@ extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_parkExit0( JNIEnv *env, jclass unused, jlong blocker, jlong unblockingSpanId) { ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); - if (current == nullptr) { - return; - } + assert(current != nullptr && "Out of order initialization"); u64 park_block_token = 0; if (!current->parkExit(park_block_token) || park_block_token == 0) { @@ -393,14 +423,13 @@ static bool decodeJavaBlockState(jint state, OSThreadState &decoded) { extern "C" DLLEXPORT jlong JNICALL Java_com_datadoghq_profiler_JavaProfiler_blockEnter0( JNIEnv *env, jclass unused, jint state) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + OSThreadState decoded; if (!decodeJavaBlockState(state, decoded)) { return 0; } - ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); - if (current == nullptr) { - return 0; - } ThreadFilter *tf = Profiler::instance()->threadFilter(); if (!tf->enabled()) { return 0; @@ -415,14 +444,13 @@ Java_com_datadoghq_profiler_JavaProfiler_blockEnter0( extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_blockExit0( JNIEnv *env, jclass unused, jlong token) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + u64 block_token = static_cast(token); if (block_token == 0) { return; } - ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); - if (current == nullptr) { - return; - } ThreadFilter::SlotID slot_id = ThreadFilter::tokenSlotId(block_token); if (current->filterSlotId() != slot_id) { return; @@ -436,12 +464,17 @@ Java_com_datadoghq_profiler_JavaProfiler_blockExit0( extern "C" DLLEXPORT jlong JNICALL Java_com_datadoghq_profiler_JavaProfiler_currentTicks0(JNIEnv *env, jclass unused) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + return TSC::ticks(); } extern "C" DLLEXPORT jlong JNICALL Java_com_datadoghq_profiler_JavaProfiler_tscFrequency0(JNIEnv *env, jclass unused) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); return TSC::frequency(); } @@ -449,6 +482,8 @@ extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_mallocArenaMax0(JNIEnv *env, jclass unused, jint maxArenas) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); OS::mallocArenaMax(maxArenas); } @@ -456,6 +491,9 @@ extern "C" DLLEXPORT jstring JNICALL Java_com_datadoghq_profiler_JVMAccess_findStringJVMFlag0(JNIEnv *env, jobject unused, jstring flagName) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + JniString flag_str(env, flagName); VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::String, VMFlag::Type::Stringlist}); if (f) { @@ -472,6 +510,9 @@ Java_com_datadoghq_profiler_JVMAccess_setStringJVMFlag0(JNIEnv *env, jobject unused, jstring flagName, jstring flagValue) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + JniString flag_str(env, flagName); JniString value_str(env, flagValue); VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::String, VMFlag::Type::Stringlist}); @@ -487,6 +528,9 @@ extern "C" DLLEXPORT jboolean JNICALL Java_com_datadoghq_profiler_JVMAccess_findBooleanJVMFlag0(JNIEnv *env, jobject unused, jstring flagName) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + JniString flag_str(env, flagName); VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::Bool}); if (f) { @@ -503,6 +547,9 @@ Java_com_datadoghq_profiler_JVMAccess_setBooleanJVMFlag0(JNIEnv *env, jobject unused, jstring flagName, jboolean flagValue) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + JniString flag_str(env, flagName); VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::Bool}); if (f) { @@ -517,6 +564,9 @@ extern "C" DLLEXPORT jlong JNICALL Java_com_datadoghq_profiler_JVMAccess_findIntJVMFlag0(JNIEnv *env, jobject unused, jstring flagName) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + JniString flag_str(env, flagName); VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::Int, VMFlag::Type::Uint, VMFlag::Type::Intx, VMFlag::Type::Uintx, VMFlag::Type::Uint64_t, VMFlag::Type::Size_t}); if (f) { @@ -532,6 +582,9 @@ extern "C" DLLEXPORT jdouble JNICALL Java_com_datadoghq_profiler_JVMAccess_findFloatJVMFlag0(JNIEnv *env, jobject unused, jstring flagName) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + JniString flag_str(env, flagName); VMFlag *f = VMFlag::find(flag_str.c_str(),{ VMFlag::Type::Double}); if (f) { @@ -546,6 +599,9 @@ Java_com_datadoghq_profiler_JVMAccess_findFloatJVMFlag0(JNIEnv *env, extern "C" DLLEXPORT jboolean JNICALL Java_com_datadoghq_profiler_JVMAccess_healthCheck0(JNIEnv *env, jobject unused) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + return true; } @@ -560,6 +616,9 @@ Java_com_datadoghq_profiler_OTelContext_setProcessCtx0(JNIEnv *env, jstring tracer_version, jobjectArray attribute_keys ) { + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + JniString env_str(env, env_data); JniString hostname_str(env, hostname); JniString runtime_id_str(env, runtime_id); @@ -634,6 +693,9 @@ Java_com_datadoghq_profiler_OTelContext_setProcessCtx0(JNIEnv *env, extern "C" DLLEXPORT jobject JNICALL Java_com_datadoghq_profiler_OTelContext_readProcessCtx0(JNIEnv *env, jclass unused) { #ifndef OTEL_PROCESS_CTX_NO_READ + ProfiledThread *current = ProfiledThread::initCurrentThreadSignalSafe(); + assert(current != nullptr && "Out of order initialization"); + otel_process_ctx_read_result result = otel_process_ctx_read(); if (!result.success) { @@ -829,10 +891,9 @@ extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_setTraceContext0(JNIEnv* env, jclass unused, jlong localRootSpanId, jlong spanId, jlong traceIdHigh, jlong traceIdLow, jint slot0, jint enc0, jbyteArray utf0, jint slot1, jint enc1, jbyteArray utf1) { - ProfiledThread* thrd = ProfiledThread::initCurrentThreadSignalSafe(); - if (thrd == nullptr) { - return; - } + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + // Contract: this is the activation path and requires a non-zero span; clearing is // clearTraceContext0. The public setTraceContext wrapper enforces this by throwing // IllegalArgumentException, so a zero span reaching here is a direct-JNI/contract violation. @@ -885,10 +946,9 @@ Java_com_datadoghq_profiler_JavaProfiler_setTraceContext0(JNIEnv* env, jclass un // detached (valid=0). extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_clearTraceContext0(JNIEnv* env, jclass unused) { - ProfiledThread* thrd = ProfiledThread::initCurrentThreadSignalSafe(); - if (thrd == nullptr) { - return; - } + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); u32* enc = thrd->getOtelTagEncodingsPtr(); u64* lrs = reinterpret_cast(enc + DD_TAGS_CAPACITY); @@ -910,8 +970,10 @@ Java_com_datadoghq_profiler_JavaProfiler_clearTraceContext0(JNIEnv* env, jclass extern "C" DLLEXPORT jboolean JNICALL Java_com_datadoghq_profiler_JavaProfiler_setContextValue0(JNIEnv* env, jclass unused, jint slot, jint encoding, jbyteArray utf8) { - ProfiledThread* thrd = ProfiledThread::initCurrentThreadSignalSafe(); - if (thrd == nullptr || slot < 0 || slot >= (jint)DD_TAGS_CAPACITY) { + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + + if (slot < 0 || slot >= (jint)DD_TAGS_CAPACITY) { return JNI_FALSE; } // See setTraceContext0: publish the OTEP TLS pointer on first native write so the sampler and @@ -949,8 +1011,10 @@ Java_com_datadoghq_profiler_JavaProfiler_setContextValue0(JNIEnv* env, jclass un // Clears a single attribute slot (zeros the sidecar encoding, compacts it out of attrs_data). extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_clearContextValue0(JNIEnv* env, jclass unused, jint slot) { - ProfiledThread* thrd = ProfiledThread::current(); - if (thrd == nullptr || slot < 0 || slot >= (jint)DD_TAGS_CAPACITY) { + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + + if (slot < 0 || slot >= (jint)DD_TAGS_CAPACITY) { return; } OtelThreadContextRecord* record = thrd->getOtelContextRecord(); @@ -978,19 +1042,21 @@ Java_com_datadoghq_profiler_JavaProfiler_copyContextTags0(JNIEnv* env, jclass un if (out == nullptr) { return; } + + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + jint len = env->GetArrayLength(out); int n = len < (jint)DD_TAGS_CAPACITY ? (int)len : (int)DD_TAGS_CAPACITY; jint tmp[DD_TAGS_CAPACITY]; for (int i = 0; i < n; i++) { tmp[i] = 0; } - ProfiledThread* thrd = ProfiledThread::current(); - if (thrd != nullptr) { - u32* enc = thrd->getOtelTagEncodingsPtr(); - for (int i = 0; i < n; i++) { - tmp[i] = (jint)enc[i]; - } + u32* enc = thrd->getOtelTagEncodingsPtr(); + for (int i = 0; i < n; i++) { + tmp[i] = (jint)enc[i]; } + if (n > 0) { env->SetIntArrayRegion(out, 0, n, tmp); } @@ -998,6 +1064,9 @@ Java_com_datadoghq_profiler_JavaProfiler_copyContextTags0(JNIEnv* env, jclass un extern "C" DLLEXPORT jint JNICALL Java_com_datadoghq_profiler_ContextValueCache_registerConstant0(JNIEnv* env, jclass unused, jstring value) { + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + JniString value_str(env, value); u32 encoding = Profiler::instance()->contextValueMap()->bounded_lookup( value_str.c_str(), value_str.length(), 1 << 16); @@ -1008,6 +1077,9 @@ Java_com_datadoghq_profiler_ContextValueCache_registerConstant0(JNIEnv* env, jcl // MAX_CONTEXT_SLOTS constant has not drifted from DD_TAGS_CAPACITY (see MaxContextSlotsTest). extern "C" DLLEXPORT jint JNICALL Java_com_datadoghq_profiler_JavaProfiler_maxContextSlots0(JNIEnv* env, jclass unused) { + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + return (jint)DD_TAGS_CAPACITY; } @@ -1018,12 +1090,18 @@ Java_com_datadoghq_profiler_JavaProfiler_maxContextSlots0(JNIEnv* env, jclass un // re-parsing in Java. extern "C" DLLEXPORT jboolean JNICALL Java_com_datadoghq_profiler_JavaProfiler_consumeContextDictionaryReset0(JNIEnv* env, jclass unused) { + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + return Profiler::instance()->consumeContextValueDictReset() ? JNI_TRUE : JNI_FALSE; } // ---- test and debug utilities extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_testlog(JNIEnv* env, jclass unused, jstring msg) { + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + JniString msg_str(env, msg); TEST_LOG("%s", msg_str.c_str()); @@ -1031,6 +1109,9 @@ Java_com_datadoghq_profiler_JavaProfiler_testlog(JNIEnv* env, jclass unused, jst extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_dumpContext(JNIEnv* env, jclass unused) { + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + u64 spanId = 0, rootSpanId = 0; ContextApi::get(spanId, rootSpanId); TEST_LOG("===> Context: tid:%lu, spanId=%lu, rootSpanId=%lu", OS::threadId(), spanId, rootSpanId); @@ -1042,10 +1123,9 @@ Java_com_datadoghq_profiler_JavaProfiler_dumpContext(JNIEnv* env, jclass unused) extern "C" DLLEXPORT jlong JNICALL Java_com_datadoghq_profiler_JavaProfiler_testGetSpanId0(JNIEnv* env, jclass unused) { - ProfiledThread* thrd = ProfiledThread::current(); - if (thrd == nullptr) { - return 0; - } + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); uint64_t beSpan; memcpy(&beSpan, record->span_id, 8); @@ -1054,10 +1134,9 @@ Java_com_datadoghq_profiler_JavaProfiler_testGetSpanId0(JNIEnv* env, jclass unus extern "C" DLLEXPORT jlong JNICALL Java_com_datadoghq_profiler_JavaProfiler_testGetRootSpanId0(JNIEnv* env, jclass unused) { - ProfiledThread* thrd = ProfiledThread::current(); - if (thrd == nullptr) { - return 0; - } + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + u32* enc = thrd->getOtelTagEncodingsPtr(); u64* lrs = reinterpret_cast(enc + DD_TAGS_CAPACITY); return (jlong)*lrs; @@ -1065,10 +1144,9 @@ Java_com_datadoghq_profiler_JavaProfiler_testGetRootSpanId0(JNIEnv* env, jclass extern "C" DLLEXPORT jstring JNICALL Java_com_datadoghq_profiler_JavaProfiler_testReadTraceId0(JNIEnv* env, jclass unused) { - ProfiledThread* thrd = ProfiledThread::current(); - if (thrd == nullptr) { - return nullptr; - } + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); static const char HEXD[16] = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; @@ -1084,10 +1162,9 @@ Java_com_datadoghq_profiler_JavaProfiler_testReadTraceId0(JNIEnv* env, jclass un extern "C" DLLEXPORT jstring JNICALL Java_com_datadoghq_profiler_JavaProfiler_testReadContextAttribute0(JNIEnv* env, jclass unused, jint slot) { - ProfiledThread* thrd = ProfiledThread::current(); - if (thrd == nullptr || slot < 0 || slot >= (jint)DD_TAGS_CAPACITY) { - return nullptr; - } + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); int targetKey = slot + 1; int size = record->attrs_data_size; @@ -1112,10 +1189,9 @@ Java_com_datadoghq_profiler_JavaProfiler_testReadContextAttribute0(JNIEnv* env, extern "C" DLLEXPORT jboolean JNICALL Java_com_datadoghq_profiler_JavaProfiler_testIsContextValid0(JNIEnv* env, jclass unused) { - ProfiledThread* thrd = ProfiledThread::current(); - if (thrd == nullptr) { - return JNI_FALSE; - } + ProfiledThread *thrd = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thrd != nullptr && "Out of order initialization"); + OtelThreadContextRecord* record = thrd->getOtelContextRecord(); return __atomic_load_n(&record->valid, __ATOMIC_ACQUIRE) ? JNI_TRUE : JNI_FALSE; } diff --git a/ddprof-lib/src/main/cpp/jvmSupport.cpp b/ddprof-lib/src/main/cpp/jvmSupport.cpp index b6277a6be..5b1833911 100644 --- a/ddprof-lib/src/main/cpp/jvmSupport.cpp +++ b/ddprof-lib/src/main/cpp/jvmSupport.cpp @@ -6,10 +6,12 @@ #include "jvmSupport.inline.h" #include "asyncSampleMutex.h" +#include "common.h" #include "frames.h" #include "os.h" #include "profiler.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" +#include "threadLocalDataPool.h" #include "vmEntry.h" #include "hotspot/hotspotSupport.h" @@ -39,7 +41,15 @@ bool JVMSupport::initialize() { } // Check ProfiledThread key, it is critical for storing per-thread metadata - return ProfiledThread::isThreadKeyValid(); + bool validKey = ProfiledThread::isThreadKeyValid(); + + if (validKey && ProfiledThread::supportPriming()) { + ThreadLocalDataPool::initialize(); + } else { + LOG_WARN("Thread priming is not supported"); + } + + return validKey; } bool JVMSupport::isInitialized() { diff --git a/ddprof-lib/src/main/cpp/mallocTracer.cpp b/ddprof-lib/src/main/cpp/mallocTracer.cpp index 18b843802..1b87060f5 100644 --- a/ddprof-lib/src/main/cpp/mallocTracer.cpp +++ b/ddprof-lib/src/main/cpp/mallocTracer.cpp @@ -10,6 +10,7 @@ #include #include #include "codeCache.h" +#include "counters.h" #include "guards.h" #include "libraries.h" #include "mallocTracer.h" @@ -41,6 +42,12 @@ static void* (*_orig_aligned_alloc)(size_t, size_t); // because the window is short. static inline void maybeRecord(void* ret, size_t size) { if (MallocTracer::running() && ret && size) { + // Even we are not in a signal handler, we cannot malloc or + // we may get into indefinite loop + if (ProfiledThread::acquireCurrent() == nullptr) { + Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); + return; + } CriticalSection cs; if (cs.entered()) { MallocTracer::recordMalloc(ret, size); diff --git a/ddprof-lib/src/main/cpp/nativeSocketSampler.cpp b/ddprof-lib/src/main/cpp/nativeSocketSampler.cpp index a8320f69d..550608369 100644 --- a/ddprof-lib/src/main/cpp/nativeSocketSampler.cpp +++ b/ddprof-lib/src/main/cpp/nativeSocketSampler.cpp @@ -264,6 +264,9 @@ void NativeSocketSampler::recordEvent(int fd, u64 t0, u64 t1, ssize_t bytes, u8 event._bytes = (u64)bytes; event._weight = weight; + // We are not in a signal handler - take this chance to ensure ProfiledThread + // is attached to the thread cheaply. + ProfiledThread::initCurrentThreadSignalSafe(); Profiler::instance()->recordSample(NULL, (u64)bytes, OS::threadId(), BCI_NATIVE_SOCKET, 0, &event); diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index b3e502813..61f964d0f 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -20,6 +20,7 @@ #include "arch.h" #include "arguments.h" #include "context.h" +#include "counters.h" #include "guards.h" #include "debugSupport.h" #include "jvmSupport.inline.h" @@ -35,7 +36,7 @@ #include "stackFrame.h" #include "stackWalker.h" #include "symbols.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "threadState.inline.h" #include #include @@ -742,16 +743,23 @@ void PerfEvents::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { return; } InflightGuard inflight; + + // A thread with no ProfiledThread attached must never enter the critical + // section below. acquireCurrent() is the only thing that can attach one; it + // must fully succeed or fail before we try to claim exclusivity, not while + // we're holding it -- otherwise a signal that interrupts us right after + // publish could observe a ProfiledThread whose critical-section state + // doesn't yet reflect reality. Drop the sample instead. + ProfiledThread *current = SIGNAL_HANDLER_CURRENT_THREAD(); + assert(current != nullptr); + // Atomically try to enter critical section - prevents all reentrancy races CriticalSection cs; if (!cs.entered()) { return; // Another critical section is active, defer profiling } - ProfiledThread *current = ProfiledThread::current(); - if (current != NULL) { - current->noteCPUSample(Profiler::instance()->recordingEpoch()); - } - int tid = current != NULL ? current->tid() : OS::threadId(); + current->noteCPUSample(Profiler::instance()->recordingEpoch()); + int tid = current->tid(); if (__atomic_load_n(&_enabled, __ATOMIC_ACQUIRE)) { Shims::instance().setSighandlerTid(tid); diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index c45a61c52..81dc6fc6b 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -35,6 +35,7 @@ #include "stackFrame.h" #include "stackWalker.h" #include "symbols.h" +#include "threadLocalData.inline.h" #include "tsc.h" #include "utils.h" #include "wallClock.h" @@ -530,6 +531,13 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain, } u64 Profiler::recordJVMTISample(u64 counter, int tid, jthread thread, jint event_type, Event *event, bool deferred) { + // Called from none signal based sampler + ProfiledThread* prof_thread = ProfiledThread::initCurrentThreadSignalSafe(); + if (prof_thread == nullptr) { + Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); + return 0; + } + // Protect JVMTI sampling operations to prevent signal handler interference CriticalSection cs; atomicIncRelaxed(_total_samples); @@ -890,10 +898,10 @@ bool Profiler::prewarmUnwinder() { // libgcc_s.so.1 has been the stable SONAME since 2002; a bump would // constitute a glibc/GCC C++ ABI break and is treated as a fixed contract. // - // INJECT_FAULT_BOOL_LIKELY lets fault-injection builds force this to + // INJECT_FAULT_BOOL_HIGH lets fault-injection builds force this to // report failure without the library actually being absent, so // checkState()'s "Missing libgcc_s.so" path can be exercised in CI. - return INJECT_FAULT_BOOL_LIKELY(dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL) != nullptr); + return INJECT_FAULT_BOOL_HIGH(dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL) != nullptr); #else return true; #endif diff --git a/ddprof-lib/src/main/cpp/refCountGuard.cpp b/ddprof-lib/src/main/cpp/refCountGuard.cpp index 5f37d74a4..7af5a3216 100644 --- a/ddprof-lib/src/main/cpp/refCountGuard.cpp +++ b/ddprof-lib/src/main/cpp/refCountGuard.cpp @@ -10,7 +10,7 @@ #include "log.h" #include "os.h" #include "primeProbing.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include #include diff --git a/ddprof-lib/src/main/cpp/safeAccess.cpp b/ddprof-lib/src/main/cpp/safeAccess.cpp index 4bcf596b1..c6f3bbbd8 100644 --- a/ddprof-lib/src/main/cpp/safeAccess.cpp +++ b/ddprof-lib/src/main/cpp/safeAccess.cpp @@ -23,7 +23,7 @@ #include #include #ifdef DEBUG -#include "threadLocalData.h" // ProfiledThread::current / isProtected +#include "threadLocalData.inline.h" // ProfiledThread::current / isProtected #endif extern "C" int safefetch32_cont(int* adr, int errValue); diff --git a/ddprof-lib/src/main/cpp/signalSafety.h b/ddprof-lib/src/main/cpp/signalSafety.h index 44d0d5d4b..091847226 100644 --- a/ddprof-lib/src/main/cpp/signalSafety.h +++ b/ddprof-lib/src/main/cpp/signalSafety.h @@ -18,7 +18,7 @@ #define _SIGNAL_SAFETY_H #include "guards.h" // isInSignalContext, SIGNAL_HANDLER_GUARD, ... -#include "threadLocalData.h" // ProfiledThread::current +#include "threadLocalData.inline.h" // ProfiledThread::current // Detect ASAN using compiler-provided macros so the ASAN_ENABLED guard below // works in every TU that includes this header, independent of include order. diff --git a/ddprof-lib/src/main/cpp/stackWalker.cpp b/ddprof-lib/src/main/cpp/stackWalker.cpp index 4de65b580..b7a30abe1 100644 --- a/ddprof-lib/src/main/cpp/stackWalker.cpp +++ b/ddprof-lib/src/main/cpp/stackWalker.cpp @@ -13,7 +13,7 @@ #include "symbols.h" #include "jvmSupport.inline.h" #include "jvmThread.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" // Use validation helpers from header (shared with tests) using StackWalkValidation::inDeadZone; @@ -46,10 +46,12 @@ int StackWalker::walkFP(void* ucontext, const void** callchain, int max_depth, S // Profiler::checkFault() from the SEGV handler and siglongjmp'd back here, // instead of crashing the process. ProfiledThread* prof_thread = ProfiledThread::current(); + assert(prof_thread != nullptr && "Should have been setup at signal handler entery"); + sigjmp_buf crash_protection_ctx; - sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr; + sigjmp_buf* prev_jmp_buf = prof_thread->getJmpCtx(); - if (prof_thread != nullptr && sigsetjmp(crash_protection_ctx, 1) != 0) { + if (sigsetjmp(crash_protection_ctx, 1) != 0) { // checkFault() does a siglongjmp from inside segvHandler, bypassing // segvHandler's SignalHandlerScope destructor. Compensate. SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP(); @@ -62,9 +64,7 @@ int StackWalker::walkFP(void* ucontext, const void** callchain, int max_depth, S } return depth; } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(&crash_protection_ctx); - } + prof_thread->setJmpCtx(&crash_protection_ctx); // Walk until the bottom of the stack or until the first Java frame while (depth < actual_max_depth) { @@ -133,10 +133,12 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth // Profiler::checkFault() from the SEGV handler and siglongjmp'd back here, // instead of crashing the process. ProfiledThread* prof_thread = ProfiledThread::current(); + assert(prof_thread != nullptr && "Should have been setup at signal handler entery"); + sigjmp_buf crash_protection_ctx; - sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr; + sigjmp_buf* prev_jmp_buf = prof_thread->getJmpCtx(); - if (prof_thread != nullptr && sigsetjmp(crash_protection_ctx, 1) != 0) { + if (sigsetjmp(crash_protection_ctx, 1) != 0) { // checkFault() does a siglongjmp from inside segvHandler, bypassing // segvHandler's SignalHandlerScope destructor. Compensate. SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP(); @@ -149,9 +151,7 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth } return depth; } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(&crash_protection_ctx); - } + prof_thread->setJmpCtx(&crash_protection_ctx); // Walk until the bottom of the stack or until the first Java frame while (depth < actual_max_depth) { @@ -229,9 +229,7 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth } } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(prev_jmp_buf); - } + prof_thread->setJmpCtx(prev_jmp_buf); if (truncated && depth > max_depth) { *truncated = true; diff --git a/ddprof-lib/src/main/cpp/threadLocal.h b/ddprof-lib/src/main/cpp/threadLocal.h index ffd06713a..764a80c68 100644 --- a/ddprof-lib/src/main/cpp/threadLocal.h +++ b/ddprof-lib/src/main/cpp/threadLocal.h @@ -86,6 +86,10 @@ class ThreadLocal { return _key != INVALID_KEY; } + pthread_key_t key() const { + return _key; + } + /** * set(nullptr) will result in the value being recreated when get() is called * when CREATE_FUNC is not nullptr. diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index 37942fe36..322975693 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -3,11 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "threadLocalData.h" +#include "faultInjection.h" +#include "threadLocalData.inline.h" +#include "threadLocalDataPool.h" #include "context_api.h" #include "guards.h" #include "otel_context.h" #include "os.h" +#include #include #include @@ -19,6 +22,25 @@ // reads it. ThreadLocal ProfiledThread::_current_thread; +bool ProfiledThread::supportPriming() { + // Key must be valid + assert(_current_thread.isKeyValid()); + if (OS::isMusl()) { + return true; + } +#ifdef __GLIBC__ + bool rc = _current_thread.key() < PTHREAD_KEY_2NDLEVEL_SIZE; + return INJECT_FAULT_BOOL_HIGH(rc); +#else + // Neither musl nor glibc (e.g. macOS libpthread): PTHREAD_KEY_2NDLEVEL_SIZE + // is a glibc NPTL implementation detail (see threadLocalData.h) that doesn't + // describe this libc's pthread_key_t allocation scheme. Fail safe by + // disabling signal-handler TLS priming rather than assuming glibc-compatible + // pthread_setspecific behavior. + return false; +#endif +} + ProfiledThread* ProfiledThread::initCurrentThread() { if (!isThreadKeyValid()) { return nullptr; @@ -49,17 +71,28 @@ ProfiledThread* ProfiledThread::initCurrentThreadSignalSafe() { void ProfiledThread::freeValue(void* value) { SignalBlocker blocker; ProfiledThread* pt = reinterpret_cast(value); - // Sole deletion site for a ProfiledThread (invoked by the ThreadLocal - // destructor callback), so the THREAD_LOCAL decrement belongs here. Record - // after the delete, consistent with the other decrement sites. - delete pt; - NativeMem::record(NM_THREAD_LOCAL, -(long long)sizeof(ProfiledThread)); + if (!ThreadLocalDataPool::release(pt)) { + // Sole deletion site for a ProfiledThread (invoked by the ThreadLocal + // destructor callback), so the THREAD_LOCAL decrement belongs here. Record + // after the delete, consistent with the other decrement sites. + delete pt; + NativeMem::record(NM_THREAD_LOCAL, -(long long)sizeof(ProfiledThread)); + } } void ProfiledThread::release() { _current_thread.clear(); } +#ifdef UNIT_TEST +void ProfiledThread::deleteForTest(ProfiledThread* pt) { + if (!ThreadLocalDataPool::release(pt)) { + delete pt; + NativeMem::record(NM_THREAD_LOCAL, -(long long)sizeof(ProfiledThread)); + } +} +#endif + int ProfiledThread::currentTid() { ProfiledThread *tls = current(); if (tls != NULL) { @@ -81,3 +114,34 @@ Context ProfiledThread::snapshotContext(size_t numAttrs) { } return ctx; } + +void ProfiledThread::resetClaimed(int tid) { + _jmp_buf = nullptr; + _pc = 0; + _sp = 0; + _span_id = 0; + _crash_depth = 0; + _tid = tid; + _wall_epoch = 0; + _call_trace_id = 0; + _recording_epoch = 0; + __atomic_store_n(&_misc_flags, FLAG_CLAIMED, __ATOMIC_RELEASE); + _park_block_token = 0; + _filter_slot_id = -1; + _init_window = 0; + _signal_depth = 0; + _otel_ctx_initialized = false; + _otel_ctx_record = {}; + for (int index = 0; index < DD_TAGS_CAPACITY; index++) { + _otel_tag_encodings[index] = 0; + } + _otel_local_root_span_id = 0; + _in_critical_section = false; + + _unwind_failures.reset(); + + #ifdef __FAULT_INJECTION__ + _fi_rng = ((u64)(uintptr_t)this) ^ (0x9e3779b97f4a7c15ULL * (u64)tid); + if (_fi_rng == 0) _fi_rng = 1; +#endif +} diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index 30f750cb1..09e3665fb 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -37,6 +37,15 @@ class ThreadLocalData { }; class ProfiledThread : public ThreadLocalData { + friend class ThreadLocalDataPool; + + // PTHREAD_KEY_2NDLEVEL_SIZE is an internal macro set to 32 in the GNU C Library (glibc) NPTL + // implementation. Slot indexes less than PTHREAD_KEY_2NDLEVEL_SIZE are pre-allocated. + // glibc-specific: only meaningful under the __GLIBC__ branch of supportPriming() + // (threadLocalData.cpp). Other libcs (musl, macOS libpthread) don't share this + // layout and must not be routed through this constant. + static constexpr int PTHREAD_KEY_2NDLEVEL_SIZE = 32; + public: enum ThreadType : u32 { TYPE_UNKNOWN = 0, @@ -46,6 +55,7 @@ class ProfiledThread : public ThreadLocalData { }; static constexpr u32 FLAG_PARKED = 0x4u; // next free bit after TYPE_MASK (0x1|0x2) + static constexpr u32 FLAG_CLAIMED = 0x8u; // Used by ThreadLocalDataPool only // We are allowing several levels of nesting because we can be // eg. in a crash handler when wallclock signal kicks in, @@ -74,7 +84,7 @@ class ProfiledThread : public ThreadLocalData { u32 _wall_epoch; u64 _call_trace_id; u32 _recording_epoch; - u32 _misc_flags; + volatile u32 _misc_flags; u64 _park_block_token; int _filter_slot_id; // Slot ID for thread filtering uint8_t _init_window; // Countdown for JVM thread init race window (PROF-13072) @@ -112,6 +122,28 @@ class ProfiledThread : public ThreadLocalData { }; virtual ~ProfiledThread() { } + + // Reset content of claimed slot + void resetClaimed(int tid); + + inline bool isClaimed() const { + return (__atomic_load_n(&_misc_flags, __ATOMIC_RELAXED) & FLAG_CLAIMED) == FLAG_CLAIMED; + } + + inline void unclaim() { + assert(isClaimed() && "Slot has been claimed"); + __atomic_fetch_and(&_misc_flags, ~FLAG_CLAIMED, __ATOMIC_RELEASE); + } + + inline bool claimAcquire() { + if (isClaimed()) { + return false; + } + + u32 flags = __atomic_fetch_or(&_misc_flags, FLAG_CLAIMED, __ATOMIC_ACQUIRE); + return (flags & FLAG_CLAIMED) == 0; + } + public: static ProfiledThread *forTid(int tid) { ProfiledThread *pt = new ProfiledThread(tid); @@ -122,6 +154,8 @@ class ProfiledThread : public ThreadLocalData { return _current_thread.isKeyValid(); } + static bool supportPriming(); + #ifdef UNIT_TEST // Simulates the moment inside release() after pthread_setspecific(NULL) but // before delete — the race window the clearCurrentThreadTLS fix covers. @@ -132,16 +166,15 @@ class ProfiledThread : public ThreadLocalData { _current_thread.set(nullptr); return pt; } - // Deletes a ProfiledThread returned by clearCurrentThreadTLS(). - // Needed because the destructor is private. This stands in for the delete - // that freeValue() performs in production, so it mirrors freeValue()'s - // NM_THREAD_LOCAL decrement to keep the accounting balanced in tests. - static void deleteForTest(ProfiledThread *pt) { - delete pt; - NativeMem::record(NM_THREAD_LOCAL, -(long long)sizeof(ProfiledThread)); - } + // Releases a ProfiledThread returned by clearCurrentThreadTLS(). + // Needed because the destructor is private. Mirrors freeValue()'s + // ThreadLocalDataPool::release()-then-delete logic (and its NM_THREAD_LOCAL + // decrement) so it's safe to call on both forTid()-obtained and pool-backed + // threads. Defined in threadLocalData.cpp, where ThreadLocalDataPool's full + // declaration is visible. + static void deleteForTest(ProfiledThread *pt); #endif - // initCurrentThread() and release() are not async-signal-safe: + // initCurrentThread() and release() are not async-signal-safe: // must be called outside of a signal handler with signal blocked static ProfiledThread* initCurrentThread(); static void release(); @@ -154,12 +187,10 @@ class ProfiledThread : public ThreadLocalData { static ProfiledThread* initCurrentThreadSignalSafe(); // Signal-handler friendly (no allocation): returns existing TLS or nullptr. - static inline ProfiledThread *current() { - if (!isThreadKeyValid()) { - return nullptr; - } - return _current_thread.get(); - } + static inline ProfiledThread *current(); + // signal-handler friendly with priming: return existing TLS or acquire and set + // ProfiledThread from ThreadLocalDataPool. + static inline ProfiledThread* acquireCurrent(); static int currentTid(); inline int tid() { return _tid; } diff --git a/ddprof-lib/src/main/cpp/threadLocalData.inline.h b/ddprof-lib/src/main/cpp/threadLocalData.inline.h new file mode 100644 index 000000000..351317618 --- /dev/null +++ b/ddprof-lib/src/main/cpp/threadLocalData.inline.h @@ -0,0 +1,44 @@ +/* + * Copyright 2026 Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef THREADLOCALDATA_INLINE_H +#define THREADLOCALDATA_INLINE_H + +#include "guards.h" +#include "os.h" +#include "threadLocalData.h" +#include "threadLocalDataPool.h" + +inline ProfiledThread* ProfiledThread::current() { + if (!isThreadKeyValid()) { + return nullptr; + } + return _current_thread.get(); +} + +ProfiledThread* ProfiledThread::acquireCurrent() { + ProfiledThread* prof_thread = current(); + if (prof_thread == nullptr) { + SignalBlocker blocker; + // Check again, in case the call is interrupted by another signal + prof_thread = current(); + if (prof_thread == nullptr) { + prof_thread = ThreadLocalDataPool::acquire(OS::threadId()); + if (prof_thread != nullptr) { + // Claim the critical section before publishing the pointer. A signal + // that interrupts us on this thread right after publish (cross-type + // nesting isn't blocked — see os_linux.cpp's empty sa_mask) would + // otherwise see a non-null ProfiledThread whose _in_critical_section + // is still false and race into the primary path. CriticalSection's + // fallback-path destructor releases this once the outer handler exits. + prof_thread->tryEnterCriticalSection(); + _current_thread.set(prof_thread); + } + } + } + return prof_thread; +} + +#endif // THREADLOCALDATA_INLINE_H diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp new file mode 100644 index 000000000..73f3bbf2f --- /dev/null +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "counters.h" +#include "threadLocalData.h" +#include "threadLocalDataPool.h" + +#include +#include + +ThreadLocalDataPool* ThreadLocalDataPool::_pool = nullptr; + +ThreadLocalDataPool::ThreadLocalDataPool(uint16_t capacity) + : _capacity(capacity), _used(0), _threads(nullptr) { + const size_t malloc_size = capacity * sizeof(ProfiledThread); + void* p = malloc(malloc_size); + if (p == nullptr) { + return; + } + + _threads = reinterpret_cast(p); + for (uint64_t index = 0; index < capacity; index++) { + new (&_threads[index]) ProfiledThread(0); + } + NativeMem::record(NM_THREAD_LOCAL, malloc_size + sizeof(ThreadLocalDataPool)); +} + +#ifdef UNIT_TEST +ThreadLocalDataPool::~ThreadLocalDataPool() { + if (_threads != nullptr) { + for (int index = 0; index < _capacity; index++) { + _threads[index].~ProfiledThread(); + } + free(reinterpret_cast(_threads)); + } +} +#endif // UNIT_TEST + + +ProfiledThread* ThreadLocalDataPool::claim(int tid) { + if (_threads == nullptr) { + return nullptr; + } + + uint16_t used = __atomic_fetch_add(&_used, 1, __ATOMIC_RELAXED); + if (used >= _capacity) { + __atomic_fetch_add(&_used, -1, __ATOMIC_RELAXED); + return nullptr; + } + + int start_pos = tid % _capacity; + int index = start_pos; + do { + if (_threads[index].claimAcquire()) { + return &_threads[index]; + } + index = (index + 1) % _capacity; + } while (index != start_pos); + __atomic_fetch_add(&_used, -1, __ATOMIC_RELAXED); + Counters::increment(SAMPLES_DROPPED_TLS_POOL_EXHAUSTED); + return nullptr; +} + +bool ThreadLocalDataPool::unclaim(ProfiledThread* t) { + if (contains(t)) { + t->unclaim(); + uint16_t used = __atomic_fetch_add(&_used, -1, __ATOMIC_RELEASE); + assert(used > 0); + return true; + } + return false; +} + +void ThreadLocalDataPool::initialize() { + // process-lifetime singleton + ThreadLocalDataPool* pool = new ThreadLocalDataPool(); + __atomic_store_n(&_pool, pool, __ATOMIC_RELEASE); +} + +ProfiledThread* ThreadLocalDataPool::acquire(int tid) { + ThreadLocalDataPool* pool = __atomic_load_n(&_pool, __ATOMIC_ACQUIRE); + if (pool == nullptr) { + return nullptr; + } else { + ProfiledThread* t = pool->claim(tid); + if (t != nullptr) { + t->resetClaimed(tid); + } + return t; + } +} + +bool ThreadLocalDataPool::release(ProfiledThread* t) { + ThreadLocalDataPool* pool = __atomic_load_n(&_pool, __ATOMIC_ACQUIRE); + if (pool != nullptr) { + return pool->unclaim(t); + } else { + return false; + } +} diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.h b/ddprof-lib/src/main/cpp/threadLocalDataPool.h new file mode 100644 index 000000000..706d58c90 --- /dev/null +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.h @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef THREADLOCALDATA_POOL_H +#define THREADLOCALDATA_POOL_H + +#include +#include +#include + +class ProfiledThread; + +class ThreadLocalDataPool { + static constexpr uint16_t DEFAULT_CAPACITY = 64; +private: + static ThreadLocalDataPool* _pool; + + const uint64_t _capacity; + volatile uint16_t _used; + ProfiledThread* _threads; + + ThreadLocalDataPool(const ThreadLocalDataPool&) = delete; + ThreadLocalDataPool& operator=(const ThreadLocalDataPool&) = delete; + + ThreadLocalDataPool(uint16_t capacity = DEFAULT_CAPACITY); +#ifdef UNIT_TEST + ~ThreadLocalDataPool(); +#else + ~ThreadLocalDataPool() = delete; +#endif // UNIT_TEST + ProfiledThread* claim(int tid); + bool unclaim(ProfiledThread* t); + + + inline bool contains(ProfiledThread* t) const { + if (_threads == nullptr || t == nullptr) return false; + const uintptr_t addr = reinterpret_cast(t); + const uintptr_t base = reinterpret_cast(_threads); + const uintptr_t end = reinterpret_cast(_threads + _capacity); + return addr >= base && addr < end; + } + +public: + static void initialize(); + static ProfiledThread* acquire(int tid); + static bool release(ProfiledThread* t); + static inline bool containsThread(ProfiledThread* t) { + ThreadLocalDataPool* pool = __atomic_load_n(&_pool, __ATOMIC_ACQUIRE); + if (pool != nullptr) { + return pool->contains(t); + } else { + return false; + } + } + +#ifdef UNIT_TEST + // Test-only: a pool isolated from the process-wide singleton (_pool), so + // contains()/boundary tests don't disturb other tests' use of + // initialize()/acquire()/release(). + static ThreadLocalDataPool* createForTest(uint64_t capacity) { + return new ThreadLocalDataPool(capacity); + } + // ThreadLocalDataPool has no destructor definition (it's a process-lifetime + // singleton in production, never freed), so `delete p` won't link. Mirror + // what a destructor would do -- destroy each placement-newed ProfiledThread + // and free() the malloc'd buffer -- then release the ThreadLocalDataPool + // object itself via the deallocation function directly, without invoking a + // (nonexistent) destructor. + static void destroyForTest(ThreadLocalDataPool* p) { + if (p->_threads != nullptr) { + for (uint64_t index = 0; index < p->_capacity; index++) { + p->_threads[index].~ProfiledThread(); + } + free(reinterpret_cast(p->_threads)); + } + ::operator delete(p); + } + + bool containsForTest(ProfiledThread* t) const { return contains(t); } + ProfiledThread* threadsForTest() const { return _threads; } + uint64_t capacityForTest() const { return _capacity; } + ProfiledThread* claimForTest(int tid) { return claim(tid); } + bool unclaimForTest(ProfiledThread* t) { return unclaim(t); } +#endif +}; + +#endif // THREADLOCALDATA_POOL_H diff --git a/ddprof-lib/src/main/cpp/unwindStats.cpp b/ddprof-lib/src/main/cpp/unwindStats.cpp index 82a38cf17..5bd21a2f7 100644 --- a/ddprof-lib/src/main/cpp/unwindStats.cpp +++ b/ddprof-lib/src/main/cpp/unwindStats.cpp @@ -1,5 +1,28 @@ +/* + * Copyright The async-profiler authors + * Copyright 2026 Datadog, Inc + * SPDX-License-Identifier: Apache-2.0 + */ + #include "unwindStats.h" // initialize static members SpinLock UnwindStats::_lock; UnwindFailures UnwindStats::_unwind_failures; + +UnwindFailures::UnwindFailures() : _nameCount(0) { + _names = new char[MAX_UNWIND_FAILURE_NAMES][MAX_NAME_LENGTH]; + _counters = new u64[MAX_UNWIND_FAILURE_NAMES][UNWIND_FAILURE_ANY + 1]; + reset(); +} + +UnwindFailures::~UnwindFailures() { + delete[] _names; + delete[] _counters; +} + +void UnwindFailures::reset() { + memset((void*)_names, 0, MAX_UNWIND_FAILURE_NAMES * MAX_NAME_LENGTH); + memset((void*)_counters, 0, MAX_UNWIND_FAILURE_NAMES * (UNWIND_FAILURE_ANY + 1) * sizeof(u64)); + _nameCount = 0; +} diff --git a/ddprof-lib/src/main/cpp/unwindStats.h b/ddprof-lib/src/main/cpp/unwindStats.h index 1eb4eab29..bf2c49cce 100644 --- a/ddprof-lib/src/main/cpp/unwindStats.h +++ b/ddprof-lib/src/main/cpp/unwindStats.h @@ -1,3 +1,9 @@ +/* + * Copyright The async-profiler authors + * Copyright 2026 Datadog, Inc + * SPDX-License-Identifier: Apache-2.0 + */ + #ifndef STUB_UNWIND_STATS_H #define STUB_UNWIND_STATS_H @@ -25,22 +31,15 @@ class UnwindFailures { volatile u64 (*_counters)[UNWIND_FAILURE_ANY + 1]; public: - UnwindFailures() : _nameCount(0) { - _names = new char[MAX_UNWIND_FAILURE_NAMES][MAX_NAME_LENGTH]; - _counters = new u64[MAX_UNWIND_FAILURE_NAMES][UNWIND_FAILURE_ANY + 1]; - memset((void*)_names, 0, MAX_UNWIND_FAILURE_NAMES * MAX_NAME_LENGTH); - memset((void*)_counters, 0, MAX_UNWIND_FAILURE_NAMES * (UNWIND_FAILURE_ANY + 1) * sizeof(u64)); - } - - ~UnwindFailures() { - delete[] _names; - delete[] _counters; - } + UnwindFailures(); + ~UnwindFailures(); // Disable copy constructor and assignment operator UnwindFailures(const UnwindFailures&) = delete; UnwindFailures& operator=(const UnwindFailures&) = delete; + void reset(); + void record(UnwindFailureKind kind, const char *name) { if (!name) return; diff --git a/ddprof-lib/src/main/cpp/vmEntry.cpp b/ddprof-lib/src/main/cpp/vmEntry.cpp index f6b6946ad..38534cba1 100644 --- a/ddprof-lib/src/main/cpp/vmEntry.cpp +++ b/ddprof-lib/src/main/cpp/vmEntry.cpp @@ -626,21 +626,26 @@ void *VM::getLibraryHandle(const char *name) { void JNICALL VM::ClassPrepare(jvmtiEnv* jvmti, JNIEnv* jni, jthread thread, jclass klass) { - ProfiledThread::initCurrentThreadSignalSafe(); + ProfiledThread* thr = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thr != nullptr); JVMSupport::loadMethodIDsIfNeeded(jvmti, jni, klass); } void JNICALL VM::ClassLoad(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread, jclass klass) { // Needed only for AsyncGetCallTrace support - ProfiledThread::initCurrentThreadSignalSafe(); + ProfiledThread* thr = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thr != nullptr); } void JNICALL VM::VMInit(jvmtiEnv* jvmti, JNIEnv* jni, jthread thread) { ready(jvmti, jni); - // initialize the heap usage tracking only after the VM is ready + ProfiledThread* thr = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thr != nullptr); + + // initialize the heap usage tracking only after the VM is ready HeapUsage::initJMXUsage(VM::jni()); // Delayed start of profiler if agent has been loaded at VM bootstrap @@ -655,6 +660,8 @@ Arguments& VM::arguments() { } void JNICALL VM::VMDeath(jvmtiEnv *jvmti, JNIEnv *jni) { + ProfiledThread* thr = ProfiledThread::initCurrentThreadSignalSafe(); + assert(thr != nullptr); Profiler::instance()->shutdown(_agent_args); } diff --git a/ddprof-lib/src/main/cpp/wallClock.cpp b/ddprof-lib/src/main/cpp/wallClock.cpp index 798da6c3d..a41029eba 100644 --- a/ddprof-lib/src/main/cpp/wallClock.cpp +++ b/ddprof-lib/src/main/cpp/wallClock.cpp @@ -21,6 +21,7 @@ #include "threadState.inline.h" #include "guards.h" #include "wallClockCounters.h" +#include #include #include #include @@ -231,18 +232,28 @@ void WallClockASGCT::sharedSignalHandler(int signo, siginfo_t *siginfo, void WallClockASGCT::signalHandler(int signo, siginfo_t *siginfo, void *ucontext, u64 last_sample) { + // A thread with no ProfiledThread attached must never enter the critical + // section below. acquireCurrent() is the only thing that can attach one; it + // must fully succeed or fail before we try to claim exclusivity, not while + // we're holding it -- otherwise a signal that interrupts us right after + // publish could observe a ProfiledThread whose critical-section state + // doesn't yet reflect reality. Drop the sample instead. + ProfiledThread *current = ProfiledThread::acquireCurrent(); + if (current == nullptr) { + Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); + return; + } + // Atomically try to enter critical section - prevents all reentrancy races CriticalSection cs; if (!cs.entered()) { return; // Another critical section is active, defer profiling } - ProfiledThread *current = ProfiledThread::current(); // Guard against the race window between Profiler::registerThread() and // thread_native_entry setting JVM TLS (PROF-13072): skip at most one signal // per thread. Pure native threads (where JVMThread::current() is always null) // are allowed through once the one-shot window expires. - if (current != nullptr && JVMThread::current() == nullptr - && current->inInitWindow()) { + if (JVMThread::current() == nullptr && current->inInitWindow()) { current->tickInitWindow(); return; } @@ -256,10 +267,10 @@ void WallClockASGCT::signalHandler(int signo, siginfo_t *siginfo, void *ucontext if (precheck.suppress) { return; } - int tid = current != NULL ? current->tid() : OS::threadId(); + int tid = current->tid(); Shims::instance().setSighandlerTid(tid); u64 call_trace_id = 0; - if (current != NULL && _collapsing) { + if (_collapsing) { StackFrame frame(ucontext); u64 spanId = 0, rootSpanId = 0; // contextValid is not redundant with (spanId==0 && rootSpanId==0): a cleared @@ -447,18 +458,21 @@ void WallClockJvmti::signalHandler(int signo, siginfo_t *siginfo, } int saved_errno = errno; ProfiledThread *current = ProfiledThread::current(); - if (current != nullptr && JVMThread::current() == nullptr + assert(current != nullptr && "Should have been setup at signal handler entery"); + + if (JVMThread::current() == nullptr && current->inInitWindow()) { current->tickInitWindow(); errno = saved_errno; return; } + WallPrecheckResult precheck = prepareWallPrecheck(current, _precheck); if (precheck.suppress) { errno = saved_errno; return; } - int tid = current != NULL ? current->tid() : OS::threadId(); + int tid = current->tid(); Shims::instance().setSighandlerTid(tid); ExecutionEvent event; diff --git a/ddprof-lib/src/main/cpp/wallClock.h b/ddprof-lib/src/main/cpp/wallClock.h index 14e3f88aa..7e2c7fefa 100644 --- a/ddprof-lib/src/main/cpp/wallClock.h +++ b/ddprof-lib/src/main/cpp/wallClock.h @@ -12,7 +12,7 @@ #include "os.h" #include "profiler.h" #include "reservoirSampler.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "threadFilter.h" #include "threadState.h" #include "tsc.h" diff --git a/ddprof-lib/src/test/cpp/faultInjection_ut.cpp b/ddprof-lib/src/test/cpp/faultInjection_ut.cpp index 46c19b953..7d64d2909 100644 --- a/ddprof-lib/src/test/cpp/faultInjection_ut.cpp +++ b/ddprof-lib/src/test/cpp/faultInjection_ut.cpp @@ -14,7 +14,7 @@ #include "faultInjection.h" #include "safeAccess.h" #include "os.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "profiler.h" #include "../../main/cpp/gtest_crash_handler.h" diff --git a/ddprof-lib/src/test/cpp/hotspot_crash_protection_ut.cpp b/ddprof-lib/src/test/cpp/hotspot_crash_protection_ut.cpp index 3739606bb..2131ecac0 100644 --- a/ddprof-lib/src/test/cpp/hotspot_crash_protection_ut.cpp +++ b/ddprof-lib/src/test/cpp/hotspot_crash_protection_ut.cpp @@ -30,7 +30,7 @@ */ #include -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "profiler.h" #include "asyncSampleMutex.h" diff --git a/ddprof-lib/src/test/cpp/stackWalker_ut.cpp b/ddprof-lib/src/test/cpp/stackWalker_ut.cpp index 1212b704d..efe19492d 100644 --- a/ddprof-lib/src/test/cpp/stackWalker_ut.cpp +++ b/ddprof-lib/src/test/cpp/stackWalker_ut.cpp @@ -3,18 +3,18 @@ */ #include -#include "../../main/cpp/stackWalker.h" -#include "../../main/cpp/gtest_crash_handler.h" +#include "stackWalker.h" +#include "gtest_crash_handler.h" #ifdef __linux__ #include #include #include -#include "../../main/cpp/counters.h" -#include "../../main/cpp/os.h" -#include "../../main/cpp/profiler.h" -#include "../../main/cpp/stackFrame.h" -#include "../../main/cpp/threadLocalData.h" +#include "counters.h" +#include "os.h" +#include "profiler.h" +#include "stackFrame.h" +#include "threadLocalData.inline.h" [[maybe_unused]] static long long* _stackwalker_ut_counters_init = Counters::getCounters(); #endif diff --git a/ddprof-lib/src/test/cpp/stress_callTraceStorage.cpp b/ddprof-lib/src/test/cpp/stress_callTraceStorage.cpp index a8706967d..4cd5690a6 100644 --- a/ddprof-lib/src/test/cpp/stress_callTraceStorage.cpp +++ b/ddprof-lib/src/test/cpp/stress_callTraceStorage.cpp @@ -8,6 +8,7 @@ #include "callTraceHashTable.h" #include "guards.h" #include "common.h" // TSAN_ENABLED (toolchain-agnostic sanitizer detection) +#include "threadLocalData.h" #include #include #include @@ -30,7 +31,7 @@ #include #include #include -#include "../../main/cpp/gtest_crash_handler.h" +#include "gtest_crash_handler.h" // Test name for crash handler static constexpr const char STRESS_TEST_NAME[] = "StressCallTraceStorage"; diff --git a/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp b/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp index 5615010e0..8ad8a336e 100644 --- a/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp +++ b/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp @@ -16,7 +16,7 @@ #include "callTraceStorage.h" #include "callTraceHashTable.h" #include "threadFilter.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "arch.h" #include "spinLock.h" diff --git a/ddprof-lib/src/test/cpp/threadFilter_lifecycle_ut.cpp b/ddprof-lib/src/test/cpp/threadFilter_lifecycle_ut.cpp index 9ca9b082e..dab8333f0 100644 --- a/ddprof-lib/src/test/cpp/threadFilter_lifecycle_ut.cpp +++ b/ddprof-lib/src/test/cpp/threadFilter_lifecycle_ut.cpp @@ -19,7 +19,7 @@ #ifdef __linux__ #include "threadFilter.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "../../main/cpp/gtest_crash_handler.h" #include diff --git a/ddprof-lib/src/test/cpp/threadLocalDataPool_ut.cpp b/ddprof-lib/src/test/cpp/threadLocalDataPool_ut.cpp new file mode 100644 index 000000000..038354e11 --- /dev/null +++ b/ddprof-lib/src/test/cpp/threadLocalDataPool_ut.cpp @@ -0,0 +1,98 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +// threadLocalData.h must precede threadLocalDataPool.h: ProfiledThread needs to +// be a complete type before contains()'s pointer arithmetic is parsed (the +// pool header only forward-declares it). +#include "threadLocalData.h" +#include "threadLocalDataPool.h" +#include "counters.h" + +// Covers ThreadLocalDataPool::contains(), whose result feeds directly into +// unclaim()'s double-release guard. Uses createForTest()/destroyForTest() to +// build a pool isolated from the process-wide singleton (_pool), so these +// boundary checks don't interact with other tests' initialize()/acquire()/ +// release() calls. +class ThreadLocalDataPoolTest : public ::testing::Test {}; + +TEST_F(ThreadLocalDataPoolTest, firstElementIsContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + ProfiledThread* base = pool->threadsForTest(); + + EXPECT_TRUE(pool->containsForTest(base)); + + ThreadLocalDataPool::destroyForTest(pool); +} + +TEST_F(ThreadLocalDataPoolTest, lastElementIsContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + ProfiledThread* base = pool->threadsForTest(); + uint64_t capacity = pool->capacityForTest(); + + EXPECT_TRUE(pool->containsForTest(base + (capacity - 1))); + + ThreadLocalDataPool::destroyForTest(pool); +} + +TEST_F(ThreadLocalDataPoolTest, onePastEndIsNotContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + ProfiledThread* base = pool->threadsForTest(); + uint64_t capacity = pool->capacityForTest(); + + EXPECT_FALSE(pool->containsForTest(base + capacity)); + + ThreadLocalDataPool::destroyForTest(pool); +} + +TEST_F(ThreadLocalDataPoolTest, oneBeforeStartIsNotContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + ProfiledThread* base = pool->threadsForTest(); + + EXPECT_FALSE(pool->containsForTest(base - 1)); + + ThreadLocalDataPool::destroyForTest(pool); +} + +TEST_F(ThreadLocalDataPoolTest, nullptrIsNotContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + + EXPECT_FALSE(pool->containsForTest(nullptr)); + + ThreadLocalDataPool::destroyForTest(pool); +} + +// Covers claim()'s (used >= _capacity) fast-path rejection once the pool is +// full. claim()'s return value is nullptr whether that guard reads `>=` or +// (bugged) `>`, because a bugged `>` still falls through to the slot-scanning +// loop, which finds every real slot already claimed and also returns nullptr. +// The observable difference is that the buggy fallthrough additionally +// increments SAMPLES_DROPPED_TLS_POOL_EXHAUSTED, which the fast path must not +// do -- that's what this test pins down. +TEST_F(ThreadLocalDataPoolTest, claimAtCapacityRejectsWithoutExhaustionScan) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(2); + + ASSERT_NE(pool->claimForTest(0), nullptr); + ASSERT_NE(pool->claimForTest(1), nullptr); + + long long before = Counters::getCounter(SAMPLES_DROPPED_TLS_POOL_EXHAUSTED); + EXPECT_EQ(pool->claimForTest(2), nullptr); + long long after = Counters::getCounter(SAMPLES_DROPPED_TLS_POOL_EXHAUSTED); + + EXPECT_EQ(after, before); + + ThreadLocalDataPool::destroyForTest(pool); +} diff --git a/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp b/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp index 02f8c3f50..9195f27ad 100644 --- a/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp +++ b/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp @@ -20,7 +20,7 @@ #include "guards.h" #include "nativeMem.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include #include diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java new file mode 100644 index 000000000..6345a25b4 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.datadoghq.profiler.cpu; + +import com.datadoghq.profiler.AbstractProfilerTest; +import com.datadoghq.profiler.Platform; +import org.junitpioneer.jupiter.RetryingTest; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +import java.lang.reflect.Method; +import java.util.Map; +import java.util.TreeSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Validates TLS priming by checking that CPU-time profiling captures samples + * from the JVM's JIT compiler threads (HotSpot's "C1 CompilerThread*", + * OpenJ9's "JIT Compilation Thread*"). + * + * Compiler threads are started very early during JVM bootstrap, usually + * before the profiler agent has attached and initialized. Because of that, + * they never go through the normal thread-registration path the profiler + * uses for application threads, so they have no ProfiledThread attached when + * the first profiling signal reaches them. TLS priming is what covers this + * gap: on that first signal, ProfiledThread::acquireCurrent() claims a slot + * from the pool and attaches it via pthread_setspecific right there in the + * signal handler (see threadLocalData.cpp/ThreadLocalDataPool). + * + * Seeing a compiler-thread eventThread alone is not sufficient evidence: the + * CPU signal handlers resolve tid via OS::threadId() before recordSample() is + * even called, and native thread names are refreshed independently of TLS + * priming, so a sample tagged with a compiler thread's name would show up + * regardless of whether ProfiledThread::acquireCurrent() actually succeeded. + * If priming fails, recordSample() still emits an event for that tid, just + * with a synthetic "no_Java_frame" stack instead of a real unwind. The + * "samples_dropped_thread_local" debug counter is incremented exactly when + * acquireCurrent() fails (see StackWalker::walkFP/walkDwarf), so asserting it + * stayed at zero for the whole run is what actually proves every signal that + * reached a stack walker — including the ones on these never-registered + * compiler threads — found or attached a ProfiledThread. + * + * The test forces JIT compilation by loading a dynamically-generated class + * with many distinct trivial methods and invoking each one past HotSpot's/ + * OpenJ9's cold-to-compiled threshold, then leaves the CPU sampler running + * long enough for the background compiler thread(s) to actually drain the + * resulting compile queue. + */ +public class TlsPrimingTest extends AbstractProfilerTest { + + private static final String HOTSPOT_COMPILER_THREAD_PREFIX = "C1 CompilerThre"; + private static final String J9_COMPILER_THREAD_PREFIX = "JIT Compilation Thread"; + + // Distinct methods, each invoked enough times to individually cross the + // JIT's cold-to-compiled invocation threshold, so the compiler queue has + // sustained work rather than a single instantly-finished compile. + private static final int METHOD_COUNT = 300; + private static final int INVOCATIONS_PER_METHOD = 1000; + + // Compilation happens asynchronously on background compiler threads, so + // after tripping the invocation thresholds we must give them real time to + // drain the queue while the CPU sampler is still active. + private static final long COMPILE_DRAIN_WAIT_MS = 4000; + + @RetryingTest(3) + public void compilerThreadSamplesArePresent() throws Exception { + triggerJitCompilation(); + + stopProfiler(); + + IItemCollection events = verifyEvents("datadog.ExecutionSample"); + String expectedPrefix = Platform.isJ9() ? J9_COMPILER_THREAD_PREFIX : HOTSPOT_COMPILER_THREAD_PREFIX; + + Set observedThreadNames = new TreeSet<>(); + boolean sawCompilerThreadSample = false; + for (IItemIterable cpuSamples : events) { + IMemberAccessor threadNameAccessor = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(cpuSamples.getType()); + for (IItem sample : cpuSamples) { + String threadName = threadNameAccessor.getMember(sample); + if (threadName == null) { + continue; + } + observedThreadNames.add(threadName); + if (threadName.startsWith(expectedPrefix)) { + sawCompilerThreadSample = true; + } + } + } + + assertTrue(sawCompilerThreadSample, + "expected a datadog.ExecutionSample with eventThread starting with \"" + expectedPrefix + + "\", but observed thread names: " + observedThreadNames); + + // A compiler-thread eventThread on its own doesn't prove a pool slot was + // ever attached (see class javadoc) — the tid and thread name are resolved + // independently of priming. Confirm no signal ever fell back to the + // "no_Java_frame" stack for lack of a ProfiledThread, on this or any other + // thread in the run. + Map debugCounters = profiler.getDebugCounters(); + assertEquals(0L, debugCounters.get("samples_dropped_thread_local"), + "TLS priming failed for at least one signal; compiler-thread samples " + + "may have used the no_Java_frame fallback instead of a real unwind"); + } + + private void triggerJitCompilation() throws Exception { + Class generated = defineWorkloadClass(); + Object instance = generated.getDeclaredConstructor().newInstance(); + Method[] methods = generated.getDeclaredMethods(); + + for (int call = 0; call < INVOCATIONS_PER_METHOD; call++) { + for (Method m : methods) { + m.invoke(instance); + } + } + + Thread.sleep(COMPILE_DRAIN_WAIT_MS); + } + + /** + * Generates a class with {@value #METHOD_COUNT} distinct no-arg int-returning + * methods (each a different constant expression, so the JIT can't fold them + * into one shared compiled method) and loads it in a fresh ClassLoader. + */ + private static Class defineWorkloadClass() throws ClassFormatError { + String internalName = "com/datadoghq/profiler/cpu/generated/CompilerThreadWorkload"; + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); + cw.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, internalName, null, "java/lang/Object", null); + + MethodVisitor ctor = cw.visitMethod(Opcodes.ACC_PUBLIC, "", "()V", null, null); + ctor.visitCode(); + ctor.visitVarInsn(Opcodes.ALOAD, 0); + ctor.visitMethodInsn(Opcodes.INVOKESPECIAL, "java/lang/Object", "", "()V", false); + ctor.visitInsn(Opcodes.RETURN); + ctor.visitMaxs(0, 0); + ctor.visitEnd(); + + for (int i = 0; i < METHOD_COUNT; i++) { + MethodVisitor mv = cw.visitMethod(Opcodes.ACC_PUBLIC, "method" + i, "()I", null, null); + mv.visitCode(); + mv.visitIntInsn(Opcodes.SIPUSH, i); + mv.visitIntInsn(Opcodes.SIPUSH, i + 1); + mv.visitInsn(Opcodes.IMUL); + mv.visitInsn(Opcodes.IRETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + } + cw.visitEnd(); + + IsolatedClassLoader loader = new IsolatedClassLoader(TlsPrimingTest.class.getClassLoader()); + return loader.defineClass(internalName.replace('/', '.'), cw.toByteArray()); + } + + private static final class IsolatedClassLoader extends ClassLoader { + IsolatedClassLoader(ClassLoader parent) { + super(parent); + } + + Class defineClass(String name, byte[] bytecode) { + return defineClass(name, bytecode, 0, bytecode.length); + } + } + + @Override + protected String getProfilerCommand() { + return "cpu=1ms"; + } +}