From 8d3cbb196e3c746389e5b47ff6457d5873c083cf Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 3 Aug 2026 19:55:44 +0000 Subject: [PATCH 01/16] Restore priming --- ddprof-lib/src/main/cpp/context_api.cpp | 2 +- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 1 + ddprof-lib/src/main/cpp/flightRecorder.cpp | 1 + ddprof-lib/src/main/cpp/guards.cpp | 2 +- .../src/main/cpp/hotspot/hotspotSupport.cpp | 23 +++--- ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp | 2 +- ddprof-lib/src/main/cpp/itimer.cpp | 2 +- ddprof-lib/src/main/cpp/javaApi.cpp | 2 +- ddprof-lib/src/main/cpp/jvmSupport.cpp | 7 +- ddprof-lib/src/main/cpp/jvmThread.h | 14 ++++ ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 2 +- ddprof-lib/src/main/cpp/refCountGuard.cpp | 2 +- ddprof-lib/src/main/cpp/stackWalker.cpp | 35 ++++---- ddprof-lib/src/main/cpp/threadLocalData.cpp | 15 ++-- ddprof-lib/src/main/cpp/threadLocalData.h | 34 ++++++-- .../src/main/cpp/threadLocalData.inline.h | 31 +++++++ .../src/main/cpp/threadLocalDataPool.cpp | 82 +++++++++++++++++++ ddprof-lib/src/main/cpp/threadLocalDataPool.h | 42 ++++++++++ ddprof-lib/src/main/cpp/wallClock.h | 2 +- 19 files changed, 250 insertions(+), 51 deletions(-) create mode 100644 ddprof-lib/src/main/cpp/threadLocalData.inline.h create mode 100644 ddprof-lib/src/main/cpp/threadLocalDataPool.cpp create mode 100644 ddprof-lib/src/main/cpp/threadLocalDataPool.h diff --git a/ddprof-lib/src/main/cpp/context_api.cpp b/ddprof-lib/src/main/cpp/context_api.cpp index 082131a921..8a4f556f58 100644 --- a/ddprof-lib/src/main/cpp/context_api.cpp +++ b/ddprof-lib/src/main/cpp/context_api.cpp @@ -19,7 +19,7 @@ #include "guards.h" #include "otel_context.h" #include "profiler.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include /** diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 11502c4883..a6a3c12ddd 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -27,6 +27,7 @@ #include "log.h" #include "profiler.h" #include "signalCookie.h" +#include "threadLocalData.inline.h" #include "threadState.inline.h" #include #include diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index 6064e594cc..b71b841415 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -28,6 +28,7 @@ #include "unwindStats.h" #include "symbols.h" #include "threadFilter.h" +#include "threadLocalData.inline.h" #include "threadState.h" #include "tsc.h" #include "hotspot/vmStructs.h" diff --git a/ddprof-lib/src/main/cpp/guards.cpp b/ddprof-lib/src/main/cpp/guards.cpp index 9905182e9a..f0651d7a2c 100644 --- a/ddprof-lib/src/main/cpp/guards.cpp +++ b/ddprof-lib/src/main/cpp/guards.cpp @@ -17,7 +17,7 @@ #include "guards.h" #include "common.h" #include "os.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" // Signal-context tracking — backed by ProfiledThread::_signal_depth; see // the comment block in guards.h for the rationale (initial-exec TLS was diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp index 66057d141a..af045189db 100644 --- a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp +++ b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp @@ -18,6 +18,7 @@ #include "profiler.h" #include "stackWalker.inline.h" #include "threadLocal.h" +#include "threadLocalData.inline.h" using StackWalkValidation::inDeadZone; using StackWalkValidation::aligned; @@ -240,7 +241,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex // VMStructs is only available for hotspot JVM assert(VM::isHotspot()); - ProfiledThread* prof_thread = ProfiledThread::current(); + ProfiledThread* prof_thread = ProfiledThread::acquire_current(); if (prof_thread == nullptr) { Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); return 0; @@ -1210,27 +1211,25 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) { // AsyncGetCallTrace. Install a jmp ctx here too, so a SIGSEGV anywhere in // walkJavaStack, except HotSpot's AsyncGetCallTrace call, is caught by // Profiler::checkFault() and siglongjmp'd back here instead of crashing the process. - ProfiledThread* prof_thread = ProfiledThread::current(); - const bool prev_unwinding_java = prof_thread != nullptr ? prof_thread->is_unwinding_Java() : false; + ProfiledThread* prof_thread = ProfiledThread::acquire_current(); + if (prof_thread == nullptr) { + Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); + return 0; + } sigjmp_buf crash_protection_ctx; sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr; - if (prof_thread != nullptr && sigsetjmp(crash_protection_ctx, 1) != 0) { + if (sigsetjmp(crash_protection_ctx, 1) != 0) { // checkFault() does a siglongjmp from inside segvHandler, bypassing // segvHandler's SignalHandlerScope destructor. Compensate. SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP(); prof_thread->setJmpCtx(prev_jmp_buf); - // A recovered siglongjmp bypasses AsyncSampleMutex destructors, so restore - // the per-thread guard to its pre-walk value. - prof_thread->set_unwinding_Java(prev_unwinding_java); if (truncated) { *truncated = true; } return java_frames; } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(&crash_protection_ctx); - } + prof_thread->setJmpCtx(&crash_protection_ctx); if (features.mixed) { java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated); @@ -1284,9 +1283,7 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) { } } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(prev_jmp_buf); - } + prof_thread->setJmpCtx(prev_jmp_buf); return java_frames; } diff --git a/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp b/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp index 73d7e2aec6..c1a1c59ecf 100644 --- a/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp +++ b/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp @@ -15,7 +15,7 @@ #include "jvmThread.h" #include "safeAccess.h" #include "spinLock.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "threadState.h" CodeCache* VMStructs::_libjvm = nullptr; diff --git a/ddprof-lib/src/main/cpp/itimer.cpp b/ddprof-lib/src/main/cpp/itimer.cpp index 0c1a134df3..820ba341b4 100644 --- a/ddprof-lib/src/main/cpp/itimer.cpp +++ b/ddprof-lib/src/main/cpp/itimer.cpp @@ -22,7 +22,7 @@ #include "profiler.h" #include "signalInflight.h" #include "stackWalker.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "threadState.inline.h" #include "guards.h" #include diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 3f71a1b74c..380b3058c7 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -31,7 +31,7 @@ #include "os.h" #include "otel_process_ctx.h" #include "profiler.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "tsc.h" #include "vmEntry.h" #include diff --git a/ddprof-lib/src/main/cpp/jvmSupport.cpp b/ddprof-lib/src/main/cpp/jvmSupport.cpp index b6277a6be4..0068d211fd 100644 --- a/ddprof-lib/src/main/cpp/jvmSupport.cpp +++ b/ddprof-lib/src/main/cpp/jvmSupport.cpp @@ -9,7 +9,8 @@ #include "frames.h" #include "os.h" #include "profiler.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" +#include "threadLocalDataPool.h" #include "vmEntry.h" #include "hotspot/hotspotSupport.h" @@ -38,6 +39,10 @@ bool JVMSupport::initialize() { return false; } + if (JVMThread::supportPriming()) { + ThreadLocalDataPool::initialize(); + } + // Check ProfiledThread key, it is critical for storing per-thread metadata return ProfiledThread::isThreadKeyValid(); } diff --git a/ddprof-lib/src/main/cpp/jvmThread.h b/ddprof-lib/src/main/cpp/jvmThread.h index 2f5bd69104..bbc3a79b9c 100644 --- a/ddprof-lib/src/main/cpp/jvmThread.h +++ b/ddprof-lib/src/main/cpp/jvmThread.h @@ -10,12 +10,16 @@ #include #include "threadLocal.h" +#include "os.h" /** * JVMThread represents a native JVM thread that is JVM implementation agnostic */ class JVMThread { private: + // PTHREAD_KEY_2NDLEVEL_SIZE is an internal macro set to 32 in the GNU C Library (glibc) NPTL + // implementation. Slot indexes less than PTHREAD_KEY_2NDLEVEL_SIZE are pre-allocated. + static constexpr int PTHREAD_KEY_2NDLEVEL_SIZE = 32; static jfieldID _tid; static ThreadLocal _jvm_thread; @@ -41,6 +45,16 @@ class JVMThread { return _jvm_thread.key(); } + static bool supportPriming() { + // Key must be valid + assert(_jvm_thread.isKeyValid()); + if (OS::isMusl()) { + return true; + } else { + return _jvm_thread.key() < PTHREAD_KEY_2NDLEVEL_SIZE; + } + } + static int nativeThreadId(JNIEnv* jni, jthread thread); static inline jlong javaThreadId(JNIEnv* env, jthread thread) { return env->GetLongField(thread, _tid); diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index b3e5028135..a111a5a14f 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -35,7 +35,7 @@ #include "stackFrame.h" #include "stackWalker.h" #include "symbols.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "threadState.inline.h" #include #include diff --git a/ddprof-lib/src/main/cpp/refCountGuard.cpp b/ddprof-lib/src/main/cpp/refCountGuard.cpp index 5f37d74a43..7af5a3216d 100644 --- a/ddprof-lib/src/main/cpp/refCountGuard.cpp +++ b/ddprof-lib/src/main/cpp/refCountGuard.cpp @@ -10,7 +10,7 @@ #include "log.h" #include "os.h" #include "primeProbing.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include #include diff --git a/ddprof-lib/src/main/cpp/stackWalker.cpp b/ddprof-lib/src/main/cpp/stackWalker.cpp index 4de65b5805..3912cfebd3 100644 --- a/ddprof-lib/src/main/cpp/stackWalker.cpp +++ b/ddprof-lib/src/main/cpp/stackWalker.cpp @@ -13,7 +13,7 @@ #include "symbols.h" #include "jvmSupport.inline.h" #include "jvmThread.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" // Use validation helpers from header (shared with tests) using StackWalkValidation::inDeadZone; @@ -45,11 +45,15 @@ int StackWalker::walkFP(void* ucontext, const void** callchain, int max_depth, S // falls inside this library while a jmp ctx is installed gets caught by // Profiler::checkFault() from the SEGV handler and siglongjmp'd back here, // instead of crashing the process. - ProfiledThread* prof_thread = ProfiledThread::current(); + ProfiledThread* prof_thread = ProfiledThread::acquire_current(); + if (prof_thread == nullptr) { + Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); + return 0; + } sigjmp_buf crash_protection_ctx; - sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr; + sigjmp_buf* prev_jmp_buf = prof_thread->getJmpCtx(); - if (prof_thread != nullptr && sigsetjmp(crash_protection_ctx, 1) != 0) { + if (sigsetjmp(crash_protection_ctx, 1) != 0) { // checkFault() does a siglongjmp from inside segvHandler, bypassing // segvHandler's SignalHandlerScope destructor. Compensate. SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP(); @@ -62,9 +66,7 @@ int StackWalker::walkFP(void* ucontext, const void** callchain, int max_depth, S } return depth; } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(&crash_protection_ctx); - } + prof_thread->setJmpCtx(&crash_protection_ctx); // Walk until the bottom of the stack or until the first Java frame while (depth < actual_max_depth) { @@ -132,11 +134,16 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth // falls inside this library while a jmp ctx is installed gets caught by // Profiler::checkFault() from the SEGV handler and siglongjmp'd back here, // instead of crashing the process. - ProfiledThread* prof_thread = ProfiledThread::current(); + ProfiledThread* prof_thread = ProfiledThread::acquire_current(); + if (prof_thread == nullptr) { + Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); + return 0; + } + sigjmp_buf crash_protection_ctx; - sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr; + sigjmp_buf* prev_jmp_buf = prof_thread->getJmpCtx(); - if (prof_thread != nullptr && sigsetjmp(crash_protection_ctx, 1) != 0) { + if (sigsetjmp(crash_protection_ctx, 1) != 0) { // checkFault() does a siglongjmp from inside segvHandler, bypassing // segvHandler's SignalHandlerScope destructor. Compensate. SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP(); @@ -149,9 +156,7 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth } return depth; } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(&crash_protection_ctx); - } + prof_thread->setJmpCtx(&crash_protection_ctx); // Walk until the bottom of the stack or until the first Java frame while (depth < actual_max_depth) { @@ -229,9 +234,7 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth } } - if (prof_thread != nullptr) { - prof_thread->setJmpCtx(prev_jmp_buf); - } + prof_thread->setJmpCtx(prev_jmp_buf); if (truncated && depth > max_depth) { *truncated = true; diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index 37942fe36f..2ff72b896b 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -3,7 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "threadLocalData.h" +#include "threadLocalData.inline.h" +#include "threadLocalDataPool.h" #include "context_api.h" #include "guards.h" #include "otel_context.h" @@ -49,11 +50,13 @@ ProfiledThread* ProfiledThread::initCurrentThreadSignalSafe() { void ProfiledThread::freeValue(void* value) { SignalBlocker blocker; ProfiledThread* pt = reinterpret_cast(value); - // Sole deletion site for a ProfiledThread (invoked by the ThreadLocal - // destructor callback), so the THREAD_LOCAL decrement belongs here. Record - // after the delete, consistent with the other decrement sites. - delete pt; - NativeMem::record(NM_THREAD_LOCAL, -(long long)sizeof(ProfiledThread)); + if (!ThreadLocalDataPool::release(pt)) { + // Sole deletion site for a ProfiledThread (invoked by the ThreadLocal + // destructor callback), so the THREAD_LOCAL decrement belongs here. Record + // after the delete, consistent with the other decrement sites. + delete pt; + NativeMem::record(NM_THREAD_LOCAL, -(long long)sizeof(ProfiledThread)); + } } void ProfiledThread::release() { diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index 30f750cb15..46fce44af3 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -37,6 +37,8 @@ class ThreadLocalData { }; class ProfiledThread : public ThreadLocalData { + friend class ThreadLocalDataPool; + public: enum ThreadType : u32 { TYPE_UNKNOWN = 0, @@ -46,6 +48,8 @@ class ProfiledThread : public ThreadLocalData { }; static constexpr u32 FLAG_PARKED = 0x4u; // next free bit after TYPE_MASK (0x1|0x2) + static constexpr u32 FLAG_CLAIMED = 0x8u; // Used by ThreadLocalDataPool only + // We are allowing several levels of nesting because we can be // eg. in a crash handler when wallclock signal kicks in, @@ -74,7 +78,7 @@ class ProfiledThread : public ThreadLocalData { u32 _wall_epoch; u64 _call_trace_id; u32 _recording_epoch; - u32 _misc_flags; + volatile u32 _misc_flags; u64 _park_block_token; int _filter_slot_id; // Slot ID for thread filtering uint8_t _init_window; // Countdown for JVM thread init race window (PROF-13072) @@ -112,6 +116,24 @@ class ProfiledThread : public ThreadLocalData { }; virtual ~ProfiledThread() { } + + inline bool isClaimed() const { + return (__atomic_load_n(&_misc_flags, __ATOMIC_RELAXED) & FLAG_CLAIMED) == FLAG_CLAIMED; + } + + inline bool claim_acquire(int tid) { + if (isClaimed()) { + return false; + } + + u32 flags = __atomic_fetch_or(&_misc_flags, FLAG_CLAIMED, __ATOMIC_ACQUIRE); + bool rc = (flags & FLAG_CLAIMED) == 0; + if (rc) { + _tid = tid; + } + return rc; +} + public: static ProfiledThread *forTid(int tid) { ProfiledThread *pt = new ProfiledThread(tid); @@ -154,12 +176,10 @@ class ProfiledThread : public ThreadLocalData { static ProfiledThread* initCurrentThreadSignalSafe(); // Signal-handler friendly (no allocation): returns existing TLS or nullptr. - static inline ProfiledThread *current() { - if (!isThreadKeyValid()) { - return nullptr; - } - return _current_thread.get(); - } + static inline ProfiledThread *current(); + // signal-handler friendly with priming: return existing TLS or acquire and set + // ProfiledThread from ThreadLocalDataPool. + static inline ProfiledThread* acquire_current(); static int currentTid(); inline int tid() { return _tid; } diff --git a/ddprof-lib/src/main/cpp/threadLocalData.inline.h b/ddprof-lib/src/main/cpp/threadLocalData.inline.h new file mode 100644 index 0000000000..a4f60c9806 --- /dev/null +++ b/ddprof-lib/src/main/cpp/threadLocalData.inline.h @@ -0,0 +1,31 @@ +/* + * Copyright 2026 Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef THREADLOCALDATA_INLINE_H +#define THREADLOCALDATA_INLINE_H + +#include "os.h" +#include "threadLocalData.h" +#include "threadLocalDataPool.h" + +ProfiledThread* ProfiledThread::current() { + if (!isThreadKeyValid()) { + return nullptr; + } + return _current_thread.get(); +} + +ProfiledThread* ProfiledThread::acquire_current() { + ProfiledThread* prof_thread = current(); + if (prof_thread == nullptr) { + prof_thread = ThreadLocalDataPool::acquire(OS::threadId()); + if (prof_thread != nullptr) { + _current_thread.set(prof_thread); + } + } + return prof_thread; +} + +#endif // THREADLOCALDATA_INLINE_H diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp new file mode 100644 index 0000000000..7b15d12448 --- /dev/null +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -0,0 +1,82 @@ + + /* + * Copyright 2026 Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "threadLocalData.h" +#include "threadLocalDataPool.h" + +#include + +ThreadLocalDataPool* ThreadLocalDataPool::_pool = nullptr; + +ThreadLocalDataPool::ThreadLocalDataPool(uint64_t capacity) : _capacity(capacity), _used(0) { + size_t malloc_size = capacity * sizeof(ProfiledThread); + void* p = malloc(malloc_size); + if (p != nullptr) { + _threads = reinterpret_cast(p); + for (int index = 0; index < capacity; index++) { + new (&_threads[index])ProfiledThread(0); + } + NativeMem::record(NM_THREAD_LOCAL, malloc_size); + } +} + +ThreadLocalDataPool::~ThreadLocalDataPool() { + if (_threads != nullptr) { + free(reinterpret_cast(_threads)); + } +} + +ProfiledThread* ThreadLocalDataPool::claim(int tid) { + uint16_t used = __atomic_fetch_add(&_used, 1, __ATOMIC_RELAXED); + if (used >= _capacity) { + __atomic_fetch_add(&_used, -1, __ATOMIC_RELAXED); + return nullptr; + } + + int start_pos = tid % _capacity; + int index = start_pos; + do { + if (_threads[index].claim_acquire(tid)) { + return &_threads[index]; + } + index = (index + 1) % _capacity; + } while (index != start_pos); + assert(false && "Should not reach here"); + return nullptr; +} + +bool ThreadLocalDataPool::unclaim(ProfiledThread* t) { + if (contains(t)) { + new (t)ProfiledThread(0); + uint16_t used = __atomic_fetch_add(&_used, -1, __ATOMIC_RELEASE); + assert(used > 0); + return true; + } + return false; +} + +void ThreadLocalDataPool::initialize() { + ThreadLocalDataPool* pool = new ThreadLocalDataPool(); + __atomic_store_n(&_pool, pool, __ATOMIC_RELEASE); +} + +ProfiledThread* ThreadLocalDataPool::acquire(int tid) { + ThreadLocalDataPool* pool = __atomic_load_n(&_pool, __ATOMIC_ACQUIRE); + if (pool == nullptr) { + return nullptr; + } else { + return pool->claim(tid); + } +} + +bool ThreadLocalDataPool::release(ProfiledThread* t) { + ThreadLocalDataPool* pool = __atomic_load_n(&_pool, __ATOMIC_ACQUIRE); + if (pool != nullptr) { + return pool->unclaim(t); + } else { + return false; + } +} \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.h b/ddprof-lib/src/main/cpp/threadLocalDataPool.h new file mode 100644 index 0000000000..1971880074 --- /dev/null +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.h @@ -0,0 +1,42 @@ +/* + * Copyright 2026 Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef THREADLOCALDATA_POOL_H +#define THREADLOCALDATA_POOL_H + +#include + +class ProfiledThread; + +class ThreadLocalDataPool { + static constexpr uint16_t DEFAULT_CAPACITY = 64; +private: + static ThreadLocalDataPool* _pool; + + const uint64_t _capacity; + volatile uint16_t _used; + ProfiledThread* _threads; + + ThreadLocalDataPool(const ThreadLocalDataPool&) = delete; + ThreadLocalDataPool& operator=(const ThreadLocalDataPool&) = delete; + + ThreadLocalDataPool(uint64_t capacity = DEFAULT_CAPACITY); + ~ThreadLocalDataPool(); + + ProfiledThread* claim(int tid); + bool unclaim(ProfiledThread* t); + + + inline bool contains(ProfiledThread* t) { + return t >= _threads && t <= &_threads[_capacity - 1]; + } + +public: + static void initialize(); + static ProfiledThread* acquire(int tid); + static bool release(ProfiledThread* t); +}; + +#endif // THREADLOCALDATA_POOL_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/wallClock.h b/ddprof-lib/src/main/cpp/wallClock.h index 14e3f88aa3..7e2c7fefa7 100644 --- a/ddprof-lib/src/main/cpp/wallClock.h +++ b/ddprof-lib/src/main/cpp/wallClock.h @@ -12,7 +12,7 @@ #include "os.h" #include "profiler.h" #include "reservoirSampler.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "threadFilter.h" #include "threadState.h" #include "tsc.h" From 856ee133a68fe770ffcfa01dd89dffccb4305ee2 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 3 Aug 2026 22:10:41 +0200 Subject: [PATCH 02/16] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- ddprof-lib/src/main/cpp/threadLocalDataPool.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.h b/ddprof-lib/src/main/cpp/threadLocalDataPool.h index 1971880074..93f0ec5a0c 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.h +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.h @@ -29,8 +29,12 @@ class ThreadLocalDataPool { bool unclaim(ProfiledThread* t); - inline bool contains(ProfiledThread* t) { - return t >= _threads && t <= &_threads[_capacity - 1]; + inline bool contains(ProfiledThread* t) const { + if (_threads == nullptr || t == nullptr) return false; + const uintptr_t addr = reinterpret_cast(t); + const uintptr_t base = reinterpret_cast(_threads); + const uintptr_t end = reinterpret_cast(_threads + _capacity); + return addr >= base && addr < end; } public: From 6a32799d44b394f56aabc01f849cf15929436f92 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 3 Aug 2026 20:54:49 +0000 Subject: [PATCH 03/16] Fix --- .../src/main/cpp/hotspot/hotspotSupport.cpp | 6 +++++- ddprof-lib/src/main/cpp/threadLocalData.h | 19 ++++++++++--------- .../src/main/cpp/threadLocalDataPool.cpp | 8 ++++++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp index af045189db..d26fbb56c2 100644 --- a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp +++ b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp @@ -1216,14 +1216,18 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) { Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); return 0; } + const bool prev_unwinding_java = prof_thread->is_unwinding_Java(); sigjmp_buf crash_protection_ctx; - sigjmp_buf* prev_jmp_buf = prof_thread != nullptr ? prof_thread->getJmpCtx() : nullptr; + sigjmp_buf* prev_jmp_buf = prof_thread->getJmpCtx(); if (sigsetjmp(crash_protection_ctx, 1) != 0) { // checkFault() does a siglongjmp from inside segvHandler, bypassing // segvHandler's SignalHandlerScope destructor. Compensate. SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP(); prof_thread->setJmpCtx(prev_jmp_buf); + // A recovered siglongjmp bypasses AsyncSampleMutex destructors, so restore + // the per-thread guard to its pre-walk value. + prof_thread->set_unwinding_Java(prev_unwinding_java); if (truncated) { *truncated = true; } diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index 46fce44af3..e5bf9d59b0 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -99,9 +99,9 @@ class ProfiledThread : public ThreadLocalData { alignas(8) u32 _otel_tag_encodings[DD_TAGS_CAPACITY]; u64 _otel_local_root_span_id; - ProfiledThread(int tid) + ProfiledThread(int tid, bool claimed = false) : ThreadLocalData(), _jmp_buf(nullptr), _pc(0), _sp(0), _span_id(0), _crash_depth(0), _tid(tid), _cpu_epoch(0), - _wall_epoch(0), _call_trace_id(0), _recording_epoch(0), _misc_flags(0), + _wall_epoch(0), _call_trace_id(0), _recording_epoch(0), _misc_flags(claimed ? FLAG_CLAIMED : 0), _park_block_token(0), _filter_slot_id(-1), _init_window(0), _signal_depth(0), _otel_ctx_initialized(false), @@ -118,20 +118,21 @@ class ProfiledThread : public ThreadLocalData { virtual ~ProfiledThread() { } inline bool isClaimed() const { - return (__atomic_load_n(&_misc_flags, __ATOMIC_RELAXED) & FLAG_CLAIMED) == FLAG_CLAIMED; + return (__atomic_load_n(&_misc_flags, __ATOMIC_RELAXED) & FLAG_CLAIMED) == FLAG_CLAIMED; } - inline bool claim_acquire(int tid) { + inline void unclaim() { + assert(isClaimed() && "Slot has been claimed"); + __atomic_fetch_and(&_misc_flags, ~FLAG_CLAIMED, __ATOMIC_RELEASE); + } + + inline bool claim_acquire() { if (isClaimed()) { return false; } u32 flags = __atomic_fetch_or(&_misc_flags, FLAG_CLAIMED, __ATOMIC_ACQUIRE); - bool rc = (flags & FLAG_CLAIMED) == 0; - if (rc) { - _tid = tid; - } - return rc; + return (flags & FLAG_CLAIMED) == 0; } public: diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index 7b15d12448..4923f04933 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -39,7 +39,7 @@ ProfiledThread* ThreadLocalDataPool::claim(int tid) { int start_pos = tid % _capacity; int index = start_pos; do { - if (_threads[index].claim_acquire(tid)) { + if (_threads[index].claim_acquire()) { return &_threads[index]; } index = (index + 1) % _capacity; @@ -68,7 +68,11 @@ ProfiledThread* ThreadLocalDataPool::acquire(int tid) { if (pool == nullptr) { return nullptr; } else { - return pool->claim(tid); + ProfiledThread* t = pool->claim(tid); + if (t != nullptr) { + new (t)ProfiledThread(tid, true /* claimed */); + } + return t; } } From 918fad4709eaf259551e91f888dbef1d99116444 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 3 Aug 2026 20:57:07 +0000 Subject: [PATCH 04/16] Fix --- ddprof-lib/src/main/cpp/signalSafety.h | 2 +- ddprof-lib/src/main/cpp/threadLocalDataPool.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/signalSafety.h b/ddprof-lib/src/main/cpp/signalSafety.h index 44d0d5d4b9..0918472260 100644 --- a/ddprof-lib/src/main/cpp/signalSafety.h +++ b/ddprof-lib/src/main/cpp/signalSafety.h @@ -18,7 +18,7 @@ #define _SIGNAL_SAFETY_H #include "guards.h" // isInSignalContext, SIGNAL_HANDLER_GUARD, ... -#include "threadLocalData.h" // ProfiledThread::current +#include "threadLocalData.inline.h" // ProfiledThread::current // Detect ASAN using compiler-provided macros so the ASAN_ENABLED guard below // works in every TU that includes this header, independent of include order. diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index 4923f04933..d7b28a9620 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -7,6 +7,7 @@ #include "threadLocalData.h" #include "threadLocalDataPool.h" +#include #include ThreadLocalDataPool* ThreadLocalDataPool::_pool = nullptr; From 0e216e5e63397344ba3d981e8e96e76e0e1cae5b Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 3 Aug 2026 22:57:31 +0200 Subject: [PATCH 05/16] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- ddprof-lib/src/main/cpp/threadLocalDataPool.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index d7b28a9620..236f78c32f 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -31,8 +31,10 @@ ThreadLocalDataPool::~ThreadLocalDataPool() { } ProfiledThread* ThreadLocalDataPool::claim(int tid) { + if (_threads == nullptr) { + return nullptr; + } uint16_t used = __atomic_fetch_add(&_used, 1, __ATOMIC_RELAXED); - if (used >= _capacity) { __atomic_fetch_add(&_used, -1, __ATOMIC_RELAXED); return nullptr; } From f4e9eecb0644690a420f5638d36740a7f32c59b2 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 3 Aug 2026 22:59:48 +0200 Subject: [PATCH 06/16] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../src/main/cpp/threadLocalDataPool.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index 236f78c32f..6bfd8ed3a0 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -12,16 +12,19 @@ ThreadLocalDataPool* ThreadLocalDataPool::_pool = nullptr; -ThreadLocalDataPool::ThreadLocalDataPool(uint64_t capacity) : _capacity(capacity), _used(0) { - size_t malloc_size = capacity * sizeof(ProfiledThread); +ThreadLocalDataPool::ThreadLocalDataPool(uint64_t capacity) + : _capacity(capacity), _used(0), _threads(nullptr) { + const size_t malloc_size = capacity * sizeof(ProfiledThread); void* p = malloc(malloc_size); - if (p != nullptr) { - _threads = reinterpret_cast(p); - for (int index = 0; index < capacity; index++) { - new (&_threads[index])ProfiledThread(0); - } - NativeMem::record(NM_THREAD_LOCAL, malloc_size); + if (p == nullptr) { + return; } + + _threads = reinterpret_cast(p); + for (uint64_t index = 0; index < capacity; index++) { + new (&_threads[index]) ProfiledThread(0); + } + NativeMem::record(NM_THREAD_LOCAL, malloc_size); } ThreadLocalDataPool::~ThreadLocalDataPool() { From 0473e711b62c725dae19b2276d9231eeb7bb95ed Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 3 Aug 2026 23:17:49 +0200 Subject: [PATCH 07/16] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- ddprof-lib/src/main/cpp/threadLocalData.inline.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/threadLocalData.inline.h b/ddprof-lib/src/main/cpp/threadLocalData.inline.h index a4f60c9806..5274d57a3a 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.inline.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.inline.h @@ -10,7 +10,7 @@ #include "threadLocalData.h" #include "threadLocalDataPool.h" -ProfiledThread* ProfiledThread::current() { +inline ProfiledThread* ProfiledThread::current() { if (!isThreadKeyValid()) { return nullptr; } From 84a5f4c7de9291cce48569eea493312b0df846f8 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 3 Aug 2026 21:18:07 +0000 Subject: [PATCH 08/16] Fix merge --- ddprof-lib/src/main/cpp/threadLocalDataPool.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index 6bfd8ed3a0..efc7621056 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -37,7 +37,9 @@ ProfiledThread* ThreadLocalDataPool::claim(int tid) { if (_threads == nullptr) { return nullptr; } + uint16_t used = __atomic_fetch_add(&_used, 1, __ATOMIC_RELAXED); + if (used >= _capacity) { __atomic_fetch_add(&_used, -1, __ATOMIC_RELAXED); return nullptr; } From ef0026e1f26826d6b33c575f079f5efcea5ec86c Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 4 Aug 2026 00:35:43 +0200 Subject: [PATCH 09/16] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- ddprof-lib/src/main/cpp/threadLocalDataPool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index efc7621056..279841bd36 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -52,7 +52,7 @@ ProfiledThread* ThreadLocalDataPool::claim(int tid) { } index = (index + 1) % _capacity; } while (index != start_pos); - assert(false && "Should not reach here"); + __atomic_fetch_add(&_used, -1, __ATOMIC_RELAXED); return nullptr; } From c1d60baffbae9da6e1f3728f4641fe10c4fa9462 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 3 Aug 2026 22:38:11 +0000 Subject: [PATCH 10/16] Check wrong thread for priming --- ddprof-lib/src/main/cpp/jvmSupport.cpp | 2 +- ddprof-lib/src/main/cpp/jvmThread.h | 13 ------------- ddprof-lib/src/main/cpp/threadLocal.h | 4 ++++ ddprof-lib/src/main/cpp/threadLocalData.cpp | 11 +++++++++++ ddprof-lib/src/main/cpp/threadLocalData.h | 6 ++++++ 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/ddprof-lib/src/main/cpp/jvmSupport.cpp b/ddprof-lib/src/main/cpp/jvmSupport.cpp index 0068d211fd..e2d6a9e24e 100644 --- a/ddprof-lib/src/main/cpp/jvmSupport.cpp +++ b/ddprof-lib/src/main/cpp/jvmSupport.cpp @@ -39,7 +39,7 @@ bool JVMSupport::initialize() { return false; } - if (JVMThread::supportPriming()) { + if (ProfiledThread::supportPriming()) { ThreadLocalDataPool::initialize(); } diff --git a/ddprof-lib/src/main/cpp/jvmThread.h b/ddprof-lib/src/main/cpp/jvmThread.h index bbc3a79b9c..d9a3d54e87 100644 --- a/ddprof-lib/src/main/cpp/jvmThread.h +++ b/ddprof-lib/src/main/cpp/jvmThread.h @@ -17,9 +17,6 @@ */ class JVMThread { private: - // PTHREAD_KEY_2NDLEVEL_SIZE is an internal macro set to 32 in the GNU C Library (glibc) NPTL - // implementation. Slot indexes less than PTHREAD_KEY_2NDLEVEL_SIZE are pre-allocated. - static constexpr int PTHREAD_KEY_2NDLEVEL_SIZE = 32; static jfieldID _tid; static ThreadLocal _jvm_thread; @@ -44,16 +41,6 @@ class JVMThread { static inline pthread_key_t key() { return _jvm_thread.key(); } - - static bool supportPriming() { - // Key must be valid - assert(_jvm_thread.isKeyValid()); - if (OS::isMusl()) { - return true; - } else { - return _jvm_thread.key() < PTHREAD_KEY_2NDLEVEL_SIZE; - } - } static int nativeThreadId(JNIEnv* jni, jthread thread); static inline jlong javaThreadId(JNIEnv* env, jthread thread) { diff --git a/ddprof-lib/src/main/cpp/threadLocal.h b/ddprof-lib/src/main/cpp/threadLocal.h index ffd06713a0..764a80c685 100644 --- a/ddprof-lib/src/main/cpp/threadLocal.h +++ b/ddprof-lib/src/main/cpp/threadLocal.h @@ -86,6 +86,10 @@ class ThreadLocal { return _key != INVALID_KEY; } + pthread_key_t key() const { + return _key; + } + /** * set(nullptr) will result in the value being recreated when get() is called * when CREATE_FUNC is not nullptr. diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index 2ff72b896b..8c3f89db34 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -9,6 +9,7 @@ #include "guards.h" #include "otel_context.h" #include "os.h" +#include #include #include @@ -20,6 +21,16 @@ // reads it. ThreadLocal ProfiledThread::_current_thread; +bool ProfiledThread::supportPriming() { + // Key must be valid + assert(_current_thread.isKeyValid()); + if (OS::isMusl()) { + return true; + } else { + return _current_thread.key() < PTHREAD_KEY_2NDLEVEL_SIZE; + } + } + ProfiledThread* ProfiledThread::initCurrentThread() { if (!isThreadKeyValid()) { return nullptr; diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index e5bf9d59b0..65955e3b03 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -39,6 +39,10 @@ class ThreadLocalData { class ProfiledThread : public ThreadLocalData { friend class ThreadLocalDataPool; + // PTHREAD_KEY_2NDLEVEL_SIZE is an internal macro set to 32 in the GNU C Library (glibc) NPTL + // implementation. Slot indexes less than PTHREAD_KEY_2NDLEVEL_SIZE are pre-allocated. + static constexpr int PTHREAD_KEY_2NDLEVEL_SIZE = 32; + public: enum ThreadType : u32 { TYPE_UNKNOWN = 0, @@ -145,6 +149,8 @@ class ProfiledThread : public ThreadLocalData { return _current_thread.isKeyValid(); } + static bool supportPriming(); + #ifdef UNIT_TEST // Simulates the moment inside release() after pthread_setspecific(NULL) but // before delete — the race window the clearCurrentThreadTLS fix covers. From 3b1f48cef146d1928737e2a1b026c6a0b55f2afe Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 4 Aug 2026 00:51:43 +0000 Subject: [PATCH 11/16] Fix pooled profiledThread reset --- ddprof-lib/src/main/cpp/threadLocalData.cpp | 24 +++++++++++++++++++ ddprof-lib/src/main/cpp/threadLocalData.h | 7 ++++-- .../src/main/cpp/threadLocalDataPool.cpp | 6 ++--- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index 8c3f89db34..5a84355acf 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -95,3 +95,27 @@ Context ProfiledThread::snapshotContext(size_t numAttrs) { } return ctx; } + +void ProfiledThread::resetClaimed(int tid) { + _jmp_buf = nullptr; + _pc = 0; + _sp = 0; + _span_id = 0; + _crash_depth = 0; + _tid = tid; + _cpu_epoch = 0; + _wall_epoch = 0; + _call_trace_id = 0; + _recording_epoch = 0; + _misc_flags = FLAG_CLAIMED; + _park_block_token = 0; + _filter_slot_id = 0; + _init_window = 0; + _signal_depth = 0; + _otel_ctx_initialized = false; + _otel_ctx_record = {}; + for (int index = 0; index < DD_TAGS_CAPACITY; index++) { + _otel_tag_encodings[index] = 0; + } + _otel_local_root_span_id = 0; +} \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index 65955e3b03..04874ff1ce 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -103,9 +103,9 @@ class ProfiledThread : public ThreadLocalData { alignas(8) u32 _otel_tag_encodings[DD_TAGS_CAPACITY]; u64 _otel_local_root_span_id; - ProfiledThread(int tid, bool claimed = false) + ProfiledThread(int tid) : ThreadLocalData(), _jmp_buf(nullptr), _pc(0), _sp(0), _span_id(0), _crash_depth(0), _tid(tid), _cpu_epoch(0), - _wall_epoch(0), _call_trace_id(0), _recording_epoch(0), _misc_flags(claimed ? FLAG_CLAIMED : 0), + _wall_epoch(0), _call_trace_id(0), _recording_epoch(0), _misc_flags(0), _park_block_token(0), _filter_slot_id(-1), _init_window(0), _signal_depth(0), _otel_ctx_initialized(false), @@ -121,6 +121,9 @@ class ProfiledThread : public ThreadLocalData { virtual ~ProfiledThread() { } + // Reset content of claimed slot + void resetClaimed(int tid); + inline bool isClaimed() const { return (__atomic_load_n(&_misc_flags, __ATOMIC_RELAXED) & FLAG_CLAIMED) == FLAG_CLAIMED; } diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index 279841bd36..b635960d18 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -24,7 +24,7 @@ ThreadLocalDataPool::ThreadLocalDataPool(uint64_t capacity) for (uint64_t index = 0; index < capacity; index++) { new (&_threads[index]) ProfiledThread(0); } - NativeMem::record(NM_THREAD_LOCAL, malloc_size); + NativeMem::record(NM_THREAD_LOCAL, malloc_size + sizeof(ThreadLocalDataPool)); } ThreadLocalDataPool::~ThreadLocalDataPool() { @@ -58,7 +58,7 @@ ProfiledThread* ThreadLocalDataPool::claim(int tid) { bool ThreadLocalDataPool::unclaim(ProfiledThread* t) { if (contains(t)) { - new (t)ProfiledThread(0); + t->unclaim(); uint16_t used = __atomic_fetch_add(&_used, -1, __ATOMIC_RELEASE); assert(used > 0); return true; @@ -78,7 +78,7 @@ ProfiledThread* ThreadLocalDataPool::acquire(int tid) { } else { ProfiledThread* t = pool->claim(tid); if (t != nullptr) { - new (t)ProfiledThread(tid, true /* claimed */); + t->resetClaimed(tid); } return t; } From 01d4b4214706754ad7d458291aea59c4986fd44c Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 4 Aug 2026 01:38:10 +0000 Subject: [PATCH 12/16] Fixes --- ddprof-lib/src/main/cpp/faultInjection.cpp | 2 +- .../src/main/cpp/hotspot/vmStructs.inline.h | 2 +- ddprof-lib/src/main/cpp/jvmThread.h | 3 +-- ddprof-lib/src/main/cpp/profiler.cpp | 1 + ddprof-lib/src/main/cpp/safeAccess.cpp | 2 +- ddprof-lib/src/main/cpp/threadLocalData.cpp | 16 ++++++++-------- ddprof-lib/src/main/cpp/threadLocalData.h | 5 ++--- ddprof-lib/src/main/cpp/threadLocalDataPool.cpp | 8 +++++--- ddprof-lib/src/main/cpp/threadLocalDataPool.h | 2 +- 9 files changed, 21 insertions(+), 20 deletions(-) diff --git a/ddprof-lib/src/main/cpp/faultInjection.cpp b/ddprof-lib/src/main/cpp/faultInjection.cpp index d61be77b68..d1389cfb49 100644 --- a/ddprof-lib/src/main/cpp/faultInjection.cpp +++ b/ddprof-lib/src/main/cpp/faultInjection.cpp @@ -22,7 +22,7 @@ #include "counters.h" // Counters::increment (FAULTS_INJECTED) #include "os.h" // OS::page_size -#include "threadLocalData.h" // ProfiledThread::current / nextFiRandom +#include "threadLocalData.inline.h" // ProfiledThread::current / nextFiRandom #include #include diff --git a/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h b/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h index 5de4c94cad..100da7fb48 100644 --- a/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h +++ b/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h @@ -11,7 +11,7 @@ #include "hotspot/vmStructs.h" #include "jvmThread.h" #include "safeAccess.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" inline bool crashProtectionActive() { ProfiledThread* pt = ProfiledThread::current(); diff --git a/ddprof-lib/src/main/cpp/jvmThread.h b/ddprof-lib/src/main/cpp/jvmThread.h index d9a3d54e87..2f5bd69104 100644 --- a/ddprof-lib/src/main/cpp/jvmThread.h +++ b/ddprof-lib/src/main/cpp/jvmThread.h @@ -10,7 +10,6 @@ #include #include "threadLocal.h" -#include "os.h" /** * JVMThread represents a native JVM thread that is JVM implementation agnostic @@ -41,7 +40,7 @@ class JVMThread { static inline pthread_key_t key() { return _jvm_thread.key(); } - + static int nativeThreadId(JNIEnv* jni, jthread thread); static inline jlong javaThreadId(JNIEnv* env, jthread thread) { return env->GetLongField(thread, _tid); diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 60aa659d8a..24c8c5d67c 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -35,6 +35,7 @@ #include "stackFrame.h" #include "stackWalker.h" #include "symbols.h" +#include "threadLocalData.inline.h" #include "tsc.h" #include "utils.h" #include "wallClock.h" diff --git a/ddprof-lib/src/main/cpp/safeAccess.cpp b/ddprof-lib/src/main/cpp/safeAccess.cpp index bd637d051a..24d1c6ff6e 100644 --- a/ddprof-lib/src/main/cpp/safeAccess.cpp +++ b/ddprof-lib/src/main/cpp/safeAccess.cpp @@ -23,7 +23,7 @@ #include #include #ifdef DEBUG -#include "threadLocalData.h" // ProfiledThread::current / isProtected +#include "threadLocalData.inline.h" // ProfiledThread::current / isProtected #endif extern "C" int safefetch32_cont(int* adr, int errValue); diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index 5a84355acf..f79f9f90aa 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -22,14 +22,14 @@ ThreadLocal ProfiledThread::_current_thread; bool ProfiledThread::supportPriming() { - // Key must be valid - assert(_current_thread.isKeyValid()); - if (OS::isMusl()) { - return true; - } else { - return _current_thread.key() < PTHREAD_KEY_2NDLEVEL_SIZE; - } + // Key must be valid + assert(_current_thread.isKeyValid()); + if (OS::isMusl()) { + return true; + } else { + return _current_thread.key() < PTHREAD_KEY_2NDLEVEL_SIZE; } +} ProfiledThread* ProfiledThread::initCurrentThread() { if (!isThreadKeyValid()) { @@ -118,4 +118,4 @@ void ProfiledThread::resetClaimed(int tid) { _otel_tag_encodings[index] = 0; } _otel_local_root_span_id = 0; -} \ No newline at end of file +} diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index 04874ff1ce..2214b0e85a 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -52,8 +52,7 @@ class ProfiledThread : public ThreadLocalData { }; static constexpr u32 FLAG_PARKED = 0x4u; // next free bit after TYPE_MASK (0x1|0x2) - static constexpr u32 FLAG_CLAIMED = 0x8u; // Used by ThreadLocalDataPool only - + static constexpr u32 FLAG_CLAIMED = 0x8u; // Used by ThreadLocalDataPool only // We are allowing several levels of nesting because we can be // eg. in a crash handler when wallclock signal kicks in, @@ -140,7 +139,7 @@ class ProfiledThread : public ThreadLocalData { u32 flags = __atomic_fetch_or(&_misc_flags, FLAG_CLAIMED, __ATOMIC_ACQUIRE); return (flags & FLAG_CLAIMED) == 0; -} + } public: static ProfiledThread *forTid(int tid) { diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index b635960d18..4ec649edc9 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -1,5 +1,4 @@ - - /* +/* * Copyright 2026 Datadog, Inc. * SPDX-License-Identifier: Apache-2.0 */ @@ -29,6 +28,9 @@ ThreadLocalDataPool::ThreadLocalDataPool(uint64_t capacity) ThreadLocalDataPool::~ThreadLocalDataPool() { if (_threads != nullptr) { + for (uint64_t index = 0; index < _capacity; index++) { + _threads[index].~ProfiledThread(); + } free(reinterpret_cast(_threads)); } } @@ -91,4 +93,4 @@ bool ThreadLocalDataPool::release(ProfiledThread* t) { } else { return false; } -} \ No newline at end of file +} diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.h b/ddprof-lib/src/main/cpp/threadLocalDataPool.h index 93f0ec5a0c..a7ed9cf4ca 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.h +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.h @@ -43,4 +43,4 @@ class ThreadLocalDataPool { static bool release(ProfiledThread* t); }; -#endif // THREADLOCALDATA_POOL_H \ No newline at end of file +#endif // THREADLOCALDATA_POOL_H From 0ef3fcf0e5315a3552529db8d0bd7b46e6befb50 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 4 Aug 2026 15:23:26 +0000 Subject: [PATCH 13/16] Review comments --- ddprof-lib/src/main/cpp/counters.h | 1 + ddprof-lib/src/main/cpp/faultInjection.h | 14 ++- .../src/main/cpp/hotspot/hotspotSupport.cpp | 4 +- ddprof-lib/src/main/cpp/jvmSupport.cpp | 3 + ddprof-lib/src/main/cpp/profiler.cpp | 4 +- ddprof-lib/src/main/cpp/stackWalker.cpp | 4 +- ddprof-lib/src/main/cpp/threadLocalData.cpp | 28 +++++- ddprof-lib/src/main/cpp/threadLocalData.h | 24 ++--- .../src/main/cpp/threadLocalData.inline.h | 4 +- .../src/main/cpp/threadLocalDataPool.cpp | 13 +-- ddprof-lib/src/main/cpp/threadLocalDataPool.h | 32 ++++++ ddprof-lib/src/main/cpp/unwindStats.cpp | 17 ++++ ddprof-lib/src/main/cpp/unwindStats.h | 15 +-- .../src/test/cpp/threadLocalDataPool_ut.cpp | 98 +++++++++++++++++++ 14 files changed, 218 insertions(+), 43 deletions(-) create mode 100644 ddprof-lib/src/test/cpp/threadLocalDataPool_ut.cpp diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h index deb6e40d9a..4d41867ba0 100644 --- a/ddprof-lib/src/main/cpp/counters.h +++ b/ddprof-lib/src/main/cpp/counters.h @@ -134,6 +134,7 @@ X(SAFECOPY_FAILED, "safecopy_failed") \ X(SAFEFETCH_FAILED, "safefetch_failed") \ X(STACKWALK_LONGJMP_RECOVERED, "stackwalk_longjmp_recovered") \ + X(SAMPLES_DROPPED_TLS_POOL_EXHAUSTED, "thread_local_pool_exhausted") \ DD_COUNTER_TABLE_FAULT_INJECTION(X) \ DD_COUNTER_TABLE_FI_DEBUG(X) \ DD_COUNTER_TABLE_DEBUG(X) diff --git a/ddprof-lib/src/main/cpp/faultInjection.h b/ddprof-lib/src/main/cpp/faultInjection.h index 5ac8ead3ba..9729668e97 100644 --- a/ddprof-lib/src/main/cpp/faultInjection.h +++ b/ddprof-lib/src/main/cpp/faultInjection.h @@ -34,8 +34,9 @@ // // return INJECT_FAULT_BOOL_LIKELY(dlopen(name, flags) != nullptr); // -// The three tiers name their firing frequency: RARE 0.01%, UNLIKELY 0.1%, -// LIKELY 1%. See faultInjection.cpp for the poison-address and PRNG details. +// The four tiers name their firing frequency: RARE 0.01%, UNLIKELY 0.1%, +// LIKELY 1%, HIGH 10%. See faultInjection.cpp for the poison-address and PRNG +// details. #ifndef _FAULT_INJECTION_H #define _FAULT_INJECTION_H @@ -56,6 +57,7 @@ namespace faultinj { constexpr u64 PROB_RARE = 1844674407370955ULL; // 1e-4 (0.01%) constexpr u64 PROB_UNLIKELY = 18446744073709552ULL; // 1e-3 (0.1%) constexpr u64 PROB_LIKELY = 184467440737095520ULL; // 1e-2 (1%) +constexpr u64 PROB_HIGH = 1844674407370955162ULL; // 1e-1 (10%) // Called once at profiler startup (off the signal path) to mmap the PROT_NONE // guard region used by poisonAddress(). Safe to call before any injection. @@ -106,6 +108,8 @@ inline T injectValue(T orig, T faulty, u64 threshold, const char* fn) { ::faultinj::injectAddress((ptr), ::faultinj::PROB_UNLIKELY, __func__) #define INJECT_FAULT_ADDRESS_LIKELY(ptr) \ ::faultinj::injectAddress((ptr), ::faultinj::PROB_LIKELY, __func__) +#define INJECT_FAULT_ADDRESS_HIGH(ptr) \ + ::faultinj::injectAddress((ptr), ::faultinj::PROB_HIGH, __func__) #define INJECT_FAULT_BOOL_RARE(v) \ ::faultinj::injectValue((v), false, ::faultinj::PROB_RARE, __func__) @@ -113,24 +117,30 @@ inline T injectValue(T orig, T faulty, u64 threshold, const char* fn) { ::faultinj::injectValue((v), false, ::faultinj::PROB_UNLIKELY, __func__) #define INJECT_FAULT_BOOL_LIKELY(v) \ ::faultinj::injectValue((v), false, ::faultinj::PROB_LIKELY, __func__) +#define INJECT_FAULT_BOOL_HIGH(v) \ + ::faultinj::injectValue((v), false, ::faultinj::PROB_HIGH, __func__) #else // __FAULT_INJECTION__ not defined — strict identity, zero cost. #define INJECT_FAULT_ADDRESS_RARE(ptr) (ptr) #define INJECT_FAULT_ADDRESS_UNLIKELY(ptr) (ptr) #define INJECT_FAULT_ADDRESS_LIKELY(ptr) (ptr) +#define INJECT_FAULT_ADDRESS_HIGH(ptr) (ptr) #define INJECT_FAULT_INT_RARE(v) (v) #define INJECT_FAULT_INT_UNLIKELY(v) (v) #define INJECT_FAULT_INT_LIKELY(v) (v) +#define INJECT_FAULT_INT_HIGH(v) (v) #define INJECT_FAULT_LONG_RARE(v) (v) #define INJECT_FAULT_LONG_UNLIKELY(v) (v) #define INJECT_FAULT_LONG_LIKELY(v) (v) +#define INJECT_FAULT_LONG_HIGH(v) (v) #define INJECT_FAULT_BOOL_RARE(v) (v) #define INJECT_FAULT_BOOL_UNLIKELY(v) (v) #define INJECT_FAULT_BOOL_LIKELY(v) (v) +#define INJECT_FAULT_BOOL_HIGH(v) (v) #define NO_INJECTION_ASSERT(a) (assert(a)) diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp index d26fbb56c2..bb2c3233bd 100644 --- a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp +++ b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp @@ -241,7 +241,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex // VMStructs is only available for hotspot JVM assert(VM::isHotspot()); - ProfiledThread* prof_thread = ProfiledThread::acquire_current(); + ProfiledThread* prof_thread = ProfiledThread::acquireCurrent(); if (prof_thread == nullptr) { Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); return 0; @@ -1211,7 +1211,7 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) { // AsyncGetCallTrace. Install a jmp ctx here too, so a SIGSEGV anywhere in // walkJavaStack, except HotSpot's AsyncGetCallTrace call, is caught by // Profiler::checkFault() and siglongjmp'd back here instead of crashing the process. - ProfiledThread* prof_thread = ProfiledThread::acquire_current(); + ProfiledThread* prof_thread = ProfiledThread::acquireCurrent(); if (prof_thread == nullptr) { Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); return 0; diff --git a/ddprof-lib/src/main/cpp/jvmSupport.cpp b/ddprof-lib/src/main/cpp/jvmSupport.cpp index e2d6a9e24e..b1e4a728d0 100644 --- a/ddprof-lib/src/main/cpp/jvmSupport.cpp +++ b/ddprof-lib/src/main/cpp/jvmSupport.cpp @@ -6,6 +6,7 @@ #include "jvmSupport.inline.h" #include "asyncSampleMutex.h" +#include "common.h" #include "frames.h" #include "os.h" #include "profiler.h" @@ -41,6 +42,8 @@ bool JVMSupport::initialize() { if (ProfiledThread::supportPriming()) { ThreadLocalDataPool::initialize(); + } else { + LOG_WARN("Thread priming is not supported"); } // Check ProfiledThread key, it is critical for storing per-thread metadata diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 24c8c5d67c..6f87abe7a6 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -866,10 +866,10 @@ bool Profiler::prewarmUnwinder() { // libgcc_s.so.1 has been the stable SONAME since 2002; a bump would // constitute a glibc/GCC C++ ABI break and is treated as a fixed contract. // - // INJECT_FAULT_BOOL_LIKELY lets fault-injection builds force this to + // INJECT_FAULT_BOOL_HIGH lets fault-injection builds force this to // report failure without the library actually being absent, so // checkState()'s "Missing libgcc_s.so" path can be exercised in CI. - return INJECT_FAULT_BOOL_LIKELY(dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL) != nullptr); + return INJECT_FAULT_BOOL_HIGH(dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL) != nullptr); #else return true; #endif diff --git a/ddprof-lib/src/main/cpp/stackWalker.cpp b/ddprof-lib/src/main/cpp/stackWalker.cpp index 3912cfebd3..9abcf2c61b 100644 --- a/ddprof-lib/src/main/cpp/stackWalker.cpp +++ b/ddprof-lib/src/main/cpp/stackWalker.cpp @@ -45,7 +45,7 @@ int StackWalker::walkFP(void* ucontext, const void** callchain, int max_depth, S // falls inside this library while a jmp ctx is installed gets caught by // Profiler::checkFault() from the SEGV handler and siglongjmp'd back here, // instead of crashing the process. - ProfiledThread* prof_thread = ProfiledThread::acquire_current(); + ProfiledThread* prof_thread = ProfiledThread::acquireCurrent(); if (prof_thread == nullptr) { Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); return 0; @@ -134,7 +134,7 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth // falls inside this library while a jmp ctx is installed gets caught by // Profiler::checkFault() from the SEGV handler and siglongjmp'd back here, // instead of crashing the process. - ProfiledThread* prof_thread = ProfiledThread::acquire_current(); + ProfiledThread* prof_thread = ProfiledThread::acquireCurrent(); if (prof_thread == nullptr) { Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); return 0; diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index f79f9f90aa..456efe9fa7 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "faultInjection.h" #include "threadLocalData.inline.h" #include "threadLocalDataPool.h" #include "context_api.h" @@ -26,9 +27,18 @@ bool ProfiledThread::supportPriming() { assert(_current_thread.isKeyValid()); if (OS::isMusl()) { return true; - } else { - return _current_thread.key() < PTHREAD_KEY_2NDLEVEL_SIZE; } +#ifdef __GLIBC__ + bool rc = _current_thread.key() < PTHREAD_KEY_2NDLEVEL_SIZE; + return INJECT_FAULT_BOOL_HIGH(rc); +#else + // Neither musl nor glibc (e.g. macOS libpthread): PTHREAD_KEY_2NDLEVEL_SIZE + // is a glibc NPTL implementation detail (see threadLocalData.h) that doesn't + // describe this libc's pthread_key_t allocation scheme. Fail safe by + // disabling signal-handler TLS priming rather than assuming glibc-compatible + // pthread_setspecific behavior. + return false; +#endif } ProfiledThread* ProfiledThread::initCurrentThread() { @@ -74,6 +84,15 @@ void ProfiledThread::release() { _current_thread.clear(); } +#ifdef UNIT_TEST +void ProfiledThread::deleteForTest(ProfiledThread* pt) { + if (!ThreadLocalDataPool::release(pt)) { + delete pt; + NativeMem::record(NM_THREAD_LOCAL, -(long long)sizeof(ProfiledThread)); + } +} +#endif + int ProfiledThread::currentTid() { ProfiledThread *tls = current(); if (tls != NULL) { @@ -109,7 +128,7 @@ void ProfiledThread::resetClaimed(int tid) { _recording_epoch = 0; _misc_flags = FLAG_CLAIMED; _park_block_token = 0; - _filter_slot_id = 0; + _filter_slot_id = -1; _init_window = 0; _signal_depth = 0; _otel_ctx_initialized = false; @@ -118,4 +137,7 @@ void ProfiledThread::resetClaimed(int tid) { _otel_tag_encodings[index] = 0; } _otel_local_root_span_id = 0; + _in_critical_section = false; + + _unwind_failures.reset(); } diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index 2214b0e85a..09e3665fbf 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -41,6 +41,9 @@ class ProfiledThread : public ThreadLocalData { // PTHREAD_KEY_2NDLEVEL_SIZE is an internal macro set to 32 in the GNU C Library (glibc) NPTL // implementation. Slot indexes less than PTHREAD_KEY_2NDLEVEL_SIZE are pre-allocated. + // glibc-specific: only meaningful under the __GLIBC__ branch of supportPriming() + // (threadLocalData.cpp). Other libcs (musl, macOS libpthread) don't share this + // layout and must not be routed through this constant. static constexpr int PTHREAD_KEY_2NDLEVEL_SIZE = 32; public: @@ -132,7 +135,7 @@ class ProfiledThread : public ThreadLocalData { __atomic_fetch_and(&_misc_flags, ~FLAG_CLAIMED, __ATOMIC_RELEASE); } - inline bool claim_acquire() { + inline bool claimAcquire() { if (isClaimed()) { return false; } @@ -163,16 +166,15 @@ class ProfiledThread : public ThreadLocalData { _current_thread.set(nullptr); return pt; } - // Deletes a ProfiledThread returned by clearCurrentThreadTLS(). - // Needed because the destructor is private. This stands in for the delete - // that freeValue() performs in production, so it mirrors freeValue()'s - // NM_THREAD_LOCAL decrement to keep the accounting balanced in tests. - static void deleteForTest(ProfiledThread *pt) { - delete pt; - NativeMem::record(NM_THREAD_LOCAL, -(long long)sizeof(ProfiledThread)); - } + // Releases a ProfiledThread returned by clearCurrentThreadTLS(). + // Needed because the destructor is private. Mirrors freeValue()'s + // ThreadLocalDataPool::release()-then-delete logic (and its NM_THREAD_LOCAL + // decrement) so it's safe to call on both forTid()-obtained and pool-backed + // threads. Defined in threadLocalData.cpp, where ThreadLocalDataPool's full + // declaration is visible. + static void deleteForTest(ProfiledThread *pt); #endif - // initCurrentThread() and release() are not async-signal-safe: + // initCurrentThread() and release() are not async-signal-safe: // must be called outside of a signal handler with signal blocked static ProfiledThread* initCurrentThread(); static void release(); @@ -188,7 +190,7 @@ class ProfiledThread : public ThreadLocalData { static inline ProfiledThread *current(); // signal-handler friendly with priming: return existing TLS or acquire and set // ProfiledThread from ThreadLocalDataPool. - static inline ProfiledThread* acquire_current(); + static inline ProfiledThread* acquireCurrent(); static int currentTid(); inline int tid() { return _tid; } diff --git a/ddprof-lib/src/main/cpp/threadLocalData.inline.h b/ddprof-lib/src/main/cpp/threadLocalData.inline.h index 5274d57a3a..2d9b1b6c26 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.inline.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.inline.h @@ -6,6 +6,7 @@ #ifndef THREADLOCALDATA_INLINE_H #define THREADLOCALDATA_INLINE_H +#include "guards.h" #include "os.h" #include "threadLocalData.h" #include "threadLocalDataPool.h" @@ -17,11 +18,12 @@ inline ProfiledThread* ProfiledThread::current() { return _current_thread.get(); } -ProfiledThread* ProfiledThread::acquire_current() { +ProfiledThread* ProfiledThread::acquireCurrent() { ProfiledThread* prof_thread = current(); if (prof_thread == nullptr) { prof_thread = ThreadLocalDataPool::acquire(OS::threadId()); if (prof_thread != nullptr) { + SignalBlocker blocker; _current_thread.set(prof_thread); } } diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index 4ec649edc9..1b25be60c0 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "counters.h" #include "threadLocalData.h" #include "threadLocalDataPool.h" @@ -26,14 +27,6 @@ ThreadLocalDataPool::ThreadLocalDataPool(uint64_t capacity) NativeMem::record(NM_THREAD_LOCAL, malloc_size + sizeof(ThreadLocalDataPool)); } -ThreadLocalDataPool::~ThreadLocalDataPool() { - if (_threads != nullptr) { - for (uint64_t index = 0; index < _capacity; index++) { - _threads[index].~ProfiledThread(); - } - free(reinterpret_cast(_threads)); - } -} ProfiledThread* ThreadLocalDataPool::claim(int tid) { if (_threads == nullptr) { @@ -49,12 +42,13 @@ ProfiledThread* ThreadLocalDataPool::claim(int tid) { int start_pos = tid % _capacity; int index = start_pos; do { - if (_threads[index].claim_acquire()) { + if (_threads[index].claimAcquire()) { return &_threads[index]; } index = (index + 1) % _capacity; } while (index != start_pos); __atomic_fetch_add(&_used, -1, __ATOMIC_RELAXED); + Counters::increment(SAMPLES_DROPPED_TLS_POOL_EXHAUSTED); return nullptr; } @@ -69,6 +63,7 @@ bool ThreadLocalDataPool::unclaim(ProfiledThread* t) { } void ThreadLocalDataPool::initialize() { + // process-lifetime singleton ThreadLocalDataPool* pool = new ThreadLocalDataPool(); __atomic_store_n(&_pool, pool, __ATOMIC_RELEASE); } diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.h b/ddprof-lib/src/main/cpp/threadLocalDataPool.h index a7ed9cf4ca..43e64749cd 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.h +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.h @@ -7,6 +7,8 @@ #define THREADLOCALDATA_POOL_H #include +#include +#include class ProfiledThread; @@ -41,6 +43,36 @@ class ThreadLocalDataPool { static void initialize(); static ProfiledThread* acquire(int tid); static bool release(ProfiledThread* t); + +#ifdef UNIT_TEST + // Test-only: a pool isolated from the process-wide singleton (_pool), so + // contains()/boundary tests don't disturb other tests' use of + // initialize()/acquire()/release(). + static ThreadLocalDataPool* createForTest(uint64_t capacity) { + return new ThreadLocalDataPool(capacity); + } + // ThreadLocalDataPool has no destructor definition (it's a process-lifetime + // singleton in production, never freed), so `delete p` won't link. Mirror + // what a destructor would do -- destroy each placement-newed ProfiledThread + // and free() the malloc'd buffer -- then release the ThreadLocalDataPool + // object itself via the deallocation function directly, without invoking a + // (nonexistent) destructor. + static void destroyForTest(ThreadLocalDataPool* p) { + if (p->_threads != nullptr) { + for (uint64_t index = 0; index < p->_capacity; index++) { + p->_threads[index].~ProfiledThread(); + } + free(reinterpret_cast(p->_threads)); + } + ::operator delete(p); + } + + bool containsForTest(ProfiledThread* t) const { return contains(t); } + ProfiledThread* threadsForTest() const { return _threads; } + uint64_t capacityForTest() const { return _capacity; } + ProfiledThread* claimForTest(int tid) { return claim(tid); } + bool unclaimForTest(ProfiledThread* t) { return unclaim(t); } +#endif }; #endif // THREADLOCALDATA_POOL_H diff --git a/ddprof-lib/src/main/cpp/unwindStats.cpp b/ddprof-lib/src/main/cpp/unwindStats.cpp index 82a38cf173..abc8123034 100644 --- a/ddprof-lib/src/main/cpp/unwindStats.cpp +++ b/ddprof-lib/src/main/cpp/unwindStats.cpp @@ -3,3 +3,20 @@ // initialize static members SpinLock UnwindStats::_lock; UnwindFailures UnwindStats::_unwind_failures; + +UnwindFailures::UnwindFailures() : _nameCount(0) { + _names = new char[MAX_UNWIND_FAILURE_NAMES][MAX_NAME_LENGTH]; + _counters = new u64[MAX_UNWIND_FAILURE_NAMES][UNWIND_FAILURE_ANY + 1]; + reset(); +} + +UnwindFailures::~UnwindFailures() { + delete[] _names; + delete[] _counters; + } + + +void UnwindFailures::reset() { + memset((void*)_names, 0, MAX_UNWIND_FAILURE_NAMES * MAX_NAME_LENGTH); + memset((void*)_counters, 0, MAX_UNWIND_FAILURE_NAMES * (UNWIND_FAILURE_ANY + 1) * sizeof(u64)); +} diff --git a/ddprof-lib/src/main/cpp/unwindStats.h b/ddprof-lib/src/main/cpp/unwindStats.h index 1eb4eab297..5a9a75845b 100644 --- a/ddprof-lib/src/main/cpp/unwindStats.h +++ b/ddprof-lib/src/main/cpp/unwindStats.h @@ -25,22 +25,15 @@ class UnwindFailures { volatile u64 (*_counters)[UNWIND_FAILURE_ANY + 1]; public: - UnwindFailures() : _nameCount(0) { - _names = new char[MAX_UNWIND_FAILURE_NAMES][MAX_NAME_LENGTH]; - _counters = new u64[MAX_UNWIND_FAILURE_NAMES][UNWIND_FAILURE_ANY + 1]; - memset((void*)_names, 0, MAX_UNWIND_FAILURE_NAMES * MAX_NAME_LENGTH); - memset((void*)_counters, 0, MAX_UNWIND_FAILURE_NAMES * (UNWIND_FAILURE_ANY + 1) * sizeof(u64)); - } - - ~UnwindFailures() { - delete[] _names; - delete[] _counters; - } + UnwindFailures(); + ~UnwindFailures(); // Disable copy constructor and assignment operator UnwindFailures(const UnwindFailures&) = delete; UnwindFailures& operator=(const UnwindFailures&) = delete; + void reset(); + void record(UnwindFailureKind kind, const char *name) { if (!name) return; diff --git a/ddprof-lib/src/test/cpp/threadLocalDataPool_ut.cpp b/ddprof-lib/src/test/cpp/threadLocalDataPool_ut.cpp new file mode 100644 index 0000000000..038354e119 --- /dev/null +++ b/ddprof-lib/src/test/cpp/threadLocalDataPool_ut.cpp @@ -0,0 +1,98 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +// threadLocalData.h must precede threadLocalDataPool.h: ProfiledThread needs to +// be a complete type before contains()'s pointer arithmetic is parsed (the +// pool header only forward-declares it). +#include "threadLocalData.h" +#include "threadLocalDataPool.h" +#include "counters.h" + +// Covers ThreadLocalDataPool::contains(), whose result feeds directly into +// unclaim()'s double-release guard. Uses createForTest()/destroyForTest() to +// build a pool isolated from the process-wide singleton (_pool), so these +// boundary checks don't interact with other tests' initialize()/acquire()/ +// release() calls. +class ThreadLocalDataPoolTest : public ::testing::Test {}; + +TEST_F(ThreadLocalDataPoolTest, firstElementIsContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + ProfiledThread* base = pool->threadsForTest(); + + EXPECT_TRUE(pool->containsForTest(base)); + + ThreadLocalDataPool::destroyForTest(pool); +} + +TEST_F(ThreadLocalDataPoolTest, lastElementIsContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + ProfiledThread* base = pool->threadsForTest(); + uint64_t capacity = pool->capacityForTest(); + + EXPECT_TRUE(pool->containsForTest(base + (capacity - 1))); + + ThreadLocalDataPool::destroyForTest(pool); +} + +TEST_F(ThreadLocalDataPoolTest, onePastEndIsNotContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + ProfiledThread* base = pool->threadsForTest(); + uint64_t capacity = pool->capacityForTest(); + + EXPECT_FALSE(pool->containsForTest(base + capacity)); + + ThreadLocalDataPool::destroyForTest(pool); +} + +TEST_F(ThreadLocalDataPoolTest, oneBeforeStartIsNotContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + ProfiledThread* base = pool->threadsForTest(); + + EXPECT_FALSE(pool->containsForTest(base - 1)); + + ThreadLocalDataPool::destroyForTest(pool); +} + +TEST_F(ThreadLocalDataPoolTest, nullptrIsNotContained) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(4); + + EXPECT_FALSE(pool->containsForTest(nullptr)); + + ThreadLocalDataPool::destroyForTest(pool); +} + +// Covers claim()'s (used >= _capacity) fast-path rejection once the pool is +// full. claim()'s return value is nullptr whether that guard reads `>=` or +// (bugged) `>`, because a bugged `>` still falls through to the slot-scanning +// loop, which finds every real slot already claimed and also returns nullptr. +// The observable difference is that the buggy fallthrough additionally +// increments SAMPLES_DROPPED_TLS_POOL_EXHAUSTED, which the fast path must not +// do -- that's what this test pins down. +TEST_F(ThreadLocalDataPoolTest, claimAtCapacityRejectsWithoutExhaustionScan) { + ThreadLocalDataPool* pool = ThreadLocalDataPool::createForTest(2); + + ASSERT_NE(pool->claimForTest(0), nullptr); + ASSERT_NE(pool->claimForTest(1), nullptr); + + long long before = Counters::getCounter(SAMPLES_DROPPED_TLS_POOL_EXHAUSTED); + EXPECT_EQ(pool->claimForTest(2), nullptr); + long long after = Counters::getCounter(SAMPLES_DROPPED_TLS_POOL_EXHAUSTED); + + EXPECT_EQ(after, before); + + ThreadLocalDataPool::destroyForTest(pool); +} From d02c156bb75f2bc8e89b7c99c51074ef7800a727 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 4 Aug 2026 16:30:38 +0000 Subject: [PATCH 14/16] Test TLS priming test --- .../profiler/cpu/TlsPrimingTest.java | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java new file mode 100644 index 0000000000..3548b965a9 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java @@ -0,0 +1,157 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.datadoghq.profiler.cpu; + +import com.datadoghq.profiler.AbstractProfilerTest; +import com.datadoghq.profiler.Platform; +import org.junitpioneer.jupiter.RetryingTest; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +import java.lang.reflect.Method; +import java.util.TreeSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Validates TLS priming by checking that CPU-time profiling captures samples + * from the JVM's JIT compiler threads (HotSpot's "C1 CompilerThread*", + * OpenJ9's "JIT Compilation Thread*"). + * + * Compiler threads are started very early during JVM bootstrap, usually + * before the profiler agent has attached and initialized. Because of that, + * they never go through the normal thread-registration path the profiler + * uses for application threads, so they have no ProfiledThread attached when + * the first profiling signal reaches them. TLS priming is what covers this + * gap: on that first signal, ProfiledThread::acquireCurrent() claims a slot + * from the pool and attaches it via pthread_setspecific right there in the + * signal handler (see threadLocalData.cpp/ThreadLocalDataPool). If priming + * were broken, these early-started threads would simply never show up as + * eventThread on a sample. So seeing compiler-thread samples here is direct + * evidence that TLS priming worked. + * + * The test forces JIT compilation by loading a dynamically-generated class + * with many distinct trivial methods and invoking each one past HotSpot's/ + * OpenJ9's cold-to-compiled threshold, then leaves the CPU sampler running + * long enough for the background compiler thread(s) to actually drain the + * resulting compile queue. + */ +public class TlsPrimingTest extends AbstractProfilerTest { + + private static final String HOTSPOT_COMPILER_THREAD_PREFIX = "C1 CompilerThre"; + private static final String J9_COMPILER_THREAD_PREFIX = "JIT Compilation Thread"; + + // Distinct methods, each invoked enough times to individually cross the + // JIT's cold-to-compiled invocation threshold, so the compiler queue has + // sustained work rather than a single instantly-finished compile. + private static final int METHOD_COUNT = 300; + private static final int INVOCATIONS_PER_METHOD = 1000; + + // Compilation happens asynchronously on background compiler threads, so + // after tripping the invocation thresholds we must give them real time to + // drain the queue while the CPU sampler is still active. + private static final long COMPILE_DRAIN_WAIT_MS = 4000; + + @RetryingTest(3) + public void compilerThreadSamplesArePresent() throws Exception { + triggerJitCompilation(); + + stopProfiler(); + + IItemCollection events = verifyEvents("datadog.ExecutionSample"); + String expectedPrefix = Platform.isJ9() ? J9_COMPILER_THREAD_PREFIX : HOTSPOT_COMPILER_THREAD_PREFIX; + + Set observedThreadNames = new TreeSet<>(); + boolean sawCompilerThreadSample = false; + for (IItemIterable cpuSamples : events) { + IMemberAccessor threadNameAccessor = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(cpuSamples.getType()); + for (IItem sample : cpuSamples) { + String threadName = threadNameAccessor.getMember(sample); + if (threadName == null) { + continue; + } + observedThreadNames.add(threadName); + if (threadName.startsWith(expectedPrefix)) { + sawCompilerThreadSample = true; + } + } + } + + assertTrue(sawCompilerThreadSample, + "expected a datadog.ExecutionSample with eventThread starting with \"" + expectedPrefix + + "\", but observed thread names: " + observedThreadNames); + } + + private void triggerJitCompilation() throws Exception { + Class generated = defineWorkloadClass(); + Object instance = generated.getDeclaredConstructor().newInstance(); + Method[] methods = generated.getDeclaredMethods(); + + for (int call = 0; call < INVOCATIONS_PER_METHOD; call++) { + for (Method m : methods) { + m.invoke(instance); + } + } + + Thread.sleep(COMPILE_DRAIN_WAIT_MS); + } + + /** + * Generates a class with {@value #METHOD_COUNT} distinct no-arg int-returning + * methods (each a different constant expression, so the JIT can't fold them + * into one shared compiled method) and loads it in a fresh ClassLoader. + */ + private static Class defineWorkloadClass() throws ClassNotFoundException { + String internalName = "com/datadoghq/profiler/cpu/generated/CompilerThreadWorkload"; + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); + cw.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, internalName, null, "java/lang/Object", null); + + MethodVisitor ctor = cw.visitMethod(Opcodes.ACC_PUBLIC, "", "()V", null, null); + ctor.visitCode(); + ctor.visitVarInsn(Opcodes.ALOAD, 0); + ctor.visitMethodInsn(Opcodes.INVOKESPECIAL, "java/lang/Object", "", "()V", false); + ctor.visitInsn(Opcodes.RETURN); + ctor.visitMaxs(0, 0); + ctor.visitEnd(); + + for (int i = 0; i < METHOD_COUNT; i++) { + MethodVisitor mv = cw.visitMethod(Opcodes.ACC_PUBLIC, "method" + i, "()I", null, null); + mv.visitCode(); + mv.visitIntInsn(Opcodes.SIPUSH, i); + mv.visitIntInsn(Opcodes.SIPUSH, i + 1); + mv.visitInsn(Opcodes.IMUL); + mv.visitInsn(Opcodes.IRETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + } + cw.visitEnd(); + + IsolatedClassLoader loader = new IsolatedClassLoader(TlsPrimingTest.class.getClassLoader()); + return loader.defineClass(internalName.replace('/', '.'), cw.toByteArray()); + } + + private static final class IsolatedClassLoader extends ClassLoader { + IsolatedClassLoader(ClassLoader parent) { + super(parent); + } + + Class defineClass(String name, byte[] bytecode) { + return defineClass(name, bytecode, 0, bytecode.length); + } + } + + @Override + protected String getProfilerCommand() { + return "cpu=1ms"; + } +} From c3e0c74e8f0f9fe75ebad1655185a68df38731b8 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 4 Aug 2026 19:13:30 +0000 Subject: [PATCH 15/16] Cleanup and test --- ddprof-lib/src/main/cpp/threadLocalData.cpp | 5 +++++ ddprof-lib/src/main/cpp/threadLocalDataPool.cpp | 13 ++++++++++++- ddprof-lib/src/main/cpp/threadLocalDataPool.h | 7 +++++-- ddprof-lib/src/main/cpp/unwindStats.cpp | 3 +-- .../com/datadoghq/profiler/cpu/TlsPrimingTest.java | 2 +- 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index 456efe9fa7..0fc7ff377b 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -140,4 +140,9 @@ void ProfiledThread::resetClaimed(int tid) { _in_critical_section = false; _unwind_failures.reset(); + + #ifdef __FAULT_INJECTION__ + _fi_rng = ((u64)(uintptr_t)this) ^ (0x9e3779b97f4a7c15ULL * (u64)tid); + if (_fi_rng == 0) _fi_rng = 1; +#endif } diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp index 1b25be60c0..73f3bbf2f9 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.cpp @@ -12,7 +12,7 @@ ThreadLocalDataPool* ThreadLocalDataPool::_pool = nullptr; -ThreadLocalDataPool::ThreadLocalDataPool(uint64_t capacity) +ThreadLocalDataPool::ThreadLocalDataPool(uint16_t capacity) : _capacity(capacity), _used(0), _threads(nullptr) { const size_t malloc_size = capacity * sizeof(ProfiledThread); void* p = malloc(malloc_size); @@ -27,6 +27,17 @@ ThreadLocalDataPool::ThreadLocalDataPool(uint64_t capacity) NativeMem::record(NM_THREAD_LOCAL, malloc_size + sizeof(ThreadLocalDataPool)); } +#ifdef UNIT_TEST +ThreadLocalDataPool::~ThreadLocalDataPool() { + if (_threads != nullptr) { + for (int index = 0; index < _capacity; index++) { + _threads[index].~ProfiledThread(); + } + free(reinterpret_cast(_threads)); + } +} +#endif // UNIT_TEST + ProfiledThread* ThreadLocalDataPool::claim(int tid) { if (_threads == nullptr) { diff --git a/ddprof-lib/src/main/cpp/threadLocalDataPool.h b/ddprof-lib/src/main/cpp/threadLocalDataPool.h index 43e64749cd..4fb4deb765 100644 --- a/ddprof-lib/src/main/cpp/threadLocalDataPool.h +++ b/ddprof-lib/src/main/cpp/threadLocalDataPool.h @@ -24,9 +24,12 @@ class ThreadLocalDataPool { ThreadLocalDataPool(const ThreadLocalDataPool&) = delete; ThreadLocalDataPool& operator=(const ThreadLocalDataPool&) = delete; - ThreadLocalDataPool(uint64_t capacity = DEFAULT_CAPACITY); + ThreadLocalDataPool(uint16_t capacity = DEFAULT_CAPACITY); +#ifdef UNIT_TEST ~ThreadLocalDataPool(); - +#else + ~ThreadLocalDataPool() = delete; +#endif // UNIT_TEST ProfiledThread* claim(int tid); bool unclaim(ProfiledThread* t); diff --git a/ddprof-lib/src/main/cpp/unwindStats.cpp b/ddprof-lib/src/main/cpp/unwindStats.cpp index abc8123034..86c99dbcea 100644 --- a/ddprof-lib/src/main/cpp/unwindStats.cpp +++ b/ddprof-lib/src/main/cpp/unwindStats.cpp @@ -13,8 +13,7 @@ UnwindFailures::UnwindFailures() : _nameCount(0) { UnwindFailures::~UnwindFailures() { delete[] _names; delete[] _counters; - } - +} void UnwindFailures::reset() { memset((void*)_names, 0, MAX_UNWIND_FAILURE_NAMES * MAX_NAME_LENGTH); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java index 3548b965a9..8ae20dc728 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/TlsPrimingTest.java @@ -111,7 +111,7 @@ private void triggerJitCompilation() throws Exception { * methods (each a different constant expression, so the JIT can't fold them * into one shared compiled method) and loads it in a fresh ClassLoader. */ - private static Class defineWorkloadClass() throws ClassNotFoundException { + private static Class defineWorkloadClass() throws ClassFormatError { String internalName = "com/datadoghq/profiler/cpu/generated/CompilerThreadWorkload"; ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); cw.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, internalName, null, "java/lang/Object", null); From 38e7faa50b81d4ef273da9c35518653ec2687bc6 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 4 Aug 2026 19:40:04 +0000 Subject: [PATCH 16/16] Cleanup includes --- ddprof-lib/src/test/cpp/faultInjection_ut.cpp | 2 +- .../src/test/cpp/hotspot_crash_protection_ut.cpp | 2 +- ddprof-lib/src/test/cpp/stackWalker_ut.cpp | 14 +++++++------- .../src/test/cpp/stress_threadLifecycle_ut.cpp | 2 +- .../src/test/cpp/threadFilter_lifecycle_ut.cpp | 2 +- .../src/test/cpp/thread_teardown_safety_ut.cpp | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ddprof-lib/src/test/cpp/faultInjection_ut.cpp b/ddprof-lib/src/test/cpp/faultInjection_ut.cpp index f0a49354e2..ab0102dee1 100644 --- a/ddprof-lib/src/test/cpp/faultInjection_ut.cpp +++ b/ddprof-lib/src/test/cpp/faultInjection_ut.cpp @@ -14,7 +14,7 @@ #include "faultInjection.h" #include "safeAccess.h" #include "os.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "profiler.h" #include "../../main/cpp/gtest_crash_handler.h" diff --git a/ddprof-lib/src/test/cpp/hotspot_crash_protection_ut.cpp b/ddprof-lib/src/test/cpp/hotspot_crash_protection_ut.cpp index 3739606bbb..2131ecac04 100644 --- a/ddprof-lib/src/test/cpp/hotspot_crash_protection_ut.cpp +++ b/ddprof-lib/src/test/cpp/hotspot_crash_protection_ut.cpp @@ -30,7 +30,7 @@ */ #include -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "profiler.h" #include "asyncSampleMutex.h" diff --git a/ddprof-lib/src/test/cpp/stackWalker_ut.cpp b/ddprof-lib/src/test/cpp/stackWalker_ut.cpp index 1212b704d2..efe19492da 100644 --- a/ddprof-lib/src/test/cpp/stackWalker_ut.cpp +++ b/ddprof-lib/src/test/cpp/stackWalker_ut.cpp @@ -3,18 +3,18 @@ */ #include -#include "../../main/cpp/stackWalker.h" -#include "../../main/cpp/gtest_crash_handler.h" +#include "stackWalker.h" +#include "gtest_crash_handler.h" #ifdef __linux__ #include #include #include -#include "../../main/cpp/counters.h" -#include "../../main/cpp/os.h" -#include "../../main/cpp/profiler.h" -#include "../../main/cpp/stackFrame.h" -#include "../../main/cpp/threadLocalData.h" +#include "counters.h" +#include "os.h" +#include "profiler.h" +#include "stackFrame.h" +#include "threadLocalData.inline.h" [[maybe_unused]] static long long* _stackwalker_ut_counters_init = Counters::getCounters(); #endif diff --git a/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp b/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp index 5615010e0a..8ad8a336e9 100644 --- a/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp +++ b/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp @@ -16,7 +16,7 @@ #include "callTraceStorage.h" #include "callTraceHashTable.h" #include "threadFilter.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "arch.h" #include "spinLock.h" diff --git a/ddprof-lib/src/test/cpp/threadFilter_lifecycle_ut.cpp b/ddprof-lib/src/test/cpp/threadFilter_lifecycle_ut.cpp index 9ca9b082e1..dab8333f04 100644 --- a/ddprof-lib/src/test/cpp/threadFilter_lifecycle_ut.cpp +++ b/ddprof-lib/src/test/cpp/threadFilter_lifecycle_ut.cpp @@ -19,7 +19,7 @@ #ifdef __linux__ #include "threadFilter.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include "../../main/cpp/gtest_crash_handler.h" #include diff --git a/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp b/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp index 02f8c3f50c..9195f27ad5 100644 --- a/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp +++ b/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp @@ -20,7 +20,7 @@ #include "guards.h" #include "nativeMem.h" -#include "threadLocalData.h" +#include "threadLocalData.inline.h" #include #include