Skip to content

Commit b5fd279

Browse files
gleocadiegleocadie
authored
[Profiler] Add timer_create-based CPU profiling on Linux (#5476)
1 parent 2bfbe4f commit b5fd279

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+988
-75
lines changed

‎profiler/build/CpuWallTime.linux.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,16 @@
3838
"DD_INTERNAL_USE_BACKTRACE2": "0",
3939
"DD_TRACE_ENABLED" : "0"
4040
}
41+
},
42+
{
43+
"name": "Profiler_cpu_walltime_timer_create",
44+
"environmentVariables": {
45+
"DD_CLR_ENABLE_NGEN": "true",
46+
"DD_PROFILING_ENABLED": "1",
47+
"DD_PROFILING_CPU_ENABLED": "1",
48+
"DD_INTERNAL_CPU_PROFILER_TYPE": "TimerCreate",
49+
"DD_TRACE_ENABLED" : "0"
50+
}
4151
}
4252
],
4353
"processName": "Samples.Computer01",

‎profiler/build/crank/Samples.AspNetCoreSimpleController.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,22 @@ scenarios:
115115
duration: 240
116116
serverPort: 5000
117117
path: /hello
118+
119+
profiler_cpu_timer_create:
120+
application:
121+
job: server
122+
environmentVariables:
123+
COR_ENABLE_PROFILING: 1
124+
CORECLR_ENABLE_PROFILING: 1
125+
DD_PROFILING_ENABLED: 1
126+
DD_PROFILING_WALLTIME_ENABLED: 0
127+
DD_PROFILING_CPU_ENABLED: 1
128+
DD_INTERNAL_CPU_PROFILER_TYPE: "TimerCreate"
129+
COMPlus_EnableDiagnostics: 1
130+
load:
131+
job: bombardier
132+
variables:
133+
warmup: 30
134+
duration: 240
135+
serverPort: 5000
136+
path: /hello

‎profiler/build/crank/run.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ elif [ "$1" = "linux" ]; then
6363
rm -f profiler_exceptions_baseline_linux.json
6464
rm -f profiler_exceptions_linux.json
6565
rm -f profiler_cpu_linux.json
66+
rm -f profiler_cpu_timer_create_linux.json
6667

6768
crank --config Samples.AspNetCoreSimpleController.yml --scenario profiler_baseline --profile linux --json profiler_baseline_linux.json $repository $commit --property name=AspNetCoreSimpleController --property scenario=profiler_baseline --property profile=linux --property arch=x64 --variable commit_hash=$commit_sha
6869
dd-trace --crank-import="profiler_baseline_linux.json"
@@ -76,8 +77,8 @@ elif [ "$1" = "linux" ]; then
7677
crank --config Samples.AspNetCoreSimpleController.yml --scenario profiler_exceptions --profile linux --json profiler_exceptions_linux.json $repository $commit --property name=AspNetCoreSimpleController --property scenario=profiler_exceptions --property profile=linux --property arch=x64 --variable commit_hash=$commit_sha
7778
dd-trace --crank-import="profiler_exceptions_linux.json"
7879

79-
crank --config Samples.AspNetCoreSimpleController.yml --scenario profiler_cpu --profile linux --json profiler_cpu_linux.json $repository $commit --property name=AspNetCoreSimpleController --property scenario=profiler_cpu --property profile=linux --property arch=x64 --variable commit_hash=$commit_sha
80-
dd-trace --crank-import="profiler_cpu_linux.json"
80+
crank --config Samples.AspNetCoreSimpleController.yml --scenario profiler_cpu_timer_create --profile linux --json profiler_cpu_timer_create_linux.json $repository $commit --property name=AspNetCoreSimpleController --property scenario=profiler_cpu --property profile=linux --property arch=x64 --variable commit_hash=$commit_sha
81+
dd-trace --crank-import="profiler_cpu_timer_create_linux.json"
8182
else
8283
echo "Unknown argument $1"
8384
exit 1

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native.Linux/Datadog.Profiler.Native.Linux.vcxproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@
119119
<ClCompile Include="OsSpecificApi.cpp" />
120120
<ClCompile Include="ProfilerSignalManager.cpp" />
121121
<ClCompile Include="SystemCallsShield.cpp" />
122+
<ClCompile Include="TimerCreateCpuProfiler.cpp" />
122123
</ItemGroup>
123124
<ItemGroup>
124125
<None Include="prepare_loader_for_linking.sh" />
@@ -129,6 +130,7 @@
129130
<ClInclude Include="LinuxThreadInfo.h" />
130131
<ClInclude Include="ProfilerSignalManager.h" />
131132
<ClInclude Include="SystemCallsShield.h" />
133+
<ClInclude Include="TimerCreateCpuProfiler.h" />
132134
</ItemGroup>
133135
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
134136
<ClCompile>

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native.Linux/Datadog.Profiler.Native.Linux.vcxproj.filters‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
<ClCompile Include="LinuxThreadInfo.cpp" />
1010
<ClCompile Include="SystemCallsShield.cpp" />
1111
<ClCompile Include="CrashReportingLinux.cpp" />
12+
<ClCompile Include="TimerCreateCpuProfiler.cpp" />
1213
</ItemGroup>
1314
<ItemGroup>
1415
<Filter Include="Scripts">
@@ -26,5 +27,6 @@
2627
<ClInclude Include="LinuxThreadInfo.h" />
2728
<ClInclude Include="SystemCallsShield.h" />
2829
<ClInclude Include="CrashReportingLinux.h" />
30+
<ClInclude Include="TimerCreateCpuProfiler.h" />
2931
</ItemGroup>
30-
</Project>
32+
</Project>

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native.Linux/LinuxStackFramesCollector.cpp‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,10 @@ LinuxStackFramesCollector::LinuxStackFramesCollector(
4040
_useBacktrace2{configuration->UseBacktrace2()},
4141
_plibrariesInfo{librariesCacheInfo}
4242
{
43-
_signalManager->RegisterHandler(LinuxStackFramesCollector::CollectStackSampleSignalHandler);
43+
if (_signalManager != nullptr)
44+
{
45+
_signalManager->RegisterHandler(LinuxStackFramesCollector::CollectStackSampleSignalHandler);
46+
}
4447
}
4548

4649
LinuxStackFramesCollector::~LinuxStackFramesCollector()
@@ -90,6 +93,10 @@ StackSnapshotResultBuffer* LinuxStackFramesCollector::CollectStackSampleImplemen
9093
{
9194
long errorCode;
9295

96+
// If there a timer associated to the managed thread, we have to disarm it.
97+
// Otherwise, the CPU consumption to collect the callstack, will be accounted as "user app CPU time"
98+
auto timerId = pThreadInfo->GetTimerId();
99+
93100
_plibrariesInfo->UpdateCache();
94101

95102
if (selfCollect)
@@ -108,12 +115,34 @@ StackSnapshotResultBuffer* LinuxStackFramesCollector::CollectStackSampleImplemen
108115
}
109116
else
110117
{
111-
if (!_signalManager->IsHandlerInPlace())
118+
if (_signalManager == nullptr || !_signalManager->IsHandlerInPlace())
112119
{
113120
*pHR = E_FAIL;
114121
return GetStackSnapshotResult();
115122
}
116123

124+
struct itimerspec old;
125+
126+
if (timerId != -1)
127+
{
128+
struct itimerspec ts;
129+
ts.it_interval.tv_sec = 0;
130+
ts.it_interval.tv_nsec = 0;
131+
ts.it_value = ts.it_interval;
132+
133+
// disarm the timer so this is not accounted for the managed thread cpu usage
134+
syscall(__NR_timer_settime, timerId, 0, &ts, &old);
135+
}
136+
137+
on_leave
138+
{
139+
if (timerId != -1)
140+
{
141+
// re-arm the timer
142+
syscall(__NR_timer_settime, timerId, 0, &old, nullptr);
143+
}
144+
};
145+
117146
std::unique_lock<std::mutex> stackWalkInProgressLock(s_stackWalkInProgressMutex);
118147

119148
const auto threadId = static_cast<::pid_t>(pThreadInfo->GetOsThreadId());
@@ -144,7 +173,7 @@ StackSnapshotResultBuffer* LinuxStackFramesCollector::CollectStackSampleImplemen
144173
if (status == std::cv_status::timeout)
145174
{
146175
_lastStackWalkErrorCode = E_ABORT;
147-
;
176+
148177
if (!_signalManager->CheckSignalHandler())
149178
{
150179
_lastStackWalkErrorCode = E_FAIL;
@@ -379,4 +408,4 @@ void LinuxStackFramesCollector::ErrorStatistics::Log()
379408
ss.str());
380409
_stats.clear();
381410
}
382-
}
411+
}

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native.Linux/MmapMemoryResource.cpp‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
2+
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2022 Datadog, Inc.
3+
4+
#include "MmapMemoryResource.h"
5+
6+
#include <sys/mman.h>
7+
#include <unistd.h>
8+
9+
inline long get_page_size()
10+
{
11+
static long page_size = 0;
12+
if (!page_size)
13+
{
14+
page_size = sysconf(_SC_PAGESIZE);
15+
}
16+
return page_size;
17+
}
18+
19+
inline uint64_t align_to_page(uint64_t x)
20+
{
21+
return ((x - 1) | (get_page_size() - 1)) + 1;
22+
}
23+
24+
void* MmapMemoryResource::do_allocate(size_t _Bytes, size_t _Align)
25+
{
26+
auto const total_length = align_to_page(_Bytes);
27+
void* region = mmap(nullptr, total_length, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
28+
if (MAP_FAILED == region || region == nullptr)
29+
{
30+
return nullptr;
31+
}
32+
33+
return region;
34+
}
35+
36+
void MmapMemoryResource::do_deallocate(void* _Ptr, size_t _Bytes, size_t _Align)
37+
{
38+
auto const total_length = align_to_page(_Bytes);
39+
munmap(_Ptr, total_length);
40+
}
41+
42+
bool MmapMemoryResource::do_is_equal(const memory_resource& _That) const noexcept
43+
{
44+
return this == &_That;
45+
}

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native.Linux/MmapMemoryResource.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
2+
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2022 Datadog, Inc.
3+
4+
#pragma once
5+
6+
#include "shared/src/native-src/dd_memory_resource.hpp"
7+
8+
class MmapMemoryResource : public shared::pmr::memory_resource
9+
{
10+
public:
11+
~MmapMemoryResource() = default;
12+
13+
private:
14+
void* do_allocate(size_t _Bytes, size_t _Align) override;
15+
void do_deallocate(void* _Ptr, size_t _Bytes, size_t _Align) override;
16+
bool do_is_equal(const memory_resource& _That) const noexcept override;
17+
};

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native.Linux/OsSpecificApi.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ std::unique_ptr<StackFramesCollectorBase> CreateNewStackFramesCollectorInstance(
6363
IConfiguration const* const pConfiguration,
6464
CallstackProvider* callstackProvider)
6565
{
66-
return std::make_unique<LinuxStackFramesCollector>(ProfilerSignalManager::Get(), pConfiguration, callstackProvider, LibrariesInfoCache::Get());
66+
return std::make_unique<LinuxStackFramesCollector>(ProfilerSignalManager::Get(SIGUSR1), pConfiguration, callstackProvider, LibrariesInfoCache::Get());
6767
}
6868

6969
// https://linux.die.net/man/5/proc
@@ -405,4 +405,4 @@ std::unique_ptr<IEtwEventsManager> CreateEtwEventsManager(
405405
// No ETW implementation on Linux
406406
return nullptr;
407407
}
408-
} // namespace OsSpecificApi
408+
} // namespace OsSpecificApi

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native.Linux/ProfilerSignalManager.cpp‎

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,15 @@
44
#include "ProfilerSignalManager.h"
55

66
#include <signal.h>
7+
#include <stdexcept>
78
#include <sys/syscall.h>
89

910
#include "Log.h"
1011
#include "OpSysTools.h"
1112

1213
ProfilerSignalManager::ProfilerSignalManager() noexcept :
1314
_canReplaceSignalHandler{true},
14-
_signalToSend{SIGUSR1},
15+
_signalToSend{-1},
1516
_handler{nullptr},
1617
_processId{OpSysTools::GetProcId()},
1718
_isHandlerInPlace{false},
@@ -30,11 +31,20 @@ ProfilerSignalManager::~ProfilerSignalManager() noexcept
3031
_handler = nullptr;
3132
}
3233

33-
ProfilerSignalManager* ProfilerSignalManager::Get()
34+
ProfilerSignalManager* ProfilerSignalManager::Get(int signal)
3435
{
35-
static ProfilerSignalManager signalManager{};
36+
static ProfilerSignalManager signalManagers[31]; // only for the standard signals
3637

37-
return &signalManager;
38+
if (signal < 1 || signal > 31)
39+
{
40+
Log::Info("Signal argument is invalid (", std::to_string(signal), " aka ", strsignal(signal), "). Value must be: 1 <= signal <= 31");
41+
return nullptr;
42+
}
43+
44+
auto* manager = &signalManagers[signal - 1]; // 0-based array
45+
manager->SetSignal(signal);
46+
47+
return manager;
3848
}
3949

4050
bool ProfilerSignalManager::RegisterHandler(HandlerFn_t handler)
@@ -64,6 +74,30 @@ bool ProfilerSignalManager::RegisterHandler(HandlerFn_t handler)
6474
return _isHandlerInPlace;
6575
}
6676

77+
bool ProfilerSignalManager::UnRegisterHandler()
78+
{
79+
if (!IsProfilerSignalHandlerInstalled())
80+
{
81+
return false;
82+
}
83+
84+
int32_t result = sigaction(_signalToSend, &_previousAction, nullptr);
85+
if (result != 0)
86+
{
87+
Log::Error("ProfilerSignalManager::UnRegisterHandler: Failed to un-register signal handler for ", _signalToSend, " signals. Reason: ",
88+
strerror(errno), ".");
89+
return false;
90+
}
91+
92+
_isHandlerInPlace = false;
93+
return true;
94+
}
95+
96+
void ProfilerSignalManager::SetSignal(int32_t signal)
97+
{
98+
_signalToSend = signal;
99+
}
100+
67101
std::int32_t ProfilerSignalManager::SendSignal(pid_t threadId)
68102
{
69103
#ifndef NDEBUG
@@ -133,18 +167,24 @@ bool ProfilerSignalManager::SetupSignalHandler()
133167
int32_t result = sigaction(_signalToSend, &sampleAction, &_previousAction);
134168
if (result != 0)
135169
{
136-
Log::Error("ProfilerSignalManager::SetupSignalHandler: Failed to setup signal handler for SIGUSR1 signals. Reason: ",
170+
Log::Error("ProfilerSignalManager::SetupSignalHandler: Failed to setup signal handler for ", strsignal(_signalToSend), " signals. Reason: ",
137171
strerror(errno), ".");
138172
return false;
139173
}
140174

141-
Log::Info("ProfilerSignalManager::SetupSignalHandler: Successfully setup signal handler for SIGUSR1 signal.");
175+
Log::Info("ProfilerSignalManager::SetupSignalHandler: Successfully setup signal handler for ", strsignal(_signalToSend), " signal.");
142176
return true;
143177
}
144178

145179
void ProfilerSignalManager::SignalHandler(int signal, siginfo_t* info, void* context)
146180
{
147-
auto* signalManager = Get();
181+
auto* signalManager = Get(signal);
182+
183+
if (signalManager == nullptr) [[unlikely]]
184+
{
185+
return;
186+
}
187+
148188
if (!signalManager->CallCustomHandler(signal, info, context))
149189
{
150190
signalManager->CallOrignalHandler(signal, info, context);
@@ -189,4 +229,9 @@ void ProfilerSignalManager::CallOrignalHandler(int32_t signal, siginfo_t* info,
189229
}
190230

191231
isExecuting = false;
232+
}
233+
234+
int32_t ProfilerSignalManager::GetSignal() const
235+
{
236+
return _signalToSend;
192237
}

0 commit comments

Comments
 (0)