Skip to content

Commit 1c38a93

Browse files
Felix Schlepperf3sch
Felix Schlepper
authored and
f3sch
committed
ITSGPU: use size estimates
Signed-off-by: Felix Schlepper <felix.schlepper@cern.ch>
1 parent 69d6252 commit 1c38a93

21 files changed

Lines changed: 1487 additions & 2330 deletions

‎Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/LaunchGeometry.h‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
///
12+
/// \file LaunchGeometry.h
13+
/// \brief Compile-time launch geometry of the ITS tracking kernels, per GPU family.
14+
///
15+
16+
#ifndef ITSTRACKINGGPU_LAUNCHGEOMETRY_H_
17+
#define ITSTRACKINGGPU_LAUNCHGEOMETRY_H_
18+
19+
namespace o2::its::gpu
20+
{
21+
22+
#if defined(GPUCA_GPUTYPE_VEGA) // gfx906: MI50, Radeon VII
23+
constexpr int ComputeUnits = 60;
24+
constexpr int WarpSize = 64;
25+
#elif defined(GPUCA_GPUTYPE_MI100) // gfx908
26+
constexpr int ComputeUnits = 120;
27+
constexpr int WarpSize = 64;
28+
#elif defined(GPUCA_GPUTYPE_MI210) // gfx90a
29+
constexpr int ComputeUnits = 104;
30+
constexpr int WarpSize = 64;
31+
#elif defined(GPUCA_GPUTYPE_MI300) // gfx942: MI300X (MI300A has 228)
32+
constexpr int ComputeUnits = 304;
33+
constexpr int WarpSize = 64;
34+
#elif defined(GPUCA_GPUTYPE_RDNA) // gfx10xx/11xx consumer parts, wave32
35+
constexpr int ComputeUnits = 60;
36+
constexpr int WarpSize = 32;
37+
#elif defined(GPUCA_GPUTYPE_BLACKWELL) // sm_120: RTX 5080
38+
constexpr int ComputeUnits = 84;
39+
constexpr int WarpSize = 32;
40+
#elif defined(GPUCA_GPUTYPE_HOPPER) // sm_90: H100
41+
constexpr int ComputeUnits = 132;
42+
constexpr int WarpSize = 32;
43+
#elif defined(GPUCA_GPUTYPE_ADA) // sm_89: RTX 4090
44+
constexpr int ComputeUnits = 128;
45+
constexpr int WarpSize = 32;
46+
#elif defined(GPUCA_GPUTYPE_AMPERE) // sm_80/86: A100 has 108, RTX 3090 has 82
47+
constexpr int ComputeUnits = 108;
48+
constexpr int WarpSize = 32;
49+
#elif defined(GPUCA_GPUTYPE_TURING) // sm_75: RTX 2080 Ti
50+
constexpr int ComputeUnits = 68;
51+
constexpr int WarpSize = 32;
52+
#else
53+
#warning "GPU architecture not available setting fallback"
54+
constexpr int ComputeUnits = 60;
55+
constexpr int WarpSize = 64;
56+
#endif
57+
58+
constexpr int GPUThreads = 256;
59+
constexpr int BlocksPerComputeUnit = 4;
60+
constexpr int GPUBlocks = ComputeUnits * BlocksPerComputeUnit;
61+
constexpr int GPUThreadsTotal = GPUBlocks * GPUThreads;
62+
63+
static_assert(GPUThreads % WarpSize == 0, "block size must be a whole number of warps/waves");
64+
static_assert(GPUBlocks > 0 && GPUThreads > 0, "degenerate launch geometry");
65+
66+
} // namespace o2::its::gpu
67+
68+
#endif // ITSTRACKINGGPU_LAUNCHGEOMETRY_H_

‎Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h‎

Lines changed: 66 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -45,68 +45,55 @@ class TimeFrameGPU : public TimeFrame<NLayers>
4545
/// Most relevant operations
4646
void pushMemoryStack(const int);
4747
void popMemoryStack(const int);
48-
void registerHostMemory(const int);
49-
void unregisterHostMemory(const int);
48+
void unregisterHostMemory();
5049
void initialise(const TrackingParameters&, int maxLayers);
5150
void initialise(const TrackingParameters&, int maxLayers, int iteration);
5251
void loadIndexTableUtils();
5352
void loadTrackingTopologies();
5453
void loadTrackingFrameInfoDevice(const int);
55-
void createTrackingFrameInfoDeviceArray();
54+
void createTrackingFrameInfoDeviceArray(const int = NLayers);
5655
void loadUnsortedClustersDevice(const int);
5756
void createUnsortedClustersDeviceArray(const int = NLayers);
5857
void loadClustersDevice(const int);
5958
void createClustersDeviceArray(const int = NLayers);
6059
void loadClustersIndexTables(const int);
61-
void createClustersIndexTablesArray();
60+
void createClustersIndexTablesArray(const int = NLayers);
6261
void createUsedClustersDevice(const int);
6362
void createUsedClustersDeviceArray(const int = NLayers);
6463
void loadUsedClustersDevice();
6564
void loadROFrameClustersDevice(const int);
66-
void createROFrameClustersDeviceArray();
65+
void createROFrameClustersDeviceArray(const int = NLayers);
6766
void loadROFCutMask(const int);
6867
void loadVertices();
6968
void loadROFOverlapTable();
7069
void loadROFVertexLookupTable();
71-
void updateROFVertexLookupTable();
70+
void uploadROFVertexLookupTable();
71+
void loadIterationParameters(const TrackingParameters&);
7272

7373
///
7474
void createTrackletsLUTDevice(bool, const int);
7575
void createTrackletsLUTDeviceArray();
76-
void loadTrackletsDevice();
77-
void loadTrackletsLUTDevice();
78-
void loadCellsDevice();
79-
void loadCellsLUTDevice();
80-
void loadTrackSeedsDevice();
81-
void loadTrackSeedsChi2Device();
82-
void loadTrackSeedsDevice(bounded_vector<TrackSeedN>&);
83-
void createTrackletsBuffers(const int);
76+
void createTrackSeedsDevice(const size_t capacity);
77+
void createTrackletsBuffers(const int, size_t capacity);
8478
void createTrackletsBuffersArray();
85-
void createCellsBuffers(const int);
79+
void createCellsBuffers(const int, size_t capacity);
8680
void createCellsBuffersArray();
87-
void createCellsDevice();
8881
void createCellsLUTDevice(const int);
8982
void createCellsLUTDeviceArray();
90-
void createNeighboursIndexTablesDevice(const int);
91-
void createNeighboursDevice(const unsigned int layer);
83+
void createNeighboursDevice(const unsigned int layer, size_t capacity);
9284
void createNeighboursLUTDevice(const int, const unsigned int);
93-
void createTrackITSExtDevice(const size_t);
85+
void createTrackITSExtDevice(const size_t capacity);
86+
void createTrackITSExtHost(const size_t nTracks);
9487
void createTrackExtensionScratchDevice(const int nThreads, const int maxHypotheses);
9588
void downloadTrackITSExtDevice();
9689
void downloadTrackIndicesDevice();
97-
void downloadCellsNeighboursDevice(std::vector<bounded_vector<CellNeighbour>>&, const int);
98-
void downloadNeighboursLUTDevice(bounded_vector<int>&, const int);
99-
void downloadCellsDevice();
100-
void downloadCellsLUTDevice();
10190

10291
/// synchronization
10392
auto& getStream(const size_t stream) { return mGpuStreams[stream]; }
10493
auto& getStreams() { return mGpuStreams; }
105-
void syncStream(const size_t stream);
10694
void syncStreams(const bool = true);
10795
void waitEvent(const int, const int);
10896
void recordEvent(const int);
109-
void recordEvents(const int = 0, const int = NLayers);
11097

11198
/// cleanup
11299
virtual void wipe() final;
@@ -115,29 +102,25 @@ class TimeFrameGPU : public TimeFrame<NLayers>
115102
virtual bool isGPU() const noexcept final { return true; }
116103
virtual const char* getName() const noexcept override final { return "GPU"; }
117104
IndexTableUtilsN* getDeviceIndexTableUtils() { return mIndexTableUtilsDevice; }
105+
const float* getDeviceLayerRadii() const { return mLayerRadiiDevice; }
106+
const float* getDeviceMinPts() const { return mMinPtsDevice; }
107+
const float* getDeviceLayerxX0() const { return mLayerxX0Device; }
118108
const auto getDeviceROFOverlapTableView() { return mDeviceROFOverlapTableView; }
119109
const auto getDeviceROFVertexLookupTableView() { return mDeviceROFVertexLookupTableView; }
120110
const auto getDeviceROFMaskTableView() { return mDeviceROFMaskTableView; }
121111
const auto getDeviceTrackingTopologyView() const { return mDeviceTrackingTopologyView; }
122-
int* getDeviceROFramesClusters(const int layer) { return mROFramesClustersDevice[layer]; }
123112
auto& getTrackITSExt() { return mTrackITSExt; }
124113
auto& getTrackIndices() { return mTrackIndices; }
125114
Vertex* getDeviceVertices() { return mPrimaryVerticesDevice; }
126-
int* getDeviceROFramesPV() { return mROFramesPVDevice; }
127-
unsigned char* getDeviceUsedClusters(const int);
128-
const o2::base::Propagator* getChainPropagator();
129115

130116
// Hybrid
131117
TrackITSExt* getDeviceTrackITSExt() { return mTrackITSExtDevice; }
132118
int* getDeviceTrackIndices() { return mTrackIndicesDevice; }
133119
TrackExtensionHypothesis<NLayers>* getDeviceActiveTrackExtensionHypotheses() { return mActiveTrackExtensionHypothesesDevice; }
134120
TrackExtensionHypothesis<NLayers>* getDeviceNextTrackExtensionHypotheses() { return mNextTrackExtensionHypothesesDevice; }
135121
int* getDeviceNeighboursLUT(const int layer) { return mNeighboursLUTDevice[layer]; }
136-
gsl::span<int*> getDeviceNeighboursLUTs() { return mNeighboursLUTDevice; }
137122
CellNeighbour** getDeviceArrayNeighbours() { return mNeighboursDeviceArray; }
138-
std::array<CellNeighbour*, MaxCells>& getDeviceNeighboursAll() { return mNeighboursDevice; }
139123
CellNeighbour* getDeviceNeighbours(const int layer) { return mNeighboursDevice[layer]; }
140-
TrackingFrameInfo* getDeviceTrackingFrameInfo(const int);
141124
const TrackingFrameInfo** getDeviceArrayTrackingFrameInfo() const { return mTrackingFrameInfoDeviceArray; }
142125
const Cluster** getDeviceArrayClusters() const { return mClustersDeviceArray; }
143126
const Cluster** getDeviceArrayUnsortedClusters() const { return mUnsortedClustersDeviceArray; }
@@ -151,11 +134,9 @@ class TimeFrameGPU : public TimeFrame<NLayers>
151134
int** getDeviceArrayNeighboursCellLUT() const { return mNeighboursCellLUTDeviceArray; }
152135
CellSeed** getDeviceArrayCells() { return mCellsDeviceArray; }
153136
TrackSeedN* getDeviceTrackSeeds() { return mTrackSeedsDevice; }
154-
int* getDeviceTrackSeedsLUT() { return mTrackSeedsLUTDevice; }
137+
int* getDeviceTrackSeedIndices() { return mTrackSeedIndicesDevice; }
138+
int* getDeviceTrackCounter() { return mTrackCounterDevice; }
155139
auto getNTrackSeeds() const { return mNTracks; }
156-
o2::track::TrackParCovF** getDeviceArrayTrackSeeds() { return mCellSeedsDeviceArray; }
157-
float** getDeviceArrayTrackSeedsChi2() { return mCellSeedsChi2DeviceArray; }
158-
int* getDeviceNeighboursIndexTables(const int layer) { return mNeighboursIndexTablesDevice[layer]; }
159140

160141
void setDevicePropagator(const o2::base::PropagatorImpl<float>* p) final { this->mPropagatorDevice = p; }
161142

@@ -164,7 +145,6 @@ class TimeFrameGPU : public TimeFrame<NLayers>
164145
gsl::span<int> getNCells() { return {mNCells.data(), static_cast<gsl::span<int>::size_type>(this->mTrackingTopologyView.nCells)}; }
165146
auto& getArrayNCells() { return mNCells; }
166147
gsl::span<int> getNNeighbours() { return {mNNeighbours.data(), static_cast<gsl::span<int>::size_type>(this->mTrackingTopologyView.nCells)}; }
167-
auto& getArrayNNeighbours() { return mNNeighbours; }
168148

169149
// Host-available device getters
170150
gsl::span<int*> getDeviceTrackletsLUTs() { return mTrackletsLUTDevice; }
@@ -178,16 +158,45 @@ class TimeFrameGPU : public TimeFrame<NLayers>
178158
size_t getNumberOfNeighbours() const final;
179159

180160
private:
181-
void allocMemAsync(void**, size_t, Stream&, bool, int32_t = o2::gpu::GPUMemoryResource::MEMORY_GPU); // Abstract owned and unowned memory allocations on specific stream
182-
void allocMem(void**, size_t, bool, int32_t = o2::gpu::GPUMemoryResource::MEMORY_GPU); // Abstract owned and unowned memory allocations on default stream
161+
enum class SlotInit {
162+
Raw, ///< whatever the allocator handed back
163+
Zero ///< cleared on the slot's stream
164+
};
165+
166+
template <typename T>
167+
T* allocDevice(size_t n, int32_t type = o2::gpu::GPUMemoryResource::MEMORY_GPU);
168+
template <typename T>
169+
T* allocDeviceAsync(size_t n, Stream&, int32_t type = o2::gpu::GPUMemoryResource::MEMORY_GPU);
170+
template <typename SlotPtr>
171+
SlotPtr* allocSlotArray(size_t n);
172+
template <typename T>
173+
void copyToDevice(T* dst, const T* src, size_t n);
174+
template <typename T>
175+
void copyFromDevice(T* dst, const T* src, size_t n);
176+
template <typename T, typename ArrayT>
177+
void publishSlot(ArrayT deviceArray, int slot, T* const& devicePtr, Stream&);
178+
template <typename T, size_t N, typename ArrayT>
179+
T* createSlot(std::array<T*, N>& slots, ArrayT deviceArray, int slot, size_t n, const char* what, SlotInit init = SlotInit::Raw, int32_t type = o2::gpu::GPUMemoryResource::MEMORY_GPU);
180+
template <typename T, size_t N, typename ArrayT, typename Container>
181+
void uploadSlot(std::array<T*, N>& slots, ArrayT deviceArray, int slot, const Container& host, const char* what);
182+
template <typename ArrayT, typename T, size_t N>
183+
void createPinnedSlotArray(ArrayT& deviceArray, std::array<T*, N>& slots, std::bitset<NLayers + 1>& pinned);
184+
template <typename Layers>
185+
void pinHostLayers(Layers& layers, std::bitset<NLayers + 1>& pinned, int maxLayers);
186+
template <typename Table>
187+
typename Table::View uploadNavigationTable(const Table& table, const typename Table::View& hostView);
183188

184189
// Host-available device buffer sizes
185190
std::array<int, MaxLinks> mNTracklets{};
186191
std::array<int, MaxCells> mNCells{};
187192
std::array<int, MaxCells> mNNeighbours{};
188193

189194
// Device pointers
190-
IndexTableUtilsN* mIndexTableUtilsDevice;
195+
IndexTableUtilsN* mIndexTableUtilsDevice{nullptr};
196+
float* mIterationParametersDevice{nullptr};
197+
const float* mLayerRadiiDevice{nullptr};
198+
const float* mMinPtsDevice{nullptr};
199+
const float* mLayerxX0Device{nullptr};
191200
// device navigation views
192201
ROFOverlapTableN::View mDeviceROFOverlapTableView;
193202
ROFVertexLookupTableN::View mDeviceROFVertexLookupTableView;
@@ -196,18 +205,17 @@ class TimeFrameGPU : public TimeFrame<NLayers>
196205
typename TrackingTopologyN::View mDeviceTrackingTopologyView;
197206

198207
// Hybrid pref
199-
Vertex* mPrimaryVerticesDevice;
200-
int* mROFramesPVDevice;
201-
std::array<Cluster*, NLayers> mClustersDevice;
202-
std::array<Cluster*, NLayers> mUnsortedClustersDevice;
203-
std::array<int*, NLayers> mClustersIndexTablesDevice;
204-
std::array<unsigned char*, NLayers> mUsedClustersDevice;
205-
std::array<int*, NLayers> mROFramesClustersDevice;
206-
const Cluster** mClustersDeviceArray;
207-
const Cluster** mUnsortedClustersDeviceArray;
208-
const int** mClustersIndexTablesDeviceArray;
209-
uint8_t** mUsedClustersDeviceArray;
210-
const int** mROFramesClustersDeviceArray;
208+
Vertex* mPrimaryVerticesDevice{nullptr};
209+
std::array<Cluster*, NLayers> mClustersDevice{};
210+
std::array<Cluster*, NLayers> mUnsortedClustersDevice{};
211+
std::array<int*, NLayers> mClustersIndexTablesDevice{};
212+
std::array<unsigned char*, NLayers> mUsedClustersDevice{};
213+
std::array<int*, NLayers> mROFramesClustersDevice{};
214+
const Cluster** mClustersDeviceArray{nullptr};
215+
const Cluster** mUnsortedClustersDeviceArray{nullptr};
216+
const int** mClustersIndexTablesDeviceArray{nullptr};
217+
uint8_t** mUsedClustersDeviceArray{nullptr};
218+
const int** mROFramesClustersDeviceArray{nullptr};
211219
std::array<Tracklet*, MaxLinks> mTrackletsDevice{};
212220
std::array<int*, MaxLinks> mTrackletsLUTDevice{};
213221
std::array<int*, MaxCells> mCellsLUTDevice{};
@@ -218,24 +226,20 @@ class TimeFrameGPU : public TimeFrame<NLayers>
218226
int** mNeighboursCellLUTDeviceArray{nullptr};
219227
int** mTrackletsLUTDeviceArray{nullptr};
220228
std::array<CellSeed*, MaxCells> mCellsDevice{};
221-
CellSeed** mCellsDeviceArray;
222-
std::array<int*, MaxCells> mNeighboursIndexTablesDevice{};
229+
CellSeed** mCellsDeviceArray{nullptr};
223230
TrackSeedN* mTrackSeedsDevice{nullptr};
224-
int* mTrackSeedsLUTDevice{nullptr};
231+
int* mTrackSeedIndicesDevice{nullptr}; ///< which seed each emitted track was fitted from
232+
int* mTrackCounterDevice{nullptr};
225233
unsigned int mNTracks{0};
226-
std::array<o2::track::TrackParCovF*, MaxCells> mCellSeedsDevice{};
227-
o2::track::TrackParCovF** mCellSeedsDeviceArray;
228-
std::array<float*, MaxCells> mCellSeedsChi2Device{};
229-
float** mCellSeedsChi2DeviceArray;
230234

231-
TrackITSExt* mTrackITSExtDevice;
235+
TrackITSExt* mTrackITSExtDevice{nullptr};
232236
int* mTrackIndicesDevice{nullptr};
233237
TrackExtensionHypothesis<NLayers>* mActiveTrackExtensionHypothesesDevice{nullptr};
234238
TrackExtensionHypothesis<NLayers>* mNextTrackExtensionHypothesesDevice{nullptr};
235239
std::array<CellNeighbour*, MaxCells> mNeighboursDevice{};
236240
CellNeighbour** mNeighboursDeviceArray{nullptr};
237-
std::array<TrackingFrameInfo*, NLayers> mTrackingFrameInfoDevice;
238-
const TrackingFrameInfo** mTrackingFrameInfoDeviceArray;
241+
std::array<TrackingFrameInfo*, NLayers> mTrackingFrameInfoDevice{};
242+
const TrackingFrameInfo** mTrackingFrameInfoDeviceArray{nullptr};
239243

240244
// State
241245
Streams mGpuStreams;

‎Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackerTraitsGPU.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ namespace o2::its
2222
template <int NLayers>
2323
class TrackerTraitsGPU final : public TrackerTraits<NLayers>
2424
{
25-
using typename TrackerTraits<NLayers>::IndexTableUtilsN;
26-
2725
public:
2826
TrackerTraitsGPU() = default;
2927
~TrackerTraitsGPU() final = default;
@@ -47,7 +45,6 @@ class TrackerTraitsGPU final : public TrackerTraits<NLayers>
4745
int getTFNumberOfCells() const override;
4846

4947
private:
50-
IndexTableUtilsN* mDeviceIndexTableUtils;
5148
gpu::TimeFrameGPU<NLayers>* mTimeFrameGPU;
5249
};
5350

0 commit comments

Comments
 (0)