Skip to content

Commit ca24036

Browse files
author
Mateusz Kopciński
committed
Implemented GlobalThreadPool and added setting xnnpack threadpool for iOS
1 parent 3504761 commit ca24036

File tree

21 files changed

+174
-518
lines changed

21 files changed

+174
-518
lines changed

‎.cspell-wordlist.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,9 @@ softmax
6666
logit
6767
logits
6868
probs
69+
threadpool
70+
chrono
71+
setpriority
72+
errno
73+
ifdef
74+
elif

‎apps/llm/android/.kotlin/errors/errors-1754380687676.log‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎apps/llm/app/llm/index.tsx‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -85,12 +85,6 @@ function LLMScreen() {
8585
</Text>
8686
</View>
8787
)}
88-
{/* <View style={styles.helloMessageContainer}>
89-
<Text style={styles.helloText}>Hello! 👋</Text>
90-
<Text style={styles.bottomHelloText}>
91-
What can I help you with?
92-
</Text>
93-
</View> */}
9488

9589
<View style={styles.bottomContainer}>
9690
<TextInput

‎apps/llm/ios/llm.xcodeproj/project.pbxproj‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,6 @@
180180
TargetAttributes = {
181181
13B07F861A680F5B00A75B9A = {
182182
LastSwiftMigration = 1250;
183-
ProvisioningStyle = Automatic;
184183
};
185184
};
186185
};
@@ -359,10 +358,7 @@
359358
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
360359
CLANG_ENABLE_MODULES = YES;
361360
CODE_SIGN_ENTITLEMENTS = llm/llm.entitlements;
362-
CODE_SIGN_IDENTITY = "Apple Development";
363-
CODE_SIGN_STYLE = Automatic;
364361
CURRENT_PROJECT_VERSION = 1;
365-
DEVELOPMENT_TEAM = J5FM626PE2;
366362
ENABLE_BITCODE = NO;
367363
GCC_PREPROCESSOR_DEFINITIONS = (
368364
"$(inherited)",
@@ -381,7 +377,7 @@
381377
"-lc++",
382378
);
383379
OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_DEBUG";
384-
PRODUCT_BUNDLE_IDENTIFIER = com.anonymous.llm2;
380+
PRODUCT_BUNDLE_IDENTIFIER = com.anonymous.llm;
385381
PRODUCT_NAME = llm;
386382
SWIFT_OBJC_BRIDGING_HEADER = "llm/llm-Bridging-Header.h";
387383
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
@@ -398,10 +394,7 @@
398394
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
399395
CLANG_ENABLE_MODULES = YES;
400396
CODE_SIGN_ENTITLEMENTS = llm/llm.entitlements;
401-
CODE_SIGN_IDENTITY = "Apple Development";
402-
CODE_SIGN_STYLE = Automatic;
403397
CURRENT_PROJECT_VERSION = 1;
404-
DEVELOPMENT_TEAM = J5FM626PE2;
405398
INFOPLIST_FILE = llm/Info.plist;
406399
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
407400
LD_RUNPATH_SEARCH_PATHS = (
@@ -415,7 +408,7 @@
415408
"-lc++",
416409
);
417410
OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_RELEASE";
418-
PRODUCT_BUNDLE_IDENTIFIER = com.anonymous.llm2;
411+
PRODUCT_BUNDLE_IDENTIFIER = com.anonymous.llm;
419412
PRODUCT_NAME = llm;
420413
SWIFT_OBJC_BRIDGING_HEADER = "llm/llm-Bridging-Header.h";
421414
SWIFT_VERSION = 5.0;

‎packages/react-native-executorch/android/src/main/cpp/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ set(RN_VERSION_LINK_LIBRARIES
3232
ReactAndroid::reactnative
3333
)
3434

35+
# Dependencies:
36+
3537
# ------- Executorch -------
3638

3739
add_library(executorch SHARED IMPORTED)

‎packages/react-native-executorch/common/rnexecutorch/Log.h‎

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -103,66 +103,66 @@ concept Fallback =
103103

104104
template <typename T>
105105
requires concepts::Streamable<T> && (!concepts::SmartPointer<T>)
106-
inline void printElement(std::ostream &os, const T &value);
106+
void printElement(std::ostream &os, const T &value);
107107

108108
template <typename T, typename U>
109-
inline void printElement(std::ostream &os, const std::pair<T, U> &p);
109+
void printElement(std::ostream &os, const std::pair<T, U> &p);
110110

111111
template <std::size_t N>
112-
inline void printElement(std::ostream &os, const char (&array)[N]);
112+
void printElement(std::ostream &os, const char (&array)[N]);
113113

114114
template <typename T, std::size_t N>
115-
inline void printElement(std::ostream &os, T (&array)[N]);
115+
void printElement(std::ostream &os, T (&array)[N]);
116116

117117
template <typename T>
118118
requires concepts::Iterable<T> && (!concepts::Streamable<T>)
119-
inline void printElement(std::ostream &os, const T &container);
119+
void printElement(std::ostream &os, const T &container);
120120

121121
template <typename T> void printSequencable(std::ostream &os, T &&container);
122122

123123
template <typename T>
124124
requires concepts::ReadOnlySequencable<T>
125-
inline void printElement(std::ostream &os, const T &container);
125+
void printElement(std::ostream &os, const T &container);
126126

127127
template <typename T>
128128
requires concepts::MutableSequencable<T>
129-
inline void printElement(std::ostream &os, T &&container);
129+
void printElement(std::ostream &os, T &&container);
130130

131131
template <typename... Args>
132-
inline void printElement(std::ostream &os, const std::tuple<Args...> &tpl);
132+
void printElement(std::ostream &os, const std::tuple<Args...> &tpl);
133133

134134
template <concepts::SmartPointer SP>
135-
inline void printElement(std::ostream &os, const SP &ptr);
135+
void printElement(std::ostream &os, const SP &ptr);
136136

137137
template <concepts::WeakPointer WP>
138-
inline void printElement(std::ostream &os, const WP &ptr);
138+
void printElement(std::ostream &os, const WP &ptr);
139139

140140
template <typename T>
141-
inline void printElement(std::ostream &os, const std::optional<T> &opt);
141+
void printElement(std::ostream &os, const std::optional<T> &opt);
142142

143143
template <typename... Ts>
144-
inline void printElement(std::ostream &os, const std::variant<Ts...> &var);
144+
void printElement(std::ostream &os, const std::variant<Ts...> &var);
145145

146-
inline void printElement(std::ostream &os, const std::exception_ptr &exPtr);
146+
void printElement(std::ostream &os, const std::exception_ptr &exPtr);
147147

148-
inline void printElement(std::ostream &os, const std::filesystem::path &path);
148+
void printElement(std::ostream &os, const std::filesystem::path &path);
149149

150-
inline void printElement(std::ostream &os,
151-
const std::filesystem::directory_iterator &dir_it);
150+
void printElement(std::ostream &os,
151+
const std::filesystem::directory_iterator &dir_it);
152152

153153
template <concepts::Fallback UnsupportedArg>
154-
inline void printElement(std::ostream &os, const UnsupportedArg &value);
154+
void printElement(std::ostream &os, const UnsupportedArg &value);
155155

156-
inline void printElement(std::ostream &os, const std::error_code &ec);
156+
void printElement(std::ostream &os, const std::error_code &ec);
157157

158158
template <typename T>
159159
requires concepts::Streamable<T> && (!concepts::SmartPointer<T>)
160-
inline void printElement(std::ostream &os, const T &value) {
160+
void printElement(std::ostream &os, const T &value) {
161161
os << value;
162162
}
163163

164164
template <typename T, typename U>
165-
inline void printElement(std::ostream &os, const std::pair<T, U> &p) {
165+
void printElement(std::ostream &os, const std::pair<T, U> &p) {
166166
os << "(";
167167
printElement(os, p.first);
168168
os << ", ";
@@ -171,7 +171,7 @@ inline void printElement(std::ostream &os, const std::pair<T, U> &p) {
171171
}
172172

173173
template <std::size_t N>
174-
inline void printElement(std::ostream &os, const char (&array)[N]) {
174+
void printElement(std::ostream &os, const char (&array)[N]) {
175175
// Treats the input as a string up to length N, drop null termination
176176
if (N > 1) {
177177
os << std::string_view(array, N - 1);
@@ -180,7 +180,7 @@ inline void printElement(std::ostream &os, const char (&array)[N]) {
180180

181181
// A special function for C-style arrays deducing size via template
182182
template <typename T, std::size_t N>
183-
inline void printElement(std::ostream &os, T (&array)[N]) {
183+
void printElement(std::ostream &os, T (&array)[N]) {
184184
os << "[";
185185
for (std::size_t i = 0; i < N; ++i) {
186186
if (i > 0) {
@@ -193,7 +193,7 @@ inline void printElement(std::ostream &os, T (&array)[N]) {
193193

194194
template <typename T>
195195
requires concepts::Iterable<T> && (!concepts::Streamable<T>)
196-
inline void printElement(std::ostream &os, const T &container) {
196+
void printElement(std::ostream &os, const T &container) {
197197
os << "[";
198198
auto it = std::begin(container);
199199
if (it != std::end(container)) {
@@ -206,8 +206,7 @@ inline void printElement(std::ostream &os, const T &container) {
206206
os << "]";
207207
}
208208

209-
template <typename T>
210-
inline void printSequencable(std::ostream &os, T &&container) {
209+
template <typename T> void printSequencable(std::ostream &os, T &&container) {
211210
os << "[";
212211
bool isFirst = true;
213212

@@ -234,7 +233,7 @@ inline void printSequencable(std::ostream &os, T &&container) {
234233

235234
template <typename T>
236235
requires concepts::ReadOnlySequencable<T>
237-
inline void printElement(std::ostream &os, const T &container) {
236+
void printElement(std::ostream &os, const T &container) {
238237
T tempContainer = container; // Make a copy to preserve original container
239238
printSequencable(
240239
os, std::move(tempContainer)); // Use std::move since tempContainer won't
@@ -243,12 +242,12 @@ inline void printElement(std::ostream &os, const T &container) {
243242

244243
template <typename T>
245244
requires concepts::MutableSequencable<T>
246-
inline void printElement(std::ostream &os, T &&container) {
245+
void printElement(std::ostream &os, T &&container) {
247246
printSequencable(os, std::forward<T>(container));
248247
}
249248

250249
template <typename... Args>
251-
inline void printElement(std::ostream &os, const std::tuple<Args...> &tpl) {
250+
void printElement(std::ostream &os, const std::tuple<Args...> &tpl) {
252251
os << "<";
253252
std::apply(
254253
[&os](const auto &...args) {
@@ -270,7 +269,7 @@ inline void printElement(std::ostream &os, const std::tuple<Args...> &tpl) {
270269
}
271270

272271
template <concepts::SmartPointer SP>
273-
inline void printElement(std::ostream &os, const SP &ptr) {
272+
void printElement(std::ostream &os, const SP &ptr) {
274273
if (ptr) {
275274
printElement(os, *ptr);
276275
} else {
@@ -279,7 +278,7 @@ inline void printElement(std::ostream &os, const SP &ptr) {
279278
}
280279

281280
template <concepts::WeakPointer WP>
282-
inline void printElement(std::ostream &os, const WP &ptr) {
281+
void printElement(std::ostream &os, const WP &ptr) {
283282
auto sp = ptr.lock();
284283
if (sp) {
285284
printElement(os, *sp);
@@ -289,7 +288,7 @@ inline void printElement(std::ostream &os, const WP &ptr) {
289288
}
290289

291290
template <typename T>
292-
inline void printElement(std::ostream &os, const std::optional<T> &opt) {
291+
void printElement(std::ostream &os, const std::optional<T> &opt) {
293292
if (opt) {
294293
os << "Optional(";
295294
printElement(os, *opt);
@@ -300,7 +299,7 @@ inline void printElement(std::ostream &os, const std::optional<T> &opt) {
300299
}
301300

302301
template <typename... Ts>
303-
inline void printElement(std::ostream &os, const std::variant<Ts...> &var) {
302+
void printElement(std::ostream &os, const std::variant<Ts...> &var) {
304303
std::visit(
305304
[&os](const auto &value) {
306305
os << "Variant(";
@@ -349,7 +348,7 @@ printElement(std::ostream &os,
349348

350349
// Fallback
351350
template <concepts::Fallback UnsupportedArg>
352-
inline void printElement(std::ostream &os, const UnsupportedArg &value) {
351+
void printElement(std::ostream &os, const UnsupportedArg &value) {
353352
const auto *typeName = typeid(UnsupportedArg).name();
354353
throw std::runtime_error(
355354
"Type "s + std::string(typeName) +

‎packages/react-native-executorch/common/rnexecutorch/RnExecutorchInstaller.cpp‎

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#include "RnExecutorchInstaller.h"
22

3-
#include <rnexecutorch/GlobalThreadPool.h>
43
#include <rnexecutorch/Log.h>
54
#include <rnexecutorch/TokenizerModule.h>
65
#include <rnexecutorch/host_objects/JsiConversions.h>
@@ -14,12 +13,8 @@
1413
#include <rnexecutorch/models/speech_to_text/SpeechToText.h>
1514
#include <rnexecutorch/models/style_transfer/StyleTransfer.h>
1615
#include <rnexecutorch/models/vertical_ocr/VerticalOCR.h>
17-
18-
#if defined(__ANDROID__) && defined(__aarch64__)
19-
#include <executorch/extension/threadpool/cpuinfo_utils.h>
20-
#include <executorch/extension/threadpool/threadpool.h>
21-
#include <rnexecutorch/Log.h>
22-
#endif
16+
#include <rnexecutorch/threads/GlobalThreadPool.h>
17+
#include <rnexecutorch/threads/ThreadUtils.h>
2318

2419
namespace rnexecutorch {
2520

@@ -94,24 +89,8 @@ void RnExecutorchInstaller::injectJSIBindings(
9489
RnExecutorchInstaller::loadModel<models::speech_to_text::SpeechToText>(
9590
jsiRuntime, jsCallInvoker, "loadSpeechToText"));
9691

97-
#if defined(__ANDROID__) && defined(__aarch64__)
98-
auto num_of_perf_cores =
99-
::executorch::extension::cpuinfo::get_num_performant_cores();
100-
log(LOG_LEVEL::Info, "Detected ", num_of_perf_cores, " performant cores");
101-
// setting num_of_cores to floor(num_of_perf_cores / 2) + 1) because depending
102-
// on cpu arch as when possible we want to leave at least 2 performant cores
103-
// for other tasks (setting more actually results in drop of performance). For
104-
// older devices (i.e. samsung s22) resolves to 3 cores, and for newer ones
105-
// (like OnePlus 12) resolves to 4, which when benchamrked gives highest
106-
// throughput.
107-
auto num_of_cores = static_cast<uint32_t>(num_of_perf_cores / 2) + 1;
108-
::executorch::extension::threadpool::get_threadpool()
109-
->_unsafe_reset_threadpool(num_of_cores);
110-
log(LOG_LEVEL::Info, "Configuring xnnpack for ", num_of_cores, " threads");
111-
#endif
112-
113-
ThreadConfig config;
114-
GlobalThreadPool::initialize(2, config);
92+
threads::ThreadUtils::unsafeSetupThreadPool();
93+
threads::GlobalThreadPool::initialize();
11594
}
11695

11796
} // namespace rnexecutorch

0 commit comments

Comments
 (0)