feat: Implemented GlobalThreadPool for async functions (#603)

Added singleton class GlobalThreadPool for single threadpool management
so that we don't have to spawn new threads for each async function and
instead we can delegate functions to the threadpool.

Also added pthreadpool and cpuinfo binaries for iOS to allow for XNNPack
threadpool configuration just like on Android

- [ ] Yes
- [x] No

- [ ] Bug fix (change which fixes an issue)
- [x] New feature (change which adds functionality)
- [ ] Documentation update (improves or adds clarity to existing
documentation)
- [ ] Other (chores, tests, code style improvements etc.)

- [x] iOS
- [x] Android

<!-- Provide step-by-step instructions on how to test your changes.
Include setup details if necessary. -->

<!-- Add screenshots here, if applicable -->

<!-- Link related issues here using #issue-number -->

- [ ] I have performed a self-review of my code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have updated the documentation accordingly
- [ ] My changes generate no new warnings

<!-- Include any additional information, assumptions, or context that
reviewers might need to understand this PR. -->

---------

Co-authored-by: Mateusz Kopciński <[email protected]>

Author: 2 people

.cspell-wordlist.txt

@@ -62,3 +62,17 @@ softmax
 logit
 logits
 probs
+unet
+Unet
+VPRED
+timesteps
+Timesteps
+denoises
+denoise
+denoising
+threadpool
+chrono
+setpriority
+errno
+ifdef
+elif

apps/llm/app.json

@@ -58,7 +58,11 @@
         "foregroundImage": "./assets/icons/adaptive-icon.png",
         "backgroundColor": "#ffffff"
       },
-      "package": "com.anonymous.llm"
+      "package": "com.anonymous.llm",
+      "permissions": [
+        "android.permission.READ_CALENDAR",
+        "android.permission.WRITE_CALENDAR"
+      ]
     },
     "web": {
       "favicon": "./assets/icons/favicon.png"

packages/react-native-executorch/common/rnexecutorch/RnExecutorchInstaller.cpp

@@ -12,12 +12,8 @@
 #include <rnexecutorch/models/speech_to_text/SpeechToText.h>
 #include <rnexecutorch/models/style_transfer/StyleTransfer.h>
 #include <rnexecutorch/models/vertical_ocr/VerticalOCR.h>
-
-#if defined(__ANDROID__) && defined(__aarch64__)
-#include <executorch/extension/threadpool/cpuinfo_utils.h>
-#include <executorch/extension/threadpool/threadpool.h>
-#include <rnexecutorch/Log.h>
-#endif
+#include <rnexecutorch/threads/GlobalThreadPool.h>
+#include <rnexecutorch/threads/utils/ThreadUtils.h>
 
 namespace rnexecutorch {
 
@@ -92,21 +88,8 @@ void RnExecutorchInstaller::injectJSIBindings(
       RnExecutorchInstaller::loadModel<models::speech_to_text::SpeechToText>(
           jsiRuntime, jsCallInvoker, "loadSpeechToText"));
 
-#if defined(__ANDROID__) && defined(__aarch64__)
-  auto num_of_perf_cores =
-      ::executorch::extension::cpuinfo::get_num_performant_cores();
-  log(LOG_LEVEL::Info, "Detected ", num_of_perf_cores, " performant cores");
-  // setting num_of_cores to floor(num_of_perf_cores / 2) + 1) because depending
-  // on cpu arch as when possible we want to leave at least 2 performant cores
-  // for other tasks (setting more actually results in drop of performance). For
-  // older devices (i.e. samsung s22) resolves to 3 cores, and for newer ones
-  // (like OnePlus 12) resolves to 4, which when benchamrked gives highest
-  // throughput.
-  auto num_of_cores = static_cast<uint32_t>(num_of_perf_cores / 2) + 1;
-  ::executorch::extension::threadpool::get_threadpool()
-      ->_unsafe_reset_threadpool(num_of_cores);
-  log(LOG_LEVEL::Info, "Configuring xnnpack for ", num_of_cores, " threads");
-#endif
+  threads::utils::unsafeSetupThreadPool();
+  threads::GlobalThreadPool::initialize();
 }
 
 } // namespace rnexecutorch

packages/react-native-executorch/common/rnexecutorch/host_objects/ModelHostObject.h

@@ -19,6 +19,7 @@
 #include <rnexecutorch/models/llm/LLM.h>
 #include <rnexecutorch/models/ocr/OCR.h>
 #include <rnexecutorch/models/vertical_ocr/VerticalOCR.h>
+#include <rnexecutorch/threads/GlobalThreadPool.h>
 
 namespace rnexecutorch {
 
@@ -195,58 +196,60 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
             // We need to dispatch a thread if we want the function to be
             // asynchronous. In this thread all accesses to jsi::Runtime need to
             // be done via the callInvoker.
-            std::thread([this, promise,
-                         argsConverted = std::move(argsConverted)]() {
-              try {
-                if constexpr (std::is_void_v<decltype(std::apply(
-                                  std::bind_front(FnPtr, model),
-                                  argsConverted))>) {
-                  // For void functions, just call the function and resolve with
-                  // undefined
-                  std::apply(std::bind_front(FnPtr, model),
-                             std::move(argsConverted));
-                  callInvoker->invokeAsync([promise](jsi::Runtime &runtime) {
-                    promise->resolve(jsi::Value::undefined());
-                  });
-                } else {
-                  // For non-void functions, capture the result and convert it
-                  auto result = std::apply(std::bind_front(FnPtr, model),
-                                           std::move(argsConverted));
-                  // The result is copied. It should either be quickly copiable,
-                  // or passed with a shared_ptr.
-                  callInvoker->invokeAsync(
-                      [promise, result](jsi::Runtime &runtime) {
-                        promise->resolve(jsi_conversion::getJsiValue(
-                            std::move(result), runtime));
-                      });
-                }
-              } catch (const std::runtime_error &e) {
-                // This catch should be merged with the next two
-                // (std::runtime_error and jsi::JSError inherits from
-                // std::exception) HOWEVER react native has broken RTTI which
-                // breaks proper exception type checking. Remove when the
-                // following change is present in our version:
-                // https://github.com/facebook/react-native/commit/3132cc88dd46f95898a756456bebeeb6c248f20e
-                callInvoker->invokeAsync([e = std::move(e), promise]() {
-                  promise->reject(e.what());
+            threads::GlobalThreadPool::detach(
+                [this, promise, argsConverted = std::move(argsConverted)]() {
+                  try {
+                    if constexpr (std::is_void_v<decltype(std::apply(
+                                      std::bind_front(FnPtr, model),
+                                      argsConverted))>) {
+                      // For void functions, just call the function and resolve
+                      // with undefined
+                      std::apply(std::bind_front(FnPtr, model),
+                                 std::move(argsConverted));
+                      callInvoker->invokeAsync(
+                          [promise](jsi::Runtime &runtime) {
+                            promise->resolve(jsi::Value::undefined());
+                          });
+                    } else {
+                      // For non-void functions, capture the result and convert
+                      // it
+                      auto result = std::apply(std::bind_front(FnPtr, model),
+                                               std::move(argsConverted));
+                      // The result is copied. It should either be quickly
+                      // copiable, or passed with a shared_ptr.
+                      callInvoker->invokeAsync(
+                          [promise, result](jsi::Runtime &runtime) {
+                            promise->resolve(jsi_conversion::getJsiValue(
+                                std::move(result), runtime));
+                          });
+                    }
+                  } catch (const std::runtime_error &e) {
+                    // This catch should be merged with the next two
+                    // (std::runtime_error and jsi::JSError inherits from
+                    // std::exception) HOWEVER react native has broken RTTI
+                    // which breaks proper exception type checking. Remove when
+                    // the following change is present in our version:
+                    // https://github.com/facebook/react-native/commit/3132cc88dd46f95898a756456bebeeb6c248f20e
+                    callInvoker->invokeAsync([e = std::move(e), promise]() {
+                      promise->reject(e.what());
+                    });
+                    return;
+                  } catch (const jsi::JSError &e) {
+                    callInvoker->invokeAsync([e = std::move(e), promise]() {
+                      promise->reject(e.what());
+                    });
+                    return;
+                  } catch (const std::exception &e) {
+                    callInvoker->invokeAsync([e = std::move(e), promise]() {
+                      promise->reject(e.what());
+                    });
+                    return;
+                  } catch (...) {
+                    callInvoker->invokeAsync(
+                        [promise]() { promise->reject("Unknown error"); });
+                    return;
+                  }
                 });
-                return;
-              } catch (const jsi::JSError &e) {
-                callInvoker->invokeAsync([e = std::move(e), promise]() {
-                  promise->reject(e.what());
-                });
-                return;
-              } catch (const std::exception &e) {
-                callInvoker->invokeAsync([e = std::move(e), promise]() {
-                  promise->reject(e.what());
-                });
-                return;
-              } catch (...) {
-                callInvoker->invokeAsync(
-                    [promise]() { promise->reject("Unknown error"); });
-                return;
-              }
-            }).detach();
           } catch (...) {
             promise->reject("Couldn't parse JS arguments in a native function");
           }

packages/react-native-executorch/common/rnexecutorch/threads/GlobalThreadPool.h

@@ -0,0 +1,79 @@
+// GlobalThreadPool.h
+#pragma once
+
+#include <executorch/extension/threadpool/cpuinfo_utils.h>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <rnexecutorch/Log.h>
+#include <rnexecutorch/threads/HighPerformanceThreadPool.h>
+
+namespace rnexecutorch::threads {
+
+class GlobalThreadPool {
+public:
+  GlobalThreadPool() = delete;
+  GlobalThreadPool(const GlobalThreadPool &) = delete;
+  GlobalThreadPool &operator=(const GlobalThreadPool &) = delete;
+  GlobalThreadPool(GlobalThreadPool &&) = delete;
+  GlobalThreadPool &operator=(GlobalThreadPool &&) = delete;
+
+  static HighPerformanceThreadPool &get() {
+    if (!instance) {
+      initialize();
+    }
+    return *instance;
+  }
+
+  static void initialize(std::optional<uint32_t> numThreads = std::nullopt,
+                         ThreadConfig config = {}) {
+    std::call_once(initFlag, [&numThreads, config]() {
+      if (!numThreads) {
+        numThreads =
+            ::executorch::extension::cpuinfo::get_num_performant_cores();
+      }
+
+      log(rnexecutorch::LOG_LEVEL::Info, "Initializing global thread pool with",
+          numThreads, "threads");
+      instance = std::make_unique<HighPerformanceThreadPool>(numThreads.value(),
+                                                             config);
+    });
+  }
+
+  // Convenience methods that mirror std::thread interface
+  template <typename Func, typename... Args>
+  static auto async(Func &&func, Args &&...args) {
+    return get().submit(std::forward<Func>(func), std::forward<Args>(args)...);
+  }
+
+  template <typename Func, typename... Args>
+  static auto async_high_priority(Func &&func, Args &&...args) {
+    return get().submitWithPriority(Priority::HIGH, std::forward<Func>(func),
+                                    std::forward<Args>(args)...);
+  }
+
+  // Fire and forget (like std::thread{}.detach())
+  template <typename Func, typename... Args>
+  static void detach(Func &&func, Args &&...args) {
+    get().submitDetached(std::forward<Func>(func), std::forward<Args>(args)...);
+  }
+
+  // Execute and wait (like std::thread{}.join())
+  template <typename Func, typename... Args>
+  static auto execute(Func &&func, Args &&...args) {
+    return get().execute(std::forward<Func>(func), std::forward<Args>(args)...);
+  }
+
+  static void shutdown() {
+    if (instance) {
+      instance->shutdown();
+      instance.reset();
+    }
+  }
+
+private:
+  inline static std::unique_ptr<HighPerformanceThreadPool> instance;
+  inline static std::once_flag initFlag;
+};
+
+} // namespace rnexecutorch::threads