From bdbf3d8671f4a938068741ee359aeb0dea09445d Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Fri, 21 Aug 2026 18:18:06 +0800 Subject: [PATCH 1/2] Add Windows ARM64 wheel support --- .github/workflows/ci.yml | 18 +++++++++++++++++- CMakeLists.txt | 2 +- docs/installation.md | 8 ++++---- .../tools/prepare_build_environment_windows.sh | 16 +++++++++++++++- src/cpu/vec_neon.h | 2 +- third_party/ruy | 2 +- 6 files changed, 39 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1611e2e79..bc0e08928 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,6 +189,9 @@ jobs: include: - os: ubuntu-22.04-arm arch: aarch64 + - os: windows-11-arm + arch: ARM64 + build: "cp31[2-4]-*" - os: macos-15 arch: arm64 - os: macos-15-intel @@ -200,7 +203,7 @@ jobs: submodules: recursive - name: Cache MKL and oneDNN (Windows) - if: runner.os == 'Windows' + if: runner.os == 'Windows' && matrix.arch != 'ARM64' uses: actions/cache@v4 with: path: | @@ -224,6 +227,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 CIBW_ARCHS: ${{ matrix.arch }} + CIBW_BUILD: ${{ matrix.build || '*' }} CIBW_SKIP: "*-musllinux_*" - name: Upload Python wheels @@ -295,6 +299,11 @@ jobs: artifact_pattern: python-wheels-Windows-auto64 wheel_pattern: "*cp312*win*amd64.whl" + - os: windows-11-arm + artifact_pattern: python-wheels-Windows-ARM64 + wheel_pattern: "*cp312*win*arm64.whl" + python_arch: arm64 + - os: macos-15 artifact_pattern: python-wheels-macOS-arm64 wheel_pattern: "*cp312*macosx*arm64.whl" @@ -304,6 +313,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + architecture: ${{ matrix.python_arch }} - uses: actions/checkout@v4 @@ -328,6 +338,12 @@ jobs: run: | pytest -v python/tests/ --ignore=python/tests/test_fairseq.py --ignore=python/tests/test_opennmt_py.py --ignore=python/tests/test_opennmt_tf.py + - name: Test multithreaded INT8 inference (Windows ARM64) + if: runner.os == 'Windows' && runner.arch == 'ARM64' + shell: pwsh + run: | + python -c "import ctranslate2; model = 'tests/data/models/transliteration-aren-all'; translator = ctranslate2.Translator(model, device='cpu', compute_type='int8', inter_threads=1, intra_threads=4); result = translator.translate_batch([['آ', 'ت', 'ز', 'م', 'و', 'ن']], beam_size=1); assert result[0].hypotheses[0] == ['a', 't', 'z', 'm', 'o', 'n']; del result, translator" + test-legacy-converters: needs: [build-python-wheels] diff --git a/CMakeLists.txt b/CMakeLists.txt index 027d09420..783f138cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -272,7 +272,7 @@ macro(ct2_compile_kernels_for_isa isa flag) list(APPEND SOURCES ${CMAKE_CURRENT_BINARY_DIR}/kernels_${isa}.cc) endmacro() -if(CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(aarch64)" +if(CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(aarch64)|(ARM64)" OR (APPLE AND CMAKE_OSX_ARCHITECTURES STREQUAL "arm64")) add_definitions(-DCT2_ARM64_BUILD) set(CT2_BUILD_ARCH "arm64") diff --git a/docs/installation.md b/docs/installation.md index 0fc4263b7..530ee24b7 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -10,18 +10,18 @@ pip install ctranslate2 The Python wheels have the following requirements: -* OS: Linux (x86-64, AArch64), macOS (x86-64, ARM64), Windows (x86-64) -* Python version: >= 3.9 +* OS: Linux (x86-64, AArch64), macOS (x86-64, ARM64), Windows (x86-64, ARM64) +* Python version: >= 3.9 (3.12–3.14 for Windows ARM64) * pip version: >= 19.3 to support `manylinux2014` wheels ```{admonition} GPU support -The Linux and Windows Python wheels support GPU execution. Install [CUDA](https://developer.nvidia.com/cuda-toolkit) 12.x to use the GPU. +The Linux and Windows x86-64 Python wheels support GPU execution. Windows ARM64 wheels are CPU-only. Install [CUDA](https://developer.nvidia.com/cuda-toolkit) 12.x to use the GPU. If you plan to run models with convolutional layers (e.g. for speech recognition), you should also install [cuDNN 8](https://developer.nvidia.com/cudnn) for CUDA 12.x. ``` ```{note} -On Windows [the Visual C++ runtime](https://www.microsoft.com/en-US/download/details.aspx?id=48145) is required. It is installed in most of the systems, but if is not the case, you have to download it and install it. +On Windows [the latest supported Visual C++ runtime](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist) is required. It is installed in most systems, but if this is not the case, you have to download and install it. ``` ## Install with Docker diff --git a/python/tools/prepare_build_environment_windows.sh b/python/tools/prepare_build_environment_windows.sh index 6160d7d08..f7c17a979 100755 --- a/python/tools/prepare_build_environment_windows.sh +++ b/python/tools/prepare_build_environment_windows.sh @@ -3,6 +3,21 @@ set -e set -x +NPROC=$(nproc) + +if [ "$CIBW_ARCHS" = "ARM64" ]; then + mkdir build + cd build + cmake -A ARM64 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$CTRANSLATE2_ROOT" -DBUILD_CLI=OFF -DWITH_MKL=OFF -DWITH_DNNL=OFF -DWITH_RUY=ON -DWITH_CUDA=OFF -DOPENMP_RUNTIME=COMP .. + cmake --build . --config Release --target install --parallel "$NPROC" --verbose + cd .. + rm -r build + + cp README.md python/ + cp "$CTRANSLATE2_ROOT/bin/ctranslate2.dll" python/ctranslate2/ + exit 0 +fi + CUDA_ROOT="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.8" curl --netrc-optional -L -nv -o cuda.exe https://developer.download.nvidia.com/compute/cuda/12.8.1/local_installers/cuda_12.8.1_572.61_windows.exe ./cuda.exe -s nvcc_12.8 cudart_12.8 cublas_dev_12.8 curand_dev_12.8 @@ -38,7 +53,6 @@ if [ ! -d "C:/Program Files (x86)/Intel/oneAPI" ]; then ./webimage_extracted/bootstrapper.exe -s --action install --components="intel.oneapi.win.mkl.devel" --eula=accept -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 --log-dir=. fi -NPROC=$(nproc) ONEDNN_VERSION=3.1.1 if [ ! -d "C:/Program Files (x86)/oneDNN" ]; then curl --netrc-optional -L -O https://github.com/oneapi-src/oneDNN/archive/refs/tags/v${ONEDNN_VERSION}.tar.gz diff --git a/src/cpu/vec_neon.h b/src/cpu/vec_neon.h index 4a407a5f1..49be7ade3 100644 --- a/src/cpu/vec_neon.h +++ b/src/cpu/vec_neon.h @@ -160,7 +160,7 @@ namespace ctranslate2 { } static inline value_type round(value_type v) { -#ifdef __aarch64__ +#if defined(__aarch64__) || defined(_M_ARM64) return vrndiq_f32(v); #else float temp[4] = {std::nearbyintf(v[0]), std::nearbyintf(v[1]), std::nearbyintf(v[2]), std::nearbyintf(v[3])}; diff --git a/third_party/ruy b/third_party/ruy index 363f25228..154138112 160000 --- a/third_party/ruy +++ b/third_party/ruy @@ -1 +1 @@ -Subproject commit 363f252289fb7a1fba1703d99196524698cb884d +Subproject commit 1541381128c2d051489f20b4d7e057ab711c33af From cdc4fc5cc9356dddd02b715d09f04e5f6aadf459 Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Fri, 21 Aug 2026 19:02:49 +0800 Subject: [PATCH 2/2] Fix Ruy context shutdown on worker threads --- .github/workflows/ci.yml | 26 +++++++++++++++++++++++++- src/cpu/backend.cc | 19 +++++++++++++++++-- src/cpu/backend.h | 1 + src/devices.cc | 6 ++++++ 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc0e08928..4c26d6048 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -342,7 +342,31 @@ jobs: if: runner.os == 'Windows' && runner.arch == 'ARM64' shell: pwsh run: | - python -c "import ctranslate2; model = 'tests/data/models/transliteration-aren-all'; translator = ctranslate2.Translator(model, device='cpu', compute_type='int8', inter_threads=1, intra_threads=4); result = translator.translate_batch([['آ', 'ت', 'ز', 'م', 'و', 'ن']], beam_size=1); assert result[0].hypotheses[0] == ['a', 't', 'z', 'm', 'o', 'n']; del result, translator" + python -c "import ctranslate2; model = 'tests/data/models/v2/aren-transliteration-i8'; translator = ctranslate2.Translator(model, device='cpu', compute_type='int8', inter_threads=1, intra_threads=4); result = translator.translate_batch([['آ', 'ت', 'ز', 'م', 'و', 'ن']], beam_size=1); assert result[0].hypotheses[0] == ['a', 't', 'z', 'm', 'o', 'n']; del result, translator" + + - name: Test multithreaded Ruy context shutdown (Windows ARM64) + if: runner.os == 'Windows' && runner.arch == 'ARM64' + timeout-minutes: 2 + shell: pwsh + run: | + @' + import ctranslate2 + + model = "tests/data/models/v2/aren-transliteration" + for _ in range(3): + translator = ctranslate2.Translator( + model, + device="cpu", + compute_type="float32", + inter_threads=1, + intra_threads=4, + ) + result = translator.translate_batch( + [["آ", "ت", "ز", "م", "و", "ن"]], beam_size=1 + ) + assert result[0].hypotheses[0] == ["a", "t", "z", "m", "o", "n"] + del result, translator + '@ | python - test-legacy-converters: diff --git a/src/cpu/backend.cc b/src/cpu/backend.cc index 02e818f8a..e25e35c57 100644 --- a/src/cpu/backend.cc +++ b/src/cpu/backend.cc @@ -1,5 +1,7 @@ #include "backend.h" +#include + #include "ctranslate2/utils.h" #include "cpu_info.h" #include "env.h" @@ -106,9 +108,22 @@ namespace ctranslate2 { } #ifdef CT2_WITH_RUY + static std::unique_ptr& get_thread_local_ruy_context() { + static thread_local std::unique_ptr context; + return context; + } + ruy::Context *get_ruy_context() { - static thread_local ruy::Context context; - return &context; + auto& context = get_thread_local_ruy_context(); + if (!context) + context = std::make_unique(); + return context.get(); + } + + void destroy_ruy_context() { + // The Ruy context owns worker threads. Destroy it before this thread + // enters TLS cleanup, where joining these workers can deadlock on Windows. + get_thread_local_ruy_context().reset(); } #endif } diff --git a/src/cpu/backend.h b/src/cpu/backend.h index 9c39769ba..057484e68 100644 --- a/src/cpu/backend.h +++ b/src/cpu/backend.h @@ -28,6 +28,7 @@ namespace ctranslate2 { bool pack_gemm_weights(ComputeType compute_type); #ifdef CT2_WITH_RUY ruy::Context *get_ruy_context(); + void destroy_ruy_context(); #endif } diff --git a/src/devices.cc b/src/devices.cc index 6bb615ea5..e6854e86f 100644 --- a/src/devices.cc +++ b/src/devices.cc @@ -8,6 +8,7 @@ # include #endif +#include "cpu/backend.h" #include "device_dispatch.h" namespace ctranslate2 { @@ -121,6 +122,11 @@ namespace ctranslate2 { } void destroy_context(Device device) { +#ifdef CT2_WITH_RUY + if (device == Device::CPU) { + cpu::destroy_ruy_context(); + } +#endif #ifdef CT2_WITH_CUDA if (device == Device::CUDA) { cuda::free_curand_states();