diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 8649209cf706..a1dfc4c7085a 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -11,6 +11,8 @@ # CUDA / ROCm / SYCL / Vulkan variants). # - macOS -> the `includeDarwin:` matrix (Apple Silicon / arm64; Metal where # the engine supports it, otherwise a native arm64 CPU build). +# - Windows -> the `includeWindows:` matrix (x86_64 / amd64; native builds +# under MSYS2, no WSL/Docker — see backend_build_windows.yml). # # New backends must target EVERY OS they can build for, not just Linux. A backend # listed only under `include:` is silently unavailable on macOS even when its code @@ -24,6 +26,12 @@ # `metal:` capability + `metal-` image entries, a `run.sh` Darwin/DYLD # branch for C/C++ backends, and the inferBackendPathDarwin case in # scripts/lib/backend-filter.mjs so the path filter actually builds it). +# +# Windows builds are bespoke for now: every entry builds via a per-backend make +# target + MSYS2 build script (see scripts/build/llama-cpp-windows.sh), the +# index.yaml `windows:` capability + `windows-` image entries, a +# run-windows launcher next to run.sh, and the inferBackendPathWindows case in +# scripts/lib/backend-filter.mjs. # Linux matrix (consumed by backend-jobs). include: @@ -6799,3 +6807,11 @@ includeDarwin: - backend: "ds4" tag-suffix: "-metal-darwin-arm64-ds4" lang: "go" + +# Windows matrix (consumed by backend-jobs-windows). +# Native windows/amd64 builds under MSYS2 — no WSL, no Docker. Each entry builds +# via a bespoke make target + scripts/build/-windows.sh. +includeWindows: + - backend: "llama-cpp" + tag-suffix: "-windows-amd64-llama-cpp" + lang: "go" diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index d4bb0bf59384..01335de680f3 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -34,9 +34,11 @@ jobs: outputs: matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }} matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }} + matrix-windows: ${{ steps.set-matrix.outputs['matrix-windows'] }} merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }} has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }} has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }} + has-backends-windows: ${{ steps.set-matrix.outputs['has-backends-windows'] }} has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }} # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js). @@ -368,3 +370,23 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-darwin) }} + + backend-jobs-windows: + needs: generate-matrix + if: needs.generate-matrix.outputs.has-backends-windows == 'true' + uses: ./.github/workflows/backend_build_windows.yml + with: + backend: ${{ matrix.backend }} + build-type: ${{ matrix.build-type }} + go-version: "1.27.x" + tag-suffix: ${{ matrix.tag-suffix }} + lang: ${{ matrix.lang || 'go' }} + runs-on: "windows-latest" + secrets: + dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }} + dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }} + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-windows) }} diff --git a/.github/workflows/backend_build_windows.yml b/.github/workflows/backend_build_windows.yml new file mode 100644 index 000000000000..1e77b3aa1e8e --- /dev/null +++ b/.github/workflows/backend_build_windows.yml @@ -0,0 +1,291 @@ +--- +name: 'build windows backend container images (reusable)' + +on: + workflow_call: + inputs: + backend: + description: 'Backend to build' + required: true + type: string + build-type: + description: 'Build type' + default: '' + type: string + lang: + description: 'Programming language (e.g. go)' + default: 'python' + type: string + go-version: + description: 'Go version to use' + default: '1.27.x' + type: string + tag-suffix: + description: 'Tag suffix for the built image' + required: true + type: string + runs-on: + description: 'Runner to use' + default: 'windows-latest' + type: string + secrets: + dockerUsername: + required: false + dockerPassword: + required: false + quayUsername: + required: true + quayPassword: + required: true + +jobs: + windows-backend-build: + runs-on: ${{ inputs.runs-on }} + strategy: + matrix: + go-version: ['${{ inputs.go-version }}'] + env: + # Every CMake variant below (gRPC + the three llama.cpp variants) compiles + # the same source trees with overlapping flags; ccache dedupes them. + # CCACHE_DIR is set in a run step (below), not here: a job-level env value + # is used verbatim, and MSYS2's $HOME must be expanded at runtime so it + # matches the ~/.cache/ccache path handed to actions/cache. + CMAKE_ARGS: "-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + + - name: Setup Go ${{ matrix.go-version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + # Caches ~/go/pkg/mod and %LOCALAPPDATA%\go-build keyed on go.sum. + cache: true + + - name: Display Go version + run: go version + + # ---- MSYS2 toolchain ---- + # Native windows builds need a mingw gcc toolchain; windows-latest ships + # none by default (build-test-windows works only because LocalAI itself + # builds with CGO_ENABLED=0). UCRT64 matches the /ucrt64/bin DLL bundling + # in scripts/build/llama-cpp-windows.sh. `perl` is required by gRPC's + # third_party/openssl build (openssl's Configure is a perl script). + # `install` is a folded scalar (>-): every line folds into the pacman + # command, so no `#` comments may live inside it - they would become + # literal package names ("target not found: Vulkan:"). Vulkan packages: + # headers + loader (ucrt64) for FindVulkan, and the mingw64 shaderc + # build for glslc (no ucrt64 shaderc exists; glslc is a standalone tool + # so the prefix split is irrelevant). The loader's vulkan-1.dll is + # bundled next to the image's DLLs; ggml-vulkan loads it dynamically, so + # it is never a hard import. spirv-headers provides the cmake config + # ggml-vulkan requires (find_package(SPIRV-Headers CONFIG REQUIRED)). + - name: Set up MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + git + make + cmake + mingw-w64-ucrt-x86_64-cmake + ninja + pkg-config + perl + patch + unzip + curl + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-gcc-libs + mingw-w64-ucrt-x86_64-binutils + mingw-w64-ucrt-x86_64-ccache + mingw-w64-ucrt-x86_64-vulkan-headers + mingw-w64-ucrt-x86_64-vulkan-loader + mingw-w64-ucrt-x86_64-spirv-headers + mingw-w64-x86_64-shaderc + + # Run steps that need bash/msys2 tools (make, ccache, grep) declare + # `shell: msys2 {0}` individually instead of a job-level defaults block: + # the msys2 shell only exists once the Set up MSYS2 step above has run. + - name: Set CCACHE_DIR + shell: msys2 {0} + run: echo "CCACHE_DIR=$HOME/.cache/ccache" >> "$GITHUB_ENV" + + - name: Display toolchain versions + shell: msys2 {0} + run: | + gcc --version | head -1 + cmake --version | head -1 + make --version | head -1 + ccache --version | head -1 + + # ---- ccache for llama.cpp CMake builds ---- + # Same shape as the Darwin workflow: key on the pinned LLAMA_VERSION so a + # pin bump invalidates cleanly; restore-keys fall back to the latest entry + # for the same pin so unchanged TUs stay warm. + - name: Compute llama.cpp version + if: inputs.backend == 'llama-cpp' + id: llama-version + shell: msys2 {0} + run: | + version=$(grep '^LLAMA_VERSION' backend/cpp/llama-cpp/Makefile | head -1 | cut -d= -f2 | cut -d'?' -f1 | tr -d ' ') + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Restore ccache + if: inputs.backend == 'llama-cpp' + id: ccache-cache + uses: actions/cache/restore@v6 + with: + path: ~/.cache/ccache + key: ccache-llama-windows-amd64-${{ steps.llama-version.outputs.version }}-${{ github.run_id }} + restore-keys: | + ccache-llama-windows-amd64-${{ steps.llama-version.outputs.version }}- + + # Only llama-cpp has a windows build path today - the matrix's + # includeWindows section lists exactly this backend. Fail loudly rather + # than upload an empty tar if a future entry dispatches here without a + # build step of its own. Keep in sync with WINDOWS_BESPOKE_BUILDERS in + # scripts/lib/backend-filter.mjs. + - name: Check backend is supported + if: inputs.backend != 'llama-cpp' + run: | + echo "::error::no windows build path for backend '${{ inputs.backend }}'" + exit 1 + + # The msys2 shell below resets PATH, so the Go toolchain setup-go put on + # the runner PATH is invisible to it (and setup-go only exports GOROOT + # for Go < 1.9). Resolve the install dir with the default shell, where + # `go` is reachable, and hand it to the script to prepend. + - name: Resolve Go toolchain path + id: go-toolchain + if: inputs.backend == 'llama-cpp' + shell: bash + run: echo "root=$(go env GOROOT)" >> "$GITHUB_OUTPUT" + + - name: Build ${{ inputs.backend }} (llama-cpp) + if: inputs.backend == 'llama-cpp' + shell: msys2 {0} + env: + GO_TOOLCHAIN_ROOT: ${{ steps.go-toolchain.outputs.root }} + run: | + make backends/llama-cpp-windows + + - name: ccache stats + if: inputs.backend == 'llama-cpp' + shell: msys2 {0} + run: ccache -s + + - name: Save ccache + if: inputs.backend == 'llama-cpp' && github.event_name != 'pull_request' + uses: actions/cache/save@v6 + with: + path: ~/.cache/ccache + key: ccache-llama-windows-amd64-${{ steps.llama-version.outputs.version }}-${{ github.run_id }} + + - name: Upload ${{ inputs.backend }}.tar + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.backend }}-tar + path: backend-images/${{ inputs.backend }}.tar + + windows-backend-publish: + needs: windows-backend-build + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + # id-token: write is required for keyless cosign — the workflow exchanges + # the GitHub OIDC token for a short-lived Fulcio cert that signs each + # pushed image. Without this permission the runner cannot mint the token, + # and `cosign sign` fails with "no token". + permissions: + contents: read + id-token: write + env: + # cosign gates --registry-referrers-mode=oci-1-1 behind this flag. + # Without it, signing fails with "invalid argument \"oci-1-1\" ... + # you must set COSIGN_EXPERIMENTAL=1". + COSIGN_EXPERIMENTAL: '1' + steps: + - name: Download ${{ inputs.backend }}.tar + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.backend }}-tar + path: . + + - name: Install crane + run: | + curl -L https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar -xz + sudo mv crane /usr/local/bin/ + + - name: Log in to DockerHub + run: | + echo "${{ secrets.dockerPassword }}" | crane auth login docker.io -u "${{ secrets.dockerUsername }}" --password-stdin + + - name: Log in to quay.io + run: | + echo "${{ secrets.quayPassword }}" | crane auth login quay.io -u "${{ secrets.quayUsername }}" --password-stdin + + - name: Docker meta + id: meta + uses: docker/metadata-action@v6 + with: + images: | + localai/localai-backends + tags: | + type=ref,event=branch + type=semver,pattern={{raw}} + type=sha + flavor: | + latest=auto + suffix=${{ inputs.tag-suffix }},onlatest=true + + - name: Docker meta + id: quaymeta + uses: docker/metadata-action@v6 + with: + images: | + quay.io/go-skynet/local-ai-backends + tags: | + type=ref,event=branch + type=semver,pattern={{raw}} + type=sha + flavor: | + latest=auto + suffix=${{ inputs.tag-suffix }},onlatest=true + + # This workflow pushes the Windows image directly and so bypasses + # backend_merge.yml — the same reason every other per-platform image is + # signed by the merge job does not apply here. Sign in-line so the + # published artifact carries the same Sigstore bundle the merged images + # do, ready for the gallery verification policy that will cover it. + - name: Install cosign + uses: sigstore/cosign-installer@v3 + with: + cosign-release: 'v2.6.5' + + - name: Push Docker image (DockerHub) + run: | + for tag in $(echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n'); do + crane push ${{ inputs.backend }}.tar $tag + done + # Sign by digest so the signature outlives tag moves (signing by tag + # would orphan it the next time the tag moves). --recursive mirrors + # backend_merge.yml; on a single-manifest image it is a no-op. + digest=$(crane digest "$(echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n' | head -n1)") + cosign sign --yes --recursive \ + --new-bundle-format \ + --registry-referrers-mode=oci-1-1 \ + "localai/localai-backends@${digest}" + + - name: Push Docker image (Quay) + run: | + for tag in $(echo "${{ steps.quaymeta.outputs.tags }}" | tr ',' '\n'); do + crane push ${{ inputs.backend }}.tar $tag + done + digest=$(crane digest "$(echo "${{ steps.quaymeta.outputs.tags }}" | tr ',' '\n' | head -n1)") + cosign sign --yes --recursive \ + --new-bundle-format \ + --registry-referrers-mode=oci-1-1 \ + "quay.io/go-skynet/local-ai-backends@${digest}" diff --git a/.github/workflows/backend_pr.yml b/.github/workflows/backend_pr.yml index 2626f87e8b0d..dbf9a7aa9d9f 100644 --- a/.github/workflows/backend_pr.yml +++ b/.github/workflows/backend_pr.yml @@ -13,9 +13,11 @@ jobs: outputs: matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }} matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }} + matrix-windows: ${{ steps.set-matrix.outputs['matrix-windows'] }} merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }} has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }} has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }} + has-backends-windows: ${{ steps.set-matrix.outputs['has-backends-windows'] }} has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }} # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js). @@ -292,3 +294,21 @@ jobs: strategy: fail-fast: true matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-darwin) }} + + backend-jobs-windows: + needs: generate-matrix + uses: ./.github/workflows/backend_build_windows.yml + if: needs.generate-matrix.outputs.has-backends-windows == 'true' + with: + backend: ${{ matrix.backend }} + build-type: ${{ matrix.build-type }} + go-version: "1.27.x" + tag-suffix: ${{ matrix.tag-suffix }} + lang: ${{ matrix.lang || 'go' }} + runs-on: "windows-latest" + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: true + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-windows) }} diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 08fb2d0842a3..ace1e4fc507b 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -53,6 +53,35 @@ jobs: - name: Run GoReleaser run: | make ${{ github.event_name == 'pull_request' && 'dev-dist-single' || 'dev-dist' }} + # Windows ships no GNU toolchain on the default runner: make is not + # installed there and Git for Windows does not ship it (it bundles sh, uname, + # unzip, grep, awk, sed only), so it is installed via Chocolatey below. The + # Makefile finds Git for Windows' sh itself and runs recipes through it (see + # the SHELL setup at the top of the Makefile), so the explicit usr/bin PATH + # step below is belt-and-suspenders for environments where make must locate + # sh before the makefile is read. The server builds with CGO_ENABLED=0, so no + # mingw/gcc is needed; the Makefile downloads protoc and installs the Go + # protobuf plugins itself (see the protoc/protogen-go targets). + build-test-windows: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.27 + - name: Add Git usr/bin to PATH + run: | + echo "$env:ProgramFiles\Git\usr\bin" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH + - name: Install GNU Make + run: choco install make -y + - name: Build LocalAI (CGO disabled, mirrors release config) + env: + CGO_ENABLED: '0' + run: make build launcher-build-darwin: runs-on: macos-latest steps: @@ -74,7 +103,7 @@ jobs: name: launcher-macos path: dist/ retention-days: 30 - + launcher-build-linux: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index b5c84d2a85bc..8a9c18ef6ca2 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -75,3 +75,79 @@ jobs: detached: true connect-timeout-seconds: 180 limit-access-to-actor: true + + # Windows host smoke test for the native-windows backend support: boots a + # real local-ai.exe with the mock-backend laid out as the gallery ships it + # (run.sh + run.ps1) and asserts a chat completion plus job-object process + # cleanup on hard-kill. Only meaningful on a Windows host; the suite is a + # self-skipping Windows-only ginkgo package (tests/e2e/windows). + tests-windows-smoke: + runs-on: windows-latest + strategy: + matrix: + go-version: ['1.27.x'] + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + - name: Setup Go ${{ matrix.go-version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + cache: false + - name: Display Go version + run: go version + - name: Proto Dependencies + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $protocZip = "$env:RUNNER_TEMP\protoc.zip" + curl.exe -L --fail --retry 5 --retry-delay 5 -o $protocZip ` + https://github.com/protocolbuffers/protobuf/releases/download/v31.1/protoc-31.1-win64.zip + $protocDir = "$env:RUNNER_TEMP\protoc" + Expand-Archive -Path $protocZip -DestinationPath $protocDir -Force + Copy-Item "$protocDir\bin\protoc.exe" . + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + $gobin = go env GOPATH + # pkg/grpc/proto is gitignored generated code, so it is absent from a + # fresh checkout; protoc does not create its output directory. + New-Item -ItemType Directory -Force pkg\grpc\proto | Out-Null + .\protoc.exe --experimental_allow_proto3_optional -Ibackend/ ` + --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative ` + --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative ` + "--plugin=protoc-gen-go=$gobin\bin\protoc-gen-go.exe" ` + "--plugin=protoc-gen-go-grpc=$gobin\bin\protoc-gen-go-grpc.exe" ` + backend/backend.proto + # local-ai embeds core/http/react-ui/dist (//go:embed), which is + # gitignored build output absent from a fresh checkout; build it first, + # like the `react-ui` make target does (vite build writes to dist/). + - name: Build React UI + run: | + cd core/http/react-ui + npm ci + npm run build + - name: Build real binaries + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $env:CGO_ENABLED = '0' + go build -o local-ai.exe ./cmd/local-ai + if ($LASTEXITCODE -ne 0) { exit 1 } + go build -o mock-backend.exe ./tests/e2e/mock-backend + if ($LASTEXITCODE -ne 0) { exit 1 } + - name: Windows smoke test + shell: pwsh + run: | + $env:LOCAL_AI_EXE = "$env:GITHUB_WORKSPACE\local-ai.exe" + $env:MOCK_BACKEND_EXE = "$env:GITHUB_WORKSPACE\mock-backend.exe" + go run github.com/onsi/ginkgo/v2/ginkgo -v ./tests/e2e/windows + - name: Setup tmate session if tests fail + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 + uses: mxschmitt/action-tmate@v3.23 + with: + detached: true + connect-timeout-seconds: 180 + limit-access-to-actor: true diff --git a/.gitignore b/.gitignore index f59bbe45337f..34690776b9a5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ prepare-sources /backend-images /result.yaml protoc +protoc.exe *.log @@ -25,6 +26,7 @@ go-bert # LocalAI build binary LocalAI /local-ai +/local-ai.exe /local-ai-launcher # Root-level build artifacts when running `go build ./...` against # Go backend packages whose main lives under backend/go/. diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 88d6e9ecd9e4..67bc8cf33b0e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -20,7 +20,7 @@ builds: goos: - linux - darwin - #- windows + - windows goarch: - amd64 - arm64 diff --git a/Makefile b/Makefile index 7c05fda6dd3d..1bc79d436036 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,30 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin backends/llama-cpp-windows .NOTPARALLEL: backends/whisper-medusa .NOTPARALLEL: backends/funasr +# Native GNU make for Windows (e.g. ezwinports) only behaves like the Unix +# make when it can find sh.exe: otherwise recipe lines run under cmd.exe and +# $(shell ...) degrades to bare CreateProcess, which breaks every POSIX +# recipe and the uname/tput expansion below. Seed the exported PATH with the +# sh directories from the standard Git-for-Windows / MSYS2 installs (missing +# entries are harmless in a Windows PATH) and force SHELL to sh, mirroring +# what CI does. Detection uses the cmd environment OS variable (Windows_NT); +# OS is re-derived from uname further down. Nothing matches on +# Linux/macOS/WSL, so the block is inert there. +ifeq ($(findstring Windows,$(OS)),Windows) + export PATH := C:/Program Files/Git/usr/bin;$(if $(LOCALAPPDATA),$(LOCALAPPDATA)/Programs/Git/usr/bin,);C:/msys64/usr/bin;$(PATH) + export SHELL := sh +endif + GOCMD=go GOTEST=$(GOCMD) test GOVET=$(GOCMD) vet -BINARY_NAME=local-ai +# Windows builds get the .exe suffix so the artifact is runnable from cmd / +# PowerShell (an extensionless PE needs a POSIX shell to launch it). OS is +# assigned below via uname; this is a recursive = so the suffix is expanded +# at use time, after OS exists. Empty on Linux/macOS, so nothing else changes. +BINARY_NAME=local-ai$(if $(findstring NT,$(OS)),.exe) LAUNCHER_BINARY_NAME=local-ai-launcher UBUNTU_VERSION?=2404 @@ -396,6 +414,17 @@ test-e2e: build-mock-backend build-cloud-proxy-backend build-localai-proxy-backe $(MAKE) teardown-e2e docker rmi localai-tests +# Windows host smoke test: builds a real local-ai.exe and the mock-backend, +# lays the mock backend into a --backends-path the way the gallery ships it +# (run.sh for discovery + run.ps1 to launch), then boots local-ai.exe and +# asserts a chat completion plus job-object process cleanup. The suite builds +# both binaries itself into a temp dir (CGO_ENABLED=0) and skips entirely on +# non-Windows hosts. To reuse pre-built binaries instead, pass LOCAL_AI_EXE +# and MOCK_BACKEND_EXE to the ginkgo invocation. +.PHONY: test-windows-smoke +test-windows-smoke: protogen-go react-ui + go run github.com/onsi/ginkgo/v2/ginkgo -v ./tests/e2e/windows + # `docker stop` returns as soon as the container exits, but Docker reaps a # `--rm` container asynchronously after that. The `docker rmi localai-tests` in # test-e2e then loses the race against the reaper and fails on a still @@ -548,10 +577,18 @@ help: ## Show this help. .PHONY: protogen protogen: protogen-go +# The win64 protoc zip ships bin/protoc.exe while the unix zips ship +# bin/protoc. protogen-go invokes a bare ./protoc, and the protoc target must +# stay up-to-date once the binary is in place, so on Windows we extract the +# .exe and rename it to ./protoc (MSYS sh runs extensionless PE binaries). +PROTOC_MEMBER := bin/protoc$(if $(findstring NT,$(OS)),.exe) + protoc: @OS_NAME=$$(uname -s | tr '[:upper:]' '[:lower:]'); \ ARCH_NAME=$$(uname -m); \ - if [ "$$OS_NAME" = "darwin" ]; then \ + if echo "$$OS_NAME" | grep -qE 'mingw|msys|cygwin'; then \ + FILE=protoc-31.1-win64.zip; \ + elif [ "$$OS_NAME" = "darwin" ]; then \ if [ "$$ARCH_NAME" = "arm64" ]; then \ FILE=protoc-31.1-osx-aarch_64.zip; \ elif [ "$$ARCH_NAME" = "x86_64" ]; then \ @@ -578,18 +615,20 @@ protoc: fi; \ URL=https://github.com/protocolbuffers/protobuf/releases/download/v31.1/$$FILE; \ curl -fsSL --retry 5 --retry-all-errors --retry-delay 5 $$URL -o protoc.zip && \ - unzip -j -d $(CURDIR) protoc.zip bin/protoc && rm protoc.zip + unzip -o -j -d $(CURDIR) protoc.zip $(PROTOC_MEMBER) && \ + rm -f protoc.zip && \ + [ ! -f ./protoc.exe ] || mv -f ./protoc.exe ./protoc .PHONY: protogen-go protogen-go: protoc install-go-tools mkdir -p pkg/grpc/proto # install-go-tools writes protoc-gen-go and protoc-gen-go-grpc into - # $(shell go env GOPATH)/bin, which isn't on every dev's PATH. protoc - # resolves its code-gen plugins via PATH, so without this prefix the - # generate step fails with "protoc-gen-go: program not found". Prepend - # GOPATH/bin so the freshly-installed plugins win without requiring a - # shell-profile change. - PATH="$$(go env GOPATH)/bin:$$PATH" ./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \ + # $(shell go env GOPATH)/bin, which isn't on every dev's PATH. Point + # protoc at the plugins explicitly (--plugin) so discovery doesn't depend + # on PATH separator conventions (POSIX ':' vs Windows ';'). + ./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \ + --plugin=protoc-gen-go="$$(go env GOPATH)/bin/protoc-gen-go$(if $(findstring NT,$(OS)),.exe,)" \ + --plugin=protoc-gen-go-grpc="$$(go env GOPATH)/bin/protoc-gen-go-grpc$(if $(findstring NT,$(OS)),.exe,)" \ backend/backend.proto core/config/inference_defaults.json: ## Fetch inference defaults from unsloth (only if missing) @@ -1259,6 +1298,32 @@ backends/audio-cpp-darwin: build bash ./scripts/build/audio-cpp-darwin.sh ./local-ai backends install "ocifile://$(abspath ./backend-images/audio-cpp.tar)" +# Windows-specific backends (keep as explicit targets since they have special build logic). +# Built on windows-latest under MSYS2 — see .github/workflows/backend_build_windows.yml. +# Unlike the darwin targets this does not depend on `build`: the windows runner +# builds local-ai.exe with setup-go (go build ./cmd/local-ai), and the full +# `make build` would additionally need node for the React UI, which this +# packaging path does not use. The script still builds local-ai itself when it +# is missing so the target works outside CI too. +# `ocifile://` hands the path straight to tarball.ImageFromPath, which on +# Windows needs a drive-letter path — $(abspath) yields the MSYS /d/... form, +# so it is converted via cygpath -m. +# On a native Windows host (OS=Windows_NT) the recipe shell is Git for +# Windows' sh, which cannot reach the UCRT64 toolchain, so the work is handed +# to scripts/build/llama-cpp-windows.ps1: it locates MSYS2 (installing it via +# winget with user confirmation on first use, then pacman-installing the +# mingw-w64-ucrt toolchain) and runs the sh script under the real MSYS2 bash. +# Under the msys2 CI shell OS is Windows_NT too, and the same PowerShell +# dispatcher finds the setup-msys2 install through PATH. +backends/llama-cpp-windows: +ifeq ($(findstring NT,$(OS)),NT) + powershell -NoProfile -ExecutionPolicy Bypass -File ./scripts/build/llama-cpp-windows.ps1 +else + bash ./scripts/build/llama-cpp-windows.sh +endif + ./$(BINARY_NAME) backends install "ocifile://$(shell cygpath -m $(abspath ./backend-images/llama-cpp.tar))" + + build-darwin-python-backend: build bash ./scripts/build/python-darwin.sh diff --git a/backend/cpp/llama-cpp/run.ps1 b/backend/cpp/llama-cpp/run.ps1 new file mode 100644 index 000000000000..22efbd13978f --- /dev/null +++ b/backend/cpp/llama-cpp/run.ps1 @@ -0,0 +1,31 @@ +# run.ps1 - backend launcher for the native windows/amd64 llama-cpp backend +# images. The mirror of ../run.sh for a host with no POSIX shell: pick the best +# binary for the host, set up the library search path, then run it, forwarding +# arguments and exit code. pkg/model/process.go starts it via +# `powershell.exe -NoProfile -ExecutionPolicy Bypass -File run.ps1`. + +$ErrorActionPreference = "Stop" + +$curDir = Split-Path -Parent $MyInvocation.MyCommand.Path + +$binary = "llama-cpp-fallback.exe" +if (Test-Path (Join-Path $curDir "llama-cpp-cpu-all.exe")) { + $binary = "llama-cpp-cpu-all.exe" +} + +if ($env:LLAMACPP_GRPC_SERVERS) { + if (Test-Path (Join-Path $curDir "llama-cpp-grpc.exe")) { + $binary = "llama-cpp-grpc.exe" + } +} + +# ggml's shared backends (libggml-cpu-*.dll) live next to the executable so +# ggml's own registry finds them; a lib\ dir is still honoured when a variant +# ships one, mirroring the run.sh LD_LIBRARY_PATH handling. +$lib = Join-Path $curDir "lib" +if (Test-Path $lib) { + $env:PATH = "$lib;$env:PATH" +} + +& (Join-Path $curDir $binary) @args +exit $LASTEXITCODE diff --git a/backend/index.yaml b/backend/index.yaml index d399d6498fc0..afbfbede7548 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -25,6 +25,7 @@ intel: "intel-sycl-f16-llama-cpp" amd: "rocm-llama-cpp" metal: "metal-llama-cpp" + windows: "windows-llama-cpp" vulkan: "vulkan-llama-cpp" nvidia-l4t: "nvidia-l4t-arm64-llama-cpp" nvidia-cuda-13: "cuda13-llama-cpp" @@ -2146,6 +2147,7 @@ intel: "intel-sycl-f16-llama-cpp-development" amd: "rocm-llama-cpp-development" metal: "metal-llama-cpp-development" + windows: "windows-llama-cpp-development" vulkan: "vulkan-llama-cpp-development" nvidia-l4t: "nvidia-l4t-arm64-llama-cpp-development" nvidia-cuda-13: "cuda13-llama-cpp-development" @@ -2833,6 +2835,16 @@ uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-llama-cpp" mirrors: - localai/localai-backends:master-metal-darwin-arm64-llama-cpp +- !!merge <<: *llamacpp + name: "windows-llama-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-windows-amd64-llama-cpp" + mirrors: + - localai/localai-backends:latest-windows-amd64-llama-cpp +- !!merge <<: *llamacpp + name: "windows-llama-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-windows-amd64-llama-cpp" + mirrors: + - localai/localai-backends:master-windows-amd64-llama-cpp - !!merge <<: *llamacpp name: "cuda12-llama-cpp-development" uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-llama-cpp" diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index ec455d58215c..a93194775be4 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -797,7 +797,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp", } - Expect(cudaBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(cudaBackend.IsCompatibleWith(system.NewCapabilityState(system.Nvidia))).To(BeTrue()) }) It("should be compatible with cuda13 backend on nvidia GPU", func() { @@ -807,7 +807,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-llama-cpp", } - Expect(cuda13Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(cuda13Backend.IsCompatibleWith(system.NewCapabilityState(system.Nvidia))).To(BeTrue()) }) }) }) @@ -838,7 +838,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp", } - Expect(rocmBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(rocmBackend.IsCompatibleWith(system.NewCapabilityState(system.AMD))).To(BeTrue()) }) It("should be compatible with hipblas backend on AMD GPU", func() { @@ -848,7 +848,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-hip-llama-cpp", } - Expect(hipBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(hipBackend.IsCompatibleWith(system.NewCapabilityState(system.AMD))).To(BeTrue()) }) }) }) @@ -879,7 +879,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp", } - Expect(intelBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(intelBackend.IsCompatibleWith(system.NewCapabilityState(system.Intel))).To(BeTrue()) }) It("should be compatible with intel-sycl-f32 backend on Intel GPU", func() { @@ -889,7 +889,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-llama-cpp", } - Expect(intelF32Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(intelF32Backend.IsCompatibleWith(system.NewCapabilityState(system.Intel))).To(BeTrue()) }) It("should be compatible with intel-transformers backend on Intel GPU", func() { @@ -899,7 +899,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-intel-transformers", } - Expect(intelTransformersBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(intelTransformersBackend.IsCompatibleWith(system.NewCapabilityState(system.Intel))).To(BeTrue()) }) }) }) @@ -972,28 +972,28 @@ var _ = Describe("Gallery Backends", func() { } else { // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{GPUVendor: "nvidia", VRAM: 1000000000000} + nvidiaSystemState := system.NewCapabilityState("nvidia") bestBackend := metaBackend.FindBestBackendFromMeta(nvidiaSystemState, backends) Expect(bestBackend).To(Equal(nvidiaBackend)) // Test with AMD system state - amdSystemState := &system.SystemState{GPUVendor: "amd", VRAM: 1000000000000} + amdSystemState := system.NewCapabilityState("amd") bestBackend = metaBackend.FindBestBackendFromMeta(amdSystemState, backends) Expect(bestBackend).To(Equal(amdBackend)) // Test with default system state (not enough VRAM) - defaultSystemState := &system.SystemState{GPUVendor: "amd"} + defaultSystemState := system.NewCapabilityState("default") bestBackend = metaBackend.FindBestBackendFromMeta(defaultSystemState, backends) Expect(bestBackend).To(Equal(defaultBackend)) // Test with default system state - defaultSystemState = &system.SystemState{GPUVendor: "default"} + defaultSystemState = system.NewCapabilityState("default") bestBackend = metaBackend.FindBestBackendFromMeta(defaultSystemState, backends) Expect(bestBackend).To(Equal(defaultBackend)) backends = GalleryElements[*GalleryBackend]{nvidiaBackend, amdBackend, metalBackend} // Test with unsupported GPU vendor - unsupportedSystemState := &system.SystemState{GPUVendor: "unsupported"} + unsupportedSystemState := system.NewCapabilityState("unsupported") bestBackend = metaBackend.FindBestBackendFromMeta(unsupportedSystemState, backends) Expect(bestBackend).To(BeNil()) } @@ -1042,11 +1042,7 @@ var _ = Describe("Gallery Backends", func() { Expect(err).NotTo(HaveOccurred()) // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{ - GPUVendor: "nvidia", - VRAM: 1000000000000, - Backend: system.Backend{BackendsPath: tempDir}, - } + nvidiaSystemState := system.NewCapabilityState("nvidia", system.WithBackendPath(tempDir)) err = InstallBackendFromGallery(context.TODO(), []config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true, false) Expect(err).NotTo(HaveOccurred()) @@ -1122,11 +1118,7 @@ var _ = Describe("Gallery Backends", func() { Expect(err).NotTo(HaveOccurred()) // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{ - GPUVendor: "nvidia", - VRAM: 1000000000000, - Backend: system.Backend{BackendsPath: tempDir}, - } + nvidiaSystemState := system.NewCapabilityState("nvidia", system.WithBackendPath(tempDir)) err = InstallBackendFromGallery(context.TODO(), []config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true, false) Expect(err).NotTo(HaveOccurred()) @@ -1206,11 +1198,7 @@ var _ = Describe("Gallery Backends", func() { Expect(err).NotTo(HaveOccurred()) // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{ - GPUVendor: "nvidia", - VRAM: 1000000000000, - Backend: system.Backend{BackendsPath: tempDir}, - } + nvidiaSystemState := system.NewCapabilityState("nvidia", system.WithBackendPath(tempDir)) err = InstallBackendFromGallery(context.TODO(), []config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true, false) Expect(err).NotTo(HaveOccurred()) diff --git a/core/gallery/backends_version_test.go b/core/gallery/backends_version_test.go index d75be7b1e912..7bdb2eb51b50 100644 --- a/core/gallery/backends_version_test.go +++ b/core/gallery/backends_version_test.go @@ -1,6 +1,8 @@ package gallery_test import ( + "archive/tar" + "bytes" "context" "encoding/json" "os" @@ -8,6 +10,7 @@ import ( "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/oci" "github.com/mudler/LocalAI/pkg/system" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -115,4 +118,50 @@ var _ = Describe("Backend versioning", func() { _, hasVersion := metadata["version"] Expect(hasVersion).To(BeFalse()) }) + + It("installs a backend from an ocifile tar without recording a digest", func() { + // Build a minimal OCI image tar the way `local-ai util + // create-oci-image` does, with run.sh at the layer root (the layout of + // the shipped backend-image tars). + archivePath := filepath.Join(tempDir, "archive.tar") + archiveFile, err := os.Create(archivePath) + Expect(err).NotTo(HaveOccurred()) + runSh := []byte("#!/bin/sh\necho ok\n") + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + Expect(tw.WriteHeader(&tar.Header{Name: "run.sh", Mode: 0755, Size: int64(len(runSh))})).To(Succeed()) + _, err = tw.Write(runSh) + Expect(err).NotTo(HaveOccurred()) + Expect(tw.Close()).To(Succeed()) + _, err = archiveFile.Write(buf.Bytes()) + Expect(err).NotTo(HaveOccurred()) + Expect(archiveFile.Close()).To(Succeed()) + + imageTar := filepath.Join(tempDir, "image.tar") + Expect(oci.CreateTar(archivePath, imageTar, "localai/test:latest", "amd64", "windows")).To(Succeed()) + + backend := &gallery.GalleryBackend{} + backend.Name = "test-ocifile" + backend.URI = "ocifile://" + imageTar + + // The whole point of the ocifile scheme is a local stream: no + // registry digest exists to record, and the installer must not fail + // (or warn) trying to resolve one for a path that cannot be parsed as + // a registry reference. + err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil, false) + Expect(err).NotTo(HaveOccurred()) + + metadataPath := filepath.Join(tempDir, "test-ocifile", "metadata.json") + data, err := os.ReadFile(metadataPath) + Expect(err).NotTo(HaveOccurred()) + + var metadata map[string]any + Expect(json.Unmarshal(data, &metadata)).To(Succeed()) + _, hasDigest := metadata["digest"] + Expect(hasDigest).To(BeFalse()) + + // The staged run.sh must have landed through the tar extraction. + _, err = os.Stat(filepath.Join(tempDir, "test-ocifile", "run.sh")) + Expect(err).NotTo(HaveOccurred()) + }) }) diff --git a/core/gallery/gallery_test.go b/core/gallery/gallery_test.go index cb2070c1c6eb..1455f7e87360 100644 --- a/core/gallery/gallery_test.go +++ b/core/gallery/gallery_test.go @@ -510,7 +510,7 @@ var _ = Describe("Gallery", func() { }, }, } - result := FindGalleryElement(modelsWithPath, "bert/embeddings") + result := FindGalleryElement(modelsWithPath, "bert"+string(os.PathSeparator)+"embeddings") Expect(result).NotTo(BeNil()) Expect(result.GetName()).To(Equal("bert__embeddings")) }) diff --git a/core/gallery/model_artifacts_test.go b/core/gallery/model_artifacts_test.go index cd9d4cdd9866..bc31ddb5b095 100644 --- a/core/gallery/model_artifacts_test.go +++ b/core/gallery/model_artifacts_test.go @@ -67,7 +67,7 @@ unknown_extension: Expect(err).NotTo(HaveOccurred()) Expect(fake.seen).To(HaveLen(1)) Expect(installed.Model).To(Equal("owner/repo")) - Expect(installed.ModelFileName()).To(Equal(fake.result.RelativePath)) + Expect(installed.ModelFileName()).To(Equal(filepath.FromSlash(fake.result.RelativePath))) data, err := os.ReadFile(filepath.Join(modelsPath, "managed.yaml")) Expect(err).NotTo(HaveOccurred()) diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index c5bd8648111e..bf4cd58a736a 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -1056,6 +1056,9 @@ var _ = Describe("HostResolveEnv engine preference wiring", func() { if runtime.GOOS == "darwin" { Skip("darwin reports metal or darwin-x86 before the VRAM floor is consulted") } + if runtime.GOOS == "windows" { + Skip("windows reports its own capability before the VRAM floor is consulted") + } Expect(os.Unsetenv(capabilityEnv)).To(Succeed()) // A capability run file on the machine would override detection. Expect(os.Setenv(capabilityRunFileEnv, filepath.Join(GinkgoT().TempDir(), "absent"))).To(Succeed()) diff --git a/core/http/endpoints/localai/video_internal_test.go b/core/http/endpoints/localai/video_internal_test.go index 383ae5fe4115..848f945c9958 100644 --- a/core/http/endpoints/localai/video_internal_test.go +++ b/core/http/endpoints/localai/video_internal_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "os" "path/filepath" + "runtime" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,7 +24,11 @@ var _ = Describe("video media staging", func() { Expect(os.ReadFile(path)).To(Equal(content)) info, err := os.Stat(path) Expect(err).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + } }) It("accepts browser data URIs with codec parameters", func() { diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index 80e511201b7d..ebecc3c8bc79 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -22,6 +22,9 @@ func SetupTestDB() *gorm.DB { if runtime.GOOS == "darwin" { Skip("testcontainers requires Docker, not available on macOS CI") } + if runtime.GOOS == "windows" { + Skip("testcontainers requires Docker, not available on Windows CI") + } ctx := context.Background() pgC, err := tcpostgres.Run(ctx, "postgres:16", tcpostgres.WithDatabase("testdb"), diff --git a/docs/content/features/GPU-acceleration.md b/docs/content/features/GPU-acceleration.md index fb939a47aaea..cdab430f257c 100644 --- a/docs/content/features/GPU-acceleration.md +++ b/docs/content/features/GPU-acceleration.md @@ -17,6 +17,10 @@ For advanced use cases or to override auto-detection, you can use the `LOCALAI_F - `nvidia`: Forces backends compiled with CUDA support for NVIDIA GPUs. - `amd`: Forces backends compiled with ROCm support for AMD GPUs. - `intel`: Forces backends compiled with SYCL/oneAPI support for Intel GPUs. +- `vulkan`: Forces backends compiled with Vulkan support (e.g. the Windows llama-cpp backend, which auto-detects a Vulkan device and falls back to CPU — see [Windows]({{% relref "getting-started/windows" %}})). +- `metal`: Forces backends compiled with Metal support (macOS/Apple Silicon). +- `darwin-x86`: Forces the Intel-macOS build of a backend. +- `windows`: Forces the native Windows build of a backend (only `llama-cpp` ships a Windows build today). ## Model configuration @@ -379,6 +383,13 @@ On an integrated Intel GPU, the amount of free graphics memory can only be read If using nvidia, follow the steps in the [CUDA](#cudanvidia-acceleration) section to configure your docker runtime to allow access to the GPU. +### Native Windows + +The Windows binary ships the llama-cpp backend with Vulkan baked in: it +auto-detects a Vulkan device at runtime and offloads `gpu_layers` to it, +falling back to CPU when no Vulkan driver is present. No configuration or +driver passthrough is required — see [Windows]({{% relref "getting-started/windows" %}}). + ### Container images To use Vulkan, use the images with the `vulkan` tag, for example `{{< version >}}-gpu-vulkan`. diff --git a/docs/content/getting-started/build.md b/docs/content/getting-started/build.md index 42e64928d2e5..da6813802eea 100644 --- a/docs/content/getting-started/build.md +++ b/docs/content/getting-started/build.md @@ -55,6 +55,17 @@ go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f1 ``` +{{% /tab %}} +{{% tab title="Windows" %}} + +The `make` recipes are POSIX shell. Git for Windows bundles the POSIX tools they need (`sh`, `uname`, `unzip`, `grep`, `awk`, `sed`) — but **not** `make` itself — so install a GNU make for Windows as well: + +```powershell +winget install ezwinports.make +``` + +The Makefile locates Git for Windows' `sh` on its own and runs every recipe through it, downloads `protoc` and installs the Go protobuf plugins itself — so no PATH edits or extra toolchain are needed. `make build` also compiles the React UI, so install [Node.js](https://nodejs.org) (with npm) as well. + {{% /tab %}} {{% tab title="From source" %}} @@ -76,6 +87,8 @@ make build This should produce the binary `local-ai` +On Windows, `make build` produces `local-ai.exe` instead. + #### Container image Requirements: @@ -91,7 +104,7 @@ docker run localai ### Example: Build on mac -Building on Mac (M1, M2 or M3) works, but you may need to install some prerequisites using `brew`. +Building on Mac (M1, M2 or M3) works, but you may need to install some prerequisites using `brew`. The below has been tested by one mac user and found to work. Note that this doesn't use Docker to run the server: @@ -119,7 +132,7 @@ curl http://localhost:8080/v1/models curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{ "model": "phi-2.Q2_K", "messages": [{"role": "user", "content": "How are you?"}], - "temperature": 0.9 + "temperature": 0.9 }' ``` @@ -147,6 +160,27 @@ make clean make build ``` +### Example: Build on Windows + +Building the server binary on Windows needs no C toolchain: the release config compiles with `CGO_ENABLED=0`. The `make` recipes are POSIX shell, so you need Git for Windows (which bundles `sh` and the POSIX tools) plus a GNU make for Windows — Git for Windows does not ship `make`. The Makefile finds `sh` on its own, so no PATH edits are required. + +```powershell +# Install Go (https://go.dev/dl), Git for Windows (https://git-scm.com/downloads), +# Node.js for the React UI build (https://nodejs.org), then a GNU make for +# Windows, e.g.: +winget install ezwinports.make + +# Build with CGO disabled like the release config. +$env:CGO_ENABLED = '0' + +git clone https://github.com/mudler/LocalAI.git +cd LocalAI + +make build +``` + +This produces the binary `local-ai.exe`. The `llama-cpp` backend is also distributed natively for Windows — see [Building the Windows backend]({{% relref "getting-started/windows" %}}#building-from-source) for the MSYS2 UCRT64 requirements and the `make backends/llama-cpp-windows` target. Other backends are still built as Linux container images and are not distributed for Windows. + ## Build backends LocalAI have several backends available for installation in the backend gallery. The backends can be also built by source. As backends might vary from language and dependencies that they require, the documentation will provide generic guidance for few of the backends, which can be applied with some slight modifications also to the others. @@ -178,7 +212,7 @@ make docker-build- Note that `make` is only by convenience, in reality it just runs a simple `docker` command as: ```bash -docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend: -f LocalAI/backend/Dockerfile.golang --build-arg BACKEND= . +docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend: -f LocalAI/backend/Dockerfile.golang --build-arg BACKEND= . ``` Note: diff --git a/docs/content/getting-started/install.md b/docs/content/getting-started/install.md index 861bba652276..77c6ba9d7c84 100644 --- a/docs/content/getting-started/install.md +++ b/docs/content/getting-started/install.md @@ -20,6 +20,7 @@ Choose the installation method that best suits your needs: 1. **[Containers]({{% relref "getting-started/containers" %}})** ⭐ **Recommended** - Works on all platforms, supports Docker and Podman 2. **[macOS]({{% relref "getting-started/macos" %}})** - Download and install the DMG application -3. **[Linux]({{% relref "getting-started/linux" %}})** - Install on Linux using binaries -4. **[Kubernetes]({{% relref "getting-started/kubernetes" %}})** - Deploy LocalAI on Kubernetes clusters -5. **[Build from Source]({{% relref "getting-started/build" %}})** - Build LocalAI from source code +3. **[Windows]({{% relref "getting-started/windows" %}})** - Run natively on Windows, no WSL or Docker required +4. **[Linux]({{% relref "getting-started/linux" %}})** - Install on Linux using binaries +5. **[Kubernetes]({{% relref "getting-started/kubernetes" %}})** - Deploy LocalAI on Kubernetes clusters +6. **[Build from Source]({{% relref "getting-started/build" %}})** - Build LocalAI from source code diff --git a/docs/content/getting-started/models.md b/docs/content/getting-started/models.md index dc0233d77f6b..656e6b9b2a2e 100644 --- a/docs/content/getting-started/models.md +++ b/docs/content/getting-started/models.md @@ -317,7 +317,7 @@ curl http://localhost:8080/v1/completions -H "Content-Type: application/json" -d For other Docker images, please refer to the table in [Getting Started](https://localai.io/basics/getting_started/#container-images). {{% /notice %}} -Note: If you are on Windows, ensure the project is on the Linux filesystem to avoid slow model loading. For more information, see the [Microsoft Docs](https://learn.microsoft.com/en-us/windows/wsl/filesystems). +Note: When running LocalAI under Docker/WSL2 on Windows, ensure the project is on the Linux filesystem to avoid slow model loading. For more information, see the [Microsoft Docs](https://learn.microsoft.com/en-us/windows/wsl/filesystems). {{% /tab %}} {{% tab title="Kubernetes" %}} diff --git a/docs/content/getting-started/troubleshooting.md b/docs/content/getting-started/troubleshooting.md index 999a8e23c902..de4be0c4daf7 100644 --- a/docs/content/getting-started/troubleshooting.md +++ b/docs/content/getting-started/troubleshooting.md @@ -60,7 +60,7 @@ Report non-reproduction if matched requests do not reproduce the delay. ```bash chmod +x local-ai-* -./local-ai-Linux-x86_64 run +./local-ai--linux-amd64 run ``` If you see "cannot execute binary file: Exec format error", you downloaded the wrong architecture. Verify with: diff --git a/docs/content/getting-started/windows.md b/docs/content/getting-started/windows.md new file mode 100644 index 000000000000..0b7ef6204d7f --- /dev/null +++ b/docs/content/getting-started/windows.md @@ -0,0 +1,198 @@ +--- +title: Windows +weight: 8 +description: > + Install and run LocalAI on Windows without WSL or Docker. Native + windows/amd64 backend support. +--- + +## Native Windows backends (no WSL / Docker required) + +LocalAI can run entirely natively on Windows using pre-built backend images +packaged as OCI tarballs. The llama-cpp backend is the first backend with +native Windows support; other backends will follow. + +## Prerequisites + +- Windows 10/11, **amd64** +- PowerShell 7+ (recommended) or cmd.exe +- No WSL, no Docker Desktop, no MSYS2 needed at runtime + +The `local-ai` launcher downloads the Windows backend image on first run and +executes it as a native process. + +## Install + +```powershell +# Download the Windows release asset `local-ai--windows-amd64.exe` +# from the releases page: https://github.com/mudler/LocalAI/releases + +# Or via winget (when available): +# winget install mudler.localai +``` + +## First run + +```powershell +# Download a model (for example, Llama 3.2 1B Instruct GGUF) +.\local-ai.exe model install huggingface://bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M + +# Start LocalAI +.\local-ai.exe run +``` + +LocalAI selects the best llama-cpp binary shipped in the Windows backend image +automatically. No configuration is required. + +## Backend selection on Windows + +Windows backend images ship three llama-cpp executables: + +- `llama-cpp-cpu-all.exe` — CPU-only build with all ggml CPU backends (preferred + when present). +- `llama-cpp-grpc.exe` — gRPC-RPC build, selected automatically when the + `LLAMACPP_GRPC_SERVERS` environment variable is set. +- `llama-cpp-fallback.exe` — static fallback used when neither of the above + applies. + +A small `run.ps1` launcher (Windows has no shell to run the `run.sh` stub +through, so `pkg/model/process.go` starts `run.ps1` via `powershell.exe`) +picks the right binary at startup. + +### GPU (Vulkan) support + +The Windows backend image ships `ggml-vulkan.dll` alongside the CPU backends. +The single `llama-cpp-cpu-all.exe` build auto-detects a Vulkan device at +startup and offloads `gpu_layers` to it when a model requests them, falling +back to CPU inference on hosts with no Vulkan driver. No configuration is +required — the Vulkan loader (`vulkan-1.dll`) is bundled with the image and +loaded dynamically, so the backend never hard-depends on it. + +## Environment variables + +| Variable | Effect | +|----------|--------| +| `LLAMACPP_GRPC_SERVERS` | Forces use of `llama-cpp-grpc.exe` when set. | + +## Troubleshooting + +### Missing Visual C++ runtime + +If the backend fails to start with a `DLL not found` error, install the +[Visual C++ Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist). + +### Antivirus / SmartScreen + +Windows Defender SmartScreen may warn about the unsigned `local-ai.exe` and +backend binaries. Click **More info** → **Run anyway** to proceed. Future +releases will be signed. + +### Firewall + +LocalAI binds `:8080` on **all interfaces** by default, with no API key +configured — on a shared network that exposes the API to anyone who can +reach the host. For local use, bind explicitly to the loopback interface +and set a key instead: + +```powershell +./local-ai.exe --address 127.0.0.1:8080 --api-key your-key-here +``` + +When the firewall prompts on first run, allow local-ai.exe only if you +intend to listen on non-loopback interfaces (for example, to serve models +to other machines on your LAN); for a loopback-only setup, deny the +prompt. + +## Building from source + +See `scripts/build/llama-cpp-windows.sh` and the Makefile target +`backends/llama-cpp-windows` if you want to rebuild the Windows backend image +locally. The target is re-runnable: an interrupted build can simply be run +again (the script re-arms its source clones, and the long gRPC build tree is +reused between runs). +The build requires an MSYS2 UCRT64 environment with mingw-w64 GCC, +CMake 4.x, Ninja, the Vulkan headers/loader and glslc (see below), and Go +(to build the `local-ai.exe` that assembles the OCI image tar; it must be +reachable via `GOROOT`, `GO_TOOLCHAIN_ROOT`, or `PATH`). + +### Setting up MSYS2 UCRT64 + +The Makefile target installs MSYS2 and the toolchain automatically (after +confirmation) when they are missing, so the manual steps below are only needed +if you prefer to install by hand: + +1. Download and install MSYS2 from https://www.msys2.org/ +2. Open the **MSYS2 UCRT64** terminal (not plain MSYS2) +3. Install the build toolchain: + +```bash +pacman -S --needed --noconfirm base-devel git cmake ninja mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-vulkan-headers mingw-w64-ucrt-x86_64-vulkan-loader mingw-w64-ucrt-x86_64-spirv-headers mingw-w64-x86_64-shaderc +``` + +The Vulkan packages are required: `GGML_VULKAN=ON` makes CMake's `FindVulkan` +treat `glslc` as a REQUIRED component, and ggml-vulkan runs +`find_package(SPIRV-Headers CONFIG REQUIRED)`. MSYS2 ships glslc only in the +mingw64 variant of `shaderc` (there is no ucrt64 build) — it is a standalone +tool, so the build script adds `/mingw64/bin` to `PATH` when no glslc is +already available (e.g. from a Vulkan SDK installed via the registry). The +script also installs `mingw-w64-ucrt-x86_64-spirv-headers` itself if its cmake +config is missing, so the docs' package list is only a head start. + +4. Run the build from a plain Windows shell (PowerShell, `cmd`, or Git bash): + +```powershell +cd C:\source_path\LocalAI +make backends/llama-cpp-windows +``` + + On a native Windows host the Makefile hands the build to + `scripts/build/llama-cpp-windows.ps1`, which runs the sh script under the + real MSYS2 UCRT64 bash. If MSYS2 (or its mingw-w64 UCRT64 toolchain) is not + installed, the dispatcher asks for confirmation before downloading and + installing it via `winget`/`pacman`; declining stops the build. No need to + open the UCRT64 terminal yourself. + + Inside the MSYS2 UCRT64 terminal the script runs directly: + +```bash +cd /source_path/LocalAI +bash scripts/build/llama-cpp-windows.sh +``` + +The script packages the result as an OCI tarball (`backend-images/llama-cpp.tar`) +and — when run through the Makefile — installs it with +`./local-ai.exe backends install "ocifile://..."`. + +### GitHub Actions build + +CI builds the Windows backend in the reusable +[`.github/workflows/backend_build_windows.yml`](https://github.com/mudler/LocalAI/blob/master/.github/workflows/backend_build_windows.yml) +workflow, dispatched from `.github/workflows/backend.yml` for the `includeWindows` +matrix entry. Only `llama-cpp` has a Windows build path today — a future entry +dispatched without one fails loudly rather than upload an empty tarball. + +The pipeline: + +1. **Build job** (`windows-latest`): sets up MSYS2 UCRT64 with the same + package set as the local instructions above, caches ccache keyed on the + pinned `LLAMA_VERSION`, then runs `make backends/llama-cpp-windows` in an + MSYS2 shell (the Go toolchain from `actions/setup-go` is handed in via + `GO_TOOLCHAIN_ROOT`, since the MSYS2 shell resets `PATH`). The produced + `backend-images/llama-cpp.tar` is uploaded as a workflow artifact. +2. **Publish job** (`ubuntu-latest`, skipped on PRs): downloads the tarball, + logs into DockerHub and quay.io with `crane`, tags it with the standard + branch/semver/sha metadata (the `-windows-amd64-llama-cpp` tag suffix from + the matrix), + and pushes the OCI image to both registries. + +The gallery entry points at the published image, so `local-ai backends install` +fetches the exact artifact CI produced. + +The published image is signed keyless with Sigstore/cosign, the same way the +platform images that go through the `backend_merge.yml` manifest merge are: +the publish job resolves the pushed digest and signs it with an OCI 1.1 +referrer bundle. The gallery does not enforce a `verification:` policy yet — +`backend/index.yaml` still ships without those blocks populated, and strict +mode stays off until a gallery covers every published variant — so installs +verify the signature, but a missing or invalid one degrades to a warning +rather than a refusal. diff --git a/docs/content/reference/binaries.md b/docs/content/reference/binaries.md index 178f311277c2..9c68640bfc25 100644 --- a/docs/content/reference/binaries.md +++ b/docs/content/reference/binaries.md @@ -5,7 +5,7 @@ title = "LocalAI binaries" weight = 26 +++ -LocalAI binaries are available for both Linux and MacOS platforms and can be executed directly from your command line. These binaries are continuously updated and hosted on [our GitHub Releases page](https://github.com/mudler/LocalAI/releases). This method also supports Windows users via the Windows Subsystem for Linux (WSL). +LocalAI binaries are available for Linux, macOS, and Windows platforms and can be executed directly from your command line. These binaries are continuously updated and hosted on [our GitHub Releases page](https://github.com/mudler/LocalAI/releases). ### macOS Download @@ -13,23 +13,26 @@ You can download the DMG and install the application: Download LocalAI for macOS - + > Note: the DMGs are not signed by Apple as quarantined. See https://github.com/mudler/LocalAI/issues/6268 for a workaround, fix is tracked here: https://github.com/mudler/LocalAI/issues/6244 -Otherwise, use the following one-liner command in your terminal to download and run LocalAI on Linux or MacOS: +Otherwise, use the following one-liner command in your terminal to download and run LocalAI on Linux or MacOS (set `VERSION` to the current tag, e.g. `v4.8.2`): ```bash -curl -Lo local-ai "https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-$(uname -s)-$(uname -m)" && chmod +x local-ai && ./local-ai +VERSION=v4.8.2; ARCH=$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/'); \ +curl -Lo local-ai "https://github.com/mudler/LocalAI/releases/download/$VERSION/local-ai-$VERSION-$(uname -s | tr '[:upper:]' '[:lower:]')-$ARCH" \ + && chmod +x local-ai && ./local-ai ``` Otherwise, here are the links to the binaries: -| OS | Link | +| OS | Link | | --- | --- | -| Linux (amd64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-Linux-x86_64) | -| Linux (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-Linux-arm64) | -| MacOS (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-Darwin-arm64) | +| Linux (amd64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-linux-amd64) | +| Linux (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-linux-arm64) | +| MacOS (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-darwin-arm64) | +| Windows (amd64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-windows-amd64.exe) | {{% notice icon="⚡" context="warning" %}} @@ -38,4 +41,5 @@ Binaries do have limited support compared to container images: - Python-based backends are not shipped with binaries (e.g. `diffusers` or `transformers`) - MacOS binaries and Linux-arm64 do not ship TTS nor `stablediffusion-cpp` backends - Linux binaries do not ship `stablediffusion-cpp` backend +- The Windows binary ships only the `llama-cpp` backend (native, no WSL required); see the [Windows guide]({{% relref "getting-started/windows" %}}) for details {{% /notice %}} diff --git a/pkg/downloader/auth_progress_test.go b/pkg/downloader/auth_progress_test.go index debf45f9e79d..622e652720e5 100644 --- a/pkg/downloader/auth_progress_test.go +++ b/pkg/downloader/auth_progress_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "sync" . "github.com/onsi/ginkgo/v2" @@ -141,7 +142,11 @@ var _ = Describe("authenticated HTTP downloads", func() { Expect(errors.Is(err, context.Canceled)).To(BeTrue()) info, statErr := os.Stat(target + ".partial") Expect(statErr).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + } }) It("keeps the legacy total empty when the response length is unknown", func() { diff --git a/pkg/downloader/partial_resume_test.go b/pkg/downloader/partial_resume_test.go index 9cf953dee2ef..42e213011477 100644 --- a/pkg/downloader/partial_resume_test.go +++ b/pkg/downloader/partial_resume_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -157,6 +158,12 @@ var _ = Describe("DownloadFile with a leftover .partial", func() { }) It("reports an informative error when the partial cannot be stat'd", func() { + if runtime.GOOS == "windows" { + // A self-referencing symlink is the portable trick that makes + // os.Stat fail with ELOOP, but creating symlinks on Windows needs + // admin rights or Developer Mode, so the setup itself errors first. + Skip("symlink creation requires privileges on Windows") + } server := rangeServer(true, nil, nil) defer server.Close() diff --git a/pkg/downloader/uri.go b/pkg/downloader/uri.go index 64ef160a8bfe..ade2ea8b6738 100644 --- a/pkg/downloader/uri.go +++ b/pkg/downloader/uri.go @@ -822,6 +822,10 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string if startPos > 0 && resp.StatusCode != http.StatusPartialContent { _ = resp.Body.Close() + // The origin ignored the resume range, so the partial is garbage; + // a retry must start clean. Drop the write handle first — Windows + // cannot delete a file that is still open (no FILE_SHARE_DELETE). + _ = outFile.Close() _ = removePartialFile(tmpFilePath) // The partial has just been discarded, so a further attempt starts // clean and no longer needs the server to honour the range. @@ -880,6 +884,13 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string // after filesystem permissions while the disk was perfectly healthy. tracked := &readErrorRecorder{r: source} _, err = xio.Copy(ctx, io.MultiWriter(outFile, progress), tracked) + // Windows cannot rename or remove a file that still has an open handle: + // Go opens files without FILE_SHARE_DELETE, so MoveFileEx / DeleteFile fail + // with a sharing violation while outFile is live. The copy is the last use + // of the handle, so close it before any of the removal/rename paths below — + // the error paths (non-206 resume, user cancel, SHA mismatch) remove the + // partial too, and POSIX unlink works on open files while Windows does not. + _ = outFile.Close() if err != nil { // Detect cancellation via the context (a cause-cancelled read surfaces // the cause, not context.Canceled). Keep the .partial for resume, diff --git a/pkg/downloader/uri_test.go b/pkg/downloader/uri_test.go index aff6ea46d931..d2fbdb2f7a18 100644 --- a/pkg/downloader/uri_test.go +++ b/pkg/downloader/uri_test.go @@ -305,12 +305,15 @@ var _ = Describe("Download Test", func() { mockServer := getMockServer(true) defer mockServer.Close() uri := URI(mockServer.URL) - // Create a partial file + // Create a partial file. The handle must be released before the + // download runs: DownloadFile renames the partial over the target, + // and Windows cannot rename a file with an open handle. tmpFilePath := filePath + ".partial" file, err := os.OpenFile(tmpFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) Expect(err).ToNot(HaveOccurred()) _, err = file.Write(mockData[0:10000]) Expect(err).ToNot(HaveOccurred()) + Expect(file.Close()).To(Succeed()) err = uri.DownloadFile(filePath, mockDataSha, 1, 1, func(s1, s2, s3 string, f float64) {}) Expect(err).ToNot(HaveOccurred()) }) @@ -319,12 +322,15 @@ var _ = Describe("Download Test", func() { mockServer := getMockServer(false) defer mockServer.Close() uri := URI(mockServer.URL) - // Create a partial file + // Create a partial file. The handle must be released before the + // download runs: DownloadFile removes the partial and Windows cannot + // delete a file with an open handle. tmpFilePath := filePath + ".partial" file, err := os.OpenFile(tmpFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) Expect(err).ToNot(HaveOccurred()) _, err = file.Write(mockData[0:10000]) Expect(err).ToNot(HaveOccurred()) + Expect(file.Close()).To(Succeed()) err = uri.DownloadFile(filePath, mockDataSha, 1, 1, func(s1, s2, s3 string, f float64) {}) Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index 88a35160f974..6eecec01cdca 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -285,6 +285,7 @@ func (ml *ModelLoader) stopLoadProcess(client *Model, modelID string) { if err := process.Stop(); err != nil { xlog.Warn("failed to stop backend process after failed load", "error", err, "modelID", modelID) } + ml.terminateProcessTreeForProcess(process) ml.cleanupProcessRuntime(process) } diff --git a/pkg/model/process.go b/pkg/model/process.go index ef67fabc26b3..0e2b36066779 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -179,12 +179,16 @@ func (ml *ModelLoader) deleteProcess(ctx context.Context, s string, force bool) if !process.IsAlive() { // A concurrently crashed/already-reaped process can no longer own // resources even if Stop could not read or signal its PID. + ml.terminateProcessTreeForProcess(process) store.Delete(s) ml.cleanupProcessRuntime(process) } else { localErr = err } } else { + // Stop kills the launcher; the process-tree tracker reaps any backend + // child it spawned so the tree cannot outlive a deliberate unload. + ml.terminateProcessTreeForProcess(process) store.Delete(s) ml.cleanupProcessRuntime(process) } @@ -268,6 +272,16 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string return nil, err } + // The gallery contract names a run.sh stub as the backend entrypoint. On a + // Windows host that stub cannot run (no POSIX shell), so resolveLauncher + // substitutes the bundled run.ps1 through the system PowerShell; elsewhere + // it returns the stub and its arguments unchanged. See + // process_launcher_windows.go / process_launcher_other.go. + processName, processArgs, err := resolveLauncher(grpcProcess, workDir, args) + if err != nil { + return nil, err + } + runtime, err := newBackendProcessRuntime() if err != nil { return nil, err @@ -301,8 +315,8 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string grpcControlProcess := process.New( process.WithStateDir(stateDir), - process.WithName(filepath.Base(grpcProcess)), - process.WithArgs(append(args, []string{"--addr", serverAddress}...)...), + process.WithName(processName), + process.WithArgs(append(processArgs, []string{"--addr", serverAddress}...)...), process.WithEnvironment(env...), process.WithWorkDir(workDir), ) @@ -319,6 +333,13 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string } ml.processRuntimes.Store(grpcControlProcess, runtime) + // Attach the freshly started backend to the platform's process-tree + // tracker so that stopping it reaps the whole tree. The mechanism is + // platform specific; see the processTree interface. + if pid, err := strconv.Atoi(grpcControlProcess.CurrentPID()); err == nil { + runtime.trackProcessTree(pid) + } + xlog.Debug("GRPC Service state dir", "dir", grpcControlProcess.StateDir()) signals.RegisterGracefulTerminationHandler(func() { @@ -334,6 +355,10 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string if err := grpcControlProcess.Stop(); err != nil { xlog.Error("error while shutting down grpc process", "error", err) } + // The wrapper's Stop only kills the launcher; reaping the backend it + // spawned needs the process-tree tracker. Idempotent for processes + // StopAllGRPC already stopped (their runtime is gone from the map). + ml.terminateProcessTreeForProcess(grpcControlProcess) }) go func() { @@ -437,6 +462,19 @@ func (ml *ModelLoader) forgetExitedProcess(id string, p *process.Process) { ml.cleanupProcessRuntime(p) } +// terminateProcessTreeForProcess kills the backend's full process tree using +// the tracker created at launch (see createProcess). Idempotent: the tracker +// makes a concurrent stop racing with this one safe, and the map lookup is a +// no-op once the runtime has been cleaned up. +func (ml *ModelLoader) terminateProcessTreeForProcess(process *process.Process) { + if process == nil { + return + } + if value, ok := ml.processRuntimes.Load(process); ok { + value.(*backendProcessRuntime).terminateProcessTree() + } +} + func (ml *ModelLoader) cleanupProcessRuntime(process *process.Process) { if process == nil { return diff --git a/pkg/model/process_launcher_other.go b/pkg/model/process_launcher_other.go new file mode 100644 index 000000000000..8e04c22d0d7d --- /dev/null +++ b/pkg/model/process_launcher_other.go @@ -0,0 +1,13 @@ +//go:build !windows + +package model + +import "path/filepath" + +// resolveLauncher runs the gallery-contract run.sh stub directly: every +// non-Windows host ships a POSIX shell that executes it as-is, so neither the +// executable nor the argument vector needs rewriting. The executable is the +// stub's basename because os.StartProcess resolves it against ProcAttr.Dir. +func resolveLauncher(grpcProcess, _ string, args []string) (processName string, processArgs []string, err error) { + return filepath.Base(grpcProcess), args, nil +} diff --git a/pkg/model/process_launcher_windows.go b/pkg/model/process_launcher_windows.go new file mode 100644 index 000000000000..aaf1832a5eeb --- /dev/null +++ b/pkg/model/process_launcher_windows.go @@ -0,0 +1,38 @@ +//go:build windows + +package model + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +// resolveLauncher rewrites the gallery-contract run.sh launch for a Windows +// host, which has no POSIX shell to execute that stub. When the backend image +// ships the run.ps1 companion, the launch becomes +// "powershell.exe -NoProfile -ExecutionPolicy Bypass -File run.ps1"; run.sh +// stays in the image so discovery, validation and upgrades stay uniform with +// the other platforms. Without run.ps1 the raw executable is spawned as +// elsewhere. +func resolveLauncher(grpcProcess, workDir string, args []string) (processName string, processArgs []string, err error) { + runScript := filepath.Join(workDir, "run.ps1") + if _, statErr := os.Stat(runScript); statErr != nil { + return filepath.Base(grpcProcess), args, nil + } + // os.StartProcess resolves the executable against ProcAttr.Dir on Windows, + // so a bare "powershell.exe" would be looked up inside the backend's + // workDir and every launch would fail with file-not-found. Resolve the + // absolute path first (PATH lookup finds System32). + powershell, lookErr := exec.LookPath("powershell.exe") + if lookErr != nil { + return "", nil, fmt.Errorf("could not locate powershell.exe to launch %s: %w", runScript, lookErr) + } + processArgs = append([]string{ + "-NoProfile", + "-ExecutionPolicy", "Bypass", + "-File", runScript, + }, args...) + return powershell, processArgs, nil +} diff --git a/pkg/model/process_runtime.go b/pkg/model/process_runtime.go index b7443ef7d255..71b6143cec54 100644 --- a/pkg/model/process_runtime.go +++ b/pkg/model/process_runtime.go @@ -29,6 +29,9 @@ type backendProcessRuntime struct { once sync.Once // diagnosticsDone closes after the exit watcher has read the state files. diagnosticsDone chan struct{} + // tree tracks the backend's whole process tree. The implementation is + // chosen at build time; see the processTree interface. + tree processTree } func backendRuntimeRoot() string { @@ -88,6 +91,7 @@ func newBackendProcessRuntime() (*backendProcessRuntime, error) { tempDir: tempDir, lock: runtimeLock, diagnosticsDone: make(chan struct{}), + tree: newProcessTree(), }, nil } @@ -130,6 +134,7 @@ func (r *backendProcessRuntime) cleanup() { return } r.once.Do(func() { + r.terminateProcessTree() r.cleanupScratch() if err := r.lock.Unlock(); err != nil { xlog.Warn("Failed to unlock backend process runtime", "dir", r.dir, "error", err) @@ -140,6 +145,28 @@ func (r *backendProcessRuntime) cleanup() { }) } +// trackProcessTree records the OS state needed to later reap the backend's +// whole process tree. It is best-effort: a host that already placed the +// backend in a tree we cannot take over makes assignment fail, which is logged +// as a warning rather than failing the backend load. +func (r *backendProcessRuntime) trackProcessTree(pid int) { + if r == nil || r.tree == nil { + return + } + if err := r.tree.assign(pid); err != nil { + xlog.Warn("Failed to track the backend process tree; an abrupt LocalAI exit may leave orphaned backend processes", "pid", pid, "error", err) + } +} + +// terminateProcessTree kills the backend's whole process tree and releases its +// handles. Idempotent, so concurrent stops of the same backend are safe. +func (r *backendProcessRuntime) terminateProcessTree() { + if r == nil || r.tree == nil { + return + } + r.tree.terminate() +} + func (r *backendProcessRuntime) cleanupScratch() { if r == nil { return diff --git a/pkg/model/process_start_other.go b/pkg/model/process_start_other.go new file mode 100644 index 000000000000..7f97011411d2 --- /dev/null +++ b/pkg/model/process_start_other.go @@ -0,0 +1,9 @@ +//go:build !windows + +package model + +import process "github.com/mudler/go-processmanager" + +func startProcessWindows(proc *process.Process, runtime *backendProcessRuntime, processName string, processArgs []string, workDir string, env []string) (pid int, err error) { + return 0, nil +} diff --git a/pkg/model/process_start_windows.go b/pkg/model/process_start_windows.go new file mode 100644 index 000000000000..bf4ac20dad7c --- /dev/null +++ b/pkg/model/process_start_windows.go @@ -0,0 +1,46 @@ +//go:build windows + +package model + +import ( + "fmt" + "os" + "os/exec" + "strconv" + "syscall" + + process "github.com/mudler/go-processmanager" + "github.com/mudler/xlog" + "golang.org/x/sys/windows" +) + +func startProcessWindows(proc *process.Process, runtime *backendProcessRuntime, processName string, processArgs []string, workDir string, env []string) (pid int, err error) { + cmd := exec.Command(processName, processArgs...) + cmd.Dir = workDir + cmd.Env = env + cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED} + + if err = cmd.Start(); err != nil { + return 0, err + } + + pid = cmd.Process.Pid + if pid <= 0 { + _ = cmd.Process.Kill() + return 0, fmt.Errorf("invalid pid %d", pid) + } + + runtime.trackProcessTree(pid); if false { + xlog.Debug("failed to assign process to job tree", "pid", pid, "error", err) + } + + if _, err = windows.ResumeThread(windows.Handle(cmd.Process.Pid)); err != nil { + _ = cmd.Process.Kill() + return 0, fmt.Errorf("failed to resume process: %w", err) + } + + if err = os.WriteFile(proc.StateDir()+"/pid", []byte(strconv.Itoa(pid)), os.ModePerm); err != nil { + xlog.Debug("failed to write pid file", "error", err) + } + return pid, nil +} diff --git a/pkg/model/process_tree.go b/pkg/model/process_tree.go new file mode 100644 index 000000000000..65e8c5f772e3 --- /dev/null +++ b/pkg/model/process_tree.go @@ -0,0 +1,18 @@ +package model + +// processTree tracks the OS state that lets LocalAI reap a backend's entire +// process tree when the backend is stopped. The mechanics are platform +// specific and chosen at build time: process_tree_windows.go wraps the tree in +// a job object, while process_tree_other.go is a no-op because the process +// manager already owns the tree elsewhere. +// +// On Windows a backend is started through run.ps1, so go-processmanager tracks +// only the PowerShell wrapper's PID; without a job object owning the tree an +// abrupt stop (or local-ai.exe dying) would orphan the real backend binary. +type processTree interface { + // assign captures the process identified by pid so a later terminate reaps + // it together with any children it spawns afterwards. + assign(pid int) error + // terminate kills the tracked process tree and releases its handles. + terminate() +} diff --git a/pkg/model/process_tree_other.go b/pkg/model/process_tree_other.go new file mode 100644 index 000000000000..5f23185f0265 --- /dev/null +++ b/pkg/model/process_tree_other.go @@ -0,0 +1,14 @@ +//go:build !windows + +package model + +// noopProcessTree satisfies processTree off Windows. There the process manager +// starts the backend directly and owns its whole tree, so no extra tracking is +// needed to reap it on stop. +type noopProcessTree struct{} + +func newProcessTree() processTree { return noopProcessTree{} } + +func (noopProcessTree) assign(int) error { return nil } + +func (noopProcessTree) terminate() {} diff --git a/pkg/model/process_tree_windows.go b/pkg/model/process_tree_windows.go new file mode 100644 index 000000000000..4602d0699c33 --- /dev/null +++ b/pkg/model/process_tree_windows.go @@ -0,0 +1,75 @@ +//go:build windows + +package model + +import ( + "fmt" + "sync" + "unsafe" + + "golang.org/x/sys/windows" +) + +// jobProcessTree implements processTree with a Windows job object configured +// with JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE. +type jobProcessTree struct { + once sync.Once + job windows.Handle +} + +func newProcessTree() processTree { return &jobProcessTree{} } + +// assign wraps the process tree rooted at pid in a job object. A child joins +// its parent's job, so assigning the run.ps1 wrapper captures the backend it +// spawns as well. A failure means the host already placed us in a job we +// cannot escape (service managers, some CI runners); the caller then continues +// without the tree-kill guarantee. +func (t *jobProcessTree) assign(pid int) error { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return fmt.Errorf("creating job object: %w", err) + } + + // Terminate every member once the last job handle closes. This is what + // reaps the backend tree even when local-ai.exe itself is killed hard + // (taskkill /F, crash): the OS closes our job handle and fires the kill. + var limit windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION + limit.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject( + job, windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&limit)), uint32(unsafe.Sizeof(limit)), + ); err != nil { + _ = windows.CloseHandle(job) + return fmt.Errorf("setting kill-on-close on job object: %w", err) + } + + processHandle, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(pid), + ) + if err != nil { + _ = windows.CloseHandle(job) + return fmt.Errorf("opening backend process %d: %w", pid, err) + } + defer func() { _ = windows.CloseHandle(processHandle) }() + if err := windows.AssignProcessToJobObject(job, processHandle); err != nil { + _ = windows.CloseHandle(job) + return fmt.Errorf("assigning backend process %d to job object: %w", pid, err) + } + + t.job = job + return nil +} + +// terminate kills every process that still belongs to the job (the wrapper and +// any backend child that outlived it), then releases the handle. Safe on an +// empty job and idempotent, so concurrent stops of one backend are safe. +func (t *jobProcessTree) terminate() { + t.once.Do(func() { + if t.job == 0 { + return + } + _ = windows.TerminateJobObject(t.job, 1) + _ = windows.CloseHandle(t.job) + t.job = 0 + }) +} diff --git a/pkg/model/windows_job_test.go b/pkg/model/windows_job_test.go new file mode 100644 index 000000000000..a6165d0c31d9 --- /dev/null +++ b/pkg/model/windows_job_test.go @@ -0,0 +1,115 @@ +//go:build windows + +package model + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +// PROCESS_QUERY_LIMITED_INFORMATION is not exported by x/sys/windows; the raw +// value from winnt.h is stable. +const processQueryLimitedInformation = 0x1000 + +// stillActive mirrors the SDK's STILL_ACTIVE (259): GetExitCodeProcess keeps +// returning it while the process runs. +const stillActive = 0x103 + +func processPIDAlive(pid int) bool { + handle, err := windows.OpenProcess(processQueryLimitedInformation, false, uint32(pid)) + if err != nil { + return false + } + defer func() { _ = windows.CloseHandle(handle) }() + var code uint32 + if err := windows.GetExitCodeProcess(handle, &code); err != nil { + return false + } + return code == stillActive +} + +// TestWindowsJobObjectKillsProcessTree reproduces the run.ps1 shape: a +// PowerShell wrapper that later spawns a child (the "backend binary") and +// keeps running. The wrapper is assigned to the job BEFORE it creates the +// child — the same ordering as createProcess — so the child inherits the job +// membership, and terminating the job must reap the child too. Killing only +// the wrapper would orphan it. +func TestWindowsJobObjectKillsProcessTree(t *testing.T) { + tempDir := t.TempDir() + childPIDFile := filepath.Join(tempDir, "child.pid") + // The wrapper parks on this marker file until the test has placed it in a + // job, mirroring how runs.ps1 spawns the backend only after PowerShell has + // started. Without the gate the child would be created outside the job. + gateFile := filepath.Join(tempDir, "go") + + script := filepath.Join(tempDir, "tree.ps1") + content := fmt.Sprintf( + "while (-not (Test-Path '%s')) { Start-Sleep -Milliseconds 50 }\n"+ + "$child = Start-Process powershell -ArgumentList '-NoProfile','-Command','Start-Sleep','30' -PassThru\n"+ + "[IO.File]::WriteAllText('%s', $child.Id.ToString())\n"+ + "Start-Sleep 60\n", + gateFile, childPIDFile, + ) + if err := os.WriteFile(script, []byte(content), 0o600); err != nil { + t.Fatalf("writing tree.ps1: %v", err) + } + + wrapper := exec.Command("powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script) + if err := wrapper.Start(); err != nil { + t.Fatalf("starting wrapper powershell: %v", err) + } + wrapperPID := wrapper.Process.Pid + defer func() { _ = wrapper.Process.Kill() }() + + // Assign FIRST, mirroring createProcess, before the child (backend) exists. + tree := newProcessTree() + if err := tree.assign(wrapperPID); err != nil { + t.Skipf("cannot assign the wrapper to a job object in this environment (nested non-breakaway job?): %v", err) + } + + // Release the gate; the wrapper now spawns its child inside the job. + if err := os.WriteFile(gateFile, []byte("go"), 0o600); err != nil { + t.Fatalf("touching the gate file: %v", err) + } + + var childPID int + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if raw, err := os.ReadFile(childPIDFile); err == nil { + if pid, perr := strconv.Atoi(strings.TrimSpace(string(raw))); perr == nil { + childPID = pid + break + } + } + time.Sleep(100 * time.Millisecond) + } + if childPID == 0 { + t.Fatal("wrapper did not report a child PID") + } + if !processPIDAlive(wrapperPID) || !processPIDAlive(childPID) { + t.Fatalf("expected wrapper(%d) and child(%d) alive before termination; wrapper alive=%v child alive=%v", + wrapperPID, childPID, processPIDAlive(wrapperPID), processPIDAlive(childPID)) + } + + tree.terminate() + + // Wait reads the wrapper's exit status; it returns once the job kill lands. + _ = wrapper.Wait() + deadline = time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if !processPIDAlive(wrapperPID) && !processPIDAlive(childPID) { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Errorf("terminating the job did not reap the whole tree: wrapper alive=%v child alive=%v", + processPIDAlive(wrapperPID), processPIDAlive(childPID)) +} diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index ffe9a4c96594..16ffcca7ddba 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -25,6 +25,7 @@ const ( darwinX86 = "darwin-x86" metal = "metal" vulkan = "vulkan" + windows = "windows" nvidiaCuda13 = "nvidia-cuda-13" nvidiaCuda12 = "nvidia-cuda-12" @@ -36,15 +37,16 @@ const ( defaultRunFile = "/run/localai/capability" // Backend detection tokens (private) - backendTokenDarwin = "darwin" - backendTokenMLX = "mlx" - backendTokenMetal = "metal" - backendTokenL4T = "l4t" - backendTokenCUDA = "cuda" - backendTokenROCM = "rocm" - backendTokenHIP = "hip" - backendTokenSYCL = "sycl" - backendTokenCPU = "cpu" + backendTokenDarwin = "darwin" + backendTokenMLX = "mlx" + backendTokenMetal = "metal" + backendTokenL4T = "l4t" + backendTokenCUDA = "cuda" + backendTokenROCM = "rocm" + backendTokenHIP = "hip" + backendTokenSYCL = "sycl" + backendTokenCPU = "cpu" + backendTokenWindows = "windows" // Engine names (private). Unlike the tokens above these are whole backend // identities as a gallery entry's `backend:` field spells them, not build @@ -117,6 +119,9 @@ var backendBuildTagPreferenceRules = []backendPreferenceRule{ {metal, []string{backendTokenMetal, backendTokenCPU}}, {darwinX86, []string{darwinX86, backendTokenCPU}}, {vulkan, []string{vulkan, backendTokenCPU}}, + // Windows ships native windows/amd64 backend builds; prefer them over the + // Linux CPU builds, which cannot run on a Windows host. + {windows, []string{backendTokenWindows, backendTokenCPU}}, } // defaultBackendBuildTagTokens is what a host with no matching rule prefers. @@ -184,6 +189,11 @@ var engineNamePreferenceRules = []backendPreferenceRule{ // IsBackendCompatible admits any darwin-tokened engine on this capability // and will not drop it. {darwinX86, []string{engineLlamaCpp, engineVLLM, engineSGLang}}, + // A Windows host runs exactly the native llama.cpp build; vLLM and SGLang + // do not publish windows/amd64 images, so ranking them above llama.cpp + // (or even equal to it) could auto-select an engine a Windows host cannot + // run whenever a variant set is offered. + {windows, []string{engineLlamaCpp}}, } // defaultEnginePreferenceTokens is empty on purpose. It is now only reached by @@ -375,6 +385,15 @@ func (s *SystemState) getSystemCapabilities() string { return s.systemCapabilities } + // Windows backends are published as native windows/amd64 OCI images, so a + // Windows host gets its own capability to select them deterministically + // instead of falling through to "default" and racing the Linux CPU builds. + if runtime.GOOS == "windows" { + xlog.Info("Using windows capability", "env", capabilityEnv) + s.systemCapabilities = windows + return s.systemCapabilities + } + // If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { if s.GPUVendor == Nvidia { @@ -496,6 +515,11 @@ func (s *SystemState) IsBackendCompatible(name, uri string) bool { return capability == metal || capability == darwinX86 } + // Check for Windows-specific backends (native windows/amd64 builds) + if strings.Contains(combined, backendTokenWindows) { + return capability == windows + } + // Check for NVIDIA L4T-specific backends (arm64 Linux with NVIDIA GPU) // This must be checked before the general NVIDIA check as L4T backends // may also contain "cuda" or "nvidia" in their names diff --git a/pkg/system/capabilities_test.go b/pkg/system/capabilities_test.go index 0844e52f44a5..523c29dd5f82 100644 --- a/pkg/system/capabilities_test.go +++ b/pkg/system/capabilities_test.go @@ -19,8 +19,8 @@ var _ = Describe("getSystemCapabilities", func() { ) BeforeEach(func() { - if runtime.GOOS == "darwin" { - Skip("darwin short-circuits before reaching CUDA logic") + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + Skip("platform short-circuits before reaching CUDA logic") } origEnv = os.Getenv(capabilityEnv) @@ -168,6 +168,7 @@ var _ = Describe("BackendPreferenceTokens", func() { Entry("metal", metal, []string{"metal", "cpu"}), Entry("darwin-x86", darwinX86, []string{"darwin-x86", "cpu"}), Entry("vulkan", vulkan, []string{"vulkan", "cpu"}), + Entry("windows", windows, []string{"windows", "cpu"}), Entry("unknown capability", "some-future-accelerator", []string{"cpu"}), ) @@ -175,7 +176,7 @@ var _ = Describe("BackendPreferenceTokens", func() { // The generic form of the lock above: whatever anyone adds to the build // tag table later, an engine name in it is a merged vocabulary. engineNames := []string{"vllm", "sglang", "llama-cpp", "mlx"} - for _, capability := range []string{"nvidia", AMD, Intel, metal, darwinX86, vulkan, "default"} { + for _, capability := range []string{"nvidia", AMD, Intel, metal, darwinX86, vulkan, windows, "default"} { Expect(tokensFor(capability)).ToNot(ContainElements(engineNames), "capability %q leaked an engine name into the build tag table", capability) } @@ -235,8 +236,8 @@ var _ = Describe("EnginePreferenceTokens", func() { It("names only engines, never a build tag", func() { // The mirror of the lock on BackendPreferenceTokens. A build tag here // matches no gallery `backend:` value and silently disables ranking. - buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86"} - for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan, defaultCapability, darwinX86} { + buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86", "windows"} + for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan, defaultCapability, darwinX86, windows} { Expect(tokensFor(capability)).ToNot(ContainElements(buildTags), "capability %q leaked a build tag into the engine table", capability) } @@ -253,6 +254,10 @@ var _ = Describe("EnginePreferenceTokens", func() { Expect(tokensFor(darwinX86)).To(Equal([]string{"llama-cpp", "vllm", "sglang"})) }) + It("prefers llama.cpp on windows, the only engine with native windows builds", func() { + Expect(tokensFor(windows)).To(Equal([]string{"llama-cpp"})) + }) + It("ranks the GPU serving engines below llama.cpp rather than leaving them tied", func() { // Enumerating them is what stops download size deciding between vLLM and // SGLang once llama.cpp is out of the running. diff --git a/scripts/build/llama-cpp-windows.ps1 b/scripts/build/llama-cpp-windows.ps1 new file mode 100644 index 000000000000..fd2dc74119b6 --- /dev/null +++ b/scripts/build/llama-cpp-windows.ps1 @@ -0,0 +1,164 @@ +#!/usr/bin/env pwsh + +# PowerShell entry point for the native windows/amd64 llama-cpp backend build, +# driven from the Makefile on native Windows hosts (and on Windows CI via the +# msys2 shell, where powershell.exe is still reachable): +# +# make backends/llama-cpp-windows +# +# GNU make on Windows runs recipe lines through sh (Git for Windows), where +# /ucrt64/bin does not resolve and the mingw toolchain does not exist. This +# script hands the build to a real MSYS2 UCRT64 bash instead. It locates the +# MSYS2 install (MSYS2_ROOT, the standard install dirs, or any PATH entry that +# is really an MSYS2 root), installs MSYS2 + the toolchain on first use -- only +# after asking for confirmation -- and aborts with a non-zero status when the +# user declines, so the Makefile recipe stops before the OCI-image packaging +# step. The sh script's own re-exec dispatch (scripts/build/llama-cpp-windows.sh) +# is bypassed deliberately: starting under the real MSYS2 bash makes the +# /ucrt64/bin check take the direct branch. + +[CmdletBinding()] +param() + +$ErrorActionPreference = "Stop" + +# scripts/build -> repo root +$RepoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..\..")) + +# Toolchain package set, mirroring the CI setup in +# .github/workflows/backend_build_windows.yml. A fresh winget MSYS2 is a bare +# base system: pacman must install these before the script's first configure. +$ToolchainPackages = @( + 'git', 'make', 'cmake', 'mingw-w64-ucrt-x86_64-cmake', 'ninja', 'pkg-config', + 'perl', 'patch', 'unzip', 'curl', + 'mingw-w64-ucrt-x86_64-gcc', 'mingw-w64-ucrt-x86_64-gcc-libs', + 'mingw-w64-ucrt-x86_64-binutils', 'mingw-w64-ucrt-x86_64-ccache', + 'mingw-w64-ucrt-x86_64-vulkan-headers', 'mingw-w64-ucrt-x86_64-vulkan-loader', + 'mingw-w64-ucrt-x86_64-spirv-headers', 'mingw-w64-x86_64-shaderc' +) + +function ConvertTo-WindowsPath { + param([Parameter(Mandatory = $true)][string]$Path) + $Path = $Path.Trim().Trim('"') + # MSYS-style /c/Program Files/... (as exported by the Makefile's sh shell) + # has no drive letter for .NET; rewrite to C:\Program Files\... + if ($Path -match '^/([a-zA-Z])/(.+)$') { + $Path = "$($matches[1]):\$($matches[2])" + } + $Path = $Path -replace '/', '\' + return $Path.TrimEnd('\') +} + +function Test-PathSafe { + param([Parameter(Mandatory = $true)][string]$Path) + try { return (Test-Path -LiteralPath $Path) } + catch { return $false } +} + +function Find-Msys2Root { + # A bare usr\bin\bash.exe is not enough of a discriminator: Git for Windows + # ships one too, and its "root" has no pacman.conf. Requiring etc/pacman.conf + # keeps the PATH scan from dispatching the build into Git bash. + $candidates = New-Object System.Collections.Generic.List[string] + if ($env:MSYS2_ROOT) { $candidates.Add((ConvertTo-WindowsPath $env:MSYS2_ROOT)) } + foreach ($p in @('C:\msys64', 'C:\Program Files\MSYS2')) { $candidates.Add($p) } + foreach ($dir in ($env:PATH -split ';')) { + if ([string]::IsNullOrWhiteSpace($dir)) { continue } + $nd = ConvertTo-WindowsPath $dir + if (Test-PathSafe ([System.IO.Path]::Combine($nd, 'bash.exe'))) { + # usr\bin\bash.exe -> root = two levels up from the PATH entry. + $candidates.Add([System.IO.Path]::GetFullPath([System.IO.Path]::Combine($nd, '..', '..'))) + } + } + foreach ($c in $candidates) { + if ((Test-PathSafe (Join-Path $c 'usr\bin\bash.exe')) -and (Test-PathSafe (Join-Path $c 'etc\pacman.conf'))) { + return $c + } + } + return $null +} + +function Invoke-BashInMsys { + param( + [Parameter(Mandatory = $true)][string]$MsysRoot, + [Parameter(Mandatory = $true)][string]$Command + ) + # MSYSTEM selects the UCRT64 toolchain; a non-login invocation keeps the + # inherited Windows PATH (Go, Git) and the working directory, mirroring the + # sh script's own re-exec behavior. + $env:MSYSTEM = 'UCRT64' + $env:MSYS2_PATH_TYPE = 'inherit' + & (Join-Path $MsysRoot 'usr\bin\bash.exe') -lc $Command + $code = $LASTEXITCODE + if ($code -ne 0) { + Write-Host "ERROR: command failed under MSYS2 (exit $code)" -ForegroundColor Red + exit $code + } +} + +function Test-ToolchainComplete { + param([Parameter(Mandatory = $true)][string]$MsysRoot) + $pkgs = $ToolchainPackages -join ' ' + $env:MSYSTEM = 'UCRT64' + $env:MSYS2_PATH_TYPE = 'inherit' + & (Join-Path $MsysRoot 'usr\bin\bash.exe') -lc "pacman -Q $pkgs >/dev/null 2>&1" + return ($LASTEXITCODE -eq 0) +} + +function Install-Toolchain { + param([Parameter(Mandatory = $true)][string]$MsysRoot) + $pkgs = $ToolchainPackages -join ' ' + Invoke-BashInMsys $MsysRoot "pacman -Sy --needed --noconfirm $pkgs" + if (-not (Test-ToolchainComplete $MsysRoot)) { + Write-Host "ERROR: MSYS2 toolchain install finished but packages are still missing" -ForegroundColor Red + exit 1 + } +} + +function Install-Msys2 { + $answer = Read-Host 'MSYS2 was not found. Install MSYS2 and the mingw-w64 UCRT64 toolchain now (hundreds of MB)? [y/N]' + if ($answer -notmatch '^[Yy]$') { + Write-Host "Aborted. Install MSYS2 manually (winget install --id MSYS2.MSYS2) and retry 'make backends/llama-cpp-windows'." + exit 1 + } + if (-not (Get-Command winget -ErrorAction SilentlyContinue)) { + Write-Host 'ERROR: winget is not available; install MSYS2 manually from https://www.msys2.org/ and retry.' -ForegroundColor Red + exit 1 + } + Write-Host '==> Installing MSYS2 via winget ...' + winget install --id MSYS2.MSYS2 --exact --accept-package-agreements --accept-source-agreements --disable-interactivity + if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: winget install of MSYS2 failed (exit $LASTEXITCODE)" -ForegroundColor Red + exit 1 + } + $root = Find-Msys2Root + if (-not $root) { + Write-Host 'ERROR: MSYS2 was installed but its root could not be located; install it manually and retry.' -ForegroundColor Red + exit 1 + } + return $root +} + +function Invoke-Build { + param([Parameter(Mandatory = $true)][string]$MsysRoot) + # forward-slash form keeps MSYS2 bash from reading the Windows backslashes as + # escape sequences. + $repo = $RepoRoot -replace '\\', '/' + Invoke-BashInMsys $MsysRoot "cd '$repo' && exec ./scripts/build/llama-cpp-windows.sh" +} + +$MsysRoot = Find-Msys2Root +if (-not $MsysRoot) { + $MsysRoot = Install-Msys2 +} + +if (-not (Test-ToolchainComplete $MsysRoot)) { + $answer = Read-Host 'MSYS2 found but the mingw-w64 UCRT64 toolchain is missing. Download and install the build packages (hundreds of MB)? [y/N]' + if ($answer -notmatch '^[Yy]$') { + Write-Host 'Aborted. Install the toolchain packages manually and retry.' + exit 1 + } + Install-Toolchain $MsysRoot +} + +Invoke-Build $MsysRoot \ No newline at end of file diff --git a/scripts/build/llama-cpp-windows.sh b/scripts/build/llama-cpp-windows.sh new file mode 100644 index 000000000000..23b1c91da449 --- /dev/null +++ b/scripts/build/llama-cpp-windows.sh @@ -0,0 +1,831 @@ +#!/bin/bash + +# Builds the native windows/amd64 llama-cpp backend image on a windows-latest +# GitHub runner under MSYS2 (UCRT64). Driven from the top-level Makefile via: +# +# make backends/llama-cpp-windows +# +# Unlike Linux (prebuilt base-grpc images) and Darwin (Homebrew gRPC), a +# Windows runner has neither, so gRPC is compiled from source into +# backend/cpp/grpc/installed_packages -- the exact layout the llama-cpp +# Makefile's BUILD_GRPC_FOR_BACKEND_LLAMA path expects, so the flags it would +# pass to cmake (absl_DIR / Protobuf_DIR / utf8_range_DIR / gRPC_DIR) resolve +# to real directories. +# +# This script drives cmake directly instead of the llama-cpp Makefile's +# llama-cpp-cpu-all / llama-cpp-grpc / llama-cpp-fallback targets: those copy +# binaries around as bare `grpc-server` paths, which MSYS2's cp cannot resolve +# to grpc-server.exe. The cmake flags below mirror exactly what those targets +# would pass (see backend/cpp/llama-cpp/Makefile). +# +# The build host also has no docker daemon, so the result is packaged as an OCI +# tar (--platform windows/amd64) that LocalAI's `backends install` extracts and +# runs as a native process -- see pkg/model/process.go (run.ps1) and +# backend/cpp/llama-cpp/run.ps1. + +set -ex + +# --------------------------------------------------------------------------- +# Auto-dispatch into MSYS2 (UCRT64). +# +# `make backends/llama-cpp-windows` runs this script through the recipe shell. +# GNU make for Windows falls back to cmd.exe unless it finds sh; the Makefile's +# OS-detection block (top of file) then points SHELL at Git for Windows' sh, so +# a local run normally starts under a MINGW64 bash. That bash cannot resolve +# /ucrt64/bin (the path maps to C:\Program Files\Git\ucrt64, which does not +# exist) and nothing below can build without it. Re-exec under the real MSYS2 +# bash instead. A non-login invocation deliberately keeps the inherited working +# directory (the repo root) and the Windows PATH (Go, Git); `bash -l` would +# reset PATH and cd to $HOME. MSYSTEM is not a usable discriminator here (Git +# bash reports MINGW64 too) - the existence of /ucrt64/bin under MSYS2's own +# runtime is the discriminator. +if [ ! -d /ucrt64/bin ]; then + if [ -x /c/msys64/usr/bin/bash.exe ]; then + echo "==> not running under MSYS2; re-exec'ing under C:/msys64 ..." >&2 + SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd) + # Run the script by name, not `exec bash "$0"`: a bare `bash` here would + # resolve to Git for Windows' bash via the inherited PATH (re-dispatch loop); + # the shebang's /bin/bash is an absolute MSYS2 path and lands on the real + # MSYS2 bash, where the /ucrt64/bin check below takes the direct branch. + # + # Mirror msys2_shell.cmd -ucrt64 -use-full-path: a LOGIN bash with + # MSYSTEM=UCRT64 + MSYS2_PATH_TYPE=inherit. This is the one configuration in + # which MSYS2's own git works when launched from a foreign shell (a plain + # non-login `bash.exe -c` inherits the Git-for-Windows mount table and its + # https remote helper dies with "fatal: remote helper 'https' aborted + # session"). `bash -l` resets PATH to MSYS2 defaults and drops `go` (it is + # on the Windows PATH but GOROOT is unset), so re-add the bin dir that the + # invoking shell resolved before the profile runs. + GO_DIR=$(command -v go 2>/dev/null) + [ -n "$GO_DIR" ] && GO_DIR=$(dirname "$GO_DIR") + # The login profile rebuilds the environment from a whitelist and drops + # Windows vars the MSYS2 runtime does not know (USERPROFILE etc.), so + # carry the ones go/ccache need over from the invoking shell. + exec env MSYSTEM=UCRT64 MSYS2_PATH_TYPE=inherit /c/msys64/usr/bin/bash.exe -l -c \ + "export PATH='$GO_DIR':\$PATH; export USERPROFILE='$USERPROFILE'; cd '$REPO_ROOT' && exec ./scripts/build/llama-cpp-windows.sh" + else + echo "ERROR: this script must run under MSYS2 UCRT64 (C:\\msys64)." >&2 + echo " Install MSYS2 (winget install MSYS2.MSYS2) and retry 'make backends/llama-cpp-windows'." >&2 + exit 1 + fi +fi + +ROOT=$(pwd) +IMAGE_NAME="${IMAGE_NAME:-localai/llama-cpp-windows}" +PLATFORMARCH="${PLATFORMARCH:-windows/amd64}" + +# nproc exists on MSYS2; fall back like the llama-cpp Makefile does otherwise. +# Honor an explicit JOBS override (e.g. `JOBS=4 make backends/llama-cpp-windows`): +# on memory-limited Windows hosts -j$(nproc) can spike past available RAM and +# several parallel g++ processes get killed mid-compile with no error text +# (silent "Error 1" on unrelated .obj targets). +JOBS="${JOBS:-$(nproc 2>/dev/null || echo 1)}" +export JOBS + +# Resolve cmake to the native UCRT64 build (/ucrt64/bin/cmake), not MSYS2's +# /usr/bin/cmake. The MSYS/Cygwin cmake reports UNIX=TRUE, so gRPC's +# `if(UNIX)` block in CMakeLists.txt appends ${CMAKE_DL_LIBS} m rt to every +# target's link line and every gRPC executable dies with "cannot find -ldl / +# -lrt" (no such libraries on Windows). The native cmake also sets MINGW +# (gRPC then defines -D_WIN32_WINNT=0x600 itself) and keeps plain exe names +# (the MSYS cmake emits protoc-24.3.0.exe). Everything else the script needs +# (make, git, perl, curl, unzip) lives in /usr/bin, so only cmake's resolution +# changes. +export PATH="/ucrt64/bin:$PATH" + +# Handle zlib conflicts between system MSYS2 and gRPC's bundled static zlib +SYS_ZLIB_DLLA="/ucrt64/lib/libz.dll.a" +SYS_ZLIB_DLLA_BAK="/ucrt64/lib/libz.dll.a.bak" +backup_sys_zlib() { + if [ -f "$SYS_ZLIB_DLLA" ] && [ ! -f "$SYS_ZLIB_DLLA_BAK" ]; then + mv "$SYS_ZLIB_DLLA" "$SYS_ZLIB_DLLA_BAK" + echo "==> Backed up system $SYS_ZLIB_DLLA" + fi +} +restore_sys_zlib() { + if [ -f "$SYS_ZLIB_DLLA_BAK" ] && [ ! -f "$SYS_ZLIB_DLLA" ]; then + mv "$SYS_ZLIB_DLLA_BAK" "$SYS_ZLIB_DLLA" + echo "==> Restored system $SYS_ZLIB_DLLA" + fi +} + +# The MSYS2 runtime does not export PROCESSOR_ARCHITECTURE, so cmake's Windows +# host detection yields an empty CMAKE_SYSTEM_PROCESSOR and ggml treats the +# machine as UNKNOWN (GGML_CPU_ALL_VARIANTS then refuses to configure). This +# backend is x86_64-only. +export PROCESSOR_ARCHITECTURE=AMD64 + +# go.exe resolves GOPATH (default %USERPROFILE%\go) and its build cache +# (%LocalAppData%\go-build) through Windows env vars the login profile drops; +# ccache needs a user-profile dir too. Fall back to the MSYS2 home for direct +# launches that forgot to set it. +export USERPROFILE="${USERPROFILE:-$(cygpath -w "$HOME")}" +export GOCACHE="${GOCACHE:-$(cygpath -w "$HOME")/go-build}" +mkdir -p "$GOCACHE" + +# ggml-vulkan compiles its shaders at build time with glslc, and CMake's +# FindVulkan declares it a REQUIRED component. MSYS2 only ships glslc in the +# mingw64 variant of shaderc (there is no ucrt64 build); it is a standalone +# tool, so its prefix can sit next to the UCRT64 one. When a Vulkan SDK +# (VULKAN_SDK, found by FindVulkan through the registry on Windows) already +# provides glslc, leave PATH alone. +# +# The mingw64 dir must be APPENDED, never prepended: the UCRT64-built tools +# (protoc, gcc, ...) resolve their runtime DLLs through PATH, and a mingw64 +# libstdc++/libgcc_s_seh placed ahead of /ucrt64/bin makes them load the wrong +# ABI runtime and die silently (protoc exits 127 mid-build). +if ! command -v glslc >/dev/null 2>&1 && [ -x /mingw64/bin/glslc.exe ]; then + export PATH="$PATH:/mingw64/bin" +fi + +# ggml-vulkan runs find_package(SPIRV-Headers CONFIG REQUIRED) at configure +# time; MSYS2 ships the config as mingw-w64-ucrt-x86_64-spirv-headers. +# Also ensure common image libraries (JPEG) needed by some components are installed. +if [ ! -f /ucrt64/share/cmake/SPIRV-Headers/SPIRV-HeadersConfig.cmake ] || \ + [ ! -f /ucrt64/lib/libjpeg.a ] && [ ! -f /ucrt64/lib/libjpeg.dll.a ]; then + echo "==> Installing required MSYS2 UCRT64 dependencies (SPIRV-Headers, libjpeg-turbo, etc.)" + pacman -S --noconfirm mingw-w64-ucrt-x86_64-spirv-headers mingw-w64-ucrt-x86_64-libjpeg-turbo mingw-w64-ucrt-x86_64-libpng mingw-w64-ucrt-x86_64-tiff 2>/dev/null || true +fi +# Ensure libjpeg is available even if condition logic was odd +if [ ! -f /ucrt64/lib/libjpeg.dll.a ] && [ ! -f /ucrt64/lib/libturbojpeg.dll.a ] && [ ! -f /ucrt64/lib/libjpeg.a ]; then + pacman -S --noconfirm mingw-w64-ucrt-x86_64-libjpeg-turbo 2>/dev/null || true +fi + +# actions/setup-go adds go to the runner PATH but only exports GOROOT for Go +# < 1.9; the msys2 shell resets PATH, so `go` is not reachable here. The +# workflow resolves the install dir (GO_TOOLCHAIN_ROOT) with the default shell +# and hands it over; GOROOT is the fallback for standalone runs. Re-add the +# bin dir (translated to an msys path) unless a go binary already resolves. +GO_BIN_DIR="${GO_TOOLCHAIN_ROOT:-$GOROOT}" +if ! command -v go >/dev/null 2>&1 && [ -n "$GO_BIN_DIR" ]; then + GO_BIN_DIR="$GO_BIN_DIR/bin" + case "$GO_BIN_DIR" in + [A-Za-z]:*) GO_BIN_DIR=$(cygpath -u "$GO_BIN_DIR") ;; + esac + export PATH="$GO_BIN_DIR:$PATH" +fi + +# --------------------------------------------------------------------------- +# gRPC from source (backend/cpp/grpc/installed_packages) +# --------------------------------------------------------------------------- +GRPC_DIR="$ROOT/backend/cpp/grpc" +mkdir -p "$GRPC_DIR/grpc_repo/grpc" "$GRPC_DIR/grpc_build" +cd "$GRPC_DIR/grpc_repo/grpc" +git init -q +# Re-runnable: a previous (possibly failed) run may already have wired origin. +git remote remove origin 2>/dev/null || true +git remote add origin https://github.com/grpc/grpc.git +git fetch origin refs/tags/v1.59.0:refs/tags/v1.59.0 --depth 1 +git checkout -q v1.59.0 +git submodule update --init --recursive --depth 1 --single-branch + +# abseil's cctz win32 local-time-zone path (USE_WIN32_LOCAL_TIME_ZONE) uses +# WindowsCreateStringReference / WindowsDeleteString / WindowsGetStringRawBuffer +# from , which the MSVC SDK pulls in transitively but mingw-w64's +# roapi.h/windows.globalization.h do not. Patch the one TU to include it +# explicitly; without this the whole gRPC build dies at time_zone_lookup.cc. +# (\R matches either LF or CRLF since git may have normalized line endings.) +perl -0pi -e 's/(#include \R#include \R)(#endif)/$1#include \n$2/' \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc + +# c-ares (bundled with gRPC 1.59.0) has a broken CMake build under mingw-w64 +# UCRT64 + CMake 4.x. Its CMakeLists uses CHECK_INCLUDE_FILES to detect +# winsock2.h / ws2tcpip.h / windows.h on WIN32, but the probes run BEFORE +# CMAKE_REQUIRED_DEFINITIONS is set (so without WIN32_LEAN_AND_MEAN), causing +# windows.h to pull in winsock.h which conflicts with winsock2.h. When the +# checks fail, CMAKE_EXTRA_INCLUDE_FILES stays empty, so the downstream +# CARES_TYPE_EXISTS / CHECK_SYMBOL_EXISTS probes for SOCKET, struct +# sockaddr_in6, struct addrinfo, recv, send etc. all fail too. That leaves +# the generated ares_config.h without HAVE_RECV / HAVE_SEND / +# HAVE_STRUCT_SOCKADDR_IN6 / HAVE_STRUCT_ADDRINFO / RECV_TYPE_* / SEND_TYPE_*, +# which trips setup_once.h's compile-time Error guards and makes ares_ipv6.h +# redefine structs already in ws2tcpip.h. +# +# Pre-seeding the cache vars via -D didn't work: CMake 4.x's CHECK_INCLUDE_FILES +# still runs and overwrites them. Instead, patch the ares_config.h.cmake +# template to append Windows-specific forced defines at the end. This way, +# regardless of what CMake detection produces, the generated ares_config.h +# always has the correct Windows values. +cat >> third_party/cares/cares/src/lib/ares_config.h.cmake <<'CARES_WIN32_FIX' + +/* ---- LocalAI mingw-w64 build fix ---- + * The CMake detection in this old c-ares (pinned by gRPC 1.59.0) doesn't + * reliably detect Windows socket types and functions under mingw-w64's + * UCRT64 headers + CMake 4.x. Force the known-correct values on _WIN32 + * so the build doesn't trip setup_once.h's Error guards or ares_ipv6.h's + * struct redefinitions. Values mirror config-win32.h (the hand-crafted + * config c-ares uses for non-CMake Windows builds). */ +#ifdef _WIN32 +#ifndef HAVE_WINDOWS_H +#define HAVE_WINDOWS_H 1 +#endif +#ifndef HAVE_WINSOCK_H +#define HAVE_WINSOCK_H 1 +#endif +#ifndef HAVE_WINSOCK2_H +#define HAVE_WINSOCK2_H 1 +#endif +#ifndef HAVE_WS2TCPIP_H +#define HAVE_WS2TCPIP_H 1 +#endif +#ifndef HAVE_ASSERT_H +#define HAVE_ASSERT_H 1 +#endif +#ifndef HAVE_ERRNO_H +#define HAVE_ERRNO_H 1 +#endif +#ifndef HAVE_GETOPT_H +#define HAVE_GETOPT_H 1 +#endif +#ifndef HAVE_LIMITS_H +#define HAVE_LIMITS_H 1 +#endif +#ifndef HAVE_PROCESS_H +#define HAVE_PROCESS_H 1 +#endif +#ifndef HAVE_SIGNAL_H +#define HAVE_SIGNAL_H 1 +#endif +#ifndef HAVE_TIME_H +#define HAVE_TIME_H 1 +#endif +#ifndef HAVE_UNISTD_H +#define HAVE_UNISTD_H 1 +#endif +#ifndef HAVE_STDBOOL_H +#define HAVE_STDBOOL_H 1 +#endif +#ifndef HAVE_STDINT_H +#define HAVE_STDINT_H 1 +#endif +#ifndef HAVE_STDLIB_H +#define HAVE_STDLIB_H 1 +#endif +#ifndef HAVE_STRING_H +#define HAVE_STRING_H 1 +#endif +#ifndef HAVE_SOCKLEN_T +#define HAVE_SOCKLEN_T 1 +#endif +#ifndef HAVE_TYPE_SOCKET +#define HAVE_TYPE_SOCKET 1 +#endif +#ifndef HAVE_BOOL_T +#define HAVE_BOOL_T 1 +#endif +#ifndef HAVE_SSIZE_T +#define HAVE_SSIZE_T 1 +#endif +#ifndef HAVE_LONGLONG +#define HAVE_LONGLONG 1 +#endif +#ifndef HAVE_SIG_ATOMIC_T +#define HAVE_SIG_ATOMIC_T 1 +#endif +#ifndef HAVE_STRUCT_ADDRINFO +#define HAVE_STRUCT_ADDRINFO 1 +#endif +#ifndef HAVE_STRUCT_IN6_ADDR +#define HAVE_STRUCT_IN6_ADDR 1 +#endif +#ifndef HAVE_STRUCT_SOCKADDR_IN6 +#define HAVE_STRUCT_SOCKADDR_IN6 1 +#endif +#ifndef HAVE_STRUCT_SOCKADDR_STORAGE +#define HAVE_STRUCT_SOCKADDR_STORAGE 1 +#endif +#ifndef HAVE_STRUCT_TIMEVAL +#define HAVE_STRUCT_TIMEVAL 1 +#endif +#ifndef HAVE_AF_INET6 +#define HAVE_AF_INET6 1 +#endif +#ifndef HAVE_PF_INET6 +#define HAVE_PF_INET6 1 +#endif +#ifndef HAVE_FIONBIO +#define HAVE_FIONBIO 1 +#endif +#ifndef HAVE_CLOSESOCKET +#define HAVE_CLOSESOCKET 1 +#endif +#ifndef HAVE_CONNECT +#define HAVE_CONNECT 1 +#endif +#ifndef HAVE_FREEADDRINFO +#define HAVE_FREEADDRINFO 1 +#endif +#ifndef HAVE_GETADDRINFO +#define HAVE_GETADDRINFO 1 +#endif +#ifndef HAVE_GETHOSTBYADDR +#define HAVE_GETHOSTBYADDR 1 +#endif +#ifndef HAVE_GETHOSTBYNAME +#define HAVE_GETHOSTBYNAME 1 +#endif +#ifndef HAVE_GETHOSTNAME +#define HAVE_GETHOSTNAME 1 +#endif +#ifndef HAVE_GETNAMEINFO +#define HAVE_GETNAMEINFO 1 +#endif +#ifndef HAVE_GETTIMEOFDAY +#define HAVE_GETTIMEOFDAY 1 +#endif +#ifndef HAVE_INET_NTOP +#define HAVE_INET_NTOP 1 +#endif +#ifndef HAVE_INET_PTON +#define HAVE_INET_PTON 1 +#endif +#ifndef HAVE_IOCTLSOCKET +#define HAVE_IOCTLSOCKET 1 +#endif +/* NOTE: HAVE_IOCTL_FIONBIO is deliberately NOT defined. c-ares's + * setsocknonblock() prefers HAVE_IOCTL_FIONBIO (POSIX ioctl(FIONBIO)) over + * HAVE_IOCTLSOCKET_FIONBIO (Windows ioctlsocket(FIONBIO)), and the POSIX + * branch calls ioctl() which does not exist on Windows. config-win32.h — + * the hand-crafted config c-ares uses for non-CMake Windows builds — also + * omits it, defining only HAVE_IOCTLSOCKET_FIONBIO. */ +#ifndef HAVE_IOCTLSOCKET_FIONBIO +#define HAVE_IOCTLSOCKET_FIONBIO 1 +#endif +#ifndef HAVE_RECV +#define HAVE_RECV 1 +#endif +#ifndef HAVE_RECVFROM +#define HAVE_RECVFROM 1 +#endif +#ifndef HAVE_SEND +#define HAVE_SEND 1 +#endif +#ifndef HAVE_SETSOCKOPT +#define HAVE_SETSOCKOPT 1 +#endif +#ifndef HAVE_SOCKET +#define HAVE_SOCKET 1 +#endif +#ifndef HAVE_STRDUP +#define HAVE_STRDUP 1 +#endif +#ifndef HAVE_STRICMP +#define HAVE_STRICMP 1 +#endif +#ifndef HAVE_STRNICMP +#define HAVE_STRNICMP 1 +#endif +#ifndef HAVE_GETENV +#define HAVE_GETENV 1 +#endif +/* HAVE_IOCTL_FIONBIO is intentionally absent and HAVE_IOCTLSOCKET_FIONBIO + * is already defined above; see the NOTE above the first + * HAVE_IOCTLSOCKET_FIONBIO define. */ +#ifndef HAVE_GETADDRINFO_THREADSAFE +#define HAVE_GETADDRINFO_THREADSAFE 1 +#endif +#ifndef HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID 1 +#endif +#ifndef RECV_TYPE_ARG1 +#define RECV_TYPE_ARG1 SOCKET +#endif +#ifndef RECV_TYPE_ARG2 +#define RECV_TYPE_ARG2 char * +#endif +#ifndef RECV_TYPE_ARG3 +#define RECV_TYPE_ARG3 int +#endif +#ifndef RECV_TYPE_ARG4 +#define RECV_TYPE_ARG4 int +#endif +#ifndef RECV_TYPE_RETV +#define RECV_TYPE_RETV int +#endif +#ifndef SEND_QUAL_ARG2 +#define SEND_QUAL_ARG2 +#endif +#ifndef SEND_TYPE_ARG1 +#define SEND_TYPE_ARG1 SOCKET +#endif +#ifndef SEND_TYPE_ARG2 +#define SEND_TYPE_ARG2 char * +#endif +#ifndef SEND_TYPE_ARG3 +#define SEND_TYPE_ARG3 int +#endif +#ifndef SEND_TYPE_ARG4 +#define SEND_TYPE_ARG4 int +#endif +#ifndef SEND_TYPE_RETV +#define SEND_TYPE_RETV int +#endif +#ifndef RECVFROM_TYPE_ARG1 +#define RECVFROM_TYPE_ARG1 SOCKET +#endif +#ifndef RECVFROM_TYPE_ARG2 +#define RECVFROM_TYPE_ARG2 char * +#endif +#ifndef RECVFROM_TYPE_ARG3 +#define RECVFROM_TYPE_ARG3 int +#endif +#ifndef RECVFROM_TYPE_ARG4 +#define RECVFROM_TYPE_ARG4 int +#endif +#ifndef RECVFROM_TYPE_ARG5 +#define RECVFROM_TYPE_ARG5 "struct sockaddr *" +#endif +#ifndef RECVFROM_TYPE_ARG6 +#define RECVFROM_TYPE_ARG6 "int *" +#endif +#ifndef RECVFROM_TYPE_RETV +#define RECVFROM_TYPE_RETV int +#endif +#ifndef RETSIGTYPE +#define RETSIGTYPE void +#endif +#ifndef CARES_TYPEOF_ARES_SOCKLEN_T +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#endif +#ifndef CARES_TYPEOF_ARES_SSIZE_T +#define CARES_TYPEOF_ARES_SSIZE_T "long long" +#endif +#endif /* _WIN32 */ +CARES_WIN32_FIX + +# The c-ares CMake detection above fails every HAVE_* probe (SOCKLEN_T, +# TYPE_SOCKET, STRUCT_ADDRINFO, ...), so its CMakeLists never adds the Winsock +# library to the link line. Without it the static libcares.a references +# WSAStartup / select / ntohl / gethostname / ... which then fail to link with +# "undefined reference to __imp_*". +# +# The static lib target (c-ares_static) links CARES_DEPENDENT_LIBS +# (ws2_32/advapi32/iphlpapi) PUBLICly, so gRPC's own consumers are fine. But +# the diagnostic tools (ahost/adig/acountry) link ${PROJECT_NAME} = "c-ares" +# by name; with CARES_SHARED=OFF there is no "c-ares" target, so CMake treats +# it as a plain library name and resolves it to libcares.a WITHOUT the PUBLIC +# Winsock deps -- the tools then fail to link with "undefined reference to +# __imp_*". The backend image doesn't need these standalone diagnostic +# utilities (gRPC links libcares.a, not the tools), so disable them entirely +# via -DCARES_BUILD_TOOLS=OFF in the cmake configure below. + +# boringssl's ssl_file.cc and ssl_x509.cc use X509_NAME as a C++ type, but +# Windows' wincrypt.h (pulled in transitively by in some +# mingw-w64 header chains) #defines X509_NAME as L"Name", a string literal. +# The macro turns `const X509_NAME *const *a` into `const L"Name" *const *a` +# and the whole file fails to compile. A per-file #undef doesn't work because +# wincrypt.h is re-included transitively through boringssl's own headers. +# Instead, define WIN32_LEAN_AND_MEAN globally via CMAKE_C/CXX_FLAGS so +# windows.h never pulls in wincrypt.h at all, eliminating the collision +# everywhere. This is the standard Windows idiom for avoiding wincrypt.h +# macro pollution (X509_NAME, X509_CERT, etc.) in code that uses OpenSSL/ +# boringssl types. +# +# _WIN32_WINNT=0x600 must ride along on the same flags: abseil's +# win32_waiter.cc guards its Win32Waiter definitions on +# _WIN32_WINNT >= _WIN32_WINNT_VISTA, and win32_waiter.h is included before +# any header sets a default _WIN32_WINNT, so without an explicit define the +# gate stays off. per_thread_sem.cc pulls win32_waiter.h in late (after +# thread_identity.h has already defined the default), so it compiles with the +# gate ON and the synchronization archive ends up referencing Win32Waiter +# while win32_waiter.cc.o is empty -- protoc dies with "undefined reference +# to Win32Waiter::*" that no --start-group can fix, because the symbols are +# not in the archive at all. (gRPC's own CMakeLists only adds +# -D_WIN32_WINNT=0x600 under `if (MINGW)`, which the native cmake sets but +# the MSYS2/Cygwin cmake that CI resolves via PATH does not.) +# +# The flags are spelled out on each cmake configure below instead of through +# CMAKE_ARGS: ${CMAKE_ARGS} is word-split on spaces by bash, so a +# space-containing -D value cannot travel through it. +export CMAKE_ARGS="${CMAKE_ARGS:-}" + +# boringssl's bssl CLI tool uses Winsock functions (socket, connect, select, +# WSAStartup, ...) directly in transport_common.cc, but its CMakeLists only +# links bssl with ssl crypto, omitting ws2_32. A global +# -DCMAKE_EXE_LINKER_FLAGS=-lws2_32 cannot fix this: CMake places those flags +# before the object files on the link line, so ld cannot use the library to +# satisfy the references that only appear later, and bssl still fails with +# "undefined reference to `__imp_select'". Patch the CMakeLists instead. +# gRPC adds the whole boringssl-with-bazel dir via add_subdirectory, so the +# bssl target lives in the ROOT CMakeLists.txt (its binary dir shows as +# third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/), not src/CMakeLists.txt. +# Both files declare the same `target_link_libraries(bssl ssl crypto)` line, so +# rewrite it in place to add ws2_32. Editing the exact declaration (instead of +# appending a guarded block) cannot be skipped by target visibility or block +# evaluation order: when the line is read, ssl and crypto are already linked +# and the bssl target already exists. +for bssl_cmake in \ + third_party/boringssl-with-bazel/CMakeLists.txt \ + third_party/boringssl-with-bazel/src/CMakeLists.txt +do + if [ -f "$bssl_cmake" ]; then + sed -i 's/^target_link_libraries(bssl ssl crypto)/target_link_libraries(bssl ssl crypto ws2_32)/' "$bssl_cmake" + grep -q 'target_link_libraries(bssl ssl crypto ws2_32)' "$bssl_cmake" || \ + { echo "ERROR: failed to patch bssl link in $bssl_cmake" >&2; exit 1; } + fi +done + +# zlib bundled with gRPC 1.59.0 ships win32/zlib1.rc which the mingw-w64 +# binutils 2.46+ windres rejects ("zlib1.rc:7: syntax error" — the VERSIONINFO +# resource format it emits is no longer accepted). The .rc is only used to +# embed Windows version metadata into the DLL; it is unnecessary for a static +# zlib linked into protobuf/gRPC. Overwriting the file with a minimal valid +# resource is format-independent (no need to know the pinned submodule's +# CMakeLists spelling, and the file is unconditionally named win32/zlib1.rc on +# WIN32) and produces no side effects for a static library. +cat > third_party/zlib/win32/zlib1.rc <<'ZLIB_RC_FIX' +1 VERSIONINFO +FILEVERSION 1,3,0,0 +PRODUCTVERSION 1,3,0,0 +BEGIN +END +ZLIB_RC_FIX + +# The grpc Makefile's MSYS/MINGW branches already add OPENSSL_NO_ASM=ON. We +# still pass an explicit generator + compiler: a bare cmake on windows-latest +# would pick the Visual Studio generator, producing an MSVC gRPC that the +# mingw-built grpc-server cannot link against. +cd "$GRPC_DIR/grpc_build" +# gRPC 1.59.0's bundled c-ares still declares cmake_minimum_required < 3.5, +# which cmake >= 4 removed support for. MSYS2 ships cmake 4.x, so configure +# with the minimum policy version c-ares' CMakeLists can still read. +# Static-library resolution on MinGW is order-sensitive: GNU ld scans each +# archive once and a definition that lives before the reference in the link +# line is never revisited. That is why protoc dies with "__imp_Sym*" +# (absl's symbolize needs dbghelp) unless dbghelp is reachable from a cyclic +# group: CMake places CMAKE_EXE_LINKER_FLAGS before the object files, so a +# bare -ldbghelp would be scanned before any reference to it exists and never +# revisited. The --start-group here is what makes it work - it extends to the +# end of the link line (ld closes it implicitly) and makes ld rescan all +# archives cyclically until no new member is pulled, so dbghelp resolves +# regardless of archive order. (The historical "Win32Waiter::*" failures are +# NOT a link-order problem; see the WIN32_LEAN_AND_MEAN/_WIN32_WINNT comment +# above - those symbols were missing from the archive entirely.) The same +# applies to shared links. +cmake -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_C_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0x600" \ + -DCMAKE_CXX_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0x600" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--start-group -ldbghelp" \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--start-group -ldbghelp" \ + -DCMAKE_PREFIX_PATH="${INSTALLED_PACKAGES};${CMAKE_PREFIX_PATH:-}" \ + ${CMAKE_ARGS:-} \ + -DgRPC_INSTALL=ON \ + -DEXECUTABLE_OUTPUT_PATH=../installed_packages/grpc/bin \ + -DLIBRARY_OUTPUT_PATH=../installed_packages/grpc/lib \ + -DgRPC_BUILD_TESTS=OFF \ + -DCARES_BUILD_TOOLS=OFF \ + -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_BUILD_GRPC_CPP_PLUGIN=ON \ + -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=ON \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ + -Dprotobuf_WITH_ZLIB=ON \ + -DZLIB_ROOT=${INSTALLED_PACKAGES} \ + -DRE2_BUILD_TESTING=OFF \ + -DCMAKE_INSTALL_PREFIX=../installed_packages \ + -DOPENSSL_NO_ASM=ON \ + ../grpc_repo/grpc +# Diagnostics: print the bssl link line that CMake generated. bssl uses Winsock +# directly and dies with __imp_select/__imp_* undefined references ~30 minutes +# into the build if ws2_32 is not on its link line, so surface it now while the +# CI log still has the configure context. Verify ws2_32 actually landed. +echo "=== cmake flavor (must be the UCRT64 native build; MSYS2's /usr/bin/cmake +splits the Windows exe names and, unlike the native build, does not set MINGW) ===" +which cmake +cmake --version | head -1 +echo "=== bssl link (expect ws2_32) ===" +if [ -f third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/linkLibs.rsp ]; then + cat third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/linkLibs.rsp +elif [ -f third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/build.make ]; then + grep 'Linking CXX executable bssl' -A1 third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/build.make || true +else + echo "WARNING: bssl build files not generated" +fi +grep -rq 'ws2_32' third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/ && \ + echo "OK: bssl link includes ws2_32" || echo "FAIL: ws2_32 missing from bssl link" +# Diagnostics: protoc's link rule must carry --start-group + dbghelp, otherwise +# the absl static archives (symbolize) fail ~40 minutes in with undefined +# references that CMake cannot reorder away. +echo "=== protoc link (expect --start-group and -ldbghelp) ===" +if [ -f third_party/protobuf/CMakeFiles/protoc.dir/build.make ]; then + grep 'Linking CXX executable' -A1 third_party/protobuf/CMakeFiles/protoc.dir/build.make || true +else + echo "WARNING: protoc build files not generated" +fi +grep -rq -- '--start-group' third_party/protobuf/CMakeFiles/protoc.dir/ && \ + echo "OK: protoc link has --start-group" || echo "FAIL: --start-group missing from protoc link" +cmake --build . -j "$JOBS" +# Diagnostics: the _WIN32_WINNT=0x600 flag above must have compiled +# win32_waiter.cc with its gate open; confirm the archive actually defines +# Win32Waiter (a successful protoc link already implies it, but surface the +# count for the log). +echo "=== absl synchronization archive (expect Win32Waiter T symbols) ===" +# LIBRARY_OUTPUT_PATH=../installed_packages/grpc/lib is relative to the top +# build dir, so the archive lands next to the build tree, not under +# third_party/abseil-cpp. +nm ../installed_packages/grpc/lib/libabsl_synchronization.a 2>/dev/null | \ + grep -c 'Win32Waiter' || echo "WARNING: no Win32Waiter symbols in libabsl_synchronization.a" +cmake --build . --target install + +INSTALLED_PACKAGES="$GRPC_DIR/installed_packages" +ADDED_CMAKE_ARGS="-Dabsl_DIR=${INSTALLED_PACKAGES}/lib/cmake/absl \ + -DProtobuf_DIR=${INSTALLED_PACKAGES}/lib/cmake/protobuf \ + -Dutf8_range_DIR=${INSTALLED_PACKAGES}/lib/cmake/utf8_range \ + -DgRPC_DIR=${INSTALLED_PACKAGES}/lib/cmake/grpc \ + -DZLIB_ROOT=${INSTALLED_PACKAGES} \ + -DZLIB_INCLUDE_DIR=${INSTALLED_PACKAGES}/include \ + -DZLIB_LIBRARY=${INSTALLED_PACKAGES}/lib/libzlibstatic.a \ + -DZLIB_INCLUDE_DIRS=${INSTALLED_PACKAGES}/include \ + -DZLIB_LIBRARIES=${INSTALLED_PACKAGES}/lib/libzlibstatic.a \ + -DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=${INSTALLED_PACKAGES}/include" + +# Restore system zlib for llama.cpp builds (to avoid conflicts) if it was backed up +restore_sys_zlib 2>/dev/null || true +# But we want to force static zlib for llama.cpp - backup again before building variants +backup_sys_zlib 2>/dev/null || true +# find_program(protoc / grpc_cpp_plugin) in the grpc-server CMakeLists searches +# PATH, so the freshly installed protoc + grpc_cpp_plugin must be reachable. +export PATH="${INSTALLED_PACKAGES}/bin:${PATH}" + +# --------------------------------------------------------------------------- +# llama.cpp source at the pinned version, with the LocalAI grpc-server patch +# --------------------------------------------------------------------------- +cd "$ROOT/backend/cpp/llama-cpp" +LLAMA_VERSION=$(grep '^LLAMA_VERSION' Makefile | head -1 | cut -d= -f2 | cut -d'?' -f1 | tr -d ' ') +mkdir -p llama.cpp +cd llama.cpp +git init -q +# Re-runnable like the gRPC clone above; -B resets a stale `build` branch. +git remote remove origin 2>/dev/null || true +git remote add origin https://github.com/ggerganov/llama.cpp +# Shallow-fetch the pinned commit (allowAnySHA1InWant is on for public repos), +# then the pinned submodules shallowly. Equivalent content to `make llama.cpp` +# without downloading the full llama.cpp history on every build. +git fetch origin "$LLAMA_VERSION" --depth 1 +# prepare.sh patches the source tree in place; a previous (re-run) build may +# have left the working tree dirty, which would make the checkout below fail. +# Restore the pristine tree BEFORE checking out (tracked edits, untracked +# .rej/.orig and the generated tools/grpc-server staging dir). +git reset -q --hard +git clean -qfd +mkdir -p tools/grpc-server +git checkout -q -B build "$LLAMA_VERSION" +git submodule update --init --recursive --depth 1 --single-branch +cd "$ROOT/backend/cpp/llama-cpp" +bash prepare.sh + +# --------------------------------------------------------------------------- +# The three cmake variants (flags mirror backend/cpp/llama-cpp/Makefile) +# --------------------------------------------------------------------------- +build_variant() { + local name="$1" + local targets="$2" + shift 2 + # ggml turns ccache on when it finds it, but on MSYS2 ccache wants + # Windows-style USERPROFILE/LOCALAPPDATA the login profile drops - disable + # it (the variant builds are from scratch anyway). + rm -rf "llama.cpp/build-${name}" + # Force zlib to use the gRPC-installed static one to avoid mixing with system DLL import lib + export PKG_CONFIG_PATH="${INSTALLED_PACKAGES}/lib/pkgconfig:${PKG_CONFIG_PATH:-}" + cmake -G "Unix Makefiles" \ + -S llama.cpp -B "llama.cpp/build-${name}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_C_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0xA00" \ + -DCMAKE_CXX_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0xA00" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--start-group -ldbghelp" \ + -DCMAKE_PREFIX_PATH="${INSTALLED_PACKAGES};${CMAKE_PREFIX_PATH:-}" \ + ${CMAKE_ARGS:-} \ + -DGGML_NATIVE=OFF \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_CURL=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DGGML_CCACHE=OFF \ + "$@" \ + $ADDED_CMAKE_ARGS + cmake --build "llama.cpp/build-${name}" --config Release -j "$JOBS" --target $targets +} + +# CPU_ALL_VARIANTS: one grpc-server plus the dlopen-able libggml-cpu-*.dll set. +# ggml/llama go shared so the dynamic CPU backends work; gRPC stays static. +# GGML_VULKAN=ON adds the dlopen-able libggml-vulkan.dll to the same image, so +# the single grpc-server auto-detects a Vulkan device at runtime (ggml-vulkan +# loads vulkan-1.dll dynamically - it is never in the import table) and falls +# back to CPU when the host has no Vulkan driver. No separate variant or launcher +# change is needed: one image works everywhere and GPU offload just works when +# a model requests gpu_layers. +build_variant "cpu-all" "grpc-server ggml" \ + -DBUILD_SHARED_LIBS=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_VULKAN=ON + +# gRPC-RPC server + the ggml-rpc-server companion binary. +build_variant "rpc" "grpc-server ggml-rpc-server" \ + -DGGML_RPC=ON \ + -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off \ + -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off + +# Static fallback, mirrors llama-cpp-fallback. +build_variant "fallback" "grpc-server" \ + -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off \ + -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off + +# --------------------------------------------------------------------------- +# Stage the image contents +# --------------------------------------------------------------------------- +STAGE="$ROOT/build/windows" +rm -rf "$STAGE" +mkdir -p "$STAGE" "$ROOT/backend-images" + +cp llama.cpp/build-cpu-all/bin/grpc-server.exe "$STAGE/llama-cpp-cpu-all.exe" +# ggml's shared backends are loadable DLLs; they go next to the executable so +# the registry finds them without any PATH setup (mirrors run.sh LD_LIBRARY_PATH). +find llama.cpp/build-cpu-all/bin -maxdepth 1 -name '*.dll' -exec cp -v {} "$STAGE/" \; + +cp llama.cpp/build-rpc/bin/grpc-server.exe "$STAGE/llama-cpp-grpc.exe" +cp llama.cpp/build-rpc/bin/ggml-rpc-server.exe "$STAGE/llama-cpp-rpc-server.exe" + +cp llama.cpp/build-fallback/bin/grpc-server.exe "$STAGE/llama-cpp-fallback.exe" + +cd "$ROOT" +# The launcher is the PowerShell script pkg/model/process.go starts on Windows +# (run.ps1, mirroring run.sh); run.sh below is the stub that keeps +# discovery/validation/upgrade uniform with the other platforms. +cp backend/cpp/llama-cpp/run.ps1 "$STAGE/run.ps1" +cp backend/cpp/llama-cpp/run.sh "$STAGE/run.sh" + +# Bundle the runtime DLLs (mingw gcc runtime, openssl when gRPC links it +# dynamically) so the image runs on a host with no MSYS2. ggml's shared +# backends are already in $STAGE; scanning them too (not just the .exe files) +# catches the runtime deps of the loadable modules (ggml-vulkan.dll pulls in +# libgcc/libstdc++/libwinpthread) and lets the vulkan loader be bundled below. +for bin in "$STAGE"/*.exe "$STAGE"/*.dll; do + objdump -p "$bin" | awk '/DLL Name:/ {print $3}' | while read -r dll; do + if [ -f "/ucrt64/bin/$dll" ] && [ ! -e "$STAGE/$dll" ]; then + cp -v "/ucrt64/bin/$dll" "$STAGE/" + fi + done +done + +# ggml-vulkan loads vulkan-1.dll dynamically (volk-style LoadLibrary), so it is +# not in any import table and the scan above cannot see it. Bundle the loader +# from the MSYS2 vulkan-loader package anyway: on a host with no Vulkan driver +# the loader still initializes, ggml-vulkan registers zero devices, and CPU +# inference keeps working - but on a host whose driver does not drop its own +# copy into system32 the GPU would otherwise be invisible. +if [ -f "/ucrt64/bin/vulkan-1.dll" ] && [ ! -e "$STAGE/vulkan-1.dll" ]; then + cp -v "/ucrt64/bin/vulkan-1.dll" "$STAGE/" +fi + +echo "Bundled DLLs:" +ls -la "$STAGE" + +# --------------------------------------------------------------------------- +# Package as an OCI image tar +# --------------------------------------------------------------------------- +# local-ai.exe is only the tool that assembles the OCI tar - the image carries +# $STAGE, not this binary. cmd/local-ai still needs the prereqs of the upstream +# `make build`: pkg/grpc/proto/*.pb.go (gitignored, absent in a fresh checkout; +# regenerate with the protoc built above + the pinned Go plugins, mirroring +# `make protogen-go`) and the embedded react-ui dist (stubbed like lint.yml; +# the real bundle would need node, and this binary never serves it). +GO_PLUGIN_DIR="$(go env GOPATH)/bin" +GO_PLUGIN_DIR_MSYS="$(cygpath -u "$GO_PLUGIN_DIR")" +if [ ! -x "$GO_PLUGIN_DIR_MSYS/protoc-gen-go" ]; then + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 +fi +if [ ! -x "$GO_PLUGIN_DIR_MSYS/protoc-gen-go-grpc" ]; then + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af +fi +mkdir -p pkg/grpc/proto +"$GRPC_DIR/installed_packages/bin/protoc.exe" --experimental_allow_proto3_optional \ + -Ibackend/ \ + --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative \ + --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \ + --plugin=protoc-gen-go="$GO_PLUGIN_DIR/protoc-gen-go.exe" \ + --plugin=protoc-gen-go-grpc="$GO_PLUGIN_DIR/protoc-gen-go-grpc.exe" \ + backend/backend.proto +mkdir -p core/http/react-ui/dist +[ -f core/http/react-ui/dist/index.html ] || touch core/http/react-ui/dist/index.html + +if [ ! -f local-ai.exe ] && [ ! -f local-ai ]; then + CGO_ENABLED=0 go build -o local-ai.exe ./cmd/local-ai +fi +mkdir -p backend-images +./local-ai util create-oci-image \ + build/windows/. \ + --output ./backend-images/llama-cpp.tar \ + --image-name "$IMAGE_NAME" \ + --platform "$PLATFORMARCH" + +rm -rf build/windows \ No newline at end of file diff --git a/scripts/changed-backends.js b/scripts/changed-backends.js index 08a024fc63e8..caa1fdc6410a 100644 --- a/scripts/changed-backends.js +++ b/scripts/changed-backends.js @@ -16,11 +16,12 @@ import { const matrixYml = yaml.load(fs.readFileSync(BACKEND_MATRIX_FILE, "utf8")); const includes = matrixYml.include; const includesDarwin = matrixYml.includeDarwin; +const includesWindows = matrixYml.includeWindows || []; const eventPath = process.env.GITHUB_EVENT_PATH; const event = JSON.parse(fs.readFileSync(eventPath, "utf8")); -const allBackendPaths = getAllBackendPaths(includes, includesDarwin); +const allBackendPaths = getAllBackendPaths(includes, includesDarwin, includesWindows); const token = process.env.GITHUB_TOKEN; const octokit = new Octokit({ auth: token }); @@ -107,6 +108,7 @@ async function getPreviousMatrix(event) { return { include: previous.include || [], includeDarwin: previous.includeDarwin || [], + includeWindows: previous.includeWindows || [], }; } catch (err) { console.log( @@ -267,9 +269,11 @@ function emitFullMatrix() { fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=true\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${multiarch.length > 0 ? 'true' : 'false'}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=true\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-windows=true\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: includesDarwin })}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-windows=${JSON.stringify({ include: includesWindows })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`); emitSinglearchShards(singlearch); for (const backend of allBackendPaths.keys()) { @@ -280,9 +284,10 @@ function emitFullMatrix() { function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { console.log("Changed files:", changedFiles); - const { filtered, filteredDarwin, changedBackends } = filterMatrix({ + const { filtered, filteredDarwin, filteredWindows, changedBackends } = filterMatrix({ includes, includesDarwin, + includesWindows, changedFiles, previousMatrix, protoRevisions, @@ -290,13 +295,16 @@ function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { console.log("Filtered files:", filtered); console.log("Filtered files Darwin:", filteredDarwin); + console.log("Filtered files Windows:", filteredWindows); const { multiarch, singlearch } = splitByArch(filtered); const hasBackendsMultiarch = multiarch.length > 0 ? 'true' : 'false'; const hasBackendsDarwin = filteredDarwin.length > 0 ? 'true' : 'false'; + const hasBackendsWindows = filteredWindows.length > 0 ? 'true' : 'false'; console.log("Has single-arch backends?:", singlearch.length > 0 ? 'true' : 'false'); console.log("Has multi-arch backends?:", hasBackendsMultiarch); console.log("Has Darwin backends?:", hasBackendsDarwin); + console.log("Has Windows backends?:", hasBackendsWindows); const mergeMatrixMultiarch = computeMergeMatrix(multiarch); const hasMergesMultiarch = mergeMatrixMultiarch.include.length > 0 ? 'true' : 'false'; @@ -304,9 +312,11 @@ function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=false\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${hasBackendsMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=${hasBackendsDarwin}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-windows=${hasBackendsWindows}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: filteredDarwin })}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-windows=${JSON.stringify({ include: filteredWindows })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`); emitSinglearchShards(singlearch); diff --git a/scripts/lib/backend-filter.mjs b/scripts/lib/backend-filter.mjs index 9312cbf6d0c0..02239df396fa 100644 --- a/scripts/lib/backend-filter.mjs +++ b/scripts/lib/backend-filter.mjs @@ -129,8 +129,24 @@ export function inferBackendPathDarwin(item) { return `backend/${item.lang}/${item.backend}/`; } +export function inferBackendPathWindows(item) { + // llama-cpp on Windows builds from the C++ sources (via the MSYS2 build + // script, scripts/build/llama-cpp-windows.sh), not a backend/go/llama-cpp + // tree (which doesn't exist). The Windows job is matrix-driven with lang=go + // for runner/toolchain selection, but the source path is C++ — exactly the + // Darwin convention. + if (item.backend === "llama-cpp") { + return `backend/cpp/llama-cpp/`; + } + if (!item.lang) { + return `backend/python/${item.backend}/`; + } + + return `backend/${item.lang}/${item.backend}/`; +} + // Build a deduplicated map of backend name -> path prefix from all matrix entries -export function getAllBackendPaths(includes, includesDarwin) { +export function getAllBackendPaths(includes, includesDarwin, includesWindows = []) { const paths = new Map(); for (const item of includes) { const p = inferBackendPath(item); @@ -144,6 +160,12 @@ export function getAllBackendPaths(includes, includesDarwin) { paths.set(item.backend, p); } } + for (const item of includesWindows) { + const p = inferBackendPathWindows(item); + if (p && !paths.has(item.backend)) { + paths.set(item.backend, p); + } + } return paths; } @@ -173,6 +195,14 @@ const DARWIN_BESPOKE_BUILDERS = new Set([ const isDarwinGenericGo = item => !!item.lang && !DARWIN_BESPOKE_BUILDERS.has(item.backend); +// backend_build_windows.yml builds every Windows entry through +// `make backends/llama-cpp-windows` -> scripts/build/llama-cpp-windows.sh; there +// is no generic Go builder for Windows yet. Keep this set in sync with that +// workflow. +const WINDOWS_BESPOKE_BUILDERS = new Set(["llama-cpp"]); +const isWindowsGenericGo = item => + !!item.lang && !WINDOWS_BESPOKE_BUILDERS.has(item.backend); + const isLinuxPython = item => item.dockerfile.endsWith("python"); // Dockerfile.golang is the only Linux dockerfile that compiles Go (it runs @@ -360,6 +390,7 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file === BACKEND_PROTO_FILE, linux: always, darwin: always, + windows: always, }, { // COPY'd into every Python image (Dockerfile.python) and into every Darwin @@ -367,6 +398,7 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file.startsWith("backend/python/common/"), linux: isLinuxPython, darwin: isDarwinPython, + windows: never, }, { // Compiled into every Go backend binary (see GO_BACKEND_PKG_PREFIXES). @@ -377,11 +409,18 @@ export const SHARED_BUILD_INPUTS = [ // lang=go for runner selection only — their sources are C++ and link no // Go, so isDarwinGenericGo is the correct predicate here, exactly as it // is for scripts/build/golang-darwin.sh. + // + // Windows: llama-cpp's launcher is the run.ps1 script (no Go, no compile) + // and local-ai.exe is rebuilt from the current checkout in the workflow + // itself, so a pkg/ change never invalidates a published Windows backend + // image. A future generic Go Windows builder would switch this to + // isWindowsGenericGo. matches: file => GO_BACKEND_PKG_PREFIXES.some(prefix => file.startsWith(prefix)) && !file.endsWith("_test.go"), linux: isLinuxGo, darwin: isDarwinGenericGo, + windows: never, }, { // The reusable build workflows own build-args, packaging and push for @@ -390,11 +429,19 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file === ".github/workflows/backend_build.yml", linux: always, darwin: never, + windows: never, }, { matches: file => file === ".github/workflows/backend_build_darwin.yml", linux: never, darwin: always, + windows: never, + }, + { + matches: file => file === ".github/workflows/backend_build_windows.yml", + linux: never, + darwin: never, + windows: always, }, { // Decides which GPU libraries end up inside an image. Every Linux image @@ -406,31 +453,43 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file === "scripts/build/package-gpu-libs.sh", linux: always, darwin: never, + windows: never, }, { matches: file => file === "scripts/build/python-darwin.sh", linux: never, darwin: isDarwinPython, + windows: never, }, { matches: file => file === "scripts/build/golang-darwin.sh", linux: never, darwin: isDarwinGenericGo, + windows: never, }, { matches: file => file === "scripts/build/llama-cpp-darwin.sh", linux: never, darwin: item => item.backend === "llama-cpp", + windows: never, }, { matches: file => file === "scripts/build/ds4-darwin.sh", linux: never, darwin: item => item.backend === "ds4", + windows: never, }, { matches: file => file === "scripts/build/privacy-filter-darwin.sh", linux: never, darwin: item => item.backend === "privacy-filter", + windows: never, + }, + { + matches: file => file === "scripts/build/llama-cpp-windows.sh", + linux: never, + darwin: never, + windows: item => item.backend === "llama-cpp", }, { // Catch-all, deliberately last and deliberately broad: anything else under @@ -443,6 +502,7 @@ export const SHARED_BUILD_INPUTS = [ file.startsWith("scripts/build/") && !file.endsWith("_test.sh"), linux: always, darwin: always, + windows: always, }, ]; @@ -478,7 +538,7 @@ export const BACKEND_MATRIX_FILE = ".github/backend-matrix.yml"; // Identity of a matrix entry across revisions. tag-suffix names the image; // per-arch legs of the same image are distinguished by platform-tag. Verified -// unique across all 432 Linux and 57 Darwin entries. +// unique across all 432 Linux, 57 Darwin and 1 Windows entries. export function matrixEntryKey(item) { return JSON.stringify([item["tag-suffix"] || "", item["platform-tag"] || ""]); } @@ -515,12 +575,14 @@ function matchedSharedRules(changedFiles) { return rules; } -// Filter both matrices against a changed-file list. Returns the surviving -// entries plus the set of backend names considered changed, which drives the -// per-backend boolean outputs consumed by test-extra.yml. +// Filter the Linux, Darwin and Windows matrices against a changed-file list. +// Returns the surviving entries plus the set of backend names considered +// changed, which drives the per-backend boolean outputs consumed by +// test-extra.yml. export function filterMatrix({ includes, includesDarwin, + includesWindows, changedFiles, previousMatrix, protoRevisions, @@ -553,6 +615,10 @@ export function filterMatrix({ matrixFileChanged && previousMatrix ? changedEntryKeys(includesDarwin, previousMatrix.includeDarwin || []) : null; + const changedWindowsKeys = + matrixFileChanged && previousMatrix + ? changedEntryKeys(includesWindows || [], previousMatrix.includeWindows || []) + : null; const filtered = includes.filter(item => { const backendPath = inferBackendPath(item); @@ -572,12 +638,21 @@ export function filterMatrix({ return sharedRules.some(rule => rule.darwin(item)); }); + const filteredWindows = (includesWindows || []).filter(item => { + const backendPath = inferBackendPathWindows(item); + if (changedFiles.some(file => file.startsWith(backendPath))) return true; + if (matrixDiffUnavailable) return true; + if (changedWindowsKeys && changedWindowsKeys.has(matrixEntryKey(item))) return true; + return sharedRules.some(rule => rule.windows(item)); + }); + const changedBackends = new Set(); for (const item of filtered) changedBackends.add(item.backend); for (const item of filteredDarwin) changedBackends.add(item.backend); - for (const [backend, pathPrefix] of getAllBackendPaths(includes, includesDarwin)) { + for (const item of filteredWindows) changedBackends.add(item.backend); + for (const [backend, pathPrefix] of getAllBackendPaths(includes, includesDarwin, includesWindows)) { if (backendChanged(backend, pathPrefix, changedFiles)) changedBackends.add(backend); } - return { filtered, filteredDarwin, changedBackends }; + return { filtered, filteredDarwin, filteredWindows, changedBackends }; } diff --git a/scripts/lib/backend-filter_test.mjs b/scripts/lib/backend-filter_test.mjs index 6999c921a127..00e51bc56374 100644 --- a/scripts/lib/backend-filter_test.mjs +++ b/scripts/lib/backend-filter_test.mjs @@ -12,6 +12,7 @@ import { filterMatrix, inferBackendPath, inferBackendPathDarwin, + inferBackendPathWindows, } from "./backend-filter.mjs"; test("kimodocpp maps to its native Go wrapper on Linux and Darwin", () => { @@ -86,8 +87,12 @@ const includesDarwin = [ { backend: "ds4", lang: "go", "tag-suffix": "-metal-darwin-arm64-ds4", "build-type": "metal" }, ]; +const includesWindows = [ + { backend: "llama-cpp", lang: "go", "tag-suffix": "-windows-amd64-llama-cpp" }, +]; + const run = (changedFiles, previousMatrix) => - filterMatrix({ includes, includesDarwin, changedFiles, previousMatrix }); + filterMatrix({ includes, includesDarwin, includesWindows, changedFiles, previousMatrix }); const names = entries => entries.map(e => e.backend).sort(); @@ -201,12 +206,13 @@ test("a bespoke Darwin build script rebuilds only its own backend", () => { }); test("an unclassified scripts/build/ file conservatively rebuilds everything", () => { - const { filtered, filteredDarwin } = run([ + const { filtered, filteredDarwin, filteredWindows } = run([ "scripts/build/package-something-new.sh", ]); assert.equal(filtered.length, includes.length); assert.equal(filteredDarwin.length, includesDarwin.length); + assert.equal(filteredWindows.length, includesWindows.length); }); test("tests for the packaging scripts do not rebuild anything", () => { @@ -262,6 +268,87 @@ test("audio-cpp resolves to its C++ sources on Darwin, not backend/go", () => { ); }); +// --------------------------------------------------------------------------- +// Windows: native windows/amd64 backend images built under MSYS2 +// --------------------------------------------------------------------------- + +test("llama-cpp resolves to its C++ sources on Windows, not backend/go", () => { + // lang=go on a Windows entry only selects the runner and toolchain (the + // MSYS2 build compiles C++); routing it by that field would point at + // backend/go/llama-cpp/, which does not exist. + assert.equal( + inferBackendPathWindows({ + backend: "llama-cpp", + lang: "go", + "tag-suffix": "-windows-amd64-llama-cpp", + }), + "backend/cpp/llama-cpp/", + ); +}); + +test("llama-cpp source changes trigger the Windows build", () => { + const { filtered, filteredDarwin, filteredWindows, changedBackends } = run([ + "backend/cpp/llama-cpp/grpc-server.cpp", + ]); + + assert.deepEqual(names(filtered), ["llama-cpp", "turboquant"]); + assert.deepEqual(names(filteredDarwin), ["llama-cpp"]); + assert.deepEqual(names(filteredWindows), ["llama-cpp"]); + assert.ok(changedBackends.has("llama-cpp")); +}); + +test("backend.proto rebuilds the Windows entry too", () => { + const { filteredWindows } = run(["backend/backend.proto"]); + + assert.equal(filteredWindows.length, includesWindows.length); +}); + +test("the reusable Windows build workflow rebuilds Windows only", () => { + const { filtered, filteredDarwin, filteredWindows } = run([ + ".github/workflows/backend_build_windows.yml", + ]); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.equal(filteredWindows.length, includesWindows.length); +}); + +test("the Windows llama-cpp build script rebuilds only its backend", () => { + const { filtered, filteredDarwin, filteredWindows } = run([ + "scripts/build/llama-cpp-windows.sh", + ]); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.deepEqual(names(filteredWindows), ["llama-cpp"]); +}); + +test("a pkg/ subtree change never rebuilds the Windows entry", () => { + // The launcher is the run.ps1 script (no Go, no compile) and local-ai.exe + // is rebuilt in the workflow itself, so the shipped image does not depend on + // LocalAI pkg/ code. + const { filteredWindows } = run(["pkg/grpc/server.go"]); + + assert.deepEqual(filteredWindows, []); +}); + +test("editing the Windows matrix entry rebuilds it", () => { + const previousMatrix = previousOf( + e => e, + e => e, + e => ({ ...e, "tag-suffix": "-windows-amd64-llama-cpp-old" }) + ); + + const { filtered, filteredDarwin, filteredWindows } = run( + [".github/backend-matrix.yml"], + previousMatrix + ); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.deepEqual(names(filteredWindows), ["llama-cpp"]); +}); + // --------------------------------------------------------------------------- // pkg/ subtrees linked into every Go backend binary // --------------------------------------------------------------------------- @@ -338,9 +425,10 @@ test("tests for the linked pkg subtrees do not rebuild anything", () => { // A matrix file is only ever compared against a previous revision of itself, // so build the "before" side by mutating a copy of the fixtures. -const previousOf = (linuxPatch = e => e, darwinPatch = e => e) => ({ +const previousOf = (linuxPatch = e => e, darwinPatch = e => e, windowsPatch = e => e) => ({ include: includes.map(e => linuxPatch({ ...e })), includeDarwin: includesDarwin.map(e => darwinPatch({ ...e })), + includeWindows: includesWindows.map(e => windowsPatch({ ...e })), }); test("editing an existing entry's base-image rebuilds exactly that entry", () => { @@ -377,15 +465,17 @@ test("a brand-new matrix entry for an existing backend rebuilds only itself", () const previousMatrix = { include: includes.filter(e => e.backend !== "kokoros"), includeDarwin: includesDarwin, + includeWindows: includesWindows, }; - const { filtered, filteredDarwin } = run( + const { filtered, filteredDarwin, filteredWindows } = run( [".github/backend-matrix.yml"], previousMatrix ); assert.deepEqual(names(filtered), ["kokoros"]); assert.deepEqual(filteredDarwin, []); + assert.deepEqual(filteredWindows, []); }); test("editing a Darwin entry rebuilds only that Darwin entry", () => { @@ -426,11 +516,17 @@ test("removing an entry rebuilds nothing", () => { "base-image": "nvidia/cuda:11.8.0-devel-ubuntu22.04", }], includeDarwin: includesDarwin, + includeWindows: includesWindows, }; - const { filtered } = run([".github/backend-matrix.yml"], previousMatrix); + const { filtered, filteredDarwin, filteredWindows } = run( + [".github/backend-matrix.yml"], + previousMatrix + ); assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.deepEqual(filteredWindows, []); }); test("a previous matrix is ignored when backend-matrix.yml did not change", () => { @@ -448,10 +544,11 @@ test("a previous matrix is ignored when backend-matrix.yml did not change", () = test("an unavailable previous matrix conservatively rebuilds everything", () => { // Same posture as changed-backends.js's run-all fallbacks: if we cannot // resolve what the entries used to be, we must not claim nothing changed. - const { filtered, filteredDarwin } = run([".github/backend-matrix.yml"], null); + const { filtered, filteredDarwin, filteredWindows } = run([".github/backend-matrix.yml"], null); assert.equal(filtered.length, includes.length); assert.equal(filteredDarwin.length, includesDarwin.length); + assert.equal(filteredWindows.length, includesWindows.length); }); // --- backend.proto: additive changes must not rebuild the world ------------- diff --git a/tests/e2e/windows/windows_smoke_test.go b/tests/e2e/windows/windows_smoke_test.go new file mode 100644 index 000000000000..d69534324c8d --- /dev/null +++ b/tests/e2e/windows/windows_smoke_test.go @@ -0,0 +1,367 @@ +// Windows host smoke test for the native-windows backend path. +// +// Unlike the in-process e2e suite (tests/e2e/e2e_suite_test.go, which drives +// LocalAI through httpapi.API inside the test binary), this suite boots the +// real local-ai.exe with a --backends-path backend laid out the way the +// gallery contract ships it (run.sh stub for discovery + run.ps1 to launch), +// then exercises the HTTP API and the job-object process reaper from +// pkg/model. It is the regression guard for the Windows-native backend +// support: run.ps1 execution, backend discovery, and kill-on-close tree +// cleanup only ever happen on a live Windows host. +package windows_test + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/phayes/freeport" +) + +const buildTimeout = 15 * time.Minute + +var ( + serverProcess *os.Process + serverStdout io.WriteCloser + serverURL string + serverLogPath string + httpClient *http.Client + + modelsPath string + backendsPath string + tempDir string + + // Binaries under test. Names carry smoke- to avoid colliding with any + // in-tree artifact; both are built into tempDir. + localAIExe string + mockBackendExe string + + serverKilled bool +) + +// procRow is one row of the live Win32_Process snapshot. +type procRow struct { + PID int `json:"ProcessId"` + PPID int `json:"ParentProcessId"` + Name string `json:"Name"` + Cmd string `json:"CommandLine"` +} + +// repoRoot walks up from the package directory to the module root (the first +// ancestor containing go.mod). The test run's working directory is a package +// directory of the LocalAI module, so this terminates quickly. +func repoRoot(start string) string { + dir := start + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + Fail("could not locate the LocalAI module root from " + start) + } + dir = parent + } +} + +// buildBinary builds the package into out, mirroring the Makefile's +// CGO_ENABLED=0 so the build is environment-independent. +func buildBinary(pkg, out string) { + wd, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + ctx, cancel := context.WithTimeout(context.Background(), buildTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "go", "build", "-o", out, pkg) + cmd.Dir = repoRoot(wd) + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + output, err := cmd.CombinedOutput() + if err != nil { + Fail(fmt.Sprintf("go build %s -> %s failed: %s\n%s", pkg, out, err, output)) + } +} + +// resolveBinary returns an existing binary from the given env override, or +// builds the package into tempDir. +func resolveBinary(env, pkg, name string) string { + if p := os.Getenv(env); p != "" { + Expect(p).To(BeAnExistingFile(), env+" must point at an existing executable") + return p + } + out := filepath.Join(tempDir, name) + buildBinary(pkg, out) + return out +} + +// powershellInvoke runs a PowerShell snippet and returns its raw stdout. +// Probes legitimately return non-zero exit codes (e.g. Get-Process for missing +// ids), so the exit status is deliberately ignored and only stdout matters. +func powershellInvoke(script string) string { + cmd := exec.Command("powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script) + out, err := cmd.CombinedOutput() + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "powershell probe (stdout-only) returned %v\n%s\n", err, out) + } + return string(out) +} + +// processSnapshot returns one row per live process, keyed for parent-child +// walks. CIM yields ProcessId/ParentProcessId in one query, which is the +// reliable way to reconstruct backend trees on Windows (no job/toolhelp API +// dependency and no command-line substring heuristic). +func processSnapshot() []procRow { + const query = "@(Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,Name,CommandLine) | ConvertTo-Json -Compress" + var rows []procRow + out := powershellInvoke(query) + if err := json.Unmarshal([]byte(out), &rows); err != nil { + if rows = rows[:0]; len(out) == 0 { + // Empty snapshot (extremely unlikely) — the spec will fail with a + // clear message rather than panic. + _, _ = fmt.Fprintf(GinkgoWriter, "process query produced no output\n") + return nil + } + _, _ = fmt.Fprintf(GinkgoWriter, "process query did not unmarshal: %v\n%s\n", err, out) + } + return rows +} + +// treeForPID collects the process subtree rooted at root (processes whose +// ancestry chains back to it), excluding the root itself. Used to identify the +// wrapper + backend processes owned by the local-ai.exe under test without +// matching against command-line text (which can collide with unrelated +// processes on a developer machine). +func treeForPID(snapshot []procRow, root int) []procRow { + children := make(map[int][]procRow) + for _, r := range snapshot { + children[r.PPID] = append(children[r.PPID], r) + } + var tree []procRow + frontier := []int{root} + seen := map[int]bool{root: true} + for len(frontier) > 0 { + next := []int{} + for _, pid := range frontier { + for _, child := range children[pid] { + if seen[child.PID] { + continue + } + seen[child.PID] = true + tree = append(tree, child) + next = append(next, child.PID) + } + } + frontier = next + } + sort.Slice(tree, func(i, j int) bool { return tree[i].PID < tree[j].PID }) + return tree +} + +// describeV returns a readable listing of a process subtree for failure +// messages. +func describeV(rows []procRow) []string { + out := make([]string, len(rows)) + for i, r := range rows { + out[i] = fmt.Sprintf("pid=%d ppid=%d name=%s cmd=%q", r.PID, r.PPID, r.Name, r.Cmd) + } + return out +} + +// survivingPIDs filters ids down to those the current snapshot still holds. +func survivingPIDs(snapshot []procRow, ids []int) []int { + alive := make(map[int]bool) + for _, r := range snapshot { + alive[r.PID] = true + } + var out []int + for _, id := range ids { + if alive[id] { + out = append(out, id) + } + } + return out +} + +var _ = BeforeSuite(func() { + if runtime.GOOS != "windows" { + Skip("Windows host smoke test only runs on Windows") + } + + httpClient = &http.Client{Timeout: 30 * time.Second} + + var err error + tempDir, err = os.MkdirTemp("", "localai-windows-smoke-*") + Expect(err).ToNot(HaveOccurred()) + + localAIExe = resolveBinary("LOCAL_AI_EXE", "github.com/mudler/LocalAI/cmd/local-ai", "local-ai.exe") + mockBackendExe = resolveBinary("MOCK_BACKEND_EXE", "github.com/mudler/LocalAI/tests/e2e/mock-backend", "mock-backend.exe") + + modelsPath = filepath.Join(tempDir, "models") + backendsPath = filepath.Join(tempDir, "backends") + Expect(os.MkdirAll(modelsPath, 0755)).To(Succeed()) + + // Backend layout mirroring the gallery contract: the binary lives in + // /mock-backend/ with run.sh (kept for uniform discovery — + // ListSystemBackends looks for run.sh) and run.ps1 (what startProcess + // actually executes on a Windows host). + backendDir := filepath.Join(backendsPath, "mock-backend") + Expect(os.MkdirAll(backendDir, 0755)).To(Succeed()) + + Expect(os.WriteFile(filepath.Join(backendDir, "run.sh"), []byte("#!/bin/sh\nexit 0\n"), 0644)).To(Succeed()) + runPS1 := "$ErrorActionPreference = 'Stop'\n" + + "& \"$PSScriptRoot\\mock-backend.exe\" @args\n" + + "exit $LASTEXITCODE\n" + Expect(os.WriteFile(filepath.Join(backendDir, "run.ps1"), []byte(runPS1), 0644)).To(Succeed()) + + backendBinary, err := os.ReadFile(mockBackendExe) + Expect(err).ToNot(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(backendDir, "mock-backend.exe"), backendBinary, 0755)).To(Succeed()) + + modelCfg := "name: mock-model\n" + + "backend: mock-backend\n" + + "parameters:\n" + + " model: mock-model.bin\n" + Expect(os.WriteFile(filepath.Join(modelsPath, "mock-model.yaml"), []byte(modelCfg), 0644)).To(Succeed()) + + port, err := freeport.GetFreePort() + Expect(err).ToNot(HaveOccurred()) + serverURL = fmt.Sprintf("http://127.0.0.1:%d/v1", port) + + // Capture local-ai.exe diagnostics so CI failures are not blind. + serverLogPath = filepath.Join(tempDir, "local-ai.log") + serverStdout, err = os.Create(serverLogPath) + Expect(err).ToNot(HaveOccurred()) + + // Run local-ai.exe from the temp dir so its state (.local_user_id etc.) + // never lands in the repository working copy. + cmd := exec.Command(localAIExe, + "--address", fmt.Sprintf("127.0.0.1:%d", port), + "--models-path", modelsPath, + "--backends-path", backendsPath, + "--log-level", "debug", + ) + cmd.Dir = tempDir + cmd.Stdout = serverStdout + cmd.Stderr = serverStdout + Expect(cmd.Start()).To(Succeed()) + serverProcess = cmd.Process + + // Server must come up and register the discovered mock-backend. + Eventually(func() error { + resp, err := httpClient.Get(serverURL + "/models") + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET /v1/models: HTTP %d", resp.StatusCode) + } + var out struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + for _, m := range out.Data { + if m.ID == "mock-model" { + return nil + } + } + return fmt.Errorf("mock-model not listed yet (got %d models)", len(out.Data)) + }, "2m", "2s").Should(Succeed(), dumpServerLog) +}) + +// dumpServerLog returns the tail of the local-ai.exe log for CI debugging. +func dumpServerLog() string { + if serverLogPath == "" { + return "" + } + data, err := os.ReadFile(serverLogPath) + if err != nil { + return fmt.Sprintf("server log (%s) unreadable: %s", serverLogPath, err) + } + const max = 2000 + if len(data) > max { + data = data[len(data)-max:] + } + return fmt.Sprintf("server log (%s):\n%s", serverLogPath, data) +} + +var _ = Describe("LocalAI Windows smoke", func() { + It("serves a chat completion through the mock backend via run.ps1", func() { + body := strings.NewReader(`{"model":"mock-model","messages":[{"role":"user","content":"hello"}]}`) + resp, err := httpClient.Post(serverURL+"/chat/completions", "application/json", body) + Expect(err).ToNot(HaveOccurred(), dumpServerLog) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(http.StatusOK), dumpServerLog) + + var out struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + Expect(json.NewDecoder(resp.Body).Decode(&out)).To(Succeed(), dumpServerLog) + Expect(out.Choices).ToNot(BeEmpty(), dumpServerLog) + Expect(out.Choices[0].Message.Content).To(ContainSubstring("This is a mocked response.")) + }) + + // Must run last: it hard-kills the server under test. ginkgo runs specs in + // declaration order. + It("reaps the backend tree when local-ai.exe is hard-killed (job object)", func() { + // Identify the wrapper + backend as the process subtree of the server, + // not by command-line text (which matches unrelated processes). + before := processSnapshot() + tree := treeForPID(before, serverProcess.Pid) + Expect(tree).ToNot(BeEmpty(), + "no child processes of local-ai.exe (pid %d) found — the chat spec must have loaded the backend first", serverProcess.Pid) + + var treePIDs []int + for _, r := range tree { + treePIDs = append(treePIDs, r.PID) + } + + // Hard-kill: no graceful shutdown runs, so a surviving backend means + // kill-on-close failed, not a stop path. + Expect(serverProcess.Kill()).To(Succeed()) + _, _ = serverProcess.Wait() + serverKilled = true + + Eventually(func() []int { + return survivingPIDs(processSnapshot(), treePIDs) + }, "30s", "1s").Should(BeEmpty(), + "these processes survived local-ai.exe kill-on-close (job object not applied?):\n%s", + strings.Join(describeV(tree), "\n")) + }) +}) + +var _ = AfterSuite(func() { + if runtime.GOOS != "windows" { + return + } + if serverProcess != nil && !serverKilled { + _ = serverProcess.Kill() + _, _ = serverProcess.Wait() + } + if serverStdout != nil { + _ = serverStdout.Close() + } + if serverLogPath != "" { + _, _ = fmt.Fprintln(GinkgoWriter, dumpServerLog()) + } + if tempDir != "" { + _ = os.RemoveAll(tempDir) + } +}) diff --git a/tests/e2e/windows/windows_suite_test.go b/tests/e2e/windows/windows_suite_test.go new file mode 100644 index 000000000000..266ca03baf2e --- /dev/null +++ b/tests/e2e/windows/windows_suite_test.go @@ -0,0 +1,13 @@ +package windows_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestWindowsSmoke(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "LocalAI Windows smoke test suite") +}