feat(3d): add Pixal3D multiview generation

.github/backend-matrix.yml

@@ -808,6 +808,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-pixal3dcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -1820,6 +1833,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-pixal3dcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -3631,6 +3657,20 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-pixal3dcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -3645,6 +3685,20 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-pixal3dcpp'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   # sam3-cpp
   - build-type: ''
     cuda-major-version: ""
@@ -3997,6 +4051,20 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-pixal3dcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'vulkan'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -4011,6 +4079,20 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-pixal3dcpp'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "0"
@@ -6586,6 +6668,10 @@ includeDarwin:
     tag-suffix: "-metal-darwin-arm64-trellis2cpp"
     build-type: "metal"
     lang: "go"
+  - backend: "pixal3dcpp"
+    tag-suffix: "-metal-darwin-arm64-pixal3dcpp"
+    build-type: "metal"
+    lang: "go"
   - backend: "whisper"
     tag-suffix: "-metal-darwin-arm64-whisper"
     build-type: "metal"

.github/workflows/bump_deps.yaml

@@ -86,6 +86,10 @@ jobs:
             variable: "STABLEDIFFUSION_GGML_VERSION"
             branch: "master"
             file: "backend/go/stablediffusion-ggml/Makefile"
+          - repository: "raven38/pixal3d.cpp"
+            variable: "PIXAL3D_VERSION"
+            branch: "main"
+            file: "backend/go/pixal3dcpp/Makefile"
           - repository: "localai-org/trellis2cpp"
             variable: "TRELLIS2CPP_VERSION"
             branch: "pbr-textures"

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
+.NOTPARALLEL: backends/pixal3dcpp backends/pixal3dcpp-darwin backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
 .NOTPARALLEL: backends/whisper-medusa
 .NOTPARALLEL: backends/funasr
 
@@ -640,6 +640,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/go/rfdetr-cpp
 	$(MAKE) -C backend/go/locate-anything-cpp
 	$(MAKE) -C backend/go/trellis2cpp
+	$(MAKE) -C backend/go/pixal3dcpp
 	$(MAKE) -C backend/go/kimodocpp
 	$(MAKE) -C backend/go/valkey-store
 
@@ -679,6 +680,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/go/vllm-cpp test
 	$(MAKE) -C backend/go/nemo-speech-cpp test
 	$(MAKE) -C backend/go/trellis2cpp test
+	$(MAKE) -C backend/go/pixal3dcpp test
 	$(MAKE) -C backend/go/kimodocpp test
 	$(MAKE) -C backend/go/valkey-store test
 
@@ -1335,6 +1337,7 @@ BACKEND_LOCALAI_PROXY = localai-proxy|golang|.|false|true
 BACKEND_HUGGINGFACE = huggingface|golang|.|false|true
 BACKEND_SILERO_VAD = silero-vad|golang|.|false|true
 BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|true
+BACKEND_PIXAL3DCPP = pixal3dcpp|golang|.|--progress=plain|true
 BACKEND_TRELLIS2CPP = trellis2cpp|golang|.|--progress=plain|true
 BACKEND_KIMODOCPP = kimodocpp|golang|.|--progress=plain|true
 BACKEND_WHISPER = whisper|golang|.|false|true
@@ -1443,9 +1446,10 @@ $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE)))
 $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD)))
 $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TRELLIS2CPP)))
+$(eval $(call generate-docker-build-target,$(BACKEND_PIXAL3DCPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_KIMODOCPP)))
 .NOTPARALLEL: backends/kimodocpp backends/kimodocpp-darwin
-docker-build-backends: docker-build-kimodocpp
+docker-build-backends: docker-build-pixal3dcpp docker-build-kimodocpp
 
 backends/kimodocpp-darwin:
 	BACKEND=kimodocpp BUILD_TYPE=cpu $(MAKE) build-darwin-go-backend
@@ -1701,3 +1705,7 @@ release-launcher-darwin: notarize-launcher-darwin
 
 build-launcher-linux:
 	cd cmd/launcher && go run fyne.io/tools/cmd/fyne@latest package -os linux -icon ../../core/http/static/logo.png --executable $(LAUNCHER_BINARY_NAME)-linux --app-version $(LAUNCHER_APP_VERSION) && mv LocalAI.tar.xz ../../$(LAUNCHER_BINARY_NAME)-linux.tar.xz
+
+backends/pixal3dcpp-darwin:
+	BACKEND=pixal3dcpp BUILD_TYPE=metal $(MAKE) build-darwin-go-backend
+	./local-ai backends install "ocifile://$(abspath ./backend-images/pixal3dcpp.tar)"

README.md

@@ -235,6 +235,10 @@ LocalAI supports **60+ backends** including llama.cpp, vLLM, SGLang, transformer
 
 See the full [Backend & Model Compatibility Table](https://localai.io/model-compatibility/) and [GPU Acceleration guide](https://localai.io/features/gpu-acceleration/).
 
+| Additional native backend | What it does |
+|---------------------------|--------------|
+| [pixal3d.cpp](https://github.com/raven38/pixal3d.cpp) (`pixal3dcpp`) | Four canonical RGBA views to a textured 3D GLB, with an explicit mesh scale. [Setup and API](https://localai.io/features/3d-generation/) |
+
 ### Backends built by us
 
 Most backends wrap a best-in-class upstream engine. A handful of them are native C/C++/GGML engines (no Python at inference) developed and maintained by the LocalAI project itself:

backend/backend.proto

@@ -744,6 +744,8 @@ message Animate3DRequest {
 }
 
 message Generate3DRequest {
+  repeated string images = 10; // Staged canonical views: front, right, back, left
+  double mesh_scale = 11; // Explicit finite positive Pixal3D object scale
   string src = 1;          // Path to the staged conditioning image (3D generation is image-conditioned)
   string dst = 2;          // Output path for the generated binary glTF (.glb) asset
   int32 seed = 3;          // <=0 lets the backend pick a random seed

backend/go/pixal3dcpp/Makefile

@@ -0,0 +1,53 @@
+.DEFAULT_GOAL := all
+PIXAL3D_REPO?=https://github.com/raven38/pixal3d.cpp
+PIXAL3D_VERSION?=d1b4926452f9e09702b891db5f05acc845e153ed
+GOCMD?=go
+GO_TAGS?=
+BUILD_TYPE?=
+JOBS?=2
+CMAKE_ARGS?=
+CMAKE_ARGS+=-DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DGGML_NATIVE=OFF -DTRELLIS_WEBP=OFF
+# A portable CPU fallback avoids instructions unavailable on the destination.
+CMAKE_ARGS+=-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_AVX512=OFF -DGGML_FMA=OFF -DGGML_F16C=OFF -DGGML_BMI2=OFF
+ifeq ($(BUILD_TYPE),cublas)
+CMAKE_ARGS+=-DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=all-major
+else ifeq ($(BUILD_TYPE),vulkan)
+CMAKE_ARGS+=-DGGML_VULKAN=ON
+else ifeq ($(BUILD_TYPE),metal)
+CMAKE_ARGS+=-DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON
+else
+CMAKE_ARGS+=-DGGML_METAL=OFF
+endif
+
+sources/pixal3d:
+	mkdir -p sources/pixal3d
+	cd sources/pixal3d && git init -q && git remote add origin $(PIXAL3D_REPO) && \
+		git fetch --depth 1 origin $(PIXAL3D_VERSION) && git checkout FETCH_HEAD && \
+		git submodule update --init --recursive --depth 1
+
+native/trellis-cli: sources/pixal3d
+	cmake -S sources/pixal3d -B native $(CMAKE_ARGS)
+	cmake --build native --target trellis-cli --config Release -j$(JOBS)
+
+pixal3dcpp: main.go pixal3d.go
+	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o pixal3dcpp ./
+
+package: pixal3dcpp native/trellis-cli
+	bash package.sh
+
+all: package
+build: package
+
+test:
+	$(GOCMD) test -v ./...
+
+purge:
+	rm -rf native sources
+clean: purge
+	rm -rf pixal3dcpp package
+.PHONY: all build package test purge clean
+
+ifeq ($(shell uname -s),Darwin)
+# Avoid a runtime dependency on Homebrew libomp in portable backend packages.
+CMAKE_ARGS+=-DGGML_OPENMP=OFF
+endif

backend/go/pixal3dcpp/main.go

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: MIT
+package main
+
+import (
+	"flag"
+	grpc "github.com/mudler/LocalAI/pkg/grpc"
+)
+
+func main() {
+	addr := flag.String("addr", "localhost:50051", "gRPC listen address")
+	flag.Parse()
+	if err := grpc.StartServer(*addr, &Pixal3D{}); err != nil {
+		panic(err)
+	}
+}

backend/go/pixal3dcpp/package.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+CURDIR=$(cd "$(dirname "$0")" && pwd)
+mkdir -p "$CURDIR/package/lib"
+cp "$CURDIR/sources/pixal3d/LICENSE" "$CURDIR/package/LICENSE.pixal3d"
+cp "$CURDIR/sources/pixal3d/thirdparty/ggml/LICENSE" "$CURDIR/package/LICENSE.ggml"
+cp "$CURDIR/pixal3dcpp" "$CURDIR/native/trellis-cli" "$CURDIR/run.sh" "$CURDIR/package/"
+find "$CURDIR/native" \( -name 'libggml*.so*' -o -name 'libggml*.dylib' \) -exec cp -a {} "$CURDIR/package/lib/" \;
+if [ "$(uname -s)" = Darwin ]; then
+    # Upstream uses Linux $ORIGIN rpaths; run.sh also exports DYLD_LIBRARY_PATH.
+    install_name_tool -add_rpath '@executable_path/lib' "$CURDIR/package/trellis-cli"
+else
+    # Include the CLI's transitive runtime dependencies. Driver libraries stay
+    # host-provided; package-gpu-libs owns the accelerator runtime inventory.
+    ldd "$CURDIR/native/trellis-cli" | awk '/=> \/|^[[:space:]]*\// {for (i=1;i<=NF;i++) if ($i ~ /^\//) print $i}' | while read -r library; do
+        case "$library" in
+            */libcuda.so*|*/libnvidia-*) continue ;;
+        esac
+        cp -Lf "$library" "$CURDIR/package/lib/"
+    done
+    for loader in /lib64/ld-linux-x86-64.so.2 /lib/ld-linux-aarch64.so.1; do
+        if [ -f "$loader" ]; then cp -Lf "$loader" "$CURDIR/package/lib/ld.so"; break; fi
+    done
+fi
+source "$CURDIR/../../../scripts/build/package-gpu-libs.sh" "$CURDIR/package/lib"
+package_gpu_libs