diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 8649209cf706..00b0f45f1395 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -808,6 +808,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "8" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-pixal3dcpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "8" @@ -1820,6 +1833,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-13-pixal3dcpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -3631,6 +3657,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-pixal3dcpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: '' cuda-major-version: "" cuda-minor-version: "" @@ -3645,6 +3685,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-pixal3dcpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' # sam3-cpp - build-type: '' cuda-major-version: "" @@ -3997,6 +4051,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-pixal3dcpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'vulkan' cuda-major-version: "" cuda-minor-version: "" @@ -4011,6 +4079,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-pixal3dcpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "0" @@ -6586,6 +6668,10 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-trellis2cpp" build-type: "metal" lang: "go" + - backend: "pixal3dcpp" + tag-suffix: "-metal-darwin-arm64-pixal3dcpp" + build-type: "metal" + lang: "go" - backend: "whisper" tag-suffix: "-metal-darwin-arm64-whisper" build-type: "metal" diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index e8569473a2be..9e1993ceeb5a 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -86,6 +86,10 @@ jobs: variable: "STABLEDIFFUSION_GGML_VERSION" branch: "master" file: "backend/go/stablediffusion-ggml/Makefile" + - repository: "raven38/pixal3d.cpp" + variable: "PIXAL3D_VERSION" + branch: "main" + file: "backend/go/pixal3dcpp/Makefile" - repository: "localai-org/trellis2cpp" variable: "TRELLIS2CPP_VERSION" branch: "pbr-textures" diff --git a/Makefile b/Makefile index 7c05fda6dd3d..0d5c13d68fb6 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin +.NOTPARALLEL: backends/pixal3dcpp backends/pixal3dcpp-darwin backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin .NOTPARALLEL: backends/whisper-medusa .NOTPARALLEL: backends/funasr @@ -640,6 +640,7 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/go/rfdetr-cpp $(MAKE) -C backend/go/locate-anything-cpp $(MAKE) -C backend/go/trellis2cpp + $(MAKE) -C backend/go/pixal3dcpp $(MAKE) -C backend/go/kimodocpp $(MAKE) -C backend/go/valkey-store @@ -679,6 +680,7 @@ test-extra: prepare-test-extra $(MAKE) -C backend/go/vllm-cpp test $(MAKE) -C backend/go/nemo-speech-cpp test $(MAKE) -C backend/go/trellis2cpp test + $(MAKE) -C backend/go/pixal3dcpp test $(MAKE) -C backend/go/kimodocpp test $(MAKE) -C backend/go/valkey-store test @@ -1335,6 +1337,7 @@ BACKEND_LOCALAI_PROXY = localai-proxy|golang|.|false|true BACKEND_HUGGINGFACE = huggingface|golang|.|false|true BACKEND_SILERO_VAD = silero-vad|golang|.|false|true BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|true +BACKEND_PIXAL3DCPP = pixal3dcpp|golang|.|--progress=plain|true BACKEND_TRELLIS2CPP = trellis2cpp|golang|.|--progress=plain|true BACKEND_KIMODOCPP = kimodocpp|golang|.|--progress=plain|true BACKEND_WHISPER = whisper|golang|.|false|true @@ -1443,9 +1446,10 @@ $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE))) $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD))) $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML))) $(eval $(call generate-docker-build-target,$(BACKEND_TRELLIS2CPP))) +$(eval $(call generate-docker-build-target,$(BACKEND_PIXAL3DCPP))) $(eval $(call generate-docker-build-target,$(BACKEND_KIMODOCPP))) .NOTPARALLEL: backends/kimodocpp backends/kimodocpp-darwin -docker-build-backends: docker-build-kimodocpp +docker-build-backends: docker-build-pixal3dcpp docker-build-kimodocpp backends/kimodocpp-darwin: BACKEND=kimodocpp BUILD_TYPE=cpu $(MAKE) build-darwin-go-backend @@ -1701,3 +1705,7 @@ release-launcher-darwin: notarize-launcher-darwin build-launcher-linux: cd cmd/launcher && go run fyne.io/tools/cmd/fyne@latest package -os linux -icon ../../core/http/static/logo.png --executable $(LAUNCHER_BINARY_NAME)-linux --app-version $(LAUNCHER_APP_VERSION) && mv LocalAI.tar.xz ../../$(LAUNCHER_BINARY_NAME)-linux.tar.xz + +backends/pixal3dcpp-darwin: + BACKEND=pixal3dcpp BUILD_TYPE=metal $(MAKE) build-darwin-go-backend + ./local-ai backends install "ocifile://$(abspath ./backend-images/pixal3dcpp.tar)" diff --git a/README.md b/README.md index f1c6c62900ab..a5ee80c28dd2 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,10 @@ LocalAI supports **60+ backends** including llama.cpp, vLLM, SGLang, transformer See the full [Backend & Model Compatibility Table](https://localai.io/model-compatibility/) and [GPU Acceleration guide](https://localai.io/features/gpu-acceleration/). +| Additional native backend | What it does | +|---------------------------|--------------| +| [pixal3d.cpp](https://github.com/raven38/pixal3d.cpp) (`pixal3dcpp`) | Four canonical RGBA views to a textured 3D GLB, with an explicit mesh scale. [Setup and API](https://localai.io/features/3d-generation/) | + ### Backends built by us Most backends wrap a best-in-class upstream engine. A handful of them are native C/C++/GGML engines (no Python at inference) developed and maintained by the LocalAI project itself: diff --git a/backend/backend.proto b/backend/backend.proto index 614f006e9fe1..fa2b355e23b4 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -744,6 +744,8 @@ message Animate3DRequest { } message Generate3DRequest { + repeated string images = 10; // Staged canonical views: front, right, back, left + double mesh_scale = 11; // Explicit finite positive Pixal3D object scale string src = 1; // Path to the staged conditioning image (3D generation is image-conditioned) string dst = 2; // Output path for the generated binary glTF (.glb) asset int32 seed = 3; // <=0 lets the backend pick a random seed diff --git a/backend/go/pixal3dcpp/Makefile b/backend/go/pixal3dcpp/Makefile new file mode 100644 index 000000000000..0356462519ff --- /dev/null +++ b/backend/go/pixal3dcpp/Makefile @@ -0,0 +1,53 @@ +.DEFAULT_GOAL := all +PIXAL3D_REPO?=https://github.com/raven38/pixal3d.cpp +PIXAL3D_VERSION?=d1b4926452f9e09702b891db5f05acc845e153ed +GOCMD?=go +GO_TAGS?= +BUILD_TYPE?= +JOBS?=2 +CMAKE_ARGS?= +CMAKE_ARGS+=-DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DGGML_NATIVE=OFF -DTRELLIS_WEBP=OFF +# A portable CPU fallback avoids instructions unavailable on the destination. +CMAKE_ARGS+=-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_AVX512=OFF -DGGML_FMA=OFF -DGGML_F16C=OFF -DGGML_BMI2=OFF +ifeq ($(BUILD_TYPE),cublas) +CMAKE_ARGS+=-DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=all-major +else ifeq ($(BUILD_TYPE),vulkan) +CMAKE_ARGS+=-DGGML_VULKAN=ON +else ifeq ($(BUILD_TYPE),metal) +CMAKE_ARGS+=-DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON +else +CMAKE_ARGS+=-DGGML_METAL=OFF +endif + +sources/pixal3d: + mkdir -p sources/pixal3d + cd sources/pixal3d && git init -q && git remote add origin $(PIXAL3D_REPO) && \ + git fetch --depth 1 origin $(PIXAL3D_VERSION) && git checkout FETCH_HEAD && \ + git submodule update --init --recursive --depth 1 + +native/trellis-cli: sources/pixal3d + cmake -S sources/pixal3d -B native $(CMAKE_ARGS) + cmake --build native --target trellis-cli --config Release -j$(JOBS) + +pixal3dcpp: main.go pixal3d.go + CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o pixal3dcpp ./ + +package: pixal3dcpp native/trellis-cli + bash package.sh + +all: package +build: package + +test: + $(GOCMD) test -v ./... + +purge: + rm -rf native sources +clean: purge + rm -rf pixal3dcpp package +.PHONY: all build package test purge clean + +ifeq ($(shell uname -s),Darwin) +# Avoid a runtime dependency on Homebrew libomp in portable backend packages. +CMAKE_ARGS+=-DGGML_OPENMP=OFF +endif diff --git a/backend/go/pixal3dcpp/main.go b/backend/go/pixal3dcpp/main.go new file mode 100644 index 000000000000..5a251fbbf990 --- /dev/null +++ b/backend/go/pixal3dcpp/main.go @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT +package main + +import ( + "flag" + grpc "github.com/mudler/LocalAI/pkg/grpc" +) + +func main() { + addr := flag.String("addr", "localhost:50051", "gRPC listen address") + flag.Parse() + if err := grpc.StartServer(*addr, &Pixal3D{}); err != nil { + panic(err) + } +} diff --git a/backend/go/pixal3dcpp/package.sh b/backend/go/pixal3dcpp/package.sh new file mode 100755 index 000000000000..fd4597486b31 --- /dev/null +++ b/backend/go/pixal3dcpp/package.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail +CURDIR=$(cd "$(dirname "$0")" && pwd) +mkdir -p "$CURDIR/package/lib" +cp "$CURDIR/sources/pixal3d/LICENSE" "$CURDIR/package/LICENSE.pixal3d" +cp "$CURDIR/sources/pixal3d/thirdparty/ggml/LICENSE" "$CURDIR/package/LICENSE.ggml" +cp "$CURDIR/pixal3dcpp" "$CURDIR/native/trellis-cli" "$CURDIR/run.sh" "$CURDIR/package/" +find "$CURDIR/native" \( -name 'libggml*.so*' -o -name 'libggml*.dylib' \) -exec cp -a {} "$CURDIR/package/lib/" \; +if [ "$(uname -s)" = Darwin ]; then + # Upstream uses Linux $ORIGIN rpaths; run.sh also exports DYLD_LIBRARY_PATH. + install_name_tool -add_rpath '@executable_path/lib' "$CURDIR/package/trellis-cli" +else + # Include the CLI's transitive runtime dependencies. Driver libraries stay + # host-provided; package-gpu-libs owns the accelerator runtime inventory. + ldd "$CURDIR/native/trellis-cli" | awk '/=> \/|^[[:space:]]*\// {for (i=1;i<=NF;i++) if ($i ~ /^\//) print $i}' | while read -r library; do + case "$library" in + */libcuda.so*|*/libnvidia-*) continue ;; + esac + cp -Lf "$library" "$CURDIR/package/lib/" + done + for loader in /lib64/ld-linux-x86-64.so.2 /lib/ld-linux-aarch64.so.1; do + if [ -f "$loader" ]; then cp -Lf "$loader" "$CURDIR/package/lib/ld.so"; break; fi + done +fi +source "$CURDIR/../../../scripts/build/package-gpu-libs.sh" "$CURDIR/package/lib" +package_gpu_libs diff --git a/backend/go/pixal3dcpp/pixal3d.go b/backend/go/pixal3dcpp/pixal3d.go new file mode 100644 index 000000000000..ec081a8d3ec8 --- /dev/null +++ b/backend/go/pixal3dcpp/pixal3d.go @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: MIT +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync" + + "github.com/mudler/LocalAI/pkg/grpc/base" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/pixal3d" + "github.com/mudler/xlog" +) + +var modelFiles = []string{"dinov3.gguf", "pixal3d_naf.gguf", "pixal3d_ss_flow_mv.gguf", "ss_dec.gguf", "pixal3d_shape_flow_512_mv.gguf", "shape_dec.gguf", "pixal3d_shape_flow_1024_mv.gguf", "pixal3d_tex_flow_1024_mv.gguf", "tex_dec.gguf"} + +// Pixal3D registers a validated model directory in the normal backend lifecycle. +// This adapter invokes the native CLI, so each request reloads +// weights and releases native memory when the child exits. +type Pixal3D struct { + base.Base + mu sync.Mutex + models string + cli string + cancel context.CancelFunc +} + +func validateModels(dir string) error { + f, err := os.Open(filepath.Join(dir, "pixal3d-models.json")) // #nosec G304 -- fixed manifest name in the administrator-configured model directory + if err != nil { + return fmt.Errorf("Pixal3D model manifest: %w", err) + } + var manifest struct { + Schema int `json:"schema_version"` + Family string `json:"model_family"` + Files []struct { + Name string `json:"name"` + Size int64 `json:"size_bytes"` + } `json:"files"` + } + err = json.NewDecoder(io.LimitReader(f, 1<<20)).Decode(&manifest) + closeErr := f.Close() + if closeErr != nil { + return fmt.Errorf("close Pixal3D manifest: %w", closeErr) + } + if err != nil || manifest.Schema != 1 || (manifest.Family != "" && manifest.Family != "mv") { + return fmt.Errorf("invalid Pixal3D MV model manifest") + } + sizes := map[string]int64{} + for _, file := range manifest.Files { + if _, exists := sizes[file.Name]; exists { + return fmt.Errorf("duplicate manifest component %s", file.Name) + } + sizes[file.Name] = file.Size + } + for _, name := range modelFiles { + f, err := os.Open(filepath.Join(dir, name)) // #nosec G304 -- name comes from the fixed modelFiles list, not manifest input + if err != nil { + return fmt.Errorf("not a Pixal3D MV model set: %s: %w", name, err) + } + stat, err := f.Stat() + var magic [4]byte + _, readErr := io.ReadFull(f, magic[:]) + closeErr := f.Close() + if closeErr != nil { + return fmt.Errorf("close Pixal3D component %s: %w", name, closeErr) + } + if err != nil || !stat.Mode().IsRegular() || readErr != nil || string(magic[:]) != "GGUF" || sizes[name] <= 0 || stat.Size() != sizes[name] { + return fmt.Errorf("invalid Pixal3D component %s (check manifest and file size)", name) + } + } + return nil +} + +func (p *Pixal3D) Load(opts *pb.ModelOptions) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.cancel != nil { + return fmt.Errorf("cannot load Pixal3D while generation is running") + } + dir := opts.ModelFile + if !filepath.IsAbs(dir) { + dir = filepath.Join(opts.ModelPath, dir) + } + if err := validateModels(dir); err != nil { + return err + } + exe, err := os.Executable() + if err != nil { + return err + } + cli := filepath.Join(filepath.Dir(exe), "trellis-cli") + if st, err := os.Stat(cli); err != nil || st.IsDir() { + return fmt.Errorf("packaged Pixal3D trellis-cli is missing") + } + p.models, p.cli = dir, cli + return nil +} + +func (p *Pixal3D) Free() error { + p.mu.Lock() + defer p.mu.Unlock() + if p.cancel != nil { + p.cancel() + } + p.models = "" + return nil +} + +func commandArgs(models, views, output string, req *pb.Generate3DRequest) ([]string, error) { + if err := pixal3d.ValidateRequest(req.Src, req.Images, req.MeshScale, req.Quality, req.Background); err != nil { + return nil, err + } + if req.Step != 0 || req.CfgScale != 0 || req.TextureSteps != 0 || req.Seed != 0 || len(req.Params) != 0 { + return nil, fmt.Errorf("Pixal3D sampling overrides and params are unsupported") + } + if req.Dst == "" { + return nil, fmt.Errorf("output path is required") + } + return []string{"--views", views, "--models", models, "--res", "1024", "--mesh-scale", strconv.FormatFloat(req.MeshScale, 'g', -1, 64), "--output", output}, nil +} + +func (p *Pixal3D) Generate3D(req *pb.Generate3DRequest) error { + p.mu.Lock() + if p.models == "" { + p.mu.Unlock() + return fmt.Errorf("Pixal3D model is not loaded") + } + if p.cancel != nil { + p.mu.Unlock() + return fmt.Errorf("Pixal3D generation already running") + } + models, cli := p.models, p.cli + ctx, cancel := context.WithCancel(context.Background()) + p.cancel = cancel + p.mu.Unlock() + defer func() { cancel(); p.mu.Lock(); p.cancel = nil; p.mu.Unlock() }() + dir, err := os.MkdirTemp("", "pixal3d-views-") + if err != nil { + return err + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + xlog.Warn("Failed to remove Pixal3D staged views", "error", err) + } + }() + // The native CLI writes texture and mesh sidecars beside its GLB. + outputDir, err := os.MkdirTemp("", "pixal3d-output-") + if err != nil { + return err + } + defer func() { + if err := os.RemoveAll(outputDir); err != nil { + xlog.Warn("Failed to remove Pixal3D staged output", "error", err) + } + }() + output := filepath.Join(outputDir, "output.glb") + args, err := commandArgs(models, dir, output, req) + if err != nil { + return err + } + names := []string{"0_front.png", "1_right.png", "2_back.png", "3_left.png"} + for i, path := range req.Images { + f, err := os.Open(path) // #nosec G304 -- images are local files staged by the server, not raw HTTP input + if err != nil { + return err + } + data, err := io.ReadAll(io.LimitReader(f, pixal3d.MaxInputBytes+1)) + closeErr := f.Close() + if closeErr != nil { + return closeErr + } + if err != nil { + return err + } + if err = pixal3d.ValidatePNG(data); err != nil { + return fmt.Errorf("view %d: %w", i, err) + } + if err = os.WriteFile(filepath.Join(dir, names[i]), data, 0600); err != nil { + return err + } + } + loader := filepath.Join(filepath.Dir(cli), "lib", "ld.so") + if _, err := os.Stat(loader); err == nil { + args = append([]string{cli}, args...) + cli = loader + } + cmd := exec.CommandContext(ctx, cli, args...) // #nosec G204 -- Load resolves the packaged CLI and loader beside this executable; arguments never pass through a shell + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("Pixal3D generation failed: %w", err) + } + f, err := os.Open(output) // #nosec G304 -- fixed output.glb name inside the private directory created above + if err != nil { + return err + } + defer func() { + if err := f.Close(); err != nil { + xlog.Warn("Failed to close Pixal3D output", "error", err) + } + }() + var header [4]byte + if _, err := io.ReadFull(f, header[:]); err != nil || string(header[:]) != "glTF" { + return fmt.Errorf("Pixal3D did not produce a GLB") + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + // Copy because the destination may be on a different filesystem. + dst, err := os.Create(req.Dst) + if err != nil { + return err + } + _, copyErr := io.Copy(dst, f) + closeErr := dst.Close() + if copyErr != nil || closeErr != nil { + if err := os.Remove(req.Dst); err != nil { + xlog.Warn("Failed to remove incomplete Pixal3D output", "error", err) + } + if copyErr != nil { + return copyErr + } + return closeErr + } + return nil +} diff --git a/backend/go/pixal3dcpp/pixal3d_test.go b/backend/go/pixal3dcpp/pixal3d_test.go new file mode 100644 index 000000000000..5e529f6d5668 --- /dev/null +++ b/backend/go/pixal3dcpp/pixal3d_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "bytes" + "fmt" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "image" + "image/png" + "os" + "path/filepath" + "testing" +) + +func TestPixal3D(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Pixal3D backend") } + +var _ = Describe("Pixal3D backend", func() { + It("rejects missing or TRELLIS-only weights", func() { + p := &Pixal3D{} + Expect(p.Load(&pb.ModelOptions{ModelFile: GinkgoT().TempDir()})).NotTo(Succeed()) + }) + It("uses the canonical multiview CLI without shell interpretation", func() { + req := &pb.Generate3DRequest{Images: []string{"a", "b", "c", "d"}, MeshScale: 0.8, Dst: "/tmp/out; touch bad.glb"} + args, err := commandArgs("/models", "/views", req.Dst, req) + Expect(err).NotTo(HaveOccurred()) + Expect(args).To(Equal([]string{"--views", "/views", "--models", "/models", "--res", "1024", "--mesh-scale", "0.8", "--output", req.Dst})) + req.Src = "single.png" + _, err = commandArgs("/models", "/views", req.Dst, req) + Expect(err).To(HaveOccurred()) + }) + It("requires every MV model component", func() { + dir := GinkgoT().TempDir() + for _, name := range modelFiles { + Expect(os.WriteFile(filepath.Join(dir, name), []byte("GGUF"), 0600)).To(Succeed()) + } + Expect(validateModels(dir)).NotTo(Succeed()) + manifest := `{"schema_version":1,"model_family":"mv","files":[` + for i, name := range modelFiles { + if i > 0 { + manifest += "," + } + manifest += fmt.Sprintf(`{"name":%q,"size_bytes":4}`, name) + } + manifest += "]}" + Expect(os.WriteFile(filepath.Join(dir, "pixal3d-models.json"), []byte(manifest), 0600)).To(Succeed()) + Expect(validateModels(dir)).To(Succeed()) + Expect(os.Remove(filepath.Join(dir, "pixal3d_naf.gguf"))).To(Succeed()) + Expect(validateModels(dir)).NotTo(Succeed()) + }) +}) + +var _ = Describe("Pixal3D native process", func() { + DescribeTable("checks the process status and GLB output, then removes all staged files", func(script string, wantError bool) { + dir := GinkgoT().TempDir() + var data bytes.Buffer + Expect(png.Encode(&data, image.NewNRGBA(image.Rect(0, 0, 2, 2)))).To(Succeed()) + views := []string{} + for i := range 4 { + path := filepath.Join(dir, fmt.Sprintf("view-%d.png", i)) + Expect(os.WriteFile(path, data.Bytes(), 0600)).To(Succeed()) + views = append(views, path) + } + cli := filepath.Join(dir, "trellis-cli") + // Native generation emits sidecars even if the process later fails. + script = `#!/bin/sh +printf '%s' "$2" > "$PIXAL_CAPTURE" +for arg do dst=$arg; done +printf '%s' "$dst" > "$PIXAL_CAPTURE.output" +printf texture > "${dst%.glb}_base.png" +printf mesh > "${dst%.glb}.ply" +` + script + GinkgoT().Setenv("PIXAL_CAPTURE", filepath.Join(dir, "capture")) + Expect(os.WriteFile(cli, []byte(script), 0700)).To(Succeed()) + p := &Pixal3D{models: dir, cli: cli} + destination := filepath.Join(dir, "destination") + Expect(os.Mkdir(destination, 0700)).To(Succeed()) + err := p.Generate3D(&pb.Generate3DRequest{Images: views, MeshScale: 1, Dst: filepath.Join(destination, "out.glb")}) + Expect(err != nil).To(Equal(wantError)) + staged, readErr := os.ReadFile(filepath.Join(dir, "capture")) + Expect(readErr).NotTo(HaveOccurred()) + _, statErr := os.Stat(string(staged)) + Expect(os.IsNotExist(statErr)).To(BeTrue()) + entries, readErr := os.ReadDir(destination) + Expect(readErr).NotTo(HaveOccurred()) + if wantError { + Expect(entries).To(BeEmpty()) + } else { + Expect(entries).To(HaveLen(1)) + Expect(entries[0].Name()).To(Equal("out.glb")) + output, readErr := os.ReadFile(filepath.Join(destination, "out.glb")) + Expect(readErr).NotTo(HaveOccurred()) + Expect(string(output)).To(Equal("glTFpayload")) + } + stagedOutput, readErr := os.ReadFile(filepath.Join(dir, "capture.output")) + Expect(readErr).NotTo(HaveOccurred()) + _, statErr = os.Stat(filepath.Dir(string(stagedOutput))) + Expect(os.IsNotExist(statErr)).To(BeTrue()) + Expect(p.Free()).To(Succeed()) + Expect(p.models).To(BeEmpty()) + }, + Entry("valid output", "test -f \"$2/0_front.png\" && test -f \"$2/3_left.png\" || exit 9\nprintf glTFpayload > \"$dst\"\n", false), + Entry("process failure", "printf glTFpartial > \"$dst\"\nexit 7\n", true), + Entry("bad output", "for arg do dst=$arg; done\nprintf nope > \"$dst\"\n", true), + ) +}) diff --git a/backend/go/pixal3dcpp/run.sh b/backend/go/pixal3dcpp/run.sh new file mode 100755 index 000000000000..73138c7d56e0 --- /dev/null +++ b/backend/go/pixal3dcpp/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail +CURDIR=$(cd "$(dirname "$0")" && pwd) +if [ "$(uname -s)" = Darwin ]; then + export DYLD_LIBRARY_PATH="$CURDIR/lib:${DYLD_LIBRARY_PATH:-}" +else + export LD_LIBRARY_PATH="$CURDIR/lib:${LD_LIBRARY_PATH:-}" +fi +exec "$CURDIR/pixal3dcpp" "$@" diff --git a/backend/index.yaml b/backend/index.yaml index d399d6498fc0..b885eaf6e031 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -539,6 +539,21 @@ amd: "vulkan-kimodocpp" intel: "vulkan-kimodocpp" metal: "cpu-darwin-arm64-kimodocpp" +- &pixal3dcpp + name: "pixal3dcpp" + alias: "pixal3dcpp" + license: mit + description: "Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI" + urls: + - https://github.com/raven38/pixal3d.cpp + tags: [image-to-3d, 3d-generation, CPU, GPU, CUDA, Vulkan, Metal] + capabilities: + default: "cpu-pixal3dcpp" + nvidia: "cuda12-pixal3dcpp" + nvidia-cuda-12: "cuda12-pixal3dcpp" + nvidia-cuda-13: "cuda13-pixal3dcpp" + vulkan: "vulkan-pixal3dcpp" + metal: "metal-pixal3dcpp" - &trellis2cpp name: "trellis2cpp" alias: "trellis2cpp" @@ -7376,3 +7391,64 @@ uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-funasr" mirrors: - localai/localai-backends:master-metal-darwin-arm64-funasr + +## pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "pixal3dcpp-development" + capabilities: + default: "cpu-pixal3dcpp-development" + nvidia: "cuda12-pixal3dcpp-development" + nvidia-cuda-12: "cuda12-pixal3dcpp-development" + nvidia-cuda-13: "cuda13-pixal3dcpp-development" + vulkan: "vulkan-pixal3dcpp-development" + metal: "metal-pixal3dcpp-development" +- !!merge <<: *pixal3dcpp + name: "cpu-pixal3dcpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-pixal3dcpp" + mirrors: + - localai/localai-backends:latest-cpu-pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "cpu-pixal3dcpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-pixal3dcpp" + mirrors: + - localai/localai-backends:master-cpu-pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "cuda12-pixal3dcpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-pixal3dcpp" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "cuda12-pixal3dcpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-pixal3dcpp" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "cuda13-pixal3dcpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-pixal3dcpp" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "cuda13-pixal3dcpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-pixal3dcpp" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "vulkan-pixal3dcpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-pixal3dcpp" + mirrors: + - localai/localai-backends:latest-gpu-vulkan-pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "vulkan-pixal3dcpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-pixal3dcpp" + mirrors: + - localai/localai-backends:master-gpu-vulkan-pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "metal-pixal3dcpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-pixal3dcpp" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-pixal3dcpp +- !!merge <<: *pixal3dcpp + name: "metal-pixal3dcpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-pixal3dcpp" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-pixal3dcpp diff --git a/core/backend/model3d.go b/core/backend/model3d.go index 7cccbc7db6c5..a7cee80ce0f7 100644 --- a/core/backend/model3d.go +++ b/core/backend/model3d.go @@ -15,6 +15,8 @@ import ( // this layer. type Model3DGenerationOptions struct { Image string + Images []string + MeshScale float64 Destination string Seed int32 Step int32 @@ -38,6 +40,8 @@ func Model3DGeneration(options Model3DGenerationOptions, loader *model.ModelLoad appConfig.Context, &proto.Generate3DRequest{ Src: options.Image, + Images: append([]string(nil), options.Images...), + MeshScale: options.MeshScale, Dst: options.Destination, Seed: options.Seed, Step: options.Step, diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index 96d53c4e255c..dc4c65f5d6cf 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -445,6 +445,12 @@ var BackendCapabilities = map[string]BackendCapability{ DefaultUsecases: []string{Usecase3DAnimation}, Description: "kimodo.cpp — text-to-motion on CPU/Vulkan, exported as animated skeleton GLB", }, + "pixal3dcpp": { + GRPCMethods: []GRPCMethod{MethodGenerate3D}, + PossibleUsecases: []string{Usecase3D}, + DefaultUsecases: []string{Usecase3D}, + Description: "Pixal3D — four pre-matted RGBA views to textured 3D mesh (GLB)", + }, "trellis2cpp": { GRPCMethods: []GRPCMethod{MethodGenerate3D}, PossibleUsecases: []string{Usecase3D}, diff --git a/core/config/model_3d.go b/core/config/model_3d.go index 5cbcd909009f..6178a3761c55 100644 --- a/core/config/model_3d.go +++ b/core/config/model_3d.go @@ -34,6 +34,20 @@ func (c *ModelConfig) ThreeDOperations() []schema.ThreeDOperation { Inputs: []schema.ThreeDInput{{Name: "prompt", Type: "text", Label: "Motion prompt", Required: true, MaxBytes: 4096}}, Parameters: params, }} + case "pixal3dcpp": + if !c.HasUsecases(FLAG_3D) { + return nil + } + return []schema.ThreeDOperation{{ + ID: "generate_multiview", Label: "Generate from four views", Endpoint: "/3d/generations", Output: "mesh", + Inputs: []schema.ThreeDInput{ + {Name: "front", Type: "image", Label: "Front RGBA PNG", Required: true}, + {Name: "right", Type: "image", Label: "Right RGBA PNG", Required: true}, + {Name: "back", Type: "image", Label: "Back RGBA PNG", Required: true}, + {Name: "left", Type: "image", Label: "Left RGBA PNG", Required: true}, + }, + Parameters: []schema.ThreeDParameter{{Name: "mesh_scale", Label: "Mesh scale (required, positive)", Type: "number", Min: 0}}, + }} case "trellis2cpp": if !c.HasUsecases(FLAG_3D) { return nil diff --git a/core/config/model_config.go b/core/config/model_config.go index b2b727628743..94e232b57516 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -2317,7 +2317,7 @@ func (c *ModelConfig) GuessUsecases(u ModelConfigUsecase) bool { } if (u & FLAG_3D) == FLAG_3D { - threeDBackends := []string{"trellis2cpp"} + threeDBackends := []string{"trellis2cpp", "pixal3dcpp"} if !slices.Contains(threeDBackends, c.Backend) { return false } diff --git a/core/config/pixal3d_test.go b/core/config/pixal3d_test.go new file mode 100644 index 000000000000..038b018efc88 --- /dev/null +++ b/core/config/pixal3d_test.go @@ -0,0 +1,17 @@ +package config + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Pixal3D discovery", func() { + It("advertises four views and explicit mesh scale", func() { + c := &ModelConfig{Backend: "pixal3dcpp"} + Expect(c.HasUsecases(FLAG_3D)).To(BeTrue()) + ops := c.ThreeDOperations() + Expect(ops).To(HaveLen(1)) + Expect(ops[0].Inputs).To(HaveLen(4)) + Expect(ops[0].Parameters[0].Name).To(Equal("mesh_scale")) + }) +}) diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index 59ecf4fccb77..b900b4bf3967 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -148,6 +148,7 @@ var defaultImporters = []Importer{ // run before LlamaCPPImporter so its GGUF sets aren't claimed by the // generic .gguf importer; matches only trellis-named URIs/repos or the // distinctive component filenames, so arbitrary GGUFs are never claimed. + &Pixal3DImporter{}, &Trellis2CppImporter{}, &KimodoCppImporter{}, &ACEStepImporter{}, diff --git a/core/gallery/importers/pixal3d.go b/core/gallery/importers/pixal3d.go new file mode 100644 index 000000000000..84641f8ba23e --- /dev/null +++ b/core/gallery/importers/pixal3d.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT +package importers + +import ( + "crypto/sha256" + _ "embed" + "encoding/json" + "fmt" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "go.yaml.in/yaml/v2" +) + +// Inventory from raven38/pixal3d.cpp d1b4926452f9e09702b891db5f05acc845e153ed (MIT). +// +//go:embed pixal3d_models.json +var pixal3dManifest []byte + +type Pixal3DImporter struct{} + +func (*Pixal3DImporter) Name() string { return "pixal3dcpp" } +func (*Pixal3DImporter) Modality() string { return "3d" } +func (*Pixal3DImporter) AutoDetects() bool { return false } +func (*Pixal3DImporter) Match(d Details) bool { + var p struct { + Backend string `json:"backend"` + } + return json.Unmarshal(d.Preferences, &p) == nil && p.Backend == "pixal3dcpp" +} +func (*Pixal3DImporter) Import(d Details) (gallery.ModelConfig, error) { + const directory = "pixal3d-q8_0-v1" + owner, repo, ok := HFOwnerRepoFromURI(d.URI) + if !ok || owner != "raven38" || repo != directory { + return gallery.ModelConfig{}, fmt.Errorf("pixal3dcpp importer supports https://huggingface.co/raven38/pixal3d-q8_0-v1; configure other complete MV directories manually") + } + var p struct { + Name string `json:"name"` + } + if err := json.Unmarshal(d.Preferences, &p); err != nil { + return gallery.ModelConfig{}, err + } + if p.Name == "" { + p.Name = "pixal3d" + } + var manifest struct { + Files []struct { + Name string `json:"name"` + SHA256 string `json:"sha256"` + } `json:"files"` + } + if err := json.Unmarshal(pixal3dManifest, &manifest); err != nil { + return gallery.ModelConfig{}, err + } + result := gallery.ModelConfig{Name: p.Name, Description: "Pixal3D multiview Q8_0: four RGBA views to GLB"} + for _, file := range manifest.Files { + result.Files = append(result.Files, gallery.File{URI: "https://huggingface.co/raven38/" + directory + "/resolve/main/" + file.Name, Filename: directory + "/" + file.Name, SHA256: file.SHA256}) + } + result.Files = append(result.Files, gallery.File{URI: "https://raw.githubusercontent.com/raven38/pixal3d.cpp/d1b4926452f9e09702b891db5f05acc845e153ed/models/" + directory + "/pixal3d-models.json", Filename: directory + "/pixal3d-models.json", SHA256: fmt.Sprintf("%x", sha256.Sum256(pixal3dManifest))}) + cfg := config.ModelConfig{Name: p.Name, Backend: "pixal3dcpp", KnownUsecaseStrings: []string{"FLAG_3D"}, PredictionOptions: schema.PredictionOptions{BasicModelRequest: schema.BasicModelRequest{Model: directory}}} + data, err := yaml.Marshal(cfg) + if err != nil { + return gallery.ModelConfig{}, err + } + result.ConfigFile = string(data) + return result, nil +} diff --git a/core/gallery/importers/pixal3d_LICENSE b/core/gallery/importers/pixal3d_LICENSE new file mode 100644 index 000000000000..f8fd8bd2f507 --- /dev/null +++ b/core/gallery/importers/pixal3d_LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2026 Piotr Wilkin (trellis.cpp, the base of this fork) +Copyright (c) 2026 raven (pixal3d.cpp changes) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/core/gallery/importers/pixal3d_models.json b/core/gallery/importers/pixal3d_models.json new file mode 100644 index 000000000000..c3625bcaa0c7 --- /dev/null +++ b/core/gallery/importers/pixal3d_models.json @@ -0,0 +1,71 @@ +{ + "schema_version": 1, + "model_set": "pixal3d-q8_0", + "version": "v1", + "files": [ + { + "name": "dinov3.gguf", + "role": "image_encoder", + "required": true, + "size_bytes": 323657920, + "sha256": "0dd4ffd4b46a248f5b7d49c35275d68461fbf73f57ddb4c1fa8afb4f7bb45a0d" + }, + { + "name": "pixal3d_naf.gguf", + "role": "naf", + "required": true, + "size_bytes": 1334656, + "sha256": "c4f6cd80e94c8d120360ba50a3633322e6ecce0163966e3b7650b24f99c18569" + }, + { + "name": "pixal3d_ss_flow_mv.gguf", + "role": "ss_flow", + "required": true, + "size_bytes": 1426559744, + "sha256": "5211bb54a3f9ec77ab9451cb37031543f516bbb3cf592026210b69a2468adf4c" + }, + { + "name": "ss_dec.gguf", + "role": "ss_decoder", + "required": true, + "size_bytes": 147379392, + "sha256": "2790b5eecb261cc877d9bf175ce2bd6dd48cd65be8c042c5f5bc023dfca01cf7" + }, + { + "name": "pixal3d_shape_flow_512_mv.gguf", + "role": "shape_flow_512", + "required": true, + "size_bytes": 1476761760, + "sha256": "79a8ff77d0da194a8d6fc348914549aa0442d07a32de7a10f065f35d50116152" + }, + { + "name": "shape_dec.gguf", + "role": "shape_decoder", + "required": true, + "size_bytes": 881361568, + "sha256": "0de7c7a675022dd8696d526a9279e5a50b2a35c8a79452f434a72dc53d40f169" + }, + { + "name": "pixal3d_shape_flow_1024_mv.gguf", + "role": "shape_flow_1024", + "required": true, + "size_bytes": 1476761760, + "sha256": "50cd892247d73d4750b720d5612cef03cc0c02ec806b9e37d0d54d7ec579a94b" + }, + { + "name": "pixal3d_tex_flow_1024_mv.gguf", + "role": "texture_flow_1024", + "required": true, + "size_bytes": 1476813984, + "sha256": "e14a590dc6c77211bd591582159252ec473165b9401c4813df6e4f1a6bf69456" + }, + { + "name": "tex_dec.gguf", + "role": "texture_decoder", + "required": true, + "size_bytes": 881344576, + "sha256": "88b4fced46455e02f316664d5c43584a311921dd9a1cdc1b7b7d981cca9214d4" + } + ], + "source": "tools/quantize_gguf.py Q8_0 applied to every file of pixal3d-f16 v1; flow: TencentARC/Pixal3D ckpts/{ss_flow_img_dit_1_3B_64_bf16_mv,slat_flow_img2shape_dit_1_3B_{512,1024}_bf16_mv,slat_flow_imgshape2tex_dit_1_3B_1024_bf16_mv}.safetensors; decoders: ckpts/{shape_dec,tex_dec}_next_dc_f16c32_fp16.safetensors + ss_dec ckpt; image_encoder: timm/vit_large_patch16_dinov3.lvd1689m model.safetensors; naf: valeoai/NAF naf_release.pth converted to safetensors; all converted with tools/convert.py" +} diff --git a/core/gallery/importers/pixal3d_test.go b/core/gallery/importers/pixal3d_test.go new file mode 100644 index 000000000000..43597eb87fc6 --- /dev/null +++ b/core/gallery/importers/pixal3d_test.go @@ -0,0 +1,24 @@ +package importers + +import ( + "encoding/json" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Pixal3D importer", func() { + It("requires explicit selection and only imports the supported multiview bundle", func() { + imp := &Pixal3DImporter{} + details := Details{URI: "https://huggingface.co/raven38/pixal3d-q8_0-v1", Preferences: json.RawMessage(`{}`)} + Expect(imp.Match(details)).To(BeFalse()) + details.Preferences = json.RawMessage(`{"backend":"pixal3dcpp"}`) + Expect(imp.Match(details)).To(BeTrue()) + cfg, err := imp.Import(details) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Files).To(HaveLen(10)) + Expect(cfg.ConfigFile).To(ContainSubstring("model: pixal3d-q8_0-v1")) + details.URI = "https://huggingface.co/microsoft/TRELLIS.2-4B" + _, err = imp.Import(details) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/core/http/endpoints/localai/model3d.go b/core/http/endpoints/localai/model3d.go index fd514c7db686..d355487cd9a6 100644 --- a/core/http/endpoints/localai/model3d.go +++ b/core/http/endpoints/localai/model3d.go @@ -18,6 +18,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/pkg/pixal3d" "github.com/mudler/xlog" @@ -53,7 +54,17 @@ func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appCon return echo.ErrBadRequest } - if input.Image == "" { + isPixal := config.Backend == "pixal3dcpp" + if isPixal { + if err := pixal3d.ValidateRequest(input.Image, input.Images, input.MeshScale, input.Quality, input.Background); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + if input.Step != 0 || input.CFGScale != 0 || input.TextureSteps != 0 || input.Seed != 0 || len(input.Params) != 0 { + return echo.NewHTTPError(http.StatusBadRequest, "Pixal3D supports images, mesh_scale and quality 1024; sampling overrides and params are not supported") + } + } else if len(input.Images) != 0 || input.MeshScale != 0 { + return echo.NewHTTPError(http.StatusBadRequest, "images and mesh_scale require the pixal3dcpp backend") + } else if input.Image == "" { return echo.NewHTTPError(http.StatusBadRequest, "image is required: 3D generation is image-conditioned") } // Reject unknown enum values here rather than surfacing an opaque @@ -65,11 +76,38 @@ func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appCon return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid background %q: must be one of auto, keep, black, white", input.Background)) } - src, err := stageVideoMediaWithLimit(c.Request().Context(), appConfig.GeneratedContentDir, input.Image, max3DInputBytes) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid image: %v", err)) + var src string + var images []string + var err error + defer func() { + if src != "" { + _ = os.Remove(src) + } + for _, path := range images { + _ = os.Remove(path) + } + }() + if isPixal { + for i, view := range input.Images { + path, stageErr := stageVideoMediaWithLimit(c.Request().Context(), appConfig.GeneratedContentDir, view, max3DInputBytes) + if stageErr != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid view %d: %v", i, stageErr)) + } + images = append(images, path) + data, readErr := os.ReadFile(path) // #nosec G304 -- stageVideoMediaWithLimit returns a server-created temporary file + if readErr != nil { + return readErr + } + if err := pixal3d.ValidatePNG(data); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid view %d: %v", i, err)) + } + } + } else { + src, err = stageVideoMediaWithLimit(c.Request().Context(), appConfig.GeneratedContentDir, input.Image, max3DInputBytes) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid image: %v", err)) + } } - defer func() { _ = os.Remove(src) }() xlog.Debug("Parameter Config", "config", config) @@ -78,11 +116,11 @@ func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appCon } step := input.Step - if step == 0 && config.Step != 0 { + if !isPixal && step == 0 && config.Step != 0 { step = int32(config.Step) } cfgScale := input.CFGScale - if cfgScale == 0 && config.CFGScale != 0 { + if !isPixal && cfgScale == 0 && config.CFGScale != 0 { cfgScale = config.CFGScale } @@ -133,6 +171,8 @@ func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appCon fn, err := backend.Model3DGeneration( backend.Model3DGenerationOptions{ Image: src, + Images: images, + MeshScale: input.MeshScale, Destination: output, Seed: input.Seed, Step: step, diff --git a/core/http/endpoints/localai/model3d_pixal3d_internal_test.go b/core/http/endpoints/localai/model3d_pixal3d_internal_test.go new file mode 100644 index 000000000000..fe3e0da83484 --- /dev/null +++ b/core/http/endpoints/localai/model3d_pixal3d_internal_test.go @@ -0,0 +1,97 @@ +package localai + +import ( + "bytes" + "context" + "encoding/base64" + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + grpcPkg "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ggrpc "google.golang.org/grpc" + "image" + "image/png" + "net/http" + "net/http/httptest" + "os" +) + +type pixalEndpointBackend struct { + grpcPkg.Backend + seen *pb.Generate3DRequest +} + +func (*pixalEndpointBackend) HealthCheck(context.Context) (bool, error) { return true, nil } +func (*pixalEndpointBackend) IsBusy() bool { return false } +func (b *pixalEndpointBackend) Generate3D(_ context.Context, r *pb.Generate3DRequest, _ ...ggrpc.CallOption) (*pb.Result, error) { + b.seen = r + for _, path := range r.Images { + _, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + } + return &pb.Result{Success: true}, os.WriteFile(r.Dst, []byte("glTF fixture"), 0600) +} + +var _ = Describe("Pixal3D HTTP generation", func() { + It("forwards four staged images and scale, ignores TRELLIS config defaults, and cleans inputs", func() { + state := &system.SystemState{} + loader := model.NewModelLoader(state) + fixture := &pixalEndpointBackend{} + loader.SetModelRouter(func(_ context.Context, id string, _, _, _, _ string, _ *pb.ModelOptions, _ bool) (*model.Model, error) { + return model.NewModelWithClient(id, "test://pixal", fixture), nil + }) + cfg := &config.ModelConfig{Backend: "pixal3dcpp", Name: "pixal"} + cfg.SetDefaults() + cfg.Model = "pixal3d" + cfg.Step = 12 + cfg.CFGScale = 7.5 + app := config.NewApplicationConfig(config.WithSystemState(state)) + app.GeneratedContentDir = GinkgoT().TempDir() + var buf bytes.Buffer + Expect(png.Encode(&buf, image.NewNRGBA(image.Rect(0, 0, 2, 2)))).To(Succeed()) + data := base64.StdEncoding.EncodeToString(buf.Bytes()) + req := &schema.Model3DRequest{BasicModelRequest: schema.BasicModelRequest{Model: "pixal"}, Images: []string{data, data, data, data}, MeshScale: 0.8, ResponseFormat: "b64_json"} + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/3d/generations", nil), rec) + c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, req) + c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, cfg) + Expect(Model3DEndpoint(nil, loader, app)(c)).To(Succeed()) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(fixture.seen.Images).To(HaveLen(4)) + Expect(fixture.seen.Src).To(BeEmpty()) + Expect(fixture.seen.MeshScale).To(Equal(0.8)) + Expect(fixture.seen.Step).To(BeZero()) + Expect(fixture.seen.CfgScale).To(BeZero()) + for _, path := range fixture.seen.Images { + _, err := os.Stat(path) + Expect(os.IsNotExist(err)).To(BeTrue()) + } + entries, err := os.ReadDir(app.GeneratedContentDir) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + It("rejects bad views and cleans partially staged input before loading", func() { + var buf bytes.Buffer + Expect(png.Encode(&buf, image.NewNRGBA(image.Rect(0, 0, 2, 2)))).To(Succeed()) + data := base64.StdEncoding.EncodeToString(buf.Bytes()) + app := &config.ApplicationConfig{GeneratedContentDir: GinkgoT().TempDir()} + req := &schema.Model3DRequest{BasicModelRequest: schema.BasicModelRequest{Model: "pixal"}, Images: []string{data, "bad%%%", data, data}, MeshScale: 1} + e := echo.New() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/3d/generations", nil), httptest.NewRecorder()) + c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, req) + c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{Backend: "pixal3dcpp"}) + err := Model3DEndpoint(nil, nil, app)(c) + Expect(err).To(BeAssignableToTypeOf(&echo.HTTPError{})) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest)) + entries, err := os.ReadDir(app.GeneratedContentDir) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) +}) diff --git a/core/http/react-ui/e2e/chat-history.spec.js b/core/http/react-ui/e2e/chat-history.spec.js index 05f024ed5283..568cd177c086 100644 --- a/core/http/react-ui/e2e/chat-history.spec.js +++ b/core/http/react-ui/e2e/chat-history.spec.js @@ -18,13 +18,15 @@ async function openMenu(page) { } test.describe('Chat conversations menu', () => { + const now = new Date('2026-10-09T12:00:00Z').getTime() test.beforeEach(async ({ page }) => { + await page.clock.setFixedTime(now) await mockChat(page, { models: [ { id: 'qwen3-8b-instruct', capabilities: ['FLAG_CHAT'] }, { id: 'gemma-4-e4b-it-qat-q4_0', capabilities: ['FLAG_CHAT'] }, { id: 'qwen2.5-vl-7b-instruct', capabilities: ['FLAG_CHAT'] }, ], loaded: ['qwen3-8b-instruct'] }) - await openChat(page, sampleChats()) + await openChat(page, sampleChats(now)) await expect(page.getByTestId('chat-message').first()).toBeVisible() }) @@ -153,7 +155,7 @@ test.describe('Chat conversations menu', () => { }) test('the undo time runs out by itself after six seconds', async ({ page }) => { - await page.clock.install() + await page.clock.install({ time: now }) await page.reload() await page.getByTestId('chat-composer').waitFor() await openMenu(page) diff --git a/core/http/react-ui/e2e/home-console.spec.js b/core/http/react-ui/e2e/home-console.spec.js index f50cb00bdee6..42a5c677d90d 100644 --- a/core/http/react-ui/e2e/home-console.spec.js +++ b/core/http/react-ui/e2e/home-console.spec.js @@ -317,9 +317,11 @@ test.describe('Home in a cluster', () => { }) test.describe('Home resume list', () => { + const now = new Date('2026-10-09T12:00:00Z').getTime() test.beforeEach(async ({ page }) => { + await page.clock.setFixedTime(now) await mockHome(page) - await seedChats(page) + await seedChats(page, sampleChats(now)) }) test('groups conversations by day, one card per day', async ({ page }) => { @@ -408,7 +410,7 @@ test.describe('Home resume list', () => { }) test('without undo the delete becomes final when the time ends', async ({ page }) => { - await page.clock.install() + await page.clock.install({ time: now }) await ready(page) const row = page.getByTestId('home-conversation').filter({ hasText: 'invoice' }) await row.hover() @@ -430,12 +432,12 @@ test.describe('Home resume list', () => { }) test('old conversations wait behind one button', async ({ page }) => { - const chats = sampleChats() + const chats = sampleChats(now) const old = Array.from({ length: 8 }, (_, i) => ({ ...chats[0], id: `old-${i}`, name: `Old chat ${i}`, - updatedAt: Date.now() - (10 + i) * 24 * 60 * 60 * 1000, + updatedAt: now - (10 + i) * 24 * 60 * 60 * 1000, })) await page.addInitScript(([key, data]) => localStorage.setItem(key, JSON.stringify(data)), [CHATS_KEY, { chats: [...chats, ...old], activeChatId: chats[0].id }]) diff --git a/core/http/react-ui/e2e/model-page.spec.js b/core/http/react-ui/e2e/model-page.spec.js index 9a07d8993040..5ee36ecb5723 100644 --- a/core/http/react-ui/e2e/model-page.spec.js +++ b/core/http/react-ui/e2e/model-page.spec.js @@ -199,15 +199,17 @@ test.describe('Model page - walking and going back', () => { const { state } = await setup(page) await expect(rows(page).first()).toBeVisible({ timeout: 15_000 }) await page.waitForTimeout(500) - const before = state.listingRequests.length + // Facet counts load in the background; they do not reload the list. + const listings = () => state.listingRequests.filter(search => new URLSearchParams(search).get('items') !== '1') + const before = listings().length await page.locator('[data-entity="qwen3-8b-instruct"] [data-row-open]').click() await expect(name(page)).toHaveText('qwen3-8b-instruct') await page.keyboard.press('Escape') await expect(rows(page).first()).toBeVisible() await page.waitForTimeout(500) // The page itself looks its own entry up once; the list asks for nothing. - expect(state.listingRequests.length - before).toBeLessThanOrEqual(1) - expect(state.listingRequests.slice(before).every(search => search.includes('term=qwen3-8b-instruct'))).toBe(true) + expect(listings().length - before).toBeLessThanOrEqual(1) + expect(listings().slice(before).every(search => search.includes('term=qwen3-8b-instruct'))).toBe(true) }) test('changing tabs keeps the walker, and Back still leaves for the list in one step', async ({ page }) => { diff --git a/core/http/react-ui/e2e/models-cleanup.spec.js b/core/http/react-ui/e2e/models-cleanup.spec.js index 08b1870fe706..11c4ee8cca03 100644 --- a/core/http/react-ui/e2e/models-cleanup.spec.js +++ b/core/http/react-ui/e2e/models-cleanup.spec.js @@ -92,11 +92,10 @@ test.describe('Cleanup sheet - what it knows and what it does not', () => { test('each tier is ranked by size and a model with no known size goes last', async ({ page }) => { await openSheet(page) - const names = await tier(page, 'call').locator('.ledger-cr__name').evaluateAll( - (els) => els.map((e) => e.firstChild.textContent.trim()), - ) - expect(names[0]).toBe('llama-3.3-70b-instruct-iq2') - expect(names[names.length - 1]).toBe('my-finetune-q4') + // Estimates arrive after the sheet becomes interactive and can reorder it. + const names = tier(page, 'call').locator('.ledger-cr__name') + await expect(names.first()).toContainText('llama-3.3-70b-instruct-iq2') + await expect(names.last()).toContainText('my-finetune-q4') await expect(tier(page, 'call').locator('li').last()).toContainText('size unknown') await expect(tier(page, 'call').locator('li').first()).toContainText(`${gb1(sizeOf('llama-3.3-70b-instruct-iq2'))} GB`) }) diff --git a/core/http/react-ui/e2e/studio-work.spec.js b/core/http/react-ui/e2e/studio-work.spec.js index a7623d0cfc99..4f1f31cc13db 100644 --- a/core/http/react-ui/e2e/studio-work.spec.js +++ b/core/http/react-ui/e2e/studio-work.spec.js @@ -135,9 +135,8 @@ test.describe('Your work', () => { test('the masonry uses fewer columns on a narrow screen', async ({ page }) => { await open(page) - const wide = await page.locator('.studio-masonry').getAttribute('data-columns') + await expect.poll(async () => Number(await page.locator('.studio-masonry').getAttribute('data-columns'))).toBeGreaterThan(1) await page.setViewportSize({ width: 390, height: 844 }) await expect.poll(() => page.locator('.studio-masonry').getAttribute('data-columns')).toBe('1') - expect(Number(wide)).toBeGreaterThan(1) }) }) diff --git a/core/http/react-ui/e2e/threed-gen.spec.js b/core/http/react-ui/e2e/threed-gen.spec.js index 934528d15f35..07f54a1249f2 100644 --- a/core/http/react-ui/e2e/threed-gen.spec.js +++ b/core/http/react-ui/e2e/threed-gen.spec.js @@ -401,3 +401,33 @@ test.describe('3D generation', () => { }) }) }) + +test('Pixal3D requires four ordered views and a positive mesh scale', async ({ page }) => { + await page.route('**/api/models/capabilities', route => route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ data: [{ id: 'pixal-test', capabilities: ['FLAG_3D'], three_d_operations: [{ + id: 'generate_multiview', endpoint: '/3d/generations', output: 'mesh', + inputs: ['front', 'right', 'back', 'left'].map(name => ({ name, type: 'image', required: true })), + parameters: [{ name: 'mesh_scale', type: 'number' }], + }] }] }), + })) + let request + await mockGeneration(page, body => { request = body }) + await mockGlbDownload(page) + await page.goto('/app/3d') + await expect(page.getByRole('button', { name: 'pixal-test' })).toBeVisible() + await expect(page.locator('#threed-image-file')).toHaveCount(0) + await expect(page.getByLabel('Mesh scale')).toBeVisible() + for (const name of ['front', 'right', 'back', 'left']) { + await page.locator(`#threed-${name}-image-file`).setInputFiles({ name: `${name}.png`, mimeType: 'image/png', buffer: TINY_PNG }) + } + await page.getByLabel('Mesh scale').fill('0.8') + await page.locator('button[type="submit"]').click() + await expect(page.getByTestId('glb-download')).toBeVisible() + expect(request.images).toHaveLength(4) + expect(request.mesh_scale).toBe(0.8) + expect(request.quality).toBe('1024') + expect(request.image).toBeUndefined() + expect(request.background).toBeUndefined() + await expect(page.getByTestId('glb-remesh')).toHaveCount(0) +}) diff --git a/core/http/react-ui/e2e/traffic-models-host.spec.js b/core/http/react-ui/e2e/traffic-models-host.spec.js index 3c707396071d..f67763286441 100644 --- a/core/http/react-ui/e2e/traffic-models-host.spec.js +++ b/core/http/react-ui/e2e/traffic-models-host.spec.js @@ -17,7 +17,8 @@ test.describe('Traffic models', () => { await expect(bge.locator('td.dk-num').last()).toHaveText('-') // The failed load of flux-dev is in the backend-operation buffer. await expect(page.locator('tr[data-entity="flux-dev"]')).toContainText('1') - await expect(page.getByText(/p50|p95|p99/)).toHaveCount(0) + // A row's concatenated text can contain llama-cpp followed by 99 requests. + await expect(page.getByText(/\bp(?:50|95|99)\b/i)).toHaveCount(0) }) test('a row opens in place with the split, the buffer and the links', async ({ page }) => { diff --git a/core/http/react-ui/src/components/models/page/model-page.css b/core/http/react-ui/src/components/models/page/model-page.css index d650afe6af23..c59a1046654b 100644 --- a/core/http/react-ui/src/components/models/page/model-page.css +++ b/core/http/react-ui/src/components/models/page/model-page.css @@ -262,8 +262,8 @@ a.modelpage-chip { cursor: pointer; text-decoration: none; color: inherit; } .modelpage-keys { display: none; } } -.modelpage-table .dk-table-sub { max-width: none; white-space: normal; overflow-wrap: anywhere; } -.modelpage-table .dk-table-sub .modelpage-inline-badge:first-child { margin-left: 0; } +.modelpage-table .dk-table .dk-table-sub { max-width: none; white-space: normal; overflow-wrap: anywhere; } +.modelpage-table .dk-table .dk-table-sub .modelpage-inline-badge:first-child { margin-left: 0; } /* A grid child is as wide as its content unless told otherwise; a table inside one would push the whole page sideways instead of scrolling in its own box. */ @@ -271,6 +271,8 @@ a.modelpage-chip { cursor: pointer; text-decoration: none; color: inherit; } .modelpage-variants__size { display: none; } @media (max-width: 640px) { .modelpage-variants__size { display: inline; } + /* Leave room for Fit and Install while long build names wrap. */ + .modelpage-table .dk-table .dk-table-sub { max-width: 14ch; } .modelpage-table .dk-table-actions .dk-btn { padding-inline: var(--dk-space-2); } } diff --git a/core/http/react-ui/src/pages/ThreeDGen.jsx b/core/http/react-ui/src/pages/ThreeDGen.jsx index 93515a001cba..72b2f28e71fd 100644 --- a/core/http/react-ui/src/pages/ThreeDGen.jsx +++ b/core/http/react-ui/src/pages/ThreeDGen.jsx @@ -78,6 +78,9 @@ export default function ThreeDGen() { const modelNames = useMemo(() => models.filter(item => item.capabilities?.some(cap => cap === CAP_3D || cap === CAP_3D_ANIMATION)).map(item => item.id), [models]) const selectedModel = models.find(item => item.id === model) const animation = selectedModel?.three_d_operations?.find(operation => operation.id === 'animate') + const multiview = selectedModel?.three_d_operations?.some(operation => operation.id === 'generate_multiview') + const [views, setViews] = useState({}) + const [meshScale, setMeshScale] = useState('') const [animationInputs, setAnimationInputs] = useState({}) const [animationParams, setAnimationParams] = useState({}) const [requestEndpoint, setRequestEndpoint] = useState('/3d/generations') @@ -126,6 +129,8 @@ export default function ThreeDGen() { setAnimationInputs({}) setAnimationParams({}) setImage(null) + setViews({}) + setMeshScale('') setQuality('auto') setBackground('auto') setSteps('') @@ -145,7 +150,10 @@ export default function ThreeDGen() { } const run = async () => { - if (!animation && !image?.base64) { addToast(t('threed.toasts.noImage'), 'warning'); return } + if (multiview && (['front', 'right', 'back', 'left'].some(name => !views[name]?.base64) || !Number.isFinite(Number(meshScale)) || Number(meshScale) <= 0)) { + addToast('Provide all four RGBA PNG views and a positive mesh scale.', 'warning'); return + } + if (!animation && !multiview && !image?.base64) { addToast(t('threed.toasts.noImage'), 'warning'); return } if (!model) { addToast(t('threed.toasts.noModel'), 'warning'); return } setLoading(true) @@ -157,8 +165,9 @@ export default function ThreeDGen() { const params = Object.fromEntries(Object.entries(animationParams).filter(([key, value]) => value !== '' && animation?.parameters.some(parameter => parameter.name === key))) const body = animation ? { model, inputs: animationInputs, params, response_format: 'url' } + : multiview ? { model, images: ['front', 'right', 'back', 'left'].map(name => views[name].base64), mesh_scale: Number(meshScale), quality: '1024', response_format: 'url' } : { model, image: image.base64, quality, background, response_format: 'url' } - if (!animation) { + if (!animation && !multiview) { if (steps) body.step = parseInt(steps) if (textureSteps) body.texture_steps = parseInt(textureSteps) if (guidance) body.cfg_scale = parseFloat(guidance) @@ -167,7 +176,7 @@ export default function ThreeDGen() { // RequestPanel renders and copies its body. Keeping a multi-megabyte image // there duplicates the upload in React and can starve the result render. - setLastRequest(animation ? body : { ...body, image: `` }) + setLastRequest(animation ? body : multiview ? { ...body, images: ['front', 'right', 'back', 'left'].map(name => `<${name} RGBA PNG omitted>`) } : { ...body, image: `` }) setRequestEndpoint(animation ? '/3d/animate' : '/3d/generations') try { @@ -187,7 +196,7 @@ export default function ThreeDGen() { const inputThumb = !animation && image?.dataUrl ? await makeThumb(image.dataUrl) : null const saved = await addEntry({ model, - params: animation ? params : { quality, background, steps, textureSteps, guidance, seed }, + params: animation ? params : multiview ? { mesh_scale: Number(meshScale), quality: '1024' } : { quality, background, steps, textureSteps, guidance, seed }, inputs: animation ? animationInputs : undefined, operation: animation ? 'animate' : 'generate', outputType, @@ -234,7 +243,7 @@ export default function ThreeDGen() { const installed = ws.installed.byType.threed const noModel = !ws.installed.loading && !ws.installed.error && installed.length === 0 const why = noModel ? t('studio.composer.whyModel', { type: t('studio.tabs.threed') }) - : (!animation && !image?.base64) ? t('studio.workspace.threed.whyImage') : '' + : (!animation && !multiview && !image?.base64) ? t('studio.workspace.threed.whyImage') : '' const f = (key) => t(`studio.workspace.fields.${key}`) const advancedSet = [steps && `${t('threed.labels.steps')} ${steps}`, textureSteps && `${t('threed.labels.textureSteps')} ${textureSteps}`, guidance && `${f('cfg')} ${guidance}`, seed && `${f('seed')} ${seed}`].filter(Boolean) const qualityOptions = QUALITIES.map(q => ({ value: q, label: t(`threed.labels.quality_${q}`) })) @@ -253,12 +262,12 @@ export default function ThreeDGen() { noModel={noModel ? { type: 'threed', label: t('studio.tabs.threed'), onChanged: ws.installed.refetch } : null} options={<> - {!animation && <> + {!animation && !multiview && <> } } - fold={animation ? null : ( + fold={animation || multiview ? null : ( )} - submit={{ label: t('threed.actions.generate'), busyLabel: t('threed.actions.generating'), busy: loading, disabled: !model || noModel || (!animation && !image?.base64), why, icon: 'cube' }} + submit={{ label: t('threed.actions.generate'), busyLabel: t('threed.actions.generating'), busy: loading, disabled: !model || noModel || (!animation && !multiview && !image?.base64), why, icon: 'cube' }} > {animation ? (
+ ) : multiview ? ( + <> +

Upload four pre-matted RGBA PNGs of the same object in the canonical turntable framing. Resolution: 1024.

+ {['front', 'right', 'back', 'left'].map(name => ( + setViews(current => ({ ...current, [name]: value }))} + onError={err => addToast(err.message, 'error')} maxBytes={MAX_3D_INPUT_BYTES} idPrefix={`threed-${name}`} /> + ))} +
+ + setMeshScale(e.target.value)} /> +

Enter the positive scale used to frame these views. Pixal3D does not estimate it.

+
+ ) : (
", + "data:image/png;base64,", + "data:image/png;base64,", + "data:image/png;base64," + ], + "mesh_scale": 0.8, + "quality": "1024", + "response_format": "url" +} +``` + +Replace the placeholders with the image data. `mesh_scale: 0.8` is an example, not an estimated default. `quality` can be omitted or set to `1024`; `background` can be omitted or set to `keep`. Both `url` and `b64_json` responses use the envelope described above. + +Pixal3D rejects `image`, sampling overrides, nonempty `params`, and other resolutions or background modes. Other backends reject the new `images` and `mesh_scale` fields. The existing TRELLIS single-image request remains unchanged. + +### Runtime and platforms + +The adapter registers and unloads models through LocalAI's normal backend lifecycle. It starts the native CLI for each generation, so each request reloads weights. Unloading cancels an active child process. This adapter does not expose upstream's persistent HTTP server or its single-view mode. + +Build definitions cover Linux amd64 and arm64 CPU/Vulkan, Linux amd64 CUDA 12/13, and Apple Silicon Metal. Some upstream operations use CPU fallbacks on Metal. CPU inference can be slow and requires substantial memory. Native builds and inference with real weights require platform validation; no throughput or memory guarantee is implied by the build definitions. diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index a9afd0ce14cf..0583b4f992b8 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -310,7 +310,7 @@ LocalAI supports various types of backends: - **Sound Generation Backends**: For music and audio generation (e.g., ACE-Step, [audio.cpp]({{%relref "features/audio-cpp" %}})) - **Sound Classification Backends**: For sound-event classification / audio tagging - identifying everyday sounds like baby cry, glass breaking, alarms (e.g., ced.cpp) - **Image & Video Generation Backends**: For diffusion and audio-conditioned avatar models (e.g., stable-diffusion.cpp, diffusers, vLLM-Omni, [MLX-Video on Apple Silicon]({{%relref "features/video-generation" %}}), [LongCat-Video]({{%relref "features/video-generation" %}}), [vllm.cpp / MiniMax-H3]({{%relref "features/video-generation" %}})) -- **3D Generation Backends**: For image-to-3D mesh generation ([trellis2.cpp]({{%relref "features/3d-generation" %}}) — Microsoft TRELLIS.2, producing GLB assets with PBR textures) +- **3D Generation Backends**: `trellis2cpp` converts a single image into a textured GLB. `pixal3dcpp` takes four canonical RGBA views. See [3D generation]({{%relref "features/3d-generation" %}}) for setup and requests. - **Vision & Detection Backends**: For object detection, segmentation, depth, and face/voice recognition (e.g., rf-detr.cpp, locate-anything.cpp, sam3.cpp, insightface) - **Audio Processing Backends**: For voice activity detection and audio enhancement (e.g., Silero VAD, LocalVQE, [audio.cpp]({{%relref "features/audio-cpp" %}})) - **Source Separation & Voice Conversion Backends**: For splitting a mix into named stems (vocals, drums, bass) and for converting speech or singing to a target voice (e.g., [audio.cpp]({{%relref "features/audio-cpp" %}})) diff --git a/docs/content/getting-started/models.md b/docs/content/getting-started/models.md index 188b34222d9c..0e29d7b52043 100644 --- a/docs/content/getting-started/models.md +++ b/docs/content/getting-started/models.md @@ -74,6 +74,9 @@ Note: The galleries available in LocalAI can be customized to point to a differe Visit [models.localai.io](https://models.localai.io) to browse all available models in your browser. +On a phone, the model page wraps build descriptions in the **Variants** table +to keep the fit status and install buttons visible. + ## Method 1.5: Import Models via WebUI The WebUI import page (**Build → Import**) takes either a source to resolve or a diff --git a/pkg/pixal3d/inputs.go b/pkg/pixal3d/inputs.go new file mode 100644 index 000000000000..0636415541d3 --- /dev/null +++ b/pkg/pixal3d/inputs.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT +// Package pixal3d defines the canonical multiview contract shared by HTTP and +// native backend entry points. +package pixal3d + +import ( + "bytes" + "fmt" + "image/png" + "math" + "strings" +) + +const MaxInputBytes = 32 << 20 + +func ValidateRequest(single string, views []string, scale float64, quality, background string) error { + if single != "" || len(views) != 4 { + return fmt.Errorf("Pixal3D requires exactly four images in front/right/back/left order and no image field") + } + for _, v := range views { + if strings.TrimSpace(v) == "" { + return fmt.Errorf("Pixal3D images must not be empty") + } + } + if scale <= 0 || math.IsNaN(scale) || math.IsInf(scale, 0) || float32(scale) == 0 || math.IsInf(float64(float32(scale)), 0) { + return fmt.Errorf("mesh_scale must be explicitly set to a finite positive number") + } + if quality != "" && quality != "1024" { + return fmt.Errorf("Pixal3D supports only quality 1024") + } + if background != "" && background != "keep" { + return fmt.Errorf("Pixal3D requires pre-matted RGBA PNG images; background must be keep") + } + return nil +} + +func ValidatePNG(data []byte) error { + // Checking IHDR before decoding bounds memory use even for compressed bombs. + if len(data) > MaxInputBytes || len(data) < 33 || !bytes.Equal(data[:8], []byte("\x89PNG\r\n\x1a\n")) || string(data[12:16]) != "IHDR" || data[24] != 8 || data[25] != 6 { + return fmt.Errorf("each Pixal3D view must be an 8-bit RGBA PNG of at most 32 MiB") + } + cfg, err := png.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("invalid PNG: %w", err) + } + if cfg.Width > 4096 || cfg.Height > 4096 { + return fmt.Errorf("Pixal3D views must be at most 4096 by 4096 pixels") + } + if _, err := png.Decode(bytes.NewReader(data)); err != nil { + return fmt.Errorf("invalid PNG: %w", err) + } + return nil +} diff --git a/pkg/pixal3d/inputs_test.go b/pkg/pixal3d/inputs_test.go new file mode 100644 index 000000000000..7af01fe709c5 --- /dev/null +++ b/pkg/pixal3d/inputs_test.go @@ -0,0 +1,62 @@ +package pixal3d + +import ( + "bytes" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "image" + "image/color" + "image/png" + "math" + "testing" +) + +func TestPixal3D(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Pixal3D input contract") } + +var _ = Describe("canonical Pixal3D inputs", func() { + It("validates all required inputs before model loading", func() { + good := []string{"front", "right", "back", "left"} + cases := []struct { + name string + image string + views []string + scale float64 + quality, background string + bad bool + }{ + {"canonical", "", good, 1, "1024", "keep", false}, + {"defaults", "", good, 0.5, "", "", false}, + {"missing views", "", nil, 1, "", "", true}, + {"three views", "", good[:3], 1, "", "", true}, + {"ambiguous", "one", good, 1, "", "", true}, + {"empty view", "", []string{"front", "", "back", "left"}, 1, "", "", true}, + {"missing scale", "", good, 0, "", "", true}, + {"negative scale", "", good, -1, "", "", true}, + {"nan scale", "", good, math.NaN(), "", "", true}, + {"infinite scale", "", good, math.Inf(1), "", "", true}, + {"wrong resolution", "", good, 1, "512", "", true}, + {"remove background", "", good, 1, "", "auto", true}, + } + for _, c := range cases { + err := ValidateRequest(c.image, c.views, c.scale, c.quality, c.background) + Expect(err != nil).To(Equal(c.bad), c.name) + } + }) + It("requires decodable RGBA PNGs", func() { + for _, transparent := range []bool{false, true} { + img := image.NewNRGBA(image.Rect(0, 0, 2, 2)) + for y := 0; y < 2; y++ { + for x := 0; x < 2; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 255, A: 255}) + } + } + if transparent { + img.SetNRGBA(0, 0, color.NRGBA{A: 0}) + } + var b bytes.Buffer + Expect(png.Encode(&b, img)).To(Succeed()) + Expect(ValidatePNG(b.Bytes()) == nil).To(Equal(transparent)) + } + Expect(ValidatePNG([]byte("not PNG"))).NotTo(Succeed()) + }) +}) diff --git a/scripts/lib/backend-filter.mjs b/scripts/lib/backend-filter.mjs index 9312cbf6d0c0..365031036617 100644 --- a/scripts/lib/backend-filter.mjs +++ b/scripts/lib/backend-filter.mjs @@ -188,14 +188,14 @@ const always = () => true; // go list -deps ./backend/go/... | grep LocalAI/pkg // // returns exactly pkg/audio, pkg/grpc (+ base, grpcerrors, proto), pkg/httpclient, -// pkg/sound, pkg/store and pkg/utils — identical for GOOS/GOARCH in +// pkg/pixal3d, pkg/sound, pkg/store and pkg/utils — identical for GOOS/GOARCH in // {linux,darwin} x {amd64,arm64}, so one list covers both matrices. Notably // absent: pkg/model, pkg/downloader, pkg/functions and the other ~21 pkg // subtrees, which are core-server-only. // // Enumerating rather than taking all of pkg/ is the whole point. All of pkg/ // changes in ~8.6% of commits and would fire a 199-entry Go matrix that often; -// these six subtrees change in 2.0%, which is the same order as the already +// the original six subtrees changed in 2.0%, the same order as the already // accepted scripts/build/ rule (1.9%). If the enumeration ever drifts from the // `go list` output the tests pin both directions — a listed subtree must // trigger, an unlisted one must not. @@ -203,6 +203,7 @@ const GO_BACKEND_PKG_PREFIXES = [ "pkg/audio/", "pkg/grpc/", // covers base/, grpcerrors/ and the generated proto/ "pkg/httpclient/", + "pkg/pixal3d/", "pkg/sound/", "pkg/store/", "pkg/utils/", diff --git a/scripts/lib/backend-filter_test.mjs b/scripts/lib/backend-filter_test.mjs index 6999c921a127..186bdc6574fa 100644 --- a/scripts/lib/backend-filter_test.mjs +++ b/scripts/lib/backend-filter_test.mjs @@ -296,6 +296,7 @@ test("every enumerated Go-backend pkg subtree triggers a Go rebuild", () => { "pkg/audio/convert.go", "pkg/grpc/client.go", "pkg/httpclient/client.go", + "pkg/pixal3d/inputs.go", "pkg/sound/sound.go", "pkg/store/client.go", "pkg/utils/path.go", @@ -596,3 +597,8 @@ test("unresolvable proto revisions conservatively rebuild everything", () => { assert.equal(filtered.length, includes.length); assert.equal(filteredDarwin.length, includesDarwin.length); }); + +test("pixal3dcpp maps to its Go wrapper on Linux and Darwin", () => { + assert.equal(inferBackendPath({ backend: "pixal3dcpp", dockerfile: "./backend/Dockerfile.golang" }), "backend/go/pixal3dcpp/"); + assert.equal(inferBackendPathDarwin({ backend: "pixal3dcpp", lang: "go" }), "backend/go/pixal3dcpp/"); +});