feat(motion): add GEM-X backend and duplex streaming API

.dockerignore

@@ -10,6 +10,8 @@ backend/go/image/stablediffusion-ggml/build/
 backend/go/*/build
 backend/go/kimodocpp/build-*
 backend/go/kimodocpp/kimodocpp
+backend/go/gemxcpp/build-*
+backend/go/gemxcpp/gemxcpp
 backend/go/*/.cache
 backend/go/*/sources
 backend/go/*/package

.github/backend-matrix.yml

@@ -3616,6 +3616,63 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  # gem-x.cpp: CPU and Vulkan, with native runners for each architecture.
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-gemxcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "gemxcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-gemxcpp'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "gemxcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-gemxcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "gemxcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-gemxcpp'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "gemxcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   # trellis2cpp
   - build-type: ''
     cuda-major-version: ""
@@ -6551,6 +6608,10 @@ includeDarwin:
     tag-suffix: "-cpu-darwin-arm64-kimodocpp"
     build-type: "cpu"
     lang: "go"
+  - backend: "gemxcpp"
+    tag-suffix: "-cpu-darwin-arm64-gemxcpp"
+    build-type: "cpu"
+    lang: "go"
   - backend: "diffusers"
     tag-suffix: "-metal-darwin-arm64-diffusers"
     build-type: "mps"

.github/workflows/bump_deps.yaml

@@ -94,6 +94,10 @@ jobs:
             variable: "KIMODO_VERSION"
             branch: "main"
             file: "backend/go/kimodocpp/Makefile"
+          - repository: "localai-org/gem-x.cpp"
+            variable: "GEMX_VERSION"
+            branch: "main"
+            file: "backend/go/gemxcpp/Makefile"
           - repository: "mudler/go-piper"
             variable: "PIPER_VERSION"
             branch: "master"

.gitignore

@@ -134,3 +134,8 @@ formal-verification/out/
 # root, which is what a contributor testing a build does. Nothing under here is
 # source: it is the instance's own models, outputs, traces and identity.
 /data/
+
+/backend/go/gemxcpp/sources/
+/backend/go/gemxcpp/build-*/
+/backend/go/gemxcpp/package/
+/backend/go/gemxcpp/gemxcpp

Dockerfile

@@ -346,6 +346,7 @@ COPY ./.git ./.git
 
 # Some of the Go backends use libs from the main src, we could further optimize the caching by building the CPP backends before here
 COPY ./pkg/grpc ./pkg/grpc
+COPY ./pkg/motion ./pkg/motion
 COPY ./pkg/utils ./pkg/utils
 
 RUN ls -l ./

Makefile

@@ -591,6 +591,7 @@ protogen-go: protoc install-go-tools
 	# shell-profile change.
 	PATH="$$(go env GOPATH)/bin:$$PATH" ./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \
     backend/backend.proto
+	PATH="$$(go env GOPATH)/bin:$$PATH" ./protoc -Ipkg/motion/proto --go_out=pkg/motion/proto --go_opt=paths=source_relative pkg/motion/proto/motion.proto
 
 core/config/inference_defaults.json: ## Fetch inference defaults from unsloth (only if missing)
 	$(GOCMD) generate ./core/config/...
@@ -604,7 +605,7 @@ generate-force: ## Re-fetch inference defaults from unsloth (always)
 
 .PHONY: protogen-go-clean
 protogen-go-clean:
-	$(RM) pkg/grpc/proto/backend.pb.go pkg/grpc/proto/backend_grpc.pb.go
+	$(RM) pkg/grpc/proto/backend.pb.go pkg/grpc/proto/backend_grpc.pb.go pkg/motion/proto/motion.pb.go
 	$(RM) bin/*
 
 prepare-test-extra: protogen-python
@@ -641,6 +642,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/go/locate-anything-cpp
 	$(MAKE) -C backend/go/trellis2cpp
 	$(MAKE) -C backend/go/kimodocpp
+	$(MAKE) -C backend/go/gemxcpp
 	$(MAKE) -C backend/go/valkey-store
 
 test-extra: prepare-test-extra
@@ -680,6 +682,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/go/nemo-speech-cpp test
 	$(MAKE) -C backend/go/trellis2cpp test
 	$(MAKE) -C backend/go/kimodocpp test
+	$(MAKE) -C backend/go/gemxcpp test
 	$(MAKE) -C backend/go/valkey-store test
 
 ##
@@ -1336,6 +1339,7 @@ BACKEND_HUGGINGFACE = huggingface|golang|.|false|true
 BACKEND_SILERO_VAD = silero-vad|golang|.|false|true
 BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|true
 BACKEND_TRELLIS2CPP = trellis2cpp|golang|.|--progress=plain|true
+BACKEND_GEMXCPP = gemxcpp|golang|.|--progress=plain|true
 BACKEND_KIMODOCPP = kimodocpp|golang|.|--progress=plain|true
 BACKEND_WHISPER = whisper|golang|.|false|true
 BACKEND_CRISPASR = crispasr|golang|.|false|true
@@ -1444,13 +1448,21 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD)))
 $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TRELLIS2CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_KIMODOCPP)))
+$(eval $(call generate-docker-build-target,$(BACKEND_GEMXCPP)))
 .NOTPARALLEL: backends/kimodocpp backends/kimodocpp-darwin
 docker-build-backends: docker-build-kimodocpp
 
 backends/kimodocpp-darwin:
 	BACKEND=kimodocpp BUILD_TYPE=cpu $(MAKE) build-darwin-go-backend
 	./local-ai backends install "ocifile://$(abspath ./backend-images/kimodocpp.tar)"
 
+.NOTPARALLEL: backends/gemxcpp backends/gemxcpp-darwin
+docker-build-backends: docker-build-gemxcpp
+
+backends/gemxcpp-darwin:
+	BACKEND=gemxcpp BUILD_TYPE=cpu $(MAKE) build-darwin-go-backend
+	./local-ai backends install "ocifile://$(abspath ./backend-images/gemxcpp.tar)"
+
 $(eval $(call generate-docker-build-target,$(BACKEND_WHISPER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_CRISPASR)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PARAKEET_CPP)))

README.md

@@ -256,6 +256,7 @@ Most backends wrap a best-in-class upstream engine. A handful of them are native
 | [face-detect.cpp](https://github.com/mudler/face-detect.cpp) | Face detection, recognition, demographics and anti-spoofing (SCRFD/ArcFace, YuNet/SFace), replacing the Python insightface backend |
 | [free-splatter.cpp](https://github.com/localai-org/free-splatter.cpp) | Pose-free 3D reconstruction (FreeSplatter): turns a handful of plain photos into 3D Gaussians, no camera poses or GPU required |
 | [trellis2.cpp](https://github.com/localai-org/trellis2cpp) | C++/GGML port of Microsoft TRELLIS.2: single-image to textured 3D mesh (GLB with PBR materials) |
+| [gem-x.cpp](https://github.com/localai-org/gem-x.cpp) | Live human motion capture with SOMA and SMPL pose streams (CPU/Vulkan) |
 | [kimodo.cpp](https://github.com/localai-org/kimodo.cpp) | C++/GGML text-to-motion on CPU and Vulkan, exported as animated skeleton GLB |
 | [privacy-filter.cpp](https://github.com/localai-org/privacy-filter.cpp) | Standalone GGML PII/NER token-classification engine powering LocalAI's PII redaction tier |
 | [LocalVQE](https://github.com/localai-org/LocalVQE) | Joint acoustic echo cancellation, noise suppression, and dereverberation |

backend/backend.proto

@@ -7,6 +7,7 @@ option java_outer_classname = "LocalAIBackend";
 
 package backend;
 
+
 service Backend {
   rpc Health(HealthMessage) returns (Reply) {}
   rpc Free(HealthMessage) returns (Result) {}
@@ -18,6 +19,7 @@ service Backend {
   rpc UpscaleImage(UpscaleImageRequest) returns (Result) {}
   rpc GenerateVideo(GenerateVideoRequest) returns (Result) {}
   rpc Generate3D(Generate3DRequest) returns (Result) {}
+  rpc MotionStream(stream MotionRequest) returns (stream MotionResponse) {}
   rpc Animate3D(Animate3DRequest) returns (Result) {}
   rpc AudioTranscription(TranscriptRequest) returns (TranscriptResult) {}
   rpc AudioTranscriptionStream(TranscriptRequest) returns (stream TranscriptStreamResponse) {}
@@ -1574,3 +1576,15 @@ message ForwardReply {
   repeated ForwardHeader headers = 2;
   bytes body_chunk = 3;
 }
+
+// Configuration is first; later messages contain input only.
+message MotionRequest {
+  string model_identity = 1;
+  string profile = 2;
+  bytes input = 3; // Serialized public localai.motion.v1.Input.
+}
+message MotionResponse {
+  bytes output = 1; // Serialized public localai.motion.v1.Output.
+  // Backend-only bounded stage timings; not part of the public wire protocol.
+  map<string, double> stage_ms = 2;
+}

backend/go/gemxcpp/CMakeLists.txt

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: MIT
+cmake_minimum_required(VERSION 3.24)
+project(localai_gemx LANGUAGES C CXX)
+set(GEMX_SANITIZERS OFF CACHE BOOL "" FORCE)
+set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
+set(GGML_METAL OFF CACHE BOOL "" FORCE)
+add_subdirectory(sources/gem-x.cpp)
+target_sources(gemx PRIVATE native/bridge.cpp)

backend/go/gemxcpp/Makefile

@@ -0,0 +1,54 @@
+GEMX_REPO?=https://github.com/localai-org/gem-x.cpp
+GEMX_VERSION?=576cab62d18829ae00df5959b8ccf0a1032e0c87
+BUILD_TYPE?=
+GOCMD?=go
+GO_TAGS?=
+JOBS?=4
+CMAKE_ARGS?=
+BUILD_DIR?=build-$(if $(filter vulkan,$(BUILD_TYPE)),vulkan,cpu)
+
+CMAKE_ARGS+=-DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DGGML_METAL=OFF
+ifeq ($(shell uname -s),Darwin)
+CMAKE_ARGS+=-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_RPATH=@loader_path
+endif
+ifeq ($(BUILD_TYPE),vulkan)
+CMAKE_ARGS+=-DGEMX_VULKAN=ON
+else
+CMAKE_ARGS+=-DGEMX_VULKAN=OFF
+endif
+
+.PHONY: all build package test clean purge FORCE
+all: build
+
+FORCE:
+
+sources/gem-x.cpp/.checkout-ready: FORCE
+	@test -d sources/gem-x.cpp/.git || git clone $(GEMX_REPO) sources/gem-x.cpp
+	@if test -f $@ && test "$$(git -C sources/gem-x.cpp rev-parse HEAD)" = "$(GEMX_VERSION)"; then exit 0; fi; \
+		cd sources/gem-x.cpp && (git diff --quiet && git diff --cached --quiet || \
+		{ echo "gem-x.cpp has local source changes; preserve them and run make clean before updating the pin" >&2; exit 1; }) && \
+		git fetch origin $(GEMX_VERSION) && git checkout $(GEMX_VERSION) && \
+		git submodule update --init --recursive --depth 1 && touch .checkout-ready
+
+$(BUILD_DIR)/.built: sources/gem-x.cpp/.checkout-ready CMakeLists.txt native/bridge.cpp Makefile
+	cmake -S . -B $(BUILD_DIR) $(CMAKE_ARGS)
+	cmake --build $(BUILD_DIR) -j$(JOBS)
+	touch $@
+
+
+gemxcpp: main.go gemx.go native.go image_projection.go $(BUILD_DIR)/.built
+	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o $@ ./
+
+package: gemxcpp
+	BUILD_DIR=$(BUILD_DIR) bash package.sh
+
+build: package
+
+test:
+	CGO_ENABLED=0 $(GOCMD) test -v ./...
+
+purge:
+	rm -rf build-native build-cpu build-vulkan build-native-avx2 build-cpu-avx2 build-vulkan-avx2
+
+clean: purge
+	rm -rf package gemxcpp sources

backend/go/gemxcpp/README.md

@@ -0,0 +1,19 @@
+# GEM-X backend
+
+Resident native `gemx_live_*` inference via PureGo. See
+[Motion Capture](../../../docs/content/features/motion.md) for configuration and
+protocol. The upstream pin lives in `Makefile` and is registered with the daily
+bump workflow. Linux CPU/Vulkan and Darwin CPU packages are built; Metal is not
+supported by the upstream runtime. Model assets are not needed for unit tests.
+
+```sh
+make -C backend/go/gemxcpp test
+make backends/gemxcpp
+```
+
+The adapter owns one native pipeline per loaded model, with one active session.
+Each instance loads GEM, ViTPose and YOLOX. No SAM3D Body, mesh export, SONIC,
+physics, or Python dependency is introduced. The C library has no immediate
+inference cancellation; cancellation discards results and releases the session
+when the native call returns. The public stream schema lives in
+`pkg/motion/proto/motion.proto`, independently of backend RPC envelopes.