From 4b89e2c9e245536e111d4135e6b528d073bc5ff3 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 9 Oct 2026 06:43:08 +0000 Subject: [PATCH] fix(gallery): place context options at the root The Qwen3 template nests context_size, f16, and mmap under parameters, where the typed model configuration ignores them. Move these options to the root and correct three model-specific context overrides. Test the real gallery data and generated qwen3-vl-4b-thinking YAML. Document manual repair because template updates do not rewrite installed configurations. Fixes #12528 Assisted-by: nib:gpt-6-astra Signed-off-by: Ettore Di Giacinto --- core/gallery/config_placement_test.go | 115 +++++++++++++++++++++++++ docs/content/features/model-gallery.md | 25 ++++++ gallery/index.yaml | 6 +- gallery/qwen3.yaml | 7 +- 4 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 core/gallery/config_placement_test.go diff --git a/core/gallery/config_placement_test.go b/core/gallery/config_placement_test.go new file mode 100644 index 000000000000..0c99a724f7ba --- /dev/null +++ b/core/gallery/config_placement_test.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: MIT + +package gallery_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +// These ModelConfig fields are not PredictionOptions. The permissive loader +// silently ignores them when a template or override nests them in parameters. +func misplacedRootOptions(cfg map[string]any, source string) []string { + params, _ := cfg["parameters"].(map[string]any) + var violations []string + for _, key := range []string{"context_size", "f16", "mmap"} { + if _, exists := params[key]; exists { + violations = append(violations, fmt.Sprintf("%s: parameters.%s belongs at the root", source, key)) + } + } + return violations +} + +var _ = Describe("gallery config placement", func() { + It("keeps known root options out of qwen3 template parameters", func() { + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "qwen3.yaml")) + Expect(err).NotTo(HaveOccurred()) + var definition gallery.ModelConfig + Expect(yaml.Unmarshal(data, &definition)).To(Succeed()) + var cfg map[string]any + Expect(yaml.Unmarshal([]byte(definition.ConfigFile), &cfg)).To(Succeed()) + Expect(misplacedRootOptions(cfg, "qwen3.yaml")).To(BeEmpty()) + }) + + It("keeps known root options out of index override parameters", func() { + entries, err := loadGalleryIndex() + Expect(err).NotTo(HaveOccurred()) + Expect(entries).NotTo(BeEmpty()) + var violations []string + for _, entry := range entries { + violations = append(violations, misplacedRootOptions(entry.Overrides, entry.Name)...) + } + Expect(violations).To(BeEmpty(), strings.Join(violations, "\n")) + }) + + DescribeTable("preserves model-specific context overrides", func(name string, contextSize int) { + entries, err := loadGalleryIndex() + Expect(err).NotTo(HaveOccurred()) + entry, found := indexEntriesByName(entries)[name] + Expect(found).To(BeTrue()) + Expect(entry.Overrides).To(HaveKeyWithValue("context_size", contextSize)) + data, err := yaml.Marshal(entry.Overrides) + Expect(err).NotTo(HaveOccurred()) + var loaded config.ModelConfig + Expect(yaml.Unmarshal(data, &loaded)).To(Succeed()) + Expect(loaded.ContextSize).To(HaveValue(Equal(contextSize))) + Expect(loaded.Model).NotTo(BeEmpty()) + }, + Entry("Supra2", "supra2-100m-instruct", 2048), + Entry("Shieldstral Q4", "shieldstral-1.0-3b", 32768), + Entry("Shieldstral Q8", "shieldstral-1.0-3b-q8", 32768), + ) + + It("installs qwen3-vl-4b-thinking with root options and nested model", func() { + entries, err := loadGalleryIndex() + Expect(err).NotTo(HaveOccurred()) + entry, found := indexEntriesByName(entries)["qwen3-vl-4b-thinking"] + Expect(found).To(BeTrue()) + Expect(entry.URL).To(Equal("github:mudler/LocalAI/gallery/qwen3.yaml@master")) + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "qwen3.yaml")) + Expect(err).NotTo(HaveOccurred()) + var definition gallery.ModelConfig + Expect(yaml.Unmarshal(data, &definition)).To(Succeed()) + + modelsPath := GinkgoT().TempDir() + state, err := system.GetSystemState(system.WithModelPath(modelsPath)) + Expect(err).NotTo(HaveOccurred()) + // Stand in for downloaded weights without changing the real overrides or + // template. Existing files with no checksum need no network access. + Expect(entry.AdditionalFiles).NotTo(BeEmpty()) + for _, file := range entry.AdditionalFiles { + path := filepath.Join(modelsPath, file.Filename) + Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed()) + Expect(os.WriteFile(path, []byte("test weights"), 0600)).To(Succeed()) + file.SHA256 = "" + definition.Files = append(definition.Files, file) + } + _, err = gallery.InstallModel(context.Background(), state, entry.Name, &definition, entry.Overrides, nil, false) + Expect(err).NotTo(HaveOccurred()) + data, err = os.ReadFile(filepath.Join(modelsPath, entry.Name+".yaml")) + Expect(err).NotTo(HaveOccurred()) + var persisted map[string]any + Expect(yaml.Unmarshal(data, &persisted)).To(Succeed()) + Expect(persisted).To(HaveKeyWithValue("context_size", 8192)) + Expect(persisted).To(HaveKeyWithValue("f16", true)) + Expect(persisted).To(HaveKeyWithValue("mmap", true)) + Expect(misplacedRootOptions(persisted, entry.Name)).To(BeEmpty()) + Expect(persisted["parameters"]).To(HaveKeyWithValue("model", "Qwen3-VL-4B-Thinking-Q4_K_M.gguf")) + Expect(persisted).NotTo(HaveKey("model")) + var loaded config.ModelConfig + Expect(yaml.Unmarshal(data, &loaded)).To(Succeed()) + Expect(loaded.ContextSize).To(HaveValue(Equal(8192))) + Expect(loaded.F16).To(HaveValue(BeTrue())) + Expect(loaded.MMap).To(HaveValue(BeTrue())) + }) +}) diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index a9e10330fa52..a5af4a904d2a 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -44,6 +44,31 @@ Both views store the view, search, filter, and selection in the URL. Installing from Explore does not move you away from the catalog; the entry updates in place when the operation finishes. +### Repairing an existing model configuration + +Gallery template updates do not rewrite YAML files for already installed models. +If an older installation puts `context_size`, `f16`, or `mmap` under +`parameters`, move those fields to the YAML root. LocalAI reads them at the +root; nested copies are ignored. Keep `model` and sampling options under +`parameters`. + +For example, the relevant fields in `models/qwen3-vl-4b-thinking.yaml` should be: + +```yaml +context_size: 8192 +f16: true +mmap: true +parameters: + model: Qwen3-VL-4B-Thinking-Q4_K_M.gguf +``` + +Back up the installed YAML before editing it, preserve its other settings, and +remove the misplaced nested copies. Use your configured models directory if it +is not `models`. Preserve model-specific context values: `supra2-100m-instruct` +uses `2048`, while `shieldstral-1.0-3b` and `shieldstral-1.0-3b-q8` use `32768`. +Restart LocalAI after editing so it reads the repaired configuration and reloads +the model. You do not need to download the weights again. + ### A model's own page Every model also has a page of its own at `/app/models/`, so it can be diff --git a/gallery/index.yaml b/gallery/index.yaml index 17a7cdfa1dfd..a68ad95e59c0 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1669,8 +1669,8 @@ - english last_checked: "2026-08-11" overrides: + context_size: 2048 parameters: - context_size: 2048 model: Supra2-100M-SFT-F16.gguf files: - filename: Supra2-100M-SFT-F16.gguf @@ -44200,8 +44200,8 @@ mmproj: llama-cpp/mmproj/Shieldstral-1.0-3B/mmproj-F16.gguf options: - use_jinja:true + context_size: 32768 parameters: - context_size: 32768 model: llama-cpp/models/Shieldstral-1.0-3B/Shieldstral-1.0-3B-Q4_K_M.gguf temperature: 0 max_tokens: 1 @@ -44228,8 +44228,8 @@ mmproj: llama-cpp/mmproj/Shieldstral-1.0-3B/mmproj-F16.gguf options: - use_jinja:true + context_size: 32768 parameters: - context_size: 32768 model: llama-cpp/models/Shieldstral-1.0-3B-q8/Shieldstral-1.0-3B-Q8_0.gguf temperature: 0 max_tokens: 1 diff --git a/gallery/qwen3.yaml b/gallery/qwen3.yaml index 62bae155a55c..e10dd81e9256 100644 --- a/gallery/qwen3.yaml +++ b/gallery/qwen3.yaml @@ -2,10 +2,9 @@ config_file: | backend: llama-cpp known_usecases: - chat - parameters: - context_size: 8192 - f16: true - mmap: true + context_size: 8192 + f16: true + mmap: true stopwords: - <|im_end|> -