From 0e525727ebb35ce285625bd465c23658786835a0 Mon Sep 17 00:00:00 2001 From: mudler <2420543+mudler@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:07:16 +0000 Subject: [PATCH 1/2] :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 2ced9a9b725f..5c02e597ba10 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=de7fa0a3c6a2e1b4cd9f22eb8d6bf5b12dbdb63b +LLAMA_VERSION?=f2cc7282ce0042641df55e0e563b75e824e2b647 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From 667f01754ecccc3c76617fa99921d7e979b70706 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:10:45 +0000 Subject: [PATCH 2/2] fix(llama-cpp): preserve chat session state Newer llama.cpp parsers return template state outside the request JSON. Pass that state to both inference tasks and response readers so grammar, tool calls, and reasoning remain available in both chat paths. Detect the parser API during preparation to preserve older fork builds. Assisted-by: Codex:gpt-6 --- backend/cpp/llama-cpp/grpc-server.cpp | 32 ++++++++++++++++++++++++ backend/cpp/llama-cpp/prepare.sh | 7 ++++++ docs/content/features/text-generation.md | 5 +++- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index a3e8f0dcbe38..f6bc0f34d2f3 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -1783,6 +1783,9 @@ class BackendServiceImpl final : public backend::Backend::Service { std::string prompt_str; std::vector files; // Declare files early so it's accessible in both branches +#if LOCALAI_HAS_CHAT_SESSION + common_chat_session chat_session; +#endif // Handle chat templates when UseTokenizerTemplate is enabled and Messages are provided if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_params.tmpls != nullptr) { // Convert proto Messages to JSON format compatible with oaicompat_chat_params_parse @@ -2038,7 +2041,12 @@ class BackendServiceImpl final : public backend::Backend::Service { } } +#if LOCALAI_HAS_CHAT_SESSION + json parsed_data = oaicompat_chat_params_parse( + ctx_server.impl->vocab, body_json, ctx_server.impl->chat_params, files, chat_session); +#else json parsed_data = oaicompat_chat_params_parse(body_json, ctx_server.impl->chat_params, files); +#endif // Debug: Log tools after template processing if (parsed_data.contains("tools")) { @@ -2157,6 +2165,10 @@ class BackendServiceImpl final : public backend::Backend::Service { ctx_server.get_meta().slot_n_ctx, ctx_server.get_meta().logit_bias_eog, data); +#endif +#if LOCALAI_HAS_CHAT_SESSION + // The parser now keeps grammar, stops and message spans in the session. + task.apply_chat_session(chat_session); #endif task.id_slot = json_value(data, "id_slot", -1); @@ -2171,7 +2183,11 @@ class BackendServiceImpl final : public backend::Backend::Service { tasks.push_back(std::move(task)); } +#if LOCALAI_HAS_CHAT_SESSION + rd.post_tasks(std::move(tasks), chat_session); +#else rd.post_tasks(std::move(tasks)); +#endif } catch (const std::exception & e) { return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, e.what()); } @@ -2368,6 +2384,9 @@ class BackendServiceImpl final : public backend::Backend::Service { std::string prompt_str; std::vector files; // Declare files early so it's accessible in both branches +#if LOCALAI_HAS_CHAT_SESSION + common_chat_session chat_session; +#endif // Handle chat templates when UseTokenizerTemplate is enabled and Messages are provided if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_params.tmpls != nullptr) { // Convert proto Messages to JSON format compatible with oaicompat_chat_params_parse @@ -2624,7 +2643,12 @@ class BackendServiceImpl final : public backend::Backend::Service { } } +#if LOCALAI_HAS_CHAT_SESSION + json parsed_data = oaicompat_chat_params_parse( + ctx_server.impl->vocab, body_json, ctx_server.impl->chat_params, files, chat_session); +#else json parsed_data = oaicompat_chat_params_parse(body_json, ctx_server.impl->chat_params, files); +#endif // Debug: Log tools after template processing if (parsed_data.contains("tools")) { @@ -2746,6 +2770,10 @@ class BackendServiceImpl final : public backend::Backend::Service { ctx_server.get_meta().slot_n_ctx, ctx_server.get_meta().logit_bias_eog, data); +#endif +#if LOCALAI_HAS_CHAT_SESSION + // The parser now keeps grammar, stops and message spans in the session. + task.apply_chat_session(chat_session); #endif task.id_slot = json_value(data, "id_slot", -1); @@ -2758,7 +2786,11 @@ class BackendServiceImpl final : public backend::Backend::Service { tasks.push_back(std::move(task)); } +#if LOCALAI_HAS_CHAT_SESSION + rd.post_tasks(std::move(tasks), chat_session); +#else rd.post_tasks(std::move(tasks)); +#endif } catch (const std::exception & e) { return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, e.what()); } diff --git a/backend/cpp/llama-cpp/prepare.sh b/backend/cpp/llama-cpp/prepare.sh index bec73a234141..1076f48a7093 100644 --- a/backend/cpp/llama-cpp/prepare.sh +++ b/backend/cpp/llama-cpp/prepare.sh @@ -109,10 +109,17 @@ if grep -q "llm_add_n_cpu_ffn_overrides" llama.cpp/common/common.h; then else HAS_N_CPU_FFN_HELPER=0 fi +# Newer servers return template state separately from the request JSON. +if grep -q "common_chat_session & out_session" llama.cpp/tools/server/server-common.h; then + HAS_CHAT_SESSION=1 +else + HAS_CHAT_SESSION=0 +fi cat > llama.cpp/tools/grpc-server/llama_compat.h <