chore: ⬆️ Update ggml-org/llama.cpp to f2cc7282ce0042641df55e0e563b75e824e2b647

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=de7fa0a3c6a2e1b4cd9f22eb8d6bf5b12dbdb63b
+LLAMA_VERSION?=f2cc7282ce0042641df55e0e563b75e824e2b647
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -1783,6 +1783,9 @@ class BackendServiceImpl final : public backend::Backend::Service {
 
             std::string prompt_str;
             std::vector<raw_buffer> files; // Declare files early so it's accessible in both branches
+#if LOCALAI_HAS_CHAT_SESSION
+            common_chat_session chat_session;
+#endif
             // Handle chat templates when UseTokenizerTemplate is enabled and Messages are provided
             if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_params.tmpls != nullptr) {
                 // Convert proto Messages to JSON format compatible with oaicompat_chat_params_parse
@@ -2038,7 +2041,12 @@ class BackendServiceImpl final : public backend::Backend::Service {
                     }
                 }
 
+#if LOCALAI_HAS_CHAT_SESSION
+                json parsed_data = oaicompat_chat_params_parse(
+                        ctx_server.impl->vocab, body_json, ctx_server.impl->chat_params, files, chat_session);
+#else
                 json parsed_data = oaicompat_chat_params_parse(body_json, ctx_server.impl->chat_params, files);
+#endif
 
                 // Debug: Log tools after template processing
                 if (parsed_data.contains("tools")) {
@@ -2157,6 +2165,10 @@ class BackendServiceImpl final : public backend::Backend::Service {
                         ctx_server.get_meta().slot_n_ctx,
                         ctx_server.get_meta().logit_bias_eog,
                         data);
+#endif
+#if LOCALAI_HAS_CHAT_SESSION
+                // The parser now keeps grammar, stops and message spans in the session.
+                task.apply_chat_session(chat_session);
 #endif
                 task.id_slot = json_value(data, "id_slot", -1);
 
@@ -2171,7 +2183,11 @@ class BackendServiceImpl final : public backend::Backend::Service {
                 tasks.push_back(std::move(task));
             }
 
+#if LOCALAI_HAS_CHAT_SESSION
+            rd.post_tasks(std::move(tasks), chat_session);
+#else
             rd.post_tasks(std::move(tasks));
+#endif
         } catch (const std::exception & e) {
             return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, e.what());
         }
@@ -2368,6 +2384,9 @@ class BackendServiceImpl final : public backend::Backend::Service {
 
             std::string prompt_str;
             std::vector<raw_buffer> files; // Declare files early so it's accessible in both branches
+#if LOCALAI_HAS_CHAT_SESSION
+            common_chat_session chat_session;
+#endif
             // Handle chat templates when UseTokenizerTemplate is enabled and Messages are provided
             if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_params.tmpls != nullptr) {
                 // Convert proto Messages to JSON format compatible with oaicompat_chat_params_parse
@@ -2624,7 +2643,12 @@ class BackendServiceImpl final : public backend::Backend::Service {
                     }
                 }
 
+#if LOCALAI_HAS_CHAT_SESSION
+                json parsed_data = oaicompat_chat_params_parse(
+                        ctx_server.impl->vocab, body_json, ctx_server.impl->chat_params, files, chat_session);
+#else
                 json parsed_data = oaicompat_chat_params_parse(body_json, ctx_server.impl->chat_params, files);
+#endif
 
                 // Debug: Log tools after template processing
                 if (parsed_data.contains("tools")) {
@@ -2746,6 +2770,10 @@ class BackendServiceImpl final : public backend::Backend::Service {
                         ctx_server.get_meta().slot_n_ctx,
                         ctx_server.get_meta().logit_bias_eog,
                         data);
+#endif
+#if LOCALAI_HAS_CHAT_SESSION
+                // The parser now keeps grammar, stops and message spans in the session.
+                task.apply_chat_session(chat_session);
 #endif
                 task.id_slot = json_value(data, "id_slot", -1);
 
@@ -2758,7 +2786,11 @@ class BackendServiceImpl final : public backend::Backend::Service {
                 tasks.push_back(std::move(task));
             }
 
+#if LOCALAI_HAS_CHAT_SESSION
+            rd.post_tasks(std::move(tasks), chat_session);
+#else
             rd.post_tasks(std::move(tasks));
+#endif
         } catch (const std::exception & e) {
             return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, e.what());
         }

backend/cpp/llama-cpp/prepare.sh

@@ -109,10 +109,17 @@ if grep -q "llm_add_n_cpu_ffn_overrides" llama.cpp/common/common.h; then
 else
     HAS_N_CPU_FFN_HELPER=0
 fi
+# Newer servers return template state separately from the request JSON.
+if grep -q "common_chat_session & out_session" llama.cpp/tools/server/server-common.h; then
+    HAS_CHAT_SESSION=1
+else
+    HAS_CHAT_SESSION=0
+fi
 cat > llama.cpp/tools/grpc-server/llama_compat.h <<EOF
 // Generated by backend/cpp/llama-cpp/prepare.sh. Do not edit.
 #pragma once
 #define LOCALAI_LEGACY_LOAD_MODE ${LEGACY_LOAD_MODE}
+#define LOCALAI_HAS_CHAT_SESSION ${HAS_CHAT_SESSION}
 #define LOCALAI_HAS_SERVER_METRICS ${HAS_SERVER_METRICS}
 #define LOCALAI_HAS_MTMD_INIT_OPT ${HAS_MTMD_INIT_OPT}
 #define LOCALAI_HAS_N_CPU_FFN_HELPER ${HAS_N_CPU_FFN_HELPER}

docs/content/features/text-generation.md

@@ -548,7 +548,10 @@ It is sufficient to copy the `ggml` or `gguf` model files in the `models` folder
 
 [You can optionally create an associated YAML]({{%relref "advanced" %}}) model config file to tune the model's parameters or apply a template to the prompt.
 
-Prompt templates are useful for models that are fine-tuned towards a specific prompt. 
+Prompt templates are useful for models that are fine-tuned towards a specific prompt.
+
+With the llama.cpp backend, streaming and non-streaming chat requests preserve
+the template's grammar, stop strings, and message boundaries.
 
 ##### Automatic setup