From ce174073ba93296b87005eec174c3ab1507a7cbb Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 18:05:59 -0500 Subject: [PATCH 01/10] Add common_chat_parse function for message parsing Fixes #17771 --- common/chat.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/common/chat.cpp b/common/chat.cpp index 41a5bb42d51..79845a2fe4d 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -2596,3 +2596,14 @@ common_chat_params common_chat_templates_apply( ? common_chat_templates_apply_jinja(tmpls, inputs) : common_chat_templates_apply_legacy(tmpls, inputs); } + +common_chat_msg common_chat_parse( + const std::string & input, + bool is_partial, + const common_chat_syntax & syntax) +{ + common_chat_msg_parser parser(input, is_partial, syntax); + common_chat_parse(parser); + return parser.result(); +} + From 2b93ea3aa3a639ed26fade95dc1a1868831b522f Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 18:56:12 -0500 Subject: [PATCH 02/10] Add common_chat_parse function declaration Fixes #17771 --- common/chat-parser.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/chat-parser.h b/common/chat-parser.h index 78c4b74c2db..ed274017f3e 100644 --- a/common/chat-parser.h +++ b/common/chat-parser.h @@ -131,3 +131,5 @@ class common_chat_msg_parser { void clear_tools(); }; + +common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax); From 82732fe002045a0807c084dc4b979bd8201dd3f7 Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 19:02:12 -0500 Subject: [PATCH 03/10] Remove unused common_chat_parse function declaration --- common/chat-parser.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/chat-parser.h b/common/chat-parser.h index ed274017f3e..78c4b74c2db 100644 --- a/common/chat-parser.h +++ b/common/chat-parser.h @@ -131,5 +131,3 @@ class common_chat_msg_parser { void clear_tools(); }; - -common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax); From f8d38061628f5389f1130cd983dd4b9c21c0b27c Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 19:05:28 -0500 Subject: [PATCH 04/10] Add new parsing functions for chat messages --- common/chat-parser.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/chat-parser.h b/common/chat-parser.h index 78c4b74c2db..1f7c5d197c2 100644 --- a/common/chat-parser.h +++ b/common/chat-parser.h @@ -131,3 +131,7 @@ class common_chat_msg_parser { void clear_tools(); }; + +common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax); +common_chat_msg common_chat_peg_parse(const common_peg_arena & parser, const std::string & input, bool is_partial, const common_chat_syntax & syntax); + From 98015388deeaad89d306cf499051be60d1b1c910 Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 19:05:43 -0500 Subject: [PATCH 05/10] Remove deprecated chat parse functions Removed deprecated chat parsing functions. --- common/chat.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/chat.h b/common/chat.h index 6085510a402..7729885961d 100644 --- a/common/chat.h +++ b/common/chat.h @@ -213,8 +213,6 @@ std::string common_chat_format_example( const char* common_chat_format_name(common_chat_format format); const char* common_reasoning_format_name(common_reasoning_format format); common_reasoning_format common_reasoning_format_from_name(const std::string & format); -common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax); -common_chat_msg common_chat_peg_parse(const common_peg_arena & parser, const std::string & input, bool is_partial, const common_chat_syntax & syntax); common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice); From f08a068206fe8627d690ba18662ee13bb2f0cb25 Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 19:10:58 -0500 Subject: [PATCH 06/10] Remove common_chat_parse function Removed the common_chat_parse function and its implementation. --- common/chat.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 79845a2fe4d..41a5bb42d51 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -2596,14 +2596,3 @@ common_chat_params common_chat_templates_apply( ? common_chat_templates_apply_jinja(tmpls, inputs) : common_chat_templates_apply_legacy(tmpls, inputs); } - -common_chat_msg common_chat_parse( - const std::string & input, - bool is_partial, - const common_chat_syntax & syntax) -{ - common_chat_msg_parser parser(input, is_partial, syntax); - common_chat_parse(parser); - return parser.result(); -} - From a2853260ffea0d622b5d8600e7bb7b2dc75cee87 Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 19:17:53 -0500 Subject: [PATCH 07/10] Update chat-parser.cpp --- common/chat-parser.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/chat-parser.cpp b/common/chat-parser.cpp index fe3e80037f4..d3f763756db 100644 --- a/common/chat-parser.cpp +++ b/common/chat-parser.cpp @@ -1399,7 +1399,7 @@ static void common_chat_parse_content_only(common_chat_msg_parser & builder) { builder.add_content(builder.consume_rest()); } -static void common_chat_parse(common_chat_msg_parser & builder) { +static void common_chat_msg_parse(common_chat_msg_parser & builder) { LOG_DBG("Parsing input with format %s: %s\n", common_chat_format_name(builder.syntax().format), builder.input().c_str()); switch (builder.syntax().format) { @@ -1492,7 +1492,7 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co } common_chat_msg_parser builder(input, is_partial, syntax); try { - common_chat_parse(builder); + common_chat_msg_parse(builder); } catch (const common_chat_msg_partial_exception & ex) { LOG_DBG("Partial parse: %s\n", ex.what()); if (!is_partial) { From dd4c0b4788bfddd5d1d86864a97ee8e5fb92fd80 Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 20:10:45 -0500 Subject: [PATCH 08/10] Add peg-parser.h include to chat-parser.h --- common/chat-parser.h | 1 + 1 file changed, 1 insertion(+) diff --git a/common/chat-parser.h b/common/chat-parser.h index 1f7c5d197c2..d35533b1604 100644 --- a/common/chat-parser.h +++ b/common/chat-parser.h @@ -1,6 +1,7 @@ #pragma once #include "chat.h" +#include "peg-parser.h" #include "chat-parser-xml-toolcall.h" #include "json-partial.h" #include "regex-partial.h" From d1de95910c65c51cceb1bca6eed5bb33d5ebab9d Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 21:09:39 -0500 Subject: [PATCH 09/10] Add chat-parser.h include to server-task.cpp --- tools/server/server-task.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index df066264778..e7b3cf8d51b 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -4,6 +4,7 @@ #include "common.h" #include "llama.h" #include "chat.h" +#include "chat-parser.h" #include "sampling.h" #include "json-schema-to-grammar.h" From 60dfa033fd9f851e2987047d7bb0c5f706afbab1 Mon Sep 17 00:00:00 2001 From: Sheldon Robinson Date: Thu, 4 Dec 2025 21:37:11 -0500 Subject: [PATCH 10/10] Add include for chat-parser.h in test-chat.cpp --- tests/test-chat.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 62dd1583fa1..10fff792498 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -6,6 +6,7 @@ // cmake -B build && cmake --build build --parallel && ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null // #include "chat.h" +#include "chat-parser.h" #include "log.h"