feat(server): add chat_template_kwargs model setting

Author: abetlen

llama_cpp/server/cli.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import argparse
+import json
 
-from typing import List, Literal, Union, Any, Type, TypeVar
+from typing import List, Literal, Union, Any, Type, TypeVar, Dict
 
 from pydantic import BaseModel
 
@@ -40,6 +41,17 @@ def _contains_list_type(annotation: Type[Any] | None) -> bool:
         return False
 
 
+def _contains_dict_type(annotation: Type[Any] | None) -> bool:
+    origin = getattr(annotation, "__origin__", None)
+
+    if origin is dict or origin is Dict:
+        return True
+    elif origin in (Literal, Union):
+        return any(_contains_dict_type(arg) for arg in annotation.__args__)  # type: ignore
+    else:
+        return False
+
+
 def _parse_bool_arg(arg: str | bytes | bool) -> bool:
     if isinstance(arg, bytes):
         arg = arg.decode("utf-8")
@@ -57,6 +69,16 @@ def _parse_bool_arg(arg: str | bytes | bool) -> bool:
         raise ValueError(f"Invalid boolean argument: {arg}")
 
 
+def _parse_json_object_arg(arg: str | bytes) -> dict[str, Any]:
+    if isinstance(arg, bytes):
+        arg = arg.decode("utf-8")
+
+    value = json.loads(arg)
+    if not isinstance(value, dict):
+        raise ValueError(f"Invalid JSON object argument: {arg}")
+    return value
+
+
 def add_args_from_model(parser: argparse.ArgumentParser, model: Type[BaseModel]):
     """Add arguments from a pydantic model to an argparse parser."""
 
@@ -68,7 +90,15 @@ def add_args_from_model(parser: argparse.ArgumentParser, model: Type[BaseModel])
             _get_base_type(field.annotation) if field.annotation is not None else str
         )
         list_type = _contains_list_type(field.annotation)
-        if base_type is not bool:
+        dict_type = _contains_dict_type(field.annotation)
+        if dict_type:
+            parser.add_argument(
+                f"--{name}",
+                dest=name,
+                type=_parse_json_object_arg,
+                help=description,
+            )
+        elif base_type is not bool:
             parser.add_argument(
                 f"--{name}",
                 dest=name,

llama_cpp/server/model.py

@@ -2,7 +2,7 @@
 
 import json
 
-from typing import Dict, Optional, Union, List
+from typing import Any, Dict, Optional, Union, List
 
 import llama_cpp
 import llama_cpp.llama_speculative as llama_speculative
@@ -11,6 +11,29 @@
 from llama_cpp.server.settings import ModelSettings
 
 
+def _resolve_chat_handler(
+    model: llama_cpp.Llama,
+) -> llama_cpp.llama_chat_format.LlamaChatCompletionHandler:
+    chat_handler = (
+        model.chat_handler
+        or model._chat_handlers.get(model.chat_format)
+        or llama_cpp.llama_chat_format.get_chat_completion_handler(model.chat_format)
+    )
+    return chat_handler
+
+
+def _chat_handler_with_kwargs(
+    chat_handler: llama_cpp.llama_chat_format.LlamaChatCompletionHandler,
+    chat_template_kwargs: Dict[str, Any],
+) -> llama_cpp.llama_chat_format.LlamaChatCompletionHandler:
+    def handler(*args: Any, **kwargs: Any):
+        merged_kwargs = dict(chat_template_kwargs)
+        merged_kwargs.update(kwargs)
+        return chat_handler(*args, **merged_kwargs)
+
+    return handler
+
+
 class LlamaProxy:
     def __init__(self, models: List[ModelSettings]) -> None:
         assert len(models) > 0, "No models provided!"
@@ -299,6 +322,10 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
             # Misc
             verbose=settings.verbose,
         )
+        if settings.chat_template_kwargs:
+            _model.chat_handler = _chat_handler_with_kwargs(
+                _resolve_chat_handler(_model), settings.chat_template_kwargs
+            )
         if settings.cache:
             if settings.cache_type == "disk":
                 if settings.verbose:

llama_cpp/server/settings.py

@@ -2,7 +2,7 @@
 
 import multiprocessing
 
-from typing import Optional, List, Literal, Union, Dict, cast
+from typing import Any, Optional, List, Literal, Union, Dict, cast
 from typing_extensions import Self
 
 from pydantic import Field, model_validator
@@ -131,6 +131,10 @@ class ModelSettings(BaseSettings):
         default=None,
         description="Chat format to use.",
     )
+    chat_template_kwargs: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description="Extra keyword arguments forwarded to chat templates at model load time. Matches llama.cpp server `chat_template_kwargs`.",
+    )
     clip_model_path: Optional[str] = Field(
         default=None,
         description="Path to a CLIP model to use for multi-modal chat completion.",

tests/test_server_model.py

@@ -0,0 +1,66 @@
+import argparse
+
+from llama_cpp.server.cli import add_args_from_model, parse_model_from_args
+from llama_cpp.server.model import _chat_handler_with_kwargs
+from llama_cpp.server.settings import ModelSettings
+
+
+def test_model_settings_accepts_chat_template_kwargs():
+    settings = ModelSettings(
+        model="test.gguf",
+        chat_template_kwargs={
+            "enable_thinking": True,
+            "reasoning_effort": "low",
+        },
+    )
+
+    assert settings.chat_template_kwargs == {
+        "enable_thinking": True,
+        "reasoning_effort": "low",
+    }
+
+
+def test_cli_parses_chat_template_kwargs_json():
+    parser = argparse.ArgumentParser()
+    add_args_from_model(parser, ModelSettings)
+
+    args = parser.parse_args(
+        [
+            "--model",
+            "test.gguf",
+            "--chat_template_kwargs",
+            '{"enable_thinking": true, "reasoning_effort": "low"}',
+        ]
+    )
+    settings = parse_model_from_args(ModelSettings, args)
+
+    assert settings.chat_template_kwargs == {
+        "enable_thinking": True,
+        "reasoning_effort": "low",
+    }
+
+
+def test_chat_handler_with_kwargs_merges_defaults_and_request_kwargs():
+    captured = {}
+
+    def base_handler(*args, **kwargs):
+        captured["args"] = args
+        captured["kwargs"] = kwargs
+        return "ok"
+
+    wrapped = _chat_handler_with_kwargs(
+        base_handler,
+        {
+            "enable_thinking": True,
+            "reasoning_effort": "medium",
+        },
+    )
+
+    result = wrapped(reasoning_effort="high", extra_flag="x")
+
+    assert result == "ok"
+    assert captured["kwargs"] == {
+        "enable_thinking": True,
+        "reasoning_effort": "high",
+        "extra_flag": "x",
+    }