From 25e43258e3abd3e70f3c46c2c4b462bb0efc8d4f Mon Sep 17 00:00:00 2001 From: Niklas Volcz Date: Sat, 7 Nov 2026 16:37:17 +0100 Subject: [PATCH] fix(server): reset request settings to the model's defaults on every request A setting one request gave, such as think or temperature, stayed on the shared engine and applied to every later request that left it out, from any client: after one request with think: true, requests without it still got thinking. The docs warned about this ("Settings stick"). The engine remembers its sampler config and whether thinking is on once the model is loaded, and the server puts them back before applying a request's settings, on the chat endpoints and on /api/generate. As the sampler is recreated, every request also starts with an empty penalty window. Qwen3.5-Omni's set_sampler override did the same as the base one, so it is gone. The reasoning_effort levels of GPT-OSS and Hy-Memory, conversational_mode and image-max-tokens live in model-specific fields and still stay in effect until the model is reloaded. The docs, a code comment and the API tests' notes describe the new behaviour, and test_reasoning.think_does_not_carry_over checks it. --- docs/docs/instructions/server/openapi.md | 2 +- .../server/reasoning_and_tools.md | 2 +- .../instructions/server/request_parameters.md | 16 ++++++++-------- src/common/AutoModel/automodel.cpp | 16 ++++++++++++++++ .../AutoModel/modeling_qwen3_5_omni.cpp | 5 ----- src/include/AutoModel/automodel.hpp | 14 ++++++++++++++ .../AutoModel/modeling_qwen3_5_omni.hpp | 3 +-- src/server/rest_handler.cpp | 6 +++++- src/test/openai_api_test/README.md | 2 +- src/test/openai_api_test/flm_api.py | 4 +--- src/test/openai_api_test/test_reasoning.py | 19 +++++++++++++++++++ src/test/openai_api_test/test_sampling.py | 2 +- 12 files changed, 68 insertions(+), 23 deletions(-) diff --git a/docs/docs/instructions/server/openapi.md b/docs/docs/instructions/server/openapi.md index 0aa387e93..a1445e247 100644 --- a/docs/docs/instructions/server/openapi.md +++ b/docs/docs/instructions/server/openapi.md @@ -325,7 +325,7 @@ import gc gc.collect() ``` -> ⚠️ Sampling settings stay in effect for later requests that don't send them, until the model is reloaded; `seed` applies to its own request only. What each field does, `seed`, the penalty window and model defaults: [Request Parameters & Long Prompts](https://fastflowlm.com/docs/instructions/server/request_parameters/). +> ℹ️ Sampling settings apply only to the request that sends them; a request that leaves one out gets the model's default. What each field does, `seed`, the penalty window and model defaults: [Request Parameters & Long Prompts](https://fastflowlm.com/docs/instructions/server/request_parameters/). --- diff --git a/docs/docs/instructions/server/reasoning_and_tools.md b/docs/docs/instructions/server/reasoning_and_tools.md index 47f1f8d53..ef93f79d0 100644 --- a/docs/docs/instructions/server/reasoning_and_tools.md +++ b/docs/docs/instructions/server/reasoning_and_tools.md @@ -242,7 +242,7 @@ client.chat.completions.create(model="qwen3:4b", messages=msgs, extra_body={"thi client.chat.completions.create(model="gpt-oss:20b", messages=msgs, reasoning_effort="low") ``` -> ⚠️ **The setting sticks.** `think` and `reasoning_effort` are applied to the loaded model and stay in effect for later requests that don't mention them. If your application mixes thinking and non-thinking calls, send the setting on **every** request. +> ℹ️ **The setting applies to its own request.** A request that doesn't mention `think` or `reasoning_effort` gets the model's default, not what an earlier request set. The `reasoning_effort` levels of GPT-OSS and Hy-Memory are the exception: they stay in effect until the model is reloaded. What changes when thinking is off: diff --git a/docs/docs/instructions/server/request_parameters.md b/docs/docs/instructions/server/request_parameters.md index 3beb85073..e9ad51965 100644 --- a/docs/docs/instructions/server/request_parameters.md +++ b/docs/docs/instructions/server/request_parameters.md @@ -63,7 +63,7 @@ response = client.chat.completions.create( print(response.choices[0].message.content) ``` -> ⚠️ **Settings stick.** Sampling fields are applied to the loaded model and stay in effect for every later request that doesn't send them, including requests from other clients. They return to the model's defaults only when the model is reloaded. If your application mixes settings, send every sampling field on **every** request. `seed` is the exception: it applies only to the request that sends it. (`think` and `reasoning_effort` [stick the same way](https://fastflowlm.com/docs/instructions/server/reasoning_and_tools/#-turning-thinking-on-and-off).) +> ℹ️ **Every request starts from the model's defaults.** A sampling field applies only to the request that sends it. A request that leaves a field out gets the model's default, not what an earlier request set. `think` and `reasoning_effort` [work the same way](https://fastflowlm.com/docs/instructions/server/reasoning_and_tools/#-turning-thinking-on-and-off), except the `reasoning_effort` levels of GPT-OSS and Hy-Memory, which stay in effect until the model is reloaded. > ⚠️ **Out-of-range values are ignored, not rejected.** A negative `temperature` or `presence_penalty`, or a `top_p` above 1, gets HTTP 200: the server logs a warning and keeps the value it had. OpenAI accepts penalties from −2 to 2, but FLM ignores negative ones, so a penalty can't be used to *encourage* repetition. `repeat_last_n` and `seed` are stricter: a value that isn't an integer, or a `repeat_last_n` below −1, is rejected with HTTP 400. @@ -77,7 +77,7 @@ A request that sends no sampling fields uses the model's own defaults, which dif | `gemma4-it:*`, `gemma4-it-core:*`, `gemma4e-flash:*`, `nemotron3.5-bolt:e4b`, `nemotron3.5-bolt-dflash:e4b`, `nemotron3.5-bolt-dflash-flash:e4b` | 1.0 | | every other model | 0 | -Sending `presence_penalty=0` turns the default off, and it stays off until the model is reloaded (see *Settings stick* above). +Sending `presence_penalty=0` turns the default off for that request. --- @@ -132,7 +132,7 @@ response = client.chat.completions.create( ) ``` -Like the other sampling fields, the window size [sticks](#️-sampling-fields) until the model is reloaded, which sets it back to 64. Making the window smaller takes effect at once: tokens it no longer covers stop being penalised from the first token of the request. Ollama-style `options.repeat_last_n` is not read by the `/v1` endpoints: send `repeat_last_n` at the top level. +Like the other sampling fields, the window size applies to its own request; a request without it uses 64. Ollama-style `options.repeat_last_n` is not read by the `/v1` endpoints: send `repeat_last_n` at the top level. --- @@ -146,7 +146,7 @@ response = client.chat.completions.create( messages=[{"role": "user", "content": "Invent a name for a pet dragon."}], temperature=0.8, top_p=0.9, - presence_penalty=0, # send every sampling field: they stick across requests + presence_penalty=0, frequency_penalty=0, seed=42, ) @@ -154,9 +154,9 @@ response = client.chat.completions.create( What `seed` does: -- **It applies to its own request only.** The sampler's random generator restarts from the seed when the request starts. A request without `seed` carries on from wherever the generator was, so it doesn't disturb the next seeded one. -- **It empties the [penalty window](#-the-penalty-window).** Without this, a conversation the server continues from its KV cache would start with the end of the previous reply in the window, and the same seeded request would sample differently depending on what ran before it. -- **It doesn't pin the other sampling fields.** They [stick](#️-sampling-fields) across requests, so another client can change them between your two calls. Send `temperature`, `top_p`, `top_k`, the penalties and `repeat_last_n` with every seeded request. +- **It applies to its own request only.** The sampler's random generator restarts from the seed when the request starts. A request without `seed` gets a generator seeded from the clock. +- **The [penalty window](#-the-penalty-window) starts empty**, as it does for every request, so the same seeded request samples the same way whatever ran before it. +- **The other sampling fields come from the request or the model's defaults**, never from an earlier request, so two calls with the same fields and `seed` give the same text. There is no `system_fingerprint` to compare: non-streaming responses have none, and each stream gets a random one. @@ -242,7 +242,7 @@ Rejected with HTTP 400 (`invalid_request_error`), for streaming requests too: | A field sent as `null` | Treated as not sent. Some clients send `null` for every option they leave unset. | | `stop`, `n`, `logit_bias`, `user` | Ignored, as above. `stop` sequences are not applied, and there is always exactly one choice. | | An `options` object (Ollama style) on a `/v1` endpoint | Ignored. Put sampling fields at the top level of the request. | -| A sampling field outside its range | Ignored with a warning in the server log. The previous value stays in effect ([see above](#️-sampling-fields)). | +| A sampling field outside its range | Ignored with a warning in the server log. The model's default is used ([see above](#️-sampling-fields)). | | A sampling field of the wrong JSON type, such as `"temperature": "0.7"` | The request fails with an error. Send numbers as JSON numbers. | | A malformed `messages`, `tool_choice`, `parallel_tool_calls`, `response_format`, `logprobs`, `top_logprobs`, `seed`, `repeat_last_n` or `continue_final_message` | Rejected with HTTP 400 and `"type": "invalid_request_error"` before anything is generated, for streaming requests too. | diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 1df8005f2..5981769a6 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -1088,6 +1088,22 @@ void AutoModel::set_sampler(sampler_config& sampler_config) { this->sampler.reset(); } this->sampler = std::make_unique(this->lm_config->get("vocab_size"), sampler_config); + this->loaded_sampler_config = sampler_config; +} + +/// \brief Remember the settings a request may change, as the model loaded them +/// \note Call after load_model() +void AutoModel::save_request_settings() { + this->loaded_thinking = this->thinking_enabled(); +} + +/// \brief Put back the settings save_request_settings() remembered: sampling and thinking +/// \note Recreates the sampler, so its penalty window starts empty and, without a seed, its +/// random generator is seeded from the clock +void AutoModel::restore_request_settings() { + sampler_config config = this->loaded_sampler_config; + this->set_sampler(config); + this->configure_parameter("enable_think", this->loaded_thinking); } /// \brief Set the max length diff --git a/src/common/AutoModel/modeling_qwen3_5_omni.cpp b/src/common/AutoModel/modeling_qwen3_5_omni.cpp index 69e6207cc..b2398e388 100644 --- a/src/common/AutoModel/modeling_qwen3_5_omni.cpp +++ b/src/common/AutoModel/modeling_qwen3_5_omni.cpp @@ -711,11 +711,6 @@ buffer Qwen3_5_Omni::say(std::string wav_out_path) { } } -void Qwen3_5_Omni::set_sampler(sampler_config& sampler_config) { - if (this->sampler != nullptr) this->sampler.reset(); - this->sampler = std::make_unique((int)this->lm_config->get("vocab_size"), sampler_config); -} - void Qwen3_5_Omni::clear_context() { this->total_tokens = 0; this->last_token = -1; diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index 5d4f7ba94..a68e4fa78 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -203,6 +203,10 @@ class AutoModel { causal_lm* lm_engine = nullptr; std::unique_ptr tokenizer = nullptr; std::unique_ptr sampler = nullptr; + /// \brief the config set_sampler() last made the sampler from, which load_model() sets + sampler_config loaded_sampler_config; + /// \brief whether thinking was on when save_request_settings() ran + bool loaded_thinking = false; bool is_model_loaded = false; std::string current_model = ""; std::vector token_history; @@ -505,6 +509,16 @@ class AutoModel { /// \param sampler_config the sampler config virtual void set_sampler(sampler_config& sampler_config); + /// \brief Remember the settings a request may change, as the model loaded them + /// \note Call after load_model() + void save_request_settings(); + + /// \brief Put back the settings save_request_settings() remembered: sampling and thinking + /// \note The server calls it before applying a request's settings, so a setting one + /// request leaves out falls back to the model's default rather than to what + /// the previous request asked for. + void restore_request_settings(); + /// \brief Set the max length /// \param MAX_L the max length virtual void set_max_length(unsigned int MAX_L); diff --git a/src/include/AutoModel/modeling_qwen3_5_omni.hpp b/src/include/AutoModel/modeling_qwen3_5_omni.hpp index 4e6f4f095..40c509c32 100644 --- a/src/include/AutoModel/modeling_qwen3_5_omni.hpp +++ b/src/include/AutoModel/modeling_qwen3_5_omni.hpp @@ -52,8 +52,7 @@ class Qwen3_5_Omni : public AutoModel { /// surfaces a clear message instead of crashing. buffer say(std::string wav_out_path = ""); - /// \brief Sampler / context / bookkeeping overrides (engine-driven). - void set_sampler(sampler_config& sampler_config) override; + /// \brief Context / bookkeeping overrides (engine-driven). void clear_context() override; void set_max_length(unsigned int MAX_L) override; int get_current_context_length() override; diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 32c7ddcad..77c771deb 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -341,7 +341,7 @@ static bool logprobs_request_from_openai(const json& request, int& top_n, std::s ///@brief The OpenAI request fields that steer sampling but are not plain numbers to set struct openai_sampling_request { - /// generated tokens the penalties look at, -1 the whole reply; sticks like the other sampling fields + /// generated tokens the penalties look at, -1 the whole reply; applies to its own request, like the other sampling fields std::optional penalty_window; /// restarts the random generator for this request only std::optional seed; @@ -689,6 +689,7 @@ bool RestHandler::ensure_model_loaded(const std::string& model_tag, engine->set_prefix_limits(this->prefix_cache_mb, this->prefix_slots, this->prefix_min_gain); const auto load_started = std::chrono::steady_clock::now(); engine->load_model(supported_models.get_load_path(new_ensure_tag), model_info, ctx_length, preemption, requested_backend); + engine->save_request_settings(); report_load_time(load_started); // Per model, not per process: the flash models take 1024-token @@ -867,6 +868,7 @@ void RestHandler::ensure_embed_model_loaded(const std::string& model_tag) { ///@param options the options JSON object ///@param request the request JSON object void RestHandler::configure_chat_engine_parameters(const json& options, const json& request) { + auto_chat_engine->restore_request_settings(); if (has_value(request, "temperature")) { float temperature = request["temperature"]; auto_chat_engine->set_temperature(temperature); @@ -1036,6 +1038,8 @@ void RestHandler::handle_generate(const json& request, return; } auto load_end_time = time_utils::now(); + // It takes no settings, but must not run with what an earlier chat request set + auto_chat_engine->restore_request_settings(); chat_meta_info_t meta_info; lm_uniform_input_t uniformed_input; diff --git a/src/test/openai_api_test/README.md b/src/test/openai_api_test/README.md index 406461c32..baeb68591 100644 --- a/src/test/openai_api_test/README.md +++ b/src/test/openai_api_test/README.md @@ -54,7 +54,7 @@ Some cases skip by design: ## Notes -- Requests pin `think`, `top_k: 1`, `temperature: 0` and `repeat_last_n: 64` explicitly. FLM applies sampling and thinking settings to the loaded model and **keeps them for later requests**, so after a run the server is left greedy with thinking as the last case set it. Send your own values, or restart the server, before using it interactively. +- Requests pin `think`, `top_k: 1`, `temperature: 0` and `repeat_last_n: 64` explicitly, so a case does not depend on the model's defaults. - The tests check guarantees, not model quality. Under `auto` a model may or may not call a tool, and the tests only check that what it returns is well formed. Under `required`, a forced function, or `response_format`, the grammar makes the shape a guarantee, so any violation is a server bug. - Shared helpers live in `flm_api.py`: the HTTP/SSE client, `fold_stream()` (merges tool call deltas by `index`, as clients should), the contract checks, and a small JSON Schema validator. Tools and schemas are in `fixtures.py`. diff --git a/src/test/openai_api_test/flm_api.py b/src/test/openai_api_test/flm_api.py index 153a42790..98452978a 100644 --- a/src/test/openai_api_test/flm_api.py +++ b/src/test/openai_api_test/flm_api.py @@ -12,9 +12,7 @@ PASS / FAIL / SKIP per case and exiting non-zero on any failure. Every request the tests send pins ``think``, ``top_k``, ``temperature`` and -``repeat_last_n`` explicitly: FastFlowLM applies those to the loaded model and -keeps them for later requests, so a test must never rely on what an earlier one -left set. +``repeat_last_n`` explicitly, so a test does not depend on the model's defaults. """ # Every check here reports on the server under test, not on a caller diff --git a/src/test/openai_api_test/test_reasoning.py b/src/test/openai_api_test/test_reasoning.py index aacaa0058..ff6fb180e 100644 --- a/src/test/openai_api_test/test_reasoning.py +++ b/src/test/openai_api_test/test_reasoning.py @@ -6,6 +6,8 @@ * With ``think: false`` (or ``reasoning_effort: "none"``) there is no ``reasoning_content``, unless the model cannot stop thinking: pass ``--always-thinks`` for those. +* A request without ``think`` gets the model's default, whatever an earlier + request set. * A stream sends every reasoning delta before the first content delta and folds to exactly the non-streaming message, so no half-written tag ever went out as text. @@ -116,7 +118,24 @@ def history_with_reasoning(api: Api) -> None: flm_api.check_message_contract(choice) +@case +def think_does_not_carry_over(api: Api) -> None: + """A request without think gets the model's default, whatever an earlier request set.""" + def thinks(**fields) -> bool: + body = api.body(user(PROMPT), max_tokens=THINK_BUDGET, **fields) + if "think" not in fields: + del body["think"] + return bool(flm_api.check_message_contract(api.ok(body)).get("reasoning_content")) + + default = thinks() + for think in (True, False): + thinks(think=think) + expect(thinks() == default, f"after think={think} a request without think " + f"{'reasoned' if not default else 'did not reason'}, unlike the model's default") + + CASES = flm_api.collect(globals()) + if __name__ == "__main__": flm_api.main(CASES, __doc__.splitlines()[0]) diff --git a/src/test/openai_api_test/test_sampling.py b/src/test/openai_api_test/test_sampling.py index 2ccdbb546..47b225adc 100644 --- a/src/test/openai_api_test/test_sampling.py +++ b/src/test/openai_api_test/test_sampling.py @@ -35,7 +35,7 @@ def no_penalties(**fields: Any) -> dict[str, Any]: - """Penalty fields set to off, overridable; they stick, so every case sends them.""" + """Penalty fields set to off, overridable; every case sends them, so none depends on the model's defaults.""" out: dict[str, Any] = {"presence_penalty": 0.0, "frequency_penalty": 0.0, "repetition_penalty": 1.0} out.update(fields) return out