Merge pull request #509 from janhq/hotfix-turnoff-all-caches

hotfix: will always be cleaning cache

Author: tikikun

context/llama_server_context.h

@@ -471,7 +471,7 @@ struct llama_server_context {
   llama_batch batch;
 
   bool multimodal = false;
-  bool clean_kv_cache = false;
+  bool clean_kv_cache = true;
   bool all_slots_are_idle = false;
   bool add_bos_token = true;
 

controllers/llamaCPP.cc

@@ -197,15 +197,16 @@ void llamaCPP::InferenceImpl(
   // To set default value
 
   // Increase number of chats received and clean the prompt
-  no_of_chats++;
-  if (no_of_chats % clean_cache_threshold == 0) {
-    LOG_INFO_REQUEST(request_id) << "Clean cache threshold reached!";
-    llama.kv_cache_clear();
-    LOG_INFO_REQUEST(request_id) << "Cache cleaned";
-  }
+  //no_of_chats++;
+  //if (no_of_chats % clean_cache_threshold == 0) {
+  //  LOG_INFO_REQUEST(request_id) << "Clean cache threshold reached!";
+  //  llama.kv_cache_clear();
+  //  LOG_INFO_REQUEST(request_id) << "Cache cleaned";
+  //}
 
   // Default values to enable auto caching
-  data["cache_prompt"] = caching_enabled;
+  //data["cache_prompt"] = caching_enabled;
+  data["cache_prompt"] = false;
   data["n_keep"] = -1;
 
   // Passing load value
@@ -655,7 +656,7 @@ bool llamaCPP::LoadModelImpl(std::shared_ptr<Json::Value> jsonBody) {
     params.cont_batching = jsonBody->get("cont_batching", false).asBool();
     this->clean_cache_threshold =
         jsonBody->get("clean_cache_threshold", 5).asInt();
-    this->caching_enabled = jsonBody->get("caching_enabled", true).asBool();
+    this->caching_enabled = jsonBody->get("caching_enabled", false).asBool();
     this->user_prompt = jsonBody->get("user_prompt", "USER: ").asString();
     this->ai_prompt = jsonBody->get("ai_prompt", "ASSISTANT: ").asString();
     this->system_prompt =