chore: cleanup

Author: sangjanai

engine/cli/command_line_parser.cc

@@ -51,9 +51,7 @@ CommandLineParser::CommandLineParser()
       download_service_{std::make_shared<DownloadService>()},
       dylib_path_manager_{std::make_shared<cortex::DylibPathManager>()},
       db_service_{std::make_shared<DatabaseService>()},
-      engine_service_{std::make_shared<EngineService>(
-          download_service_, dylib_path_manager_, db_service_,
-          std::make_shared<cortex::TaskQueue>(1, "q"))} {}
+      engine_service_{std::make_shared<EngineService>(dylib_path_manager_)} {}
 
 bool CommandLineParser::SetupCommand(int argc, char** argv) {
   app_.usage("Usage:\n" + commands::GetCortexBinary() +
@@ -124,14 +122,14 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
     }
   }
 #endif
-  // auto config = file_manager_utils::GetCortexConfig();
-  // if (!config.llamacppVersion.empty() &&
-  //     config.latestLlamacppRelease != config.llamacppVersion) {
-  //   CLI_LOG(
-  //       "\nNew llama.cpp version available: " << config.latestLlamacppRelease);
-  //   CLI_LOG("To update, run: " << commands::GetCortexBinary()
-  //                              << " engines update llama-cpp");
-  // }
+  auto config = file_manager_utils::GetCortexConfig();
+  if (!config.llamacppVersion.empty() &&
+      config.latestLlamacppRelease != config.llamacppVersion) {
+    CLI_LOG(
+        "\nNew llama.cpp version available: " << config.latestLlamacppRelease);
+    CLI_LOG("To update, run: " << commands::GetCortexBinary()
+                               << " engines update llama-cpp");
+  }
 
   return true;
 }

engine/cli/commands/cortex_upd_cmd.cc

@@ -515,10 +515,10 @@ bool CortexUpdCmd::GetLinuxInstallScript(const std::string& v,
                                          const std::string& channel) {
   std::vector<std::string> path_list;
   if (channel == "nightly") {
-    path_list = {"menloresearch",     "cortex.cpp", "dev",       "engine",
+    path_list = {kMenloOrg,     "cortex.cpp", "dev",       "engine",
                  "templates", "linux",      "install.sh"};
   } else {
-    path_list = {"menloresearch",     "cortex.cpp", "main",      "engine",
+    path_list = {kMenloOrg,     "cortex.cpp", "main",      "engine",
                  "templates", "linux",      "install.sh"};
   }
   auto url_obj = url_parser::Url{

engine/cli/commands/cortex_upd_cmd.h

@@ -79,9 +79,9 @@ inline std::vector<std::string> GetReleasePath() {
   if (CORTEX_VARIANT == file_manager_utils::kNightlyVariant) {
     return {"cortex", "latest", "version.json"};
   } else if (CORTEX_VARIANT == file_manager_utils::kBetaVariant) {
-    return {"repos", "menloresearch", "cortex.cpp", "releases"};
+    return {"repos", kMenloOrg, "cortex.cpp", "releases"};
   } else {
-    return {"repos", "menloresearch", "cortex.cpp", "releases", "latest"};
+    return {"repos", kMenloOrg, "cortex.cpp", "releases", "latest"};
   }
 }
 

engine/cli/commands/engine_install_cmd.cc

@@ -92,8 +92,8 @@ bool EngineInstallCmd::Exec(const std::string& engine,
       auto v_name = variant["name"].asString();
       if ((string_utils::StringContainsIgnoreCase(v_name,
                                                   hw_inf_.sys_inf->os) ||
-           (hw_inf_.sys_inf->os == "linux" &&
-            string_utils::StringContainsIgnoreCase(v_name, "ubuntu"))) &&
+           (hw_inf_.sys_inf->os == kLinuxOs &&
+            string_utils::StringContainsIgnoreCase(v_name, kUbuntuOs))) &&
           string_utils::StringContainsIgnoreCase(v_name,
                                                  hw_inf_.sys_inf->arch)) {
         variant_selections.push_back(variant["name"].asString());

engine/cli/commands/server_start_cmd.cc

@@ -106,11 +106,7 @@ bool ServerStartCmd::Exec(const std::string& host, int port,
 #else
   std::vector<std::string> commands;
   // Some engines requires to add lib search path before process being created
-  auto download_srv = std::make_shared<DownloadService>();
-  auto dylib_path_mng = std::make_shared<cortex::DylibPathManager>();
-  auto db_srv = std::make_shared<DatabaseService>();
-  EngineService(download_srv, dylib_path_mng, db_srv,
-                std::make_shared<cortex::TaskQueue>(1, "task_queue"))
+  EngineService(std::make_shared<cortex::DylibPathManager>())
       .RegisterEngineLibPath();
 
   std::string p = cortex_utils::GetCurrentPath() + "/" + exe;

engine/cli/main.cc

@@ -148,14 +148,14 @@ int main(int argc, char* argv[]) {
         std::chrono::hours(24);
     should_check_for_latest_llamacpp_version = now > last_check;
   }
-
-  if (false) {
+  
+  if (should_check_for_latest_llamacpp_version) {
     std::thread t1([]() {
       // TODO: namh current we only check for llamacpp. Need to add support for other engine
       auto get_latest_version = []() -> cpp::result<std::string, std::string> {
         try {
           auto res = github_release_utils::GetReleaseByVersion(
-              "menloresearch", "cortex.llamacpp", "latest");
+              kGgmlOrg, kLlamaRepo, "latest");
           if (res.has_error()) {
             CTL_ERR("Failed to get latest llama.cpp version: " << res.error());
             return cpp::fail("Failed to get latest llama.cpp version: " +
@@ -171,6 +171,7 @@ int main(int argc, char* argv[]) {
       };
 
       auto res = get_latest_version();
+      
       if (res.has_error()) {
         CTL_ERR("Failed to get latest llama.cpp version: " << res.error());
         return;

engine/extensions/local-engine/local_engine.cc

@@ -447,13 +447,10 @@ void LocalEngine::HandleEmbedding(std::shared_ptr<Json::Value> json_body,
   if (server_map_.find(model_id) != server_map_.end()) {
     auto& s = server_map_[model_id];
     auto url = url_parser::Url{
-        .protocol = "http",
-        .host = s.host + ":" + std::to_string(s.port),
-        .pathParams =
-            {
-                "v1",
-                "embeddings",
-            },
+        /*.protocol*/ "http",
+        /*.host*/ s.host + ":" + std::to_string(s.port),
+        /*.pathParams*/ {"v1", "embeddings"},
+        /* .queries = */ {},
     };
 
     auto response = curl_utils::SimplePostJson(url.ToFullPath(),
@@ -495,9 +492,10 @@ void LocalEngine::LoadModel(std::shared_ptr<Json::Value> json_body,
   auto wait_for_server_up = [this](const std::string& model,
                                    const std::string& host, int port) {
     auto url = url_parser::Url{
-        .protocol = "http",
-        .host = host + ":" + std::to_string(port),
-        .pathParams = {"health"},
+        /*.protocol*/ "http",
+        /*.host*/ host + ":" + std::to_string(port),
+        /*.pathParams*/ {"health"},
+        /*.queries*/ {},
     };
     while (server_map_.find(model) != server_map_.end()) {
       auto res = curl_utils::SimpleGet(url.ToFullPath());
@@ -519,6 +517,11 @@ void LocalEngine::LoadModel(std::shared_ptr<Json::Value> json_body,
   server_map_[model_id].host = "127.0.0.1";
   server_map_[model_id].port = GenerateRandomInteger(39400, 39999);
   auto& s = server_map_[model_id];
+  s.pre_prompt = json_body->get("pre_prompt", "").asString();
+  s.user_prompt = json_body->get("user_prompt", "USER: ").asString();
+  s.ai_prompt = json_body->get("ai_prompt", "ASSISTANT: ").asString();
+  s.system_prompt =
+      json_body->get("system_prompt", "ASSISTANT's RULE: ").asString();
   std::vector<std::string> params = ConvertJsonToParamsVector(*json_body);
   params.push_back("--host");
   params.push_back(s.host);
@@ -530,21 +533,21 @@ void LocalEngine::LoadModel(std::shared_ptr<Json::Value> json_body,
 
   std::vector<std::string> v;
   v.reserve(params.size() + 1);
-  auto engine_dir = engine_service_.GetEngineDirPath("llama.cpp");
+  auto engine_dir = engine_service_.GetEngineDirPath(kLlamaRepo);
   if (engine_dir.has_error()) {
     CTL_WRN(engine_dir.error());
     server_map_.erase(model_id);
     return;
   }
-  auto exe = (engine_dir.value().first / "llama-server").string();
+  auto exe = (engine_dir.value().first / kLlamaServer).string();
 
   v.push_back(exe);
   v.insert(v.end(), params.begin(), params.end());
   engine_service_.RegisterEngineLibPath();
 
   auto log_path =
       (file_manager_utils::GetCortexLogPath() / "logs" / "cortex.log").string();
-  CTL_INF("log: " << log_path);
+  CTL_DBG("log: " << log_path);
   auto result = cortex::process::SpawnProcess(v, log_path, log_path);
   if (result.has_error()) {
     CTL_ERR("Fail to spawn process. " << result.error());
@@ -696,18 +699,24 @@ void LocalEngine::HandleOpenAiChatCompletion(
   // llama.cpp server only supports n = 1
   (*json_body)["n"] = 1;
 
+  auto url = url_parser::Url{
+      /*.protocol*/ "http",
+      /*.host*/ s.host + ":" + std::to_string(s.port),
+      /*.pathParams*/ {"v1", "chat", "completions"},
+      /*.queries*/ {},
+  };
+
   if (is_stream) {
-    q_.RunInQueue([s, json_body, callback, model] {
+    q_.RunInQueue([s, json_body, callback, model, url = std::move(url)] {
       auto curl = curl_easy_init();
       if (!curl) {
         CTL_WRN("Failed to initialize CURL");
         return;
       }
 
-      auto url = "http://" + s.host + ":" + std::to_string(s.port) +
-                 "/v1/chat/completions";
-      curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+      curl_easy_setopt(curl, CURLOPT_URL, url.ToFullPath().c_str());
       curl_easy_setopt(curl, CURLOPT_POST, 1L);
+      CTL_INF(url.ToFullPath());
 
       struct curl_slist* headers = nullptr;
       headers = curl_slist_append(headers, "Content-Type: application/json");
@@ -754,16 +763,6 @@ void LocalEngine::HandleOpenAiChatCompletion(
     });
 
   } else {
-    auto url = url_parser::Url{
-        .protocol = "http",
-        .host = s.host + ":" + std::to_string(s.port),
-        .pathParams =
-            {
-                "v1",
-                "chat",
-                "completions",
-            },
-    };
     Json::Value result;
     // multiple choices
     for (int i = 0; i < n; i++) {
@@ -810,6 +809,8 @@ void LocalEngine::HandleOpenAiChatCompletion(
   }
 }
 
+// (sang) duplicate code but it is easier to clean when
+// llama-server upstream is fully OpenAI API Compatible
 void LocalEngine::HandleNonOpenAiChatCompletion(
     std::shared_ptr<Json::Value> json_body, http_callback&& callback,
     const std::string& model) {
@@ -881,17 +882,23 @@ void LocalEngine::HandleNonOpenAiChatCompletion(
   (*json_body)["n"] = 1;
   int n_probs = json_body->get("n_probs", 0).asInt();
 
+  auto url = url_parser::Url{
+      /*.protocol*/ "http",
+      /*.host*/ s.host + ":" + std::to_string(s.port),
+      /*.pathParams*/ {"v1", "completions"},
+      /*.queries*/ {},
+  };
+
   if (is_stream) {
-    q_.RunInQueue([s, json_body, callback, n_probs, model] {
+    q_.RunInQueue([s, json_body, callback, n_probs, model,
+                   url = std::move(url)] {
       auto curl = curl_easy_init();
       if (!curl) {
         CTL_WRN("Failed to initialize CURL");
         return;
       }
 
-      auto url =
-          "http://" + s.host + ":" + std::to_string(s.port) + "/v1/completions";
-      curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+      curl_easy_setopt(curl, CURLOPT_URL, url.ToFullPath().c_str());
       curl_easy_setopt(curl, CURLOPT_POST, 1L);
 
       struct curl_slist* headers = nullptr;
@@ -939,15 +946,7 @@ void LocalEngine::HandleNonOpenAiChatCompletion(
     });
 
   } else {
-    auto url = url_parser::Url{
-        .protocol = "http",
-        .host = s.host + ":" + std::to_string(s.port),
-        .pathParams =
-            {
-                "v1",
-                "completions",
-            },
-    };
+
     Json::Value result;
     int prompt_tokens = 0;
     int predicted_tokens = 0;