Merge pull request #1650 from janhq/j/update-engine-mng

chore: some update for engine management

Author: namchuai

engine/cli/command_line_parser.cc

@@ -88,34 +88,39 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
 
   // Check new update
 #ifdef CORTEX_CPP_VERSION
-  if (cml_data_.check_upd) {
-    if (strcmp(CORTEX_CPP_VERSION, "default_version") != 0) {
-      // TODO(sang) find a better way to handle
-      // This is an extremely ugly way to deal with connection
-      // hang when network down
-      std::atomic<bool> done = false;
-      std::thread t([&]() {
-        if (auto latest_version =
-                commands::CheckNewUpdate(commands::kTimeoutCheckUpdate);
-            latest_version.has_value() &&
-            *latest_version != CORTEX_CPP_VERSION) {
-          CLI_LOG("\nNew Cortex release available: "
-                  << CORTEX_CPP_VERSION << " -> " << *latest_version);
-          CLI_LOG("To update, run: " << commands::GetRole()
-                                     << commands::GetCortexBinary()
-                                     << " update");
-        }
-        done = true;
-      });
-      // Do not wait for http connection timeout
-      t.detach();
-      int retry = 10;
-      while (!done && retry--) {
-        std::this_thread::sleep_for(commands::kTimeoutCheckUpdate / 10);
+  if (cml_data_.check_upd &&
+      strcmp(CORTEX_CPP_VERSION, "default_version") != 0) {
+    // TODO(sang) find a better way to handle
+    // This is an extremely ugly way to deal with connection
+    // hang when network down
+    std::atomic<bool> done = false;
+    std::thread t([&]() {
+      if (auto latest_version =
+              commands::CheckNewUpdate(commands::kTimeoutCheckUpdate);
+          latest_version.has_value() && *latest_version != CORTEX_CPP_VERSION) {
+        CLI_LOG("\nNew Cortex release available: "
+                << CORTEX_CPP_VERSION << " -> " << *latest_version);
+        CLI_LOG("To update, run: " << commands::GetRole()
+                                   << commands::GetCortexBinary() << " update");
       }
+      done = true;
+    });
+    // Do not wait for http connection timeout
+    t.detach();
+    int retry = 10;
+    while (!done && retry--) {
+      std::this_thread::sleep_for(commands::kTimeoutCheckUpdate / 10);
     }
   }
 #endif
+  auto config = file_manager_utils::GetCortexConfig();
+  if (!config.llamacppVersion.empty() &&
+      config.latestLlamacppRelease != config.llamacppVersion) {
+    CLI_LOG(
+        "\nNew llama.cpp version available: " << config.latestLlamacppRelease);
+    CLI_LOG("To update, run: " << commands::GetCortexBinary()
+                               << " engines update llama-cpp");
+  }
 
   return true;
 }

engine/cli/commands/cortex_upd_cmd.cc

@@ -174,8 +174,13 @@ std::optional<std::string> CheckNewUpdate(
         CTL_INF("Got the latest release, update to the config file: "
                 << latest_version)
         config.latestRelease = latest_version;
-        config_yaml_utils::DumpYamlConfig(
+        auto result = config_yaml_utils::DumpYamlConfig(
             config, file_manager_utils::GetConfigurationPath().string());
+        if (result.has_error()) {
+          CTL_ERR("Error update "
+                  << file_manager_utils::GetConfigurationPath().string()
+                  << result.error());
+        }
         if (current_version != latest_version) {
           return latest_version;
         }

engine/cli/commands/engine_install_cmd.cc

@@ -84,12 +84,25 @@ bool EngineInstallCmd::Exec(const std::string& engine,
     std::vector<std::string> variant_selections;
     for (const auto& variant : variant_result.value()) {
       auto v_name = variant["name"].asString();
-      if (string_utils::StringContainsIgnoreCase(v_name, hw_inf_.sys_inf->os)) {
+      if (string_utils::StringContainsIgnoreCase(v_name, hw_inf_.sys_inf->os) &&
+          string_utils::StringContainsIgnoreCase(v_name,
+                                                 hw_inf_.sys_inf->arch)) {
         variant_selections.push_back(variant["name"].asString());
       }
     }
-    auto selected_variant =
-        cli_selection_utils::PrintSelection(variant_selections);
+    if (variant_selections.empty()) {
+      CTL_ERR("No suitable variant found for " << hw_inf_.sys_inf->os << " "
+                                               << hw_inf_.sys_inf->arch);
+      return false;
+    }
+
+    std::optional<std::string> selected_variant = std::nullopt;
+    if (variant_selections.size() == 1) {
+      selected_variant = variant_selections[0];
+    } else {
+      selected_variant =
+          cli_selection_utils::PrintSelection(variant_selections);
+    }
     if (selected_variant == std::nullopt) {
       CTL_ERR("Invalid variant selection");
       return false;

engine/cli/commands/engine_update_cmd.cc

@@ -4,6 +4,7 @@
 #include "utils/cli_selection_utils.h"
 #include "utils/curl_utils.h"
 #include "utils/download_progress.h"
+#include "utils/json_helper.h"
 #include "utils/logging_utils.h"
 #include "utils/system_info_utils.h"
 #include "utils/url_parser.h"
@@ -26,7 +27,6 @@ bool EngineUpdateCmd::Exec(const std::string& host, int port,
   auto dp_res = std::async(std::launch::deferred, [&dp, &engine] {
     return dp.Handle(DownloadType::Engine);
   });
-  CLI_LOG("Validating download items, please wait..")
 
   auto update_url = url_parser::Url{
       .protocol = "http",
@@ -35,7 +35,13 @@ bool EngineUpdateCmd::Exec(const std::string& host, int port,
   };
   auto update_result = curl_utils::SimplePostJson(update_url.ToFullPath());
   if (update_result.has_error()) {
-    CTL_ERR(update_result.error());
+    try {
+      Json::Value json = json_helper::ParseJsonString(update_result.error());
+      std::cout << json["message"].asString() << std::endl;
+    } catch (const std::exception& e) {
+      CTL_ERR(update_result.error());
+    }
+
     return false;
   }
 

engine/cli/main.cc

@@ -101,6 +101,68 @@ int main(int argc, char* argv[]) {
 
   RemoveBinaryTempFileIfExists();
 
+  auto should_check_for_latest_llamacpp_version = true;
+  auto now = std::chrono::system_clock::now();
+
+  // read the yaml to see the last time we check for update
+  auto config = file_manager_utils::GetCortexConfig();
+  if (config.checkedForLlamacppUpdateAt != 0) {
+    // if it passed a day, then we should check
+    auto last_check =
+        std::chrono::system_clock::time_point(
+            std::chrono::milliseconds(config.checkedForLlamacppUpdateAt)) +
+        std::chrono::hours(24);
+    should_check_for_latest_llamacpp_version = now > last_check;
+  }
+
+  if (should_check_for_latest_llamacpp_version) {
+    std::thread t1([]() {
+      auto config = file_manager_utils::GetCortexConfig();
+      // TODO: namh current we only check for llamacpp. Need to add support for other engine
+      auto get_latest_version = []() -> cpp::result<std::string, std::string> {
+        try {
+          auto res = github_release_utils::GetReleaseByVersion(
+              "janhq", "cortex.llamacpp", "latest");
+          if (res.has_error()) {
+            CTL_ERR("Failed to get latest llama.cpp version: " << res.error());
+            return cpp::fail("Failed to get latest llama.cpp version: " +
+                             res.error());
+          }
+          CTL_INF("Latest llamacpp version: " << res->tag_name);
+          return res->tag_name;
+        } catch (const std::exception& e) {
+          CTL_ERR("Failed to get latest llama.cpp version: " << e.what());
+          return cpp::fail("Failed to get latest llama.cpp version: " +
+                           std::string(e.what()));
+        }
+      };
+
+      auto res = get_latest_version();
+      if (res.has_error()) {
+        CTL_ERR("Failed to get latest llama.cpp version: " << res.error());
+        return;
+      }
+
+      auto now = std::chrono::system_clock::now();
+      CTL_DBG("latest llama.cpp version: " << res.value());
+      config.checkedForLlamacppUpdateAt =
+          std::chrono::duration_cast<std::chrono::milliseconds>(
+              now.time_since_epoch())
+              .count();
+      config.latestLlamacppRelease = res.value();
+
+      auto upd_config_res = config_yaml_utils::DumpYamlConfig(
+          config, file_manager_utils::GetConfigurationPath().string());
+      if (upd_config_res.has_error()) {
+        CTL_ERR("Failed to update config file: " << upd_config_res.error());
+      } else {
+        CTL_INF("Updated config file with latest llama.cpp version: "
+                << res.value());
+      }
+    });
+    t1.detach();
+  }
+
   trantor::FileLogger async_file_logger;
   SetupLogger(async_file_logger, verbose);
 

engine/controllers/engines.h

@@ -24,8 +24,10 @@ class Engines : public drogon::HttpController<Engines, false> {
   METHOD_ADD(Engines::UnloadEngine, "/{1}/load", Options, Delete);
   METHOD_ADD(Engines::UpdateEngine, "/{1}/update", Post);
   METHOD_ADD(Engines::ListEngine, "", Get);
+
   METHOD_ADD(Engines::GetEngineVersions, "/{1}/versions", Get);
   METHOD_ADD(Engines::GetEngineVariants, "/{1}/versions/{2}", Get);
+  METHOD_ADD(Engines::GetLatestEngineVersion, "/{1}/latest", Get);
 
   ADD_METHOD_TO(Engines::GetInstalledEngineVariants, "/v1/engines/{1}", Get);
   ADD_METHOD_TO(Engines::InstallEngine,

engine/services/engine_service.cc

@@ -181,6 +181,16 @@ cpp::result<bool, std::string> EngineService::UninstallEngineVariant(
     const std::string& engine, const std::optional<std::string> version,
     const std::optional<std::string> variant) {
   auto ne = NormalizeEngine(engine);
+  if (IsEngineLoaded(ne)) {
+    CTL_INF("Engine " << ne << " is already loaded, unloading it");
+    auto unload_res = UnloadEngine(ne);
+    if (unload_res.has_error()) {
+      CTL_INF("Failed to unload engine: " << unload_res.error());
+      return cpp::fail(unload_res.error());
+    } else {
+      CTL_INF("Engine " << ne << " unloaded successfully");
+    }
+  }
 
   std::optional<std::filesystem::path> path_to_remove = std::nullopt;
   if (version == std::nullopt && variant == std::nullopt) {
@@ -264,6 +274,16 @@ cpp::result<void, std::string> EngineService::DownloadEngineV2(
   if (selected_variant == std::nullopt) {
     return cpp::fail("Failed to find a suitable variant for " + engine);
   }
+  if (IsEngineLoaded(engine)) {
+    CTL_INF("Engine " << engine << " is already loaded, unloading it");
+    auto unload_res = UnloadEngine(engine);
+    if (unload_res.has_error()) {
+      CTL_INF("Failed to unload engine: " << unload_res.error());
+      return cpp::fail(unload_res.error());
+    } else {
+      CTL_INF("Engine " << engine << " unloaded successfully");
+    }
+  }
   auto normalize_version = "v" + selected_variant->version;
 
   auto variant_folder_name = engine_matcher_utils::GetVariantFromNameAndVersion(
@@ -276,7 +296,7 @@ cpp::result<void, std::string> EngineService::DownloadEngineV2(
   auto variant_path = variant_folder_path / selected_variant->name;
   std::filesystem::create_directories(variant_folder_path);
   CLI_LOG("variant_folder_path: " + variant_folder_path.string());
-  auto on_finished = [this, engine, selected_variant,
+  auto on_finished = [this, engine, selected_variant, variant_folder_path,
                       normalize_version](const DownloadTask& finishedTask) {
     // try to unzip the downloaded file
     CLI_LOG("Engine zip path: " << finishedTask.items[0].localPath.string());
@@ -299,6 +319,22 @@ cpp::result<void, std::string> EngineService::DownloadEngineV2(
       CTL_INF("Set default engine variant: " << res.value().variant);
     }
 
+    // remove other engines
+    auto engine_directories = file_manager_utils::GetEnginesContainerPath() /
+                              engine / selected_variant->name;
+
+    for (const auto& entry : std::filesystem::directory_iterator(
+             variant_folder_path.parent_path())) {
+      if (entry.is_directory() &&
+          entry.path().filename() != normalize_version) {
+        try {
+          std::filesystem::remove_all(entry.path());
+        } catch (const std::exception& e) {
+          CTL_WRN("Could not delete directory: " << e.what());
+        }
+      }
+    }
+
     // remove the downloaded file
     try {
       std::filesystem::remove(finishedTask.items[0].localPath);
@@ -364,6 +400,16 @@ cpp::result<bool, std::string> EngineService::DownloadEngine(
         CTL_INF("Creating " << engine_folder_path.string());
         std::filesystem::create_directories(engine_folder_path);
       }
+      if (IsEngineLoaded(engine)) {
+        CTL_INF("Engine " << engine << " is already loaded, unloading it");
+        auto unload_res = UnloadEngine(engine);
+        if (unload_res.has_error()) {
+          CTL_INF("Failed to unload engine: " << unload_res.error());
+          return cpp::fail(unload_res.error());
+        } else {
+          CTL_INF("Engine " << engine << " unloaded successfully");
+        }
+      }
       CTL_INF("Engine folder path: " << engine_folder_path.string() << "\n");
       auto local_path = engine_folder_path / asset.name;
       auto downloadTask{
@@ -918,6 +964,17 @@ cpp::result<EngineUpdateResult, std::string> EngineService::UpdateEngine(
   CTL_INF("Default variant: " << default_variant->variant
                               << ", version: " + default_variant->version);
 
+  if (IsEngineLoaded(ne)) {
+    CTL_INF("Engine " << ne << " is already loaded, unloading it");
+    auto unload_res = UnloadEngine(ne);
+    if (unload_res.has_error()) {
+      CTL_INF("Failed to unload engine: " << unload_res.error());
+      return cpp::fail(unload_res.error());
+    } else {
+      CTL_INF("Engine " << ne << " unloaded successfully");
+    }
+  }
+
   auto latest_version = GetLatestEngineVersion(ne);
   if (latest_version.has_error()) {
     // if can't get latest version, stop

engine/test/components/test_cortex_config.cc

@@ -18,6 +18,7 @@ class CortexConfigTest : public ::testing::Test {
                       kDefaultHost,
                       kDefaultPort,
                       kDefaultCheckedForUpdateAt,
+                      kDefaultCheckedForLlamacppUpdateAt,
                       kDefaultLatestRelease};
   }
 
@@ -39,9 +40,11 @@ TEST_F(CortexConfigTest, DumpYamlConfig_WritesCorrectly) {
                          "localhost",
                          "8080",
                          123456789,
+                         123456789,
                          "v1.0.0"};
 
-  DumpYamlConfig(config, test_file_path);
+  auto result = DumpYamlConfig(config, test_file_path);
+  EXPECT_FALSE(result.has_error());
 
   // Verify that the file was created and contains the expected data
   YAML::Node node = YAML::LoadFile(test_file_path);
@@ -66,9 +69,11 @@ TEST_F(CortexConfigTest, FromYaml_ReadsCorrectly) {
                          "localhost",
                          "8080",
                          123456789,
+                         123456789,
                          "v1.0.0"};
 
-  DumpYamlConfig(config, test_file_path);
+  auto result = DumpYamlConfig(config, test_file_path);
+  EXPECT_FALSE(result.has_error());
 
   // Now read from the YAML file
   CortexConfig loaded_config = FromYaml(test_file_path, default_config);
@@ -115,4 +120,4 @@ TEST_F(CortexConfigTest, FromYaml_IncompleteConfigUsesDefaults) {
             default_config.latestRelease);  // Default value
 }
 
-}  // namespace config_yaml_utils
+}  // namespace config_yaml_utils

engine/test/components/test_file_manager_config_yaml_utils.cc

@@ -37,14 +37,16 @@ TEST_F(FileManagerConfigTest, GetDefaultDataFolderName) {
 
 TEST_F(FileManagerConfigTest, CreateConfigFileIfNotExist) {
 
-  file_manager_utils::CreateConfigFileIfNotExist();
+  auto result = file_manager_utils::CreateConfigFileIfNotExist();
+  EXPECT_FALSE(result.has_error());
   EXPECT_TRUE(
       std::filesystem::exists(file_manager_utils::GetConfigurationPath()));
   std::filesystem::remove(file_manager_utils::GetConfigurationPath());
 }
 
 TEST_F(FileManagerConfigTest, GetCortexConfig) {
-  file_manager_utils::CreateConfigFileIfNotExist();
+  auto result = file_manager_utils::CreateConfigFileIfNotExist();
+  EXPECT_FALSE(result.has_error());
   auto config = file_manager_utils::GetCortexConfig();
   EXPECT_FALSE(config.dataFolderPath.empty());
   EXPECT_FALSE(config.logFolderPath.empty());
@@ -61,8 +63,8 @@ TEST_F(FileManagerConfigTest, DumpYamlConfig) {
                                          .apiServerPort = "8080"};
 
   std::string test_file = "test_config.yaml";
-  config_yaml_utils::DumpYamlConfig(config, test_file);
-
+  auto result = config_yaml_utils::DumpYamlConfig(config, test_file);
+  EXPECT_FALSE(result.has_error());
   EXPECT_TRUE(std::filesystem::exists(test_file));
 
   // Clean up
@@ -91,4 +93,4 @@ TEST_F(FileManagerConfigTest, FromYaml) {
 
   // Clean up
   std::filesystem::remove(test_file);
-}
+}