Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 3932539

Browse files
hiento09hiento09
authored
Merge pull request #1702 from janhq/dev
Sync dev to main
2 parents 8d60c95 + 70e25b7 commit 3932539

File tree

16 files changed

+1010
-735
lines changed

16 files changed

+1010
-735
lines changed

‎docs/static/openapi/cortex.json‎

Lines changed: 283 additions & 646 deletions
Large diffs are not rendered by default.

‎engine/cli/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ add_executable(${TARGET_NAME} main.cc
7373
${CMAKE_CURRENT_SOURCE_DIR}/../utils/cpuid/cpu_info.cc
7474
${CMAKE_CURRENT_SOURCE_DIR}/../utils/file_logger.cc
7575
${CMAKE_CURRENT_SOURCE_DIR}/command_line_parser.cc
76+
${CMAKE_CURRENT_SOURCE_DIR}/../services/config_service.cc
7677
${CMAKE_CURRENT_SOURCE_DIR}/../services/download_service.cc
7778
${CMAKE_CURRENT_SOURCE_DIR}/../services/engine_service.cc
7879
${CMAKE_CURRENT_SOURCE_DIR}/../services/model_service.cc

‎engine/cli/command_line_parser.cc‎

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -326,14 +326,8 @@ void CommandLineParser::SetupModelCommands() {
326326
void CommandLineParser::SetupConfigsCommands() {
327327
auto config_cmd =
328328
app_.add_subcommand("config", "Subcommands for managing configurations");
329-
config_cmd->usage(
330-
"Usage:\n" + commands::GetCortexBinary() +
331-
" config status for listing all API server configuration.\n" +
332-
commands::GetCortexBinary() +
333-
" config --cors [on/off] to toggle CORS.\n" +
334-
commands::GetCortexBinary() +
335-
" config --allowed_origins [comma separated origin] to set a list of "
336-
"allowed origin");
329+
config_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
330+
" config [option] [value]");
337331
config_cmd->group(kConfigGroup);
338332
auto config_status_cmd =
339333
config_cmd->add_subcommand("status", "Print all configurations");
@@ -344,18 +338,18 @@ void CommandLineParser::SetupConfigsCommands() {
344338
std::stoi(cml_data_.config.apiServerPort));
345339
});
346340

347-
// TODO: this can be improved
348-
std::vector<std::string> avai_opts{"cors", "allowed_origins"};
349-
std::unordered_map<std::string, std::string> description{
350-
{"cors", "[on/off] Toggling CORS."},
351-
{"allowed_origins",
352-
"Allowed origins for CORS. Comma separated. E.g. "
353-
"http://localhost,https://cortex.so"}};
354-
for (const auto& opt : avai_opts) {
355-
std::string option = "--" + opt;
356-
config_cmd->add_option(option, config_update_opts_[opt], description[opt])
357-
->expected(0, 1)
358-
->default_str("*");
341+
for (const auto& [key, opt] : CONFIGURATIONS) {
342+
std::string option = "--" + opt.name;
343+
auto option_cmd =
344+
config_cmd->add_option(option, config_update_opts_[opt.name], opt.desc)
345+
->group(opt.group)
346+
->default_str(opt.default_value);
347+
348+
if (opt.allow_empty) {
349+
option_cmd->expected(0, 1);
350+
} else {
351+
option_cmd->expected(1);
352+
}
359353
}
360354

361355
config_cmd->callback([this, config_cmd] {

‎engine/cli/commands/config_upd_cmd.cc‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,39 @@
11
#include "config_upd_cmd.h"
22
#include "commands/server_start_cmd.h"
3+
#include "common/api_server_configuration.h"
34
#include "utils/curl_utils.h"
45
#include "utils/logging_utils.h"
56
#include "utils/string_utils.h"
67
#include "utils/url_parser.h"
78

89
namespace {
9-
const std::vector<std::string> config_keys{"cors", "allowed_origins"};
10-
1110
inline Json::Value NormalizeJson(
1211
const std::unordered_map<std::string, std::string> options) {
1312
Json::Value root;
1413
for (const auto& [key, value] : options) {
15-
if (std::find(config_keys.begin(), config_keys.end(), key) ==
16-
config_keys.end()) {
14+
if (CONFIGURATIONS.find(key) == CONFIGURATIONS.end()) {
1715
continue;
1816
}
17+
auto config = CONFIGURATIONS.at(key);
1918

20-
if (key == "cors") {
19+
if (config.accept_value == "[on|off]") {
2120
if (string_utils::EqualsIgnoreCase("on", value)) {
22-
root["cors"] = true;
21+
root[key] = true;
2322
} else if (string_utils::EqualsIgnoreCase("off", value)) {
24-
root["cors"] = false;
23+
root[key] = false;
2524
}
26-
} else if (key == "allowed_origins") {
25+
} else if (config.accept_value == "comma separated") {
2726
auto origins = string_utils::SplitBy(value, ",");
2827
Json::Value origin_array(Json::arrayValue);
2928
for (const auto& origin : origins) {
3029
origin_array.append(origin);
3130
}
3231
root[key] = origin_array;
32+
} else if (config.accept_value == "string") {
33+
root[key] = value;
34+
} else {
35+
CTL_ERR("Not support configuration type: " << config.accept_value
36+
<< " for config key: " << key);
3337
}
3438
}
3539

@@ -50,13 +54,21 @@ void commands::ConfigUpdCmd::Exec(
5054
}
5155
}
5256

57+
auto non_null_opts = std::unordered_map<std::string, std::string>();
58+
for (const auto& [key, value] : options) {
59+
if (value.empty()) {
60+
continue;
61+
}
62+
non_null_opts[key] = value;
63+
}
64+
5365
auto url = url_parser::Url{
5466
.protocol = "http",
5567
.host = host + ":" + std::to_string(port),
5668
.pathParams = {"v1", "configs"},
5769
};
5870

59-
auto json = NormalizeJson(options);
71+
auto json = NormalizeJson(non_null_opts);
6072
if (json.empty()) {
6173
CLI_LOG_ERROR("Invalid configuration options provided");
6274
return;

‎engine/common/api_server_configuration.h‎

Lines changed: 197 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,144 @@
55
#include <unordered_map>
66
#include <vector>
77

8+
// current only support basic auth
9+
enum class ProxyAuthMethod {
10+
Basic,
11+
Digest,
12+
DigestIe,
13+
Bearer,
14+
Negotiate,
15+
Ntlm,
16+
NtlmWb,
17+
Any,
18+
AnySafe,
19+
AuthOnly,
20+
AwsSigV4
21+
};
22+
23+
struct ApiConfigurationMetadata {
24+
std::string name;
25+
std::string desc;
26+
std::string group;
27+
std::string accept_value;
28+
std::string default_value;
29+
30+
bool allow_empty = false;
31+
};
32+
33+
static const std::unordered_map<std::string, ApiConfigurationMetadata>
34+
CONFIGURATIONS = {
35+
{"cors",
36+
ApiConfigurationMetadata{
37+
.name = "cors",
38+
.desc = "Cross-Origin Resource Sharing configuration.",
39+
.group = "CORS",
40+
.accept_value = "[on|off]",
41+
.default_value = "on"}},
42+
{"allowed_origins",
43+
ApiConfigurationMetadata{
44+
.name = "allowed_origins",
45+
.desc = "Allowed origins for CORS. Comma separated. E.g. "
46+
"http://localhost,https://cortex.so",
47+
.group = "CORS",
48+
.accept_value = "comma separated",
49+
.default_value = "*",
50+
.allow_empty = true}},
51+
{"proxy_url", ApiConfigurationMetadata{.name = "proxy_url",
52+
.desc = "Proxy URL",
53+
.group = "Proxy",
54+
.accept_value = "string",
55+
.default_value = ""}},
56+
{"proxy_username", ApiConfigurationMetadata{.name = "proxy_username",
57+
.desc = "Proxy Username",
58+
.group = "Proxy",
59+
.accept_value = "string",
60+
.default_value = ""}},
61+
{"proxy_password", ApiConfigurationMetadata{.name = "proxy_password",
62+
.desc = "Proxy Password",
63+
.group = "Proxy",
64+
.accept_value = "string",
65+
.default_value = ""}},
66+
{"verify_proxy_ssl",
67+
ApiConfigurationMetadata{.name = "verify_proxy_ssl",
68+
.desc = "Verify SSL for proxy",
69+
.group = "Proxy",
70+
.accept_value = "[on|off]",
71+
.default_value = "on"}},
72+
{"verify_proxy_host_ssl",
73+
ApiConfigurationMetadata{.name = "verify_proxy_host_ssl",
74+
.desc = "Verify SSL for proxy",
75+
.group = "Proxy",
76+
.accept_value = "[on|off]",
77+
.default_value = "on"}},
78+
{"no_proxy", ApiConfigurationMetadata{.name = "no_proxy",
79+
.desc = "No proxy for hosts",
80+
.group = "Proxy",
81+
.accept_value = "string",
82+
.default_value = ""}},
83+
{"verify_peer_ssl", ApiConfigurationMetadata{.name = "verify_peer_ssl",
84+
.desc = "Verify peer SSL",
85+
.group = "Proxy",
86+
.accept_value = "[on|off]",
87+
.default_value = "on"}},
88+
{"verify_host_ssl", ApiConfigurationMetadata{.name = "verify_host_ssl",
89+
.desc = "Verify host SSL",
90+
.group = "Proxy",
91+
.accept_value = "[on|off]",
92+
.default_value = "on"}},
93+
};
94+
895
class ApiServerConfiguration {
996
public:
10-
ApiServerConfiguration(bool cors = true,
11-
std::vector<std::string> allowed_origins = {})
12-
: cors{cors}, allowed_origins{allowed_origins} {}
97+
ApiServerConfiguration(
98+
bool cors = true, std::vector<std::string> allowed_origins = {},
99+
bool verify_proxy_ssl = true, bool verify_proxy_host_ssl = true,
100+
const std::string& proxy_url = "", const std::string& proxy_username = "",
101+
const std::string& proxy_password = "", const std::string& no_proxy = "",
102+
bool verify_peer_ssl = true, bool verify_host_ssl = true)
103+
: cors{cors},
104+
allowed_origins{allowed_origins},
105+
verify_proxy_ssl{verify_proxy_ssl},
106+
verify_proxy_host_ssl{verify_proxy_host_ssl},
107+
proxy_url{proxy_url},
108+
proxy_username{proxy_username},
109+
proxy_password{proxy_password},
110+
no_proxy{no_proxy},
111+
verify_peer_ssl{verify_peer_ssl},
112+
verify_host_ssl{verify_host_ssl} {}
13113

114+
// cors
14115
bool cors{true};
15116
std::vector<std::string> allowed_origins;
16117

118+
// proxy
119+
bool verify_proxy_ssl{true};
120+
bool verify_proxy_host_ssl{true};
121+
ProxyAuthMethod proxy_auth_method{ProxyAuthMethod::Basic};
122+
std::string proxy_url{""};
123+
std::string proxy_username{""};
124+
std::string proxy_password{""};
125+
std::string no_proxy{""};
126+
127+
bool verify_peer_ssl{true};
128+
bool verify_host_ssl{true};
129+
17130
Json::Value ToJson() const {
18131
Json::Value root;
19132
root["cors"] = cors;
20133
root["allowed_origins"] = Json::Value(Json::arrayValue);
21134
for (const auto& origin : allowed_origins) {
22135
root["allowed_origins"].append(origin);
23136
}
137+
root["verify_proxy_ssl"] = verify_proxy_ssl;
138+
root["verify_proxy_host_ssl"] = verify_proxy_host_ssl;
139+
root["proxy_url"] = proxy_url;
140+
root["proxy_username"] = proxy_username;
141+
root["proxy_password"] = proxy_password;
142+
root["no_proxy"] = no_proxy;
143+
root["verify_peer_ssl"] = verify_peer_ssl;
144+
root["verify_host_ssl"] = verify_host_ssl;
145+
24146
return root;
25147
}
26148

@@ -31,6 +153,78 @@ class ApiServerConfiguration {
31153
const std::unordered_map<std::string,
32154
std::function<bool(const Json::Value&)>>
33155
field_updater{
156+
{"verify_peer_ssl",
157+
[this](const Json::Value& value) -> bool {
158+
if (!value.isBool()) {
159+
return false;
160+
}
161+
verify_peer_ssl = value.asBool();
162+
return true;
163+
}},
164+
165+
{"verify_host_ssl",
166+
[this](const Json::Value& value) -> bool {
167+
if (!value.isBool()) {
168+
return false;
169+
}
170+
verify_host_ssl = value.asBool();
171+
return true;
172+
}},
173+
174+
{"verify_proxy_host_ssl",
175+
[this](const Json::Value& value) -> bool {
176+
if (!value.isBool()) {
177+
return false;
178+
}
179+
verify_proxy_host_ssl = value.asBool();
180+
return true;
181+
}},
182+
183+
{"verify_proxy_ssl",
184+
[this](const Json::Value& value) -> bool {
185+
if (!value.isBool()) {
186+
return false;
187+
}
188+
verify_proxy_ssl = value.asBool();
189+
return true;
190+
}},
191+
192+
{"no_proxy",
193+
[this](const Json::Value& value) -> bool {
194+
if (!value.isString()) {
195+
return false;
196+
}
197+
no_proxy = value.asString();
198+
return true;
199+
}},
200+
201+
{"proxy_url",
202+
[this](const Json::Value& value) -> bool {
203+
if (!value.isString()) {
204+
return false;
205+
}
206+
proxy_url = value.asString();
207+
return true;
208+
}},
209+
210+
{"proxy_username",
211+
[this](const Json::Value& value) -> bool {
212+
if (!value.isString()) {
213+
return false;
214+
}
215+
proxy_username = value.asString();
216+
return true;
217+
}},
218+
219+
{"proxy_password",
220+
[this](const Json::Value& value) -> bool {
221+
if (!value.isString()) {
222+
return false;
223+
}
224+
proxy_password = value.asString();
225+
return true;
226+
}},
227+
34228
{"cors",
35229
[this](const Json::Value& value) -> bool {
36230
if (!value.isBool()) {

‎engine/config/yaml_config.cc‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include "utils/file_manager_utils.h"
99
#include "utils/format_utils.h"
1010
#include "yaml_config.h"
11+
1112
namespace config {
1213
// Method to read YAML file
1314
void YamlHandler::Reset() {
@@ -44,6 +45,7 @@ void YamlHandler::ReadYamlFile(const std::string& file_path) {
4445
throw;
4546
}
4647
}
48+
4749
void YamlHandler::SplitPromptTemplate(ModelConfig& mc) {
4850
if (mc.prompt_template.size() > 0) {
4951
auto& pt = mc.prompt_template;
@@ -220,7 +222,7 @@ void YamlHandler::UpdateModelConfig(ModelConfig new_model_config) {
220222
yaml_node_["ngl"] = model_config_.ngl;
221223
if (!std::isnan(static_cast<double>(model_config_.ctx_len)))
222224
yaml_node_["ctx_len"] = model_config_.ctx_len;
223-
if (!std::isnan(static_cast<double>(model_config_.n_parallel)))
225+
if (!std::isnan(static_cast<double>(model_config_.n_parallel)))
224226
yaml_node_["n_parallel"] = model_config_.n_parallel;
225227
if (!std::isnan(static_cast<double>(model_config_.tp)))
226228
yaml_node_["tp"] = model_config_.tp;
@@ -377,7 +379,8 @@ void YamlHandler::WriteYamlFile(const std::string& file_path) const {
377379
outFile << format_utils::writeKeyValue(
378380
"ctx_len", yaml_node_["ctx_len"],
379381
"llama.context_length | 0 or undefined = loaded from model");
380-
outFile << format_utils::writeKeyValue("n_parallel", yaml_node_["n_parallel"]);
382+
outFile << format_utils::writeKeyValue("n_parallel",
383+
yaml_node_["n_parallel"]);
381384
outFile << format_utils::writeKeyValue("ngl", yaml_node_["ngl"],
382385
"Undefined = loaded from model");
383386
outFile << "# END OPTIONAL\n";

0 commit comments

Comments
 (0)