Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 824de03

Browse files
namchuainamchuai
committed
feat: add proxy support
1 parent 6892823 commit 824de03

File tree

12 files changed

+506
-703
lines changed

12 files changed

+506
-703
lines changed

‎docs/static/openapi/cortex.json‎

Lines changed: 205 additions & 645 deletions
Large diffs are not rendered by default.

‎engine/cli/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ add_executable(${TARGET_NAME} main.cc
7373
${CMAKE_CURRENT_SOURCE_DIR}/../utils/cpuid/cpu_info.cc
7474
${CMAKE_CURRENT_SOURCE_DIR}/../utils/file_logger.cc
7575
${CMAKE_CURRENT_SOURCE_DIR}/command_line_parser.cc
76+
${CMAKE_CURRENT_SOURCE_DIR}/../services/config_service.cc
7677
${CMAKE_CURRENT_SOURCE_DIR}/../services/download_service.cc
7778
${CMAKE_CURRENT_SOURCE_DIR}/../services/engine_service.cc
7879
${CMAKE_CURRENT_SOURCE_DIR}/../services/model_service.cc

‎engine/common/api_server_configuration.h‎

Lines changed: 125 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,72 @@
55
#include <unordered_map>
66
#include <vector>
77

8+
// current only support basic auth
9+
enum class ProxyAuthMethod {
10+
Basic,
11+
Digest,
12+
DigestIe,
13+
Bearer,
14+
Negotiate,
15+
Ntlm,
16+
NtlmWb,
17+
Any,
18+
AnySafe,
19+
AuthOnly,
20+
AwsSigV4
21+
};
22+
823
class ApiServerConfiguration {
924
public:
10-
ApiServerConfiguration(bool cors = true,
11-
std::vector<std::string> allowed_origins = {})
12-
: cors{cors}, allowed_origins{allowed_origins} {}
25+
ApiServerConfiguration(
26+
bool cors = true, std::vector<std::string> allowed_origins = {},
27+
bool verify_proxy_ssl = true, bool verify_proxy_host_ssl = true,
28+
const std::string& proxy_url = "", const std::string& proxy_username = "",
29+
const std::string& proxy_password = "", const std::string& no_proxy = "",
30+
bool verify_peer_ssl = true, bool verify_host_ssl = true)
31+
: cors{cors},
32+
allowed_origins{allowed_origins},
33+
verify_proxy_ssl{verify_proxy_ssl},
34+
verify_proxy_host_ssl{verify_proxy_host_ssl},
35+
proxy_url{proxy_url},
36+
proxy_username{proxy_username},
37+
proxy_password{proxy_password},
38+
no_proxy{no_proxy},
39+
verify_peer_ssl{verify_peer_ssl},
40+
verify_host_ssl{verify_host_ssl} {}
1341

42+
// cors
1443
bool cors{true};
1544
std::vector<std::string> allowed_origins;
1645

46+
// proxy
47+
bool verify_proxy_ssl{true};
48+
bool verify_proxy_host_ssl{true};
49+
ProxyAuthMethod proxy_auth_method{ProxyAuthMethod::Basic};
50+
std::string proxy_url{""};
51+
std::string proxy_username{""};
52+
std::string proxy_password{""};
53+
std::string no_proxy{""};
54+
55+
bool verify_peer_ssl{true};
56+
bool verify_host_ssl{true};
57+
1758
Json::Value ToJson() const {
1859
Json::Value root;
1960
root["cors"] = cors;
2061
root["allowed_origins"] = Json::Value(Json::arrayValue);
2162
for (const auto& origin : allowed_origins) {
2263
root["allowed_origins"].append(origin);
2364
}
65+
root["verify_proxy_ssl"] = verify_proxy_ssl;
66+
root["verify_proxy_host_ssl"] = verify_proxy_host_ssl;
67+
root["proxy_url"] = proxy_url;
68+
root["proxy_username"] = proxy_username;
69+
root["proxy_password"] = proxy_password;
70+
root["no_proxy"] = no_proxy;
71+
root["verify_peer_ssl"] = verify_peer_ssl;
72+
root["verify_host_ssl"] = verify_host_ssl;
73+
2474
return root;
2575
}
2676

@@ -31,6 +81,78 @@ class ApiServerConfiguration {
3181
const std::unordered_map<std::string,
3282
std::function<bool(const Json::Value&)>>
3383
field_updater{
84+
{"verify_peer_ssl",
85+
[this](const Json::Value& value) -> bool {
86+
if (!value.isBool()) {
87+
return false;
88+
}
89+
verify_peer_ssl = value.asBool();
90+
return true;
91+
}},
92+
93+
{"verify_host_ssl",
94+
[this](const Json::Value& value) -> bool {
95+
if (!value.isBool()) {
96+
return false;
97+
}
98+
verify_host_ssl = value.asBool();
99+
return true;
100+
}},
101+
102+
{"verify_proxy_host_ssl",
103+
[this](const Json::Value& value) -> bool {
104+
if (!value.isBool()) {
105+
return false;
106+
}
107+
verify_proxy_host_ssl = value.asBool();
108+
return true;
109+
}},
110+
111+
{"verify_proxy_ssl",
112+
[this](const Json::Value& value) -> bool {
113+
if (!value.isBool()) {
114+
return false;
115+
}
116+
verify_proxy_ssl = value.asBool();
117+
return true;
118+
}},
119+
120+
{"no_proxy",
121+
[this](const Json::Value& value) -> bool {
122+
if (!value.isString()) {
123+
return false;
124+
}
125+
no_proxy = value.asString();
126+
return true;
127+
}},
128+
129+
{"proxy_url",
130+
[this](const Json::Value& value) -> bool {
131+
if (!value.isString()) {
132+
return false;
133+
}
134+
proxy_url = value.asString();
135+
return true;
136+
}},
137+
138+
{"proxy_username",
139+
[this](const Json::Value& value) -> bool {
140+
if (!value.isString()) {
141+
return false;
142+
}
143+
proxy_username = value.asString();
144+
return true;
145+
}},
146+
147+
{"proxy_password",
148+
[this](const Json::Value& value) -> bool {
149+
if (!value.isString()) {
150+
return false;
151+
}
152+
proxy_password = value.asString();
153+
return true;
154+
}},
155+
34156
{"cors",
35157
[this](const Json::Value& value) -> bool {
36158
if (!value.isBool()) {

‎engine/config/yaml_config.cc‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include "utils/file_manager_utils.h"
99
#include "utils/format_utils.h"
1010
#include "yaml_config.h"
11+
1112
namespace config {
1213
// Method to read YAML file
1314
void YamlHandler::Reset() {
@@ -44,6 +45,7 @@ void YamlHandler::ReadYamlFile(const std::string& file_path) {
4445
throw;
4546
}
4647
}
48+
4749
void YamlHandler::SplitPromptTemplate(ModelConfig& mc) {
4850
if (mc.prompt_template.size() > 0) {
4951
auto& pt = mc.prompt_template;
@@ -220,7 +222,7 @@ void YamlHandler::UpdateModelConfig(ModelConfig new_model_config) {
220222
yaml_node_["ngl"] = model_config_.ngl;
221223
if (!std::isnan(static_cast<double>(model_config_.ctx_len)))
222224
yaml_node_["ctx_len"] = model_config_.ctx_len;
223-
if (!std::isnan(static_cast<double>(model_config_.n_parallel)))
225+
if (!std::isnan(static_cast<double>(model_config_.n_parallel)))
224226
yaml_node_["n_parallel"] = model_config_.n_parallel;
225227
if (!std::isnan(static_cast<double>(model_config_.tp)))
226228
yaml_node_["tp"] = model_config_.tp;
@@ -377,7 +379,8 @@ void YamlHandler::WriteYamlFile(const std::string& file_path) const {
377379
outFile << format_utils::writeKeyValue(
378380
"ctx_len", yaml_node_["ctx_len"],
379381
"llama.context_length | 0 or undefined = loaded from model");
380-
outFile << format_utils::writeKeyValue("n_parallel", yaml_node_["n_parallel"]);
382+
outFile << format_utils::writeKeyValue("n_parallel",
383+
yaml_node_["n_parallel"]);
381384
outFile << format_utils::writeKeyValue("ngl", yaml_node_["ngl"],
382385
"Undefined = loaded from model");
383386
outFile << "# END OPTIONAL\n";

‎engine/main.cc‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,13 +106,14 @@ void RunServer(std::optional<int> port, bool ignore_cout) {
106106
auto event_queue_ptr = std::make_shared<EventQueue>();
107107
cortex::event::EventProcessor event_processor(event_queue_ptr);
108108

109-
auto download_service = std::make_shared<DownloadService>(event_queue_ptr);
109+
auto config_service = std::make_shared<ConfigService>();
110+
auto download_service =
111+
std::make_shared<DownloadService>(event_queue_ptr, config_service);
110112
auto engine_service = std::make_shared<EngineService>(download_service);
111113
auto inference_svc =
112114
std::make_shared<services::InferenceService>(engine_service);
113115
auto model_service = std::make_shared<ModelService>(
114116
download_service, inference_svc, engine_service);
115-
auto config_service = std::make_shared<ConfigService>();
116117

117118
// initialize custom controllers
118119
auto engine_ctl = std::make_shared<Engines>(engine_service);

‎engine/services/config_service.cc‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,12 @@
55
cpp::result<ApiServerConfiguration, std::string>
66
ConfigService::UpdateApiServerConfiguration(const Json::Value& json) {
77
auto config = file_manager_utils::GetCortexConfig();
8-
ApiServerConfiguration api_server_config{config.enableCors,
9-
config.allowedOrigins};
8+
ApiServerConfiguration api_server_config{
9+
config.enableCors, config.allowedOrigins, config.verifyProxySsl,
10+
config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername,
11+
config.proxyPassword, config.noProxy, config.verifyPeerSsl,
12+
config.verifyHostSsl};
13+
1014
std::vector<std::string> updated_fields;
1115
std::vector<std::string> invalid_fields;
1216
std::vector<std::string> unknown_fields;
@@ -20,13 +24,27 @@ ConfigService::UpdateApiServerConfiguration(const Json::Value& json) {
2024

2125
config.enableCors = api_server_config.cors;
2226
config.allowedOrigins = api_server_config.allowed_origins;
27+
config.verifyProxySsl = api_server_config.verify_proxy_ssl;
28+
config.verifyProxyHostSsl = api_server_config.verify_proxy_host_ssl;
29+
30+
config.proxyUrl = api_server_config.proxy_url;
31+
config.proxyUsername = api_server_config.proxy_username;
32+
config.proxyPassword = api_server_config.proxy_password;
33+
config.noProxy = api_server_config.no_proxy;
34+
35+
config.verifyPeerSsl = api_server_config.verify_peer_ssl;
36+
config.verifyHostSsl = api_server_config.verify_host_ssl;
2337

2438
auto result = file_manager_utils::UpdateCortexConfig(config);
2539
return api_server_config;
2640
}
2741

2842
cpp::result<ApiServerConfiguration, std::string>
29-
ConfigService::GetApiServerConfiguration() const {
43+
ConfigService::GetApiServerConfiguration() {
3044
auto config = file_manager_utils::GetCortexConfig();
31-
return ApiServerConfiguration{config.enableCors, config.allowedOrigins};
45+
return ApiServerConfiguration{
46+
config.enableCors, config.allowedOrigins, config.verifyProxySsl,
47+
config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername,
48+
config.proxyPassword, config.noProxy, config.verifyPeerSsl,
49+
config.verifyHostSsl};
3250
}

‎engine/services/config_service.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,5 @@ class ConfigService {
88
cpp::result<ApiServerConfiguration, std::string> UpdateApiServerConfiguration(
99
const Json::Value& json);
1010

11-
cpp::result<ApiServerConfiguration, std::string> GetApiServerConfiguration()
12-
const;
11+
cpp::result<ApiServerConfiguration, std::string> GetApiServerConfiguration();
1312
};

‎engine/services/download_service.cc‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ cpp::result<uint64_t, std::string> DownloadService::GetFileSize(
8787
return cpp::fail(static_cast<std::string>("Failed to init CURL"));
8888
}
8989

90+
// TODO: namh add header here
9091
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
9192
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
9293
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
@@ -189,6 +190,7 @@ cpp::result<bool, std::string> DownloadService::Download(
189190

190191
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers);
191192
}
193+
// TODO: namh add proxy setting here
192194
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback);
193195
curl_easy_setopt(curl, CURLOPT_WRITEDATA, file);
194196
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
@@ -337,6 +339,7 @@ void DownloadService::ProcessTask(DownloadTask& task, int worker_id) {
337339
});
338340
worker_data->downloading_data_map[item.id] = dl_data_ptr;
339341

342+
CTL_ERR("Namh Setup curl");
340343
SetUpCurlHandle(handle, item, file, dl_data_ptr.get());
341344
curl_multi_add_handle(worker_data->multi_handle, handle);
342345
task_handles.push_back(std::make_pair(handle, file));
@@ -407,13 +410,57 @@ cpp::result<void, ProcessDownloadFailed> DownloadService::ProcessMultiDownload(
407410

408411
void DownloadService::SetUpCurlHandle(CURL* handle, const DownloadItem& item,
409412
FILE* file, DownloadingData* dl_data) {
413+
auto configuration = config_service_->GetApiServerConfiguration();
414+
if (configuration.has_value()) {
415+
if (!configuration->proxy_url.empty()) {
416+
auto proxy_url = configuration->proxy_url;
417+
auto verify_proxy_ssl = configuration->verify_proxy_ssl;
418+
auto verify_proxy_host_ssl = configuration->verify_proxy_host_ssl;
419+
420+
auto verify_ssl = configuration->verify_peer_ssl;
421+
auto verify_host_ssl = configuration->verify_host_ssl;
422+
423+
auto proxy_username = configuration->proxy_username;
424+
auto proxy_password = configuration->proxy_password;
425+
426+
CTL_ERR("=== Proxy configuration ===");
427+
CTL_ERR("Proxy url: " << proxy_url);
428+
CTL_ERR("Verify proxy ssl: " << verify_proxy_ssl);
429+
CTL_ERR("Verify proxy host ssl: " << verify_proxy_host_ssl);
430+
CTL_ERR("Verify ssl: " << verify_ssl);
431+
CTL_ERR("Verify host ssl: " << verify_host_ssl);
432+
433+
curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str());
434+
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1L : 0L);
435+
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST,
436+
verify_host_ssl ? 2L : 0L);
437+
438+
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER,
439+
verify_proxy_ssl ? 1L : 0L);
440+
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST,
441+
verify_proxy_host_ssl ? 2L : 0L);
442+
443+
if (!proxy_username.empty()) {
444+
std::string proxy_auth = proxy_username + ":" + proxy_password;
445+
CTL_ERR("Proxy auth: " << proxy_auth);
446+
curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, proxy_auth.c_str());
447+
}
448+
449+
curl_easy_setopt(handle, CURLOPT_NOPROXY,
450+
configuration->no_proxy.c_str());
451+
}
452+
} else {
453+
CTL_ERR("Failed to get configuration");
454+
}
455+
410456
curl_easy_setopt(handle, CURLOPT_URL, item.downloadUrl.c_str());
411457
curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, WriteCallback);
412458
curl_easy_setopt(handle, CURLOPT_WRITEDATA, file);
413459
curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L);
414460
curl_easy_setopt(handle, CURLOPT_NOPROGRESS, 0L);
415461
curl_easy_setopt(handle, CURLOPT_XFERINFOFUNCTION, ProgressCallback);
416462
curl_easy_setopt(handle, CURLOPT_XFERINFODATA, dl_data);
463+
curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
417464

418465
auto headers = curl_utils::GetHeaders(item.downloadUrl);
419466
if (headers) {

‎engine/services/download_service.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include <unordered_set>
99
#include "common/download_task_queue.h"
1010
#include "common/event.h"
11+
#include "services/config_service.h"
1112
#include "utils/result.hpp"
1213

1314
struct ProcessDownloadFailed {
@@ -20,6 +21,8 @@ class DownloadService {
2021
private:
2122
static constexpr int MAX_CONCURRENT_TASKS = 4;
2223

24+
std::shared_ptr<ConfigService> config_service_;
25+
2326
struct DownloadingData {
2427
std::string task_id;
2528
std::string item_id;
@@ -82,8 +85,9 @@ class DownloadService {
8285

8386
explicit DownloadService() = default;
8487

85-
explicit DownloadService(std::shared_ptr<EventQueue> event_queue)
86-
: event_queue_{event_queue} {
88+
explicit DownloadService(std::shared_ptr<EventQueue> event_queue,
89+
std::shared_ptr<ConfigService> config_service)
90+
: event_queue_{event_queue}, config_service_{config_service} {
8791
InitializeWorkers();
8892
};
8993

‎engine/test/components/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ add_executable(${PROJECT_NAME}
99
${CMAKE_CURRENT_SOURCE_DIR}/../../config/gguf_parser.cc
1010
${CMAKE_CURRENT_SOURCE_DIR}/../../cli/commands/cortex_upd_cmd.cc
1111
${CMAKE_CURRENT_SOURCE_DIR}/../../cli/commands/server_stop_cmd.cc
12+
${CMAKE_CURRENT_SOURCE_DIR}/../../services/config_service.cc
1213
${CMAKE_CURRENT_SOURCE_DIR}/../../services/download_service.cc
1314
${CMAKE_CURRENT_SOURCE_DIR}/../../database/models.cc
1415
)

0 commit comments

Comments
 (0)