Skip to content

Commit d887f04

Browse files
committed
fix: thread timeout_secs through multi-model paths and add system route guard
- Add timeout_secs parameter to gateway_inference_set_multi and pass through to SetClusterInferenceRequest - Add print_timeout to multi-model output display - Add timeout field to router test helper make_route (upstream added timeout to ResolvedRoute) - Add system route guard: upsert_multi_model_route rejects route_name == sandbox-system with InvalidArgument - Add timeout_secs: 0 to multi-model test ClusterInferenceConfig structs - Add upsert_multi_model_route_rejects_system_route test Signed-off-by: Lyle Hopkins <lyle@cosmicnetworks.com>
1 parent 3e02f49 commit d887f04

File tree

3 files changed

+42
-0
lines changed

3 files changed

+42
-0
lines changed

crates/openshell-cli/src/run.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3628,6 +3628,15 @@ pub async fn gateway_inference_set_multi(
36283628
}
36293629
print_timeout(configured.timeout_secs);
36303630
if configured.validation_performed {
3631+
println!(" {}", "Validated Endpoints:".dimmed());
3632+
for endpoint in configured.validated_endpoints {
3633+
println!(" - {} ({})", endpoint.url, endpoint.protocol);
3634+
}
3635+
}
3636+
Ok(())
3637+
}
3638+
3639+
pub async fn gateway_inference_update(
36313640
server: &str,
36323641
provider_name: Option<&str>,
36333642
model_id: Option<&str>,

crates/openshell-router/src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ mod tests {
218218
protocols: protocols.into_iter().map(String::from).collect(),
219219
auth: config::AuthHeader::Bearer,
220220
default_headers: Vec::new(),
221+
timeout: std::time::Duration::from_secs(60),
221222
}
222223
}
223224

crates/openshell-server/src/inference.rs

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,14 @@ async fn upsert_multi_model_route(
254254
return Err(Status::invalid_argument("models list is empty"));
255255
}
256256

257+
// The system route must remain single-model so the sandbox can always
258+
// resolve it to exactly one backend without alias selection.
259+
if route_name == SANDBOX_SYSTEM_ROUTE_NAME {
260+
return Err(Status::invalid_argument(
261+
"multi-model is not supported on the system route",
262+
));
263+
}
264+
257265
// Names reserved for internal route partitioning (sandbox uses
258266
// `name == "sandbox-system"` to split user vs system caches).
259267
const RESERVED_ALIASES: &[&str] = &["sandbox-system", "inference.local"];
@@ -1228,6 +1236,7 @@ mod tests {
12281236
config: Some(ClusterInferenceConfig {
12291237
provider_name: "openai-dev".to_string(),
12301238
model_id: "gpt-4o".to_string(),
1239+
timeout_secs: 0,
12311240
models: vec![
12321241
InferenceModelEntry {
12331242
alias: "my-gpt".to_string(),
@@ -1278,6 +1287,7 @@ mod tests {
12781287
config: Some(ClusterInferenceConfig {
12791288
provider_name: "openai-dev".to_string(),
12801289
model_id: "gpt-4o".to_string(),
1290+
timeout_secs: 0,
12811291
models: vec![
12821292
InferenceModelEntry {
12831293
alias: "my-gpt".to_string(),
@@ -1363,4 +1373,26 @@ mod tests {
13631373
assert_eq!(err.code(), tonic::Code::InvalidArgument);
13641374
assert!(err.message().contains("reserved"));
13651375
}
1376+
1377+
#[tokio::test]
1378+
async fn upsert_multi_model_route_rejects_system_route() {
1379+
let store = Store::connect("sqlite::memory:?cache=shared")
1380+
.await
1381+
.expect("store");
1382+
1383+
let openai = make_provider("openai-dev", "openai", "OPENAI_API_KEY", "sk-openai");
1384+
store.put_message(&openai).await.expect("persist");
1385+
1386+
let models = vec![InferenceModelEntry {
1387+
alias: "fast-gpt".to_string(),
1388+
provider_name: "openai-dev".to_string(),
1389+
model_id: "gpt-4o-mini".to_string(),
1390+
}];
1391+
1392+
let err = upsert_multi_model_route(&store, SANDBOX_SYSTEM_ROUTE_NAME, &models, false)
1393+
.await
1394+
.expect_err("should reject system route");
1395+
assert_eq!(err.code(), tonic::Code::InvalidArgument);
1396+
assert!(err.message().contains("system route"));
1397+
}
13661398
}

0 commit comments

Comments
 (0)