From eba3b0a5e427315c9e600b4e633be7e32fd2e3d2 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Fri, 6 Mar 2026 18:26:04 -0600 Subject: [PATCH 1/2] try dropshot with gzip --- Cargo.lock | 39 ++++++++++++++++++++++---- Cargo.toml | 4 ++- dns-server/src/lib.rs | 2 +- dns-server/tests/basic_test.rs | 2 +- dns-server/tests/cross_version_test.rs | 2 +- gateway/src/lib.rs | 2 +- installinator-api/src/lib.rs | 2 +- internal-dns/resolver/src/resolver.rs | 2 +- nexus/src/app/background/init.rs | 2 +- nexus/src/lib.rs | 10 +++++-- oximeter/producer/src/lib.rs | 2 +- sled-agent/src/services.rs | 6 ++-- sled-agent/src/sim/storage.rs | 2 +- wicketd/src/lib.rs | 2 +- 14 files changed, 58 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 334e640228a..5e44cd898fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,6 +284,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-compression" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-lock" version = "3.4.2" @@ -1772,6 +1784,23 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compression-codecs" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -3109,9 +3138,9 @@ dependencies = [ [[package]] name = "dropshot" version = "0.16.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69fd85c8dfc67252d02f260595f6b62b5abceb1b88b4b9722369d27936e5fa4" +source = "git+https://github.com/oxidecomputer/dropshot?rev=d707f541#d707f541f1f26326177f7bd64cf897001b657ec9" dependencies = [ + "async-compression", "async-stream", "async-trait", "base64 0.22.1", @@ -3150,7 +3179,8 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-rustls 0.25.0", - "toml 0.9.12+spec-1.1.0", + "tokio-util", + "toml 1.0.3+spec-1.1.0", "usdt 0.6.0", "uuid", "version_check", @@ -3206,8 +3236,7 @@ dependencies = [ [[package]] name = "dropshot_endpoint" version = "0.16.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67d106478e4a4782556981d028a667f41c4845cdaa6e2d3a9f58c5d15e725401" +source = "git+https://github.com/oxidecomputer/dropshot?rev=d707f541#d707f541f1f26326177f7bd64cf897001b657ec9" dependencies = [ "heck 0.5.0", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index c9c9abe8611..330b155a4c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -478,7 +478,7 @@ dns-server = { path = "dns-server" } dns-server-api = { path = "dns-server-api" } dns-service-client = { path = "clients/dns-service-client" } dpd-client = { git = "https://github.com/oxidecomputer/dendrite", rev = "f20f786e67c86388dfaaf0ef3aa9d2693dfe1da4" } -dropshot = { version = "0.16.6", features = [ "usdt-probes" ] } +dropshot = { git = "https://github.com/oxidecomputer/dropshot", rev = "d707f541", features = [ "usdt-probes" ] } dropshot-api-manager = "0.5.2" dropshot-api-manager-types = "0.5.2" dyn-clone = "1.0.20" @@ -1040,6 +1040,8 @@ opt-level = 3 # drift = { path = "../drift" } # dropshot = { path = "../dropshot/dropshot" } # dropshot_endpoint = { path = "../dropshot/dropshot_endpoint" } +dropshot = { git = "https://github.com/oxidecomputer/dropshot", rev = "d707f541" } +dropshot_endpoint = { git = "https://github.com/oxidecomputer/dropshot", rev = "d707f541" } # dropshot-api-manager = { path = "../dropshot-api-manager/crates/dropshot-api-manager" } # dropshot-api-manager-types = { path = "../dropshot-api-manager/crates/dropshot-api-manager-types" } # progenitor = { path = "../progenitor/progenitor" } diff --git a/dns-server/src/lib.rs b/dns-server/src/lib.rs index dde10a6387a..0a6dbe764ec 100644 --- a/dns-server/src/lib.rs +++ b/dns-server/src/lib.rs @@ -154,7 +154,7 @@ impl TransientServer { bind_address: "[::1]:0".parse().unwrap(), default_request_body_max_bytes: 4 * 1024 * 1024, default_handler_task_mode: dropshot::HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }, ) .await?; diff --git a/dns-server/tests/basic_test.rs b/dns-server/tests/basic_test.rs index 62aa73b1b19..20870298631 100644 --- a/dns-server/tests/basic_test.rs +++ b/dns-server/tests/basic_test.rs @@ -732,7 +732,7 @@ fn test_config( bind_address: "[::1]:0".to_string().parse().unwrap(), default_request_body_max_bytes: 1024, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }; Ok((tmp_dir, config_storage, config_dropshot, logctx)) diff --git a/dns-server/tests/cross_version_test.rs b/dns-server/tests/cross_version_test.rs index a2c1c323826..ba61ef075b8 100644 --- a/dns-server/tests/cross_version_test.rs +++ b/dns-server/tests/cross_version_test.rs @@ -275,7 +275,7 @@ fn test_config( bind_address: "[::1]:0".to_string().parse().unwrap(), default_request_body_max_bytes: 1024, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }; Ok((tmp_dir, config_storage, config_dropshot, logctx)) diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 86a05687e1b..a445f406162 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -94,7 +94,7 @@ fn start_dropshot_server( bind_address: SocketAddr::V6(addr), default_request_body_max_bytes, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }; let http_server = dropshot::ServerBuilder::new( diff --git a/installinator-api/src/lib.rs b/installinator-api/src/lib.rs index 3576643e272..6067c6a20c0 100644 --- a/installinator-api/src/lib.rs +++ b/installinator-api/src/lib.rs @@ -125,6 +125,6 @@ pub fn default_config(bind_address: std::net::SocketAddr) -> ConfigDropshot { bind_address, default_request_body_max_bytes: 1024, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() } } diff --git a/internal-dns/resolver/src/resolver.rs b/internal-dns/resolver/src/resolver.rs index 9ce3d6aa48d..38f59de1818 100644 --- a/internal-dns/resolver/src/resolver.rs +++ b/internal-dns/resolver/src/resolver.rs @@ -520,7 +520,7 @@ mod test { bind_address: "[::1]:0".parse().unwrap(), default_request_body_max_bytes: 8 * 1024, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }, ) .await diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 32aeefca3d5..c30d87d904f 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -1454,7 +1454,7 @@ pub mod test { bind_address: "[::1]:0".parse().unwrap(), default_request_body_max_bytes: 8 * 1024, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }, ) .await diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index f77751ce32f..5cadbdd803a 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -214,6 +214,12 @@ impl Server { // (but a different port) as the `internal` server. The latter is // available for proxied connections via the tech port in the event the // rack has lost connectivity (see RFD 431). + let external_dropshot_config = { + let mut c = config.deployment.dropshot_external.dropshot.clone(); + c.compression = dropshot::CompressionConfig::Gzip; + c + }; + let techport_server_bind_addr = { let mut addr = http_server_internal.local_addr(); addr.set_port(config.deployment.techport_external_server_port); @@ -221,7 +227,7 @@ impl Server { }; let techport_server_config = ConfigDropshot { bind_address: techport_server_bind_addr, - ..config.deployment.dropshot_external.dropshot.clone() + ..external_dropshot_config.clone() }; let http_server_external = { @@ -230,7 +236,7 @@ impl Server { apictx.for_external(), log.new(o!("component" => "dropshot_external")), ) - .config(config.deployment.dropshot_external.dropshot.clone()) + .config(external_dropshot_config.clone()) .version_policy(dropshot::VersionPolicy::Dynamic(Box::new( dropshot::ClientSpecifiesVersionInHeader::new( omicron_common::api::VERSION_HEADER, diff --git a/oximeter/producer/src/lib.rs b/oximeter/producer/src/lib.rs index 7d56a1657e2..26456d35476 100644 --- a/oximeter/producer/src/lib.rs +++ b/oximeter/producer/src/lib.rs @@ -220,7 +220,7 @@ impl Server { bind_address: server_info.address, default_request_body_max_bytes, default_handler_task_mode: dropshot::HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }; let server = Self::build_dropshot_server(&log, ®istry, &dropshot)?; diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 89e1e81adec..79d3362557b 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -2446,14 +2446,14 @@ impl ServiceManager { default_request_body_max_bytes: 1048576, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }, }, dropshot_internal: dropshot::ConfigDropshot { bind_address: (*internal_address).into(), default_request_body_max_bytes: 1048576, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }, dropshot_lockstep: dropshot::ConfigDropshot { bind_address: SocketAddr::new( @@ -2462,7 +2462,7 @@ impl ServiceManager { ), default_request_body_max_bytes: 1048576, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }, internal_dns: nexus_config::InternalDns::FromSubnet { subnet: Ipv6Subnet::::new( diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index a287868c2c3..4ac914e9052 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -2316,7 +2316,7 @@ impl PantryServer { // - bulk writes into disks default_request_body_max_bytes: 8192 * 1024, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }) .start() .expect("Could not initialize pantry server"); diff --git a/wicketd/src/lib.rs b/wicketd/src/lib.rs index d6cfcb6277d..0dd0ef0fd2a 100644 --- a/wicketd/src/lib.rs +++ b/wicketd/src/lib.rs @@ -135,7 +135,7 @@ impl Server { bind_address: SocketAddr::V6(args.address), default_request_body_max_bytes: 8 * 1024 * 1024, default_handler_task_mode: HandlerTaskMode::Detached, - log_headers: vec![], + ..Default::default() }; let mgs_manager = MgsManager::new(&log, args.mgs_address); From 4e5301c2bd66f71780176af43b60fa1ac8a8f1a0 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Fri, 6 Mar 2026 18:32:48 -0600 Subject: [PATCH 2/2] test it --- Cargo.lock | 1 + nexus/Cargo.toml | 1 + nexus/test-utils/src/http_testing.rs | 2 + nexus/tests/integration_tests/basic.rs | 53 ++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 5e44cd898fa..427ef20c03a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8521,6 +8521,7 @@ dependencies = [ "ereport-types", "expectorate", "fatfs", + "flate2", "futures", "gateway-client", "gateway-messages", diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 757229a1b4f..09939b4939e 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -168,6 +168,7 @@ camino-tempfile.workspace = true criterion.workspace = true diesel.workspace = true dns-server.workspace = true +flate2.workspace = true expectorate.workspace = true gateway-messages.workspace = true gateway-test-utils.workspace = true diff --git a/nexus/test-utils/src/http_testing.rs b/nexus/test-utils/src/http_testing.rs index 50a9ccdcb83..8c622348e23 100644 --- a/nexus/test-utils/src/http_testing.rs +++ b/nexus/test-utils/src/http_testing.rs @@ -97,6 +97,8 @@ impl<'a> RequestBuilder<'a> { http::header::DATE, http::header::LOCATION, http::header::SET_COOKIE, + http::header::TRANSFER_ENCODING, + http::header::VARY, http::header::HeaderName::from_static("x-request-id"), ]), expected_response_headers: http::HeaderMap::default(), diff --git a/nexus/tests/integration_tests/basic.rs b/nexus/tests/integration_tests/basic.rs index 87c65816060..2e663428976 100644 --- a/nexus/tests/integration_tests/basic.rs +++ b/nexus/tests/integration_tests/basic.rs @@ -557,3 +557,56 @@ async fn test_ping(cptestctx: &ControlPlaneTestContext) { .await; assert_eq!(health.status, system::PingStatus::Ok); } + +/// Test that the external API returns gzip-compressed responses when the +/// client sends Accept-Encoding: gzip. +#[nexus_test] +async fn test_gzip_compression(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + + // Create several projects so the response body exceeds the minimum + // compression threshold (512 bytes). + for i in 0..10 { + create_project(&client, &format!("project-{i}")).await; + } + + // With Accept-Encoding: gzip, response should be compressed. + let response = NexusRequest::new( + RequestBuilder::new(client, Method::GET, "/v1/projects") + .header(http::header::ACCEPT_ENCODING, "gzip") + .expect_status(Some(StatusCode::OK)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + + assert_eq!( + response.headers.get(http::header::CONTENT_ENCODING).unwrap(), + "gzip", + ); + + // Decompress and verify the body is valid JSON. + let compressed_len = response.body.len(); + let mut decoder = flate2::read::GzDecoder::new(&response.body[..]); + let mut decompressed = String::new(); + std::io::Read::read_to_string(&mut decoder, &mut decompressed).unwrap(); + let page: dropshot::ResultsPage = + serde_json::from_str(&decompressed).unwrap(); + assert_eq!(page.items.len(), 10); + + // Without Accept-Encoding: gzip, response should not be compressed. + let response = NexusRequest::object_get(client, "/v1/projects") + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + assert!(response.headers.get(http::header::CONTENT_ENCODING).is_none()); + + let uncompressed_len = response.body.len(); + assert!( + compressed_len < uncompressed_len, + "compressed body ({compressed_len} bytes) should be smaller \ + than uncompressed body ({uncompressed_len} bytes)" + ); +}