extract docs_rs_headers crate with custom headers:

Author: syphar

Cargo.lock

@@ -22,11 +22,13 @@ exclude = [
 
 [workspace.dependencies]
 anyhow = { version = "1.0.42", features = ["backtrace"]}
+axum-extra = { version = "0.12.0", features = ["typed-header", "routing", "middleware"] }
 askama = "0.14.0"
 bincode = "2.0.1"
 chrono = { version = "0.4.11", default-features = false, features = ["clock", "serde"] }
 derive_more = { version = "2.0.0", features = ["display", "deref", "from", "into", "from_str"] }
 http = "1.0.0"
+itertools = { version = "0.14.0" }
 opentelemetry = "0.31.0"
 opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic",  "metrics"] }
 opentelemetry-resource-detectors = "0.10.0"
@@ -43,6 +45,7 @@ url = { version = "2.1.1", features = ["serde"] }
 
 [dependencies]
 docs_rs_env_vars = { path = "crates/lib/docs_rs_env_vars" }
+docs_rs_headers = { path = "crates/lib/docs_rs_headers" }
 docs_rs_logging = { path = "crates/lib/docs_rs_logging" }
 docs_rs_opentelemetry = { path = "crates/lib/docs_rs_opentelemetry" }
 docs_rs_types = { path = "crates/lib/docs_rs_types" }
@@ -85,7 +88,7 @@ dashmap = "6.0.0"
 zip = {version = "6.0.0", default-features = false, features = ["bzip2"]}
 bzip2 = "0.6.0"
 getrandom = "0.3.1"
-itertools = { version = "0.14.0" }
+itertools = { workspace = true }
 hex = "0.4.3"
 derive_more = { workspace = true }
 sysinfo = { version = "0.37.2", default-features = false, features = ["system"] }
@@ -111,7 +114,7 @@ bincode = { workspace = true }
 # axum dependencies
 async-trait = "0.1.83"
 axum = { version = "0.8.1", features = ["macros"] }
-axum-extra = { version = "0.12.0", features = ["typed-header", "routing", "middleware"] }
+axum-extra = { workspace = true }
 tower = "0.5.1"
 tower-http = { version = "0.6.0", features = ["fs", "trace", "timeout", "catch-panic"] }
 mime = "0.3.16"
@@ -129,11 +132,9 @@ chrono = { workspace = true }
 
 # Transitive dependencies we don't use directly but need to have specific versions of
 constant_time_eq = "0.4.2"
-md5 = "0.8.0"
-
-crates_io_validation = { path = "crates/lib/crates_io_validation" }
 
 [dev-dependencies]
+docs_rs_headers = { path = "crates/lib/docs_rs_headers", features = ["testing"] }
 docs_rs_opentelemetry = { path = "crates/lib/docs_rs_opentelemetry", features = ["testing"] }
 docs_rs_types = { path = "crates/lib/docs_rs_types", features = ["testing"] }
 criterion = "0.8.0"

Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "docs_rs_headers"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+axum-core = "0.5.5"
+axum-extra = { workspace = true }
+askama = { workspace = true }
+derive_more = { workspace = true }
+docs_rs_types = { path = "../docs_rs_types" }
+docs_rs_uri = { path = "../docs_rs_uri" }
+http = { workspace = true }
+itertools = { workspace = true }
+md5 = "0.8.0"
+serde = { workspace = true }
+
+[dev-dependencies]
+serde_json = { workspace = true }
+test-case = { workspace = true }
+tokio = { workspace = true }
+
+[features]
+testing = []

crates/lib/docs_rs_headers/Cargo.toml

@@ -1,8 +1,8 @@
 use anyhow::Result;
 use askama::filters::HtmlSafe;
-use axum::http::uri::Uri;
 use axum_extra::headers::{Header, HeaderName, HeaderValue};
 use docs_rs_uri::EscapedURI;
+use http::uri::Uri;
 use serde::Serialize;
 use std::{fmt, ops::Deref};
 
@@ -96,9 +96,8 @@ impl HtmlSafe for CanonicalUrl {}
 #[cfg(test)]
 mod tests {
     use super::*;
-
-    use axum::http::HeaderMap;
     use axum_extra::headers::HeaderMapExt;
+    use http::HeaderMap;
 
     #[test]
     fn test_serialize_canonical_from_uri() {

src/web/headers/canonical_url.rs …lib/docs_rs_headers/src/canonical_url.rssrc/web/headers/canonical_url.rs renamed to crates/lib/docs_rs_headers/src/canonical_url.rs src/web/headers/canonical_url.rs renamed to crates/lib/docs_rs_headers/src/canonical_url.rs

@@ -1,22 +1,6 @@
-mod canonical_url;
-mod if_none_match;
-mod surrogate_key;
-
 use axum_extra::headers::ETag;
-use http::HeaderName;
 use std::io::{self, Write};
 
-pub use canonical_url::CanonicalUrl;
-pub(crate) use if_none_match::IfNoneMatch;
-pub use surrogate_key::{SURROGATE_KEY, SurrogateKey, SurrogateKeys};
-
-/// Fastly's Surrogate-Control header
-/// https://www.fastly.com/documentation/reference/http/http-headers/Surrogate-Control/
-pub static SURROGATE_CONTROL: HeaderName = HeaderName::from_static("surrogate-control");
-
-/// X-Robots-Tag header for search engines.
-pub static X_ROBOTS_TAG: HeaderName = HeaderName::from_static("x-robots-tag");
-
 /// compute our etag header value from some content
 ///
 /// Has to match the implementation in our build-script.
@@ -30,7 +14,8 @@ pub fn compute_etag<T: AsRef<[u8]>>(content: T) -> ETag {
 ///
 /// Works the same way as the inner `md5::Context`,
 /// but produces an `ETag` when finalized.
-pub(crate) struct ETagComputer(md5::Context);
+#[derive(Default)]
+pub struct ETagComputer(md5::Context);
 
 impl ETagComputer {
     pub fn new() -> Self {

src/web/headers/mod.rs crates/lib/docs_rs_headers/src/etag.rssrc/web/headers/mod.rs renamed to crates/lib/docs_rs_headers/src/etag.rs

@@ -28,7 +28,7 @@ mod header_impl {
     use derive_more::Deref;
 
     #[derive(Debug, Clone, PartialEq, Deref)]
-    pub(crate) struct IfNoneMatch(pub axum_extra::headers::IfNoneMatch);
+    pub struct IfNoneMatch(pub axum_extra::headers::IfNoneMatch);
 
     impl Header for IfNoneMatch {
         fn name() -> &'static http::HeaderName {
@@ -65,13 +65,13 @@ mod header_impl {
     }
 }
 
-pub(crate) use header_impl::IfNoneMatch;
+pub use header_impl::IfNoneMatch;
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use anyhow::Result;
-    use axum::{RequestPartsExt, body::Body, extract::Request};
+    use axum_core::{RequestPartsExt as _, body::Body, extract::Request};
     use axum_extra::{
         TypedHeader,
         headers::{ETag, HeaderMapExt as _},

src/web/headers/if_none_match.rs …lib/docs_rs_headers/src/if_none_match.rssrc/web/headers/if_none_match.rs renamed to crates/lib/docs_rs_headers/src/if_none_match.rs src/web/headers/if_none_match.rs renamed to crates/lib/docs_rs_headers/src/if_none_match.rs

@@ -0,0 +1,21 @@
+mod canonical_url;
+mod etag;
+mod if_none_match;
+mod surrogate_key;
+#[cfg(test)]
+mod testing;
+
+pub use axum_extra::headers::ETag;
+pub use canonical_url::CanonicalUrl;
+pub use etag::{ETagComputer, compute_etag};
+pub use if_none_match::IfNoneMatch;
+pub use surrogate_key::{SURROGATE_KEY, SurrogateKey, SurrogateKeys};
+
+use http::HeaderName;
+
+/// Fastly's Surrogate-Control header
+/// https://www.fastly.com/documentation/reference/http/http-headers/Surrogate-Control/
+pub static SURROGATE_CONTROL: HeaderName = HeaderName::from_static("surrogate-control");
+
+/// X-Robots-Tag header for search engines.
+pub static X_ROBOTS_TAG: HeaderName = HeaderName::from_static("x-robots-tag");

crates/lib/docs_rs_headers/src/lib.rs

@@ -96,7 +96,7 @@ impl From<KrateName> for SurrogateKey {
 }
 
 /// A full Fastly Surrogate-Key header, containing zero or more keys.
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Default)]
 pub struct SurrogateKeys(BTreeSet<SurrogateKey>);
 
 impl IntoIterator for SurrogateKeys {
@@ -239,7 +239,7 @@ impl SurrogateKeys {
         Ok(())
     }
 
-    #[cfg(test)]
+    #[cfg(feature = "testing")]
     pub fn try_from_iter<I>(iter: I) -> anyhow::Result<Self>
     where
         I: IntoIterator<Item = SurrogateKey>,
@@ -258,7 +258,7 @@ impl SurrogateKeys {
     }
 }
 
-#[cfg(test)]
+#[cfg(feature = "testing")]
 impl FromStr for SurrogateKeys {
     type Err = anyhow::Error;
 
@@ -274,7 +274,7 @@ impl FromStr for SurrogateKeys {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test::headers::{test_typed_decode, test_typed_encode};
+    use crate::testing::{test_typed_decode, test_typed_encode};
     use std::ops::RangeInclusive;
     use test_case::test_case;
 

src/web/headers/surrogate_key.rs …lib/docs_rs_headers/src/surrogate_key.rssrc/web/headers/surrogate_key.rs renamed to crates/lib/docs_rs_headers/src/surrogate_key.rs src/web/headers/surrogate_key.rs renamed to crates/lib/docs_rs_headers/src/surrogate_key.rs

@@ -0,0 +1,24 @@
+use axum_extra::headers::{self, Header, HeaderMapExt};
+use http::{HeaderMap, HeaderValue};
+
+pub(crate) fn test_typed_decode<H, V>(value: V) -> Result<Option<H>, headers::Error>
+where
+    H: Header,
+    V: TryInto<HeaderValue>,
+    <V as TryInto<http::HeaderValue>>::Error: std::fmt::Debug,
+{
+    let mut map = HeaderMap::new();
+    map.append(
+        H::name(),
+        // this `.try_into` only generates the `HeaderValue` items.
+        value.try_into().unwrap(),
+    );
+    // parsing errors from the typed header end up here.
+    map.typed_try_get()
+}
+
+pub(crate) fn test_typed_encode<H: Header>(header: H) -> HeaderValue {
+    let mut map = HeaderMap::new();
+    map.typed_insert(header);
+    map.get(H::name()).cloned().unwrap()
+}

crates/lib/docs_rs_headers/src/testing.rs

@@ -1,11 +1,7 @@
-use crate::{
-    APP_USER_AGENT,
-    cdn::CdnMetrics,
-    config::Config,
-    web::headers::{SURROGATE_KEY, SurrogateKey, SurrogateKeys},
-};
+use crate::{APP_USER_AGENT, cdn::CdnMetrics, config::Config};
 use anyhow::{Result, anyhow, bail};
 use chrono::{DateTime, TimeZone as _, Utc};
+use docs_rs_headers::{SURROGATE_KEY, SurrogateKey, SurrogateKeys};
 use http::{
     HeaderMap, HeaderName, HeaderValue,
     header::{ACCEPT, USER_AGENT},