From 38422a83b098447a30b2195c5d3711934e24beb4 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Tue, 18 Mar 2025 02:32:13 +0000 Subject: [PATCH 01/52] Use `from_static` instead of `from_string` --- src/main.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index e1b010df..e4af4b70 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,7 @@ use cached::proc_macro::cached; use clap::{Arg, ArgAction, Command}; use std::str::FromStr; - +use bincode::Error; use futures_lite::FutureExt; use hyper::Uri; use hyper::{header::HeaderValue, Body, Request, Response}; @@ -83,9 +83,7 @@ async fn resource(body: &str, content_type: &str, cache: bool) -> Result Result, String> { #[cached(time = 600)] async fn fetch_commit_info() -> String { - let uri = Uri::from_str("https://github.com/redlib-org/redlib/commits/main.atom").expect("Invalid URI"); + let uri = Uri::from_static("https://github.com/redlib-org/redlib/commits/main.atom"); let resp: Body = CLIENT.get(uri).await.expect("Failed to request GitHub").into_body(); @@ -452,7 +450,7 @@ pub async fn proxy_instances() -> Result, String> { #[cached(time = 600)] async fn fetch_instances() -> String { - let uri = Uri::from_str("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json").expect("Invalid URI"); + let uri = Uri::from_static("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json"); let resp: Body = CLIENT.get(uri).await.expect("Failed to request GitHub").into_body(); From 87206772b88b5fde6e8c4f55fd7fe62d30419866 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Tue, 18 Mar 2025 03:02:07 +0000 Subject: [PATCH 02/52] On error, fallback to cache; Use `hyper::Error` The interaction with the rest of the codebase is a little ugly, but functionality is practically the same. --- src/main.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main.rs b/src/main.rs index e4af4b70..fc1480eb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,8 +4,6 @@ use cached::proc_macro::cached; use clap::{Arg, ArgAction, Command}; -use std::str::FromStr; -use bincode::Error; use futures_lite::FutureExt; use hyper::Uri; use hyper::{header::HeaderValue, Body, Request, Response}; @@ -259,7 +257,7 @@ async fn main() { app.at("/copy.js").get(|_| resource(include_str!("../static/copy.js"), "text/javascript", false).boxed()); app.at("/commits.atom").get(|_| async move { proxy_commit_info().await }.boxed()); - app.at("/instances.json").get(|_| async move { proxy_instances().await }.boxed()); + app.at("/instances.json").get(|_| async move { Ok(proxy_instances().await.unwrap()) }.boxed()); // TODO: In the process of migrating error handling. (I recommend thiserror crate for dynamic errors.) No proper error handling yes, so functionality unimpacted. // Proxy media through Redlib app.at("/vid/:id/:size").get(|r| proxy(r, "https://v.redd.it/{id}/DASH_{size}").boxed()); @@ -438,21 +436,21 @@ async fn fetch_commit_info() -> String { hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect() } -pub async fn proxy_instances() -> Result, String> { +pub async fn proxy_instances() -> Result, hyper::Error> { Ok( Response::builder() .status(200) .header("content-type", "application/json") - .body(Body::from(fetch_instances().await)) + .body(Body::from(fetch_instances().await?)) .unwrap_or_default(), ) } -#[cached(time = 600)] -async fn fetch_instances() -> String { +#[cached(time = 600, result = true, result_fallback = true)] +async fn fetch_instances() -> Result { let uri = Uri::from_static("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json"); - let resp: Body = CLIENT.get(uri).await.expect("Failed to request GitHub").into_body(); + let resp: Body = CLIENT.get(uri).await?.into_body(); // Could fail if there is no internet - hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect() + Ok(hyper::body::to_bytes(resp).await?.iter().copied().map(|x| x as char).collect()) } From 47771b57c06984ca6d8530bc9772e7131132a03a Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Tue, 18 Mar 2025 03:33:52 +0000 Subject: [PATCH 03/52] Same fix for another function --- src/main.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/main.rs b/src/main.rs index fc1480eb..d56d06ef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -256,7 +256,7 @@ async fn main() { .get(|_| resource(include_str!("../static/check_update.js"), "text/javascript", false).boxed()); app.at("/copy.js").get(|_| resource(include_str!("../static/copy.js"), "text/javascript", false).boxed()); - app.at("/commits.atom").get(|_| async move { proxy_commit_info().await }.boxed()); + app.at("/commits.atom").get(|_| async move { Ok(proxy_instances().await.unwrap()) }.boxed()); // TODO: see below app.at("/instances.json").get(|_| async move { Ok(proxy_instances().await.unwrap()) }.boxed()); // TODO: In the process of migrating error handling. (I recommend thiserror crate for dynamic errors.) No proper error handling yes, so functionality unimpacted. // Proxy media through Redlib @@ -417,23 +417,21 @@ async fn main() { } } -pub async fn proxy_commit_info() -> Result, String> { +pub async fn proxy_commit_info() -> Result, hyper::Error> { Ok( Response::builder() .status(200) .header("content-type", "application/atom+xml") - .body(Body::from(fetch_commit_info().await)) + .body(Body::from(fetch_commit_info().await?)) .unwrap_or_default(), ) } -#[cached(time = 600)] -async fn fetch_commit_info() -> String { +#[cached(time = 600, result = true, result_fallback = true)] +async fn fetch_commit_info() -> Result { let uri = Uri::from_static("https://github.com/redlib-org/redlib/commits/main.atom"); - - let resp: Body = CLIENT.get(uri).await.expect("Failed to request GitHub").into_body(); - - hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect() + let resp: Body = CLIENT.get(uri).await?.into_body(); // Could fail if there is no internet + Ok(hyper::body::to_bytes(resp).await?.iter().copied().map(|x| x as char).collect()) } pub async fn proxy_instances() -> Result, hyper::Error> { @@ -441,7 +439,7 @@ pub async fn proxy_instances() -> Result, hyper::Error> { Response::builder() .status(200) .header("content-type", "application/json") - .body(Body::from(fetch_instances().await?)) + .body(Body::from(fetch_instances().await?))// Could fail if no internet .unwrap_or_default(), ) } @@ -449,8 +447,6 @@ pub async fn proxy_instances() -> Result, hyper::Error> { #[cached(time = 600, result = true, result_fallback = true)] async fn fetch_instances() -> Result { let uri = Uri::from_static("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json"); - - let resp: Body = CLIENT.get(uri).await?.into_body(); // Could fail if there is no internet - + let resp: Body = CLIENT.get(uri).await?.into_body(); // Could fail if no internet Ok(hyper::body::to_bytes(resp).await?.iter().copied().map(|x| x as char).collect()) } From d6d399d97b8064b2b55fdaac839db2968cd71ba5 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Tue, 18 Mar 2025 03:38:51 +0000 Subject: [PATCH 04/52] cargo fmt --- src/main.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index d56d06ef..3cd0d655 100644 --- a/src/main.rs +++ b/src/main.rs @@ -81,7 +81,9 @@ async fn resource(body: &str, content_type: &str, cache: bool) -> Result Result, hyper::Error> { Response::builder() .status(200) .header("content-type", "application/json") - .body(Body::from(fetch_instances().await?))// Could fail if no internet + .body(Body::from(fetch_instances().await?)) // Could fail if no internet .unwrap_or_default(), ) } From f3043932a5cda1305fbf365d5d4fa865965a353a Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Tue, 18 Mar 2025 19:36:33 +0000 Subject: [PATCH 05/52] Fix wrong content served in previous commit --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 3cd0d655..623729ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -258,7 +258,7 @@ async fn main() { .get(|_| resource(include_str!("../static/check_update.js"), "text/javascript", false).boxed()); app.at("/copy.js").get(|_| resource(include_str!("../static/copy.js"), "text/javascript", false).boxed()); - app.at("/commits.atom").get(|_| async move { Ok(proxy_instances().await.unwrap()) }.boxed()); // TODO: see below + app.at("/commits.atom").get(|_| async move { Ok(proxy_commit_info().await.unwrap()) }.boxed()); // TODO: see below app.at("/instances.json").get(|_| async move { Ok(proxy_instances().await.unwrap()) }.boxed()); // TODO: In the process of migrating error handling. (I recommend thiserror crate for dynamic errors.) No proper error handling yes, so functionality unimpacted. // Proxy media through Redlib From eb7fbf4682910b30b6be1c703afddf4cbe50371b Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Tue, 18 Mar 2025 19:38:07 +0000 Subject: [PATCH 06/52] Begin removing some deprecated `to_bytes` Refactor `main::fetch_instances()` and `main::fetch_commit_info()` Enabled deprecated warnings in hyper crate --- Cargo.toml | 2 +- src/main.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c7b6d4a5..2c916b5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ regex = "1.10.2" serde = { version = "1.0.193", features = ["derive"] } cookie = "0.18.0" futures-lite = "2.2.0" -hyper = { version = "0.14.31", features = ["full"] } +hyper = { version = "0.14.32", features = ["stream", "backports", "deprecated"] } percent-encoding = "2.3.1" route-recognizer = "0.3.1" serde_json = "1.0.133" diff --git a/src/main.rs b/src/main.rs index 623729ed..32cf0bf5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,7 @@ use cached::proc_macro::cached; use clap::{Arg, ArgAction, Command}; use futures_lite::FutureExt; use hyper::Uri; -use hyper::{header::HeaderValue, Body, Request, Response}; +use hyper::{body::Bytes, header::HeaderValue, Body, Request, Response}; use log::{info, warn}; use once_cell::sync::Lazy; use redlib::client::{canonical_path, proxy, rate_limit_check, CLIENT}; @@ -430,10 +430,10 @@ pub async fn proxy_commit_info() -> Result, hyper::Error> { } #[cached(time = 600, result = true, result_fallback = true)] -async fn fetch_commit_info() -> Result { +async fn fetch_commit_info() -> Result { let uri = Uri::from_static("https://github.com/redlib-org/redlib/commits/main.atom"); let resp: Body = CLIENT.get(uri).await?.into_body(); // Could fail if there is no internet - Ok(hyper::body::to_bytes(resp).await?.iter().copied().map(|x| x as char).collect()) + Ok(hyper::body::HttpBody::collect(resp).await?.to_bytes()) } pub async fn proxy_instances() -> Result, hyper::Error> { @@ -447,8 +447,8 @@ pub async fn proxy_instances() -> Result, hyper::Error> { } #[cached(time = 600, result = true, result_fallback = true)] -async fn fetch_instances() -> Result { +async fn fetch_instances() -> Result { let uri = Uri::from_static("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json"); - let resp: Body = CLIENT.get(uri).await?.into_body(); // Could fail if no internet - Ok(hyper::body::to_bytes(resp).await?.iter().copied().map(|x| x as char).collect()) + let resp = CLIENT.get(uri).await?.into_body(); // Could fail if no internet + Ok(hyper::body::HttpBody::collect(resp).await?.to_bytes()) } From bac01e1866c1f64b8105f815f10cf85b92725b38 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 02:49:59 +0000 Subject: [PATCH 07/52] Start using http_api_problem Settings encoded_restore was updated main.rs now temporarily uses map_error to map into strings. Later, the ApiError should be converted into a Response using the crate's utilities. --- Cargo.lock | 158 ++++++++++++++++++++++++++++++++++++++---------- Cargo.toml | 2 + src/main.rs | 11 ++-- src/server.rs | 9 +++ src/settings.rs | 56 +++++++++++++---- src/utils.rs | 8 +-- 6 files changed, 191 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d03edade..62eb62f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,7 +85,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -96,7 +96,7 @@ checksum = "d556ec1359574147ec0c4fc5eb525f3f23263a592b1a9c07e0a75b427de55c97" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -257,7 +257,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -319,7 +319,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -412,7 +412,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.99", ] [[package]] @@ -423,7 +423,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -459,7 +459,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -469,7 +469,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn", + "syn 2.0.99", +] + +[[package]] +name = "derive_utils" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7590f99468735a318c254ca9158d0c065aa9b5312896b5a043b5e39bc96f5fa2" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -499,7 +510,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -622,6 +633,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -641,10 +663,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", + "futures-macro", "futures-sink", "futures-task", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -710,7 +734,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", "indexmap", "slab", "tokio", @@ -763,6 +787,39 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-api-problem" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "000bed434eb9b5dfb8ea5ed904f02ffc63aa0ac7ac034d480dc24fbb97b748d8" +dependencies = [ + "http 1.3.1", + "http-api-problem-derive", + "hyper 1.6.0", + "serde", + "serde_json", +] + +[[package]] +name = "http-api-problem-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb363d7867c5d2b8d719b857b9807a19ceb917703d99fb3a6616ec66ab24a8dd" +dependencies = [ + "derive_utils", +] + [[package]] name = "http-body" version = "0.4.6" @@ -770,10 +827,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", "pin-project-lite", ] +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.3.1", +] + [[package]] name = "httparse" version = "1.10.1" @@ -803,8 +870,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -816,6 +883,18 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "http 1.3.1", + "http-body 1.0.1", + "tokio", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -823,8 +902,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.32", "log", "rustls", "rustls-native-certs", @@ -947,7 +1026,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -1372,8 +1451,10 @@ dependencies = [ "dotenvy", "fastrand", "futures-lite", + "futures-util", "htmlescape", - "hyper", + "http-api-problem", + "hyper 0.14.32", "hyper-rustls", "libflate", "lipsum", @@ -1457,7 +1538,7 @@ checksum = "5f0ec466e5d8dca9965eb6871879677bef5590cf7525ad96cae14376efb75073" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -1497,7 +1578,7 @@ dependencies = [ "quote", "rinja_parser", "rustc-hash", - "syn", + "syn 2.0.99", ] [[package]] @@ -1554,7 +1635,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn", + "syn 2.0.99", "walkdir", ] @@ -1714,7 +1795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77253fb2d4451418d07025826028bcb96ee42d3e58859689a70ce62908009db6" dependencies = [ "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -1757,7 +1838,7 @@ checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -1819,7 +1900,7 @@ checksum = "aafbefbe175fa9bf03ca83ef89beecff7d2a95aaacd5732325b90ac8c3bd7b90" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -1919,6 +2000,17 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.99" @@ -1938,7 +2030,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -1999,7 +2091,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -2010,7 +2102,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -2082,7 +2174,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -2305,7 +2397,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.99", "wasm-bindgen-shared", ] @@ -2327,7 +2419,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2492,7 +2584,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", "synstructure", ] @@ -2514,7 +2606,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] [[package]] @@ -2534,7 +2626,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", "synstructure", ] @@ -2557,5 +2649,5 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.99", ] diff --git a/Cargo.toml b/Cargo.toml index 2c916b5b..d281219e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,8 @@ htmlescape = "0.3.1" bincode = "1.3.3" base2048 = "2.0.2" revision = "0.10.0" +http-api-problem = { version = "0.60", features = ["hyper", "api-error"] } +futures-util = "0.3.31" [dev-dependencies] diff --git a/src/main.rs b/src/main.rs index 32cf0bf5..43b946a5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,8 @@ use redlib::{config, duplicates, headers, instance_info, post, search, settings, use redlib::client::OAUTH_CLIENT; +use futures_util::future::TryFutureExt; + // Create Services // Required for the manifest to be valid @@ -258,9 +260,8 @@ async fn main() { .get(|_| resource(include_str!("../static/check_update.js"), "text/javascript", false).boxed()); app.at("/copy.js").get(|_| resource(include_str!("../static/copy.js"), "text/javascript", false).boxed()); - app.at("/commits.atom").get(|_| async move { Ok(proxy_commit_info().await.unwrap()) }.boxed()); // TODO: see below - app.at("/instances.json").get(|_| async move { Ok(proxy_instances().await.unwrap()) }.boxed()); // TODO: In the process of migrating error handling. (I recommend thiserror crate for dynamic errors.) No proper error handling yes, so functionality unimpacted. - + app.at("/commits.atom").get(|_| proxy_commit_info().map_err(|e| e.to_string()).boxed()); + app.at("/instances.json").get(|_| proxy_instances().map_err(|e| e.to_string()).boxed()); // Proxy media through Redlib app.at("/vid/:id/:size").get(|r| proxy(r, "https://v.redd.it/{id}/DASH_{size}").boxed()); app.at("/hls/:id/*path").get(|r| proxy(r, "https://v.redd.it/{id}/{path}").boxed()); @@ -295,7 +296,9 @@ async fn main() { // Configure settings app.at("/settings").get(|r| settings::get(r).boxed()).post(|r| settings::set(r).boxed()); app.at("/settings/restore").get(|r| settings::restore(r).boxed()); - app.at("/settings/encoded-restore").post(|r| settings::encoded_restore(r).boxed()); + app + .at("/settings/encoded-restore") + .post(|r| settings::encoded_restore(r).map_err(|e| e.to_string()).boxed()); app.at("/settings/update").get(|r| settings::update(r).boxed()); // RSS Subscriptions diff --git a/src/server.rs b/src/server.rs index a287de23..684d87e1 100644 --- a/src/server.rs +++ b/src/server.rs @@ -31,6 +31,15 @@ use crate::dbg_msg; type BoxResponse = Pin, String>> + Send>>; +// thiserror error combing hyper::Error and http:Error +/*#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + Hyper(#[from] hyper::Error), + #[error(transparent)] + Http(#[from] http::Error), +}*/ + /// Compressors for the response Body, in ascending order of preference. #[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] enum CompressionType { diff --git a/src/settings.rs b/src/settings.rs index 6649f694..ab90e4f8 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -8,6 +8,7 @@ use crate::subreddit::join_until_size_limit; use crate::utils::{deflate_decompress, redirect, template, Preferences}; use cookie::Cookie; use futures_lite::StreamExt; +use http_api_problem::ApiError; use hyper::{Body, Request, Response}; use rinja::Template; use time::{Duration, OffsetDateTime}; @@ -264,24 +265,55 @@ pub async fn update(req: Request) -> Result, String> { Ok(set_cookies_method(req, false)) } -pub async fn encoded_restore(req: Request) -> Result, String> { - let body = hyper::body::to_bytes(req.into_body()) - .await - .map_err(|e| format!("Failed to get bytes from request body: {}", e))?; +pub async fn encoded_restore(req: Request) -> Result, ApiError> { + let body = hyper::body::to_bytes(req.into_body()).await.map_err(|e| { + ApiError::builder(http_api_problem::StatusCode::REQUEST_TIMEOUT) // 408 + .title("Request Timeout") + .message(format!("The server could not receive the request: {e}")) + .source(e) + .finish() + })?; let encoded_prefs = form_urlencoded::parse(&body) .find(|(key, _)| key == "encoded_prefs") .map(|(_, value)| value) - .ok_or_else(|| "encoded_prefs parameter not found in request body".to_string())?; - - let bytes = base2048::decode(&encoded_prefs).ok_or_else(|| "Failed to decode base2048 encoded preferences".to_string())?; - - let out = deflate_decompress(bytes)?; - - let mut prefs: Preferences = bincode::deserialize(&out).map_err(|e| format!("Failed to deserialize bytes into Preferences struct: {}", e))?; + .ok_or_else(|| { + ApiError::builder(http_api_problem::StatusCode::BAD_REQUEST) // 400 + .title("Bad Request") + .message("Missing encoded_prefs parameter") + .finish() + })?; + + let bytes = base2048::decode(&encoded_prefs).ok_or_else(|| { + ApiError::builder(http_api_problem::StatusCode::BAD_REQUEST) // 400 + .title("Invalid base2048") + .message(format!("Given invalid base2048 string as encoded preferences: {}", &encoded_prefs)) + })?; + + let out = deflate_decompress(bytes).map_err(|e| { + ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) // 500 + .title("Could not read deflate stream") + .message(e.to_string()) + .source(e) + .finish() + })?; + + let mut prefs: Preferences = bincode::deserialize(&out).map_err(|e| { + ApiError::builder(http_api_problem::StatusCode::BAD_REQUEST) // 400 + .title("Invalid preference string") + .message(format!("Failed to deserialize bytes into Preferences struct: {}", e)) + .source(e) + })?; prefs.available_themes = vec![]; - let url = format!("/settings/restore/?{}", prefs.to_urlencoded()?); + let url = format!( + "/settings/restore/?{}", + prefs.to_urlencoded().map_err(|e| ApiError::builder(http_api_problem::StatusCode::BAD_REQUEST) // 400 + .title("Invalid preference string") + .message(format!("Given preference struct is not url-encodable: {}", e)) + .source(e) + .finish())? + ); Ok(redirect(&url)) } diff --git a/src/utils.rs b/src/utils.rs index f5046cb3..22a49391 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -731,8 +731,8 @@ impl Preferences { } } - pub fn to_urlencoded(&self) -> Result { - serde_urlencoded::to_string(self).map_err(|e| e.to_string()) + pub fn to_urlencoded(&self) -> Result { + serde_urlencoded::to_string(self) } pub fn to_bincode(&self) -> Result, String> { @@ -752,10 +752,10 @@ pub fn deflate_compress(i: Vec) -> Result, String> { e.finish().into_result().map_err(|e| e.to_string()) } -pub fn deflate_decompress(i: Vec) -> Result, String> { +pub fn deflate_decompress(i: Vec) -> Result, std::io::Error> { let mut decoder = Decoder::new(&i[..]); let mut out = Vec::new(); - decoder.read_to_end(&mut out).map_err(|e| format!("Failed to read from gzip decoder: {}", e))?; + decoder.read_to_end(&mut out)?; Ok(out) } From 8580a16df3bd8d85a6b8ff9a4200315903c15dd4 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:43:15 +0000 Subject: [PATCH 08/52] Migrate /style.css --- Cargo.lock | 170 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 4 +- src/main.rs | 48 ++++++++++++++- 3 files changed, 220 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62eb62f7..c47390bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,6 +118,60 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "axum" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +dependencies = [ + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -841,6 +895,19 @@ dependencies = [ "http 1.3.1", ] +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.10.1" @@ -890,8 +957,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", + "futures-channel", + "futures-util", "http 1.3.1", "http-body 1.0.1", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", "tokio", ] @@ -911,6 +985,22 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +dependencies = [ + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.6.0", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -1170,6 +1260,12 @@ version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.7.4" @@ -1314,6 +1410,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1439,6 +1555,7 @@ version = "0.35.1" dependencies = [ "arc-swap", "async-recursion", + "axum", "base2048", "base64 0.22.1", "bincode", @@ -1479,6 +1596,7 @@ dependencies = [ "time", "tokio", "toml", + "tower-default-headers", "url", "uuid", ] @@ -1903,6 +2021,16 @@ dependencies = [ "syn 2.0.99", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -2022,6 +2150,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.1" @@ -2234,6 +2368,41 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-default-headers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c0ee01216c770b930e0d0b23e7a7b8c30b61f4ae550dd9c8d543b488863c40d" +dependencies = [ + "futures-util", + "http 1.3.1", + "pin-project", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -2246,6 +2415,7 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ + "log", "pin-project-lite", "tracing-core", ] diff --git a/Cargo.toml b/Cargo.toml index d281219e..7c2a73d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ regex = "1.10.2" serde = { version = "1.0.193", features = ["derive"] } cookie = "0.18.0" futures-lite = "2.2.0" -hyper = { version = "0.14.32", features = ["stream", "backports", "deprecated"] } +hyper = { version = "0.14.32", features = ["stream", "backports"] } percent-encoding = "2.3.1" route-recognizer = "0.3.1" serde_json = "1.0.133" @@ -58,6 +58,8 @@ base2048 = "2.0.2" revision = "0.10.0" http-api-problem = { version = "0.60", features = ["hyper", "api-error"] } futures-util = "0.3.31" +axum= { version = "0.8" } +tower-default-headers = "0.2.0" [dev-dependencies] diff --git a/src/main.rs b/src/main.rs index 43b946a5..fe94553e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,6 +18,10 @@ use redlib::client::OAUTH_CLIENT; use futures_util::future::TryFutureExt; +use axum::{routing::get, routing::post, }; +use tower_default_headers::DefaultHeadersLayer; +use axum::http; +use axum::http::header::{HeaderMap, HeaderName, HeaderValue as HeaderValuex}; // Create Services // Required for the manifest to be valid @@ -108,6 +112,20 @@ async fn style() -> Result, String> { ) } +async fn stylex() -> impl axum::response::IntoResponse { + let mut res = include_str!("../static/style.css").to_string(); + for file in ThemeAssets::iter() { + res.push('\n'); + let theme = ThemeAssets::get(file.as_ref()).unwrap(); + res.push_str(std::str::from_utf8(theme.data.as_ref()).unwrap()); + } + let mut headers = axum::http::HeaderMap::new(); + headers.insert(axum::http::header::CONTENT_TYPE, axum::http::HeaderValue::from_static("text/css")); + headers.insert(axum::http::header::CACHE_CONTROL, axum::http::HeaderValue::from_static("public, max-age=1209600, s-maxage=86400")); + info!("stylex called"); + (headers, res) +} + #[tokio::main] async fn main() { // Load environment variables @@ -204,7 +222,7 @@ async fn main() { info!("Evaluating instance info."); Lazy::force(&instance_info::INSTANCE_INFO); info!("Creating OAUTH client."); - Lazy::force(&OAUTH_CLIENT); + Lazy::force(&OAUTH_CLIENT); // TODO: Redlib hangs when launched offline due to this. // Define default headers (added to all responses) app.default_headers = headers! { @@ -214,12 +232,30 @@ async fn main() { "Content-Security-Policy" => "default-src 'none'; font-src 'self'; script-src 'self' blob:; manifest-src 'self'; media-src 'self' data: blob: about:; style-src 'self' 'unsafe-inline'; base-uri 'none'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none'; connect-src 'self'; worker-src blob:;" }; + let mut default_headersx: HeaderMap = HeaderMap::from_iter([ + (axum::http::header::REFERRER_POLICY, HeaderValuex::from_static("no-referrer")), + (axum::http::header::X_CONTENT_TYPE_OPTIONS, HeaderValuex::from_static("nosniff")), + (axum::http::header::X_FRAME_OPTIONS, HeaderValuex::from_static("DENY")), + (axum::http::header::CONTENT_SECURITY_POLICY, HeaderValuex::from_static("default-src 'none'; font-src 'self'; script-src 'self' blob:; manifest-src 'self'; media-src 'self' data: blob: about:; style-src 'self' 'unsafe-inline'; base-uri 'none'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none'; connect-src 'self'; worker-src blob:;"), ) + ].into_iter()); + // http::header::HeaderMap::new(); + // default_headersx.insert(http::header::REFERRER_POLICY, http::header::HeaderValue::from_static("no-referrer")); + + + if let Some(expire_time) = hsts { if let Ok(val) = HeaderValue::from_str(&format!("max-age={expire_time}")) { app.default_headers.insert("Strict-Transport-Security", val); } } + //axum + if let Some(expire_time) = hsts { + if let Ok(val) = HeaderValuex::from_str(&format!("max-age={expire_time}")) { + default_headersx.insert(axum::http::header::STRICT_TRANSPORT_SECURITY, val); + } + } + // Read static files app.at("/style.css").get(|_| style().boxed()); app @@ -412,6 +448,16 @@ async fn main() { // Default service in case no routes match app.at("/*").get(|req| error(req, "Nothing here").boxed()); + let appx: axum::routing::Router<()> = axum::routing::Router::new() + .route("/style.css", get (stylex)) + .route("/", get(|| async { "hello, world!" })) + .layer(DefaultHeadersLayer::new(default_headersx)); + + + // Temporary listener for the axum server: + let listenerx = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + axum::serve(listenerx, appx).await.unwrap(); + println!("Running Redlib v{} on {listener}!", env!("CARGO_PKG_VERSION")); let server = app.listen(&listener); From 901123b8095ba8cf356b93550042ad335998a9ae Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 15:51:50 +0000 Subject: [PATCH 09/52] Migrate /manifest.json --- src/main.rs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main.rs b/src/main.rs index fe94553e..2c745234 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,10 +18,10 @@ use redlib::client::OAUTH_CLIENT; use futures_util::future::TryFutureExt; -use axum::{routing::get, routing::post, }; -use tower_default_headers::DefaultHeadersLayer; use axum::http; use axum::http::header::{HeaderMap, HeaderName, HeaderValue as HeaderValuex}; +use axum::{routing::get, routing::post}; +use tower_default_headers::DefaultHeadersLayer; // Create Services // Required for the manifest to be valid @@ -121,7 +121,10 @@ async fn stylex() -> impl axum::response::IntoResponse { } let mut headers = axum::http::HeaderMap::new(); headers.insert(axum::http::header::CONTENT_TYPE, axum::http::HeaderValue::from_static("text/css")); - headers.insert(axum::http::header::CACHE_CONTROL, axum::http::HeaderValue::from_static("public, max-age=1209600, s-maxage=86400")); + headers.insert( + axum::http::header::CACHE_CONTROL, + axum::http::HeaderValue::from_static("public, max-age=1209600, s-maxage=86400"), + ); info!("stylex called"); (headers, res) } @@ -222,7 +225,7 @@ async fn main() { info!("Evaluating instance info."); Lazy::force(&instance_info::INSTANCE_INFO); info!("Creating OAUTH client."); - Lazy::force(&OAUTH_CLIENT); // TODO: Redlib hangs when launched offline due to this. + Lazy::force(&OAUTH_CLIENT); // TODO: Redlib hangs when launched offline due to this. // Define default headers (added to all responses) app.default_headers = headers! { @@ -238,10 +241,6 @@ async fn main() { (axum::http::header::X_FRAME_OPTIONS, HeaderValuex::from_static("DENY")), (axum::http::header::CONTENT_SECURITY_POLICY, HeaderValuex::from_static("default-src 'none'; font-src 'self'; script-src 'self' blob:; manifest-src 'self'; media-src 'self' data: blob: about:; style-src 'self' 'unsafe-inline'; base-uri 'none'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none'; connect-src 'self'; worker-src blob:;"), ) ].into_iter()); - // http::header::HeaderMap::new(); - // default_headersx.insert(http::header::REFERRER_POLICY, http::header::HeaderValue::from_static("no-referrer")); - - if let Some(expire_time) = hsts { if let Ok(val) = HeaderValue::from_str(&format!("max-age={expire_time}")) { @@ -449,11 +448,20 @@ async fn main() { app.at("/*").get(|req| error(req, "Nothing here").boxed()); let appx: axum::routing::Router<()> = axum::routing::Router::new() - .route("/style.css", get (stylex)) + .route("/style.css", get(stylex)) + .route( + "/manifest.json", + get(( + [ + (axum::http::header::CONTENT_TYPE, "application/json"), + (axum::http::header::CACHE_CONTROL, "public, max-age=86400, s-maxage=3600"), + ], + include_bytes!("../static/manifest.json"), + )), + ) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); - // Temporary listener for the axum server: let listenerx = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); axum::serve(listenerx, appx).await.unwrap(); From 57592e2eca62b9fd33857981f05fbe39eab6f3fa Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 16:45:55 +0000 Subject: [PATCH 10/52] Migrate /robots.txt --- src/main.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/main.rs b/src/main.rs index 2c745234..5a39dd2c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -129,6 +129,22 @@ async fn stylex() -> impl axum::response::IntoResponse { (headers, res) } +async fn robots() -> impl axum::response::IntoResponse { + let headers = [ + (axum::http::header::CONTENT_TYPE, "text/plain"), + (axum::http::header::CACHE_CONTROL, "public, max-age=1209600, s-maxage=86400"), + ]; + let robots_txt = if match config::get_setting("REDLIB_ROBOTS_DISABLE_INDEXING") { + Some(val) => val == "on", + None => false, + } { + "User-agent: *\nDisallow: /" + } else { + "User-agent: *\nDisallow: /u/\nDisallow: /user/" + }; + (headers, robots_txt) +} + #[tokio::main] async fn main() { // Load environment variables @@ -459,6 +475,7 @@ async fn main() { include_bytes!("../static/manifest.json"), )), ) + .route("/robots.txt", get(robots)) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); From 84bf573c03abed9ac7d955533bd6d5bb1baa2d4b Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 16:51:45 +0000 Subject: [PATCH 11/52] Migrate /favicon.ico --- src/main.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main.rs b/src/main.rs index 5a39dd2c..9f42c8b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -57,6 +57,15 @@ async fn favicon() -> Result, String> { ) } +async fn faviconx() -> impl axum::response::IntoResponse { + let headers = [ + (axum::http::header::CONTENT_TYPE, "image/vnd.microsoft.icon"), + (axum::http::header::CACHE_CONTROL, "public, max-age=1209600, s-maxage=86400"), + ]; + let favicon = include_bytes!("../static/favicon.ico"); + (headers, favicon) +} + async fn font() -> Result, String> { Ok( Response::builder() @@ -476,6 +485,7 @@ async fn main() { )), ) .route("/robots.txt", get(robots)) + .route("/favicon.ico", get(faviconx)) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); From bb152823f2242a30a8e566b6ea8c660cc7ec6371 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 16:54:47 +0000 Subject: [PATCH 12/52] Migrate /logo.png --- src/main.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main.rs b/src/main.rs index 9f42c8b3..785396d1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,6 +35,15 @@ async fn pwa_logo() -> Result, String> { ) } +async fn pwa_logox() -> impl axum::response::IntoResponse { + let headers = [ + (axum::http::header::CONTENT_TYPE, "image/png"), + (axum::http::header::CACHE_CONTROL, "public, max-age=1209600, s-maxage=86400"), + ]; + let logo = include_bytes!("../static/logo.png"); + (headers, logo) +} + // Required for iOS App Icons async fn iphone_logo() -> Result, String> { Ok( @@ -486,6 +495,7 @@ async fn main() { ) .route("/robots.txt", get(robots)) .route("/favicon.ico", get(faviconx)) + .route("/logo.png", get(pwa_logox)) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); From 77e7e07d1fff857f62ed5ce299a7375e8b7c8de3 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 17:42:44 +0000 Subject: [PATCH 13/52] Create helper macro cached_static_resource --- src/main.rs | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/main.rs b/src/main.rs index 785396d1..5d18d893 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,14 +35,25 @@ async fn pwa_logo() -> Result, String> { ) } -async fn pwa_logox() -> impl axum::response::IntoResponse { +macro_rules! cached_static_resource { + ($path:expr, $content_type:expr) => { + ( + [ + (axum::http::header::CONTENT_TYPE, $content_type), + (axum::http::header::CACHE_CONTROL, "public, max-age=1209600, s-maxage=86400"), + ], + include_bytes!($path), + ) + }; +} +/*async fn static_resource(path: &'static str, content_type: axum::http::header::HeaderValue) -> impl axum::response::IntoResponse { let headers = [ - (axum::http::header::CONTENT_TYPE, "image/png"), - (axum::http::header::CACHE_CONTROL, "public, max-age=1209600, s-maxage=86400"), + (axum::http::header::CONTENT_TYPE, content_type), + (axum::http::header::CACHE_CONTROL, axum::http::header::HeaderValue::from_static("public, max-age=1209600, s-maxage=86400")), ]; - let logo = include_bytes!("../static/logo.png"); - (headers, logo) -} + let image = include_bytes!(path); + (headers, image) +}*/ // Required for iOS App Icons async fn iphone_logo() -> Result, String> { @@ -66,14 +77,6 @@ async fn favicon() -> Result, String> { ) } -async fn faviconx() -> impl axum::response::IntoResponse { - let headers = [ - (axum::http::header::CONTENT_TYPE, "image/vnd.microsoft.icon"), - (axum::http::header::CACHE_CONTROL, "public, max-age=1209600, s-maxage=86400"), - ]; - let favicon = include_bytes!("../static/favicon.ico"); - (headers, favicon) -} async fn font() -> Result, String> { Ok( @@ -483,19 +486,11 @@ async fn main() { let appx: axum::routing::Router<()> = axum::routing::Router::new() .route("/style.css", get(stylex)) - .route( - "/manifest.json", - get(( - [ - (axum::http::header::CONTENT_TYPE, "application/json"), - (axum::http::header::CACHE_CONTROL, "public, max-age=86400, s-maxage=3600"), - ], - include_bytes!("../static/manifest.json"), - )), - ) + .route("/manifest.json", get(cached_static_resource!("../static/manifest.json", "application/json"))) .route("/robots.txt", get(robots)) - .route("/favicon.ico", get(faviconx)) - .route("/logo.png", get(pwa_logox)) + .route("/favicon.ico", get(cached_static_resource!("../static/favicon.ico", "image/vnd.microsoft.icon"))) + .route("/logo.png", get(cached_static_resource!("../static/logo.png", "image/png"))) + .route("/Inter.var.woff2", get(cached_static_resource!("../static/Inter.var.woff2", "font/woff2"))) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); From b0eedbafb7c76981f595293d5fe405a1b61a3d52 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 17:57:06 +0000 Subject: [PATCH 14/52] Migrate multiple static resources --- src/main.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 5d18d893..3a2fcbd5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,7 +77,6 @@ async fn favicon() -> Result, String> { ) } - async fn font() -> Result, String> { Ok( Response::builder() @@ -491,6 +490,17 @@ async fn main() { .route("/favicon.ico", get(cached_static_resource!("../static/favicon.ico", "image/vnd.microsoft.icon"))) .route("/logo.png", get(cached_static_resource!("../static/logo.png", "image/png"))) .route("/Inter.var.woff2", get(cached_static_resource!("../static/Inter.var.woff2", "font/woff2"))) + .route("/touch-icon-iphone.png", get(cached_static_resource!("../static/apple-touch-icon.png", "image/png"))) + .route("/apple-touch-icon.png", get(cached_static_resource!("../static/apple-touch-icon.png", "image/png"))) + .route( + "/opensearch.xml", + get(cached_static_resource!("../static/opensearch.xml", "application/opensearchdescription+xml")), + ) + .route("/playHLSVideo.js", get(cached_static_resource!("../static/playHLSVideo.js", "text/javascript"))) + .route("/hls.min.js", get(cached_static_resource!("../static/hls.min.js", "text/javascript"))) + .route("/highlighted.js", get(cached_static_resource!("../static/highlighted.js", "text/javascript"))) + .route("/check_update.js", get(cached_static_resource!("../static/check_update.js", "text/javascript"))) + .route("/copy.js", get(cached_static_resource!("../static/copy.js", "text/javascript"))) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); From 7cdfe90100e051921e8ae1b30ab1cb26f42bbe65 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 21:33:55 +0000 Subject: [PATCH 15/52] Use reqwest HTTP client; Migrate /commits.atom --- Cargo.lock | 473 +++++++++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 3 +- src/client.rs | 30 ++++ src/main.rs | 20 ++- 4 files changed, 503 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c47390bc..9a822d3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,6 +112,12 @@ dependencies = [ "quick-xml", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.4.0" @@ -184,7 +190,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -623,6 +629,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -717,9 +738,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -755,7 +778,7 @@ dependencies = [ "cfg-if", "libc", "wasi 0.13.3+wasi-0.2.2", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -796,6 +819,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.3.1", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -858,9 +900,9 @@ version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "000bed434eb9b5dfb8ea5ed904f02ffc63aa0ac7ac034d480dc24fbb97b748d8" dependencies = [ + "axum-core", "http 1.3.1", "http-api-problem-derive", - "hyper 1.6.0", "serde", "serde_json", ] @@ -936,7 +978,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -959,6 +1001,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2 0.4.8", "http 1.3.1", "http-body 1.0.1", "httparse", @@ -967,6 +1010,7 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", ] [[package]] @@ -979,10 +1023,43 @@ dependencies = [ "http 0.2.12", "hyper 0.14.32", "log", - "rustls", + "rustls 0.21.12", "rustls-native-certs", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +dependencies = [ + "futures-util", + "http 1.3.1", + "hyper 1.6.0", + "hyper-util", + "rustls 0.23.25", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.2", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", ] [[package]] @@ -992,13 +1069,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", + "futures-channel", "futures-util", "http 1.3.1", "http-body 1.0.1", "hyper 1.6.0", "pin-project-lite", + "socket2", "tokio", "tower-service", + "tracing", ] [[package]] @@ -1165,6 +1245,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + [[package]] name = "is-terminal" version = "0.4.16" @@ -1314,6 +1400,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "never" version = "0.1.0" @@ -1369,12 +1472,50 @@ version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +[[package]] +name = "openssl" +version = "0.10.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", +] + [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-sys" +version = "0.9.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking" version = "2.2.1" @@ -1401,7 +1542,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1442,6 +1583,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "powerfmt" version = "0.2.0" @@ -1572,7 +1719,7 @@ dependencies = [ "htmlescape", "http-api-problem", "hyper 0.14.32", - "hyper-rustls", + "hyper-rustls 0.24.2", "libflate", "lipsum", "log", @@ -1581,6 +1728,7 @@ dependencies = [ "pretty_env_logger", "pulldown-cmark", "regex", + "reqwest", "revision", "rinja", "route-recognizer", @@ -1639,6 +1787,52 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "reqwest" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.8", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-rustls 0.27.5", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 2.2.0", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "windows-registry", +] + [[package]] name = "revision" version = "0.10.0" @@ -1801,10 +1995,23 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.103.0", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" version = "0.6.3" @@ -1812,7 +2019,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "schannel", "security-framework", ] @@ -1826,6 +2033,21 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -1836,6 +2058,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.20" @@ -2128,6 +2361,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "1.0.109" @@ -2155,6 +2394,9 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -2167,6 +2409,27 @@ dependencies = [ "syn 2.0.99", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tegen" version = "0.1.4" @@ -2311,13 +2574,33 @@ dependencies = [ "syn 2.0.99", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls 0.23.25", "tokio", ] @@ -2497,6 +2780,12 @@ dependencies = [ "getrandom 0.3.1", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2554,6 +2843,7 @@ checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] @@ -2571,6 +2861,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.100" @@ -2603,6 +2906,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" @@ -2622,13 +2948,48 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-registry" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.53.0", +] + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -2637,7 +2998,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -2646,14 +3007,30 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -2662,48 +3039,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.7.3" @@ -2800,6 +3225,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + [[package]] name = "zerovec" version = "0.10.4" diff --git a/Cargo.toml b/Cargo.toml index 7c2a73d3..e25eb6cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,10 +56,11 @@ htmlescape = "0.3.1" bincode = "1.3.3" base2048 = "2.0.2" revision = "0.10.0" -http-api-problem = { version = "0.60", features = ["hyper", "api-error"] } +http-api-problem = { version = "0.60", features = ["axum", "api-error"] } futures-util = "0.3.31" axum= { version = "0.8" } tower-default-headers = "0.2.0" +reqwest = { version = "0.12.15", features = ["stream"] } [dev-dependencies] diff --git a/src/client.rs b/src/client.rs index 76369cad..a95227b0 100644 --- a/src/client.rs +++ b/src/client.rs @@ -12,6 +12,8 @@ use once_cell::sync::Lazy; use percent_encoding::{percent_encode, CONTROLS}; use serde_json::Value; +use http_api_problem::ApiError; +use reqwest; use std::sync::atomic::Ordering; use std::sync::atomic::{AtomicBool, AtomicU16}; use std::{io, result::Result}; @@ -34,6 +36,34 @@ pub static HTTPS_CONNECTOR: Lazy> = Lazy::new(|| hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http2().build()); pub static CLIENT: Lazy>> = Lazy::new(|| Client::builder().build::<_, Body>(HTTPS_CONNECTOR.clone())); +pub static CLIENTX: Lazy = Lazy::new(|| reqwest::Client::new()); + +pub fn into_api_error(e: reqwest::Error) -> ApiError { + if e.is_timeout() || e.is_connect() { + ApiError::builder(http_api_problem::StatusCode::GATEWAY_TIMEOUT) // 504 + .title("Gateway Timeout") + .message(format!("{e}")) + .source(e) + .finish() + } else if e.is_decode() { + ApiError::builder(http_api_problem::StatusCode::BAD_GATEWAY) // 502 + .title("Bad Gateway") + .message(format!("{e}")) + .source(e) + .finish() + } else if e.is_status() && e.status().is_some() { + let status = e.status().unwrap(); + ApiError::try_builder(status.as_u16()) + .expect("reqwest considers this HTTP status to be an error status, but http_api_problem does not.") + .finish() + } else { + ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) // 500 + .title("Internal Server Error") + .message(format!("{e}")) + .source(e) + .finish() + } +} pub static OAUTH_CLIENT: Lazy> = Lazy::new(|| { let client = block_on(Oauth::new()); diff --git a/src/main.rs b/src/main.rs index 3a2fcbd5..b0c32f85 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,11 +5,12 @@ use cached::proc_macro::cached; use clap::{Arg, ArgAction, Command}; use futures_lite::FutureExt; +use http_api_problem::ApiError; use hyper::Uri; use hyper::{body::Bytes, header::HeaderValue, Body, Request, Response}; use log::{info, warn}; use once_cell::sync::Lazy; -use redlib::client::{canonical_path, proxy, rate_limit_check, CLIENT}; +use redlib::client::{canonical_path, into_api_error, proxy, rate_limit_check, CLIENT, CLIENTX}; use redlib::server::{self, RequestExt}; use redlib::utils::{error, redirect, ThemeAssets}; use redlib::{config, duplicates, headers, instance_info, post, search, settings, subreddit, user}; @@ -501,6 +502,7 @@ async fn main() { .route("/highlighted.js", get(cached_static_resource!("../static/highlighted.js", "text/javascript"))) .route("/check_update.js", get(cached_static_resource!("../static/check_update.js", "text/javascript"))) .route("/copy.js", get(cached_static_resource!("../static/copy.js", "text/javascript"))) + .route("/commits.atom", get(proxy_commit_infox)) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); @@ -527,6 +529,18 @@ pub async fn proxy_commit_info() -> Result, hyper::Error> { .unwrap_or_default(), ) } +#[cached(time = 600, result = true, result_fallback = true)] +async fn proxy_commit_infox() -> Result<([(axum::http::header::HeaderName, &'static str); 2], Bytes), ApiError> { + let response = fetch_reqwest("https://github.com/redlib-org/redlib/commits/main.atom").await.map_err(&into_api_error)?; + // Note: Want to use Result::and_then(...), but that method does not allow async, so can't call await inside. + // Hence forced to call `into_api_error` twice. + let data = response.bytes().await.map_err(&into_api_error)?; + let headers = [ + (axum::http::header::CONTENT_TYPE, "application/atom+xml"), + (axum::http::header::CACHE_CONTROL, "public, max-age=86400, s-maxage=600"), + ]; + Ok((headers, data)) +} #[cached(time = 600, result = true, result_fallback = true)] async fn fetch_commit_info() -> Result { @@ -545,6 +559,10 @@ pub async fn proxy_instances() -> Result, hyper::Error> { ) } +async fn fetch_reqwest(url: &str) -> Result { + CLIENTX.get(url).send().await?.error_for_status() // Forward HTTP errors as an error +} + #[cached(time = 600, result = true, result_fallback = true)] async fn fetch_instances() -> Result { let uri = Uri::from_static("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json"); From d342c78a7001043501e3a4a1b876d441f0e4624b Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 22:32:06 +0000 Subject: [PATCH 16/52] Migrate /instances.json --- src/main.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index b0c32f85..ef4ef3f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -503,6 +503,7 @@ async fn main() { .route("/check_update.js", get(cached_static_resource!("../static/check_update.js", "text/javascript"))) .route("/copy.js", get(cached_static_resource!("../static/copy.js", "text/javascript"))) .route("/commits.atom", get(proxy_commit_infox)) + .route("/instances.json", get(proxy_instancesx)) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); @@ -530,7 +531,7 @@ pub async fn proxy_commit_info() -> Result, hyper::Error> { ) } #[cached(time = 600, result = true, result_fallback = true)] -async fn proxy_commit_infox() -> Result<([(axum::http::header::HeaderName, &'static str); 2], Bytes), ApiError> { +pub async fn proxy_commit_infox() -> Result<([(axum::http::header::HeaderName, &'static str); 2], Bytes), ApiError> { let response = fetch_reqwest("https://github.com/redlib-org/redlib/commits/main.atom").await.map_err(&into_api_error)?; // Note: Want to use Result::and_then(...), but that method does not allow async, so can't call await inside. // Hence forced to call `into_api_error` twice. @@ -559,6 +560,21 @@ pub async fn proxy_instances() -> Result, hyper::Error> { ) } +#[cached(time = 600, result = true, result_fallback = true)] +pub async fn proxy_instancesx() -> Result<([(axum::http::header::HeaderName, &'static str); 2], Bytes), ApiError> { + let response = fetch_reqwest("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json") + .await + .map_err(&into_api_error)?; + // NOTE: Want to use Result::and_then(...), but that method does not allow async, so can't call .await inside. + // Hence forced to call `into_api_error` twice. + let data = response.bytes().await.map_err(&into_api_error)?; + let headers = [ + (axum::http::header::CONTENT_TYPE, "application/json"), + (axum::http::header::CACHE_CONTROL, "public, max-age=86400, s-maxage=600"), + ]; + Ok((headers, data)) +} + async fn fetch_reqwest(url: &str) -> Result { CLIENTX.get(url).send().await?.error_for_status() // Forward HTTP errors as an error } From d709c62e1b8ad8a4f8e383d9d203784b369e407e Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Wed, 19 Mar 2025 23:23:59 +0000 Subject: [PATCH 17/52] Add a proxy helper function --- src/client.rs | 15 ++++++++++++++- src/main.rs | 18 ++++++++++-------- src/utils.rs | 5 +++++ 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/client.rs b/src/client.rs index a95227b0..e3274756 100644 --- a/src/client.rs +++ b/src/client.rs @@ -21,7 +21,7 @@ use std::{io, result::Result}; use crate::dbg_msg; use crate::oauth::{force_refresh_token, token_daemon, Oauth}; use crate::server::RequestExt; -use crate::utils::{format_url, Post}; +use crate::utils::{fetch_reqwest, format_url, Post}; const REDDIT_URL_BASE: &str = "https://oauth.reddit.com"; const REDDIT_URL_BASE_HOST: &str = "oauth.reddit.com"; @@ -179,6 +179,19 @@ pub async fn proxy(req: Request, format: &str) -> Result, S stream(&url, &req).await } +pub async fn proxyx(url: impl reqwest::IntoUrl) -> impl axum::response::IntoResponse { + let response: reqwest::Response = fetch_reqwest(url).await.map_err(&into_api_error)?; + let mut response_builder = axum::http::Response::builder().status(response.status()); // See https://github.com/tokio-rs/axum/blob/main/examples/reqwest-response/src/main.rs#L62C5-L67C18 + *response_builder.headers_mut().unwrap() = response.headers().clone(); + Ok::<_, ApiError>(response_builder.body(axum::body::Body::from_stream(response.bytes_stream())).map_err(|e| { + ApiError::builder(http_api_problem::StatusCode::BAD_GATEWAY) //502 + .title("Bad Gateway") + .message(format!("Could not forward response: {e}")) + .source(e) + .finish() + })?) +} + async fn stream(url: &str, req: &Request) -> Result, String> { // First parameter is target URL (mandatory). let parsed_uri = url.parse::().map_err(|_| "Couldn't parse URL".to_string())?; diff --git a/src/main.rs b/src/main.rs index ef4ef3f1..9ef378a1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,10 +10,10 @@ use hyper::Uri; use hyper::{body::Bytes, header::HeaderValue, Body, Request, Response}; use log::{info, warn}; use once_cell::sync::Lazy; -use redlib::client::{canonical_path, into_api_error, proxy, rate_limit_check, CLIENT, CLIENTX}; +use redlib::client::{canonical_path, into_api_error, proxy, proxyx, rate_limit_check, CLIENT}; use redlib::server::{self, RequestExt}; use redlib::utils::{error, redirect, ThemeAssets}; -use redlib::{config, duplicates, headers, instance_info, post, search, settings, subreddit, user}; +use redlib::{config, duplicates, headers, instance_info, post, search, settings, subreddit, user, utils}; use redlib::client::OAUTH_CLIENT; @@ -504,6 +504,10 @@ async fn main() { .route("/copy.js", get(cached_static_resource!("../static/copy.js", "text/javascript"))) .route("/commits.atom", get(proxy_commit_infox)) .route("/instances.json", get(proxy_instancesx)) + .route( + "/vid/{id}/{size}", + get(|axum::extract::Path((id, size)): axum::extract::Path<(String, String)>| proxyx(format!("https://v.redd.it/{id}/DASH_{size}"))), + ) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); @@ -532,7 +536,9 @@ pub async fn proxy_commit_info() -> Result, hyper::Error> { } #[cached(time = 600, result = true, result_fallback = true)] pub async fn proxy_commit_infox() -> Result<([(axum::http::header::HeaderName, &'static str); 2], Bytes), ApiError> { - let response = fetch_reqwest("https://github.com/redlib-org/redlib/commits/main.atom").await.map_err(&into_api_error)?; + let response = utils::fetch_reqwest("https://github.com/redlib-org/redlib/commits/main.atom") + .await + .map_err(&into_api_error)?; // Note: Want to use Result::and_then(...), but that method does not allow async, so can't call await inside. // Hence forced to call `into_api_error` twice. let data = response.bytes().await.map_err(&into_api_error)?; @@ -562,7 +568,7 @@ pub async fn proxy_instances() -> Result, hyper::Error> { #[cached(time = 600, result = true, result_fallback = true)] pub async fn proxy_instancesx() -> Result<([(axum::http::header::HeaderName, &'static str); 2], Bytes), ApiError> { - let response = fetch_reqwest("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json") + let response = utils::fetch_reqwest("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json") .await .map_err(&into_api_error)?; // NOTE: Want to use Result::and_then(...), but that method does not allow async, so can't call .await inside. @@ -575,10 +581,6 @@ pub async fn proxy_instancesx() -> Result<([(axum::http::header::HeaderName, &'s Ok((headers, data)) } -async fn fetch_reqwest(url: &str) -> Result { - CLIENTX.get(url).send().await?.error_for_status() // Forward HTTP errors as an error -} - #[cached(time = 600, result = true, result_fallback = true)] async fn fetch_instances() -> Result { let uri = Uri::from_static("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json"); diff --git a/src/utils.rs b/src/utils.rs index 22a49391..c59c6fae 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -5,6 +5,7 @@ use crate::config::{self, get_setting}; // // CRATES // +use crate::client::CLIENTX; use crate::{client::json, server::RequestExt}; use cookie::Cookie; use hyper::{Body, Request, Response}; @@ -1692,3 +1693,7 @@ fn test_round_trip(input: &Preferences, compression: bool) { let deserialized: Preferences = bincode::deserialize(&decompressed).unwrap(); assert_eq!(*input, deserialized); } + +pub async fn fetch_reqwest(url: impl reqwest::IntoUrl) -> Result { + CLIENTX.get(url).send().await?.error_for_status() // Forward HTTP errors as an error +} From 083841eb557ef12fce9cb9b98880b48bf275c544 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Thu, 20 Mar 2025 02:43:13 +0000 Subject: [PATCH 18/52] Add Path extractor to proxy function --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/client.rs | 22 +++++++++++++++------- src/main.rs | 6 +----- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a822d3e..042dd635 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1740,6 +1740,7 @@ dependencies = [ "serde_json_path", "serde_urlencoded", "serde_yaml", + "strfmt", "tegen", "time", "tokio", @@ -2355,6 +2356,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strfmt" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a8348af2d9fc3258c8733b8d9d8db2e56f54b2363a4b5b81585c7875ed65e65" + [[package]] name = "strsim" version = "0.11.1" diff --git a/Cargo.toml b/Cargo.toml index e25eb6cf..fc0d2604 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ futures-util = "0.3.31" axum= { version = "0.8" } tower-default-headers = "0.2.0" reqwest = { version = "0.12.15", features = ["stream"] } +strfmt = "0.2.4" [dev-dependencies] diff --git a/src/client.rs b/src/client.rs index e3274756..fb1f5094 100644 --- a/src/client.rs +++ b/src/client.rs @@ -12,16 +12,16 @@ use once_cell::sync::Lazy; use percent_encoding::{percent_encode, CONTROLS}; use serde_json::Value; +use crate::dbg_msg; +use crate::oauth::{force_refresh_token, token_daemon, Oauth}; +use crate::server::RequestExt; +use crate::utils::{fetch_reqwest, format_url, Post}; use http_api_problem::ApiError; use reqwest; use std::sync::atomic::Ordering; use std::sync::atomic::{AtomicBool, AtomicU16}; use std::{io, result::Result}; - -use crate::dbg_msg; -use crate::oauth::{force_refresh_token, token_daemon, Oauth}; -use crate::server::RequestExt; -use crate::utils::{fetch_reqwest, format_url, Post}; +use strfmt; const REDDIT_URL_BASE: &str = "https://oauth.reddit.com"; const REDDIT_URL_BASE_HOST: &str = "oauth.reddit.com"; @@ -179,9 +179,17 @@ pub async fn proxy(req: Request, format: &str) -> Result, S stream(&url, &req).await } -pub async fn proxyx(url: impl reqwest::IntoUrl) -> impl axum::response::IntoResponse { +pub async fn proxyx(axum::extract::Path(parameters): axum::extract::Path>, fmtstr: &str) -> impl axum::response::IntoResponse { + let url = strfmt::strfmt(fmtstr, ¶meters).map_err(|e| { + // Should fail only if the passed fmtstr parameter if formatted wrong. See fmtstr docs. + ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) + .title("Internal Server Error") + .message("Could not rewrite url: {e}") + .source(e) + .finish() + })?; let response: reqwest::Response = fetch_reqwest(url).await.map_err(&into_api_error)?; - let mut response_builder = axum::http::Response::builder().status(response.status()); // See https://github.com/tokio-rs/axum/blob/main/examples/reqwest-response/src/main.rs#L62C5-L67C18 + let mut response_builder = axum::http::Response::builder().status(response.status()); // See https://github.com/tokio-rs/axum/blob/15917c6dbcb4a48707a20e9cfd021992a279a662/examples/reqwest-response/src/main.rs#L62-L67 *response_builder.headers_mut().unwrap() = response.headers().clone(); Ok::<_, ApiError>(response_builder.body(axum::body::Body::from_stream(response.bytes_stream())).map_err(|e| { ApiError::builder(http_api_problem::StatusCode::BAD_GATEWAY) //502 diff --git a/src/main.rs b/src/main.rs index 9ef378a1..6c6890f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -504,10 +504,7 @@ async fn main() { .route("/copy.js", get(cached_static_resource!("../static/copy.js", "text/javascript"))) .route("/commits.atom", get(proxy_commit_infox)) .route("/instances.json", get(proxy_instancesx)) - .route( - "/vid/{id}/{size}", - get(|axum::extract::Path((id, size)): axum::extract::Path<(String, String)>| proxyx(format!("https://v.redd.it/{id}/DASH_{size}"))), - ) + .route("/vid/{id}/{size}", get(|path: axum::extract::Path<_>| proxyx(path, "https://v.redd.it/{id}/DASH_{size}"))) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); @@ -524,7 +521,6 @@ async fn main() { eprintln!("Server error: {e}"); } } - pub async fn proxy_commit_info() -> Result, hyper::Error> { Ok( Response::builder() From 8c42fdb6ccf1a51c63205b52bcab610eada35393 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Thu, 20 Mar 2025 02:44:09 +0000 Subject: [PATCH 19/52] Cargo fix --- src/main.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 6c6890f1..b9692047 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,9 +19,8 @@ use redlib::client::OAUTH_CLIENT; use futures_util::future::TryFutureExt; -use axum::http; -use axum::http::header::{HeaderMap, HeaderName, HeaderValue as HeaderValuex}; -use axum::{routing::get, routing::post}; +use axum::http::header::{HeaderMap, HeaderValue as HeaderValuex}; +use axum::routing::get; use tower_default_headers::DefaultHeadersLayer; // Create Services From 2202218ef5ecf3aa31c465478f4eff8c1392b641 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Thu, 20 Mar 2025 14:23:37 +0000 Subject: [PATCH 20/52] Migrate proxy routes Note: current implementation is broken, since the url query isn't forwarded. --- src/main.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main.rs b/src/main.rs index b9692047..db37b624 100644 --- a/src/main.rs +++ b/src/main.rs @@ -483,7 +483,15 @@ async fn main() { // Default service in case no routes match app.at("/*").get(|req| error(req, "Nothing here").boxed()); + // Helper macro to create a direct reverse proxy with formatted outcome + macro_rules! proxy { + ($fmtstr:expr) => { + |path: axum::extract::Path<_>| proxyx(path, $fmtstr) + }; + } + let appx: axum::routing::Router<()> = axum::routing::Router::new() + // Static resources .route("/style.css", get(stylex)) .route("/manifest.json", get(cached_static_resource!("../static/manifest.json", "application/json"))) .route("/robots.txt", get(robots)) @@ -503,7 +511,23 @@ async fn main() { .route("/copy.js", get(cached_static_resource!("../static/copy.js", "text/javascript"))) .route("/commits.atom", get(proxy_commit_infox)) .route("/instances.json", get(proxy_instancesx)) + // Direct proxies .route("/vid/{id}/{size}", get(|path: axum::extract::Path<_>| proxyx(path, "https://v.redd.it/{id}/DASH_{size}"))) + .route("/hls/{id}/{*path}", get(proxy!("https://v.redd.it/{id}/{path}"))) + .route("/img/{*path}", get(proxy!("https://i.redd.it/{path}"))) + .route("/thumb/{point}/{id}", get(proxy!("https://{point}.thumbs.redditmedia.com/{id}"))) + .route("/emoji/{id}/{name}", get(proxy!("https://emoji.redditmedia.com/{id}/{name}"))) + .route( + "/emote/{subreddit_id}/{filename}", + get(proxy!("https://reddit-econ-prod-assets-permanent.s3.amazonaws.com/asset-manager/{subreddit_id}/{filename}")), + ) + .route( + "/preview/{loc}/award_images/{fullname}/{id}", + get(proxy!("https://{loc}view.redd.it/award_images/{fullname}/{id}")), + ) + .route("/preview/{loc}/{id}", get(proxy!("https://{loc}view.redd.it/{id}"))) + .route("/style/{*path}", get(proxy!("https://styles.redditmedia.com/{path}"))) + .route("/static/{*path}", get(proxy!("https://www.redditstatic.com/{path}"))) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); From f8059cc4d6f0cad41916ba176cefd1005cbba5b2 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Thu, 20 Mar 2025 14:45:02 +0000 Subject: [PATCH 21/52] axum::Router::remove_v07_checks() --- src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.rs b/src/main.rs index db37b624..4dfd9831 100644 --- a/src/main.rs +++ b/src/main.rs @@ -491,6 +491,7 @@ async fn main() { } let appx: axum::routing::Router<()> = axum::routing::Router::new() + .without_v07_checks() // Remove unnecessary backward compatibility // Static resources .route("/style.css", get(stylex)) .route("/manifest.json", get(cached_static_resource!("../static/manifest.json", "application/json"))) From ff2165da31d10508dead183d5533353c8003183c Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Thu, 20 Mar 2025 17:33:09 +0000 Subject: [PATCH 22/52] Reimplement proxy function, fix bugs --- src/client.rs | 45 +++++++++++++++++++++++++++++---------------- src/main.rs | 10 +++++++--- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/client.rs b/src/client.rs index fb1f5094..e20340e1 100644 --- a/src/client.rs +++ b/src/client.rs @@ -179,25 +179,38 @@ pub async fn proxy(req: Request, format: &str) -> Result, S stream(&url, &req).await } -pub async fn proxyx(axum::extract::Path(parameters): axum::extract::Path>, fmtstr: &str) -> impl axum::response::IntoResponse { - let url = strfmt::strfmt(fmtstr, ¶meters).map_err(|e| { - // Should fail only if the passed fmtstr parameter if formatted wrong. See fmtstr docs. - ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) - .title("Internal Server Error") - .message("Could not rewrite url: {e}") +pub async fn proxyx( + axum::extract::Path(parameters): axum::extract::Path>, + mut req: axum::extract::Request, + fmtstr: &str, +) -> impl axum::response::IntoResponse { + // Format URI from fmtstr, then append any queries from the request. + let uri = format!( + "{}?{}", + strfmt::strfmt(fmtstr, ¶meters) // Format given uri, then append any queries + .map_err(|e| { + // Should fail only if the passed fmtstr parameter if formatted wrong. See fmtstr docs. + ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) + .title("Internal Server Error") + .message(format!("Could not rewrite url: {e}")) + .source(e) + .finish() + })?, + req.uri().query().unwrap_or_default() + ); + log::debug!("Forwarding {} request: {} to {}", req.method(), req.uri(), uri); + // Change req URI + *req.uri_mut() = axum::http::Uri::try_from(uri).map_err(|e| { + ApiError::builder(http_api_problem::StatusCode::BAD_REQUEST) // 400 + .title("Bad Request") + .message(format!("Could not read uri: {e}")) .source(e) .finish() })?; - let response: reqwest::Response = fetch_reqwest(url).await.map_err(&into_api_error)?; - let mut response_builder = axum::http::Response::builder().status(response.status()); // See https://github.com/tokio-rs/axum/blob/15917c6dbcb4a48707a20e9cfd021992a279a662/examples/reqwest-response/src/main.rs#L62-L67 - *response_builder.headers_mut().unwrap() = response.headers().clone(); - Ok::<_, ApiError>(response_builder.body(axum::body::Body::from_stream(response.bytes_stream())).map_err(|e| { - ApiError::builder(http_api_problem::StatusCode::BAD_GATEWAY) //502 - .title("Bad Gateway") - .message(format!("Could not forward response: {e}")) - .source(e) - .finish() - })?) + + let req: reqwest::Request = req.map(|body| reqwest::Body::wrap_stream(body.into_data_stream())).try_into().map_err(&into_api_error)?; + let response: reqwest::Response = CLIENTX.execute(req).await.and_then(reqwest::Response::error_for_status).map_err(&into_api_error)?; + Ok::, ApiError>(response.into()) } async fn stream(url: &str, req: &Request) -> Result, String> { diff --git a/src/main.rs b/src/main.rs index 4dfd9831..c300784c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -483,10 +483,14 @@ async fn main() { // Default service in case no routes match app.at("/*").get(|req| error(req, "Nothing here").boxed()); - // Helper macro to create a direct reverse proxy with formatted outcome + /// Helper macro to create a direct reverse proxy with formatted outcome + /// Effectively, we want to curry the proxy function. This is achieved manually using this macro. + // There are two other "currying" crates: + // `currying`, which unfortunately depends on unstable features + // `auto_curry`, which uses procedural macros, and isn't very featureful macro_rules! proxy { ($fmtstr:expr) => { - |path: axum::extract::Path<_>| proxyx(path, $fmtstr) + |parameters: axum::extract::Path>, req: axum::extract::Request| proxyx(parameters, req, $fmtstr) }; } @@ -513,7 +517,7 @@ async fn main() { .route("/commits.atom", get(proxy_commit_infox)) .route("/instances.json", get(proxy_instancesx)) // Direct proxies - .route("/vid/{id}/{size}", get(|path: axum::extract::Path<_>| proxyx(path, "https://v.redd.it/{id}/DASH_{size}"))) + .route("/vid/{id}/{size}", get(proxy!("https://v.redd.it/{id}/DASH_{size}"))) .route("/hls/{id}/{*path}", get(proxy!("https://v.redd.it/{id}/{path}"))) .route("/img/{*path}", get(proxy!("https://i.redd.it/{path}"))) .route("/thumb/{point}/{id}", get(proxy!("https://{point}.thumbs.redditmedia.com/{id}"))) From 0d868b74721d3ffb606c92f0ef7dc521bac54656 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Thu, 20 Mar 2025 18:20:34 +0000 Subject: [PATCH 23/52] Modify headers at proxy (Tested working!) --- src/client.rs | 37 ++++++++++++++++++++++++++++++++++--- src/main.rs | 4 ++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/client.rs b/src/client.rs index e20340e1..ab8d643c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -15,7 +15,7 @@ use serde_json::Value; use crate::dbg_msg; use crate::oauth::{force_refresh_token, token_daemon, Oauth}; use crate::server::RequestExt; -use crate::utils::{fetch_reqwest, format_url, Post}; +use crate::utils::{format_url, Post}; use http_api_problem::ApiError; use reqwest; use std::sync::atomic::Ordering; @@ -179,7 +179,7 @@ pub async fn proxy(req: Request, format: &str) -> Result, S stream(&url, &req).await } -pub async fn proxyx( +pub async fn proxy_get( axum::extract::Path(parameters): axum::extract::Path>, mut req: axum::extract::Request, fmtstr: &str, @@ -208,8 +208,39 @@ pub async fn proxyx( .finish() })?; + // Filter request headers + let mut new_headers = axum::http::HeaderMap::new(); + + // NOTE: These header values are old Redlib code, and are only tested on GET requests. + for &key in &["Range", "If-Modified-Since", "Cache-Control"] { + if let Some(value) = req.headers().get(key) { + new_headers.insert(key, value.clone()); + } + } + *req.headers_mut() = new_headers; + let req: reqwest::Request = req.map(|body| reqwest::Body::wrap_stream(body.into_data_stream())).try_into().map_err(&into_api_error)?; - let response: reqwest::Response = CLIENTX.execute(req).await.and_then(reqwest::Response::error_for_status).map_err(&into_api_error)?; + let response: reqwest::Response = CLIENTX + .execute(req) + .await + .and_then(reqwest::Response::error_for_status) + .map(|mut res| { + let mut rm = |key: &str| res.headers_mut().remove(key); + rm("access-control-expose-headers"); + rm("server"); + rm("vary"); + rm("etag"); + rm("x-cdn"); + rm("x-cdn-client-region"); + rm("x-cdn-name"); + rm("x-cdn-server-region"); + rm("x-reddit-cdn"); + rm("x-reddit-video-features"); + rm("Nel"); + rm("Report-To"); + res + }) + .map_err(&into_api_error)?; Ok::, ApiError>(response.into()) } diff --git a/src/main.rs b/src/main.rs index c300784c..b0a235f9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ use hyper::Uri; use hyper::{body::Bytes, header::HeaderValue, Body, Request, Response}; use log::{info, warn}; use once_cell::sync::Lazy; -use redlib::client::{canonical_path, into_api_error, proxy, proxyx, rate_limit_check, CLIENT}; +use redlib::client::{canonical_path, into_api_error, proxy, proxy_get, rate_limit_check, CLIENT}; use redlib::server::{self, RequestExt}; use redlib::utils::{error, redirect, ThemeAssets}; use redlib::{config, duplicates, headers, instance_info, post, search, settings, subreddit, user, utils}; @@ -490,7 +490,7 @@ async fn main() { // `auto_curry`, which uses procedural macros, and isn't very featureful macro_rules! proxy { ($fmtstr:expr) => { - |parameters: axum::extract::Path>, req: axum::extract::Request| proxyx(parameters, req, $fmtstr) + |parameters: axum::extract::Path>, req: axum::extract::Request| proxy_get(parameters, req, $fmtstr) }; } From f0b1abe02b07ec9394a68f021540d828109eaeaf Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Fri, 21 Mar 2025 17:37:51 +0000 Subject: [PATCH 24/52] Migrate redirect --- src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index b0a235f9..e9024d74 100644 --- a/src/main.rs +++ b/src/main.rs @@ -495,7 +495,7 @@ async fn main() { } let appx: axum::routing::Router<()> = axum::routing::Router::new() - .without_v07_checks() // Remove unnecessary backward compatibility + .without_v07_checks() // Remove unnecessary backward compatibility checks // Static resources .route("/style.css", get(stylex)) .route("/manifest.json", get(cached_static_resource!("../static/manifest.json", "application/json"))) @@ -533,6 +533,11 @@ async fn main() { .route("/preview/{loc}/{id}", get(proxy!("https://{loc}view.redd.it/{id}"))) .route("/style/{*path}", get(proxy!("https://styles.redditmedia.com/{path}"))) .route("/static/{*path}", get(proxy!("https://www.redditstatic.com/{path}"))) + // User profile + .route( + "/u/{name}", + get(|axum::extract::Path(name): axum::extract::Path| async move { axum::response::Redirect::temporary(format!("/user/{}", name).as_str()) }), + ) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); From c29089341667cf16c3139bb04ad542254c045674 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sun, 23 Mar 2025 02:00:41 +0000 Subject: [PATCH 25/52] Update oauth.rs to use reqwest and shared client --- Cargo.toml | 2 +- src/oauth.rs | 28 ++++++++++------------------ 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0c5acc9f..d0095cac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ http-api-problem = { version = "0.60", features = ["axum", "api-error"] } futures-util = "0.3.31" axum= { version = "0.8" } tower-default-headers = "0.2.0" -reqwest = { version = "0.12.15", features = ["stream"] } +reqwest = { version = "0.12.15", features = ["stream", "json"] } strfmt = "0.2.4" diff --git a/src/oauth.rs b/src/oauth.rs index 56279008..fb3fe833 100644 --- a/src/oauth.rs +++ b/src/oauth.rs @@ -1,11 +1,10 @@ use std::{collections::HashMap, sync::atomic::Ordering, time::Duration}; use crate::{ - client::{CLIENT, OAUTH_CLIENT, OAUTH_IS_ROLLING_OVER, OAUTH_RATELIMIT_REMAINING}, + client::{CLIENTX, OAUTH_CLIENT, OAUTH_IS_ROLLING_OVER, OAUTH_RATELIMIT_REMAINING}, oauth_resources::ANDROID_APP_VERSION_LIST, }; use base64::{engine::general_purpose, Engine as _}; -use hyper::{client, Body, Method, Request}; use log::{error, info, trace}; use serde_json::json; use tegen::tegen::TextGenerator; @@ -69,12 +68,10 @@ impl Oauth { async fn login(&mut self) -> Option<()> { // Construct URL for OAuth token let url = format!("{AUTH_ENDPOINT}/auth/v2/oauth/access-token/loid"); - let mut builder = Request::builder().method(Method::POST).uri(&url); - + let mut builder = CLIENTX.request(reqwest::Method::POST, &url); // Add headers from spoofed client - for (key, value) in &self.initial_headers { - builder = builder.header(key, value); - } + builder = builder.headers((&self.initial_headers).try_into().unwrap()); + // Set up HTTP Basic Auth - basically just the const OAuth ID's with no password, // Base64-encoded. https://en.wikipedia.org/wiki/Basic_access_authentication // This could be constant, but I don't think it's worth it. OAuth ID's can change @@ -83,19 +80,14 @@ impl Oauth { builder = builder.header("Authorization", format!("Basic {auth}")); // Set JSON body. I couldn't tell you what this means. But that's what the client sends - let json = json!({ - "scopes": ["*","email", "pii"] - }); - let body = Body::from(json.to_string()); - - // Build request - let request = builder.body(body).unwrap(); + let builder = builder.json(&json!( + {"scopes": ["*","email", "pii"]} + )); - trace!("Sending token request...\n\n{request:?}"); + trace!("Sending token request...\n\n{builder:?}"); // Send request - let client: &once_cell::sync::Lazy> = &CLIENT; - let resp = client.request(request).await.ok()?; + let resp = builder.send().await.ok()?; trace!("Received response with status {} and length {:?}", resp.status(), resp.headers().get("content-length")); trace!("OAuth headers: {:#?}", resp.headers()); @@ -117,7 +109,7 @@ impl Oauth { trace!("Serializing response..."); // Serialize response - let body_bytes = hyper::body::to_bytes(resp.into_body()).await.ok()?; + let body_bytes = resp.bytes().await.ok()?; let json: serde_json::Value = serde_json::from_slice(&body_bytes).ok()?; trace!("Accessing relevant fields..."); From 01aaf39f04aca9a9aae3c1b202d48d48b76d433c Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Thu, 3 Apr 2025 22:06:54 +0100 Subject: [PATCH 26/52] Refactor cookie-getter idiomatic --- src/utils.rs | 40 ++++++++++++---------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index c59c6fae..8f0cde0b 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -914,52 +914,36 @@ pub fn setting(req: &Request, name: &str) -> String { // Parse a cookie value from request // If this was called with "subscriptions" and the "subscriptions" cookie has a value - if name == "subscriptions" && req.cookie("subscriptions").is_some() { + if let ("subscriptions", Some(subscriptions_cookie)) = (name, req.cookie("subscriptions")) { + // NOTE: the numbered subcriptions will not be registered if subscriptions cookie is empty. Unclear whether intended behaviour. // Create subscriptions string let mut subscriptions = String::new(); + subscriptions.push_str(subscriptions_cookie.value_trimmed()); - // Default subscriptions cookie - if req.cookie("subscriptions").is_some() { - subscriptions.push_str(req.cookie("subscriptions").unwrap().value()); - } - - // Start with first numbered subscription cookie + // append all subscriptionsNUMBER cookies let mut subscriptions_number = 1; - - // While whatever subscriptionsNUMBER cookie we're looking at has a value - while req.cookie(&format!("subscriptions{}", subscriptions_number)).is_some() { + while let Some(cookie_i) = req.cookie(&format!("subscriptions{}", subscriptions_number)) { // Push whatever subscriptionsNUMBER cookie we're looking at into the subscriptions string - subscriptions.push_str(req.cookie(&format!("subscriptions{}", subscriptions_number)).unwrap().value()); - - // Increment subscription cookie number + subscriptions.push_str(cookie_i.value_trimmed()); subscriptions_number += 1; } - // Return the subscriptions cookies as one large string subscriptions } // If this was called with "filters" and the "filters" cookie has a value - else if name == "filters" && req.cookie("filters").is_some() { + else if let ("filters", Some(filters_cookie)) = (name, req.cookie("filters")) { + // NOTE: the numbered filters will not be registered if filters cookie is empty. Unclear whether intended behaviour. // Create filters string let mut filters = String::new(); + filters.push_str(filters_cookie.value_trimmed()); - // Default filters cookie - if req.cookie("filters").is_some() { - filters.push_str(req.cookie("filters").unwrap().value()); - } - - // Start with first numbered filters cookie + // append all filtersNUMBER cookies let mut filters_number = 1; - - // While whatever filtersNUMBER cookie we're looking at has a value - while req.cookie(&format!("filters{}", filters_number)).is_some() { + while let Some(cookie_i) = req.cookie(&format!("filters{}", filters_number)) { // Push whatever filtersNUMBER cookie we're looking at into the filters string - filters.push_str(req.cookie(&format!("filters{}", filters_number)).unwrap().value()); - - // Increment filters cookie number + filters.push_str(cookie_i.value_trimmed()); filters_number += 1; } - // Return the filters cookies as one large string filters } From f897f9a7862de17151d2d5ed1b76604cbb0ae41f Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Thu, 3 Apr 2025 22:14:30 +0100 Subject: [PATCH 27/52] Port some Reddit client stuff Preparations to port post::item --- Cargo.lock | 48 ++++++++++++++++++++++++++ Cargo.toml | 5 +-- src/client.rs | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 144 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e526e5fd..dc5d7976 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,6 +77,19 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "async-compression" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a194f9d963d8099596278594b3107448656ba73831c9d8c783e613ce86da64" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-recursion" version = "1.1.1" @@ -178,6 +191,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-extra" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b" +dependencies = [ + "axum", + "axum-core", + "bytes", + "cookie", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "serde", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -394,6 +429,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" dependencies = [ + "percent-encoding", "time", "version_check", ] @@ -623,6 +659,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "flate2" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1703,6 +1749,7 @@ dependencies = [ "arc-swap", "async-recursion", "axum", + "axum-extra", "base2048", "base64 0.22.1", "bincode", @@ -1794,6 +1841,7 @@ version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ + "async-compression", "base64 0.22.1", "bytes", "encoding_rs", diff --git a/Cargo.toml b/Cargo.toml index d0095cac..51feb3b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,10 +58,11 @@ base2048 = "2.0.2" revision = "0.10.0" http-api-problem = { version = "0.60", features = ["axum", "api-error"] } futures-util = "0.3.31" -axum= { version = "0.8" } +axum= { version = "0.8", features = [] } tower-default-headers = "0.2.0" -reqwest = { version = "0.12.15", features = ["stream", "json"] } +reqwest = { version = "0.12.15", features = ["stream", "json", "gzip"] } strfmt = "0.2.4" +axum-extra = { version = "0.10.0", features = ["cookie"] } [dev-dependencies] diff --git a/src/client.rs b/src/client.rs index ab8d643c..304ac455 100644 --- a/src/client.rs +++ b/src/client.rs @@ -210,7 +210,6 @@ pub async fn proxy_get( // Filter request headers let mut new_headers = axum::http::HeaderMap::new(); - // NOTE: These header values are old Redlib code, and are only tested on GET requests. for &key in &["Range", "If-Modified-Since", "Cache-Control"] { if let Some(value) = req.headers().get(key) { @@ -225,6 +224,7 @@ pub async fn proxy_get( .await .and_then(reqwest::Response::error_for_status) .map(|mut res| { + // Remove unwanted headers let mut rm = |key: &str| res.headers_mut().remove(key); rm("access-control-expose-headers"); rm("server"); @@ -292,6 +292,10 @@ fn reddit_get(path: String, quarantine: bool) -> Boxed, St request(&Method::GET, path, true, quarantine, REDDIT_URL_BASE, REDDIT_URL_BASE_HOST) } +async fn reddit_getx(path: &str, quarantine: bool) -> Result { + reddit_request(reqwest::Method::GET, path, quarantine, REDDIT_SHORT_URL_BASE, REDDIT_SHORT_URL_BASE_HOST).await +} + /// Makes a HEAD request to Reddit at `path, using the short URL base. This will not follow redirects. fn reddit_short_head(path: String, quarantine: bool, base_path: &'static str, host: &'static str) -> Boxed, String>> { request(&Method::HEAD, path, false, quarantine, base_path, host) @@ -303,6 +307,35 @@ fn reddit_short_head(path: String, quarantine: bool, base_path: &'static str, ho // } // Unused - reddit_head is only ever called in the context of a short URL +async fn reddit_request(method: reqwest::Method, path: &str, quarantine: bool, base_path: &'static str, host: &'static str) -> Result { + let url = format!("{base_path}{path}"); + + // Build request to Reddit. Reqwest handles gzip encoding. Reddit does not yet support Brotli encoding. + use reqwest::header; + use reqwest::header::{HeaderMap, HeaderValue}; + let mut headers: HeaderMap = HeaderMap::new(); + headers.append(header::HOST, HeaderValue::from_static(host)); // FIXME: Reddit can fingerprint. Either shuffle headers, or add dynamic headers. + if quarantine { + headers.append( + header::COOKIE, + HeaderValue::from_static("_options=%7B%22pref_quarantine_optin%22%3A%20true%2C%20%22pref_gated_sr_optin%22%3A%20true%7D"), + ); + } + + let client = OAUTH_CLIENT.load(); + let headermap2: HeaderMap = HeaderMap::try_from(&client.headers_map).expect("Invalid hashmap of headers"); + headers.extend(headermap2); + + let result = CLIENTX + .request(method, url) + .headers(headers) + .send() + .await + .and_then(reqwest::Response::error_for_status) + .map_err(&into_api_error)?; + Ok(result) +} + /// Makes a request to Reddit. If `redirect` is `true`, `request_with_redirect` /// will recurse on the URL that Reddit provides in the Location HTTP header /// in its response. @@ -451,6 +484,65 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo .boxed() } +#[cached(size = 100, time = 30, result = true)] +pub async fn jsonx(path: String, quarantine: bool) -> Result { + // First, handle rolling over the OAUTH_CLIENT if need be. + let current_rate_limit = OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst); + let is_rolling_over = OAUTH_IS_ROLLING_OVER.load(Ordering::SeqCst); + if current_rate_limit < 10 && !is_rolling_over { + log::info!("Rate limit {current_rate_limit} is low. Spawning force_refresh_token()"); + tokio::spawn(force_refresh_token()); + } + OAUTH_RATELIMIT_REMAINING.fetch_sub(1, Ordering::SeqCst); + + let response = reddit_getx(&path, quarantine).await?; + + // Handle OAUTH stuff + let reset = response.headers().get("x-ratelimit-reset").map(|val| String::from_utf8_lossy(val.as_bytes())); + let remaining = response.headers().get("x-ratelimit-remaining").map(|val| String::from_utf8_lossy(val.as_bytes())); + let used = response.headers().get("x-ratelimit-used").map(|val| String::from_utf8_lossy(val.as_bytes())); + trace!( + "Ratelimit remaining: Header says {}, we have {current_rate_limit}. Resets in {}. Rollover: {}. Ratelimit used: {}", + remaining.as_deref().unwrap_or_default(), + reset.as_deref().unwrap_or_default(), + if is_rolling_over { "yes" } else { "no" }, + used.as_deref().unwrap_or_default(), + ); + if let Some(val) = remaining.and_then(|s| s.parse::().ok()) { + OAUTH_RATELIMIT_REMAINING.store(val.round() as u16, Ordering::SeqCst); + } + + // Work with the JSON. + let json: Value = response.json().await.map_err(|e| { + ApiError::builder(http_api_problem::StatusCode::BAD_GATEWAY) // 502 + .title("Bad Gateway") + .message(format!("Failed to parse page JSON data: {e}")) + .source(e) + .finish() + })?; + + // FIXME: If we get http 401, with error message "Unauthorized", force a token refresh. Currently 401 is handled by `into_api_error`. Perphaps reddit_request can handle oauth stuff + + if let Some(true) = json["data"]["is_suspended"].as_bool() { + return Err( + ApiError::builder(http_api_problem::StatusCode::NOT_FOUND) + .title("Suspended") + .message("user is suspended") + .finish(), + ); + } + if let Some(error_code) = json["error"].as_i64() { + // Err(format!("Reddit error {} \"{}\": {} | {path}", json["error"], json["reason"], json["message"])); + return Err( + ApiError::builder(http_api_problem::StatusCode::NOT_FOUND) + .title(format!("Reddit Error {error_code}: {}", json["reason"])) + .message(json["message"].as_str().unwrap_or_default()) + .finish(), + ); + } + Ok(json) +} + // Make a request to a Reddit API and parse the JSON response #[cached(size = 100, time = 30, result = true)] pub async fn json(path: String, quarantine: bool) -> Result { From ca6ef1eec6394739b6dd21372eacc773b055c32e Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Thu, 3 Apr 2025 22:49:00 +0100 Subject: [PATCH 28/52] Remove some cloning --- src/config.rs | 61 ++++++++++++++++++++++++++++----------------------- src/post.rs | 2 +- src/utils.rs | 2 +- 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/config.rs b/src/config.rs index 7b1c95cc..05174545 100644 --- a/src/config.rs +++ b/src/config.rs @@ -133,7 +133,10 @@ impl Config { // Return the first non-`None` value // If all are `None`, return `None` let legacy_key = key.replace("REDLIB_", "LIBREDDIT_"); - var(key).ok().or_else(|| var(legacy_key).ok()).or_else(|| get_setting_from_config(key, &config)) + var(key) + .ok() + .or_else(|| var(legacy_key).ok()) + .or_else(|| get_setting_from_config(key, &config).map(|s| s.to_string())) }; Self { sfw_only: parse("REDLIB_SFW_ONLY"), @@ -164,38 +167,40 @@ impl Config { } } -fn get_setting_from_config(name: &str, config: &Config) -> Option { +fn get_setting_from_config<'a>(name: &str, config: &'a Config) -> Option<&'a str> { match name { - "REDLIB_SFW_ONLY" => config.sfw_only.clone(), - "REDLIB_DEFAULT_THEME" => config.default_theme.clone(), - "REDLIB_DEFAULT_FRONT_PAGE" => config.default_front_page.clone(), - "REDLIB_DEFAULT_LAYOUT" => config.default_layout.clone(), - "REDLIB_DEFAULT_COMMENT_SORT" => config.default_comment_sort.clone(), - "REDLIB_DEFAULT_POST_SORT" => config.default_post_sort.clone(), - "REDLIB_DEFAULT_BLUR_SPOILER" => config.default_blur_spoiler.clone(), - "REDLIB_DEFAULT_SHOW_NSFW" => config.default_show_nsfw.clone(), - "REDLIB_DEFAULT_BLUR_NSFW" => config.default_blur_nsfw.clone(), - "REDLIB_DEFAULT_USE_HLS" => config.default_use_hls.clone(), - "REDLIB_DEFAULT_HIDE_HLS_NOTIFICATION" => config.default_hide_hls_notification.clone(), - "REDLIB_DEFAULT_WIDE" => config.default_wide.clone(), - "REDLIB_DEFAULT_HIDE_AWARDS" => config.default_hide_awards.clone(), - "REDLIB_DEFAULT_HIDE_SIDEBAR_AND_SUMMARY" => config.default_hide_sidebar_and_summary.clone(), - "REDLIB_DEFAULT_HIDE_SCORE" => config.default_hide_score.clone(), - "REDLIB_DEFAULT_SUBSCRIPTIONS" => config.default_subscriptions.clone(), - "REDLIB_DEFAULT_FILTERS" => config.default_filters.clone(), - "REDLIB_DEFAULT_DISABLE_VISIT_REDDIT_CONFIRMATION" => config.default_disable_visit_reddit_confirmation.clone(), - "REDLIB_BANNER" => config.banner.clone(), - "REDLIB_ROBOTS_DISABLE_INDEXING" => config.robots_disable_indexing.clone(), - "REDLIB_PUSHSHIFT_FRONTEND" => config.pushshift.clone(), - "REDLIB_ENABLE_RSS" => config.enable_rss.clone(), - "REDLIB_FULL_URL" => config.full_url.clone(), - "REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS" => config.default_remove_default_feeds.clone(), - _ => None, + "REDLIB_SFW_ONLY" => &config.sfw_only, + "REDLIB_DEFAULT_THEME" => &config.default_theme, + "REDLIB_DEFAULT_FRONT_PAGE" => &config.default_front_page, + "REDLIB_DEFAULT_LAYOUT" => &config.default_layout, + "REDLIB_DEFAULT_COMMENT_SORT" => &config.default_comment_sort, + "REDLIB_DEFAULT_POST_SORT" => &config.default_post_sort, + "REDLIB_DEFAULT_BLUR_SPOILER" => &config.default_blur_spoiler, + "REDLIB_DEFAULT_SHOW_NSFW" => &config.default_show_nsfw, + "REDLIB_DEFAULT_BLUR_NSFW" => &config.default_blur_nsfw, + "REDLIB_DEFAULT_USE_HLS" => &config.default_use_hls, + "REDLIB_DEFAULT_HIDE_HLS_NOTIFICATION" => &config.default_hide_hls_notification, + "REDLIB_DEFAULT_WIDE" => &config.default_wide, + "REDLIB_DEFAULT_HIDE_AWARDS" => &config.default_hide_awards, + "REDLIB_DEFAULT_HIDE_SIDEBAR_AND_SUMMARY" => &config.default_hide_sidebar_and_summary, + "REDLIB_DEFAULT_HIDE_SCORE" => &config.default_hide_score, + "REDLIB_DEFAULT_SUBSCRIPTIONS" => &config.default_subscriptions, + "REDLIB_DEFAULT_FILTERS" => &config.default_filters, + "REDLIB_DEFAULT_DISABLE_VISIT_REDDIT_CONFIRMATION" => &config.default_disable_visit_reddit_confirmation, + "REDLIB_BANNER" => &config.banner, + "REDLIB_ROBOTS_DISABLE_INDEXING" => &config.robots_disable_indexing, + "REDLIB_PUSHSHIFT_FRONTEND" => &config.pushshift, + "REDLIB_ENABLE_RSS" => &config.enable_rss, + "REDLIB_FULL_URL" => &config.full_url, + "REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS" => &config.default_remove_default_feeds, + _ => &None, } + .as_ref() + .map(String::as_str) } /// Retrieves setting from environment variable or config file. -pub fn get_setting(name: &str) -> Option { +pub fn get_setting(name: &str) -> Option<&'static str> { get_setting_from_config(name, &CONFIG) } diff --git a/src/post.rs b/src/post.rs index 20b917da..a9ff4a85 100644 --- a/src/post.rs +++ b/src/post.rs @@ -177,7 +177,7 @@ fn build_comment( let body = if (val(comment, "author") == "[deleted]" && val(comment, "body") == "[removed]") || val(comment, "body") == "[ Removed by Reddit ]" { format!( "

[removed] — view removed comment

", - get_setting("REDLIB_PUSHSHIFT_FRONTEND").unwrap_or_else(|| String::from(crate::config::DEFAULT_PUSHSHIFT_FRONTEND)), + get_setting("REDLIB_PUSHSHIFT_FRONTEND").unwrap_or(crate::config::DEFAULT_PUSHSHIFT_FRONTEND), ) } else { rewrite_emotes(&data["media_metadata"], val(comment, "body_html")) diff --git a/src/utils.rs b/src/utils.rs index 8f0cde0b..10052993 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -810,7 +810,7 @@ pub async fn parse_post(post: &Value) -> Post { let body = if val(post, "removed_by_category") == "moderator" { format!( "

[removed] — view removed post

", - get_setting("REDLIB_PUSHSHIFT_FRONTEND").unwrap_or_else(|| String::from(crate::config::DEFAULT_PUSHSHIFT_FRONTEND)), + get_setting("REDLIB_PUSHSHIFT_FRONTEND").unwrap_or(config::DEFAULT_PUSHSHIFT_FRONTEND), ) } else { let selftext = val(post, "selftext"); From 4f36dd39c9121b83af918d9899abab8ef28833b0 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Fri, 4 Apr 2025 01:14:59 +0100 Subject: [PATCH 29/52] Use CookieJar extractor --- src/utils.rs | 58 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index 10052993..cba86cd7 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -2,12 +2,13 @@ #![allow(clippy::cmp_owned)] use crate::config::{self, get_setting}; +use std::borrow::Cow; // // CRATES // use crate::client::CLIENTX; use crate::{client::json, server::RequestExt}; -use cookie::Cookie; +use axum_extra::extract::CookieJar; use hyper::{Body, Request, Response}; use libflate::deflate::{Decoder, Encoder}; use log::error; @@ -910,38 +911,50 @@ pub fn param(path: &str, value: &str) -> Option { } // Retrieve the value of a setting by name +// Deprecated. Please use setting_from_cookiejar pub fn setting(req: &Request, name: &str) -> String { + // http crate versions do not match. use this "unsafe" conversion. + let cookies = CookieJar::from_headers(&{ + let mut headers = axum::http::header::HeaderMap::new(); + if let Some(cookie_header) = req.headers().get("Cookie") { + headers.insert("Cookie", cookie_header.as_bytes().try_into().unwrap()); // This should never panic + } + headers + }); + setting_from_cookiejar(&cookies, name).to_string() +} + +// Retrieve the value of a setting by name +pub fn setting_from_cookiejar<'a>(cookies: &'a CookieJar, name: &str) -> Cow<'a, str> { // Parse a cookie value from request // If this was called with "subscriptions" and the "subscriptions" cookie has a value - if let ("subscriptions", Some(subscriptions_cookie)) = (name, req.cookie("subscriptions")) { - // NOTE: the numbered subcriptions will not be registered if subscriptions cookie is empty. Unclear whether intended behaviour. + if let ("subscriptions", Some(subscriptions_cookie)) = (name, cookies.get("subscriptions")) { + // NOTE: the numbered subcription cookies will not be registered if "subscriptions" cookie is empty. Unclear whether intended behaviour. // Create subscriptions string - let mut subscriptions = String::new(); - subscriptions.push_str(subscriptions_cookie.value_trimmed()); + let mut subscriptions = Cow::from(subscriptions_cookie.value_trimmed()); // append all subscriptionsNUMBER cookies let mut subscriptions_number = 1; - while let Some(cookie_i) = req.cookie(&format!("subscriptions{}", subscriptions_number)) { + while let Some(cookie_i) = cookies.get(&format!("subscriptions{}", subscriptions_number)) { // Push whatever subscriptionsNUMBER cookie we're looking at into the subscriptions string - subscriptions.push_str(cookie_i.value_trimmed()); + subscriptions.to_mut().push_str(cookie_i.value_trimmed()); subscriptions_number += 1; } // Return the subscriptions cookies as one large string subscriptions } // If this was called with "filters" and the "filters" cookie has a value - else if let ("filters", Some(filters_cookie)) = (name, req.cookie("filters")) { - // NOTE: the numbered filters will not be registered if filters cookie is empty. Unclear whether intended behaviour. + else if let ("filters", Some(filters_cookie)) = (name, cookies.get("filters")) { + // NOTE: the numbered filter cookies will not be registered if "filters" cookie is empty. Unclear whether intended behaviour. // Create filters string - let mut filters = String::new(); - filters.push_str(filters_cookie.value_trimmed()); + let mut filters = Cow::from(filters_cookie.value_trimmed()); // append all filtersNUMBER cookies let mut filters_number = 1; - while let Some(cookie_i) = req.cookie(&format!("filters{}", filters_number)) { + while let Some(cookie_i) = cookies.get(&format!("filters{}", filters_number)) { // Push whatever filtersNUMBER cookie we're looking at into the filters string - filters.push_str(cookie_i.value_trimmed()); + filters.to_mut().push_str(cookie_i.value_trimmed()); filters_number += 1; } // Return the filters cookies as one large string @@ -949,18 +962,13 @@ pub fn setting(req: &Request, name: &str) -> String { } // The above two still come to this if there was no existing value else { - req - .cookie(name) - .unwrap_or_else(|| { - // If there is no cookie for this setting, try receiving a default from the config - if let Some(default) = get_setting(&format!("REDLIB_DEFAULT_{}", name.to_uppercase())) { - Cookie::new(name, default) - } else { - Cookie::from(name) - } - }) - .value() - .to_string() + Cow::from( + cookies + .get(name) + .map(|cookie| cookie.value()) + .or_else(|| get_setting(&format!("REDLIB_DEFAULT_{}", name.to_uppercase()))) + .unwrap_or_default(), + ) } } From 6004e6a6be8d199ad56254875acfe8988f66a203 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Fri, 4 Apr 2025 18:00:34 +0100 Subject: [PATCH 30/52] Refactor parse_comments and query_comments --- src/post.rs | 58 ++++++++++++++++++++++------------------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/src/post.rs b/src/post.rs index a9ff4a85..b8a39a0e 100644 --- a/src/post.rs +++ b/src/post.rs @@ -114,25 +114,28 @@ pub async fn item(req: Request) -> Result, String> { // COMMENTS +/// A Vec of all comments defined in a json response fn parse_comments(json: &serde_json::Value, post_link: &str, post_author: &str, highlighted_comment: &str, filters: &HashSet, req: &Request) -> Vec { - // Parse the comment JSON into a Vector of Comments - let comments = json["data"]["children"].as_array().map_or(Vec::new(), std::borrow::ToOwned::to_owned); - - // For each comment, retrieve the values to build a Comment object - comments - .into_iter() - .map(|comment| { - let data = &comment["data"]; - let replies: Vec = if data["replies"].is_object() { - parse_comments(&data["replies"], post_link, post_author, highlighted_comment, filters, req) - } else { - Vec::new() - }; - build_comment(&comment, data, replies, post_link, post_author, highlighted_comment, filters, req) - }) - .collect() + let comments = json["data"]["children"].as_array(); + if let Some(comments) = comments { + comments + .into_iter() + .map(|comment| { + let data = &comment["data"]; + let replies: Vec = if data["replies"].is_object() { + parse_comments(&data["replies"], post_link, post_author, highlighted_comment, filters, req) + } else { + Vec::new() + }; + build_comment(&comment, data, replies, post_link, post_author, highlighted_comment, filters, req) + }) + .collect() + } else { + Vec::new() + } } +/// like parse_comments, but filters comment body by query parameter. fn query_comments( json: &serde_json::Value, post_link: &str, @@ -142,24 +145,11 @@ fn query_comments( query: &str, req: &Request, ) -> Vec { - let comments = json["data"]["children"].as_array().map_or(Vec::new(), std::borrow::ToOwned::to_owned); - let mut results = Vec::new(); - - for comment in comments { - let data = &comment["data"]; - - // If this comment contains replies, handle those too - if data["replies"].is_object() { - results.append(&mut query_comments(&data["replies"], post_link, post_author, highlighted_comment, filters, query, req)); - } - - let c = build_comment(&comment, data, Vec::new(), post_link, post_author, highlighted_comment, filters, req); - if c.body.to_lowercase().contains(&query.to_lowercase()) { - results.push(c); - } - } - - results + let query_lc = query.to_lowercase(); + parse_comments(json, post_link, post_author, highlighted_comment, filters, req) + .into_iter() + .filter(|c| c.body.to_lowercase().contains(&query_lc)) + .collect() } #[allow(clippy::too_many_arguments)] fn build_comment( From 47c59907dfbae56344bb4857741bfb9e0a0cb18a Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Fri, 4 Apr 2025 22:45:52 +0100 Subject: [PATCH 31/52] Refactor Comment struct initialiser --- src/utils.rs | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 191 insertions(+), 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index cba86cd7..39a06c72 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -25,6 +25,7 @@ use std::env; use std::io::{Read, Write}; use std::str::FromStr; use std::string::ToString; +use std::sync::OnceLock; use time::{macros::format_description, Duration, OffsetDateTime}; use url::Url; @@ -493,6 +494,102 @@ pub struct Comment { pub prefs: Preferences, } +impl Comment { + fn build( + comment: &serde_json::Value, + data: &serde_json::Value, + replies: Vec, + post_link: &str, + post_author: &str, + highlighted_comment: &str, + filters: &HashSet, + cookies: &CookieJar, + ) -> Self { + let id = val(comment, "id"); + + let body = if (val(comment, "author") == "[deleted]" && val(comment, "body") == "[removed]") || val(comment, "body") == "[ Removed by Reddit ]" { + format!( + "

[removed] — view removed comment

", + get_setting("REDLIB_PUSHSHIFT_FRONTEND").unwrap_or(crate::config::DEFAULT_PUSHSHIFT_FRONTEND), + ) + } else { + rewrite_emotes(&data["media_metadata"], val(comment, "body_html")) + }; + let kind = comment["kind"].as_str().unwrap_or_default().to_string(); + + let unix_time = data["created_utc"].as_f64().unwrap_or_default(); + let (rel_time, created) = time(unix_time); + + let edited = data["edited"].as_f64().map_or((String::new(), String::new()), time); + + let score = data["score"].as_i64().unwrap_or(0); + + // The JSON API only provides comments up to some threshold. + // Further comments have to be loaded by subsequent requests. + // The "kind" value will be "more" and the "count" + // shows how many more (sub-)comments exist in the respective nesting level. + // Note that in certain (seemingly random) cases, the count is simply wrong. + let more_count = data["count"].as_i64().unwrap_or_default(); + + let awards: Awards = Awards::parse(&data["all_awardings"]); + + let parent_kind_and_id = val(comment, "parent_id"); + let parent_info = parent_kind_and_id.split('_').collect::>(); + + let highlighted = id == highlighted_comment; + + let author = Author { + name: val(comment, "author"), + flair: Flair { + flair_parts: FlairPart::parse( + data["author_flair_type"].as_str().unwrap_or_default(), + data["author_flair_richtext"].as_array(), + data["author_flair_text"].as_str(), + ), + text: val(comment, "link_flair_text"), + background_color: val(comment, "author_flair_background_color"), + foreground_color: val(comment, "author_flair_text_color"), + }, + distinguished: val(comment, "distinguished"), + }; + let is_filtered = filters.contains(&["u_", author.name.as_str()].concat()); + + // Many subreddits have a default comment posted about the sub's rules etc. + // Many Redlib users do not wish to see this kind of comment by default. + // Reddit does not tell us which users are "bots", so a good heuristic is to + // collapse stickied moderator comments. + let is_moderator_comment = data["distinguished"].as_str().unwrap_or_default() == "moderator"; + let is_stickied = data["stickied"].as_bool().unwrap_or_default(); + let collapsed = (is_moderator_comment && is_stickied) || is_filtered; + + Comment { + id, + kind, + parent_id: parent_info[1].to_string(), + parent_kind: parent_info[0].to_string(), + post_link: post_link.to_string(), + post_author: post_author.to_string(), + body, + author, + score: if data["score_hidden"].as_bool().unwrap_or_default() { + ("\u{2022}".to_string(), "Hidden".to_string()) + } else { + format_num(score) + }, + rel_time, + created, + edited, + replies, + highlighted, + awards, + collapsed, + is_filtered, + more_count, + prefs: Preferences::build(cookies), + } + } +} + #[derive(Default, Clone, Serialize)] pub struct Award { pub name: String, @@ -733,6 +830,88 @@ impl Preferences { } } + pub fn build(cookies: &CookieJar) -> Self { + // Read available theme names from embedded css files. + // Always make the default "system" theme available. + /*let mut themes = vec!["system".to_string()]; + for file in ThemeAssets::iter() { + let chunks: Vec<&str> = file.as_ref().split(".css").collect(); + themes.push(chunks[0].to_owned()); + }*/ + static THEMES: OnceLock> = OnceLock::new(); + Self { + available_themes: THEMES + .get_or_init(|| ThemeAssets::iter().map(|f| f.split(".css").collect::>()[0].to_owned()).collect()) + .clone(), + theme: setting_from_cookiejar(cookies, "theme").into_owned(), + front_page: setting_from_cookiejar(cookies, "front_page").into_owned(), + layout: setting_from_cookiejar(cookies, "layout").into_owned(), + wide: setting_from_cookiejar(cookies, "wide").into_owned(), + blur_spoiler: setting_from_cookiejar(cookies, "blur_spoiler").into_owned(), + show_nsfw: setting_from_cookiejar(cookies, "show_nsfw").into_owned(), + hide_sidebar_and_summary: setting_from_cookiejar(cookies, "hide_sidebar_and_summary").into_owned(), + blur_nsfw: setting_from_cookiejar(cookies, "blur_nsfw").into_owned(), + use_hls: setting_from_cookiejar(cookies, "use_hls").into_owned(), + hide_hls_notification: setting_from_cookiejar(cookies, "hide_hls_notification").into_owned(), + video_quality: setting_from_cookiejar(cookies, "video_quality").into_owned(), + autoplay_videos: setting_from_cookiejar(cookies, "autoplay_videos").into_owned(), + fixed_navbar: setting_from_cookiejar_or_default(cookies, "fixed_navbar", Cow::from("on")).into_owned(), + disable_visit_reddit_confirmation: setting_from_cookiejar(cookies, "disable_visit_reddit_confirmation").into_owned(), + comment_sort: setting_from_cookiejar(cookies, "comment_sort").into_owned(), + post_sort: setting_from_cookiejar(cookies, "post_sort").into_owned(), + subscriptions: setting_from_cookiejar(cookies, "subscriptions") + .split('+') + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(), + filters: setting_from_cookiejar(cookies, "filters").split('+').filter(|s| !s.is_empty()).map(String::from).collect(), + hide_awards: setting_from_cookiejar(cookies, "hide_awards").into_owned(), + hide_score: setting_from_cookiejar(cookies, "hide_score").into_owned(), + remove_default_feeds: setting_from_cookiejar(cookies, "remove_default_feeds").into_owned(), + } + } + + pub fn build(cookies: &CookieJar) -> Self { + // Read available theme names from embedded css files. + // Always make the default "system" theme available. + /*let mut themes = vec!["system".to_string()]; + for file in ThemeAssets::iter() { + let chunks: Vec<&str> = file.as_ref().split(".css").collect(); + themes.push(chunks[0].to_owned()); + }*/ + static THEMES: OnceLock> = OnceLock::new(); + Self { + available_themes: THEMES + .get_or_init(|| ThemeAssets::iter().map(|f| f.split(".css").collect::>()[0].to_owned()).collect()) + .clone(), + theme: setting_from_cookiejar(cookies, "theme").into_owned(), + front_page: setting_from_cookiejar(cookies, "front_page").into_owned(), + layout: setting_from_cookiejar(cookies, "layout").into_owned(), + wide: setting_from_cookiejar(cookies, "wide").into_owned(), + blur_spoiler: setting_from_cookiejar(cookies, "blur_spoiler").into_owned(), + show_nsfw: setting_from_cookiejar(cookies, "show_nsfw").into_owned(), + hide_sidebar_and_summary: setting_from_cookiejar(cookies, "hide_sidebar_and_summary").into_owned(), + blur_nsfw: setting_from_cookiejar(cookies, "blur_nsfw").into_owned(), + use_hls: setting_from_cookiejar(cookies, "use_hls").into_owned(), + hide_hls_notification: setting_from_cookiejar(cookies, "hide_hls_notification").into_owned(), + video_quality: setting_from_cookiejar(cookies, "video_quality").into_owned(), + autoplay_videos: setting_from_cookiejar(cookies, "autoplay_videos").into_owned(), + fixed_navbar: setting_from_cookiejar_or_default(cookies, "fixed_navbar", Cow::from("on")).into_owned(), + disable_visit_reddit_confirmation: setting_from_cookiejar(cookies, "disable_visit_reddit_confirmation").into_owned(), + comment_sort: setting_from_cookiejar(cookies, "comment_sort").into_owned(), + post_sort: setting_from_cookiejar(cookies, "post_sort").into_owned(), + subscriptions: setting_from_cookiejar(cookies, "subscriptions") + .split('+') + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(), + filters: setting_from_cookiejar(cookies, "filters").split('+').filter(|s| !s.is_empty()).map(String::from).collect(), + hide_awards: setting_from_cookiejar(cookies, "hide_awards").into_owned(), + hide_score: setting_from_cookiejar(cookies, "hide_score").into_owned(), + remove_default_feeds: setting_from_cookiejar(cookies, "remove_default_feeds").into_owned(), + } + } + pub fn to_urlencoded(&self) -> Result { serde_urlencoded::to_string(self) } @@ -925,6 +1104,8 @@ pub fn setting(req: &Request, name: &str) -> String { } // Retrieve the value of a setting by name +// If cookie is empty, fetches it from config +// Defaults to empty string pub fn setting_from_cookiejar<'a>(cookies: &'a CookieJar, name: &str) -> Cow<'a, str> { // Parse a cookie value from request @@ -962,7 +1143,7 @@ pub fn setting_from_cookiejar<'a>(cookies: &'a CookieJar, name: &str) -> Cow<'a, } // The above two still come to this if there was no existing value else { - Cow::from( + Cow::from( cookies .get(name) .map(|cookie| cookie.value()) @@ -982,6 +1163,15 @@ pub fn setting_or_default(req: &Request, name: &str, default: String) -> S } } +pub fn setting_from_cookiejar_or_default<'a>(cookies: &'a CookieJar, name: &str, default: Cow<'a, str>) -> Cow<'a, str> { + let value = setting_from_cookiejar(cookies, name); + if value.is_empty() { + default + } else { + value + } +} + // Detect and redirect in the event of a random subreddit pub async fn catch_random(sub: &str, additional: &str) -> Result, String> { if sub == "random" || sub == "randnsfw" { From d198bd6e08375f231086405bc7a7aead9f850733 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Fri, 4 Apr 2025 22:57:33 +0100 Subject: [PATCH 32/52] UTIL: convert old Request->Cookiejar --- src/utils.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index 39a06c72..4564af28 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -940,6 +940,18 @@ pub fn deflate_decompress(i: Vec) -> Result, std::io::Error> { Ok(out) } +/// Convert an old http<1 req into a CookieJar +/// DEPRECATED +pub fn cookie_jar_from_oldreq(req: &Request) -> CookieJar { + CookieJar::from_headers(&{ + let mut headers = axum::http::header::HeaderMap::new(); + if let Some(cookie_header) = req.headers().get("Cookie") { + headers.insert("Cookie", cookie_header.as_bytes().try_into().unwrap()); // This should never panic + } + headers + }) +} + /// Gets a `HashSet` of filters from the cookie in the given `Request`. pub fn get_filters(req: &Request) -> HashSet { setting(req, "filters").split('+').map(String::from).filter(|s| !s.is_empty()).collect::>() @@ -1093,13 +1105,7 @@ pub fn param(path: &str, value: &str) -> Option { // Deprecated. Please use setting_from_cookiejar pub fn setting(req: &Request, name: &str) -> String { // http crate versions do not match. use this "unsafe" conversion. - let cookies = CookieJar::from_headers(&{ - let mut headers = axum::http::header::HeaderMap::new(); - if let Some(cookie_header) = req.headers().get("Cookie") { - headers.insert("Cookie", cookie_header.as_bytes().try_into().unwrap()); // This should never panic - } - headers - }); + let cookies = cookie_jar_from_oldreq(req); setting_from_cookiejar(&cookies, name).to_string() } From 44e1fc8cccaa237f3af12a138c4760db0b1c81f4 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Fri, 4 Apr 2025 23:06:45 +0100 Subject: [PATCH 33/52] Migrate old code new Comment init --- src/post.rs | 131 +++++++++++---------------------------------------- src/utils.rs | 8 ++++ 2 files changed, 35 insertions(+), 104 deletions(-) diff --git a/src/post.rs b/src/post.rs index b8a39a0e..36e1a200 100644 --- a/src/post.rs +++ b/src/post.rs @@ -5,15 +5,17 @@ use crate::client::json; use crate::config::get_setting; use crate::server::RequestExt; use crate::subreddit::{can_access_quarantine, quarantine}; -use crate::utils::{ - error, format_num, get_filters, nsfw_landing, param, parse_post, rewrite_emotes, setting, template, time, val, Author, Awards, Comment, Flair, FlairPart, Post, Preferences, -}; +use crate::utils::{cookie_jar_from_oldreq, error, get_filters, nsfw_landing, param, parse_post, setting, setting_from_cookiejar, template, Comment, Post, Preferences}; use hyper::{Body, Request, Response}; +use axum::RequestExt as AxumRequestExt; +use axum_extra::extract::cookie::CookieJar; use once_cell::sync::Lazy; use regex::Regex; use rinja::Template; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; +use unwrap_infallible::UnwrapInfallible; // STRUCTS #[derive(Template)] @@ -84,8 +86,23 @@ pub async fn item(req: Request) -> Result, String> { let query = form.get("q").unwrap().clone().to_string(); let comments = match query.as_str() { - "" => parse_comments(&response[1], &post.permalink, &post.author.name, highlighted_comment, &get_filters(&req), &req), - _ => query_comments(&response[1], &post.permalink, &post.author.name, highlighted_comment, &get_filters(&req), &query, &req), + "" => parse_comments( + &response[1], + &post.permalink, + &post.author.name, + highlighted_comment, + &get_filters(&req), + &cookie_jar_from_oldreq(&req), + ), + _ => query_comments( + &response[1], + &post.permalink, + &post.author.name, + highlighted_comment, + &get_filters(&req), + &query, + &cookie_jar_from_oldreq(&req), + ), }; // Use the Post and Comment structs to generate a website to show users @@ -115,7 +132,7 @@ pub async fn item(req: Request) -> Result, String> { // COMMENTS /// A Vec of all comments defined in a json response -fn parse_comments(json: &serde_json::Value, post_link: &str, post_author: &str, highlighted_comment: &str, filters: &HashSet, req: &Request) -> Vec { +fn parse_comments(json: &serde_json::Value, post_link: &str, post_author: &str, highlighted_comment: &str, filters: &HashSet, cookies: &CookieJar) -> Vec { let comments = json["data"]["children"].as_array(); if let Some(comments) = comments { comments @@ -123,11 +140,11 @@ fn parse_comments(json: &serde_json::Value, post_link: &str, post_author: &str, .map(|comment| { let data = &comment["data"]; let replies: Vec = if data["replies"].is_object() { - parse_comments(&data["replies"], post_link, post_author, highlighted_comment, filters, req) + parse_comments(&data["replies"], post_link, post_author, highlighted_comment, filters, cookies) } else { Vec::new() }; - build_comment(&comment, data, replies, post_link, post_author, highlighted_comment, filters, req) + Comment::build(&comment, data, replies, post_link, post_author, highlighted_comment, filters, cookies) }) .collect() } else { @@ -143,105 +160,11 @@ fn query_comments( highlighted_comment: &str, filters: &HashSet, query: &str, - req: &Request, + cookies: &CookieJar, ) -> Vec { let query_lc = query.to_lowercase(); - parse_comments(json, post_link, post_author, highlighted_comment, filters, req) + parse_comments(json, post_link, post_author, highlighted_comment, filters, cookies) .into_iter() .filter(|c| c.body.to_lowercase().contains(&query_lc)) .collect() } -#[allow(clippy::too_many_arguments)] -fn build_comment( - comment: &serde_json::Value, - data: &serde_json::Value, - replies: Vec, - post_link: &str, - post_author: &str, - highlighted_comment: &str, - filters: &HashSet, - req: &Request, -) -> Comment { - let id = val(comment, "id"); - - let body = if (val(comment, "author") == "[deleted]" && val(comment, "body") == "[removed]") || val(comment, "body") == "[ Removed by Reddit ]" { - format!( - "

[removed] — view removed comment

", - get_setting("REDLIB_PUSHSHIFT_FRONTEND").unwrap_or(crate::config::DEFAULT_PUSHSHIFT_FRONTEND), - ) - } else { - rewrite_emotes(&data["media_metadata"], val(comment, "body_html")) - }; - let kind = comment["kind"].as_str().unwrap_or_default().to_string(); - - let unix_time = data["created_utc"].as_f64().unwrap_or_default(); - let (rel_time, created) = time(unix_time); - - let edited = data["edited"].as_f64().map_or((String::new(), String::new()), time); - - let score = data["score"].as_i64().unwrap_or(0); - - // The JSON API only provides comments up to some threshold. - // Further comments have to be loaded by subsequent requests. - // The "kind" value will be "more" and the "count" - // shows how many more (sub-)comments exist in the respective nesting level. - // Note that in certain (seemingly random) cases, the count is simply wrong. - let more_count = data["count"].as_i64().unwrap_or_default(); - - let awards: Awards = Awards::parse(&data["all_awardings"]); - - let parent_kind_and_id = val(comment, "parent_id"); - let parent_info = parent_kind_and_id.split('_').collect::>(); - - let highlighted = id == highlighted_comment; - - let author = Author { - name: val(comment, "author"), - flair: Flair { - flair_parts: FlairPart::parse( - data["author_flair_type"].as_str().unwrap_or_default(), - data["author_flair_richtext"].as_array(), - data["author_flair_text"].as_str(), - ), - text: val(comment, "link_flair_text"), - background_color: val(comment, "author_flair_background_color"), - foreground_color: val(comment, "author_flair_text_color"), - }, - distinguished: val(comment, "distinguished"), - }; - let is_filtered = filters.contains(&["u_", author.name.as_str()].concat()); - - // Many subreddits have a default comment posted about the sub's rules etc. - // Many Redlib users do not wish to see this kind of comment by default. - // Reddit does not tell us which users are "bots", so a good heuristic is to - // collapse stickied moderator comments. - let is_moderator_comment = data["distinguished"].as_str().unwrap_or_default() == "moderator"; - let is_stickied = data["stickied"].as_bool().unwrap_or_default(); - let collapsed = (is_moderator_comment && is_stickied) || is_filtered; - - Comment { - id, - kind, - parent_id: parent_info[1].to_string(), - parent_kind: parent_info[0].to_string(), - post_link: post_link.to_string(), - post_author: post_author.to_string(), - body, - author, - score: if data["score_hidden"].as_bool().unwrap_or_default() { - ("\u{2022}".to_string(), "Hidden".to_string()) - } else { - format_num(score) - }, - rel_time, - created, - edited, - replies, - highlighted, - awards, - collapsed, - is_filtered, - more_count, - prefs: Preferences::new(req), - } -} diff --git a/src/utils.rs b/src/utils.rs index 4564af28..e3d2ddd1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -957,6 +957,14 @@ pub fn get_filters(req: &Request) -> HashSet { setting(req, "filters").split('+').map(String::from).filter(|s| !s.is_empty()).collect::>() } +pub fn get_filtersx(cookies: &CookieJar) -> HashSet { + setting_from_cookiejar(cookies, "filters") + .split('+') + .filter(|s| !s.is_empty()) + .map(String::from) + .collect::>() +} + /// Filters a `Vec` by the given `HashSet` of filters (each filter being /// a subreddit name or a user name). If a `Post`'s subreddit or author is /// found in the filters, it is removed. From 5bbc9792546cf60f1b48f31f08619e67bc06c675 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Fri, 4 Apr 2025 23:22:57 +0100 Subject: [PATCH 34/52] Begin migrating `item` fn --- Cargo.lock | 7 ++++++ Cargo.toml | 1 + src/main.rs | 1 + src/post.rs | 64 +++++++++++++++++++++++++++++++++++++++++++++++++--- src/utils.rs | 54 ++++++++++---------------------------------- 5 files changed, 82 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc5d7976..3402f2f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1793,6 +1793,7 @@ dependencies = [ "tokio", "toml", "tower-default-headers", + "unwrap-infallible", "url", "uuid", ] @@ -2803,6 +2804,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unwrap-infallible" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "151ac09978d3c2862c4e39b557f4eceee2cc72150bc4cb4f16abf061b6e381fb" + [[package]] name = "url" version = "2.5.4" diff --git a/Cargo.toml b/Cargo.toml index 51feb3b5..235cdab9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,7 @@ tower-default-headers = "0.2.0" reqwest = { version = "0.12.15", features = ["stream", "json", "gzip"] } strfmt = "0.2.4" axum-extra = { version = "0.10.0", features = ["cookie"] } +unwrap-infallible = "0.1.5" [dev-dependencies] diff --git a/src/main.rs b/src/main.rs index e9024d74..185f9529 100644 --- a/src/main.rs +++ b/src/main.rs @@ -538,6 +538,7 @@ async fn main() { "/u/{name}", get(|axum::extract::Path(name): axum::extract::Path| async move { axum::response::Redirect::temporary(format!("/user/{}", name).as_str()) }), ) + .route("/u/{:name}/comments/{:id}/{:title}", get(post::itemx)) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); diff --git a/src/post.rs b/src/post.rs index 36e1a200..b522cab2 100644 --- a/src/post.rs +++ b/src/post.rs @@ -1,11 +1,12 @@ #![allow(clippy::cmp_owned)] // CRATES -use crate::client::json; -use crate::config::get_setting; +use crate::client::{json, jsonx}; use crate::server::RequestExt; use crate::subreddit::{can_access_quarantine, quarantine}; -use crate::utils::{cookie_jar_from_oldreq, error, get_filters, nsfw_landing, param, parse_post, setting, setting_from_cookiejar, template, Comment, Post, Preferences}; +use crate::utils::{ + cookie_jar_from_oldreq, error, get_filters, get_filtersx, nsfw_landing, param, parse_post, setting, setting_from_cookiejar, template, Comment, Post, Preferences, +}; use hyper::{Body, Request, Response}; use axum::RequestExt as AxumRequestExt; @@ -33,6 +34,63 @@ struct PostTemplate { static COMMENT_SEARCH_CAPTURE: Lazy = Lazy::new(|| Regex::new(r"\?q=(.*)&type=comment").unwrap()); +pub async fn itemx( + axum::extract::Path((name, id, title, comment_id)): axum::extract::Path<(String, String, String, Option)>, + axum::extract::RawQuery(raw_query): axum::extract::RawQuery, + axum::extract::Query(query): axum::extract::Query>, + cookies: CookieJar, + mut req: axum::extract::Request, +) -> impl axum::response::IntoResponse { + let mut url: String = format!("u/{name}/comments/{id}/{title}.json?{}&raw_json=1", raw_query.unwrap_or_default()); //FIXME: /u or /r?; Query? + + let quarantined: bool = setting_from_cookiejar(&cookies, &format!("allow_quaran_{}", name.to_lowercase())) + .parse::() + .unwrap_or_default(); // default is false + + // Set sort to sort query parameter + let sort: Cow = query + .get("sort") // NOTE: as a cookie value 'y', not a whole 'x=y' parameter + .map(Cow::from) + .unwrap_or_else(|| { + // Grab default comment sort method from Cookies + let res = setting_from_cookiejar(&cookies, "comment_sort"); + if !res.is_empty() { + // If the query does not have a sort parameter, add it so that it can be forwarded to reddit + url.push_str("&sort="); // NOTE: path already has '?' to start query parameters. + url.push_str(res.as_ref()); + } + res + }); + + let json = jsonx(url, quarantined).await?; + + let post = parse_post(&json[0]["data"]["children"][0]).await; + + if post.nsfw && crate::utils::should_be_nsfw_gatedx(&cookies, req.extract_parts::().await.unwrap_infallible()) { + return Ok("nsfw_landing"); // FIXME + } + + let comments = match query.get("q").map(String::as_str) { + None | Some("") => parse_comments( + &json[1], + &post.permalink, + &post.author.name, + &comment_id.unwrap_or_default(), + &get_filtersx(&cookies), + &cookies, + ), + Some(pattern) => query_comments( + &json[1], + &post.permalink, + &post.author.name, + &comment_id.unwrap_or_default(), + &get_filtersx(&cookies), + pattern, + &cookies, + ), + }; + Ok::<_, http_api_problem::ApiError>("Response from post and comment struct") // FIXME +} pub async fn item(req: Request) -> Result, String> { // Build Reddit API path let mut path: String = format!("{}.json?{}&raw_json=1", req.uri().path(), req.uri().query().unwrap_or_default()); diff --git a/src/utils.rs b/src/utils.rs index e3d2ddd1..576de504 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -8,6 +8,7 @@ use std::borrow::Cow; // use crate::client::CLIENTX; use crate::{client::json, server::RequestExt}; +use axum::extract::RawQuery; use axum_extra::extract::CookieJar; use hyper::{Body, Request, Response}; use libflate::deflate::{Decoder, Encoder}; @@ -495,7 +496,7 @@ pub struct Comment { } impl Comment { - fn build( + pub fn build( comment: &serde_json::Value, data: &serde_json::Value, replies: Vec, @@ -871,47 +872,6 @@ impl Preferences { } } - pub fn build(cookies: &CookieJar) -> Self { - // Read available theme names from embedded css files. - // Always make the default "system" theme available. - /*let mut themes = vec!["system".to_string()]; - for file in ThemeAssets::iter() { - let chunks: Vec<&str> = file.as_ref().split(".css").collect(); - themes.push(chunks[0].to_owned()); - }*/ - static THEMES: OnceLock> = OnceLock::new(); - Self { - available_themes: THEMES - .get_or_init(|| ThemeAssets::iter().map(|f| f.split(".css").collect::>()[0].to_owned()).collect()) - .clone(), - theme: setting_from_cookiejar(cookies, "theme").into_owned(), - front_page: setting_from_cookiejar(cookies, "front_page").into_owned(), - layout: setting_from_cookiejar(cookies, "layout").into_owned(), - wide: setting_from_cookiejar(cookies, "wide").into_owned(), - blur_spoiler: setting_from_cookiejar(cookies, "blur_spoiler").into_owned(), - show_nsfw: setting_from_cookiejar(cookies, "show_nsfw").into_owned(), - hide_sidebar_and_summary: setting_from_cookiejar(cookies, "hide_sidebar_and_summary").into_owned(), - blur_nsfw: setting_from_cookiejar(cookies, "blur_nsfw").into_owned(), - use_hls: setting_from_cookiejar(cookies, "use_hls").into_owned(), - hide_hls_notification: setting_from_cookiejar(cookies, "hide_hls_notification").into_owned(), - video_quality: setting_from_cookiejar(cookies, "video_quality").into_owned(), - autoplay_videos: setting_from_cookiejar(cookies, "autoplay_videos").into_owned(), - fixed_navbar: setting_from_cookiejar_or_default(cookies, "fixed_navbar", Cow::from("on")).into_owned(), - disable_visit_reddit_confirmation: setting_from_cookiejar(cookies, "disable_visit_reddit_confirmation").into_owned(), - comment_sort: setting_from_cookiejar(cookies, "comment_sort").into_owned(), - post_sort: setting_from_cookiejar(cookies, "post_sort").into_owned(), - subscriptions: setting_from_cookiejar(cookies, "subscriptions") - .split('+') - .filter(|s| !s.is_empty()) - .map(String::from) - .collect(), - filters: setting_from_cookiejar(cookies, "filters").split('+').filter(|s| !s.is_empty()).map(String::from).collect(), - hide_awards: setting_from_cookiejar(cookies, "hide_awards").into_owned(), - hide_score: setting_from_cookiejar(cookies, "hide_score").into_owned(), - remove_default_feeds: setting_from_cookiejar(cookies, "remove_default_feeds").into_owned(), - } - } - pub fn to_urlencoded(&self) -> Result { serde_urlencoded::to_string(self) } @@ -1592,6 +1552,16 @@ pub fn should_be_nsfw_gated(req: &Request, req_url: &str) -> bool { gate_nsfw && !bypass_gate } +pub fn should_be_nsfw_gatedx(cookies: &CookieJar, RawQuery(query): RawQuery) -> bool { + let sfw_instance = sfw_only(); + let gate_nsfw = setting_from_cookiejar(cookies, "show_nsfw") != "on"; + + // Nsfw landing gate should not be bypassed on a sfw only instance, + let bypass_gate: bool = query.map_or(false, |s| s.starts_with("bypass_nsfw_landing=") || s.contains("&bypass_nsfw_landing=")); // TODO: Test if the equals sign breaks the pattern matching + + sfw_instance || (gate_nsfw && !bypass_gate) +} + /// Renders the landing page for NSFW content when the user has not enabled /// "show NSFW posts" in settings. pub async fn nsfw_landing(req: Request, req_url: String) -> Result, String> { From 7eea58eee27811f03dd93620dc117bd1f20e1884 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 00:07:32 +0100 Subject: [PATCH 35/52] Create a PathParameters struct --- src/post.rs | 16 +++++++--------- src/utils.rs | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/post.rs b/src/post.rs index b522cab2..357f0462 100644 --- a/src/post.rs +++ b/src/post.rs @@ -4,9 +4,7 @@ use crate::client::{json, jsonx}; use crate::server::RequestExt; use crate::subreddit::{can_access_quarantine, quarantine}; -use crate::utils::{ - cookie_jar_from_oldreq, error, get_filters, get_filtersx, nsfw_landing, param, parse_post, setting, setting_from_cookiejar, template, Comment, Post, Preferences, -}; +use crate::utils::{cookie_jar_from_oldreq, error, get_filters, get_filtersx, nsfw_landing, param, parse_post, setting, setting_from_cookiejar, template, Comment, PathParameters, Post, Preferences}; use hyper::{Body, Request, Response}; use axum::RequestExt as AxumRequestExt; @@ -35,15 +33,15 @@ struct PostTemplate { static COMMENT_SEARCH_CAPTURE: Lazy = Lazy::new(|| Regex::new(r"\?q=(.*)&type=comment").unwrap()); pub async fn itemx( - axum::extract::Path((name, id, title, comment_id)): axum::extract::Path<(String, String, String, Option)>, + axum::extract::Path(parameters): axum::extract::Path, axum::extract::RawQuery(raw_query): axum::extract::RawQuery, - axum::extract::Query(query): axum::extract::Query>, + query: axum::extract::Query>, cookies: CookieJar, mut req: axum::extract::Request, ) -> impl axum::response::IntoResponse { - let mut url: String = format!("u/{name}/comments/{id}/{title}.json?{}&raw_json=1", raw_query.unwrap_or_default()); //FIXME: /u or /r?; Query? + let mut url: String = format!("u/{}/comments/{}/{}.json?{}&raw_json=1", parameters.name, parameters.id, parameters.title, raw_query.unwrap_or_default()); //FIXME: /u or /r?; Query? - let quarantined: bool = setting_from_cookiejar(&cookies, &format!("allow_quaran_{}", name.to_lowercase())) + let quarantined: bool = setting_from_cookiejar(&cookies, &format!("allow_quaran_{}", parameters.name.to_lowercase())) .parse::() .unwrap_or_default(); // default is false @@ -75,7 +73,7 @@ pub async fn itemx( &json[1], &post.permalink, &post.author.name, - &comment_id.unwrap_or_default(), + ¶meters.comment_id.unwrap_or_default(), &get_filtersx(&cookies), &cookies, ), @@ -83,7 +81,7 @@ pub async fn itemx( &json[1], &post.permalink, &post.author.name, - &comment_id.unwrap_or_default(), + ¶meters.comment_id.unwrap_or_default(), &get_filtersx(&cookies), pattern, &cookies, diff --git a/src/utils.rs b/src/utils.rs index 576de504..59d8f5a1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -27,6 +27,7 @@ use std::io::{Read, Write}; use std::str::FromStr; use std::string::ToString; use std::sync::OnceLock; +use axum::response::IntoResponse; use time::{macros::format_description, Duration, OffsetDateTime}; use url::Url; @@ -1564,6 +1565,7 @@ pub fn should_be_nsfw_gatedx(cookies: &CookieJar, RawQuery(query): RawQuery) -> /// Renders the landing page for NSFW content when the user has not enabled /// "show NSFW posts" in settings. +/// DEPRECATED pub async fn nsfw_landing(req: Request, req_url: String) -> Result, String> { let res_type: ResourceType; @@ -1592,6 +1594,22 @@ pub async fn nsfw_landing(req: Request, req_url: String) -> Result>, req_url: String) -> impl IntoResponse { + let res_type: ResourceType; + + // Determine from the request URL if the resource is a subreddit, a user + // page, or a post. +} + +#[derive(Deserialize)] +pub struct PathParameters { + pub name: String, + pub id: String, + pub title: String, + pub comment_id: Option, +} + + // Returns the last (non-empty) segment of a path string pub fn url_path_basename(path: &str) -> String { let url_result = Url::parse(format!("https://libredd.it/{path}").as_str()); From 4b9f940a7025032af73012feab0b46e8b070eb48 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 02:08:33 +0100 Subject: [PATCH 36/52] Deps: Migrate rinja -> askama --- Cargo.lock | 99 +++++++++++++++++++++++--------------------- Cargo.toml | 2 +- src/duplicates.rs | 2 +- src/instance_info.rs | 2 +- src/post.rs | 15 +++++-- src/search.rs | 2 +- src/settings.rs | 2 +- src/subreddit.rs | 2 +- src/user.rs | 2 +- src/utils.rs | 50 +++++++++++++++------- 10 files changed, 106 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3402f2f6..b9b09c19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,6 +77,48 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "askama" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a4e46abb203e00ef226442d452769233142bbfdd79c3941e84c8e61c4112543" +dependencies = [ + "askama_derive", + "itoa", + "percent-encoding", + "serde", + "serde_json", +] + +[[package]] +name = "askama_derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54398906821fd32c728135f7b351f0c7494ab95ae421d41b6f5a020e158f28a6" +dependencies = [ + "askama_parser", + "basic-toml", + "memchr", + "proc-macro2", + "quote", + "rustc-hash", + "serde", + "serde_derive", + "syn 2.0.99", +] + +[[package]] +name = "askama_parser" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf315ce6524c857bb129ff794935cf6d42c82a6cff60526fe2a63593de4d0d4f" +dependencies = [ + "memchr", + "serde", + "serde_derive", + "winnow", +] + [[package]] name = "async-compression" version = "0.4.22" @@ -246,6 +288,15 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + [[package]] name = "bincode" version = "1.3.3" @@ -1410,16 +1461,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1747,6 +1788,7 @@ name = "redlib" version = "0.36.0" dependencies = [ "arc-swap", + "askama", "async-recursion", "axum", "axum-extra", @@ -1777,7 +1819,6 @@ dependencies = [ "regex", "reqwest", "revision", - "rinja", "route-recognizer", "rss", "rust-embed", @@ -1917,42 +1958,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rinja" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc4940d00595430b3d7d5a01f6222b5e5b51395d1120bdb28d854bb8abb17a5" -dependencies = [ - "itoa", - "rinja_derive", -] - -[[package]] -name = "rinja_derive" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d9ed0146aef6e2825f1b1515f074510549efba38d71f4554eec32eb36ba18b" -dependencies = [ - "memchr", - "mime", - "mime_guess", - "proc-macro2", - "quote", - "rinja_parser", - "rustc-hash", - "syn 2.0.99", -] - -[[package]] -name = "rinja_parser" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f9a866e2e00a7a1fb27e46e9e324a6f7c0e7edc4543cae1d38f4e4a100c610" -dependencies = [ - "memchr", - "nom", -] - [[package]] name = "rle-decode-fast" version = "1.0.3" diff --git a/Cargo.toml b/Cargo.toml index 235cdab9..c9341333 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,6 @@ edition = "2021" default-run = "redlib" [dependencies] -rinja = { version = "0.3.4", default-features = false } cached = { version = "0.54.0", features = ["async"] } clap = { version = "4.4.11", default-features = false, features = [ "std", @@ -64,6 +63,7 @@ reqwest = { version = "0.12.15", features = ["stream", "json", "gzip"] } strfmt = "0.2.4" axum-extra = { version = "0.10.0", features = ["cookie"] } unwrap-infallible = "0.1.5" +askama = "0.13.0" [dev-dependencies] diff --git a/src/duplicates.rs b/src/duplicates.rs index b533198c..5a7653ef 100644 --- a/src/duplicates.rs +++ b/src/duplicates.rs @@ -5,8 +5,8 @@ use crate::server::RequestExt; use crate::subreddit::{can_access_quarantine, quarantine}; use crate::utils::{error, filter_posts, get_filters, nsfw_landing, parse_post, template, Post, Preferences}; +use askama::Template; use hyper::{Body, Request, Response}; -use rinja::Template; use serde_json::Value; use std::borrow::ToOwned; use std::collections::HashSet; diff --git a/src/instance_info.rs b/src/instance_info.rs index a573953d..6fd165b5 100644 --- a/src/instance_info.rs +++ b/src/instance_info.rs @@ -3,10 +3,10 @@ use crate::{ server::RequestExt, utils::{ErrorTemplate, Preferences}, }; +use askama::Template; use build_html::{Container, Html, HtmlContainer, Table}; use hyper::{http::Error, Body, Request, Response}; use once_cell::sync::Lazy; -use rinja::Template; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; diff --git a/src/post.rs b/src/post.rs index 357f0462..891bb8ab 100644 --- a/src/post.rs +++ b/src/post.rs @@ -4,14 +4,17 @@ use crate::client::{json, jsonx}; use crate::server::RequestExt; use crate::subreddit::{can_access_quarantine, quarantine}; -use crate::utils::{cookie_jar_from_oldreq, error, get_filters, get_filtersx, nsfw_landing, param, parse_post, setting, setting_from_cookiejar, template, Comment, PathParameters, Post, Preferences}; +use crate::utils::{ + cookie_jar_from_oldreq, error, get_filters, get_filtersx, nsfw_landing, param, parse_post, setting, setting_from_cookiejar, template, Comment, PathParameters, Post, + Preferences, +}; use hyper::{Body, Request, Response}; +use askama::Template; use axum::RequestExt as AxumRequestExt; use axum_extra::extract::cookie::CookieJar; use once_cell::sync::Lazy; use regex::Regex; -use rinja::Template; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use unwrap_infallible::UnwrapInfallible; @@ -39,7 +42,13 @@ pub async fn itemx( cookies: CookieJar, mut req: axum::extract::Request, ) -> impl axum::response::IntoResponse { - let mut url: String = format!("u/{}/comments/{}/{}.json?{}&raw_json=1", parameters.name, parameters.id, parameters.title, raw_query.unwrap_or_default()); //FIXME: /u or /r?; Query? + let mut url: String = format!( + "u/{}/comments/{}/{}.json?{}&raw_json=1", + parameters.name, + parameters.id, + parameters.title, + raw_query.unwrap_or_default() + ); //FIXME: /u or /r?; Query? let quarantined: bool = setting_from_cookiejar(&cookies, &format!("allow_quaran_{}", parameters.name.to_lowercase())) .parse::() diff --git a/src/search.rs b/src/search.rs index 88dcfdd8..34597605 100644 --- a/src/search.rs +++ b/src/search.rs @@ -7,10 +7,10 @@ use crate::{ server::RequestExt, subreddit::{can_access_quarantine, quarantine}, }; +use askama::Template; use hyper::{Body, Request, Response}; use once_cell::sync::Lazy; use regex::Regex; -use rinja::Template; // STRUCTS struct SearchParams { diff --git a/src/settings.rs b/src/settings.rs index e144b78e..3be15360 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -6,11 +6,11 @@ use std::collections::HashMap; use crate::server::ResponseExt; use crate::subreddit::join_until_size_limit; use crate::utils::{deflate_decompress, redirect, template, Preferences}; +use askama::Template; use cookie::Cookie; use futures_lite::StreamExt; use http_api_problem::ApiError; use hyper::{Body, Request, Response}; -use rinja::Template; use time::{Duration, OffsetDateTime}; use tokio::time::timeout; use url::form_urlencoded; diff --git a/src/subreddit.rs b/src/subreddit.rs index f84cca31..3d49374b 100644 --- a/src/subreddit.rs +++ b/src/subreddit.rs @@ -7,10 +7,10 @@ use crate::utils::{ Subreddit, }; use crate::{client::json, server::RequestExt, server::ResponseExt}; +use askama::Template; use cookie::Cookie; use htmlescape::decode_html; use hyper::{Body, Request, Response}; -use rinja::Template; use chrono::DateTime; use once_cell::sync::Lazy; diff --git a/src/user.rs b/src/user.rs index 592389d7..de870af1 100644 --- a/src/user.rs +++ b/src/user.rs @@ -5,10 +5,10 @@ use crate::client::json; use crate::server::RequestExt; use crate::utils::{error, filter_posts, format_url, get_filters, nsfw_landing, param, setting, template, Post, Preferences, User}; use crate::{config, utils}; +use askama::Template; use chrono::DateTime; use htmlescape::decode_html; use hyper::{Body, Request, Response}; -use rinja::Template; use time::{macros::format_description, OffsetDateTime}; // STRUCTS diff --git a/src/utils.rs b/src/utils.rs index 59d8f5a1..2dc46f53 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -8,7 +8,10 @@ use std::borrow::Cow; // use crate::client::CLIENTX; use crate::{client::json, server::RequestExt}; -use axum::extract::RawQuery; +use askama::Template; +use axum::extract::{FromRequestParts, RawQuery}; +use axum::http::request::Parts; +use axum::response::IntoResponse; use axum_extra::extract::CookieJar; use hyper::{Body, Request, Response}; use libflate::deflate::{Decoder, Encoder}; @@ -16,18 +19,17 @@ use log::error; use once_cell::sync::Lazy; use regex::Regex; use revision::revisioned; -use rinja::Template; use rust_embed::RustEmbed; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; use serde_json_path::{JsonPath, JsonPathExt}; use std::collections::{HashMap, HashSet}; +use std::convert::Infallible; use std::env; use std::io::{Read, Write}; use std::str::FromStr; use std::string::ToString; use std::sync::OnceLock; -use axum::response::IntoResponse; use time::{macros::format_description, Duration, OffsetDateTime}; use url::Url; @@ -773,6 +775,19 @@ pub struct Preferences { pub remove_default_feeds: String, } +impl FromRequestParts for Preferences +where + S: Send + Sync, +{ + type Rejection = Infallible; + + async fn from_request_parts(parts: &mut Parts, _: &S) -> Result { + Ok( + Preferences::build(parts.extensions.get::().unwrap_or(&CookieJar::new())) + ) + } +} + fn serialize_vec_with_plus(vec: &[String], serializer: S) -> Result where S: Serializer, @@ -835,11 +850,6 @@ impl Preferences { pub fn build(cookies: &CookieJar) -> Self { // Read available theme names from embedded css files. // Always make the default "system" theme available. - /*let mut themes = vec!["system".to_string()]; - for file in ThemeAssets::iter() { - let chunks: Vec<&str> = file.as_ref().split(".css").collect(); - themes.push(chunks[0].to_owned()); - }*/ static THEMES: OnceLock> = OnceLock::new(); Self { available_themes: THEMES @@ -1594,12 +1604,7 @@ pub async fn nsfw_landing(req: Request, req_url: String) -> Result>, req_url: String) -> impl IntoResponse { - let res_type: ResourceType; - - // Determine from the request URL if the resource is a subreddit, a user - // page, or a post. -} +pub async fn nsfw_landingx(cookies: &CookieJar, query: axum::extract::Query>, req_url: String) -> impl IntoResponse {} #[derive(Deserialize)] pub struct PathParameters { @@ -1607,9 +1612,24 @@ pub struct PathParameters { pub id: String, pub title: String, pub comment_id: Option, + pub sub: Option, +} + +impl PathParameters { + /// Different paths correspond to different resource types + pub fn resource_type(&self) -> Option { + if !self.name.is_empty() { + Some(ResourceType::User) + } else if !self.id.is_empty() { + Some(ResourceType::Post) + } else if self.sub.is_some() { + Some(ResourceType::Subreddit) + } else { + None + } + } } - // Returns the last (non-empty) segment of a path string pub fn url_path_basename(path: &str) -> String { let url_result = Url::parse(format!("https://libredd.it/{path}").as_str()); From a70896e39af16369af5ab431330cca099c9df70a Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 03:33:39 +0100 Subject: [PATCH 37/52] Create nsfw_landingx --- src/post.rs | 2 +- src/utils.rs | 43 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/post.rs b/src/post.rs index 891bb8ab..b4cad0e6 100644 --- a/src/post.rs +++ b/src/post.rs @@ -73,7 +73,7 @@ pub async fn itemx( let post = parse_post(&json[0]["data"]["children"][0]).await; - if post.nsfw && crate::utils::should_be_nsfw_gatedx(&cookies, req.extract_parts::().await.unwrap_infallible()) { + if post.nsfw && crate::utils::should_be_nsfw_gatedx(&cookies, &query) { return Ok("nsfw_landing"); // FIXME } diff --git a/src/utils.rs b/src/utils.rs index 2dc46f53..2b79be7a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -9,10 +9,11 @@ use std::borrow::Cow; use crate::client::CLIENTX; use crate::{client::json, server::RequestExt}; use askama::Template; -use axum::extract::{FromRequestParts, RawQuery}; +use axum::extract::{FromRequestParts, Path, Query}; use axum::http::request::Parts; use axum::response::IntoResponse; use axum_extra::extract::CookieJar; +use http_api_problem::ApiError; use hyper::{Body, Request, Response}; use libflate::deflate::{Decoder, Encoder}; use log::error; @@ -782,9 +783,7 @@ where type Rejection = Infallible; async fn from_request_parts(parts: &mut Parts, _: &S) -> Result { - Ok( - Preferences::build(parts.extensions.get::().unwrap_or(&CookieJar::new())) - ) + Ok(Preferences::build(parts.extensions.get::().unwrap_or(&CookieJar::new()))) } } @@ -1563,12 +1562,12 @@ pub fn should_be_nsfw_gated(req: &Request, req_url: &str) -> bool { gate_nsfw && !bypass_gate } -pub fn should_be_nsfw_gatedx(cookies: &CookieJar, RawQuery(query): RawQuery) -> bool { +pub fn should_be_nsfw_gatedx(cookies: &CookieJar, query: &Query>) -> bool { let sfw_instance = sfw_only(); let gate_nsfw = setting_from_cookiejar(cookies, "show_nsfw") != "on"; // Nsfw landing gate should not be bypassed on a sfw only instance, - let bypass_gate: bool = query.map_or(false, |s| s.starts_with("bypass_nsfw_landing=") || s.contains("&bypass_nsfw_landing=")); // TODO: Test if the equals sign breaks the pattern matching + let bypass_gate: bool = query.get("bypass_nsfw_landing").is_some(); sfw_instance || (gate_nsfw && !bypass_gate) } @@ -1604,7 +1603,37 @@ pub async fn nsfw_landing(req: Request, req_url: String) -> Result>, req_url: String) -> impl IntoResponse {} +pub async fn nsfw_landingx( + prefs: Preferences, + Path(params): Path, + axum::extract::OriginalUri(uri): axum::extract::OriginalUri, +) -> Result { + let res_type = params.resource_type().unwrap_or(ResourceType::Subreddit); + let res = match res_type { + ResourceType::User => params.name, + ResourceType::Post => params.id, + ResourceType::Subreddit => params.sub.unwrap_or_default(), + }; + let body = NSFWLandingTemplate { + res, + res_type, + prefs, + url: uri.to_string(), + } + .render() //render into HTML String + // Handle rendering errors + .map_err(|e| { + ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) + .title("Error rendering NSFW landing page") + .message(&e) + .source(e) + .finish() + })?; + Ok(( + axum::http::status::StatusCode::FORBIDDEN, // set status code 403 + axum::response::Html(body), + )) +} #[derive(Deserialize)] pub struct PathParameters { From 114a244438bffb96ff7f94ced641c22b61f8891d Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 05:27:38 +0100 Subject: [PATCH 38/52] Use Preference struct as extractor --- src/post.rs | 28 +++++++++++++++++----------- src/utils.rs | 5 +++-- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/post.rs b/src/post.rs index b4cad0e6..c2403e0c 100644 --- a/src/post.rs +++ b/src/post.rs @@ -5,14 +5,17 @@ use crate::client::{json, jsonx}; use crate::server::RequestExt; use crate::subreddit::{can_access_quarantine, quarantine}; use crate::utils::{ - cookie_jar_from_oldreq, error, get_filters, get_filtersx, nsfw_landing, param, parse_post, setting, setting_from_cookiejar, template, Comment, PathParameters, Post, - Preferences, + cookie_jar_from_oldreq, error, get_filters, get_filtersx, nsfw_landing, nsfw_landingx, param, parse_post, setting, setting_from_cookiejar, template, Comment, + PathParameters, Post, Preferences, }; +use axum::response::IntoResponse; use hyper::{Body, Request, Response}; use askama::Template; +use axum::extract::{OriginalUri, Path}; use axum::RequestExt as AxumRequestExt; use axum_extra::extract::cookie::CookieJar; +use http_api_problem::ApiError; use once_cell::sync::Lazy; use regex::Regex; use std::borrow::Cow; @@ -40,8 +43,10 @@ pub async fn itemx( axum::extract::RawQuery(raw_query): axum::extract::RawQuery, query: axum::extract::Query>, cookies: CookieJar, + prefs: Preferences, + original_uri: OriginalUri, mut req: axum::extract::Request, -) -> impl axum::response::IntoResponse { +) -> Result { let mut url: String = format!( "u/{}/comments/{}/{}.json?{}&raw_json=1", parameters.name, @@ -60,21 +65,22 @@ pub async fn itemx( .map(Cow::from) .unwrap_or_else(|| { // Grab default comment sort method from Cookies - let res = setting_from_cookiejar(&cookies, "comment_sort"); + let res = &prefs.comment_sort; if !res.is_empty() { // If the query does not have a sort parameter, add it so that it can be forwarded to reddit url.push_str("&sort="); // NOTE: path already has '?' to start query parameters. - url.push_str(res.as_ref()); + url.push_str(res); } - res + Cow::from(res) }); let json = jsonx(url, quarantined).await?; let post = parse_post(&json[0]["data"]["children"][0]).await; - if post.nsfw && crate::utils::should_be_nsfw_gatedx(&cookies, &query) { - return Ok("nsfw_landing"); // FIXME + if post.nsfw && crate::utils::should_be_nsfw_gatedx(&prefs, &query) { + // nsfw_landingx is an axum::Handler, but we don't have to reallocate memory + return Ok(nsfw_landingx(prefs, Path(parameters), original_uri).await?.into_response()); } let comments = match query.get("q").map(String::as_str) { @@ -83,7 +89,7 @@ pub async fn itemx( &post.permalink, &post.author.name, ¶meters.comment_id.unwrap_or_default(), - &get_filtersx(&cookies), + &HashSet::from_iter(prefs.filters), &cookies, ), Some(pattern) => query_comments( @@ -91,12 +97,12 @@ pub async fn itemx( &post.permalink, &post.author.name, ¶meters.comment_id.unwrap_or_default(), - &get_filtersx(&cookies), + &HashSet::from_iter(prefs.filters), pattern, &cookies, ), }; - Ok::<_, http_api_problem::ApiError>("Response from post and comment struct") // FIXME + Ok::<_, ApiError>("Response from post and comment struct".into_response()) // FIXME } pub async fn item(req: Request) -> Result, String> { // Build Reddit API path diff --git a/src/utils.rs b/src/utils.rs index 2b79be7a..d612f91f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -927,6 +927,7 @@ pub fn get_filters(req: &Request) -> HashSet { setting(req, "filters").split('+').map(String::from).filter(|s| !s.is_empty()).collect::>() } +// If available, use Preferences.filters pub fn get_filtersx(cookies: &CookieJar) -> HashSet { setting_from_cookiejar(cookies, "filters") .split('+') @@ -1562,9 +1563,9 @@ pub fn should_be_nsfw_gated(req: &Request, req_url: &str) -> bool { gate_nsfw && !bypass_gate } -pub fn should_be_nsfw_gatedx(cookies: &CookieJar, query: &Query>) -> bool { +pub fn should_be_nsfw_gatedx(prefs: &Preferences, query: &Query>) -> bool { let sfw_instance = sfw_only(); - let gate_nsfw = setting_from_cookiejar(cookies, "show_nsfw") != "on"; + let gate_nsfw = prefs.show_nsfw != "on"; // Nsfw landing gate should not be bypassed on a sfw only instance, let bypass_gate: bool = query.get("bypass_nsfw_landing").is_some(); From 517370d7ef69c67022d76d14081b224261ab151e Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 07:00:03 +0100 Subject: [PATCH 39/52] Improve memory sharing Preferences --- src/post.rs | 66 ++++++++++++---------------------------------------- src/utils.rs | 25 +++++++++----------- 2 files changed, 26 insertions(+), 65 deletions(-) diff --git a/src/post.rs b/src/post.rs index c2403e0c..e2027d56 100644 --- a/src/post.rs +++ b/src/post.rs @@ -5,8 +5,7 @@ use crate::client::{json, jsonx}; use crate::server::RequestExt; use crate::subreddit::{can_access_quarantine, quarantine}; use crate::utils::{ - cookie_jar_from_oldreq, error, get_filters, get_filtersx, nsfw_landing, nsfw_landingx, param, parse_post, setting, setting_from_cookiejar, template, Comment, - PathParameters, Post, Preferences, + cookie_jar_from_oldreq, error, nsfw_landing, nsfw_landingx, param, parse_post, setting_from_cookiejar, template, Comment, PathParameters, Post, Preferences, }; use axum::response::IntoResponse; use hyper::{Body, Request, Response}; @@ -19,7 +18,8 @@ use http_api_problem::ApiError; use once_cell::sync::Lazy; use regex::Regex; use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; +use std::sync::Arc; use unwrap_infallible::UnwrapInfallible; // STRUCTS @@ -47,6 +47,7 @@ pub async fn itemx( original_uri: OriginalUri, mut req: axum::extract::Request, ) -> Result { + let prefs = Arc::new(prefs); let mut url: String = format!( "u/{}/comments/{}/{}.json?{}&raw_json=1", parameters.name, @@ -84,27 +85,13 @@ pub async fn itemx( } let comments = match query.get("q").map(String::as_str) { - None | Some("") => parse_comments( - &json[1], - &post.permalink, - &post.author.name, - ¶meters.comment_id.unwrap_or_default(), - &HashSet::from_iter(prefs.filters), - &cookies, - ), - Some(pattern) => query_comments( - &json[1], - &post.permalink, - &post.author.name, - ¶meters.comment_id.unwrap_or_default(), - &HashSet::from_iter(prefs.filters), - pattern, - &cookies, - ), + None | Some("") => parse_comments(&json[1], &post.permalink, &post.author.name, ¶meters.comment_id.unwrap_or_default(), prefs), + Some(pattern) => query_comments(&json[1], &post.permalink, &post.author.name, ¶meters.comment_id.unwrap_or_default(), prefs, pattern), }; Ok::<_, ApiError>("Response from post and comment struct".into_response()) // FIXME } pub async fn item(req: Request) -> Result, String> { + let prefs = Arc::new(Preferences::build(&cookie_jar_from_oldreq(&req))); // Build Reddit API path let mut path: String = format!("{}.json?{}&raw_json=1", req.uri().path(), req.uri().query().unwrap_or_default()); let sub = req.param("sub").unwrap_or_default(); @@ -114,7 +101,7 @@ pub async fn item(req: Request) -> Result, String> { // Set sort to sort query parameter let sort = param(&path, "sort").unwrap_or_else(|| { // Grab default comment sort method from Cookies - let default_sort = setting(&req, "comment_sort"); + let default_sort = prefs.comment_sort.clone(); // If there's no sort query but there's a default sort, set sort to default_sort if default_sort.is_empty() { @@ -157,23 +144,8 @@ pub async fn item(req: Request) -> Result, String> { let query = form.get("q").unwrap().clone().to_string(); let comments = match query.as_str() { - "" => parse_comments( - &response[1], - &post.permalink, - &post.author.name, - highlighted_comment, - &get_filters(&req), - &cookie_jar_from_oldreq(&req), - ), - _ => query_comments( - &response[1], - &post.permalink, - &post.author.name, - highlighted_comment, - &get_filters(&req), - &query, - &cookie_jar_from_oldreq(&req), - ), + "" => parse_comments(&response[1], &post.permalink, &post.author.name, highlighted_comment, prefs), + _ => query_comments(&response[1], &post.permalink, &post.author.name, highlighted_comment, prefs, &query), }; // Use the Post and Comment structs to generate a website to show users @@ -203,7 +175,7 @@ pub async fn item(req: Request) -> Result, String> { // COMMENTS /// A Vec of all comments defined in a json response -fn parse_comments(json: &serde_json::Value, post_link: &str, post_author: &str, highlighted_comment: &str, filters: &HashSet, cookies: &CookieJar) -> Vec { +fn parse_comments(json: &serde_json::Value, post_link: &str, post_author: &str, highlighted_comment: &str, prefs: Arc) -> Vec { let comments = json["data"]["children"].as_array(); if let Some(comments) = comments { comments @@ -211,11 +183,11 @@ fn parse_comments(json: &serde_json::Value, post_link: &str, post_author: &str, .map(|comment| { let data = &comment["data"]; let replies: Vec = if data["replies"].is_object() { - parse_comments(&data["replies"], post_link, post_author, highlighted_comment, filters, cookies) + parse_comments(&data["replies"], post_link, post_author, highlighted_comment, prefs.clone()) } else { Vec::new() }; - Comment::build(&comment, data, replies, post_link, post_author, highlighted_comment, filters, cookies) + Comment::build(&comment, data, replies, post_link, post_author, highlighted_comment, prefs.clone()) }) .collect() } else { @@ -224,17 +196,9 @@ fn parse_comments(json: &serde_json::Value, post_link: &str, post_author: &str, } /// like parse_comments, but filters comment body by query parameter. -fn query_comments( - json: &serde_json::Value, - post_link: &str, - post_author: &str, - highlighted_comment: &str, - filters: &HashSet, - query: &str, - cookies: &CookieJar, -) -> Vec { +fn query_comments(json: &serde_json::Value, post_link: &str, post_author: &str, highlighted_comment: &str, prefs: Arc, query: &str) -> Vec { let query_lc = query.to_lowercase(); - parse_comments(json, post_link, post_author, highlighted_comment, filters, cookies) + parse_comments(json, post_link, post_author, highlighted_comment, prefs) .into_iter() .filter(|c| c.body.to_lowercase().contains(&query_lc)) .collect() diff --git a/src/utils.rs b/src/utils.rs index d612f91f..6db61888 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -30,7 +30,7 @@ use std::env; use std::io::{Read, Write}; use std::str::FromStr; use std::string::ToString; -use std::sync::OnceLock; +use std::sync::{Arc, LazyLock}; use time::{macros::format_description, Duration, OffsetDateTime}; use url::Url; @@ -496,7 +496,7 @@ pub struct Comment { pub collapsed: bool, pub is_filtered: bool, pub more_count: i64, - pub prefs: Preferences, + pub prefs: Arc, } impl Comment { @@ -507,8 +507,7 @@ impl Comment { post_link: &str, post_author: &str, highlighted_comment: &str, - filters: &HashSet, - cookies: &CookieJar, + prefs: Arc, ) -> Self { let id = val(comment, "id"); @@ -557,7 +556,7 @@ impl Comment { }, distinguished: val(comment, "distinguished"), }; - let is_filtered = filters.contains(&["u_", author.name.as_str()].concat()); + let is_filtered = prefs.filters.contains(&["u_", author.name.as_str()].concat()); // Many subreddits have a default comment posted about the sub's rules etc. // Many Redlib users do not wish to see this kind of comment by default. @@ -590,7 +589,7 @@ impl Comment { collapsed, is_filtered, more_count, - prefs: Preferences::build(cookies), + prefs, } } } @@ -680,7 +679,7 @@ pub struct NSFWLandingTemplate { pub res_type: ResourceType, /// User preferences. - pub prefs: Preferences, + pub prefs: Arc, /// Request URL. pub url: String, @@ -724,7 +723,7 @@ pub struct Params { pub before: Option, } -#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] #[revisioned(revision = 1)] pub struct Preferences { #[revision(start = 1)] @@ -849,11 +848,9 @@ impl Preferences { pub fn build(cookies: &CookieJar) -> Self { // Read available theme names from embedded css files. // Always make the default "system" theme available. - static THEMES: OnceLock> = OnceLock::new(); + static THEMES: LazyLock> = LazyLock::new(|| ThemeAssets::iter().map(|f| f.split(".css").collect::>()[0].to_owned()).collect()); Self { - available_themes: THEMES - .get_or_init(|| ThemeAssets::iter().map(|f| f.split(".css").collect::>()[0].to_owned()).collect()) - .clone(), + available_themes: THEMES.clone(), theme: setting_from_cookiejar(cookies, "theme").into_owned(), front_page: setting_from_cookiejar(cookies, "front_page").into_owned(), layout: setting_from_cookiejar(cookies, "layout").into_owned(), @@ -1595,7 +1592,7 @@ pub async fn nsfw_landing(req: Request, req_url: String) -> Result, req_url: String) -> Result, Path(params): Path, axum::extract::OriginalUri(uri): axum::extract::OriginalUri, ) -> Result { From 7c66b9cbbb4f1cd2d1d5f7bdeb8fbcd64da71880 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 07:45:48 +0100 Subject: [PATCH 40/52] Finish implementing itemx --- src/main.rs | 2 +- src/post.rs | 54 ++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/main.rs b/src/main.rs index 185f9529..9be3a156 100644 --- a/src/main.rs +++ b/src/main.rs @@ -538,7 +538,7 @@ async fn main() { "/u/{name}", get(|axum::extract::Path(name): axum::extract::Path| async move { axum::response::Redirect::temporary(format!("/user/{}", name).as_str()) }), ) - .route("/u/{:name}/comments/{:id}/{:title}", get(post::itemx)) + .route("/u/{name}/comments/{id}/{title}", get(post::itemx)) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); diff --git a/src/post.rs b/src/post.rs index e2027d56..3c65035a 100644 --- a/src/post.rs +++ b/src/post.rs @@ -7,12 +7,11 @@ use crate::subreddit::{can_access_quarantine, quarantine}; use crate::utils::{ cookie_jar_from_oldreq, error, nsfw_landing, nsfw_landingx, param, parse_post, setting_from_cookiejar, template, Comment, PathParameters, Post, Preferences, }; -use axum::response::IntoResponse; +use axum::response::{Html, IntoResponse}; use hyper::{Body, Request, Response}; use askama::Template; -use axum::extract::{OriginalUri, Path}; -use axum::RequestExt as AxumRequestExt; +use axum::extract::{OriginalUri, Path, Query, RawQuery}; use axum_extra::extract::cookie::CookieJar; use http_api_problem::ApiError; use once_cell::sync::Lazy; @@ -20,7 +19,6 @@ use regex::Regex; use std::borrow::Cow; use std::collections::HashMap; use std::sync::Arc; -use unwrap_infallible::UnwrapInfallible; // STRUCTS #[derive(Template)] @@ -29,7 +27,7 @@ struct PostTemplate { comments: Vec, post: Post, sort: String, - prefs: Preferences, + prefs: Arc, single_thread: bool, url: String, url_without_query: String, @@ -39,13 +37,12 @@ struct PostTemplate { static COMMENT_SEARCH_CAPTURE: Lazy = Lazy::new(|| Regex::new(r"\?q=(.*)&type=comment").unwrap()); pub async fn itemx( - axum::extract::Path(parameters): axum::extract::Path, - axum::extract::RawQuery(raw_query): axum::extract::RawQuery, - query: axum::extract::Query>, + Path(parameters): Path, + RawQuery(raw_query): RawQuery, + query: Query>, cookies: CookieJar, prefs: Preferences, original_uri: OriginalUri, - mut req: axum::extract::Request, ) -> Result { let prefs = Arc::new(prefs); let mut url: String = format!( @@ -85,10 +82,41 @@ pub async fn itemx( } let comments = match query.get("q").map(String::as_str) { - None | Some("") => parse_comments(&json[1], &post.permalink, &post.author.name, ¶meters.comment_id.unwrap_or_default(), prefs), - Some(pattern) => query_comments(&json[1], &post.permalink, &post.author.name, ¶meters.comment_id.unwrap_or_default(), prefs, pattern), + None | Some("") => parse_comments( + &json[1], + &post.permalink, + &post.author.name, + parameters.comment_id.as_ref().map(String::as_str).unwrap_or_default(), + prefs.clone(), + ), + Some(pattern) => query_comments( + &json[1], + &post.permalink, + &post.author.name, + parameters.comment_id.as_ref().map(String::as_str).unwrap_or_default(), + prefs.clone(), + pattern, + ), }; - Ok::<_, ApiError>("Response from post and comment struct".into_response()) // FIXME + let body = PostTemplate { + comments, + post, + url_without_query: original_uri.to_string(), + sort: sort.to_string(), + prefs: prefs.clone(), + single_thread: parameters.comment_id.is_some(), + url: original_uri.to_string(), + comment_query: query.get("q").cloned().unwrap_or_default(), + } + .render() + .map_err(|e| { + ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) + .title("Error rendering Post") + .message(&e) + .source(e) + .finish() + })?; + Ok(Html(body).into_response()) } pub async fn item(req: Request) -> Result, String> { let prefs = Arc::new(Preferences::build(&cookie_jar_from_oldreq(&req))); @@ -154,7 +182,7 @@ pub async fn item(req: Request) -> Result, String> { post, url_without_query: url.clone().trim_end_matches(&format!("?q={query}&type=comment")).to_string(), sort, - prefs: Preferences::new(&req), + prefs: Arc::new(Preferences::build(&cookie_jar_from_oldreq(&req))), single_thread, url: req_url, comment_query: query, From 172e32c00caa7f433fbcca13fe2351d447429227 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 08:10:54 +0100 Subject: [PATCH 41/52] Routing typo bugfixes --- src/client.rs | 5 ++++- src/main.rs | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/client.rs b/src/client.rs index 304ac455..9ae39a62 100644 --- a/src/client.rs +++ b/src/client.rs @@ -55,6 +55,9 @@ pub fn into_api_error(e: reqwest::Error) -> ApiError { let status = e.status().unwrap(); ApiError::try_builder(status.as_u16()) .expect("reqwest considers this HTTP status to be an error status, but http_api_problem does not.") + .title(format!("Reddit Error {}", status)) + .message(format!("{e}")) + .source(e) .finish() } else { ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) // 500 @@ -308,7 +311,7 @@ fn reddit_short_head(path: String, quarantine: bool, base_path: &'static str, ho // Unused - reddit_head is only ever called in the context of a short URL async fn reddit_request(method: reqwest::Method, path: &str, quarantine: bool, base_path: &'static str, host: &'static str) -> Result { - let url = format!("{base_path}{path}"); + let url = format!("{base_path}/{path}"); // Build request to Reddit. Reqwest handles gzip encoding. Reddit does not yet support Brotli encoding. use reqwest::header; diff --git a/src/main.rs b/src/main.rs index 9be3a156..a557e2e3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -535,10 +535,10 @@ async fn main() { .route("/static/{*path}", get(proxy!("https://www.redditstatic.com/{path}"))) // User profile .route( - "/u/{name}", + "/u/{*name}", get(|axum::extract::Path(name): axum::extract::Path| async move { axum::response::Redirect::temporary(format!("/user/{}", name).as_str()) }), ) - .route("/u/{name}/comments/{id}/{title}", get(post::itemx)) + .route("/user/{name}/comments/{id}/{title}", get(post::itemx)) .route("/", get(|| async { "hello, world!" })) .layer(DefaultHeadersLayer::new(default_headersx)); From d303c5016c1c2528d5b519bd46955c7d961d633e Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 08:17:36 +0100 Subject: [PATCH 42/52] Change reddit_getx host to oauth --- src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index 9ae39a62..f78652df 100644 --- a/src/client.rs +++ b/src/client.rs @@ -296,7 +296,7 @@ fn reddit_get(path: String, quarantine: bool) -> Boxed, St } async fn reddit_getx(path: &str, quarantine: bool) -> Result { - reddit_request(reqwest::Method::GET, path, quarantine, REDDIT_SHORT_URL_BASE, REDDIT_SHORT_URL_BASE_HOST).await + reddit_request(reqwest::Method::GET, path, quarantine, REDDIT_URL_BASE, REDDIT_URL_BASE_HOST).await } /// Makes a HEAD request to Reddit at `path, using the short URL base. This will not follow redirects. From 52085c06c8e4d367d61b7fd5393586d1cf325d38 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 08:42:44 +0100 Subject: [PATCH 43/52] Handle trailing slash in path --- Cargo.lock | 16 ++++++++++++++++ Cargo.toml | 2 ++ src/main.rs | 8 ++++++-- src/post.rs | 16 ++++++++-------- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9b09c19..d8d14cf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1833,7 +1833,9 @@ dependencies = [ "time", "tokio", "toml", + "tower", "tower-default-headers", + "tower-http", "unwrap-infallible", "url", "uuid", @@ -2741,6 +2743,20 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +dependencies = [ + "bitflags", + "bytes", + "http 1.3.1", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" diff --git a/Cargo.toml b/Cargo.toml index c9341333..e0c9252c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,8 @@ strfmt = "0.2.4" axum-extra = { version = "0.10.0", features = ["cookie"] } unwrap-infallible = "0.1.5" askama = "0.13.0" +tower = "0.5.2" +tower-http = { version = "0.6.2", features = ["normalize-path"] } [dev-dependencies] diff --git a/src/main.rs b/src/main.rs index a557e2e3..7444ee78 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,8 @@ use futures_util::future::TryFutureExt; use axum::http::header::{HeaderMap, HeaderValue as HeaderValuex}; use axum::routing::get; +use axum::ServiceExt; +use tower::Layer; use tower_default_headers::DefaultHeadersLayer; // Create Services @@ -539,12 +541,14 @@ async fn main() { get(|axum::extract::Path(name): axum::extract::Path| async move { axum::response::Redirect::temporary(format!("/user/{}", name).as_str()) }), ) .route("/user/{name}/comments/{id}/{title}", get(post::itemx)) - .route("/", get(|| async { "hello, world!" })) + .route("/user/{name}/comments/{id}/{title}/{comment_id}", get(post::itemx)) .layer(DefaultHeadersLayer::new(default_headersx)); + let appx = tower_http::normalize_path::NormalizePathLayer::trim_trailing_slash().layer(appx); + // Temporary listener for the axum server: let listenerx = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - axum::serve(listenerx, appx).await.unwrap(); + axum::serve(listenerx, ServiceExt::::into_make_service(appx)).await.unwrap(); println!("Running Redlib v{} on {listener}!", env!("CARGO_PKG_VERSION")); diff --git a/src/post.rs b/src/post.rs index 3c65035a..1450db47 100644 --- a/src/post.rs +++ b/src/post.rs @@ -108,14 +108,14 @@ pub async fn itemx( url: original_uri.to_string(), comment_query: query.get("q").cloned().unwrap_or_default(), } - .render() - .map_err(|e| { - ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) - .title("Error rendering Post") - .message(&e) - .source(e) - .finish() - })?; + .render() + .map_err(|e| { + ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) + .title("Error rendering Post") + .message(&e) + .source(e) + .finish() + })?; Ok(Html(body).into_response()) } pub async fn item(req: Request) -> Result, String> { From e69c12b24ee2f7a693a8c26189390262ba200090 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 08:51:50 +0100 Subject: [PATCH 44/52] Modify pathparameters to optional --- src/main.rs | 1 + src/post.rs | 4 ++-- src/utils.rs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 7444ee78..680ac9c5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -542,6 +542,7 @@ async fn main() { ) .route("/user/{name}/comments/{id}/{title}", get(post::itemx)) .route("/user/{name}/comments/{id}/{title}/{comment_id}", get(post::itemx)) + .route("/user/{name}/comments/{id}", get(post::itemx)) .layer(DefaultHeadersLayer::new(default_headersx)); let appx = tower_http::normalize_path::NormalizePathLayer::trim_trailing_slash().layer(appx); diff --git a/src/post.rs b/src/post.rs index 1450db47..00f03777 100644 --- a/src/post.rs +++ b/src/post.rs @@ -46,10 +46,10 @@ pub async fn itemx( ) -> Result { let prefs = Arc::new(prefs); let mut url: String = format!( - "u/{}/comments/{}/{}.json?{}&raw_json=1", + "u/{}/comments/{}{}.json?{}&raw_json=1", parameters.name, parameters.id, - parameters.title, + parameters.title.as_ref().map(|t| format!("/{}", t)).unwrap_or_default(), raw_query.unwrap_or_default() ); //FIXME: /u or /r?; Query? diff --git a/src/utils.rs b/src/utils.rs index 6db61888..5d017a83 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1637,7 +1637,7 @@ pub async fn nsfw_landingx( pub struct PathParameters { pub name: String, pub id: String, - pub title: String, + pub title: Option, pub comment_id: Option, pub sub: Option, } From b36e8c61cecc416938519baa546f070fa0a51a7a Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Sat, 5 Apr 2025 08:53:54 +0100 Subject: [PATCH 45/52] Remove unused dependency --- Cargo.lock | 7 ------- Cargo.toml | 1 - 2 files changed, 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8d14cf4..936a713b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1836,7 +1836,6 @@ dependencies = [ "tower", "tower-default-headers", "tower-http", - "unwrap-infallible", "url", "uuid", ] @@ -2825,12 +2824,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "unwrap-infallible" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "151ac09978d3c2862c4e39b557f4eceee2cc72150bc4cb4f16abf061b6e381fb" - [[package]] name = "url" version = "2.5.4" diff --git a/Cargo.toml b/Cargo.toml index e0c9252c..14345ec0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,7 +62,6 @@ tower-default-headers = "0.2.0" reqwest = { version = "0.12.15", features = ["stream", "json", "gzip"] } strfmt = "0.2.4" axum-extra = { version = "0.10.0", features = ["cookie"] } -unwrap-infallible = "0.1.5" askama = "0.13.0" tower = "0.5.2" tower-http = { version = "0.6.2", features = ["normalize-path"] } From 01556e87635b0108f4efa0c64192701e47f6f6bf Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Fri, 18 Apr 2025 12:54:18 +0100 Subject: [PATCH 46/52] Refactor nsfwlandingx less abstract --- src/post.rs | 7 +++---- src/utils.rs | 24 ++++++++---------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/post.rs b/src/post.rs index 00f03777..19bb41c5 100644 --- a/src/post.rs +++ b/src/post.rs @@ -43,7 +43,7 @@ pub async fn itemx( cookies: CookieJar, prefs: Preferences, original_uri: OriginalUri, -) -> Result { +) -> Result { let prefs = Arc::new(prefs); let mut url: String = format!( "u/{}/comments/{}{}.json?{}&raw_json=1", @@ -77,8 +77,7 @@ pub async fn itemx( let post = parse_post(&json[0]["data"]["children"][0]).await; if post.nsfw && crate::utils::should_be_nsfw_gatedx(&prefs, &query) { - // nsfw_landingx is an axum::Handler, but we don't have to reallocate memory - return Ok(nsfw_landingx(prefs, Path(parameters), original_uri).await?.into_response()); + return nsfw_landingx(prefs, parameters.id, ResourceType::Post, original_uri.to_string()).await; } let comments = match query.get("q").map(String::as_str) { @@ -116,7 +115,7 @@ pub async fn itemx( .source(e) .finish() })?; - Ok(Html(body).into_response()) + Ok(Html(body)) } pub async fn item(req: Request) -> Result, String> { let prefs = Arc::new(Preferences::build(&cookie_jar_from_oldreq(&req))); diff --git a/src/utils.rs b/src/utils.rs index 5d017a83..c7d690dc 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1603,20 +1603,15 @@ pub async fn nsfw_landing(req: Request, req_url: String) -> Result, - Path(params): Path, - axum::extract::OriginalUri(uri): axum::extract::OriginalUri, -) -> Result { - let res_type = params.resource_type().unwrap_or(ResourceType::Subreddit); - let res = match res_type { - ResourceType::User => params.name, - ResourceType::Post => params.id, - ResourceType::Subreddit => params.sub.unwrap_or_default(), - }; + resource_id: String, + resource_type: ResourceType, + uri: String, +) -> Result, ApiError> { let body = NSFWLandingTemplate { - res, - res_type, + res: resource_id, + res_type: resource_type, prefs, - url: uri.to_string(), + url: uri, } .render() //render into HTML String // Handle rendering errors @@ -1627,10 +1622,7 @@ pub async fn nsfw_landingx( .source(e) .finish() })?; - Ok(( - axum::http::status::StatusCode::FORBIDDEN, // set status code 403 - axum::response::Html(body), - )) + Ok(axum::response::Html(body)) } #[derive(Deserialize)] From afd51fd12c752bf0bd08b4d6e926e0918a1d2a18 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Mon, 21 Apr 2025 19:53:58 +0100 Subject: [PATCH 47/52] Refactor profile page --- Cargo.lock | 12 +++++ Cargo.toml | 1 + src/duplicates.rs | 4 +- src/main.rs | 2 + src/post.rs | 2 +- src/user.rs | 110 +++++++++++++++++++++++++++++++++++++++++++--- src/utils.rs | 37 ++++++++++------ 7 files changed, 146 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 936a713b..d905de89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1824,6 +1824,7 @@ dependencies = [ "rust-embed", "sealed_test", "serde", + "serde-inline-default", "serde_json", "serde_json_path", "serde_urlencoded", @@ -2237,6 +2238,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-inline-default" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59fb1bedd774187d304179493b0d3c41fbe97b04b14305363f68d2bdf5e47cb9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", +] + [[package]] name = "serde_derive" version = "1.0.218" diff --git a/Cargo.toml b/Cargo.toml index 14345ec0..07ed6883 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ axum-extra = { version = "0.10.0", features = ["cookie"] } askama = "0.13.0" tower = "0.5.2" tower-http = { version = "0.6.2", features = ["normalize-path"] } +serde-inline-default = "0.2.3" [dev-dependencies] diff --git a/src/duplicates.rs b/src/duplicates.rs index 5a7653ef..0a8cfd97 100644 --- a/src/duplicates.rs +++ b/src/duplicates.rs @@ -43,7 +43,7 @@ struct DuplicatesTemplate { /// num_posts_filtered counts how many posts were filtered from the /// duplicates list. - num_posts_filtered: u64, + num_posts_filtered: usize, /// all_posts_filtered is true if every duplicate was filtered. This is an /// edge case but can still happen. @@ -221,7 +221,7 @@ pub async fn item(req: Request) -> Result, String> { } // DUPLICATES -async fn parse_duplicates(json: &Value, filters: &HashSet) -> (Vec, u64, bool) { +async fn parse_duplicates(json: &Value, filters: &HashSet) -> (Vec, usize, bool) { let post_duplicates: &Vec = &json["data"]["children"].as_array().map_or(Vec::new(), ToOwned::to_owned); let mut duplicates: Vec = Vec::new(); diff --git a/src/main.rs b/src/main.rs index 680ac9c5..f30da4e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -543,6 +543,8 @@ async fn main() { .route("/user/{name}/comments/{id}/{title}", get(post::itemx)) .route("/user/{name}/comments/{id}/{title}/{comment_id}", get(post::itemx)) .route("/user/{name}/comments/{id}", get(post::itemx)) + .route("/user/{name}", get(user::profilex)) + .route("/user/{name}/{listing}", get(user::profilex)) .layer(DefaultHeadersLayer::new(default_headersx)); let appx = tower_http::normalize_path::NormalizePathLayer::trim_trailing_slash().layer(appx); diff --git a/src/post.rs b/src/post.rs index 19bb41c5..e96b50fa 100644 --- a/src/post.rs +++ b/src/post.rs @@ -5,7 +5,7 @@ use crate::client::{json, jsonx}; use crate::server::RequestExt; use crate::subreddit::{can_access_quarantine, quarantine}; use crate::utils::{ - cookie_jar_from_oldreq, error, nsfw_landing, nsfw_landingx, param, parse_post, setting_from_cookiejar, template, Comment, PathParameters, Post, Preferences, + cookie_jar_from_oldreq, error, nsfw_landing, nsfw_landingx, param, parse_post, setting_from_cookiejar, template, Comment, PathParameters, Post, Preferences, ResourceType, }; use axum::response::{Html, IntoResponse}; use hyper::{Body, Request, Response}; diff --git a/src/user.rs b/src/user.rs index de870af1..5dbf71a8 100644 --- a/src/user.rs +++ b/src/user.rs @@ -3,12 +3,19 @@ // CRATES use crate::client::json; use crate::server::RequestExt; -use crate::utils::{error, filter_posts, format_url, get_filters, nsfw_landing, param, setting, template, Post, Preferences, User}; +use crate::utils::{error, filter_posts, format_url, get_filters, nsfw_landing, nsfw_landingx, param, setting, template, Post, Preferences, ResourceType, User}; use crate::{config, utils}; use askama::Template; +use axum::extract::{OriginalUri, Path, Query, RawQuery}; +use axum::response::Html; use chrono::DateTime; use htmlescape::decode_html; +use http_api_problem::ApiError; use hyper::{Body, Request, Response}; +use serde::Deserialize; +use serde_inline_default::serde_inline_default; +use std::collections::HashMap; +use std::sync::Arc; use time::{macros::format_description, OffsetDateTime}; // STRUCTS @@ -20,8 +27,8 @@ struct UserTemplate { sort: (String, String), ends: (String, String), /// "overview", "comments", or "submitted" - listing: String, - prefs: Preferences, + listing: String, // TODO turn into an enum + prefs: Arc, url: String, redirect_url: String, /// Whether the user themself is filtered. @@ -70,7 +77,7 @@ pub async fn profile(req: Request) -> Result, String> { sort: (sort, param(&path, "t").unwrap_or_default()), ends: (param(&path, "after").unwrap_or_default(), String::new()), listing, - prefs: Preferences::new(&req), + prefs: Arc::new(Preferences::new(&req)), url, redirect_url, is_filtered: true, @@ -91,7 +98,7 @@ pub async fn profile(req: Request) -> Result, String> { sort: (sort, param(&path, "t").unwrap_or_default()), ends: (param(&path, "after").unwrap_or_default(), after), listing, - prefs: Preferences::new(&req), + prefs: Arc::new(Preferences::new(&req)), url, redirect_url, is_filtered: false, @@ -106,7 +113,100 @@ pub async fn profile(req: Request) -> Result, String> { } } +#[serde_inline_default] +#[derive(Deserialize)] +pub struct ProfilePathParameters { + pub name: String, + #[serde_inline_default("overview".into())] + pub listing: String, // TODO convert to enum +} +pub async fn profilex( + Path(parameters): Path, + RawQuery(raw_query): RawQuery, + query: Query>, + prefs: Preferences, + original_uri: OriginalUri, +) -> Result, ApiError> { + let prefs = Arc::new(prefs); + let url: String = format!("/user/{}/{}.json?{}&raw_json=1", parameters.name, parameters.listing, raw_query.unwrap_or_default()); + + let path_and_query_url = original_uri + .path_and_query() + .expect("We had path parameters, therefore we should have a path at least") + .as_str(); + // As the above should never be empty, the below code (designed to remove leading "/") should never panic. + // Using a url_encoding crate is potentially expensive, so use this for now (which has worked fine for many years since LibReddit). + // TODO: if the query contains url-encoded "?" or "&", they will probably be decoded too early. + // TODO: This should proabably be redesigned to avoid this unsafe slice. `user.html` needs to be refactored. + let redirect_url = path_and_query_url[1..].replace('?', "%3F").replace('&', "%26"); + let user = user(¶meters.name).await.unwrap_or_default(); + let sort = query.get("sort").cloned().unwrap_or_default(); + + if user.nsfw && crate::utils::should_be_nsfw_gatedx(&prefs, &query) { + return nsfw_landingx(prefs, parameters.name, ResourceType::User, original_uri.to_string()).await; + } + + //RANDOM STUFF BELOW !!! + + if prefs.filters.contains(&format!("u_{}", parameters.name)) { + Ok(Html( + UserTemplate { + user, + posts: Vec::new(), // Return empty vector because user is filtered + sort: (sort, query.get("t").cloned().unwrap_or_default()), + ends: (query.get("after").cloned().unwrap_or_default(), String::new()), + listing: parameters.listing, + prefs, // We can move prefs. Otherwise stick to prefs.clone() + url: path_and_query_url.to_string(), + redirect_url, + is_filtered: true, + all_posts_filtered: false, + all_posts_hidden_nsfw: false, + no_posts: false, + } + .render() + .unwrap(), + )) //FIXME: is this unwrap safe? + } else { + // Request user posts/comments from Reddit + match Post::fetch(&url, false).await { + Ok((mut posts, after)) => { + let (_, all_posts_filtered) = filter_posts(&mut posts, &prefs.get_filters_hashset()); + let no_posts = posts.is_empty(); + //TODO: Currently the actual nsfw filtering happens in the user template. It's better to bring that functionality here. + let all_posts_hidden_nsfw = !no_posts && prefs.show_nsfw != "on" && posts.iter().all(|p| p.flags.nsfw); + Ok(Html( + UserTemplate { + user, + posts, + sort: (sort, query.get("t").cloned().unwrap_or_default()), + ends: (query.get("after").cloned().unwrap_or_default(), after), + listing: parameters.listing, + prefs, + url: path_and_query_url.to_string(), + redirect_url, + is_filtered: false, + all_posts_filtered, + all_posts_hidden_nsfw, + no_posts, + } + .render() + .unwrap(), + )) //FIXME: is this unwrap safe? + } + // If there is an error show error page + Err(msg) => Err( + ApiError::builder(http_api_problem::StatusCode::INTERNAL_SERVER_ERROR) + .title("Temporary error while error messages are migrated") //FIXME + .message(msg) + .finish(), + ), + } + } +} + // USER +// TODO: reimplement this helper function async fn user(name: &str) -> Result { // Build the Reddit JSON API path let path: String = format!("/user/{name}/about.json?raw_json=1"); diff --git a/src/utils.rs b/src/utils.rs index c7d690dc..eb7ee051 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -9,9 +9,8 @@ use std::borrow::Cow; use crate::client::CLIENTX; use crate::{client::json, server::RequestExt}; use askama::Template; -use axum::extract::{FromRequestParts, Path, Query}; +use axum::extract::{FromRequestParts, Query}; use axum::http::request::Parts; -use axum::response::IntoResponse; use axum_extra::extract::CookieJar; use http_api_problem::ApiError; use hyper::{Body, Request, Response}; @@ -52,7 +51,7 @@ macro_rules! dbg_msg { /// Identifies whether or not the page is a subreddit, a user page, or a post. /// This is used by the NSFW landing template to determine the mesage to convey /// to the user. -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Clone, Copy)] pub enum ResourceType { Subreddit, User, @@ -363,7 +362,8 @@ pub struct Post { } impl Post { - // Fetch posts of a user or subreddit and return a vector of posts and the "after" value + // TODO: Wow, look at this spaghetti. Migrate to reqwest and serde. + /// Fetch posts of a user or subreddit and return a vector of posts and the "after" value pub async fn fetch(path: &str, quarantine: bool) -> Result<(Vec, String), String> { // Send a request to the url let res = match json(path.to_string(), quarantine).await { @@ -740,6 +740,7 @@ pub struct Preferences { #[revision(start = 1)] pub blur_spoiler: String, #[revision(start = 1)] + //TODO: convert to boolean pub show_nsfw: String, #[revision(start = 1)] pub blur_nsfw: String, @@ -775,6 +776,16 @@ pub struct Preferences { pub remove_default_feeds: String, } +impl Preferences { + // TODO: It's better if filters is of type Hashset from the start + // The builder definitely supports this + // Currently not implemented because there are quite a few instances of using filters as an iterator in legacy code. + /// Will be deprecated + pub fn get_filters_hashset(&self) -> HashSet { + self.filters.clone().into_iter().collect() + } +} + impl FromRequestParts for Preferences where S: Send + Sync, @@ -939,18 +950,20 @@ pub fn get_filtersx(cookies: &CookieJar) -> HashSet { /// /// The first value of the return tuple is the number of posts filtered. The /// second return value is `true` if all posts were filtered. -pub fn filter_posts(posts: &mut Vec, filters: &HashSet) -> (u64, bool) { +/// The second value always returns `false` if the posts argument is empty. +/// This is a MUTATING method. +pub fn filter_posts(posts: &mut Vec, filters: &HashSet) -> (usize, bool) { // This is the length of the Vec prior to applying the filter. - let lb: u64 = posts.len().try_into().unwrap_or(0); + let lb = posts.len(); if posts.is_empty() { (0, false) } else { - posts.retain(|p| !(filters.contains(&p.community) || filters.contains(&["u_", &p.author.name].concat()))); + posts.retain(|p| !(filters.contains(&p.community) || filters.contains(&format!("u_{}", p.author.name)))); // Get the length of the Vec after applying the filter. // If lb > la, then at least one post was removed. - let la: u64 = posts.len().try_into().unwrap_or(0); + let la = posts.len(); (lb - la, posts.is_empty()) } @@ -1601,12 +1614,7 @@ pub async fn nsfw_landing(req: Request, req_url: String) -> Result, - resource_id: String, - resource_type: ResourceType, - uri: String, -) -> Result, ApiError> { +pub async fn nsfw_landingx(prefs: Arc, resource_id: String, resource_type: ResourceType, uri: String) -> Result, ApiError> { let body = NSFWLandingTemplate { res: resource_id, res_type: resource_type, @@ -1628,6 +1636,7 @@ pub async fn nsfw_landingx( #[derive(Deserialize)] pub struct PathParameters { pub name: String, + pub listing: Option, pub id: String, pub title: Option, pub comment_id: Option, From a8e8863e0d47178299e55a56dce9928666b9bf7d Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Mon, 21 Apr 2025 21:37:22 +0100 Subject: [PATCH 48/52] Add /u/[deleted] error page --- src/main.rs | 2 ++ src/user.rs | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/main.rs b/src/main.rs index f30da4e8..0328cdf5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,7 @@ use futures_util::future::TryFutureExt; use axum::http::header::{HeaderMap, HeaderValue as HeaderValuex}; use axum::routing::get; use axum::ServiceExt; +use redlib::user::user_deleted_error; use tower::Layer; use tower_default_headers::DefaultHeadersLayer; // Create Services @@ -543,6 +544,7 @@ async fn main() { .route("/user/{name}/comments/{id}/{title}", get(post::itemx)) .route("/user/{name}/comments/{id}/{title}/{comment_id}", get(post::itemx)) .route("/user/{name}/comments/{id}", get(post::itemx)) + .route("/user/[deleted]", get(user_deleted_error)) .route("/user/{name}", get(user::profilex)) .route("/user/{name}/{listing}", get(user::profilex)) .layer(DefaultHeadersLayer::new(default_headersx)); diff --git a/src/user.rs b/src/user.rs index 5dbf71a8..2b795c76 100644 --- a/src/user.rs +++ b/src/user.rs @@ -234,6 +234,14 @@ async fn user(name: &str) -> Result { }) } +#[inline] +pub async fn user_deleted_error() -> ApiError { + ApiError::builder(http_api_problem::StatusCode::NOT_FOUND) + .title("User has deleted their account") + .message("The user you are looking for does not exist or has been deleted.") + .finish() +} + pub async fn rss(req: Request) -> Result, String> { if config::get_setting("REDLIB_ENABLE_RSS").is_none() { return Ok(error(req, "RSS is disabled on this instance.").await.unwrap_or_default()); From 8fd2bfdce568804180f18d8d3a203dbdcd1ab0e3 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Tue, 22 Apr 2025 17:36:49 +0100 Subject: [PATCH 49/52] Migrate user rss feed --- src/main.rs | 4 ++++ src/user.rs | 50 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 0328cdf5..ea1189fa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] #![allow(clippy::cmp_owned)] +// use crate::user::rssx; use cached::proc_macro::cached; use clap::{Arg, ArgAction, Command}; use futures_lite::FutureExt; @@ -545,6 +546,9 @@ async fn main() { .route("/user/{name}/comments/{id}/{title}/{comment_id}", get(post::itemx)) .route("/user/{name}/comments/{id}", get(post::itemx)) .route("/user/[deleted]", get(user_deleted_error)) + // FIXME: Axum has not yet merged MatchIt 0.8.6 for routing, which introduces postfix matching + // - Waiting for https://github.com/tokio-rs/axum/issues/3140 + // .route("/user/{name}.rss", get(rssx)) .route("/user/{name}", get(user::profilex)) .route("/user/{name}/{listing}", get(user::profilex)) .layer(DefaultHeadersLayer::new(default_headersx)); diff --git a/src/user.rs b/src/user.rs index 2b795c76..281ba6c6 100644 --- a/src/user.rs +++ b/src/user.rs @@ -2,15 +2,16 @@ // CRATES use crate::client::json; +use crate::config::CONFIG; use crate::server::RequestExt; -use crate::utils::{error, filter_posts, format_url, get_filters, nsfw_landing, nsfw_landingx, param, setting, template, Post, Preferences, ResourceType, User}; +use crate::utils::{error, filter_posts, format_url, get_filters, nsfw_landing, nsfw_landingx, param, rewrite_urls, setting, template, Post, Preferences, ResourceType, User}; use crate::{config, utils}; use askama::Template; use axum::extract::{OriginalUri, Path, Query, RawQuery}; -use axum::response::Html; +use axum::response::{Html, IntoResponse}; use chrono::DateTime; use htmlescape::decode_html; -use http_api_problem::ApiError; +use http_api_problem::{ApiError, StatusCode}; use hyper::{Body, Request, Response}; use serde::Deserialize; use serde_inline_default::serde_inline_default; @@ -205,6 +206,49 @@ pub async fn profilex( } } +// TODO: Why not just use the Reddit RSS Api? This way is very roundabout and I would avoid it if possible. +pub async fn rssx(Path(params): Path, RawQuery(raw_query): RawQuery) -> Result { + if CONFIG.enable_rss.is_none() { + // I'm a teapot, yay! + return Err( + ApiError::builder(http_api_problem::StatusCode::IM_A_TEAPOT) + .title("RSS is disabled on this instance") + .message("The RSS feed is disabled on this instance.") + .finish(), + ); + } + + use rss::{ChannelBuilder, Item}; + let path = format!("/user/{}/{}.json?{}&raw_json=1", ¶ms.name, ¶ms.listing, &raw_query.unwrap_or_default()); + + //FIXME: This unwrap is in original code. Should be fixed once this function is reimplemented. + let user = user(¶ms.name).await.unwrap_or_default(); + let (posts, _) = Post::fetch(&path, false) + .await + .map_err(|e| ApiError::builder(StatusCode::BAD_GATEWAY).title("Could Not Fetch Posts").message(e))?; + + let channel = ChannelBuilder::default() + .title(params.name) + .description(user.description) + .items( + posts + .into_iter() + .map(|post| Item { + title: Some(post.title.to_string()), + link: Some(format_url(&utils::get_post_url(&post))), + author: Some(post.author.name), + pub_date: Some(DateTime::from_timestamp(post.created_ts as i64, 0).unwrap_or_default().to_rfc2822()), + content: Some(rewrite_urls(&decode_html(&post.body).unwrap())), + ..Default::default() + }) + .collect::>(), + ) + .build(); + + // set content-type + Ok(([(axum::http::header::CONTENT_TYPE, "application/rss+xml")], channel.to_string())) +} + // USER // TODO: reimplement this helper function async fn user(name: &str) -> Result { From adb63385311a63fb2de55199deb2b6fd5dc25fe4 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Fri, 25 Apr 2025 00:05:22 +0100 Subject: [PATCH 50/52] Small routing reorder --- src/main.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index ea1189fa..3c84e89c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -538,19 +538,24 @@ async fn main() { .route("/style/{*path}", get(proxy!("https://styles.redditmedia.com/{path}"))) .route("/static/{*path}", get(proxy!("https://www.redditstatic.com/{path}"))) // User profile + // TODO: I have a small emotional attachment to seeing /u in the browser URL bar + // This can be fixed by either manually definining the same routes for /u + // or using a nested router + // NOTE: Redlib previously did a 302 redirect. + // 307 does not support technology from the Cretaceous Era (as well as HTTP 1.0) .route( "/u/{*name}", get(|axum::extract::Path(name): axum::extract::Path| async move { axum::response::Redirect::temporary(format!("/user/{}", name).as_str()) }), ) - .route("/user/{name}/comments/{id}/{title}", get(post::itemx)) - .route("/user/{name}/comments/{id}/{title}/{comment_id}", get(post::itemx)) - .route("/user/{name}/comments/{id}", get(post::itemx)) .route("/user/[deleted]", get(user_deleted_error)) // FIXME: Axum has not yet merged MatchIt 0.8.6 for routing, which introduces postfix matching // - Waiting for https://github.com/tokio-rs/axum/issues/3140 // .route("/user/{name}.rss", get(rssx)) .route("/user/{name}", get(user::profilex)) .route("/user/{name}/{listing}", get(user::profilex)) + .route("/user/{name}/comments/{id}", get(post::itemx)) + .route("/user/{name}/comments/{id}/{title}", get(post::itemx)) + .route("/user/{name}/comments/{id}/{title}/{comment_id}", get(post::itemx)) .layer(DefaultHeadersLayer::new(default_headersx)); let appx = tower_http::normalize_path::NormalizePathLayer::trim_trailing_slash().layer(appx); From 77e2b09e397cc4e822322721c152f8f6d16526b5 Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Mon, 28 Apr 2025 12:50:02 +0100 Subject: [PATCH 51/52] Add preferences page Fix bug with Preferences::from_request --- src/main.rs | 4 ++++ src/settings.rs | 49 ++++++++++++++++++++++++++++++++--------- src/utils.rs | 5 +++-- templates/settings.html | 2 +- 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3c84e89c..99c3f250 100644 --- a/src/main.rs +++ b/src/main.rs @@ -556,6 +556,10 @@ async fn main() { .route("/user/{name}/comments/{id}", get(post::itemx)) .route("/user/{name}/comments/{id}/{title}", get(post::itemx)) .route("/user/{name}/comments/{id}/{title}/{comment_id}", get(post::itemx)) + // Settings + .route("/settings", get(settings::getx).put(settings::setx)) // The post endpoint is backwards-compatible with Redlib v0.x + .route("/settings/set", get(settings::setx)) + // Other layers .layer(DefaultHeadersLayer::new(default_headersx)); let appx = tower_http::normalize_path::NormalizePathLayer::trim_trailing_slash().layer(appx); diff --git a/src/settings.rs b/src/settings.rs index 3be15360..685b1beb 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -7,6 +7,9 @@ use crate::server::ResponseExt; use crate::subreddit::join_until_size_limit; use crate::utils::{deflate_decompress, redirect, template, Preferences}; use askama::Template; +use axum::extract::OriginalUri; +use axum::http::header::HeaderMap; +use axum::response::{Html, IntoResponse}; use cookie::Cookie; use futures_lite::StreamExt; use http_api_problem::ApiError; @@ -25,7 +28,7 @@ struct SettingsTemplate { // CONSTANTS -const PREFS: [&str; 19] = [ +const PREFS: [&'static str; 19] = [ "theme", "front_page", "layout", @@ -58,19 +61,45 @@ pub async fn get(req: Request) -> Result, String> { })) } +pub async fn getx(OriginalUri(uri): OriginalUri, prefs: Preferences) -> impl IntoResponse { + Html(SettingsTemplate { prefs, url: uri.to_string() }.render().unwrap()) +} + +// Surpringly, the most appropriate http method for this handler is GET, because this handler is Safe and Indempotent. +// Second-best is PUT. Redlib will generally be like this, because there is currently zero server-side state. +pub async fn setx(form: axum::extract::Form>) -> impl IntoResponse { + let mut resp_headers = HeaderMap::with_capacity(PREFS.len()); + // Don't let client set arbitrary cookie values + for name in PREFS { + match form.get(name).map(String::as_str) { + Some("") => { + resp_headers.append( + axum::http::header::SET_COOKIE, + axum::http::HeaderValue::try_from(format!("{name}=; Path=/; HttpOnly; Max-Age=0")).expect("This should have been checked statically"), + ); + } + Some(value) => { + // 400 days = 34,560,000 seconds + // NOTE: AFAICT all Redlib settings are not secure, and there is no reason not to let Javascript access them. + // Hence the HttpOnly cookie flag is unnecessary. + // It might be proper to set settings cookies client-side anyway. + // TODO: let clients set preferences client-side. + if let Ok(val) = axum::http::HeaderValue::try_from(format!("{}={}; Path=/; HttpOnly; Max-Age=34560000", name, value)) { + resp_headers.append(axum::http::header::SET_COOKIE, val); + } + } + None => {} // Do nothing. The client can specify just one cookie at a time without wiping eveything. + }; + } + + // HTTP 303 See Other + (resp_headers, axum::response::Redirect::to("/settings")) +} + // Set cookies using response "Set-Cookie" header pub async fn set(req: Request) -> Result, String> { // Split the body into parts let (parts, mut body) = req.into_parts(); - - // Grab existing cookies - let _cookies: Vec> = parts - .headers - .get_all("Cookie") - .iter() - .filter_map(|header| Cookie::parse(header.to_str().unwrap_or_default()).ok()) - .collect(); - // Aggregate the body... // let whole_body = hyper::body::aggregate(req).await.map_err(|e| e.to_string())?; let body_bytes = body diff --git a/src/utils.rs b/src/utils.rs index eb7ee051..9e3bb921 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -724,6 +724,7 @@ pub struct Params { } #[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] +#[serde(default)] #[revisioned(revision = 1)] pub struct Preferences { #[revision(start = 1)] @@ -792,8 +793,8 @@ where { type Rejection = Infallible; - async fn from_request_parts(parts: &mut Parts, _: &S) -> Result { - Ok(Preferences::build(parts.extensions.get::().unwrap_or(&CookieJar::new()))) + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + Ok(Preferences::build(&CookieJar::from_request_parts(parts, state).await?)) } } diff --git a/templates/settings.html b/templates/settings.html index c3d8086b..1e95d0d6 100644 --- a/templates/settings.html +++ b/templates/settings.html @@ -13,7 +13,7 @@ {% block content %}
-
+
Appearance From 6eb08c2a564ba7a2953e30912aeac8807f59b0ef Mon Sep 17 00:00:00 2001 From: Tokarak <63452145+Tokarak@users.noreply.github.com> Date: Mon, 28 Apr 2025 13:17:26 +0100 Subject: [PATCH 52/52] Add placeholder system.css theme --- src/utils.rs | 6 +++--- static/themes/system.css | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 static/themes/system.css diff --git a/src/utils.rs b/src/utils.rs index 9e3bb921..47e10947 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -825,8 +825,8 @@ impl Preferences { // Build preferences from cookies pub fn new(req: &Request) -> Self { // Read available theme names from embedded css files. - // Always make the default "system" theme available. - let mut themes = vec!["system".to_string()]; + // "system" theme is has a placeholder CSS file. + let mut themes = vec![]; for file in ThemeAssets::iter() { let chunks: Vec<&str> = file.as_ref().split(".css").collect(); themes.push(chunks[0].to_owned()); @@ -859,7 +859,7 @@ impl Preferences { pub fn build(cookies: &CookieJar) -> Self { // Read available theme names from embedded css files. - // Always make the default "system" theme available. + // "system" theme has a placeholder CSS file. static THEMES: LazyLock> = LazyLock::new(|| ThemeAssets::iter().map(|f| f.split(".css").collect::>()[0].to_owned()).collect()); Self { available_themes: THEMES.clone(), diff --git a/static/themes/system.css b/static/themes/system.css new file mode 100644 index 00000000..328986fc --- /dev/null +++ b/static/themes/system.css @@ -0,0 +1,2 @@ +/* Placeholder css to automatically generate "system" theme */ +.system {} \ No newline at end of file