diff --git a/Cargo.lock b/Cargo.lock index 474425f8..1b5ca2f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,7 +69,7 @@ dependencies = [ "rustc-hash", "serde", "serde_derive", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -284,7 +284,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -322,7 +322,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.6.0", + "hyper 1.8.1", "hyper-util", "itoa 1.0.15", "matchit", @@ -396,7 +396,7 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -837,7 +837,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -867,7 +867,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -878,7 +878,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -909,7 +909,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -919,7 +919,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -932,7 +932,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -973,7 +973,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.0", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -995,7 +995,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -1202,7 +1202,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -1305,7 +1305,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -1449,9 +1449,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.4" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] name = "hermit-abi" @@ -1623,13 +1623,14 @@ dependencies = [ [[package]] name = "hyper" -version = "1.6.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ + "atomic-waker", "bytes", "futures-channel", - "futures-util", + "futures-core", "h2", "http 1.3.1", "http-body 1.0.1", @@ -1637,6 +1638,7 @@ dependencies = [ "httpdate", "itoa 1.0.15", "pin-project-lite", + "pin-utils", "smallvec", "tokio", "want", @@ -1649,7 +1651,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", - "hyper 1.6.0", + "hyper 1.8.1", "hyper-util", "rustls", "rustls-pki-types", @@ -1667,7 +1669,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.6.0", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -1688,7 +1690,7 @@ dependencies = [ "futures-util", "http 1.3.1", "http-body 1.0.1", - "hyper 1.6.0", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", @@ -1816,9 +1818,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.9.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown", @@ -1917,9 +1919,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ "once_cell", "wasm-bindgen", @@ -2101,7 +2103,7 @@ checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -2179,15 +2181,44 @@ name = "mlm" version = "0.4.6" dependencies = [ "anyhow", - "askama", "async-trait", "axum", - "axum-extra", - "bytes", - "cookie", "dirs", "embed-resource", "figment", + "mlm_core", + "mlm_db", + "mlm_mam", + "mlm_meta", + "mlm_parse", + "mlm_web_askama", + "native_db", + "once_cell", + "open", + "qbit", + "serde", + "serde_json", + "tempfile", + "time", + "tokio", + "tracing", + "tracing-appender", + "tracing-panic", + "tracing-subscriber", + "tray-item", + "url", + "winsafe", +] + +[[package]] +name = "mlm_core" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "bytes", + "dirs", + "figment", "file-id", "futures", "htmlentity", @@ -2202,7 +2233,6 @@ dependencies = [ "native_db", "native_model", "once_cell", - "open", "openssl", "qbit", "quick-xml", @@ -2216,7 +2246,6 @@ dependencies = [ "serde_derive", "serde_json", "sublime_fuzzy", - "tempfile", "thiserror 2.0.17", "time", "tokio", @@ -2224,17 +2253,13 @@ dependencies = [ "tokio-util", "toml 0.8.23", "tower", - "tower-http", "tracing", "tracing-appender", "tracing-panic", "tracing-subscriber", - "tray-item", "unidecode", - "url", "urlencoding", "uuid", - "winsafe", ] [[package]] @@ -2314,6 +2339,40 @@ dependencies = [ "unidecode", ] +[[package]] +name = "mlm_web_askama" +version = "0.1.0" +dependencies = [ + "anyhow", + "askama", + "axum", + "axum-extra", + "futures", + "itertools 0.14.0", + "mlm_core", + "mlm_db", + "mlm_mam", + "mlm_parse", + "native_db", + "once_cell", + "qbit", + "regex", + "reqwest", + "serde", + "serde_json", + "sublime_fuzzy", + "thiserror 2.0.17", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "toml 0.8.23", + "tower", + "tower-http", + "tracing", + "urlencoding", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -2353,7 +2412,7 @@ source = "git+https://github.com/StirlingMouse/native_db.git?branch=0.8.x#cddaaf dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -2379,7 +2438,7 @@ checksum = "2f385f3d57adaea8d8868e65a0bc821bcb8ba2228bbf87a1c3c6144ac48f3791" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -2516,7 +2575,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -2620,7 +2679,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -2753,7 +2812,7 @@ dependencies = [ "phf_shared 0.11.3", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -2870,9 +2929,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] @@ -2885,7 +2944,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", "version_check", "yansi", ] @@ -3010,9 +3069,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" dependencies = [ "proc-macro2", ] @@ -3264,7 +3323,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.6.0", + "hyper 1.8.1", "hyper-rustls", "hyper-tls", "hyper-util", @@ -3324,9 +3383,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.25" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" @@ -3353,7 +3412,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3393,9 +3452,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -3571,7 +3630,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -3628,7 +3687,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -3848,9 +3907,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.104" +version = "2.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "6e614ed320ac28113fa64972c4262d5dbc89deacdfd00c34a3e4cea073243c12" dependencies = [ "proc-macro2", "quote", @@ -3874,7 +3933,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -3908,7 +3967,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3965,7 +4024,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -3976,7 +4035,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -4066,7 +4125,6 @@ dependencies = [ "io-uring", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "slab", @@ -4083,7 +4141,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -4108,9 +4166,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -4299,7 +4357,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -4553,37 +4611,25 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.104", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-util", "js-sys", "once_cell", "wasm-bindgen", @@ -4592,9 +4638,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4602,31 +4648,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.104", - "wasm-bindgen-backend", + "syn 2.0.115", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", @@ -4953,7 +4999,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", "synstructure", ] @@ -4974,7 +5020,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] [[package]] @@ -4994,7 +5040,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", "synstructure", ] @@ -5034,5 +5080,5 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.115", ] diff --git a/Cargo.toml b/Cargo.toml index bf5b68df..5df95d3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,21 @@ [workspace] resolver = "3" -members = ["server", "mlm_db", "mlm_parse", "mlm_mam", "mlm_meta"] +members = [ + "server", + "mlm_db", + "mlm_parse", + "mlm_mam", + "mlm_meta", + "mlm_core", + "mlm_web_askama", +] + +# Faster dev builds for WASM +[profile.dev.package."*"] +opt-level = 1 + +# Keep debug builds fast for the server +[profile.dev] +opt-level = 0 +codegen-units = 256 +debug = 1 diff --git a/mlm_core/Cargo.toml b/mlm_core/Cargo.toml new file mode 100644 index 00000000..b69541e6 --- /dev/null +++ b/mlm_core/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "mlm_core" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.100" +axum = { version = "0.8.4", features = ["query", "macros"] } +bytes = "1.11.0" +dirs = "6.0" +figment = { version = "0.10", features = ["toml", "env"] } +file-id = "0.2.2" +futures = "0.3" +htmlentity = "1.3.2" +itertools = "0.14.0" +lava_torrent = { git = "https://github.com/StirlingMouse/lava_torrent.git" } +log = "0.4.27" +matchr = "0.2.5" +mlm_db = { path = "../mlm_db" } +mlm_mam = { path = "../mlm_mam" } +mlm_parse = { path = "../mlm_parse" } +mlm_meta = { path = "../mlm_meta" } +native_db = { git = "https://github.com/StirlingMouse/native_db.git", branch = "0.8.x" } +native_model = "0.4.20" +once_cell = "1.21.3" +openssl = { version = "0.10.73", features = ["vendored"] } +qbit = { git = "https://github.com/StirlingMouse/qbittorrent-webui-api.git" } +quick-xml = { version = "0.38.0", features = ["serialize"] } +regex = "1.12.2" +reqwest = { version = "0.12.20", default-features = false, features = ["json", "rustls-tls"] } +reqwest_cookie_store = "0.8.0" +sanitize-filename = { git = "https://github.com/StirlingMouse/sanitize-filename.git" } +scraper = "0.23.1" +serde = "1.0.136" +serde_derive = "1.0.136" +serde_json = "1.0.140" +serde-nested-json = "0.1.3" +sublime_fuzzy = "0.7.0" +thiserror = "2.0.17" +time = { version = "0.3.41", features = [ + "formatting", + "local-offset", + "macros", + "serde", +] } +tokio = { version = "1.45.1", features = ["fs", "macros", "rt-multi-thread", "sync", "time"] } +tokio-stream = { version = "0.1.17", features = ["sync"] } +tokio-util = "0.7" +toml = "0.8.23" +tower = { version = "0.5.2" } +tracing = "0.1" +tracing-appender = { version = "0.2.3" } +tracing-subscriber = { version = "0.3", features = [ + "local-time", + "env-filter", + "tracing-log", +] } +tracing-panic = { version = "0.1.2" } +unidecode = "0.3.0" +urlencoding = "2.1.3" +uuid = { version = "1.17.0", features = ["serde", "v4", "js"] } diff --git a/server/src/audiobookshelf.rs b/mlm_core/src/audiobookshelf.rs similarity index 99% rename from server/src/audiobookshelf.rs rename to mlm_core/src/audiobookshelf.rs index 55fea902..82de20ad 100644 --- a/server/src/audiobookshelf.rs +++ b/mlm_core/src/audiobookshelf.rs @@ -675,7 +675,7 @@ pub fn create_metadata(meta: &TorrentMeta) -> serde_json::Value { "series": &meta.series.iter().map(format_serie).collect::>(), "title": title, "subtitle": subtitle, - "description": meta.description, + "description": clean_html(&meta.description), "isbn": meta.ids.get(ids::ISBN), "asin": meta.ids.get(ids::ASIN), "tags": if flags.lgbt == Some(true) { Some(vec!["LGBT"]) } else { None }, diff --git a/server/src/autograbber.rs b/mlm_core/src/autograbber.rs similarity index 91% rename from server/src/autograbber.rs rename to mlm_core/src/autograbber.rs index 16498e4f..3f18591f 100644 --- a/server/src/autograbber.rs +++ b/mlm_core/src/autograbber.rs @@ -47,6 +47,7 @@ pub async fn run_autograbber( autograb_trigger: Sender<()>, index: usize, autograb_config: Arc, + events: &crate::stats::Events, ) -> Result<()> { // Make sure we are only running one autograbber at a time let _guard = AUTOGRABBER_MUTEX.lock().await; @@ -96,6 +97,7 @@ pub async fn run_autograbber( }, &mam, max_torrents, + events, ) .await .context("search_torrents")?; @@ -116,6 +118,7 @@ pub async fn search_and_select_torrents( fields: SearchFields, mam: &MaM<'_>, max_torrents: u64, + events: &crate::stats::Events, ) -> Result { let torrents = search_torrents(torrent_search, fields, mam) .await @@ -123,7 +126,7 @@ pub async fn search_and_select_torrents( if torrent_search.mark_removed { let torrents = torrents.collect::>(); - mark_removed_torrents(db, mam, &torrents) + mark_removed_torrents(db, mam, &torrents, events) .await .context("mark_removed_torrents")?; @@ -140,6 +143,7 @@ pub async fn search_and_select_torrents( torrent_search.dry_run, max_torrents, None, + events, ) .await .context("select_torrents"); @@ -158,6 +162,7 @@ pub async fn search_and_select_torrents( torrent_search.dry_run, max_torrents, None, + events, ) .await .context("select_torrents") @@ -305,6 +310,7 @@ pub async fn mark_removed_torrents( db: &Database<'_>, mam: &MaM<'_>, torrents: &[MaMTorrent], + events: &crate::stats::Events, ) -> Result<()> { if let (Some(first), Some(last)) = (torrents.first(), torrents.last()) { let ids = first.id.min(last.id)..=first.id.max(last.id); @@ -324,8 +330,12 @@ pub async fn mark_removed_torrents( rw.upsert(torrent)?; rw.commit()?; drop(guard); - write_event(db, Event::new(tid, Some(id), EventType::RemovedFromTracker)) - .await; + write_event( + db, + events, + Event::new(tid, Some(id), EventType::RemovedFromTracker), + ) + .await; } sleep(Duration::from_millis(400)).await; } @@ -350,6 +360,7 @@ pub async fn select_torrents>( dry_run: bool, max_torrents: u64, goodreads_id: Option, + events: &crate::stats::Events, ) -> Result { let mut selected_torrents = 0; 'torrent: for torrent in torrents { @@ -390,13 +401,15 @@ pub async fn select_torrents>( let mut updated = old_selected.clone(); updated.unsat_buffer = Some(unsat_buffer); if updated.meta != meta { - update_selected_torrent_meta(db, rw_opt.unwrap(), mam, updated, meta).await?; + update_selected_torrent_meta(db, rw_opt.unwrap(), mam, updated, meta, events) + .await?; } else { rw.update(old_selected, updated)?; rw_opt.unwrap().1.commit()?; } } else if old_selected.meta != meta { - update_selected_torrent_meta(db, rw_opt.unwrap(), mam, old_selected, meta).await?; + update_selected_torrent_meta(db, rw_opt.unwrap(), mam, old_selected, meta, events) + .await?; } trace!("Torrent {} is already selected", torrent.id); continue; @@ -420,6 +433,7 @@ pub async fn select_torrents>( meta, false, cost == Cost::MetadataOnlyAdd, + events, ) .await?; } @@ -467,7 +481,8 @@ pub async fn select_torrents>( for old in old_selected { if old.mam_id == torrent.id { if old.meta != meta { - update_selected_torrent_meta(db, rw_opt.unwrap(), mam, old, meta).await?; + update_selected_torrent_meta(db, rw_opt.unwrap(), mam, old, meta, events) + .await?; } trace!("Torrent {} is already selected2", torrent.id); continue 'torrent; @@ -536,6 +551,7 @@ pub async fn select_torrents>( meta, false, false, + events, ) .await?; } @@ -671,22 +687,36 @@ pub async fn add_metadata_only_torrent( Ok(()) } +pub enum PreparedTorrentMetaUpdate { + Unchanged, + Silent, + Pending(Box), +} + +pub struct PendingTorrentMetaUpdate { + torrent: mlm_db::Torrent, + meta: TorrentMeta, + diff: Vec, + mam_id: Option, +} + #[allow(clippy::too_many_arguments)] -pub async fn update_torrent_meta( - config: &Config, - db: &Database<'_>, - (guard, rw): (MutexGuard<'_, ()>, RwTransaction<'_>), +pub fn queue_torrent_meta_update( + rw: &RwTransaction<'_>, mam_torrent: Option<&MaMTorrent>, mut torrent: mlm_db::Torrent, mut meta: TorrentMeta, allow_non_mam: bool, linker_is_owner: bool, -) -> Result<()> { +) -> Result { meta.ids.extend(torrent.meta.ids.clone()); meta.tags = torrent.meta.tags.clone(); if meta.description.is_empty() { meta.description = torrent.meta.description.clone(); } + if meta == torrent.meta { + return Ok(PreparedTorrentMetaUpdate::Unchanged); + } if !allow_non_mam && torrent.meta.source != MetadataSource::Mam { // Update VIP status and uploaded_at still @@ -695,10 +725,9 @@ pub async fn update_torrent_meta( { torrent.meta.vip_status = meta.vip_status; torrent.meta.uploaded_at = meta.uploaded_at; - rw.upsert(torrent.clone())?; - rw.commit()?; + rw.upsert(torrent)?; } - return Ok(()); + return Ok(PreparedTorrentMetaUpdate::Silent); } // Check expiring VIP @@ -713,9 +742,8 @@ pub async fn update_torrent_meta( torrent.meta.vip_status = meta.vip_status.clone(); // If expiring VIP was the only change, just silently update the database if torrent.meta == meta { - rw.upsert(torrent.clone())?; - rw.commit()?; - return Ok(()); + rw.upsert(torrent)?; + return Ok(PreparedTorrentMetaUpdate::Silent); } } @@ -725,9 +753,8 @@ pub async fn update_torrent_meta( torrent.meta.num_files = meta.num_files; // If uploaded_at or num_files was the only change, just silently update the database if torrent.meta == meta { - rw.upsert(torrent.clone())?; - rw.commit()?; - return Ok(()); + rw.upsert(torrent)?; + return Ok(PreparedTorrentMetaUpdate::Silent); } } @@ -750,8 +777,29 @@ pub async fn update_torrent_meta( torrent.meta = meta.clone(); torrent.title_search = normalize_title(&meta.title); rw.upsert(torrent.clone())?; - rw.commit()?; - drop(guard); + Ok(PreparedTorrentMetaUpdate::Pending(Box::new( + PendingTorrentMetaUpdate { + torrent, + meta, + diff, + mam_id: mam_torrent.map(|m| m.id), + }, + ))) +} + +pub async fn finalize_torrent_meta_update( + config: &Config, + db: &Database<'_>, + pending: PendingTorrentMetaUpdate, + events: &crate::stats::Events, +) -> Result<()> { + let PendingTorrentMetaUpdate { + torrent, + meta, + diff, + mam_id, + } = pending; + let id = torrent.id.clone(); if let Some(library_path) = &torrent.library_path && let serde_json::Value::Object(new) = abs::create_metadata(&meta) @@ -782,9 +830,9 @@ pub async fn update_torrent_meta( } if !diff.is_empty() { - let mam_id = mam_torrent.map(|m| m.id); write_event( db, + events, Event::new( Some(id), mam_id, @@ -799,12 +847,45 @@ pub async fn update_torrent_meta( Ok(()) } +#[allow(clippy::too_many_arguments)] +pub async fn update_torrent_meta( + config: &Config, + db: &Database<'_>, + (guard, rw): (MutexGuard<'_, ()>, RwTransaction<'_>), + mam_torrent: Option<&MaMTorrent>, + torrent: mlm_db::Torrent, + meta: TorrentMeta, + allow_non_mam: bool, + linker_is_owner: bool, + events: &crate::stats::Events, +) -> Result<()> { + let prepared = queue_torrent_meta_update( + &rw, + mam_torrent, + torrent, + meta, + allow_non_mam, + linker_is_owner, + )?; + + if !matches!(prepared, PreparedTorrentMetaUpdate::Unchanged) { + rw.commit()?; + } + drop(guard); + + if let PreparedTorrentMetaUpdate::Pending(pending) = prepared { + finalize_torrent_meta_update(config, db, *pending, events).await?; + } + Ok(()) +} + async fn update_selected_torrent_meta( db: &Database<'_>, (guard, rw): (MutexGuard<'_, ()>, RwTransaction<'_>), mam: &MaM<'_>, torrent: SelectedTorrent, meta: TorrentMeta, + events: &crate::stats::Events, ) -> Result<()> { let mam_id = torrent.mam_id; let diff = torrent.meta.diff(&meta); @@ -824,6 +905,7 @@ async fn update_selected_torrent_meta( drop(guard); write_event( db, + events, Event::new( hash, Some(mam_id), diff --git a/server/src/cleaner.rs b/mlm_core/src/cleaner.rs similarity index 91% rename from server/src/cleaner.rs rename to mlm_core/src/cleaner.rs index f4b1827a..b915c8f3 100644 --- a/server/src/cleaner.rs +++ b/mlm_core/src/cleaner.rs @@ -7,16 +7,20 @@ use mlm_db::{ use native_db::Database; use tracing::{debug, info, instrument, trace, warn}; +use crate::config::Config; use crate::{ audiobookshelf::Abs, - config::Config, linker::rank_torrents, logging::{TorrentMetaError, update_errored_torrent, write_event}, qbittorrent::ensure_category_exists, }; #[instrument(skip_all)] -pub async fn run_library_cleaner(config: Arc, db: Arc>) -> Result<()> { +pub async fn run_library_cleaner( + config: Arc, + db: Arc>, + events: &crate::stats::Events, +) -> Result<()> { let torrents: Vec = { let r = db.r_transaction()?; let torrents = r.scan().secondary::(TorrentKey::title_search)?; @@ -30,21 +34,26 @@ pub async fn run_library_cleaner(config: Arc, db: Arc>) -> for torrent in torrents { if let Some(current) = batch.first() { if !current.matches(&torrent) { - process_batch(&config, &db, mem::take(&mut batch)).await?; + process_batch(&config, &db, mem::take(&mut batch), events).await?; } batch.push(torrent); } else { batch.push(torrent); } } - process_batch(&config, &db, batch).await?; + process_batch(&config, &db, batch, events).await?; Ok(()) } #[instrument(skip_all)] -async fn process_batch(config: &Config, db: &Database<'_>, batch: Vec) -> Result<()> { - if batch.len() == 1 { +async fn process_batch( + config: &Config, + db: &Database<'_>, + batch: Vec, + events: &crate::stats::Events, +) -> Result<()> { + if batch.len() <= 1 { return Ok(()); }; let mut batch = rank_torrents(config, batch); @@ -60,6 +69,7 @@ async fn process_batch(config: &Config, db: &Database<'_>, batch: Vec) db, remove.clone(), keep.library_path.is_some() && keep.library_path != remove.library_path, + events, ) .await .map_err(|err| anyhow::Error::new(TorrentMetaError(remove.meta.clone(), err))); @@ -81,11 +91,12 @@ pub async fn clean_torrent( db: &Database<'_>, mut remove: Torrent, delete_in_abs: bool, + events: &crate::stats::Events, ) -> Result<()> { if remove.id_is_hash { for qbit_conf in config.qbittorrent.iter() { if let Some(on_cleaned) = &qbit_conf.on_cleaned { - let qbit = qbit::Api::new_login_username_password( + let qbit: qbit::Api = qbit::Api::new_login_username_password( &qbit_conf.url, &qbit_conf.username, &qbit_conf.password, @@ -127,6 +138,7 @@ pub async fn clean_torrent( if let Some(library_path) = library_path { write_event( db, + events, Event::new( Some(id), mam_id, diff --git a/server/src/config.rs b/mlm_core/src/config.rs similarity index 91% rename from server/src/config.rs rename to mlm_core/src/config.rs index a40b85bb..212bc5ad 100644 --- a/server/src/config.rs +++ b/mlm_core/src/config.rs @@ -1,5 +1,3 @@ -use std::{collections::BTreeMap, path::PathBuf}; - use mlm_db::{ Flags, Language, MediaType, OldDbMainCat, Size, impls::{parse, parse_opt, parse_vec}, @@ -9,9 +7,10 @@ use mlm_mam::{ serde::parse_opt_date, }; use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, path::PathBuf}; use time::Date; -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(tag = "id", rename_all = "lowercase")] pub enum ProviderConfig { Hardcover(HardcoverConfig), @@ -19,7 +18,7 @@ pub enum ProviderConfig { OpenLibrary(OpenLibraryConfig), } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct HardcoverConfig { #[serde(default = "default_provider_enabled")] @@ -29,7 +28,7 @@ pub struct HardcoverConfig { pub api_key: Option, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct RomanceIoConfig { #[serde(default = "default_provider_enabled")] @@ -38,7 +37,7 @@ pub struct RomanceIoConfig { pub timeout_secs: Option, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct OpenLibraryConfig { #[serde(default = "default_provider_enabled")] @@ -77,7 +76,7 @@ fn default_provider_enabled() -> bool { true } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Config { pub mam_id: String, @@ -144,14 +143,14 @@ pub struct Config { pub metadata_providers: Vec, } -#[derive(Clone, Debug, Default, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct SearchConfig { #[serde(deserialize_with = "parse_opt")] pub wedge_over: Option, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct AudiobookShelfConfig { pub url: String, @@ -160,8 +159,7 @@ pub struct AudiobookShelfConfig { pub interval: u64, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct TorrentSearch { #[serde(rename = "type")] pub kind: Type, @@ -174,8 +172,6 @@ pub struct TorrentSearch { pub max_pages: Option, #[serde(flatten)] pub filter: TorrentFilter, - #[serde(flatten)] - pub edition: EditionFilter, pub search_interval: Option, pub unsat_buffer: Option, @@ -207,8 +203,7 @@ pub enum SortBy { Random, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct SnatchlistSearch { #[serde(rename = "type")] pub kind: SnatchlistType, @@ -217,15 +212,13 @@ pub struct SnatchlistSearch { pub max_pages: Option, #[serde(flatten)] pub filter: TorrentFilter, - #[serde(flatten)] - pub edition: EditionFilter, pub search_interval: Option, #[serde(default)] pub dry_run: bool, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct GoodreadsList { pub url: String, @@ -242,7 +235,7 @@ pub struct GoodreadsList { pub dry_run: bool, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct NotionList { pub data_source: String, @@ -258,32 +251,25 @@ pub struct NotionList { pub dry_run: bool, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct Grab { #[serde(default)] pub cost: Cost, #[serde(flatten)] pub filter: TorrentFilter, - #[serde(flatten)] - pub edition: EditionFilter, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct TagFilter { #[serde(flatten)] pub filter: TorrentFilter, - #[serde(flatten)] - pub edition: EditionFilter, #[serde(default)] pub category: Option, #[serde(default)] pub tags: Vec, } -#[derive(Clone, Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct TorrentFilter { #[serde(default)] pub name: Option, @@ -304,13 +290,11 @@ pub struct TorrentFilter { pub min_snatched: Option, pub max_snatched: Option, - // TODO: READ from parent - #[serde(skip)] + #[serde(flatten)] pub edition: EditionFilter, } -#[derive(Clone, Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct EditionFilter { #[serde(default)] #[serde(deserialize_with = "parse_vec")] @@ -365,7 +349,7 @@ pub struct QbitUpdate { pub tags: Vec, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(untagged)] #[allow(clippy::enum_variant_names)] pub enum Library { @@ -374,7 +358,7 @@ pub enum Library { ByCategory(LibraryByCategory), } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct LibraryByRipDir { pub rip_dir: PathBuf, @@ -384,7 +368,7 @@ pub struct LibraryByRipDir { pub filter: EditionFilter, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct LibraryByDownloadDir { pub download_dir: PathBuf, @@ -394,7 +378,7 @@ pub struct LibraryByDownloadDir { pub tag_filters: LibraryTagFilters, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct LibraryByCategory { pub category: String, @@ -413,7 +397,7 @@ pub struct LibraryTagFilters { pub deny_tags: Vec, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct LibraryOptions { #[serde(default)] diff --git a/server/src/config_impl.rs b/mlm_core/src/config_impl.rs similarity index 78% rename from server/src/config_impl.rs rename to mlm_core/src/config_impl.rs index e872a3ed..221ac035 100644 --- a/server/src/config_impl.rs +++ b/mlm_core/src/config_impl.rs @@ -1,14 +1,22 @@ use std::sync::OnceLock; use anyhow::{Result, ensure}; -use mlm_db::{Flags, Language, MediaType, OldCategory, Size, Torrent, TorrentMeta}; -use mlm_mam::{search::MaMTorrent, serde::DATE_TIME_FORMAT, user_torrent::UserDetailsTorrent}; +use mlm_db::{ + AudiobookCategory, EbookCategory, Flags, Language, MediaType, OldCategory, Size, Torrent, + TorrentMeta, +}; +use mlm_mam::{ + search::MaMTorrent, + serde::{DATE_FORMAT, DATE_TIME_FORMAT}, + user_torrent::UserDetailsTorrent, +}; use reqwest::Url; use time::UtcDateTime; use tracing::error; use crate::config::{ - Config, EditionFilter, GoodreadsList, Library, LibraryOptions, LibraryTagFilters, TorrentFilter, + Config, Cost, EditionFilter, GoodreadsList, Library, LibraryOptions, LibraryTagFilters, + TorrentFilter, TorrentSearch, Type, }; static EMPTY_LIBRARY_TAG_FILTERS: OnceLock = OnceLock::new(); @@ -29,6 +37,10 @@ impl Config { } impl TorrentFilter { + pub fn display_name(&self, i: usize) -> String { + self.name.clone().unwrap_or_else(|| format!("{i}")) + } + pub fn matches(&self, torrent: &MaMTorrent) -> bool { if !self.edition.matches(torrent) { return false; @@ -139,11 +151,11 @@ impl TorrentFilter { true } - pub(crate) fn matches_lib(&self, torrent: &Torrent) -> Result { + pub fn matches_lib(&self, torrent: &Torrent) -> Result { self.matches_meta(&torrent.meta) } - pub(crate) fn matches_meta(&self, meta: &TorrentMeta) -> Result { + pub fn matches_meta(&self, meta: &TorrentMeta) -> Result { if !self.edition.matches_meta(meta)? { return Ok(false); } @@ -162,7 +174,206 @@ impl TorrentFilter { } } -#[allow(dead_code)] +impl TorrentSearch { + pub fn mam_search(&self) -> String { + let mut url: Url = "https://www.myanonamouse.net/tor/browse.php?thumbnail=true" + .parse() + .unwrap(); + + { + let mut query = url.query_pairs_mut(); + if let Some(text) = &self.query { + query.append_pair("tor[text]", text); + } + + for srch_in in &self.search_in { + query.append_pair(&format!("tor[srchIn][{}]", srch_in.as_str()), "true"); + } + let search_in = match self.kind { + Type::Bookmarks => "bookmarks", + Type::Mine => "mine", + Type::Uploader(id) => { + query.append_pair("tor[user]", &id.to_string()); + "uploader" + } + _ => "torrents", + }; + query.append_pair("tor[searchIn]", search_in); + let search_type = match (self.kind, self.cost) { + (Type::Freeleech, _) => "fl", + (_, Cost::Free) => "fl-VIP", + _ => "all", + }; + query.append_pair("tor[searchType]", search_type); + let sort_type = self + .sort_by + .map(|sort_by| match sort_by { + crate::config::SortBy::LowSeeders => "seedersAsc", + crate::config::SortBy::LowSnatches => "snatchedAsc", + crate::config::SortBy::OldestFirst => "dateAsc", + crate::config::SortBy::Random => "random", + }) + .unwrap_or(match self.kind { + Type::New => "dateDesc", + _ => "", + }); + if !sort_type.is_empty() { + query.append_pair("tor[sort_type]", sort_type); + } + + for cat in self.mam_search_category_ids() { + query.append_pair("tor[cat][]", &cat.to_string()); + } + for lang in &self.filter.edition.languages { + query.append_pair("tor[browse_lang][]", &lang.to_id().to_string()); + } + + let (flags_is_hide, flags) = self.filter.edition.flags.as_search_bitfield(); + if !flags.is_empty() { + query.append_pair( + "tor[browseFlagsHideVsShow]", + if flags_is_hide { "0" } else { "1" }, + ); + } + for flag in flags { + query.append_pair("tor[browseFlags][]", &flag.to_string()); + } + + if self.filter.edition.min_size.bytes() > 0 || self.filter.edition.max_size.bytes() > 0 + { + query.append_pair("tor[unit]", "1"); + } + if self.filter.edition.min_size.bytes() > 0 { + query.append_pair( + "tor[minSize]", + &self.filter.edition.min_size.bytes().to_string(), + ); + } + if self.filter.edition.max_size.bytes() > 0 { + query.append_pair( + "tor[maxSize]", + &self.filter.edition.max_size.bytes().to_string(), + ); + } + + if let Some(uploaded_after) = self.filter.uploaded_after { + query.append_pair( + "tor[startDate]", + &uploaded_after.format(&DATE_FORMAT).unwrap(), + ); + } + if let Some(uploaded_before) = self.filter.uploaded_before { + query.append_pair( + "tor[endDate]", + &uploaded_before.format(&DATE_FORMAT).unwrap(), + ); + } + if let Some(min_seeders) = self.filter.min_seeders { + query.append_pair("tor[minSeeders]", &min_seeders.to_string()); + } + if let Some(max_seeders) = self.filter.max_seeders { + query.append_pair("tor[maxSeeders]", &max_seeders.to_string()); + } + if let Some(min_leechers) = self.filter.min_leechers { + query.append_pair("tor[minLeechers]", &min_leechers.to_string()); + } + if let Some(max_leechers) = self.filter.max_leechers { + query.append_pair("tor[maxLeechers]", &max_leechers.to_string()); + } + if let Some(min_snatched) = self.filter.min_snatched { + query.append_pair("tor[minSnatched]", &min_snatched.to_string()); + } + if let Some(max_snatched) = self.filter.max_snatched { + query.append_pair("tor[maxSnatched]", &max_snatched.to_string()); + } + } + + url.to_string() + } + + fn mam_search_category_ids(&self) -> Vec { + let include_audio = self.filter.edition.media_type.is_empty() + || self.filter.edition.media_type.iter().any(|media_type| { + matches!( + media_type, + MediaType::Audiobook | MediaType::PeriodicalAudiobook + ) + }); + let include_ebook = self.filter.edition.media_type.is_empty() + || self.filter.edition.media_type.iter().any(|media_type| { + matches!( + media_type, + MediaType::Ebook + | MediaType::Manga + | MediaType::ComicBook + | MediaType::PeriodicalEbook + ) + }); + let include_musicology = self.filter.edition.media_type.is_empty() + || self + .filter + .edition + .media_type + .contains(&MediaType::Musicology); + let include_radio = self.filter.edition.media_type.is_empty() + || self.filter.edition.media_type.contains(&MediaType::Radio); + + let mut categories = Vec::new(); + if include_audio { + categories.extend( + self.filter + .edition + .categories + .audio + .clone() + .unwrap_or_else(AudiobookCategory::all) + .into_iter() + .map(|category| category.to_id()), + ); + } + if include_ebook { + categories.extend( + self.filter + .edition + .categories + .ebook + .clone() + .unwrap_or_else(EbookCategory::all) + .into_iter() + .map(|category| category.to_id()), + ); + } + if include_musicology { + categories.extend( + self.filter + .edition + .categories + .musicology + .clone() + .unwrap_or_else(mlm_db::MusicologyCategory::all) + .into_iter() + .map(|category| category.to_id()), + ); + } + if include_radio { + categories.extend( + self.filter + .edition + .categories + .radio + .clone() + .unwrap_or_else(mlm_db::RadioCategory::all) + .into_iter() + .map(|category| category.to_id()), + ); + } + + categories.sort_unstable(); + categories.dedup(); + categories + } +} + impl EditionFilter { pub fn matches(&self, torrent: &MaMTorrent) -> bool { if !self.media_type.is_empty() @@ -251,11 +462,11 @@ impl EditionFilter { true } - pub(crate) fn matches_lib(&self, torrent: &Torrent) -> Result { + pub fn matches_lib(&self, torrent: &Torrent) -> Result { self.matches_meta(&torrent.meta) } - pub(crate) fn matches_meta(&self, meta: &TorrentMeta) -> Result { + pub fn matches_meta(&self, meta: &TorrentMeta) -> Result { if !self.media_type.is_empty() && !self.media_type.contains(&meta.media_type) { return Ok(false); } @@ -378,7 +589,7 @@ impl GoodreadsList { pub fn allow_audio(&self) -> bool { self.grab .iter() - .any(|g| match g.edition.categories.audio.as_ref() { + .any(|g| match g.filter.edition.categories.audio.as_ref() { None => true, Some(c) => !c.is_empty(), }) @@ -387,7 +598,7 @@ impl GoodreadsList { pub fn allow_ebook(&self) -> bool { self.grab .iter() - .any(|g| match g.edition.categories.ebook.as_ref() { + .any(|g| match g.filter.edition.categories.ebook.as_ref() { None => true, Some(c) => !c.is_empty(), }) @@ -424,13 +635,124 @@ impl Library { #[cfg(test)] mod tests { - use mlm_db::{AudiobookCategory, FlagBits, Timestamp, TorrentMeta}; + use std::collections::BTreeMap; + + use mlm_db::{ + AudiobookCategory, EbookCategory, FlagBits, MusicologyCategory, RadioCategory, Timestamp, + TorrentMeta, + }; use time::macros::date; - use mlm_mam::enums::Categories; + use mlm_mam::enums::{Categories, SearchIn}; use super::*; + fn mam_query(search: TorrentSearch) -> BTreeMap> { + let url = Url::parse(&search.mam_search()).unwrap(); + let mut query = BTreeMap::>::new(); + for (key, value) in url.query_pairs() { + query + .entry(key.into_owned()) + .or_default() + .push(value.into_owned()); + } + query + } + + #[test] + fn test_mam_search_encodes_sort_target_and_categories() { + let query = mam_query(TorrentSearch { + kind: Type::Mine, + cost: Cost::Ratio, + query: Some("book".to_string()), + search_in: vec![SearchIn::Title], + sort_by: Some(crate::config::SortBy::LowSeeders), + max_pages: None, + filter: TorrentFilter { + edition: EditionFilter { + media_type: vec![MediaType::Musicology, MediaType::Radio], + categories: Categories { + audio: Some(vec![]), + ebook: Some(vec![]), + musicology: Some(vec![MusicologyCategory::MusicBook]), + radio: Some(vec![RadioCategory::Drama]), + }, + ..Default::default() + }, + ..Default::default() + }, + search_interval: None, + unsat_buffer: None, + max_active_downloads: None, + wedge_buffer: None, + dry_run: false, + mark_removed: false, + category: None, + }); + + assert_eq!( + query.get("tor[searchIn]").unwrap(), + &vec!["mine".to_string()] + ); + assert_eq!( + query.get("tor[sort_type]").unwrap(), + &vec!["seedersAsc".to_string()] + ); + assert_eq!( + query.get("tor[srchIn][title]").unwrap(), + &vec!["true".to_string()] + ); + assert_eq!( + query.get("tor[cat][]").unwrap(), + &vec![ + MusicologyCategory::MusicBook.to_id().to_string(), + RadioCategory::Drama.to_id().to_string(), + ] + ); + } + + #[test] + fn test_mam_search_encodes_uploader_and_ebook_media_type() { + let query = mam_query(TorrentSearch { + kind: Type::Uploader(42), + cost: Cost::Ratio, + query: None, + search_in: vec![], + sort_by: None, + max_pages: None, + filter: TorrentFilter { + edition: EditionFilter { + media_type: vec![MediaType::Ebook], + categories: Categories { + audio: Some(vec![]), + ebook: Some(vec![EbookCategory::ActionAdventure]), + musicology: Some(vec![]), + radio: Some(vec![]), + }, + ..Default::default() + }, + ..Default::default() + }, + search_interval: None, + unsat_buffer: None, + max_active_downloads: None, + wedge_buffer: None, + dry_run: false, + mark_removed: false, + category: None, + }); + + assert_eq!( + query.get("tor[searchIn]").unwrap(), + &vec!["uploader".to_string()] + ); + assert_eq!(query.get("tor[user]").unwrap(), &vec!["42".to_string()]); + assert_eq!( + query.get("tor[cat][]").unwrap(), + &vec![EbookCategory::ActionAdventure.to_id().to_string()] + ); + } + #[test] fn test_uploaded_after() { let torrent = MaMTorrent { @@ -1010,7 +1332,7 @@ mod tests { narrators: vec![], series: vec![], source: MetadataSource::Mam, - uploaded_at: Timestamp::now(), + uploaded_at: Some(Timestamp::now()), } } @@ -1380,10 +1702,7 @@ mod tests { )), ..default_meta() }); - assert!( - filter.matches_lib(&torrent).unwrap(), - "Torrent should pass all checks when all allowed filter criteria match." - ); + assert!(filter.matches_lib(&torrent).unwrap()); } } } diff --git a/mlm_core/src/context.rs b/mlm_core/src/context.rs new file mode 100644 index 00000000..d157a643 --- /dev/null +++ b/mlm_core/src/context.rs @@ -0,0 +1,33 @@ +use crate::config::Config; +use crate::stats::{Events, Stats}; +use std::sync::Arc; +use tokio::sync::Mutex; + +pub trait Backend: Send + Sync { + fn as_any(&self) -> &dyn std::any::Any; +} + +#[derive(Clone)] +pub struct Context { + pub config: Arc>>, + pub stats: Stats, + pub events: Events, + pub backend: Option>, + pub triggers: Triggers, +} + +#[derive(Clone, Default)] +pub struct Triggers { + pub search_tx: std::collections::BTreeMap>, + pub import_tx: std::collections::BTreeMap>, + pub torrent_linker_tx: Option>, + pub folder_linker_tx: Option>, + pub downloader_tx: Option>, + pub audiobookshelf_tx: Option>, +} + +impl Context { + pub async fn config(&self) -> Arc { + self.config.lock().await.clone() + } +} diff --git a/server/src/exporter.rs b/mlm_core/src/exporter.rs similarity index 100% rename from server/src/exporter.rs rename to mlm_core/src/exporter.rs diff --git a/mlm_core/src/lib.rs b/mlm_core/src/lib.rs new file mode 100644 index 00000000..77f772a2 --- /dev/null +++ b/mlm_core/src/lib.rs @@ -0,0 +1,72 @@ +pub mod audiobookshelf; +pub mod autograbber; +pub mod cleaner; +pub mod config; +pub mod config_impl; +pub mod context; +pub mod exporter; +pub mod linker; +pub mod lists; +pub mod logging; +pub mod metadata; +pub mod qbittorrent; +pub mod runner; +pub mod snatchlist; +pub mod stats; +pub mod torrent_downloader; + +pub use crate::config::Config; +pub use crate::context::{Backend, Context, Triggers}; +pub use crate::stats::{Events, Stats, StatsValues}; + +// Re-export types from mlm_db for convenience +pub use mlm_db::{ + ClientStatus, Event, EventKey, EventType, Flags, Language, LibraryMismatch, MetadataSource, + OldCategory, Timestamp, Torrent, TorrentCost, TorrentKey, ids, +}; + +pub struct SsrBackend { + pub db: std::sync::Arc>, + pub mam: std::sync::Arc>>>, + pub metadata: std::sync::Arc, +} + +impl Backend for SsrBackend { + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +pub trait ContextExt { + fn ssr(&self) -> &SsrBackend; + fn db(&self) -> &std::sync::Arc>; + fn mam(&self) -> anyhow::Result>>; + fn metadata(&self) -> &std::sync::Arc; +} + +impl ContextExt for Context { + fn ssr(&self) -> &SsrBackend { + self.backend + .as_ref() + .expect("Backend not initialized") + .as_any() + .downcast_ref::() + .expect("Failed to downcast to SsrBackend") + } + + fn db(&self) -> &std::sync::Arc> { + &self.ssr().db + } + + fn mam(&self) -> anyhow::Result>> { + let mam = self.ssr().mam.as_ref(); + match mam { + Ok(m) => Ok(m.clone()), + Err(e) => Err(anyhow::anyhow!(e.to_string())), + } + } + + fn metadata(&self) -> &std::sync::Arc { + &self.ssr().metadata + } +} diff --git a/server/src/linker/common.rs b/mlm_core/src/linker/common.rs similarity index 100% rename from server/src/linker/common.rs rename to mlm_core/src/linker/common.rs diff --git a/server/src/linker/duplicates.rs b/mlm_core/src/linker/duplicates.rs similarity index 97% rename from server/src/linker/duplicates.rs rename to mlm_core/src/linker/duplicates.rs index 511adfbc..d0c225f4 100644 --- a/server/src/linker/duplicates.rs +++ b/mlm_core/src/linker/duplicates.rs @@ -44,7 +44,7 @@ pub fn rank_torrents(config: &Config, batch: Vec) -> Vec { let mut size = 0; if let Some(library_path) = &torrent.library_path { for file in &torrent.library_files { - let path = library_path.join(file); + let path: std::path::PathBuf = library_path.join(file); size += fs::metadata(path).map_or(0, |s| file_size(&s)); } } @@ -83,7 +83,7 @@ mod tests { media_type: MediaType::Audiobook, main_cat: Some(MainCat::Fiction), source: MetadataSource::Mam, - uploaded_at: Timestamp::now(), + uploaded_at: Some(Timestamp::now()), authors: vec!["Author".to_string()], language: Some(Language::English), ids: BTreeMap::new(), diff --git a/server/src/linker/folder.rs b/mlm_core/src/linker/folder.rs similarity index 93% rename from server/src/linker/folder.rs rename to mlm_core/src/linker/folder.rs index 20296008..24167dbb 100644 --- a/server/src/linker/folder.rs +++ b/mlm_core/src/linker/folder.rs @@ -28,12 +28,16 @@ use crate::linker::{ use crate::logging::{update_errored_torrent, write_event}; #[instrument(skip_all)] -pub async fn link_folders_to_library(config: Arc, db: Arc>) -> Result<()> { +pub async fn link_folders_to_library( + config: Arc, + db: Arc>, + events: &crate::stats::Events, +) -> Result<()> { for library in &config.libraries { if let Library::ByRipDir(l) = library { let mut entries = read_dir(&l.rip_dir).await?; while let Some(folder) = entries.next_entry().await? { - link_folder(&config, library, &db, folder).await?; + link_folder(&config, library, &db, folder, events).await?; } } } @@ -46,6 +50,7 @@ async fn link_folder( library: &Library, db: &Database<'_>, folder: DirEntry, + events: &crate::stats::Events, ) -> Result<()> { let span = span!( Level::TRACE, @@ -95,9 +100,16 @@ async fn link_folder( trace!("Linking libation folder"); let asin = libation_meta.asin.clone(); let title = libation_meta.title.clone(); - let result = - link_libation_folder(config, library, db, libation_meta, audio_files, ebook_files) - .await; + let result = link_libation_folder( + config, + library, + db, + libation_meta, + audio_files, + ebook_files, + events, + ) + .await; update_errored_torrent(db, ErroredTorrentId::Linker(asin), title, result).await; return Ok(()); } @@ -105,9 +117,16 @@ async fn link_folder( trace!("Linking nextory folder"); let id = nextory_torrent_id(nextory_meta.id); let title = nextory_meta.title.clone(); - let result = - link_nextory_folder(config, library, db, nextory_meta, audio_files, ebook_files) - .await; + let result = link_nextory_folder( + config, + library, + db, + nextory_meta, + audio_files, + ebook_files, + events, + ) + .await; update_errored_torrent(db, ErroredTorrentId::Linker(id), title, result).await; return Ok(()); } @@ -128,10 +147,20 @@ async fn link_libation_folder( libation_meta: Libation, audio_files: Vec, ebook_files: Vec, + events: &crate::stats::Events, ) -> Result<()> { let torrent = build_libation_torrent(library, libation_meta, &audio_files, &ebook_files).await?; - link_prepared_folder_torrent(config, library, db, torrent, audio_files, ebook_files).await + link_prepared_folder_torrent( + config, + library, + db, + torrent, + audio_files, + ebook_files, + events, + ) + .await } async fn link_nextory_folder( @@ -141,9 +170,19 @@ async fn link_nextory_folder( nextory_meta: NextoryRaw, audio_files: Vec, ebook_files: Vec, + events: &crate::stats::Events, ) -> Result<()> { let torrent = build_nextory_torrent(library, nextory_meta, &audio_files, &ebook_files).await?; - link_prepared_folder_torrent(config, library, db, torrent, audio_files, ebook_files).await + link_prepared_folder_torrent( + config, + library, + db, + torrent, + audio_files, + ebook_files, + events, + ) + .await } async fn build_libation_torrent( @@ -355,6 +394,7 @@ async fn link_prepared_folder_torrent( mut torrent: Torrent, audio_files: Vec, ebook_files: Vec, + events: &crate::stats::Events, ) -> Result<()> { let torrent_id = torrent.id.clone(); let r = db.r_transaction()?; @@ -458,6 +498,7 @@ async fn link_prepared_folder_torrent( if let Some(library_path) = library_path { write_event( db, + events, Event::new( Some(torrent_id), None, diff --git a/server/src/linker/libation_cats.rs b/mlm_core/src/linker/libation_cats.rs similarity index 99% rename from server/src/linker/libation_cats.rs rename to mlm_core/src/linker/libation_cats.rs index f058206f..86ea24c8 100644 --- a/server/src/linker/libation_cats.rs +++ b/mlm_core/src/linker/libation_cats.rs @@ -27,6 +27,7 @@ impl CategoryMapping { } } + #[cfg(test)] fn extend(&mut self, other: CategoryMapping) { for category in other.categories { self.push_category(category); diff --git a/server/src/linker/mod.rs b/mlm_core/src/linker/mod.rs similarity index 100% rename from server/src/linker/mod.rs rename to mlm_core/src/linker/mod.rs diff --git a/server/src/linker/torrent.rs b/mlm_core/src/linker/torrent.rs similarity index 89% rename from server/src/linker/torrent.rs rename to mlm_core/src/linker/torrent.rs index 32f5f0d9..a1de367b 100644 --- a/server/src/linker/torrent.rs +++ b/mlm_core/src/linker/torrent.rs @@ -61,6 +61,7 @@ use tokio::fs::create_dir_all; use tracing::{Level, debug, instrument, span, trace}; use crate::{ + Events, audiobookshelf::{self as abs}, autograbber::update_torrent_meta, cleaner::remove_library_files, @@ -256,6 +257,7 @@ pub async fn link_torrents_to_library( db: Arc>, qbit: (&QbitConfig, &Q), mam: &M, + events: &Events, ) -> Result<()> where Q: QbitApi + ?Sized, @@ -271,7 +273,6 @@ where if torrent.progress < 1.0 { continue; } - let library = find_library(&config, &torrent); let r = db.r_transaction()?; let mut existing_torrent: Option = r.get().primary(torrent.hash.clone())?; @@ -309,7 +310,7 @@ where rw.commit()?; } for event in update.events { - write_event(&db, event).await; + write_event(&db, events, event).await; } if t.library_path.is_some() || t.replaced_with.is_some() { @@ -338,12 +339,10 @@ where &torrent, library, existing_torrent, + events, ) .await .context("match_torrent"); - if let Err(e) = &result { - debug!("match_torrent error for {}: {:#}", torrent.hash, e); - } update_errored_torrent( &db, ErroredTorrentId::Linker(torrent.hash.clone()), @@ -391,6 +390,7 @@ async fn match_torrent( torrent: &QbitTorrent, library: &Library, mut existing_torrent: Option, + events: &Events, ) -> Result<()> where Q: QbitApi + ?Sized, @@ -453,6 +453,7 @@ where library, existing_torrent.as_ref(), &meta, + events, ) .await .context("link_torrent") @@ -465,6 +466,7 @@ pub async fn refresh_mam_metadata( db: &Database<'_>, mam: &M, id: String, + events: &Events, ) -> Result<(Torrent, MaMTorrent)> where M: MaMApi + ?Sized, @@ -498,6 +500,7 @@ where meta.clone(), true, false, + events, ) .await?; torrent.meta = meta; @@ -506,7 +509,12 @@ where } #[instrument(skip_all)] -pub async fn relink(config: &Config, db: &Database<'_>, hash: String) -> Result<()> { +pub async fn relink( + config: &Config, + db: &Database<'_>, + hash: String, + events: &Events, +) -> Result<()> { for qbit_conf in &config.qbittorrent { let qbit = match qbit::Api::new_login_username_password( &qbit_conf.url, @@ -537,7 +545,7 @@ pub async fn relink(config: &Config, db: &Database<'_>, hash: String) -> Result< let Some(qbit_torrent) = torrents.pop() else { continue; }; - return relink_internal(config, qbit_conf, db, &qbit, qbit_torrent, hash).await; + return relink_internal(config, qbit_conf, db, &qbit, qbit_torrent, hash, events).await; } bail!("Could not find torrent in qbit"); } @@ -550,6 +558,7 @@ pub async fn relink_internal( qbit: &Q, qbit_torrent: QbitTorrent, hash: String, + events: &Events, ) -> Result<()> where Q: QbitApi + ?Sized, @@ -588,6 +597,7 @@ where library, Some(&torrent), &torrent.meta, + events, ) .await .context("link_torrent") @@ -600,6 +610,7 @@ pub async fn refresh_metadata_relink( db: &Database<'_>, mam: &M, hash: String, + events: &Events, ) -> Result<()> where M: MaMApi + ?Sized, @@ -642,6 +653,7 @@ where mam, qbit_torrent, hash, + events, ) .await; } @@ -649,6 +661,7 @@ where } #[instrument(skip_all)] +#[allow(clippy::too_many_arguments)] pub async fn refresh_metadata_relink_internal( config: &Config, qbit_config: &QbitConfig, @@ -657,6 +670,7 @@ pub async fn refresh_metadata_relink_internal( mam: &M, qbit_torrent: QbitTorrent, hash: String, + events: &Events, ) -> Result<()> where Q: QbitApi + ?Sized, @@ -674,7 +688,8 @@ where if selected_audio_format.is_none() && selected_ebook_format.is_none() { bail!("Could not find any wanted formats in torrent"); } - let (torrent, _mam_torrent) = refresh_mam_metadata(config, db, mam, hash.clone()).await?; + let (torrent, _mam_torrent) = + refresh_mam_metadata(config, db, mam, hash.clone(), events).await?; let library_path_changed = torrent.library_path != library_dir( config.exclude_narrator_in_library_dir, @@ -694,6 +709,7 @@ where library, Some(&torrent), &torrent.meta, + events, ) .await .context("link_torrent") @@ -714,11 +730,10 @@ async fn link_torrent( library: &Library, existing_torrent: Option<&Torrent>, meta: &TorrentMeta, + events: &Events, ) -> Result<()> { let mut library_files = vec![]; - // Removed temporary debug prints that were used during investigation. - let library_path = if library.options().method != LibraryLinkMethod::NoLink { let Some(mut dir) = library_dir(config.exclude_narrator_in_library_dir, library, meta) else { @@ -813,6 +828,7 @@ async fn link_torrent( if let Some(library_path) = library_path { write_event( db, + events, Event::new( Some(hash.to_owned()), meta.mam_id(), @@ -846,7 +862,7 @@ pub fn find_library<'a>(config: &'a Config, torrent: &QbitTorrent) -> Option<&'a if filters .deny_tags .iter() - .any(|tag| torrent.tags.split(", ").any(|t| t == tag.as_str())) + .any(|tag| torrent.tags.split(",").any(|t| t.trim() == tag.as_str())) { return false; } @@ -856,7 +872,7 @@ pub fn find_library<'a>(config: &'a Config, torrent: &QbitTorrent) -> Option<&'a filters .allow_tags .iter() - .any(|tag| torrent.tags.split(", ").any(|t| t == tag.as_str())) + .any(|tag| torrent.tags.split(",").any(|t| t.trim() == tag.as_str())) }) } @@ -882,18 +898,44 @@ mod tests { fn test_find_library_by_download_dir() { let cfg = Config { mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], libraries: vec![Library::ByDownloadDir(LibraryByDownloadDir { download_dir: PathBuf::from("/downloads"), options: LibraryOptions { name: None, - library_dir: PathBuf::from("/lib"), + library_dir: PathBuf::from("/library"), method: LibraryLinkMethod::Hardlink, audio_types: None, ebook_types: None, }, - tag_filters: LibraryTagFilters::default(), + tag_filters: LibraryTagFilters { + allow_tags: vec![], + deny_tags: vec![], + }, })], - ..Default::default() + metadata_providers: vec![], }; let qbit_torrent = qbit::models::Torrent { @@ -913,6 +955,29 @@ mod tests { fn test_find_library_by_category() { let cfg = Config { mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], libraries: vec![Library::ByCategory(LibraryByCategory { category: "audiobooks".to_string(), options: LibraryOptions { @@ -927,7 +992,7 @@ mod tests { deny_tags: vec![], }, })], - ..Default::default() + metadata_providers: vec![], }; let qbit_torrent = qbit::models::Torrent { @@ -947,6 +1012,29 @@ mod tests { fn test_find_library_skips_rip_dir() { let cfg = Config { mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], libraries: vec![Library::ByRipDir(crate::config::LibraryByRipDir { rip_dir: PathBuf::from("/rip"), options: LibraryOptions { @@ -958,7 +1046,7 @@ mod tests { }, filter: crate::config::EditionFilter::default(), })], - ..Default::default() + metadata_providers: vec![], }; let qbit_torrent = qbit::models::Torrent { @@ -1099,7 +1187,7 @@ mod tests { narrators: vec![], series: vec![], source: mlm_db::MetadataSource::Mam, - uploaded_at: mlm_db::Timestamp::now(), + uploaded_at: Some(mlm_db::Timestamp::now()), }, created_at: mlm_db::Timestamp::now(), replaced_with: None, @@ -1112,7 +1200,31 @@ mod tests { }; let cfg = Config { mam_id: "m".to_string(), - ..Default::default() + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![], + metadata_providers: vec![], }; let update = check_torrent_updates(&mut torrent, &qbit_torrent, None, &cfg, &[]); @@ -1387,7 +1499,7 @@ mod tests { narrators: vec![], series: vec![], source: mlm_db::MetadataSource::Mam, - uploaded_at: mlm_db::Timestamp::now(), + uploaded_at: Some(mlm_db::Timestamp::now()), }, created_at: mlm_db::Timestamp::now(), replaced_with: None, @@ -1399,7 +1511,31 @@ mod tests { fn mock_config() -> Config { Config { mam_id: "m".to_string(), - ..Default::default() + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![], + metadata_providers: vec![], } } diff --git a/server/src/lists/goodreads.rs b/mlm_core/src/lists/goodreads.rs similarity index 96% rename from server/src/lists/goodreads.rs rename to mlm_core/src/lists/goodreads.rs index 4a4761b7..5a476b56 100644 --- a/server/src/lists/goodreads.rs +++ b/mlm_core/src/lists/goodreads.rs @@ -17,9 +17,9 @@ use serde::Deserialize; use tokio::time::sleep; use tracing::{debug, instrument, trace, warn}; +use crate::config::{Config, Cost, GoodreadsList, Grab}; use crate::{ autograbber::select_torrents, - config::{Config, Cost, GoodreadsList, Grab}, lists::{search_grab, search_library}, }; @@ -35,6 +35,7 @@ pub async fn run_goodreads_import( mam: Arc>, list: &GoodreadsList, max_torrents: u64, + events: &crate::stats::Events, ) -> Result<()> { // Make sure we are only running one import at a time let _guard = IMPORT_MUTEX.lock().await; @@ -131,15 +132,25 @@ pub async fn run_goodreads_import( } }; trace!("Searching for book {} from Goodreads list", item.title); - search_item(&config, &db, &mam, list, &item, db_item, max_torrents) - .await - .context("search goodreads book")?; + search_item( + &config, + &db, + &mam, + list, + &item, + db_item, + max_torrents, + events, + ) + .await + .context("search goodreads book")?; sleep(Duration::from_millis(400)).await; } Ok(()) } +#[allow(clippy::too_many_arguments)] #[instrument(skip_all)] async fn search_item( config: &Config, @@ -149,6 +160,7 @@ async fn search_item( item: &Item, mut db_item: ListItem, max_torrents: u64, + events: &crate::stats::Events, ) -> Result { if !db_item.want_audio() && !db_item.want_ebook() { return Ok(0); @@ -257,6 +269,7 @@ async fn search_item( list.dry_run, max_torrents, item.book_id, + events, ) .await .context("select_torrents")?; @@ -275,6 +288,7 @@ async fn search_item( list.dry_run, max_torrents, item.book_id, + events, ) .await .context("select_torrents")?; diff --git a/server/src/lists/mod.rs b/mlm_core/src/lists/mod.rs similarity index 98% rename from server/src/lists/mod.rs rename to mlm_core/src/lists/mod.rs index 19ad9b37..d4541887 100644 --- a/server/src/lists/mod.rs +++ b/mlm_core/src/lists/mod.rs @@ -3,6 +3,8 @@ mod notion; use std::{borrow::Cow, sync::Arc}; +use crate::config::{Config, GoodreadsList, Grab, NotionList}; +use crate::lists::{goodreads::run_goodreads_import, notion::run_notion_import}; use anyhow::{Context, Result}; use itertools::Itertools; use matchr::score; @@ -23,11 +25,6 @@ use serde_json::Value; use tokio::sync::watch::Sender; use tracing::{debug, instrument, trace}; -use crate::{ - config::{Config, GoodreadsList, Grab, NotionList}, - lists::{goodreads::run_goodreads_import, notion::run_notion_import}, -}; - pub enum List { Goodreads(GoodreadsList), Notion(NotionList), @@ -82,6 +79,7 @@ pub async fn run_list_import( list: Arc, index: usize, autograb_trigger: Sender<()>, + events: &crate::stats::Events, ) -> Result<()> { let user_info = mam.user_info().await?; let max_torrents = user_info.unsat.limit.saturating_sub(user_info.unsat.count); @@ -98,10 +96,10 @@ pub async fn run_list_import( if max_torrents > 0 { match list.as_ref() { List::Goodreads(list) => { - run_goodreads_import(config, db, mam, list, max_torrents).await?; + run_goodreads_import(config, db, mam, list, max_torrents, events).await?; } List::Notion(list) => { - run_notion_import(config, db, mam, list, max_torrents).await?; + run_notion_import(config, db, mam, list, max_torrents, events).await?; } } } diff --git a/server/src/lists/notion.rs b/mlm_core/src/lists/notion.rs similarity index 97% rename from server/src/lists/notion.rs rename to mlm_core/src/lists/notion.rs index c47e9dd7..2d86e6e1 100644 --- a/server/src/lists/notion.rs +++ b/mlm_core/src/lists/notion.rs @@ -9,10 +9,8 @@ use serde_json::Value; use tokio::time::sleep; use tracing::{instrument, trace}; -use crate::{ - autograbber::select_torrents, - config::{Config, NotionList}, -}; +use crate::autograbber::select_torrents; +use crate::config::{Config, NotionList}; static IMPORT_MUTEX: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); @@ -23,6 +21,7 @@ pub async fn run_notion_import( mam: Arc>, list: &NotionList, max_torrents: u64, + events: &crate::stats::Events, ) -> Result<()> { // Make sure we are only running one import at a time let _guard = IMPORT_MUTEX.lock().await; @@ -85,6 +84,7 @@ pub async fn run_notion_import( list.dry_run, max_torrents, None, + events, ) .await .context("select_torrents")?; diff --git a/server/src/logging.rs b/mlm_core/src/logging.rs similarity index 92% rename from server/src/logging.rs rename to mlm_core/src/logging.rs index 3ffabf69..773c0d60 100644 --- a/server/src/logging.rs +++ b/mlm_core/src/logging.rs @@ -6,6 +6,8 @@ use tracing::{error, warn}; use mlm_db::{DatabaseExt, ErroredTorrent, ErroredTorrentId, Event, Timestamp, TorrentMeta}; +use crate::stats::Events; + #[derive(Debug)] pub struct TorrentMetaError(pub TorrentMeta, pub anyhow::Error); impl Display for TorrentMetaError { @@ -59,12 +61,14 @@ pub async fn update_errored_torrent( } } -pub async fn write_event(db: &Database<'_>, event: Event) { +pub async fn write_event(db: &Database<'_>, events: &Events, event: Event) { if let Err(err) = db.rw_async().await.and_then(|(_guard, rw)| { rw.upsert(event.clone())?; rw.commit()?; Ok(()) }) { error!("Error writing event: {err:?}, event: {event:?}"); + } else { + let _ = events.event.0.send(Some(event)); } } diff --git a/server/src/metadata/mam_meta.rs b/mlm_core/src/metadata/mam_meta.rs similarity index 95% rename from server/src/metadata/mam_meta.rs rename to mlm_core/src/metadata/mam_meta.rs index f7280417..adf8a5ee 100644 --- a/server/src/metadata/mam_meta.rs +++ b/mlm_core/src/metadata/mam_meta.rs @@ -1,4 +1,4 @@ -use crate::stats::Context; +use crate::{Context, ContextExt}; use anyhow::Result; use mlm_db::TorrentMeta; @@ -26,7 +26,11 @@ pub async fn match_meta( // centralized MetadataService attached to the Context. This keeps // provider configuration in one place and avoids duplicating instantiation // logic here. - let fetched = ctx.metadata.fetch_provider(ctx, query, provider_id).await?; + let fetched = ctx + .ssr() + .metadata + .fetch_provider(ctx, query, provider_id) + .await?; // Merge fetched metadata into original meta: only overwrite fields when // the provider supplied non-empty / non-default values. This preserves diff --git a/server/src/metadata/mod.rs b/mlm_core/src/metadata/mod.rs similarity index 98% rename from server/src/metadata/mod.rs rename to mlm_core/src/metadata/mod.rs index 1e56db25..e385aaf3 100644 --- a/server/src/metadata/mod.rs +++ b/mlm_core/src/metadata/mod.rs @@ -1,4 +1,4 @@ -use crate::stats::Context; +use crate::{Context, ContextExt}; use anyhow::Result; use mlm_db::DatabaseExt as _; use mlm_db::{Event, EventType, MetadataSource, TorrentMeta}; @@ -158,7 +158,7 @@ impl MetadataService { }; // Insert event into DB using async rw transaction helper from mlm_db - let (guard, rw) = ctx.db.rw_async().await?; + let (guard, rw) = ctx.db().rw_async().await?; rw.insert(ev)?; rw.commit()?; drop(guard); diff --git a/server/src/qbittorrent.rs b/mlm_core/src/qbittorrent.rs similarity index 88% rename from server/src/qbittorrent.rs rename to mlm_core/src/qbittorrent.rs index ea03d17e..a56c74da 100644 --- a/server/src/qbittorrent.rs +++ b/mlm_core/src/qbittorrent.rs @@ -101,9 +101,11 @@ impl QbitApi for Arc { const CATEGORY_CACHE_TTL_SECS: u64 = 60; +type CategoryCacheValue = (HashMap, Instant); + #[derive(Clone)] pub struct CategoryCache { - cache: Arc, Instant)>>>, + cache: Arc>>, } impl CategoryCache { @@ -118,18 +120,22 @@ impl CategoryCache { qbit: &Q, url: &str, ) -> Result> { - let now = Instant::now(); let cache = self.cache.read().await; - if let Some((categories, cached_at)) = cache.get(url) { - if now.duration_since(*cached_at) < Duration::from_secs(CATEGORY_CACHE_TTL_SECS) { - return Ok(categories.clone()); - } + if let Some((categories, cached_at)) = cache.get(url) + && now.duration_since(*cached_at) < Duration::from_secs(CATEGORY_CACHE_TTL_SECS) + { + return Ok(categories.clone()); } drop(cache); - let categories: HashMap = qbit.categories().await?.into_keys().map(|k| (k, ())).collect(); + let categories: HashMap = qbit + .categories() + .await? + .into_keys() + .map(|k| (k, ())) + .collect(); let mut cache = self.cache.write().await; cache.insert(url.to_string(), (categories.clone(), now)); @@ -178,16 +184,18 @@ pub async fn add_torrent_with_category( url: &str, add_torrent: AddTorrent, ) -> Result<()> { - if let Some(ref category) = add_torrent.category { - if !category.is_empty() { - ensure_category_exists(qbit, url, category).await?; - } + if let Some(ref category) = add_torrent.category + && !category.is_empty() + { + ensure_category_exists(qbit, url, category).await?; } - qbit.add_torrent(add_torrent).await.map_err(|e| anyhow::Error::new(e)) + qbit.add_torrent(add_torrent) + .await + .map_err(anyhow::Error::new) } -pub async fn get_torrent<'a, 'b>( +pub async fn get_torrent<'a>( config: &'a Config, hash: &str, ) -> Result> { @@ -201,15 +209,13 @@ pub async fn get_torrent<'a, 'b>( else { continue; }; - let Some(torrent) = qbit + let torrents = qbit .torrents(Some(TorrentListParams { hashes: Some(vec![hash.to_string()]), ..TorrentListParams::default() })) - .await? - .into_iter() - .next() - else { + .await?; + let Some(torrent) = torrents.into_iter().next() else { continue; }; return Ok(Some((torrent, qbit, qbit_conf))); diff --git a/mlm_core/src/runner.rs b/mlm_core/src/runner.rs new file mode 100644 index 00000000..407b6732 --- /dev/null +++ b/mlm_core/src/runner.rs @@ -0,0 +1,594 @@ +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context as _, Result}; +use mlm_mam::api::MaM; +use qbit; +use time::OffsetDateTime; +use tokio::{ + select, + sync::{Mutex, watch}, + time::sleep, +}; +use tracing::error; + +use crate::{ + Context, SsrBackend, Stats, Triggers, + audiobookshelf::match_torrents_to_abs, + autograbber::run_autograbber, + cleaner::run_library_cleaner, + config::Config, + linker::{folder::link_folders_to_library, torrent::link_torrents_to_library}, + lists::{get_lists, run_list_import}, + snatchlist::run_snatchlist_search, + stats::Events, + torrent_downloader::grab_selected_torrents, +}; + +pub fn spawn_tasks( + config: Arc, + db: Arc>, + mam: Arc>>>, + stats: Stats, + metadata: Arc, +) -> Context { + let events = Events::new(); + let (mut search_tx, mut search_rx) = (BTreeMap::new(), BTreeMap::new()); + let (mut import_tx, mut import_rx) = (BTreeMap::new(), BTreeMap::new()); + let (torrent_linker_tx, torrent_linker_rx) = watch::channel(()); + let (folder_linker_tx, folder_linker_rx) = watch::channel(()); + let (downloader_tx, mut downloader_rx) = watch::channel(()); + let (audiobookshelf_tx, mut audiobookshelf_rx) = watch::channel(()); + + for (i, _) in config.autograbs.iter().enumerate() { + let (tx, rx) = watch::channel(()); + search_tx.insert(i, tx); + search_rx.insert(i, rx); + } + for (i, _) in config.snatchlist.iter().enumerate() { + let (tx, rx) = watch::channel(()); + let idx = i + config.autograbs.len(); + search_tx.insert(idx, tx); + search_rx.insert(idx, rx); + } + for (i, _) in get_lists(&config).iter().enumerate() { + let (tx, rx) = watch::channel(()); + import_tx.insert(i, tx); + import_rx.insert(i, rx); + } + + // Downloader task + { + let config = config.clone(); + let db = db.clone(); + let mam = mam.clone(); + let stats = stats.clone(); + let events = events.clone(); + tokio::spawn(async move { + if let Some(qbit_conf) = config.qbittorrent.first() { + let mut qbit: Option = None; + loop { + if downloader_rx.changed().await.is_err() { + break; + } + if qbit.is_none() { + match qbit::Api::new_login_username_password( + &qbit_conf.url, + &qbit_conf.username, + &qbit_conf.password, + ) + .await + { + Ok(q) => qbit = Some(q), + Err(err) => { + error!("Error logging in to qbit {}: {err}", qbit_conf.url); + stats + .update(|stats| { + stats.downloader_run_at = Some(OffsetDateTime::now_utc()); + stats.downloader_result = Some(Err(err.into())); + }) + .await; + } + }; + } + let Some(qbit_api) = &qbit else { + continue; + }; + let Ok(mam_api) = mam.as_ref() else { + continue; + }; + { + stats + .update(|stats| { + stats.downloader_run_at = Some(OffsetDateTime::now_utc()); + stats.downloader_result = None; + }) + .await; + } + let result = grab_selected_torrents( + &config, + &db, + qbit_api, + &qbit_conf.url, + mam_api, + &events, + ) + .await + .context("grab_selected_torrents"); + + if let Err(err) = &result { + error!("Error grabbing selected torrents: {err:?}"); + } + { + stats + .update(|stats| { + stats.downloader_result = Some(result); + }) + .await; + } + } + } + }); + } + + // Autograbber tasks + for (i, grab) in config.autograbs.iter().enumerate() { + let config = config.clone(); + let db = db.clone(); + let mam = mam.clone(); + let downloader_tx = downloader_tx.clone(); + let mut rx = search_rx.remove(&i).unwrap(); + let stats = stats.clone(); + let events = events.clone(); + let grab = Arc::new(grab.clone()); + tokio::spawn(async move { + loop { + let interval = grab.search_interval.unwrap_or(config.search_interval); + if interval > 0 { + select! { + _ = sleep(Duration::from_secs(60 * interval)) => {}, + result = rx.changed() => { + if let Err(err) = result { + error!("Error listening on search_rx: {err:?}"); + stats.update(|stats| { + stats.autograbber_result.insert(i, Err(err.into())); + }).await; + break; + } + }, + } + } else { + let result = rx.changed().await; + if let Err(err) = result { + error!("Error listening on search_rx: {err:?}"); + stats + .update(|stats| { + stats.autograbber_result.insert(i, Err(err.into())); + }) + .await; + break; + } + } + { + stats + .update(|stats| { + stats + .autograbber_run_at + .insert(i, OffsetDateTime::now_utc()); + stats.autograbber_result.remove(&i); + }) + .await; + } + let Ok(mam_api) = mam.as_ref() else { + continue; + }; + let result = run_autograbber( + config.clone(), + db.clone(), + mam_api.clone(), + downloader_tx.clone(), + i, + grab.clone(), + &events, + ) + .await + .context("autograbbers"); + if let Err(err) = &result { + error!("Error running autograbbers: {err:?}"); + } + { + stats + .update(|stats| { + stats.autograbber_result.insert(i, result); + }) + .await; + } + } + }); + } + + // Snatchlist tasks + for (i, grab) in config.snatchlist.iter().enumerate() { + let idx = i + config.autograbs.len(); + let config = config.clone(); + let db = db.clone(); + let mam = mam.clone(); + let mut rx = search_rx.remove(&idx).unwrap(); + let stats = stats.clone(); + let grab = Arc::new(grab.clone()); + let events = events.clone(); + tokio::spawn(async move { + loop { + let interval = grab.search_interval.unwrap_or(config.search_interval); + if interval > 0 { + select! { + _ = sleep(Duration::from_secs(60 * interval)) => {}, + result = rx.changed() => { + if let Err(err) = result { + error!("Error listening on search_rx for snatchlist: {err:?}"); + stats.update(|stats| { + stats.autograbber_result.insert(idx, Err(err.into())); + }).await; + break; + } + }, + } + } else { + let result = rx.changed().await; + if let Err(err) = result { + error!("Error listening on search_rx for snatchlist: {err:?}"); + stats + .update(|stats| { + stats.autograbber_result.insert(idx, Err(err.into())); + }) + .await; + break; + } + } + { + stats + .update(|stats| { + stats + .autograbber_run_at + .insert(idx, OffsetDateTime::now_utc()); + stats.autograbber_result.remove(&idx); + }) + .await; + } + let Ok(mam_api) = mam.as_ref() else { + continue; + }; + let result = run_snatchlist_search( + config.clone(), + db.clone(), + mam_api.clone(), + idx, + grab.clone(), + &events, + ) + .await + .context("snatchlist_search"); + if let Err(err) = &result { + error!("Error running snatchlist_search: {err:?}"); + } + { + stats + .update(|stats| { + stats.autograbber_result.insert(idx, result); + }) + .await; + } + } + }); + } + + // List import tasks + for (i, list) in get_lists(&config).into_iter().enumerate() { + let config = config.clone(); + let db = db.clone(); + let mam = mam.clone(); + let downloader_tx = downloader_tx.clone(); + let mut rx = import_rx.remove(&i).unwrap(); + let stats = stats.clone(); + let list = Arc::new(list); + let events = events.clone(); + tokio::spawn(async move { + loop { + let interval = list.search_interval().unwrap_or(config.import_interval); + if interval > 0 { + select! { + _ = sleep(Duration::from_secs(60 * interval)) => {}, + result = rx.changed() => { + if let Err(err) = result { + error!("Error listening on import_rx: {err:?}"); + stats.update(|stats| { + stats.import_result.insert(i, Err(err.into())); + }).await; + break; + } + }, + } + } else { + let result = rx.changed().await; + if let Err(err) = result { + error!("Error listening on import_rx: {err:?}"); + stats + .update(|stats| { + stats.import_result.insert(i, Err(err.into())); + }) + .await; + break; + } + } + { + stats + .update(|stats| { + stats.import_run_at.insert(i, OffsetDateTime::now_utc()); + stats.import_result.remove(&i); + }) + .await; + } + let Ok(mam_api) = mam.as_ref() else { + continue; + }; + let result = run_list_import( + config.clone(), + db.clone(), + mam_api.clone(), + list.clone(), + i, + downloader_tx.clone(), + &events, + ) + .await + .context("import"); + if let Err(err) = &result { + error!("Error running import: {err:?}"); + } + { + stats + .update(|stats| { + stats.import_result.insert(i, result); + }) + .await; + } + } + }); + } + + // Torrent linker tasks + for qbit_conf in config.qbittorrent.clone() { + let config = config.clone(); + let db = db.clone(); + let mam = mam.clone(); + let stats = stats.clone(); + let events = events.clone(); + let mut rx = torrent_linker_rx.clone(); + tokio::spawn(async move { + loop { + if config.link_interval > 0 { + select! { + _ = sleep(Duration::from_secs(60 * config.link_interval)) => {}, + result = rx.changed() => { + if let Err(err) = result { + error!("Error listening on link_rx: {err:?}"); + stats + .update(|stats| { + stats.torrent_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.torrent_linker_result = Some(Err(err.into())); + }).await; + break; + } + }, + } + } else if let Err(err) = rx.changed().await { + error!("Error listening on link_rx: {err:?}"); + stats + .update(|stats| { + stats.torrent_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.torrent_linker_result = Some(Err(err.into())); + }) + .await; + break; + } + { + stats + .update(|stats| { + stats.torrent_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.torrent_linker_result = None; + }) + .await; + } + let qbit = match qbit::Api::new_login_username_password( + &qbit_conf.url, + &qbit_conf.username, + &qbit_conf.password, + ) + .await + { + Ok(qbit) => qbit, + Err(err) => { + error!("Error logging in to qbit {}: {err}", qbit_conf.url); + stats + .update(|stats| { + stats.torrent_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.torrent_linker_result = Some(Err(anyhow::Error::msg( + format!("Error logging in to qbit {}: {err}", qbit_conf.url,), + ))); + }) + .await; + continue; + } + }; + let Ok(mam_api) = mam.as_ref() else { + continue; + }; + let result = link_torrents_to_library( + config.clone(), + db.clone(), + (&qbit_conf, &qbit), + mam_api, + &events, + ) + .await + .context("link_torrents_to_library"); + if let Err(err) = &result { + error!("Error running linker: {err:?}"); + } + { + stats + .update(|stats| { + stats.torrent_linker_result = Some(result); + stats.cleaner_run_at = Some(OffsetDateTime::now_utc()); + stats.cleaner_result = None; + }) + .await; + } + let result = run_library_cleaner(config.clone(), db.clone(), &events) + .await + .context("library_cleaner"); + if let Err(err) = &result { + error!("Error running library_cleaner: {err:?}"); + } + { + stats + .update(|stats| { + stats.cleaner_result = Some(result); + }) + .await; + } + } + }); + } + + // Folder linker task + { + let config = config.clone(); + let db = db.clone(); + let stats = stats.clone(); + let events = events.clone(); + let mut rx = folder_linker_rx.clone(); + tokio::spawn(async move { + loop { + select! { + _ = sleep(Duration::from_secs(60 * config.link_interval)) => {}, + result = rx.changed() => { + if let Err(err) = result { + error!("Error listening on link_rx: {err:?}"); + stats + .update(|stats| { + stats.folder_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.folder_linker_result = Some(Err(err.into())); + }).await; + break; + } + }, + } + { + stats + .update(|stats| { + stats.folder_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.folder_linker_result = None; + }) + .await; + } + let result = link_folders_to_library(config.clone(), db.clone(), &events) + .await + .context("link_folders_to_library"); + if let Err(err) = &result { + error!("Error running linker: {err:?}"); + } + { + stats + .update(|stats| { + stats.folder_linker_result = Some(result); + stats.cleaner_run_at = Some(OffsetDateTime::now_utc()); + stats.cleaner_result = None; + }) + .await; + } + let result = run_library_cleaner(config.clone(), db.clone(), &events) + .await + .context("library_cleaner"); + if let Err(err) = &result { + error!("Error running library_cleaner: {err:?}"); + } + { + stats + .update(|stats| { + stats.cleaner_result = Some(result); + }) + .await; + } + } + }); + } + + // Audiobookshelf task + if let Some(abs_config) = &config.audiobookshelf { + let abs_config = abs_config.clone(); + let db = db.clone(); + let stats = stats.clone(); + tokio::spawn(async move { + loop { + select! { + _ = sleep(Duration::from_secs(60 * abs_config.interval)) => {}, + result = audiobookshelf_rx.changed() => { + if let Err(err) = result { + error!("Error listening on audiobookshelf_rx: {err:?}"); + stats + .update(|stats| { + stats.audiobookshelf_result = Some(Err(err.into())); + }) + .await; + break; + } + }, + } + { + stats + .update(|stats| { + stats.audiobookshelf_run_at = Some(OffsetDateTime::now_utc()); + stats.audiobookshelf_result = None; + }) + .await; + } + let result = match_torrents_to_abs(&abs_config, db.clone()) + .await + .context("audiobookshelf_matcher"); + if let Err(err) = &result { + error!("Error running audiobookshelf matcher: {err:?}"); + } + { + stats + .update(|stats| { + stats.audiobookshelf_result = Some(result); + }) + .await; + } + } + }); + } + + let backend = Arc::new(SsrBackend { + db: db.clone(), + mam: mam.clone(), + metadata: metadata.clone(), + }); + + Context { + config: Arc::new(Mutex::new(config)), + stats, + events, + backend: Some(backend), + triggers: Triggers { + search_tx, + import_tx, + torrent_linker_tx: Some(torrent_linker_tx), + folder_linker_tx: Some(folder_linker_tx), + downloader_tx: Some(downloader_tx), + audiobookshelf_tx: Some(audiobookshelf_tx), + }, + } +} diff --git a/server/src/snatchlist.rs b/mlm_core/src/snatchlist.rs similarity index 96% rename from server/src/snatchlist.rs rename to mlm_core/src/snatchlist.rs index 49e191d7..21d1b56d 100644 --- a/server/src/snatchlist.rs +++ b/mlm_core/src/snatchlist.rs @@ -14,10 +14,8 @@ use tokio::{sync::MutexGuard, time::sleep}; use tracing::{Level, debug, enabled, info, instrument, trace, warn}; use uuid::Uuid; -use crate::{ - config::{Config, Cost, SnatchlistSearch, TorrentFilter}, - logging::write_event, -}; +use crate::config::{Config, Cost, SnatchlistSearch, TorrentFilter}; +use crate::logging::write_event; #[instrument(skip_all)] pub async fn run_snatchlist_search( @@ -26,6 +24,7 @@ pub async fn run_snatchlist_search( mam: Arc>, index: usize, snatchlist_config: Arc, + events: &crate::stats::Events, ) -> Result<()> { if !snatchlist_config.filter.edition.languages.is_empty() { bail!("Language filtering is not supported in snatchlist searches"); @@ -48,7 +47,7 @@ pub async fn run_snatchlist_search( .unwrap_or_else(|| index.to_string()); debug!("snatchlist {}", name); - search_and_update_torrents(&config, &db, &snatchlist_config, &mam) + search_and_update_torrents(&config, &db, &snatchlist_config, &mam, events) .await .context("search_torrents")?; @@ -61,6 +60,7 @@ async fn search_and_update_torrents( db: &Database<'_>, torrent_search: &SnatchlistSearch, mam: &MaM<'_>, + events: &crate::stats::Events, ) -> Result<()> { let max_pages = torrent_search.max_pages.unwrap_or(100); let now = UtcDateTime::now(); @@ -99,6 +99,7 @@ async fn search_and_update_torrents( &torrent_search.filter, torrent_search.cost, torrent_search.dry_run, + events, ) .await .context("update_torrents")?; @@ -121,6 +122,7 @@ async fn update_torrents>( grabber: &TorrentFilter, cost: Cost, dry_run: bool, + events: &crate::stats::Events, ) -> Result<()> { 'torrent: for torrent in torrents { if config.ignore_torrents.contains(&torrent.id) { @@ -159,6 +161,7 @@ async fn update_torrents>( old, meta, cost == Cost::MetadataOnlyAdd, + events, ) .await?; } @@ -237,6 +240,7 @@ async fn update_torrent_meta( mut torrent: Torrent, mut meta: TorrentMeta, linker_is_owner: bool, + events: &crate::stats::Events, ) -> Result<()> { // These are missing in user details torrent response, so keep the old values meta.ids = torrent.meta.ids.clone(); @@ -303,6 +307,7 @@ async fn update_torrent_meta( let mam_id = mam_torrent.map(|m| m.id); write_event( db, + events, Event::new( Some(id), mam_id, diff --git a/server/src/stats.rs b/mlm_core/src/stats.rs similarity index 59% rename from server/src/stats.rs rename to mlm_core/src/stats.rs index e22e7784..12561b95 100644 --- a/server/src/stats.rs +++ b/mlm_core/src/stats.rs @@ -1,18 +1,11 @@ -use std::{collections::BTreeMap, sync::Arc}; - use anyhow::Result; -use mlm_db::Event; -use mlm_mam::api::MaM; -use native_db::Database; +use std::{collections::BTreeMap, sync::Arc}; use time::{OffsetDateTime, UtcDateTime}; use tokio::sync::{ Mutex, watch::{self, Receiver, Sender}, }; -use crate::config::Config; -use crate::metadata::MetadataService; - #[derive(Default)] pub struct StatsValues { pub autograbber_run_at: BTreeMap, @@ -31,6 +24,28 @@ pub struct StatsValues { pub audiobookshelf_result: Option>, } +#[derive(Clone)] +pub struct Events { + pub event: ( + Sender>, + Receiver>, + ), +} + +impl Events { + pub fn new() -> Self { + Self { + event: watch::channel(None), + } + } +} + +impl Default for Events { + fn default() -> Self { + Self::new() + } +} + #[derive(Clone)] pub struct Stats { pub values: Arc>, @@ -48,7 +63,7 @@ impl Stats { pub async fn update(&self, f: impl FnOnce(&mut StatsValues)) { let mut data = self.values.lock().await; f(&mut data); - self.values_updated.0.send(UtcDateTime::now()).unwrap(); + let _ = self.values_updated.0.send(UtcDateTime::now()); } pub fn updates(&self) -> Receiver { @@ -61,43 +76,3 @@ impl Default for Stats { Self::new() } } - -#[derive(Clone)] -pub struct Events { - pub event: (Sender>, Receiver>), -} - -#[derive(Clone)] -pub struct Triggers { - pub search_tx: BTreeMap>, - pub import_tx: BTreeMap>, - pub torrent_linker_tx: Sender<()>, - pub folder_linker_tx: Sender<()>, - pub downloader_tx: Sender<()>, - pub audiobookshelf_tx: Sender<()>, -} - -#[derive(Clone)] -pub struct Context { - pub config: Arc>>, - pub db: Arc>, - pub mam: Arc>>>, - pub stats: Stats, - pub metadata: Arc, - // pub events: Events, - pub triggers: Triggers, -} - -impl Context { - pub async fn config(&self) -> Arc { - self.config.lock().await.clone() - } - - pub fn mam(&self) -> Result>> { - let Ok(mam) = self.mam.as_ref() else { - return Err(anyhow::Error::msg("mam_id error")); - }; - - Ok(mam.clone()) - } -} diff --git a/server/src/torrent_downloader.rs b/mlm_core/src/torrent_downloader.rs similarity index 98% rename from server/src/torrent_downloader.rs rename to mlm_core/src/torrent_downloader.rs index 459d2271..26ec835f 100644 --- a/server/src/torrent_downloader.rs +++ b/mlm_core/src/torrent_downloader.rs @@ -29,6 +29,7 @@ pub async fn grab_selected_torrents( qbit: &qbit::Api, qbit_url: &str, mam: &MaM<'_>, + events: &crate::stats::Events, ) -> Result<()> { let selected_torrents = { let r = db.r_transaction()?; @@ -77,7 +78,7 @@ pub async fn grab_selected_torrents( continue; } - let result = grab_torrent(config, db, qbit, qbit_url, mam, torrent.clone()) + let result = grab_torrent(config, db, qbit, qbit_url, mam, torrent.clone(), events) .await .map_err(|err| anyhow::Error::new(TorrentMetaError(torrent.meta.clone(), err))); @@ -107,6 +108,7 @@ async fn grab_torrent( qbit_url: &str, mam: &MaM<'_>, torrent: SelectedTorrent, + events: &crate::stats::Events, ) -> Result<()> { info!( "Grabbing torrent \"{}\", with category {:?} and tags {:?}", @@ -276,6 +278,7 @@ async fn grab_torrent( write_event( db, + events, Event::new( Some(hash), Some(mam_id), diff --git a/mlm_db/Cargo.toml b/mlm_db/Cargo.toml index a39f6887..fc974395 100644 --- a/mlm_db/Cargo.toml +++ b/mlm_db/Cargo.toml @@ -6,7 +6,6 @@ edition = "2024" [dependencies] anyhow = "1.0.100" itertools = "0.14.0" -# native_db = "0.8.2" native_db = { git = "https://github.com/StirlingMouse/native_db.git", branch = "0.8.x" } native_model = "0.4.20" matchr = "0.2.5" @@ -22,6 +21,6 @@ time = { version = "0.3.41", features = [ "macros", "serde", ] } -tokio = { version = "1.45.1", features = ["full"] } +tokio = { version = "1.45.1", features = ["sync"] } tracing = "0.1" uuid = { version = "1.17.0", features = ["serde", "v4"] } diff --git a/mlm_db/src/lib.rs b/mlm_db/src/lib.rs index b1094cc7..eeb53b17 100644 --- a/mlm_db/src/lib.rs +++ b/mlm_db/src/lib.rs @@ -190,7 +190,7 @@ pub enum OldMainCat { Radio, } -#[derive(Clone, Default, Debug, Deserialize)] +#[derive(Clone, Default, Debug, Deserialize, Serialize)] #[serde(try_from = "HashMap")] pub struct Flags { pub crude_language: Option, diff --git a/mlm_db/src/v03.rs b/mlm_db/src/v03.rs index 42b67abf..cd47710c 100644 --- a/mlm_db/src/v03.rs +++ b/mlm_db/src/v03.rs @@ -249,7 +249,7 @@ impl ToKey for Timestamp { } #[derive(Serialize, Deserialize, Eq, PartialEq, Debug, Clone, Hash)] -pub struct Uuid(uuid::Uuid); +pub struct Uuid(pub uuid::Uuid); impl Uuid { pub fn new() -> Self { Self(uuid::Uuid::new_v4()) diff --git a/mlm_mam/Cargo.toml b/mlm_mam/Cargo.toml index d0a7e033..ca4b47ed 100644 --- a/mlm_mam/Cargo.toml +++ b/mlm_mam/Cargo.toml @@ -8,7 +8,7 @@ anyhow = "1.0.100" bytes = "1.11.0" cookie = "0.18.1" itertools = "0.14.0" -mlm_db = { path = "../mlm_db" } +mlm_db = { path = "../mlm_db", default-features = false } mlm_parse = { path = "../mlm_parse" } native_db = { git = "https://github.com/StirlingMouse/native_db.git", branch = "0.8.x" } native_model = "0.4.20" @@ -29,5 +29,5 @@ time = { version = "0.3.41", features = [ "macros", "serde", ] } -tokio = { version = "1.45.1", features = ["full"] } +tokio = { version = "1.45.1", features = ["sync"] } tracing = "0.1" diff --git a/mlm_meta/Cargo.toml b/mlm_meta/Cargo.toml index bc6d4c0f..e427ba07 100644 --- a/mlm_meta/Cargo.toml +++ b/mlm_meta/Cargo.toml @@ -11,11 +11,10 @@ reqwest = { version = "0.12.20", default-features = false, features = ["json", " tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros"] } serde_json = "1.0" scraper = "0.14" -mlm_db = { path = "../mlm_db" } +mlm_db = { path = "../mlm_db", default-features = false } mlm_parse = { path = "../mlm_parse" } strsim = "0.11" tracing = "0.1" - urlencoding = "2.1" url = "2.4" diff --git a/mlm_meta/tests/mock_openlibrary.rs b/mlm_meta/tests/mock_openlibrary.rs index 2067fc20..3239792b 100644 --- a/mlm_meta/tests/mock_openlibrary.rs +++ b/mlm_meta/tests/mock_openlibrary.rs @@ -2,42 +2,33 @@ use anyhow::Result; use mlm_meta::http::HttpClient; use std::sync::Arc; -fn resolve_plan_file(rel: &str) -> std::io::Result { - let mut dir = std::env::current_dir()?; - loop { - let candidate = dir.join(rel); - if candidate.exists() { - return Ok(candidate); - } - if !dir.pop() { - break; - } - } - Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("could not find {}", rel), - )) -} - pub struct MockOpenLibraryClient; #[async_trait::async_trait] impl HttpClient for MockOpenLibraryClient { async fn get(&self, url: &str) -> Result { let u = url::Url::parse(url).map_err(|e| anyhow::anyhow!(e))?; - let rel = if u.host_str().is_some_and(|h| h.contains("openlibrary.org")) { - if u.path().starts_with("/search.json") { - "plan/openlibrary/search.json" - } else { - return Err(anyhow::anyhow!("unexpected path: {}", u.path())); - } - } else { + if !u.host_str().is_some_and(|h| h.contains("openlibrary.org")) { return Err(anyhow::anyhow!("unexpected host in test fetch")); - }; + } + if !u.path().starts_with("/search.json") { + return Err(anyhow::anyhow!("unexpected path: {}", u.path())); + } - let p = resolve_plan_file(rel).map_err(|e| anyhow::anyhow!(e))?; - let s = std::fs::read_to_string(p).map_err(|e| anyhow::anyhow!(e))?; - Ok(s) + Ok(r#"{ + "numFound": 1, + "docs": [ + { + "title": "The Lord of the Rings", + "author_name": ["J.R.R. Tolkien"], + "isbn": ["9780261102385", "0261102389"], + "subject": ["Fantasy fiction", "Middle Earth", "Epic fantasy"], + "first_publish_year": 1954, + "edition_count": 120 + } + ] +}"# + .to_string()) } async fn post( diff --git a/mlm_web_askama/Cargo.toml b/mlm_web_askama/Cargo.toml new file mode 100644 index 00000000..416cedde --- /dev/null +++ b/mlm_web_askama/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "mlm_web_askama" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.100" +askama = { version = "0.14.0", features = ["code-in-doc", "serde_json"] } +axum = { version = "0.8.4", features = ["query", "macros"] } +axum-extra = { version = "0.10.1", features = ["form"] } +futures = "0.3" +itertools = "0.14.0" +mlm_core = { path = "../mlm_core" } +mlm_db = { path = "../mlm_db" } +mlm_mam = { path = "../mlm_mam" } +mlm_parse = { path = "../mlm_parse" } +native_db = { git = "https://github.com/StirlingMouse/native_db.git", branch = "0.8.x" } +once_cell = "1.21.3" +qbit = { git = "https://github.com/StirlingMouse/qbittorrent-webui-api.git" } +regex = "1.12.2" +reqwest = { version = "0.12.20", default-features = false, features = [ + "json", + "rustls-tls", +] } +serde = "1.0.136" +serde_json = "1.0.140" +sublime_fuzzy = "0.7.0" +thiserror = "2.0.17" +time = { version = "0.3.41", features = [ + "formatting", + "local-offset", + "macros", + "serde", +] } +tokio = { version = "1.45.1", features = ["fs", "sync"] } +tokio-stream = { version = "0.1.17", features = ["sync"] } +tokio-util = "0.7" +tower = "0.5.2" +tower-http = { version = "0.6.6", features = ["fs"] } +toml = "0.8.23" +tracing = "0.1" +urlencoding = "2.1.3" diff --git a/mlm_web_askama/build.rs b/mlm_web_askama/build.rs new file mode 100644 index 00000000..c8a19665 --- /dev/null +++ b/mlm_web_askama/build.rs @@ -0,0 +1,11 @@ +use std::time::SystemTime; + +fn main() { + let now = SystemTime::now(); + println!( + "cargo:rustc-env=DATE={}", + now.duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + ); +} diff --git a/server/src/web/api/mod.rs b/mlm_web_askama/src/api/mod.rs similarity index 100% rename from server/src/web/api/mod.rs rename to mlm_web_askama/src/api/mod.rs diff --git a/server/src/web/api/search.rs b/mlm_web_askama/src/api/search.rs similarity index 93% rename from server/src/web/api/search.rs rename to mlm_web_askama/src/api/search.rs index f5d17c07..62a580db 100644 --- a/server/src/web/api/search.rs +++ b/mlm_web_askama/src/api/search.rs @@ -10,11 +10,11 @@ use mlm_mam::search::{MaMTorrent, SearchFields}; use serde::{Deserialize, Serialize}; use tokio::fs::create_dir_all; -use crate::{ +use crate::{AppError, MaMState}; +use mlm_core::config::{Cost, TorrentSearch}; +use mlm_core::{ + Context, ContextExt, autograbber::{mark_removed_torrents, search_torrents, select_torrents}, - config::{Cost, TorrentSearch}, - stats::Context, - web::{AppError, MaMState}, }; pub async fn search_api( @@ -84,13 +84,13 @@ pub async fn search_api_post( } if form.mark_removed { - mark_removed_torrents(&context.db, &mam, &torrents).await?; + mark_removed_torrents(context.db(), &mam, &torrents, &context.events).await?; } if form.add { select_torrents( &config, - &context.db, + context.db(), &mam, torrents.into_iter(), &search.filter, @@ -101,6 +101,7 @@ pub async fn search_api_post( search.dry_run, u64::MAX, None, + &context.events, ) .await?; return Ok::<_, AppError>(Json(SearchApiResponse { diff --git a/server/src/web/api/torrent.rs b/mlm_web_askama/src/api/torrent.rs similarity index 92% rename from server/src/web/api/torrent.rs rename to mlm_web_askama/src/api/torrent.rs index 2f83c69b..483baa81 100644 --- a/server/src/web/api/torrent.rs +++ b/mlm_web_askama/src/api/torrent.rs @@ -5,10 +5,10 @@ use axum::{ use mlm_db::{Torrent, TorrentKey}; use serde_json::json; -use crate::{ +use crate::AppError; +use mlm_core::{ + Context, ContextExt, qbittorrent::{self}, - stats::Context, - web::AppError, }; pub async fn torrent_api( @@ -27,7 +27,7 @@ async fn torrent_api_mam_id( Path(mam_id): Path, ) -> std::result::Result, AppError> { if let Some(torrent) = context - .db + .db() .r_transaction()? .get() .secondary::(TorrentKey::mam_id, mam_id)? @@ -52,7 +52,7 @@ async fn torrent_api_id( Path(id): Path, ) -> std::result::Result, AppError> { let config = context.config().await; - let Some(torrent) = context.db.r_transaction()?.get().primary::(id)? else { + let Some(torrent) = context.db().r_transaction()?.get().primary::(id)? else { return Err(AppError::NotFound); }; let mut qbit_torrent = None; diff --git a/server/src/web/mod.rs b/mlm_web_askama/src/lib.rs similarity index 88% rename from server/src/web/mod.rs rename to mlm_web_askama/src/lib.rs index e61f24d8..228e0b1c 100644 --- a/server/src/web/mod.rs +++ b/mlm_web_askama/src/lib.rs @@ -26,7 +26,6 @@ use pages::{ duplicate::{duplicate_page, duplicate_torrents_page_post}, errors::{errors_page, errors_page_post}, events::event_page, - index::{index_page, index_page_post}, list::{list_page, list_page_post}, lists::lists_page, replaced::{replaced_torrents_page, replaced_torrents_page_post}, @@ -48,28 +47,22 @@ use tower::ServiceBuilder; use tower_http::services::{ServeDir, ServeFile}; use crate::{ - config::{SearchConfig, TorrentFilter}, - stats::Context, - web::{ - api::{ - search::{search_api, search_api_post}, - torrent::torrent_api, - }, - pages::{ - index::stats_updates, - search::{search_page, search_page_post}, - }, + api::{ + search::{search_api, search_api_post}, + torrent::torrent_api, + }, + pages::{ + index::stats_updates, + search::{search_page, search_page_post}, }, }; +use mlm_core::config::{SearchConfig, TorrentFilter}; +use mlm_core::{Context, ContextExt}; pub type MaMState = Arc>>>; -pub async fn start_webserver(context: Context) -> Result<()> { - let config = context.config().await; - +pub fn router(context: Context) -> Router { let app = Router::new() - .route("/", get(index_page).with_state(context.clone())) - .route("/", post(index_page_post).with_state(context.clone())) .route( "/stats-updates", get(stats_updates).with_state(context.clone()), @@ -89,7 +82,7 @@ pub async fn start_webserver(context: Context) -> Result<()> { ) .route( "/torrents/{id}/edit", - get(torrent_edit_page).with_state(context.db.clone()), + get(torrent_edit_page).with_state(context.db().clone()), ) .route( "/torrents/{id}/edit", @@ -99,7 +92,7 @@ pub async fn start_webserver(context: Context) -> Result<()> { "/torrents/{id}/{filename}", get(torrent_file).with_state(context.clone()), ) - .route("/events", get(event_page).with_state(context.db.clone())) + .route("/events", get(event_page).with_state(context.db().clone())) .route("/search", get(search_page).with_state(context.clone())) .route( "/search", @@ -108,21 +101,21 @@ pub async fn start_webserver(context: Context) -> Result<()> { .route("/lists", get(lists_page).with_state(context.clone())) .route( "/lists/{list_id}", - get(list_page).with_state(context.db.clone()), + get(list_page).with_state(context.db().clone()), ) .route( "/lists/{list_id}", - post(list_page_post).with_state(context.db.clone()), + post(list_page_post).with_state(context.db().clone()), ) - .route("/errors", get(errors_page).with_state(context.db.clone())) + .route("/errors", get(errors_page).with_state(context.db().clone())) .route( "/errors", - post(errors_page_post).with_state(context.db.clone()), + post(errors_page_post).with_state(context.db().clone()), ) .route("/selected", get(selected_page).with_state(context.clone())) .route( "/selected", - post(selected_torrents_page_post).with_state(context.db.clone()), + post(selected_torrents_page_post).with_state(context.db().clone()), ) .route( "/replaced", @@ -140,14 +133,14 @@ pub async fn start_webserver(context: Context) -> Result<()> { "/duplicate", post(duplicate_torrents_page_post).with_state(context.clone()), ) - .route("/config", get(config_page).with_state(config.clone())) + .route("/config", get(config_page).with_state(context.clone())) .route( "/config", post(config_page_post).with_state(context.clone()), ) .route( "/api/search", - get(search_api).with_state(context.mam.clone()), + get(search_api).with_state(Arc::new(context.mam())), ) .route( "/api/search", @@ -172,11 +165,7 @@ pub async fn start_webserver(context: Context) -> Result<()> { .service(ServeFile::new("server/assets/favicon_dev.png")), ); - let listener = - tokio::net::TcpListener::bind((config.web_host.clone(), config.web_port)).await?; - axum::serve(listener, app).await?; - - Ok(()) + app } pub trait Page { @@ -274,7 +263,33 @@ pub static TIME_FORMAT: Lazy = Lazy::new(|| { format_description::parse_owned::<2>("[year]-[month]-[day] [hour]:[minute]:[second]").unwrap() }); -fn time(time: &Timestamp) -> String { +trait TimeValue { + fn as_timestamp(&self) -> Option<&Timestamp>; +} + +impl TimeValue for &T { + fn as_timestamp(&self) -> Option<&Timestamp> { + (*self).as_timestamp() + } +} + +impl TimeValue for Timestamp { + fn as_timestamp(&self) -> Option<&Timestamp> { + Some(self) + } +} + +impl TimeValue for Option { + fn as_timestamp(&self) -> Option<&Timestamp> { + self.as_ref() + } +} + +fn time(time: &T) -> String { + let Some(time) = time.as_timestamp() else { + return String::new(); + }; + time.0 .to_offset(UtcOffset::current_local_offset().unwrap_or(UtcOffset::UTC)) .replace_nanosecond(0) diff --git a/mlm_web_askama/src/pages/config.rs b/mlm_web_askama/src/pages/config.rs new file mode 100644 index 00000000..b170bed3 --- /dev/null +++ b/mlm_web_askama/src/pages/config.rs @@ -0,0 +1,217 @@ +use std::{ops::Deref, sync::Arc}; + +use anyhow::Result; +use askama::Template; +use axum::{ + extract::{OriginalUri, Query, State}, + response::{Html, Redirect}, +}; +use axum_extra::extract::Form; +use mlm_db::{DatabaseExt as _, Torrent}; +use serde::Deserialize; +use tokio::sync::Semaphore; +use tracing::{info, warn}; + +use crate::{AppError, Page, filter, yaml_items, yaml_nums}; +use mlm_core::{ + Context, ContextExt, + autograbber::update_torrent_meta, + config::{Config, Library}, + qbittorrent::ensure_category_exists, +}; + +pub async fn config_page( + State(context): State, + Query(query): Query, +) -> std::result::Result, AppError> { + let config = context.config().await; + let template = ConfigPageTemplate { + config, + show_apply_tags: query.show_apply_tags.unwrap_or_default(), + }; + Ok::<_, AppError>(Html(template.to_string())) +} + +pub async fn config_page_post( + State(context): State, + uri: OriginalUri, + Form(form): Form, +) -> Result { + let config = context.config().await; + match form.action.as_str() { + "apply" => { + const BATCH_SIZE: usize = 100; + const MAX_CONCURRENT_MAM_REQUESTS: usize = 5; + + let tag_filter = form + .tag_filter + .ok_or(anyhow::Error::msg("apply requires tag_filter"))?; + let tag_filter = config + .tags + .get(tag_filter) + .ok_or(anyhow::Error::msg("invalid tag_filter"))?; + let qbit_conf = config + .qbittorrent + .get(form.qbit_index.unwrap_or_default()) + .ok_or(anyhow::Error::msg("requires a qbit config"))?; + let qbit = qbit::Api::new_login_username_password( + &qbit_conf.url, + &qbit_conf.username, + &qbit_conf.password, + ) + .await?; + + // Collect all torrents first + let torrents: Vec = context + .db() + .r_transaction()? + .scan() + .primary::()? + .all()? + .collect::, _>>()?; + + let total_torrents = torrents.len(); + info!("Processing {} torrents for tag filter", total_torrents); + + // Create semaphore to limit concurrent MAM API calls + let mam_semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT_MAM_REQUESTS)); + let mut processed_count = 0; + + // Process torrents in batches + for (batch_idx, batch) in torrents.chunks(BATCH_SIZE).enumerate() { + let batch_start = batch_idx * BATCH_SIZE; + let batch_end = (batch_start + batch.len()).min(total_torrents); + info!( + "Processing batch {}/{} (torrents {}-{})", + batch_idx + 1, + total_torrents.div_ceil(BATCH_SIZE), + batch_start + 1, + batch_end + ); + + // First pass: Process MAM API calls with concurrency limiting + // Collect torrents that match the filter + let mut matched_torrents: Vec<&Torrent> = Vec::with_capacity(batch.len()); + + for torrent in batch { + match tag_filter.filter.matches_lib(torrent) { + Ok(matches) => { + if matches { + matched_torrents.push(torrent); + } + } + Err(err) => { + let Some(mam_id) = torrent.mam_id else { + continue; + }; + info!("need to ask mam due to: {err}"); + + let permit = mam_semaphore + .clone() + .acquire_owned() + .await + .map_err(anyhow::Error::new)?; + + let mam = context.mam()?; + let Some(mam_torrent) = mam.get_torrent_info_by_id(mam_id).await? + else { + drop(permit); + warn!("could not get torrent from mam"); + continue; + }; + + drop(permit); + + let new_meta = mam_torrent.as_meta()?; + if new_meta != torrent.meta { + update_torrent_meta( + &config, + context.db(), + context.db().rw_async().await?, + Some(&mam_torrent), + torrent.clone(), + new_meta, + false, + false, + &context.events, + ) + .await?; + } + + if tag_filter.filter.matches(&mam_torrent) { + matched_torrents.push(torrent); + } + } + } + } + + if !matched_torrents.is_empty() { + let matched_count = matched_torrents.len(); + info!( + "Applying category/tags to {} torrents in batch", + matched_count + ); + + let hashes: Vec<&str> = + matched_torrents.iter().map(|t| t.id.as_str()).collect(); + + if let Some(category) = &tag_filter.category { + ensure_category_exists(&qbit, &qbit_conf.url, category).await?; + qbit.set_category(Some(hashes.clone()), category).await?; + info!("Set category '{}' for {} torrents", category, matched_count); + } + + if !tag_filter.tags.is_empty() { + let tags: Vec<&str> = tag_filter.tags.iter().map(Deref::deref).collect(); + qbit.add_tags(Some(hashes), tags).await?; + info!( + "Added tags {:?} to {} torrents", + tag_filter.tags, matched_count + ); + } + + processed_count += matched_count; + } + + info!( + "Completed batch {}/{}, total processed: {}/{}", + batch_idx + 1, + total_torrents.div_ceil(BATCH_SIZE), + processed_count, + total_torrents + ); + } + + info!( + "Tag filter application complete: {} torrents processed", + processed_count + ); + } + action => { + eprintln!("unknown action: {action}"); + } + } + + Ok(Redirect::to(&uri.to_string())) +} + +#[derive(Debug, Deserialize)] +pub struct ConfigPageQuery { + show_apply_tags: Option, +} + +#[derive(Debug, Deserialize)] +pub struct ConfigPageForm { + action: String, + qbit_index: Option, + tag_filter: Option, +} + +#[derive(Template)] +#[template(path = "pages/config.html")] +struct ConfigPageTemplate { + config: Arc, + show_apply_tags: bool, +} + +impl Page for ConfigPageTemplate {} diff --git a/server/src/web/pages/duplicate.rs b/mlm_web_askama/src/pages/duplicate.rs similarity index 93% rename from server/src/web/pages/duplicate.rs rename to mlm_web_askama/src/pages/duplicate.rs index 209c6e54..f2eaa7dc 100644 --- a/server/src/web/pages/duplicate.rs +++ b/mlm_web_askama/src/pages/duplicate.rs @@ -13,14 +13,11 @@ use serde::{Deserialize, Serialize}; use tracing::info; use crate::{ - cleaner::clean_torrent, - stats::Context, - web::{ - AppError, Page, - tables::{Key, SortOn, Sortable, table_styles_rows}, - time, - }, + AppError, Page, + tables::{Key, SortOn, Sortable, table_styles_rows}, + time, }; +use mlm_core::{Context, ContextExt, cleaner::clean_torrent}; pub async fn duplicate_page( State(context): State, @@ -29,7 +26,7 @@ pub async fn duplicate_page( ) -> std::result::Result, AppError> { let config = context.config().await; let mut duplicate_torrents = context - .db + .db() .r_transaction()? .scan() .primary::()? @@ -75,7 +72,7 @@ pub async fn duplicate_page( let Some(with) = &torrent.duplicate_of else { continue; }; - let Some(duplicate) = context.db.r_transaction()?.get().primary(with.clone())? else { + let Some(duplicate) = context.db().r_transaction()?.get().primary(with.clone())? else { continue; }; torrents.push((torrent, duplicate)); @@ -98,7 +95,7 @@ pub async fn duplicate_torrents_page_post( "replace" => { let mam = context.mam()?; for torrent in form.torrents { - let r = context.db.r_transaction()?; + let r = context.db().r_transaction()?; let Some(duplicate_torrent) = r.get().primary::(torrent)? else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; @@ -151,7 +148,7 @@ pub async fn duplicate_torrents_page_post( ); { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; rw.insert(SelectedTorrent { mam_id: mam_torrent.id, hash: None, @@ -177,12 +174,12 @@ pub async fn duplicate_torrents_page_post( rw.remove(duplicate_torrent)?; rw.commit()?; } - clean_torrent(&config, &context.db, duplicate_of, false).await?; + clean_torrent(&config, context.db(), duplicate_of, false, &context.events).await?; } } "remove" => { for torrent in form.torrents { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; let Some(torrent) = rw.get().primary::(torrent)? else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; diff --git a/server/src/web/pages/errors.rs b/mlm_web_askama/src/pages/errors.rs similarity index 99% rename from server/src/web/pages/errors.rs rename to mlm_web_askama/src/pages/errors.rs index 3bd360a9..ee85ec12 100644 --- a/server/src/web/pages/errors.rs +++ b/mlm_web_askama/src/pages/errors.rs @@ -13,7 +13,7 @@ use mlm_db::{DatabaseExt as _, ErroredTorrent, ErroredTorrentId, ErroredTorrentK use native_db::Database; use serde::{Deserialize, Serialize}; -use crate::web::{ +use crate::{ AppError, Page, tables::{self, Flex, HidableColumns, Key, SortOn, Sortable}, time, diff --git a/server/src/web/pages/events.rs b/mlm_web_askama/src/pages/events.rs similarity index 98% rename from server/src/web/pages/events.rs rename to mlm_web_askama/src/pages/events.rs index 4a4b5b86..8b5cae93 100644 --- a/server/src/web/pages/events.rs +++ b/mlm_web_askama/src/pages/events.rs @@ -9,7 +9,7 @@ use mlm_db::{Event, EventKey, EventType, Torrent, TorrentCost, TorrentKey}; use native_db::Database; use serde::{Deserialize, Serialize}; -use crate::web::{ +use crate::{ AppError, Conditional, Page, TorrentLink, tables::{Key, Pagination, PaginationParams, table_styles}, time, @@ -36,7 +36,7 @@ pub async fn event_page( EventType::Linked { .. } => value == "linker", EventType::Cleaned { .. } => value == "cleaner", EventType::Updated { .. } => value == "updated", - EventType::RemovedFromTracker { .. } => value == "removed", + EventType::RemovedFromTracker => value == "removed", }, EventPageFilter::Grabber => match t.event { EventType::Grabbed { ref grabber, .. } => { diff --git a/server/src/web/pages/index.rs b/mlm_web_askama/src/pages/index.rs similarity index 86% rename from server/src/web/pages/index.rs rename to mlm_web_askama/src/pages/index.rs index 4872635d..5812bf9c 100644 --- a/server/src/web/pages/index.rs +++ b/mlm_web_askama/src/pages/index.rs @@ -1,3 +1,7 @@ +// NOTE: This file is kept for reference but is no longer used. +// The home page has been replaced with a Dioxus implementation in mlm_web_dioxus/src/home.rs +#![allow(dead_code)] + use std::{collections::BTreeMap, convert::Infallible, sync::Arc, time::Duration}; use anyhow::Result; @@ -15,26 +19,26 @@ use mlm_db::Timestamp; use serde::Deserialize; use tokio_stream::{StreamExt as _, wrappers::WatchStream}; -use crate::{ - config::{Config, TorrentFilter}, +use crate::{AppError, Page, time}; +use mlm_core::config::Config; +use mlm_core::{ + Context, ContextExt, lists::{List, get_lists}, - stats::Context, - web::{AppError, Page, time}, }; pub async fn index_page( State(context): State, ) -> std::result::Result, AppError> { let stats = context.stats.values.lock().await; - let username = match context.mam.as_ref() { + let username = match context.mam() { Ok(mam) => mam.cached_user_info().await.map(|u| u.username), Err(_) => None, }; - let config = context.config.lock().await; + let config = context.config().await; let template = IndexPageTemplate { config: config.clone(), lists: get_lists(&config), - mam_error: context.mam.as_ref().as_ref().err().map(|e| format!("{e}")), + mam_error: context.mam().err().map(|e| format!("{e}")), has_no_qbits: config.qbittorrent.is_empty(), username, autograbber_run_at: stats @@ -101,10 +105,14 @@ pub async fn index_page_post( ) -> Result { match form.action.as_str() { "run_torrent_linker" => { - context.triggers.torrent_linker_tx.send(())?; + if let Some(tx) = &context.triggers.torrent_linker_tx { + tx.send(())?; + } } "run_folder_linker" => { - context.triggers.folder_linker_tx.send(())?; + if let Some(tx) = &context.triggers.folder_linker_tx { + tx.send(())?; + } } "run_search" => { if let Some(tx) = context.triggers.search_tx.get( @@ -129,10 +137,14 @@ pub async fn index_page_post( } } "run_downloader" => { - context.triggers.downloader_tx.send(())?; + if let Some(tx) = &context.triggers.downloader_tx { + tx.send(())?; + } } "run_abs_matcher" => { - context.triggers.audiobookshelf_tx.send(())?; + if let Some(tx) = &context.triggers.audiobookshelf_tx { + tx.send(())?; + } } action => { eprintln!("unknown action: {action}"); @@ -173,9 +185,3 @@ pub struct IndexPageForm { action: String, index: Option, } - -impl TorrentFilter { - fn display_name(&self, i: usize) -> String { - self.name.clone().unwrap_or_else(|| format!("{i}")) - } -} diff --git a/server/src/web/pages/list.rs b/mlm_web_askama/src/pages/list.rs similarity index 99% rename from server/src/web/pages/list.rs rename to mlm_web_askama/src/pages/list.rs index 92348246..dd9f781f 100644 --- a/server/src/web/pages/list.rs +++ b/mlm_web_askama/src/pages/list.rs @@ -14,7 +14,7 @@ use native_db::Database; use reqwest::Url; use serde::{Deserialize, Serialize}; -use crate::web::{AppError, Page, time}; +use crate::{AppError, Page, time}; pub async fn list_page( State(db): State>>, diff --git a/server/src/web/pages/lists.rs b/mlm_web_askama/src/pages/lists.rs similarity index 92% rename from server/src/web/pages/lists.rs rename to mlm_web_askama/src/pages/lists.rs index a480dff5..f0f43d69 100644 --- a/server/src/web/pages/lists.rs +++ b/mlm_web_askama/src/pages/lists.rs @@ -3,18 +3,16 @@ use axum::{extract::State, response::Html}; use itertools::Itertools as _; use mlm_db::{List, ListKey}; -use crate::{ - config::GoodreadsList, - stats::Context, - web::{AppError, Page, time}, -}; +use crate::{AppError, Page, time}; +use mlm_core::config::GoodreadsList; +use mlm_core::{Context, ContextExt}; pub async fn lists_page( State(context): State, ) -> std::result::Result, AppError> { let config = context.config().await; let db_lists = context - .db + .db() .r_transaction()? .scan() .secondary::(ListKey::title)?; diff --git a/server/src/web/pages/mod.rs b/mlm_web_askama/src/pages/mod.rs similarity index 100% rename from server/src/web/pages/mod.rs rename to mlm_web_askama/src/pages/mod.rs diff --git a/server/src/web/pages/replaced.rs b/mlm_web_askama/src/pages/replaced.rs similarity index 94% rename from server/src/web/pages/replaced.rs rename to mlm_web_askama/src/pages/replaced.rs index f2ba71b1..1bc8f59a 100644 --- a/server/src/web/pages/replaced.rs +++ b/mlm_web_askama/src/pages/replaced.rs @@ -13,15 +13,14 @@ use axum_extra::extract::Form; use mlm_db::{Language, Torrent, TorrentKey, ids}; use serde::{Deserialize, Serialize}; -use crate::stats::Context; -use crate::web::{Page, tables}; use crate::{ + AppError, Page, + tables::{self, Flex, HidableColumns, Key, Pagination, PaginationParams, SortOn, Sortable}, + time, +}; +use mlm_core::{ + Context, ContextExt, linker::{refresh_mam_metadata, refresh_metadata_relink}, - web::{ - AppError, - tables::{Flex, HidableColumns, Key, Pagination, PaginationParams, SortOn, Sortable}, - time, - }, }; use mlm_db::DatabaseExt as _; @@ -35,7 +34,7 @@ pub async fn replaced_torrents_page( ) -> std::result::Result { let config = context.config().await; let torrents = context - .db + .db() .r_transaction()? .scan() .secondary::(TorrentKey::created_at)?; @@ -114,7 +113,7 @@ pub async fn replaced_torrents_page( let Some((with, _)) = &torrent.replaced_with else { continue; }; - let Some(replacement) = context.db.r_transaction()?.get().primary(with.clone())? else { + let Some(replacement) = context.db().r_transaction()?.get().primary(with.clone())? else { continue; }; torrents.push((torrent, replacement)); @@ -141,18 +140,19 @@ pub async fn replaced_torrents_page_post( "refresh" => { let mam = context.mam()?; for torrent in form.torrents { - refresh_mam_metadata(&config, &context.db, &mam, torrent).await?; + refresh_mam_metadata(&config, context.db(), &mam, torrent, &context.events).await?; } } "refresh-relink" => { let mam = context.mam()?; for torrent in form.torrents { - refresh_metadata_relink(&config, &context.db, &mam, torrent).await?; + refresh_metadata_relink(&config, context.db(), &mam, torrent, &context.events) + .await?; } } "remove" => { for torrent in form.torrents { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; let Some(torrent) = rw.get().primary::(torrent)? else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; diff --git a/server/src/web/pages/search.rs b/mlm_web_askama/src/pages/search.rs similarity index 94% rename from server/src/web/pages/search.rs rename to mlm_web_askama/src/pages/search.rs index dde2f46c..57de0664 100644 --- a/server/src/web/pages/search.rs +++ b/mlm_web_askama/src/pages/search.rs @@ -14,10 +14,8 @@ use mlm_parse::normalize_title; use serde::Deserialize; use tracing::info; -use crate::{ - stats::Context, - web::{AppError, MaMTorrentsTemplate, Page}, -}; +use crate::{AppError, MaMTorrentsTemplate, Page}; +use mlm_core::{Context, ContextExt}; pub async fn search_page( State(context): State, @@ -39,7 +37,7 @@ pub async fn search_page( }) .await?; - let r = context.db.r_transaction()?; + let r = context.db().r_transaction()?; let mut torrents = result .data .into_iter() @@ -146,7 +144,7 @@ pub async fn select_torrent(context: &Context, mam_id: u64, wedge: bool) -> Resu torrent.title, torrent.filetype, cost, category, tags ); { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; rw.insert(SelectedTorrent { mam_id: torrent.id, hash: None, @@ -168,7 +166,9 @@ pub async fn select_torrent(context: &Context, mam_id: u64, wedge: bool) -> Resu })?; rw.commit()?; } - context.triggers.downloader_tx.send(())?; + if let Some(tx) = &context.triggers.downloader_tx { + tx.send(())?; + } Ok(()) } diff --git a/server/src/web/pages/selected.rs b/mlm_web_askama/src/pages/selected.rs similarity index 98% rename from server/src/web/pages/selected.rs rename to mlm_web_askama/src/pages/selected.rs index 4e025e87..5765bb73 100644 --- a/server/src/web/pages/selected.rs +++ b/mlm_web_askama/src/pages/selected.rs @@ -18,13 +18,11 @@ use serde::{Deserialize, Serialize}; use tracing::info; use crate::{ - stats::Context, - web::{ - AppError, Page, flag_icons, - tables::{self, Flex, HidableColumns, Key, SortOn, Sortable}, - time, - }, + AppError, Page, flag_icons, + tables::{self, Flex, HidableColumns, Key, SortOn, Sortable}, + time, }; +use mlm_core::{Context, ContextExt}; pub async fn selected_page( State(context): State, @@ -36,7 +34,7 @@ pub async fn selected_page( let show = show.show.unwrap_or_default(); let mut torrents = context - .db + .db() .r_transaction()? .scan() .primary::()? @@ -135,7 +133,7 @@ pub async fn selected_page( }); } let downloading_size: f64 = context - .db + .db() .r_transaction()? .scan() .primary::()? @@ -150,7 +148,7 @@ pub async fn selected_page( }) .map(|t| t.meta.size.bytes() as f64) .sum(); - let user_info = match context.mam.as_ref() { + let user_info = match context.mam() { Ok(mam) => mam.user_info().await.ok(), _ => None, }; diff --git a/server/src/web/pages/torrent.rs b/mlm_web_askama/src/pages/torrent.rs similarity index 87% rename from server/src/web/pages/torrent.rs rename to mlm_web_askama/src/pages/torrent.rs index 93147bb3..42f9d9d3 100644 --- a/server/src/web/pages/torrent.rs +++ b/mlm_web_askama/src/pages/torrent.rs @@ -29,22 +29,22 @@ use serde::Deserialize; use time::UtcDateTime; use tokio_util::io::ReaderStream; -use crate::metadata::mam_meta::match_meta; use crate::{ + AppError, Conditional, MaMTorrentsTemplate, Page, TorrentLink, flag_icons, + pages::{search::select_torrent, torrents::TorrentsPageFilter}, + tables::table_styles, + time, +}; +use mlm_core::config::Config; +use mlm_core::metadata::mam_meta::match_meta; +use mlm_core::{ + Context, ContextExt, audiobookshelf::{Abs, LibraryItemMinified}, cleaner::clean_torrent, - config::Config, linker::{ find_library, library_dir, map_path, refresh_mam_metadata, refresh_metadata_relink, relink, }, qbittorrent::{self, ensure_category_exists}, - stats::Context, - web::{ - AppError, Conditional, MaMTorrentsTemplate, Page, TorrentLink, flag_icons, - pages::{search::select_torrent, torrents::TorrentsPageFilter}, - tables::table_styles, - time, - }, }; use mlm_db::MetadataSource; @@ -53,7 +53,7 @@ pub async fn torrent_file( Path((id, filename)): Path<(String, String)>, ) -> impl IntoResponse { let config = context.config().await; - let Some(torrent) = context.db.r_transaction()?.get().primary::(id)? else { + let Some(torrent) = context.db().r_transaction()?.get().primary::(id)? else { return Err(AppError::NotFound); }; let Some(path) = (if let (Some(library_path), Some(library_file)) = ( @@ -111,7 +111,7 @@ async fn torrent_page_mam_id( Path(mam_id): Path, ) -> std::result::Result, AppError> { if let Some(torrent) = context - .db + .db() .r_transaction()? .get() .secondary::(TorrentKey::mam_id, mam_id)? @@ -127,8 +127,8 @@ async fn torrent_page_mam_id( println!("mam_torrent: {:?}", mam_torrent); println!("mam_meta: {:?}", meta); - let config = context.config.lock().await.clone(); - let other_torrents = other_torrents(&config, &context.db, &mam, &meta).await?; + let config = context.config().await; + let other_torrents = other_torrents(&config, context.db(), &mam, &meta).await?; let template = TorrentMamPageTemplate { mam_torrent, @@ -144,7 +144,7 @@ async fn torrent_page_id( ) -> std::result::Result, AppError> { let config = context.config().await; let abs = config.audiobookshelf.as_ref().map(Abs::new); - let Some(mut torrent) = context.db.r_transaction()?.get().primary::(id)? else { + let Some(mut torrent) = context.db().r_transaction()?.get().primary::(id)? else { return Err(AppError::NotFound); }; let replacement_torrent = torrent @@ -152,7 +152,7 @@ async fn torrent_page_id( .as_ref() .map(|(id, _)| { context - .db + .db() .r_transaction()? .get() .primary::(id.to_string()) @@ -161,7 +161,7 @@ async fn torrent_page_id( .flatten(); if replacement_torrent.is_none() && torrent.replaced_with.is_some() { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; torrent.replaced_with = None; rw.upsert(torrent.clone())?; rw.commit()?; @@ -172,7 +172,7 @@ async fn torrent_page_id( }; let events = context - .db + .db() .r_transaction()? .scan() .secondary::(EventKey::torrent_id)?; @@ -199,7 +199,7 @@ async fn torrent_page_id( .as_ref() .is_none_or(|t| t.0 == UtcDateTime::UNIX_EPOCH) { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; torrent.meta.uploaded_at = mam_meta.uploaded_at; rw.upsert(torrent.clone())?; rw.commit()?; @@ -250,12 +250,12 @@ async fn torrent_page_id( && qbit_data.is_none() && torrent.client_status != Some(ClientStatus::NotInClient) { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; torrent.client_status = Some(ClientStatus::NotInClient); rw.upsert(torrent.clone())?; rw.commit()?; } - let other_torrents = other_torrents(&config, &context.db, &mam, &torrent.meta).await?; + let other_torrents = other_torrents(&config, context.db(), &mam, &torrent.meta).await?; let template = TorrentPageTemplate { abs_url: config @@ -273,7 +273,7 @@ async fn torrent_page_id( wanted_path, qbit_files, other_torrents, - metadata_providers: context.metadata.enabled_providers(), + metadata_providers: context.metadata().enabled_providers(), }; Ok::<_, AppError>(Html(template.to_string())) } @@ -299,7 +299,7 @@ pub async fn torrent_page_post_mam_id( ) -> Result { let mam_id = form.mam_id.unwrap_or(mam_id); if let Some(torrent) = context - .db + .db() .r_transaction()? .get() .secondary::(TorrentKey::mam_id, mam_id)? @@ -337,25 +337,25 @@ pub async fn torrent_page_post_id( select_torrent(&context, mam_id, form.action == "wedge").await?; } "clean" => { - let Some(torrent) = context.db.r_transaction()?.get().primary(id)? else { + let Some(torrent) = context.db().r_transaction()?.get().primary(id)? else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; - clean_torrent(&config, &context.db, torrent, true).await?; + clean_torrent(&config, context.db(), torrent, true, &context.events).await?; } "refresh" => { let mam = context.mam()?; - refresh_mam_metadata(&config, &context.db, &mam, id).await?; + refresh_mam_metadata(&config, context.db(), &mam, id, &context.events).await?; } "relink" => { - relink(&config, &context.db, id).await?; + relink(&config, context.db(), id, &context.events).await?; } "refresh-relink" => { let mam = context.mam()?; - refresh_metadata_relink(&config, &context.db, &mam, id).await?; + refresh_metadata_relink(&config, context.db(), &mam, id, &context.events).await?; } "match" => { // Build a query from existing torrent metadata - let Some(mut torrent) = context.db.r_transaction()?.get().primary::(id)? + let Some(mut torrent) = context.db().r_transaction()?.get().primary::(id)? else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; @@ -370,18 +370,7 @@ pub async fn torrent_page_post_id( match match_meta(&context, &torrent.meta, provider_id).await { Ok((new_meta, pid, fields)) => { - let ev = Event { - id: mlm_db::Uuid::new(), - torrent_id: Some(torrent.id.clone()), - mam_id: torrent.mam_id, - created_at: mlm_db::Timestamp::now(), - event: EventType::Updated { - fields: fields.clone(), - source: (MetadataSource::Match, pid.clone()), - }, - }; - - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; // apply meta updates let mut meta = new_meta; meta.source = MetadataSource::Match; @@ -389,9 +378,24 @@ pub async fn torrent_page_post_id( // update title_search to normalized title torrent.title_search = mlm_parse::normalize_title(&torrent.meta.title); - rw.upsert(torrent)?; - rw.insert(ev)?; + rw.upsert(torrent.clone())?; rw.commit()?; + drop(_guard); + + // Write event through the proper channel so SSE gets notified + mlm_core::logging::write_event( + context.db(), + &context.events, + Event::new( + Some(torrent.id.clone()), + torrent.mam_id, + EventType::Updated { + fields: fields.clone(), + source: (MetadataSource::Match, pid.clone()), + }, + ), + ) + .await; } Err(e) => { tracing::error!("metadata match failed for provider {}: {}", provider_id, e) @@ -399,7 +403,7 @@ pub async fn torrent_page_post_id( } } "remove" => { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; let Some(torrent) = rw.get().primary::(id)? else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; @@ -419,7 +423,7 @@ pub async fn torrent_page_post_id( qbit.stop(vec![&id]).await?; } "clear-replacement" => { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; let Some(mut torrent) = rw.get().primary::(id)? else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; @@ -467,7 +471,7 @@ pub async fn torrent_page_post_id( } } "remove-torrent" => { - // let Some(torrent) = context.db.r_transaction()?.get().primary(id)? else { + // let Some(torrent) = context.db().r_transaction()?.get().primary(id)? else { // return Err(anyhow::Error::msg("Could not find torrent").into()); // }; // remove_library_files(&torrent)?; diff --git a/server/src/web/pages/torrent_edit.rs b/mlm_web_askama/src/pages/torrent_edit.rs similarity index 95% rename from server/src/web/pages/torrent_edit.rs rename to mlm_web_askama/src/pages/torrent_edit.rs index 2d09e677..c156974b 100644 --- a/server/src/web/pages/torrent_edit.rs +++ b/mlm_web_askama/src/pages/torrent_edit.rs @@ -14,11 +14,8 @@ use mlm_db::{ use native_db::Database; use serde::Deserialize; -use crate::{ - autograbber::update_torrent_meta, - stats::Context, - web::{AppError, Page}, -}; +use crate::{AppError, Page}; +use mlm_core::{Context, ContextExt, autograbber::update_torrent_meta}; pub async fn torrent_edit_page( State(db): State>>, @@ -43,7 +40,7 @@ pub async fn torrent_edit_page_post( let config = context.config().await; let _mam = context.mam()?; let Some(torrent) = context - .db + .db() .r_transaction()? .get() .primary::(hash.clone())? @@ -105,13 +102,14 @@ pub async fn torrent_edit_page_post( update_torrent_meta( &config, - &context.db, - context.db.rw_async().await?, + context.db(), + context.db().rw_async().await?, None, torrent, meta, true, false, + &context.events, ) .await?; diff --git a/server/src/web/pages/torrents.rs b/mlm_web_askama/src/pages/torrents.rs similarity index 97% rename from server/src/web/pages/torrents.rs rename to mlm_web_askama/src/pages/torrents.rs index 7f5c7624..f96f9d44 100644 --- a/server/src/web/pages/torrents.rs +++ b/mlm_web_askama/src/pages/torrents.rs @@ -17,15 +17,14 @@ use serde::{Deserialize, Serialize}; use sublime_fuzzy::FuzzySearch; use crate::{ + AppError, Page, flag_icons, tables, + tables::{Flex, HidableColumns, Key, Pagination, PaginationParams, SortOn, Sortable}, + time, +}; +use mlm_core::{ + Context, ContextExt, cleaner::clean_torrent, linker::{refresh_mam_metadata, refresh_metadata_relink}, - stats::Context, - web::{ - AppError, - tables::{Flex, HidableColumns, Key, Pagination, PaginationParams, SortOn, Sortable}, - time, - }, - web::{Page, flag_icons, tables}, }; use mlm_db::{ ClientStatus, DatabaseExt as _, Flags, MediaType, MetadataSource, OldCategory, Series, @@ -40,7 +39,7 @@ pub async fn torrents_page( Query(show): Query, Query(paging): Query, ) -> std::result::Result { - let r = context.db.r_transaction()?; + let r = context.db().r_transaction()?; let torrent_count = r.len().secondary::(TorrentKey::created_at)?; let torrents = r.scan().secondary::(TorrentKey::created_at)?; @@ -631,33 +630,28 @@ pub async fn torrents_page_post( match form.action.as_str() { "clean" => { for torrent in form.torrents { - let Some(torrent) = context.db.r_transaction()?.get().primary(torrent)? else { + let Some(torrent) = context.db().r_transaction()?.get().primary(torrent)? else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; - clean_torrent(&config, &context.db, torrent, true).await?; + clean_torrent(&config, context.db(), torrent, true, &context.events).await?; } } "refresh" => { let mam = context.mam()?; for torrent in form.torrents { - refresh_mam_metadata(&config, &context.db, &mam, torrent).await?; + refresh_mam_metadata(&config, context.db(), &mam, torrent, &context.events).await?; } } "refresh-relink" => { let mam = context.mam()?; for torrent in form.torrents { - refresh_metadata_relink( - context.config.lock().await.as_ref(), - &context.db, - &mam, - torrent, - ) - .await?; + refresh_metadata_relink(&config, context.db(), &mam, torrent, &context.events) + .await?; } } "remove" => { for torrent in form.torrents { - let (_guard, rw) = context.db.rw_async().await?; + let (_guard, rw) = context.db().rw_async().await?; let Some(torrent) = rw.get().primary::(torrent)? else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; diff --git a/server/src/web/tables.rs b/mlm_web_askama/src/tables.rs similarity index 100% rename from server/src/web/tables.rs rename to mlm_web_askama/src/tables.rs diff --git a/server/templates/base.html b/mlm_web_askama/templates/base.html similarity index 100% rename from server/templates/base.html rename to mlm_web_askama/templates/base.html diff --git a/server/templates/pages/config.html b/mlm_web_askama/templates/pages/config.html similarity index 100% rename from server/templates/pages/config.html rename to mlm_web_askama/templates/pages/config.html diff --git a/server/templates/pages/duplicate.html b/mlm_web_askama/templates/pages/duplicate.html similarity index 100% rename from server/templates/pages/duplicate.html rename to mlm_web_askama/templates/pages/duplicate.html diff --git a/server/templates/pages/errors.html b/mlm_web_askama/templates/pages/errors.html similarity index 100% rename from server/templates/pages/errors.html rename to mlm_web_askama/templates/pages/errors.html diff --git a/server/templates/pages/events.html b/mlm_web_askama/templates/pages/events.html similarity index 100% rename from server/templates/pages/events.html rename to mlm_web_askama/templates/pages/events.html diff --git a/server/templates/pages/index.html b/mlm_web_askama/templates/pages/index.html similarity index 100% rename from server/templates/pages/index.html rename to mlm_web_askama/templates/pages/index.html diff --git a/server/templates/pages/list.html b/mlm_web_askama/templates/pages/list.html similarity index 100% rename from server/templates/pages/list.html rename to mlm_web_askama/templates/pages/list.html diff --git a/server/templates/pages/lists.html b/mlm_web_askama/templates/pages/lists.html similarity index 100% rename from server/templates/pages/lists.html rename to mlm_web_askama/templates/pages/lists.html diff --git a/server/templates/pages/replaced.html b/mlm_web_askama/templates/pages/replaced.html similarity index 100% rename from server/templates/pages/replaced.html rename to mlm_web_askama/templates/pages/replaced.html diff --git a/server/templates/pages/search.html b/mlm_web_askama/templates/pages/search.html similarity index 100% rename from server/templates/pages/search.html rename to mlm_web_askama/templates/pages/search.html diff --git a/server/templates/pages/selected.html b/mlm_web_askama/templates/pages/selected.html similarity index 100% rename from server/templates/pages/selected.html rename to mlm_web_askama/templates/pages/selected.html diff --git a/server/templates/pages/torrent.html b/mlm_web_askama/templates/pages/torrent.html similarity index 100% rename from server/templates/pages/torrent.html rename to mlm_web_askama/templates/pages/torrent.html diff --git a/server/templates/pages/torrent_edit.html b/mlm_web_askama/templates/pages/torrent_edit.html similarity index 100% rename from server/templates/pages/torrent_edit.html rename to mlm_web_askama/templates/pages/torrent_edit.html diff --git a/server/templates/pages/torrent_mam.html b/mlm_web_askama/templates/pages/torrent_mam.html similarity index 100% rename from server/templates/pages/torrent_mam.html rename to mlm_web_askama/templates/pages/torrent_mam.html diff --git a/server/templates/pages/torrents.html b/mlm_web_askama/templates/pages/torrents.html similarity index 100% rename from server/templates/pages/torrents.html rename to mlm_web_askama/templates/pages/torrents.html diff --git a/server/templates/partials/cost_icon.html b/mlm_web_askama/templates/partials/cost_icon.html similarity index 100% rename from server/templates/partials/cost_icon.html rename to mlm_web_askama/templates/partials/cost_icon.html diff --git a/server/templates/partials/filter.html b/mlm_web_askama/templates/partials/filter.html similarity index 100% rename from server/templates/partials/filter.html rename to mlm_web_askama/templates/partials/filter.html diff --git a/server/templates/partials/flag_icons.html b/mlm_web_askama/templates/partials/flag_icons.html similarity index 100% rename from server/templates/partials/flag_icons.html rename to mlm_web_askama/templates/partials/flag_icons.html diff --git a/server/templates/partials/mam_torrents.html b/mlm_web_askama/templates/partials/mam_torrents.html similarity index 100% rename from server/templates/partials/mam_torrents.html rename to mlm_web_askama/templates/partials/mam_torrents.html diff --git a/server/Cargo.toml b/server/Cargo.toml index 9ac21718..10183a0a 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -7,63 +7,31 @@ edition = "2024" before-packaging-command = "cargo build --release" product-name = "Myanonamouse Library Manager" identifier = "io.github.stirlingmouse.mlm" -resources = ["Cargo.toml", "src", "assets"] +resources = ["Cargo.toml", "src"] icons = ["assets/favicon.png"] nsis = { installer-icon = "assets/tray.ico" } [dependencies] anyhow = "1.0.100" -askama = { version = "0.14.0", features = ["code-in-doc", "serde_json"] } axum = { version = "0.8.4", features = ["query", "macros"] } -axum-extra = { version = "0.10.1", features = ["form"] } -bytes = "1.11.0" -cookie = "0.18.1" dirs = "6.0" figment = { version = "0.10", features = ["toml", "env"] } -file-id = "0.2.2" -futures = "0.3" -# goodreads-metadata-scraper = "0.2.4" -# goodreads-metadata-scraper = { path = "../goodreads-metadata-scraper" } -htmlentity = "1.3.2" -itertools = "0.14.0" -lava_torrent = { git = "https://github.com/StirlingMouse/lava_torrent.git" } -log = "0.4.27" -matchr = "0.2.5" +mlm_core = { path = "../mlm_core" } mlm_db = { path = "../mlm_db" } mlm_mam = { path = "../mlm_mam" } -mlm_parse = { path = "../mlm_parse" } -mlm_meta = { path = "../mlm_meta" } +mlm_web_askama = { path = "../mlm_web_askama" } native_db = { git = "https://github.com/StirlingMouse/native_db.git", branch = "0.8.x" } -native_model = "0.4.20" once_cell = "1.21.3" -openssl = { version = "0.10.73", features = ["vendored"] } -# qbit = "=0.2.0" qbit = { git = "https://github.com/StirlingMouse/qbittorrent-webui-api.git" } -# qbit = { path = "../qbittorrent-webui-api" } -quick-xml = { version = "0.38.0", features = ["serialize"] } -regex = "1.12.2" -reqwest = "0.12.20" -reqwest_cookie_store = "0.8.0" -sanitize-filename = { git = "https://github.com/StirlingMouse/sanitize-filename.git" } -scraper = "0.23.1" serde = "1.0.136" -serde_derive = "1.0.136" -serde_json = "1.0.140" -serde-nested-json = "0.1.3" -sublime_fuzzy = "0.7.0" -thiserror = "2.0.17" +serde_json = "1.0" time = { version = "0.3.41", features = [ "formatting", "local-offset", "macros", "serde", ] } -tokio = { version = "1.45.1", features = ["full"] } -tokio-stream = { version = "0.1.17", features = ["sync"] } -tokio-util = "0.7" -toml = "0.8.23" -tower = "0.5.2" -tower-http = { version = "0.6.6", features = ["fs"] } +tokio = { version = "1.45.1", features = ["macros", "net", "rt-multi-thread", "sync"] } tracing = "0.1" tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3", features = [ @@ -72,9 +40,6 @@ tracing-subscriber = { version = "0.3", features = [ "tracing-log", ] } tracing-panic = "0.1.2" -unidecode = "0.3.0" -urlencoding = "2.1.3" -uuid = { version = "1.17.0", features = ["serde", "v4"] } [target.'cfg(windows)'.dependencies] open = "5.3.2" @@ -88,3 +53,5 @@ embed-resource = "3.0.5" tempfile = "3.24.0" async-trait = "0.1" url = "2.4" +mlm_parse = { path = "../mlm_parse" } +mlm_meta = { path = "../mlm_meta" } diff --git a/server/src/bin/libation_unmapped_categories.rs b/server/src/bin/libation_unmapped_categories.rs index 04c96c9e..c3dfbbe5 100644 --- a/server/src/bin/libation_unmapped_categories.rs +++ b/server/src/bin/libation_unmapped_categories.rs @@ -5,7 +5,7 @@ use std::{ }; use anyhow::{Context, Result, bail}; -use mlm::linker::{ +use mlm_core::linker::{ folder::Libation, libation_cats::{MappingDepth, three_plus_override_candidates}, }; diff --git a/server/src/lib.rs b/server/src/lib.rs index 40eba759..7d26df1a 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,17 +1,2 @@ -pub mod audiobookshelf; -pub mod autograbber; -pub mod cleaner; -pub mod config; -pub mod config_impl; -pub mod exporter; -pub mod linker; -pub mod lists; -pub mod logging; -pub mod metadata; -pub mod qbittorrent; -pub mod snatchlist; -pub mod stats; -pub mod torrent_downloader; -pub mod web; #[cfg(target_family = "windows")] pub mod windows; diff --git a/server/src/main.rs b/server/src/main.rs index 6034e56d..4b144858 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,29 +1,22 @@ #![windows_subsystem = "windows"] use std::{ - collections::BTreeMap, env, fs::{self, create_dir_all}, - io, mem, + io, path::PathBuf, process, sync::Arc, time::Duration, }; -use anyhow::{Context as _, Result}; +use anyhow::Result; use dirs::{config_dir, data_local_dir}; use figment::{ Figment, providers::{Env, Format, Toml}, }; use mlm_mam::api::MaM; -use time::OffsetDateTime; -use tokio::{ - select, - sync::{Mutex, watch}, - time::sleep, -}; use tracing::error; use tracing_appender::rolling::{RollingFileAppender, Rotation}; use tracing_subscriber::{ @@ -31,19 +24,8 @@ use tracing_subscriber::{ util::SubscriberInitExt as _, }; -use mlm::{ - audiobookshelf::match_torrents_to_abs, - autograbber::run_autograbber, - cleaner::run_library_cleaner, - config::Config, - linker::{folder::link_folders_to_library, torrent::link_torrents_to_library}, - lists::{get_lists, run_list_import}, - metadata::MetadataService, - snatchlist::run_snatchlist_search, - stats::{Context, Stats, Triggers}, - torrent_downloader::grab_selected_torrents, - web::start_webserver, -}; +use mlm_core::{config::Config, metadata::MetadataService, stats::Stats}; +use mlm_web_askama::router as askama_router; #[cfg(target_family = "windows")] use mlm::windows; @@ -173,27 +155,7 @@ async fn app_main() -> Result<()> { .unwrap(); return Ok(()); } - let mut config = config?; - for autograb in &mut config.autograbs { - autograb.filter.edition = mem::take(&mut autograb.edition); - } - for snatchlist in &mut config.snatchlist { - snatchlist.filter.edition = mem::take(&mut snatchlist.edition); - } - for list in &mut config.goodreads_lists { - for grab in &mut list.grab { - grab.filter.edition = mem::take(&mut grab.edition); - } - } - for list in &mut config.notion_lists { - for grab in &mut list.grab { - grab.filter.edition = mem::take(&mut grab.edition); - } - } - for tag in &mut config.tags { - tag.filter.edition = mem::take(&mut tag.edition); - } - let config = Arc::new(config); + let config = Arc::new(config?); let db = native_db::Builder::new().create(&mlm_db::MODELS, database_file)?; mlm_db::migrate(&db)?; @@ -213,25 +175,25 @@ async fn app_main() -> Result<()> { // Instantiate metadata service from config provider settings let default_timeout = Duration::from_secs(5); // Convert Config's ProviderConfig -> metadata::ProviderSetting - let provider_settings: Vec = config + let provider_settings: Vec = config .metadata_providers .iter() .map(|p| match p { - mlm::config::ProviderConfig::Hardcover(c) => { - mlm::metadata::ProviderSetting::Hardcover { + mlm_core::config::ProviderConfig::Hardcover(c) => { + mlm_core::metadata::ProviderSetting::Hardcover { enabled: c.enabled, timeout_secs: c.timeout_secs, api_key: c.api_key.clone(), } } - mlm::config::ProviderConfig::RomanceIo(c) => { - mlm::metadata::ProviderSetting::RomanceIo { + mlm_core::config::ProviderConfig::RomanceIo(c) => { + mlm_core::metadata::ProviderSetting::RomanceIo { enabled: c.enabled, timeout_secs: c.timeout_secs, } } - mlm::config::ProviderConfig::OpenLibrary(c) => { - mlm::metadata::ProviderSetting::OpenLibrary { + mlm_core::config::ProviderConfig::OpenLibrary(c) => { + mlm_core::metadata::ProviderSetting::OpenLibrary { enabled: c.enabled, timeout_secs: c.timeout_secs, } @@ -241,506 +203,21 @@ async fn app_main() -> Result<()> { let metadata_service = MetadataService::from_settings(&provider_settings, default_timeout); let metadata_service = Arc::new(metadata_service); - let (mut search_tx, mut search_rx) = (BTreeMap::new(), BTreeMap::new()); - let (mut import_tx, mut import_rx) = (BTreeMap::new(), BTreeMap::new()); - let (torrent_linker_tx, torrent_linker_rx) = watch::channel(()); - let (folder_linker_tx, folder_linker_rx) = watch::channel(()); - let (downloader_tx, mut downloader_rx) = watch::channel(()); - let (audiobookshelf_tx, mut audiobookshelf_rx) = watch::channel(()); - let mam = if config.mam_id.is_empty() { Err(anyhow::Error::msg("No mam_id set")) } else { MaM::new(&config.mam_id, db.clone()).await.map(Arc::new) }; - if let Ok(mam) = &mam { - { - let config = config.clone(); - let db = db.clone(); - let mam = mam.clone(); - let stats = stats.clone(); - tokio::spawn(async move { - if let Some(qbit_conf) = config.qbittorrent.first() { - loop { - let mut qbit: Option = None; - if downloader_rx.changed().await.is_err() { - break; - } - if qbit.is_none() { - match qbit::Api::new_login_username_password( - &qbit_conf.url, - &qbit_conf.username, - &qbit_conf.password, - ) - .await - { - Ok(q) => qbit = Some(q), - Err(err) => { - error!("Error logging in to qbit {}: {err}", qbit_conf.url); - stats - .update(|stats| { - stats.downloader_run_at = - Some(OffsetDateTime::now_utc()); - stats.downloader_result = Some(Err(err.into())); - }) - .await; - } - }; - } - let Some(qbit) = &qbit else { - continue; - }; - { - stats - .update(|stats| { - stats.downloader_run_at = Some(OffsetDateTime::now_utc()); - stats.downloader_result = None; - }) - .await; - } - let result = - grab_selected_torrents(&config, &db, qbit, &qbit_conf.url, &mam) - .await - .context("grab_selected_torrents"); - - if let Err(err) = &result { - error!("Error grabbing selected torrents: {err:?}"); - } - { - stats - .update(|stats| { - stats.downloader_result = Some(result); - }) - .await; - } - } - } - }); - } - - for (i, grab) in config.autograbs.iter().enumerate() { - let config = config.clone(); - let db = db.clone(); - let mam = mam.clone(); - let downloader_tx = downloader_tx.clone(); - let (tx, mut rx) = watch::channel(()); - search_tx.insert(i, tx); - search_rx.insert(i, rx.clone()); - let stats = stats.clone(); - let grab = Arc::new(grab.clone()); - tokio::spawn(async move { - loop { - let interval = grab.search_interval.unwrap_or(config.search_interval); - if interval > 0 { - select! { - () = sleep(Duration::from_secs(60 * grab.search_interval.unwrap_or(config.search_interval))) => {}, - result = rx.changed() => { - if let Err(err) = result { - error!("Error listening on search_rx: {err:?}"); - stats.update(|stats| { - stats.autograbber_result.insert(i, Err(err.into())); - }).await; - } - }, - } - } else { - let result = rx.changed().await; - if let Err(err) = result { - error!("Error listening on search_rx: {err:?}"); - stats - .update(|stats| { - stats.autograbber_result.insert(i, Err(err.into())); - }) - .await; - } - } - { - stats - .update(|stats| { - stats - .autograbber_run_at - .insert(i, OffsetDateTime::now_utc()); - stats.autograbber_result.remove(&i); - }) - .await; - } - let result = run_autograbber( - config.clone(), - db.clone(), - mam.clone(), - downloader_tx.clone(), - i, - grab.clone(), - ) - .await - .context("autograbbers"); - if let Err(err) = &result { - error!("Error running autograbbers: {err:?}"); - } - { - stats - .update(|stats| { - stats.autograbber_result.insert(i, result); - }) - .await; - } - } - }); - } - - for (i, grab) in config.snatchlist.iter().enumerate() { - let i = i + config.autograbs.len(); - let config = config.clone(); - let db = db.clone(); - let mam = mam.clone(); - let (tx, mut rx) = watch::channel(()); - search_tx.insert(i, tx); - search_rx.insert(i, rx.clone()); - let stats = stats.clone(); - let grab = Arc::new(grab.clone()); - tokio::spawn(async move { - loop { - let interval = grab.search_interval.unwrap_or(config.search_interval); - if interval > 0 { - select! { - () = sleep(Duration::from_secs(60 * interval)) => {}, - result = rx.changed() => { - if let Err(err) = result { - error!("Error listening on search_rx for snatchlist: {err:?}"); - stats.update(|stats| { - stats.autograbber_result.insert(i, Err(err.into())); - }).await; - } - }, - } - } else { - let result = rx.changed().await; - if let Err(err) = result { - error!("Error listening on search_rx for snatchlist: {err:?}"); - stats - .update(|stats| { - stats.autograbber_result.insert(i, Err(err.into())); - }) - .await; - } - } - { - stats - .update(|stats| { - stats - .autograbber_run_at - .insert(i, OffsetDateTime::now_utc()); - stats.autograbber_result.remove(&i); - }) - .await; - } - let result = run_snatchlist_search( - config.clone(), - db.clone(), - mam.clone(), - i, - grab.clone(), - ) - .await - .context("snatchlist_search"); - if let Err(err) = &result { - error!("Error running snatchlist_search: {err:?}"); - } - { - stats - .update(|stats| { - stats.autograbber_result.insert(i, result); - }) - .await; - } - } - }); - } - - for (i, list) in get_lists(&config).into_iter().enumerate() { - let config = config.clone(); - let db = db.clone(); - let mam = mam.clone(); - let downloader_tx = downloader_tx.clone(); - let (tx, mut rx) = watch::channel(()); - import_tx.insert(i, tx); - import_rx.insert(i, rx.clone()); - let stats = stats.clone(); - let list = Arc::new(list); - tokio::spawn(async move { - loop { - let interval = list.search_interval().unwrap_or(config.import_interval); - if interval > 0 { - select! { - () = sleep(Duration::from_secs(60 * interval)) => {}, - result = rx.changed() => { - if let Err(err) = result { - error!("Error listening on import_rx: {err:?}"); - stats.update(|stats| { - stats.import_result.insert(i, Err(err.into())); - }).await; - } - }, - } - } else { - let result = rx.changed().await; - if let Err(err) = result { - error!("Error listening on import_rx: {err:?}"); - stats - .update(|stats| { - stats.import_result.insert(i, Err(err.into())); - }) - .await; - } - } - { - stats - .update(|stats| { - stats.import_run_at.insert(i, OffsetDateTime::now_utc()); - stats.import_result.remove(&i); - }) - .await; - } - let result = run_list_import( - config.clone(), - db.clone(), - mam.clone(), - list.clone(), - i, - downloader_tx.clone(), - ) - .await - .context("import"); - if let Err(err) = &result { - error!("Error running import: {err:?}"); - } - { - stats - .update(|stats| { - stats.import_result.insert(i, result); - }) - .await; - } - } - }); - } - { - for qbit_conf in config.qbittorrent.clone() { - let config = config.clone(); - let db = db.clone(); - let mam = mam.clone(); - let stats = stats.clone(); - let mut linker_rx = torrent_linker_rx.clone(); - tokio::spawn(async move { - loop { - select! { - () = sleep(Duration::from_secs(60 * config.link_interval)) => {}, - result = linker_rx.changed() => { - if let Err(err) = result { - error!("Error listening on link_rx: {err:?}"); - stats - .update(|stats| { - stats.torrent_linker_run_at = Some(OffsetDateTime::now_utc()); - stats.torrent_linker_result = Some(Err(err.into())); - }).await; - } - }, - } - { - stats - .update(|stats| { - stats.torrent_linker_run_at = Some(OffsetDateTime::now_utc()); - stats.torrent_linker_result = None; - }) - .await; - } - let qbit = match qbit::Api::new_login_username_password( - &qbit_conf.url, - &qbit_conf.username, - &qbit_conf.password, - ) - .await - { - Ok(qbit) => qbit, - Err(err) => { - error!("Error logging in to qbit {}: {err}", qbit_conf.url); - stats - .update(|stats| { - stats.torrent_linker_run_at = - Some(OffsetDateTime::now_utc()); - stats.torrent_linker_result = - Some(Err(anyhow::Error::msg(format!( - "Error logging in to qbit {}: {err}", - qbit_conf.url, - )))); - }) - .await; - continue; - } - }; - let result = link_torrents_to_library( - config.clone(), - db.clone(), - (&qbit_conf, &qbit), - &mam, - ) - .await - .context("link_torrents_to_library"); - if let Err(err) = &result { - error!("Error running linker: {err:?}"); - } - { - stats - .update(|stats| { - stats.torrent_linker_result = Some(result); - stats.cleaner_run_at = Some(OffsetDateTime::now_utc()); - stats.cleaner_result = None; - }) - .await; - } - let result = run_library_cleaner(config.clone(), db.clone()) - .await - .context("library_cleaner"); - if let Err(err) = &result { - error!("Error running library_cleaner: {err:?}"); - } - { - stats - .update(|stats| { - stats.cleaner_result = Some(result); - }) - .await; - } - } - }); - } - } - } - { - let config = config.clone(); - let db = db.clone(); - let stats = stats.clone(); - let mut linker_rx = folder_linker_rx.clone(); - tokio::spawn(async move { - loop { - select! { - () = sleep(Duration::from_secs(60 * config.link_interval)) => {}, - result = linker_rx.changed() => { - if let Err(err) = result { - error!("Error listening on link_rx: {err:?}"); - stats - .update(|stats| { - stats.folder_linker_run_at = Some(OffsetDateTime::now_utc()); - stats.folder_linker_result = Some(Err(err.into())); - }).await; - } - }, - } - { - stats - .update(|stats| { - stats.folder_linker_run_at = Some(OffsetDateTime::now_utc()); - stats.folder_linker_result = None; - }) - .await; - } - let result = link_folders_to_library(config.clone(), db.clone()) - .await - .context("link_torrents_to_library"); - if let Err(err) = &result { - error!("Error running linker: {err:?}"); - } - { - stats - .update(|stats| { - stats.folder_linker_result = Some(result); - stats.cleaner_run_at = Some(OffsetDateTime::now_utc()); - stats.cleaner_result = None; - }) - .await; - } - let result = run_library_cleaner(config.clone(), db.clone()) - .await - .context("library_cleaner"); - if let Err(err) = &result { - error!("Error running library_cleaner: {err:?}"); - } - { - stats - .update(|stats| { - stats.cleaner_result = Some(result); - }) - .await; - } - } - }); - } + let web_port = config.web_port; + let web_host = config.web_host.clone(); - if let Some(config) = &config.audiobookshelf { - let config = config.clone(); - let db = db.clone(); - let stats = stats.clone(); - tokio::spawn(async move { - loop { - select! { - () = sleep(Duration::from_secs(60 * config.interval)) => {}, - result = audiobookshelf_rx.changed() => { - if let Err(err) = result { - error!("Error listening on audiobookshelf_rx: {err:?}"); - stats - .update(|stats| { - stats.audiobookshelf_result = Some(Err(err.into())); - }) - .await; - } - }, - } - { - stats - .update(|stats| { - stats.audiobookshelf_run_at = Some(OffsetDateTime::now_utc()); - stats.audiobookshelf_result = None; - }) - .await; - } - let result = match_torrents_to_abs(&config, db.clone()) - .await - .context("audiobookshelf_matcher"); - if let Err(err) = &result { - error!("Error running audiobookshelf matcher: {err:?}"); - } - { - stats - .update(|stats| { - stats.audiobookshelf_result = Some(result); - }) - .await; - } - } - }); - } + let context = mlm_core::runner::spawn_tasks(config, db, Arc::new(mam), stats, metadata_service); - let triggers = Triggers { - search_tx, - import_tx, - torrent_linker_tx, - folder_linker_tx, - downloader_tx, - audiobookshelf_tx, - }; - #[cfg(target_family = "windows")] - let web_port = config.web_port; - let context = Context { - config: Arc::new(Mutex::new(config)), - db, - mam: Arc::new(mam), - stats, - metadata: metadata_service, - triggers, - }; + let app = askama_router(context.clone()); - let result = start_webserver(context).await; + let listener = tokio::net::TcpListener::bind((web_host, web_port)).await?; + let result: Result<()> = axum::serve(listener, app).await.map_err(|e| e.into()); #[cfg(target_family = "windows")] if let Err(err) = &result { diff --git a/server/src/web/pages/config.rs b/server/src/web/pages/config.rs deleted file mode 100644 index 782391ea..00000000 --- a/server/src/web/pages/config.rs +++ /dev/null @@ -1,273 +0,0 @@ -use std::{ops::Deref, sync::Arc}; - -use anyhow::Result; -use askama::Template; -use axum::{ - extract::{OriginalUri, Query, State}, - response::{Html, Redirect}, -}; -use axum_extra::extract::Form; -use mlm_db::{AudiobookCategory, DatabaseExt as _, EbookCategory, Torrent}; -use mlm_mam::serde::DATE_FORMAT; -use reqwest::Url; -use serde::Deserialize; -use tracing::{info, warn}; - -use crate::{ - autograbber::update_torrent_meta, - config::{Config, Cost, Library, TorrentSearch, Type}, - qbittorrent::ensure_category_exists, - stats::Context, - web::{AppError, Page, filter, yaml_items, yaml_nums}, -}; - -pub async fn config_page( - State(config): State>, - Query(query): Query, -) -> std::result::Result, AppError> { - let template = ConfigPageTemplate { - config, - show_apply_tags: query.show_apply_tags.unwrap_or_default(), - }; - Ok::<_, AppError>(Html(template.to_string())) -} - -pub async fn config_page_post( - State(context): State, - uri: OriginalUri, - Form(form): Form, -) -> Result { - let config = context.config().await; - match form.action.as_str() { - "apply" => { - let tag_filter = form - .tag_filter - .ok_or(anyhow::Error::msg("apply requires tag_filter"))?; - let tag_filter = config - .tags - .get(tag_filter) - .ok_or(anyhow::Error::msg("invalid tag_filter"))?; - let qbit_conf = config - .qbittorrent - .get(form.qbit_index.unwrap_or_default()) - .ok_or(anyhow::Error::msg("requires a qbit config"))?; - let qbit = qbit::Api::new_login_username_password( - &qbit_conf.url, - &qbit_conf.username, - &qbit_conf.password, - ) - .await?; - let torrents = context.db.r_transaction()?.scan().primary::()?; - for torrent in torrents.all()? { - let torrent = torrent?; - match tag_filter.filter.matches_lib(&torrent) { - Ok(matches) => { - if !matches { - continue; - } - } - Err(err) => { - let Some(mam_id) = torrent.mam_id else { - continue; - }; - info!("need to ask mam due to: {err}"); - let mam = context.mam()?; - let Some(mam_torrent) = mam.get_torrent_info_by_id(mam_id).await? else { - warn!("could not get torrent from mam"); - continue; - }; - let new_meta = mam_torrent.as_meta()?; - if new_meta != torrent.meta { - update_torrent_meta( - &config, - &context.db, - context.db.rw_async().await?, - Some(&mam_torrent), - torrent.clone(), - new_meta, - false, - false, - ) - .await?; - } - if !tag_filter.filter.matches(&mam_torrent) { - continue; - } - } - }; - if let Some(category) = &tag_filter.category { - ensure_category_exists(&qbit, &qbit_conf.url, category).await?; - qbit.set_category(Some(vec![torrent.id.as_str()]), category) - .await?; - info!( - "set category {} on torrent {}", - category, torrent.meta.title - ); - } - - if !tag_filter.tags.is_empty() { - qbit.add_tags( - Some(vec![torrent.id.as_str()]), - tag_filter.tags.iter().map(Deref::deref).collect(), - ) - .await?; - info!( - "set tags {:?} on torrent {}", - tag_filter.tags, torrent.meta.title - ); - } - } - } - action => { - eprintln!("unknown action: {action}"); - } - } - - Ok(Redirect::to(&uri.to_string())) -} - -#[derive(Debug, Deserialize)] -pub struct ConfigPageQuery { - show_apply_tags: Option, -} - -#[derive(Debug, Deserialize)] -pub struct ConfigPageForm { - action: String, - qbit_index: Option, - tag_filter: Option, -} - -#[derive(Template)] -#[template(path = "pages/config.html")] -struct ConfigPageTemplate { - config: Arc, - show_apply_tags: bool, -} - -impl Page for ConfigPageTemplate {} - -impl TorrentSearch { - fn mam_search(&self) -> String { - let mut url: Url = "https://www.myanonamouse.net/tor/browse.php?thumbnail=true" - .parse() - .unwrap(); - - { - let mut query = url.query_pairs_mut(); - if let Some(text) = &self.query { - query.append_pair("tor[text]", text); - } - - for srch_in in &self.search_in { - query.append_pair(&format!("tor[srchIn][{}]", srch_in.as_str()), "true"); - } - let search_in = match self.kind { - Type::Bookmarks => "bookmarks", - _ => "torrents", - }; - query.append_pair("tor[searchIn]", search_in); - let search_type = match (self.kind, self.cost) { - (Type::Freeleech, _) => "fl", - (_, Cost::Free) => "fl-VIP", - _ => "all", - }; - query.append_pair("tor[searchType]", search_type); - let sort_type = match self.kind { - Type::New => "dateDesc", - _ => "", - }; - if !sort_type.is_empty() { - query.append_pair("tor[sort_type]", search_type); - } - for cat in self - .filter - .edition - .categories - .audio - .clone() - .unwrap_or_else(AudiobookCategory::all) - .into_iter() - .map(AudiobookCategory::to_id) - { - query.append_pair("tor[cat][]", &cat.to_string()); - } - for cat in self - .filter - .edition - .categories - .ebook - .clone() - .unwrap_or_else(EbookCategory::all) - .into_iter() - .map(EbookCategory::to_id) - { - query.append_pair("tor[cat][]", &cat.to_string()); - } - for lang in &self.filter.edition.languages { - query.append_pair("tor[browse_lang][]", &lang.to_id().to_string()); - } - - let (flags_is_hide, flags) = self.filter.edition.flags.as_search_bitfield(); - if !flags.is_empty() { - query.append_pair( - "tor[browseFlagsHideVsShow]", - if flags_is_hide { "0" } else { "1" }, - ); - } - for flag in flags { - query.append_pair("tor[browseFlags][]", &flag.to_string()); - } - - if self.filter.edition.min_size.bytes() > 0 || self.filter.edition.max_size.bytes() > 0 - { - query.append_pair("tor[unit]", "1"); - } - if self.filter.edition.min_size.bytes() > 0 { - query.append_pair( - "tor[minSize]", - &self.filter.edition.min_size.bytes().to_string(), - ); - } - if self.filter.edition.max_size.bytes() > 0 { - query.append_pair( - "tor[maxSize]", - &self.filter.edition.max_size.bytes().to_string(), - ); - } - - if let Some(uploaded_after) = self.filter.uploaded_after { - query.append_pair( - "tor[startDate]", - &uploaded_after.format(&DATE_FORMAT).unwrap(), - ); - } - if let Some(uploaded_before) = self.filter.uploaded_before { - query.append_pair( - "tor[endDate]", - &uploaded_before.format(&DATE_FORMAT).unwrap(), - ); - } - if let Some(min_seeders) = self.filter.min_seeders { - query.append_pair("tor[minSeeders]", &min_seeders.to_string()); - } - if let Some(max_seeders) = self.filter.max_seeders { - query.append_pair("tor[maxSeeders]", &max_seeders.to_string()); - } - if let Some(min_leechers) = self.filter.min_leechers { - query.append_pair("tor[minLeechers]", &min_leechers.to_string()); - } - if let Some(max_leechers) = self.filter.max_leechers { - query.append_pair("tor[maxLeechers]", &max_leechers.to_string()); - } - if let Some(min_snatched) = self.filter.min_snatched { - query.append_pair("tor[minSnatched]", &min_snatched.to_string()); - } - if let Some(max_snatched) = self.filter.max_snatched { - query.append_pair("tor[maxSnatched]", &max_snatched.to_string()); - } - } - - url.to_string() - } -} diff --git a/server/tests/cleaner_test.rs b/server/tests/cleaner_test.rs index 769d343c..9ca3da55 100644 --- a/server/tests/cleaner_test.rs +++ b/server/tests/cleaner_test.rs @@ -1,7 +1,7 @@ mod common; use common::{MockFs, MockTorrentBuilder, TestDb, mock_config}; -use mlm::cleaner::run_library_cleaner; +use mlm_core::{Events, cleaner::run_library_cleaner}; use mlm_db::{DatabaseExt, Torrent}; use std::sync::Arc; @@ -45,7 +45,7 @@ async fn test_run_library_cleaner() -> anyhow::Result<()> { rw.commit()?; } - run_library_cleaner(config.clone(), test_db.db.clone()).await?; + run_library_cleaner(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let t1_after: Torrent = r.get().primary("ID1".to_string())?.unwrap(); diff --git a/server/tests/common/mod.rs b/server/tests/common/mod.rs index 3acf9462..47049660 100644 --- a/server/tests/common/mod.rs +++ b/server/tests/common/mod.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use mlm::config::{Config, Library, LibraryByRipDir, LibraryLinkMethod, LibraryOptions}; +use mlm_core::config::{Config, Library, LibraryByRipDir, LibraryLinkMethod, LibraryOptions}; use mlm_db::{ Database, MODELS, MainCat, MediaType, MetadataSource, Size, Timestamp, Torrent, TorrentMeta, migrate, @@ -272,8 +272,8 @@ pub fn mock_config(rip_dir: PathBuf, library_dir: PathBuf) -> Config { }, filter: Default::default(), })], - metadata_providers: vec![mlm::config::ProviderConfig::RomanceIo( - mlm::config::RomanceIoConfig { + metadata_providers: vec![mlm_core::config::ProviderConfig::RomanceIo( + mlm_core::config::RomanceIoConfig { enabled: true, timeout_secs: None, }, diff --git a/server/tests/config_impl.rs b/server/tests/config_impl.rs index f88a7bea..4d9b4607 100644 --- a/server/tests/config_impl.rs +++ b/server/tests/config_impl.rs @@ -1,4 +1,4 @@ -use mlm::config::{Cost, EditionFilter, GoodreadsList, Grab, TorrentFilter}; +use mlm_core::config::{Cost, EditionFilter, GoodreadsList, Grab, TorrentFilter}; use mlm_db::AudiobookCategory; use mlm_mam::enums::Categories; @@ -63,7 +63,6 @@ fn test_allow_audio_true_when_none_or_nonempty() { grab: vec![Grab { cost: Cost::default(), filter: TorrentFilter::default(), - edition: EditionFilter::default(), }], search_interval: None, unsat_buffer: None, @@ -83,9 +82,11 @@ fn test_allow_audio_true_when_none_or_nonempty() { prefer_format: None, grab: vec![Grab { cost: Cost::default(), - filter: TorrentFilter::default(), - edition: EditionFilter { - categories: cats, + filter: TorrentFilter { + edition: EditionFilter { + categories: cats, + ..Default::default() + }, ..Default::default() }, }], @@ -110,9 +111,11 @@ fn test_allow_audio_false_when_all_grabs_empty_audio() { prefer_format: None, grab: vec![Grab { cost: Cost::default(), - filter: TorrentFilter::default(), - edition: EditionFilter { - categories: cats, + filter: TorrentFilter { + edition: EditionFilter { + categories: cats, + ..Default::default() + }, ..Default::default() }, }], @@ -135,7 +138,6 @@ fn test_allow_ebook_behaviour() { grab: vec![Grab { cost: Cost::default(), filter: TorrentFilter::default(), - edition: EditionFilter::default(), }], search_interval: None, unsat_buffer: None, @@ -155,9 +157,11 @@ fn test_allow_ebook_behaviour() { prefer_format: None, grab: vec![Grab { cost: Cost::default(), - filter: TorrentFilter::default(), - edition: EditionFilter { - categories: cats, + filter: TorrentFilter { + edition: EditionFilter { + categories: cats, + ..Default::default() + }, ..Default::default() }, }], diff --git a/server/tests/linker_folder_test.rs b/server/tests/linker_folder_test.rs index 2790c2cc..2e0fab43 100644 --- a/server/tests/linker_folder_test.rs +++ b/server/tests/linker_folder_test.rs @@ -1,8 +1,8 @@ mod common; use common::{MockFs, TestDb, mock_config}; -use mlm::linker::folder::link_folders_to_library; -use mlm_db::{Category, DatabaseExt, Torrent}; +use mlm_core::{Events, linker::folder::link_folders_to_library}; +use mlm_db::{DatabaseExt, Torrent}; use std::{fs, sync::Arc}; #[tokio::test] @@ -16,7 +16,7 @@ async fn test_link_folders_to_library() -> anyhow::Result<()> { mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("B00TEST1".to_string())?; @@ -62,7 +62,7 @@ async fn test_link_folders_to_library_duplicate_skipping() -> anyhow::Result<()> // Libation folder files will have small size "fake audio data" = 15 bytes mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("B00TEST1".to_string())?; @@ -78,14 +78,14 @@ async fn test_link_folders_to_library_filter_size_too_small() -> anyhow::Result< let mock_fs = MockFs::new()?; let mut config = mock_config(mock_fs.rip_dir.clone(), mock_fs.library_dir.clone()); - if let mlm::config::Library::ByRipDir(ref mut l) = config.libraries[0] { + if let mlm_core::config::Library::ByRipDir(ref mut l) = config.libraries[0] { l.filter.min_size = mlm_db::Size::from_bytes(100); // Libation folder is 15 bytes } let config = Arc::new(config); mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("B00TEST1".to_string())?; @@ -103,14 +103,14 @@ async fn test_link_folders_to_library_filter_media_type_mismatch() -> anyhow::Re let mock_fs = MockFs::new()?; let mut config = mock_config(mock_fs.rip_dir.clone(), mock_fs.library_dir.clone()); - if let mlm::config::Library::ByRipDir(ref mut l) = config.libraries[0] { + if let mlm_core::config::Library::ByRipDir(ref mut l) = config.libraries[0] { l.filter.media_type = vec![mlm_db::MediaType::Ebook]; // Libation is Audiobook } let config = Arc::new(config); mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("B00TEST1".to_string())?; @@ -128,14 +128,14 @@ async fn test_link_folders_to_library_filter_language_mismatch() -> anyhow::Resu let mock_fs = MockFs::new()?; let mut config = mock_config(mock_fs.rip_dir.clone(), mock_fs.library_dir.clone()); - if let mlm::config::Library::ByRipDir(ref mut l) = config.libraries[0] { + if let mlm_core::config::Library::ByRipDir(ref mut l) = config.libraries[0] { l.filter.languages = vec![mlm_db::Language::German]; // Libation is English } let config = Arc::new(config); mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("B00TEST1".to_string())?; @@ -186,7 +186,7 @@ async fn test_link_folders_to_library_libation_missing_subtitle() -> anyhow::Res "fake audio data", )?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("1977386733".to_string())?; @@ -238,7 +238,7 @@ async fn test_link_folders_to_library_libation_missing_publication_name() -> any "fake audio data", )?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("B0DZ3R4CCN".to_string())?; @@ -283,7 +283,7 @@ async fn test_link_folders_to_library_libation_missing_narrators() -> anyhow::Re )?; fs::write(folder.join("Fat Ham [B0CM7V5MSN].m4b"), "fake audio data")?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("B0CM7V5MSN".to_string())?; @@ -375,7 +375,7 @@ async fn test_link_folders_to_library_nextory_wrapped_metadata() -> anyhow::Resu mock_fs.create_nextory_folder("nextory_wrapped", true)?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("nextory_424242".to_string())?; @@ -415,7 +415,7 @@ async fn test_link_folders_to_library_nextory_raw_metadata() -> anyhow::Result<( mock_fs.create_nextory_folder("nextory_raw_only", false)?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary("nextory_424242".to_string())?; @@ -495,7 +495,7 @@ async fn test_link_folders_to_library_libation_series_subtitle_does_not_overwrit "fake audio data", )?; - link_folders_to_library(config.clone(), test_db.db.clone()).await?; + link_folders_to_library(config.clone(), test_db.db.clone(), &Events::new()).await?; let r = test_db.db.r_transaction()?; let torrent: Option = r.get().primary(asin.to_string())?; diff --git a/server/tests/linker_torrent_test.rs b/server/tests/linker_torrent_test.rs index a7b48d1e..903bef47 100644 --- a/server/tests/linker_torrent_test.rs +++ b/server/tests/linker_torrent_test.rs @@ -2,11 +2,12 @@ mod common; use anyhow::Result; use common::{MockFs, TestDb, mock_config}; -use mlm::config::{ +use mlm_core::Events; +use mlm_core::config::{ Library, LibraryByDownloadDir, LibraryLinkMethod, LibraryOptions, LibraryTagFilters, QbitConfig, }; -use mlm::linker::torrent::{MaMApi, link_torrents_to_library}; -use mlm::qbittorrent::QbitApi; +use mlm_core::linker::torrent::{MaMApi, link_torrents_to_library}; +use mlm_core::qbittorrent::QbitApi; use mlm_db::DatabaseExt as _; use mlm_mam::search::MaMTorrent; use qbit::models::{Torrent as QbitTorrent, TorrentContent, Tracker}; @@ -166,6 +167,7 @@ async fn test_link_torrent_audiobook() -> anyhow::Result<()> { db.db.clone(), (qbit_config, &mock_qbit), &mock_mam, + &Events::new(), ) .await?; @@ -229,6 +231,7 @@ async fn test_skip_incomplete_torrent() -> anyhow::Result<()> { db.db.clone(), (qbit_config, &mock_qbit), &mock_mam, + &Events::new(), ) .await?; @@ -320,6 +323,7 @@ async fn test_remove_selected_torrent() -> anyhow::Result<()> { db.db.clone(), (qbit_config, &mock_qbit), &mock_mam, + &Events::new(), ) .await; @@ -408,6 +412,7 @@ async fn test_link_torrent_ebook() -> anyhow::Result<()> { db.db.clone(), (qbit_config, &mock_qbit), &mock_mam, + &Events::new(), ) .await?; @@ -504,13 +509,14 @@ async fn test_relink() -> anyhow::Result<()> { ..Default::default() }; - mlm::linker::torrent::relink_internal( + mlm_core::linker::torrent::relink_internal( &config, &qbit_config, &db.db, &mock_qbit, qbit_torrent, torrent_hash.to_string(), + &Events::new(), ) .await?; @@ -617,7 +623,7 @@ async fn test_refresh_metadata_relink() -> anyhow::Result<()> { ..Default::default() }; - mlm::linker::torrent::refresh_metadata_relink_internal( + mlm_core::linker::torrent::refresh_metadata_relink_internal( &config, &qbit_config, &db.db, @@ -625,6 +631,7 @@ async fn test_refresh_metadata_relink() -> anyhow::Result<()> { &mock_mam, qbit_torrent, torrent_hash.to_string(), + &Events::new(), ) .await?; diff --git a/server/tests/metadata_integration.rs b/server/tests/metadata_integration.rs index a50b69fc..89cf8d59 100644 --- a/server/tests/metadata_integration.rs +++ b/server/tests/metadata_integration.rs @@ -8,8 +8,8 @@ use mlm_db::{Event, EventKey, EventType, TorrentMeta as MetadataQuery}; use async_trait::async_trait; use common::{TestDb, mock_config}; -use mlm::metadata::MetadataService; -use mlm::stats::Context; +use mlm_core::Context; +use mlm_core::metadata::MetadataService; use url::Url; // Simple mock fetcher that returns inline mock data for tests. @@ -91,24 +91,24 @@ async fn test_metadata_fetch_and_persist_romanceio() -> Result<()> { let _default_timeout = StdDuration::from_secs(5); let providers = cfg.metadata_providers.clone(); // convert provider config to server metadata provider settings - let provider_settings: Vec = providers + let provider_settings: Vec = providers .iter() .map(|p| match p { - mlm::config::ProviderConfig::Hardcover(c) => { - mlm::metadata::ProviderSetting::Hardcover { + mlm_core::config::ProviderConfig::Hardcover(c) => { + mlm_core::metadata::ProviderSetting::Hardcover { enabled: c.enabled, timeout_secs: c.timeout_secs, api_key: c.api_key.clone(), } } - mlm::config::ProviderConfig::RomanceIo(c) => { - mlm::metadata::ProviderSetting::RomanceIo { + mlm_core::config::ProviderConfig::RomanceIo(c) => { + mlm_core::metadata::ProviderSetting::RomanceIo { enabled: c.enabled, timeout_secs: c.timeout_secs, } } - mlm::config::ProviderConfig::OpenLibrary(c) => { - mlm::metadata::ProviderSetting::OpenLibrary { + mlm_core::config::ProviderConfig::OpenLibrary(c) => { + mlm_core::metadata::ProviderSetting::OpenLibrary { enabled: c.enabled, timeout_secs: c.timeout_secs, } @@ -120,18 +120,21 @@ async fn test_metadata_fetch_and_persist_romanceio() -> Result<()> { let metadata = Arc::new(metadata); let ctx = Context { + backend: Some(Arc::new(mlm_core::SsrBackend { + db: test_db.db.clone(), + mam: Arc::new(Err(anyhow::anyhow!("no mam"))), + metadata: metadata.clone(), + })), config: Arc::new(tokio::sync::Mutex::new(Arc::new(cfg))), - db: test_db.db.clone(), - mam: Arc::new(Err(anyhow::anyhow!("no mam"))), - stats: mlm::stats::Stats::new(), - metadata: metadata.clone(), - triggers: mlm::stats::Triggers { + stats: mlm_core::stats::Stats::new(), + events: mlm_core::Events::new(), + triggers: mlm_core::Triggers { search_tx: std::collections::BTreeMap::new(), import_tx: std::collections::BTreeMap::new(), - torrent_linker_tx: tokio::sync::watch::channel(()).0, - folder_linker_tx: tokio::sync::watch::channel(()).0, - downloader_tx: tokio::sync::watch::channel(()).0, - audiobookshelf_tx: tokio::sync::watch::channel(()).0, + torrent_linker_tx: Some(tokio::sync::watch::channel(()).0), + folder_linker_tx: Some(tokio::sync::watch::channel(()).0), + downloader_tx: Some(tokio::sync::watch::channel(()).0), + audiobookshelf_tx: Some(tokio::sync::watch::channel(()).0), }, }; @@ -143,20 +146,26 @@ async fn test_metadata_fetch_and_persist_romanceio() -> Result<()> { let mock_fetcher = std::sync::Arc::new(MockFetcher); // Rebuild a metadata service with a RomanceIo using the mock fetcher. let rom = mlm_meta::providers::RomanceIo::with_client(mock_fetcher.clone()); - let svc = mlm::metadata::MetadataService::new( + let svc = mlm_core::metadata::MetadataService::new( vec![(std::sync::Arc::new(rom), std::time::Duration::from_secs(5))], std::time::Duration::from_secs(5), ); let metadata = Arc::new(svc); let ctx = Context { - metadata: metadata.clone(), + backend: Some(Arc::new(mlm_core::SsrBackend { + db: test_db.db.clone(), + mam: Arc::new(Err(anyhow::anyhow!("no mam"))), + metadata: metadata.clone(), + })), ..ctx }; // Use a title known to the plan/romanceio mock - let mut q: MetadataQuery = Default::default(); - q.title = "Of Ink and Alchemy".to_string(); + let q = MetadataQuery { + title: "Of Ink and Alchemy".to_string(), + ..Default::default() + }; let meta = metadata.fetch_and_persist(&ctx, q).await?; // Expect meta to contain some categories/tags