diff --git a/Cargo.lock b/Cargo.lock index a5507c96..19cea592 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1396,9 +1396,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libredox" @@ -1412,9 +1412,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" @@ -1587,6 +1587,7 @@ dependencies = [ "serde_derive", "serde_json", "sublime_fuzzy", + "tempfile", "thiserror 2.0.17", "time", "tokio", @@ -1640,6 +1641,7 @@ dependencies = [ "native_db", "native_model", "once_cell", + "openssl", "reqwest", "reqwest_cookie_store", "serde", @@ -2367,15 +2369,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.7" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ "bitflags 2.9.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2835,15 +2837,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.20.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] diff --git a/mlm_db/src/impls.rs b/mlm_db/src/impls.rs index 433ea964..98576cdb 100644 --- a/mlm_db/src/impls.rs +++ b/mlm_db/src/impls.rs @@ -1,6 +1,7 @@ pub mod categories; pub mod flags; pub mod language; +pub mod lists; pub mod meta; pub mod old_categories; pub mod series; diff --git a/mlm_db/src/impls/categories.rs b/mlm_db/src/impls/categories.rs index c2f714e1..751b778e 100644 --- a/mlm_db/src/impls/categories.rs +++ b/mlm_db/src/impls/categories.rs @@ -1,6 +1,6 @@ use std::str::FromStr; -use crate::{Category, MainCat, MediaType, OldMainCat}; +use crate::{Category, MainCat, MediaType, OldCategory, OldMainCat}; impl MediaType { pub fn from_id(id: u8) -> Option { @@ -127,222 +127,879 @@ impl std::fmt::Display for MainCat { } impl Category { - pub fn from_id(id: u8) -> Option { - match id { - 1 => Some(Category::Action), - 2 => Some(Category::Art), - 3 => Some(Category::Biographical), - 4 => Some(Category::Business), - 5 => Some(Category::Comedy), - 6 => Some(Category::CompleteEditionsMusic), - 7 => Some(Category::Computer), - 8 => Some(Category::Crafts), - 9 => Some(Category::Crime), - 10 => Some(Category::Dramatization), - 11 => Some(Category::Education), - 12 => Some(Category::FactualNews), - 13 => Some(Category::Fantasy), - 14 => Some(Category::Food), - 15 => Some(Category::Guitar), - 16 => Some(Category::Health), - 17 => Some(Category::Historical), - 18 => Some(Category::Home), - 19 => Some(Category::Horror), - 20 => Some(Category::Humor), - 21 => Some(Category::IndividualSheet), - 22 => Some(Category::Instructional), - 23 => Some(Category::Juvenile), - 24 => Some(Category::Language), - 25 => Some(Category::Lgbt), - 26 => Some(Category::LickLibraryLTP), - 27 => Some(Category::LickLibraryTechniques), - 28 => Some(Category::LiteraryClassics), - 29 => Some(Category::LitRPG), - 30 => Some(Category::Math), - 31 => Some(Category::Medicine), - 32 => Some(Category::Music), - 33 => Some(Category::MusicBook), - 34 => Some(Category::Mystery), - 35 => Some(Category::Nature), - 36 => Some(Category::Paranormal), - 37 => Some(Category::Philosophy), - 38 => Some(Category::Poetry), - 39 => Some(Category::Politics), - 40 => Some(Category::Reference), - 41 => Some(Category::Religion), - 42 => Some(Category::Romance), - 43 => Some(Category::Rpg), - 44 => Some(Category::Science), - 45 => Some(Category::ScienceFiction), - 46 => Some(Category::SelfHelp), - 47 => Some(Category::SheetCollection), - 48 => Some(Category::SheetCollectionMP3), - 49 => Some(Category::Sports), - 50 => Some(Category::Technology), - 51 => Some(Category::Thriller), - 52 => Some(Category::Travel), - 53 => Some(Category::UrbanFantasy), - 54 => Some(Category::Western), - 55 => Some(Category::YoungAdult), - 56 => Some(Category::Superheroes), - 57 => Some(Category::LiteraryFiction), - 58 => Some(Category::ProgressionFantasy), - 59 => Some(Category::ContemporaryFiction), - 60 => Some(Category::DramaPlays), - id => Some(Category::Unknown(id)), + pub fn from_old_category(category: OldCategory) -> Vec { + match category { + OldCategory::Audio(cat) => { + let mut mapped = vec![Category::Audiobook]; + mapped.extend(match cat { + crate::AudiobookCategory::ActionAdventure => vec![Category::ActionAdventure], + crate::AudiobookCategory::Art => vec![Category::ArtPhotography], + crate::AudiobookCategory::Biographical => vec![Category::Biography], + crate::AudiobookCategory::Business => vec![Category::Business], + crate::AudiobookCategory::ComputerInternet => vec![Category::ComputerScience], + crate::AudiobookCategory::Crafts => vec![Category::CraftsDiy], + crate::AudiobookCategory::CrimeThriller => { + vec![Category::Crime, Category::Thriller] + } + crate::AudiobookCategory::Fantasy => vec![Category::Fantasy], + crate::AudiobookCategory::Food => vec![Category::CookingFood], + crate::AudiobookCategory::GeneralNonFic => vec![], + crate::AudiobookCategory::HistoricalFiction => vec![Category::Historical], + crate::AudiobookCategory::History => vec![Category::History], + crate::AudiobookCategory::HomeGarden => vec![Category::HomeGarden], + crate::AudiobookCategory::Horror => vec![Category::Horror], + crate::AudiobookCategory::Humor => vec![Category::Funny, Category::Humor], + crate::AudiobookCategory::Instructional => vec![Category::GuideManual], + crate::AudiobookCategory::Juvenile => vec![Category::Children], + crate::AudiobookCategory::Language => vec![Category::LanguageLinguistics], + crate::AudiobookCategory::MathScienceTech => { + vec![Category::Science, Category::Technology] + } + crate::AudiobookCategory::Medical => vec![Category::Medicine], + crate::AudiobookCategory::Mystery => vec![Category::Mystery], + crate::AudiobookCategory::Nature => vec![Category::NatureEnvironment], + crate::AudiobookCategory::Philosophy => vec![Category::Philosophy], + crate::AudiobookCategory::PolSocRelig => { + vec![Category::PoliticsSociety, Category::ReligionSpirituality] + } + crate::AudiobookCategory::Recreation => vec![Category::SportsOutdoors], + crate::AudiobookCategory::Romance => vec![Category::Romance], + crate::AudiobookCategory::ScienceFiction => vec![Category::ScienceFiction], + crate::AudiobookCategory::SelfHelp => vec![Category::SelfHelp], + crate::AudiobookCategory::TravelAdventure => vec![Category::Travel], + crate::AudiobookCategory::TrueCrime => vec![Category::TrueCrime], + crate::AudiobookCategory::UrbanFantasy => { + vec![Category::Fantasy, Category::UrbanFantasy] + } + crate::AudiobookCategory::Western => vec![Category::Western], + crate::AudiobookCategory::YoungAdult => vec![Category::YoungAdult], + _ => vec![], + }); + mapped + } + OldCategory::Ebook(cat) => { + let mut mapped = vec![Category::Ebook]; + mapped.extend(match cat { + crate::EbookCategory::ActionAdventure => vec![Category::ActionAdventure], + crate::EbookCategory::Art => vec![Category::ArtPhotography], + crate::EbookCategory::Biographical => vec![Category::Biography], + crate::EbookCategory::Business => vec![Category::Business], + crate::EbookCategory::ComicsGraphicnovels => { + vec![Category::GraphicNovelsComics] + } + crate::EbookCategory::ComputerInternet => vec![Category::ComputerScience], + crate::EbookCategory::Crafts => vec![Category::CraftsDiy], + crate::EbookCategory::CrimeThriller => { + vec![Category::Crime, Category::Thriller] + } + crate::EbookCategory::Fantasy => vec![Category::Fantasy], + crate::EbookCategory::Food => vec![Category::CookingFood], + crate::EbookCategory::GeneralNonFiction => vec![], + crate::EbookCategory::HistoricalFiction => vec![Category::Historical], + crate::EbookCategory::History => vec![Category::History], + crate::EbookCategory::HomeGarden => vec![Category::HomeGarden], + crate::EbookCategory::Horror => vec![Category::Horror], + crate::EbookCategory::Humor => vec![Category::Funny, Category::Humor], + crate::EbookCategory::IllusionMagic => vec![Category::MythologyFolklore], + crate::EbookCategory::Instructional => vec![Category::GuideManual], + crate::EbookCategory::Juvenile => vec![Category::Children], + crate::EbookCategory::Language => vec![Category::LanguageLinguistics], + crate::EbookCategory::MagazinesNewspapers => vec![], + crate::EbookCategory::MathScienceTech => { + vec![Category::Science, Category::Technology] + } + crate::EbookCategory::Medical => vec![Category::Medicine], + crate::EbookCategory::MixedCollections => vec![Category::Anthology], + crate::EbookCategory::Mystery => vec![Category::Mystery], + crate::EbookCategory::Nature => vec![Category::NatureEnvironment], + crate::EbookCategory::Philosophy => vec![Category::Philosophy], + crate::EbookCategory::PolSocRelig => { + vec![Category::PoliticsSociety, Category::ReligionSpirituality] + } + crate::EbookCategory::Recreation => vec![Category::SportsOutdoors], + crate::EbookCategory::Romance => vec![Category::Romance], + crate::EbookCategory::ScienceFiction => vec![Category::ScienceFiction], + crate::EbookCategory::SelfHelp => vec![Category::SelfHelp], + crate::EbookCategory::TravelAdventure => vec![Category::Travel], + crate::EbookCategory::TrueCrime => vec![Category::TrueCrime], + crate::EbookCategory::UrbanFantasy => { + vec![Category::Fantasy, Category::UrbanFantasy] + } + crate::EbookCategory::Western => vec![Category::Western], + crate::EbookCategory::YoungAdult => vec![Category::YoungAdult], + _ => vec![], + }); + mapped + } + OldCategory::Musicology(cat) => match cat { + crate::MusicologyCategory::GuitarBassTabs + | crate::MusicologyCategory::IndividualSheet + | crate::MusicologyCategory::IndividualSheetMP3 + | crate::MusicologyCategory::SheetCollection + | crate::MusicologyCategory::SheetCollectionMP3 => { + vec![Category::Music, Category::SheetMusicScores] + } + crate::MusicologyCategory::MusicCompleteEditions + | crate::MusicologyCategory::MusicBook + | crate::MusicologyCategory::MusicBookMP3 => vec![Category::Music], + crate::MusicologyCategory::InstructionalBookWithVideo + | crate::MusicologyCategory::InstructionalMediaMusic + | crate::MusicologyCategory::LickLibraryLTPJamWith + | crate::MusicologyCategory::LickLibraryTechniquesQL => { + vec![Category::Music, Category::GuideManual] + } + }, + OldCategory::Radio(cat) => match cat { + crate::RadioCategory::Comedy => vec![Category::Funny], + crate::RadioCategory::Drama => vec![Category::DramaPlays], + _ => vec![], + }, } } - pub fn as_str(&self) -> &'static str { - match self { - // Since there is no replacement for general fiction, literary fiction is being used - // instead. - Category::LiteraryFiction => "General Fiction", - Category::ContemporaryFiction => "Contemporary", - _ => self.as_raw_str(), + pub fn from_legacy_v15_id(id: u8) -> Option<(Vec, Vec)> { + let category = match id { + 1 => crate::v15::Category::Action, + 2 => crate::v15::Category::Art, + 3 => crate::v15::Category::Biographical, + 4 => crate::v15::Category::Business, + 5 => crate::v15::Category::Comedy, + 6 => crate::v15::Category::CompleteEditionsMusic, + 7 => crate::v15::Category::Computer, + 8 => crate::v15::Category::Crafts, + 9 => crate::v15::Category::Crime, + 10 => crate::v15::Category::Dramatization, + 11 => crate::v15::Category::Education, + 12 => crate::v15::Category::FactualNews, + 13 => crate::v15::Category::Fantasy, + 14 => crate::v15::Category::Food, + 15 => crate::v15::Category::Guitar, + 16 => crate::v15::Category::Health, + 17 => crate::v15::Category::Historical, + 18 => crate::v15::Category::Home, + 19 => crate::v15::Category::Horror, + 20 => crate::v15::Category::Humor, + 21 => crate::v15::Category::IndividualSheet, + 22 => crate::v15::Category::Instructional, + 23 => crate::v15::Category::Juvenile, + 24 => crate::v15::Category::Language, + 25 => crate::v15::Category::Lgbt, + 26 => crate::v15::Category::LickLibraryLTP, + 27 => crate::v15::Category::LickLibraryTechniques, + 28 => crate::v15::Category::LiteraryClassics, + 29 => crate::v15::Category::LitRPG, + 30 => crate::v15::Category::Math, + 31 => crate::v15::Category::Medicine, + 32 => crate::v15::Category::Music, + 33 => crate::v15::Category::MusicBook, + 34 => crate::v15::Category::Mystery, + 35 => crate::v15::Category::Nature, + 36 => crate::v15::Category::Paranormal, + 37 => crate::v15::Category::Philosophy, + 38 => crate::v15::Category::Poetry, + 39 => crate::v15::Category::Politics, + 40 => crate::v15::Category::Reference, + 41 => crate::v15::Category::Religion, + 42 => crate::v15::Category::Romance, + 43 => crate::v15::Category::Rpg, + 44 => crate::v15::Category::Science, + 45 => crate::v15::Category::ScienceFiction, + 46 => crate::v15::Category::SelfHelp, + 47 => crate::v15::Category::SheetCollection, + 48 => crate::v15::Category::SheetCollectionMP3, + 49 => crate::v15::Category::Sports, + 50 => crate::v15::Category::Technology, + 51 => crate::v15::Category::Thriller, + 52 => crate::v15::Category::Travel, + 53 => crate::v15::Category::UrbanFantasy, + 54 => crate::v15::Category::Western, + 55 => crate::v15::Category::YoungAdult, + 56 => crate::v15::Category::Superheroes, + 57 => crate::v15::Category::LiteraryFiction, + 58 => crate::v15::Category::ProgressionFantasy, + 59 => crate::v15::Category::ContemporaryFiction, + 60 => crate::v15::Category::DramaPlays, + 61 => crate::v15::Category::Unknown(61), + 62 => crate::v15::Category::Unknown(62), + _ => return None, + }; + Some(Self::from_legacy_v15_category(category, &[], &[])) + } + + pub fn from_legacy_v15_category( + category: crate::v15::Category, + legacy_categories: &[crate::v15::Category], + existing_categories: &[Category], + ) -> (Vec, Vec) { + let mut mapped = Vec::new(); + let mut tags = Vec::new(); + + match category { + crate::v15::Category::Action => mapped.extend([Category::ActionAdventure]), + crate::v15::Category::Art => mapped.extend([Category::ArtPhotography]), + crate::v15::Category::Biographical => mapped.extend([Category::Biography]), + crate::v15::Category::Business => mapped.extend([Category::Business]), + crate::v15::Category::Comedy | crate::v15::Category::Humor => { + mapped.extend([Category::Funny, Category::Humor]); + } + crate::v15::Category::CompleteEditionsMusic + | crate::v15::Category::Music + | crate::v15::Category::MusicBook => mapped.extend([Category::Music]), + crate::v15::Category::Computer => mapped.extend([Category::ComputerScience]), + crate::v15::Category::Crafts => mapped.extend([Category::CraftsDiy]), + crate::v15::Category::Crime => mapped.extend([Category::Crime]), + crate::v15::Category::Dramatization => { + mapped.extend([Category::DramatizedAdaptation, Category::FullCast]); + } + crate::v15::Category::Education => mapped.extend([Category::Textbook]), + crate::v15::Category::FactualNews => mapped.extend([Category::PoliticsSociety]), + crate::v15::Category::Fantasy => mapped.extend([Category::Fantasy]), + crate::v15::Category::Food => mapped.extend([Category::CookingFood]), + crate::v15::Category::Guitar + | crate::v15::Category::IndividualSheet + | crate::v15::Category::SheetCollection + | crate::v15::Category::SheetCollectionMP3 => { + mapped.extend([Category::SheetMusicScores]); + } + crate::v15::Category::Health => mapped.extend([Category::HealthWellness]), + crate::v15::Category::Historical => mapped.extend([Category::Historical]), + crate::v15::Category::Home => mapped.extend([Category::HomeGarden]), + crate::v15::Category::Horror => mapped.extend([Category::Horror]), + crate::v15::Category::Lgbt => mapped.extend([Category::Lgbtqia]), + crate::v15::Category::Instructional + | crate::v15::Category::LickLibraryLTP + | crate::v15::Category::LickLibraryTechniques => { + mapped.extend([Category::GuideManual]); + } + crate::v15::Category::Juvenile => mapped.extend([Category::Children]), + crate::v15::Category::Language => mapped.extend([Category::LanguageLinguistics]), + crate::v15::Category::LiteraryClassics => { + tags.push(Self::legacy_v15_label(category)); + } + crate::v15::Category::LiteraryFiction => { + mapped.extend([Category::CharacterDriven]); + tags.push(Self::legacy_v15_label(category)); + } + crate::v15::Category::LitRPG | crate::v15::Category::ProgressionFantasy => { + mapped.extend([Category::Fantasy, Category::ProgressionFantasy]); + } + crate::v15::Category::Math => mapped.extend([Category::Mathematics]), + crate::v15::Category::Medicine => { + mapped.extend([Category::Medicine, Category::Psychology]); + } + crate::v15::Category::Mystery => mapped.extend([Category::Mystery]), + crate::v15::Category::Nature => mapped.extend([Category::NatureEnvironment]), + crate::v15::Category::Philosophy => mapped.extend([Category::Philosophy]), + crate::v15::Category::Poetry => mapped.extend([Category::Poetry]), + crate::v15::Category::Politics => mapped.extend([Category::PoliticsSociety]), + crate::v15::Category::Reference => mapped.extend([Category::Reference]), + crate::v15::Category::Religion => { + mapped.extend([Category::ReligionSpirituality]); + } + crate::v15::Category::Romance => { + mapped.extend([Category::Romance]); + if legacy_categories.contains(&crate::v15::Category::Thriller) { + mapped.extend([Category::RomanticSuspense]); + } + if legacy_categories.contains(&crate::v15::Category::Humor) { + mapped.extend([Category::RomanticComedy]); + } + } + crate::v15::Category::Rpg => mapped.extend([Category::Fantasy]), + crate::v15::Category::Science => mapped.extend([Category::Science]), + crate::v15::Category::ScienceFiction => mapped.extend([Category::ScienceFiction]), + crate::v15::Category::SelfHelp => mapped.extend([Category::SelfHelp]), + crate::v15::Category::Sports => mapped.extend([Category::SportsOutdoors]), + crate::v15::Category::Technology => mapped.extend([Category::Technology]), + crate::v15::Category::Thriller => mapped.extend([Category::Thriller]), + crate::v15::Category::Travel => mapped.extend([Category::Travel]), + crate::v15::Category::UrbanFantasy => { + mapped.extend([Category::Fantasy, Category::UrbanFantasy]); + } + crate::v15::Category::Western => mapped.extend([Category::Western]), + crate::v15::Category::YoungAdult => mapped.extend([Category::YoungAdult]), + crate::v15::Category::ContemporaryFiction => { + mapped.extend([Category::ContemporaryRealist]); + } + crate::v15::Category::DramaPlays => mapped.extend([Category::DramaPlays]), + crate::v15::Category::Paranormal => { + if !existing_categories.contains(&Category::ParanormalRomance) + && !existing_categories.contains(&Category::ParanormalHorror) + { + if legacy_categories.contains(&crate::v15::Category::Romance) { + mapped.extend([Category::ParanormalRomance]); + } else if legacy_categories.contains(&crate::v15::Category::Horror) { + mapped.extend([Category::ParanormalHorror]); + } else { + tags.push("Paranormal".to_string()); + } + } + } + crate::v15::Category::Unknown(61) => { + mapped.extend([Category::OccultEsotericism]); + } + crate::v15::Category::Unknown(62) => { + mapped.extend([Category::ContemporaryRealist, Category::CharacterDriven]); + } + _ => tags.push(Self::legacy_v15_label(category)), } + + (mapped, tags) } - pub fn as_raw_str(&self) -> &'static str { + pub fn from_id(id: u8) -> Option { + if let Some(legacy) = Category::legacy_label_by_id(id) + && let Some(mapped) = Category::from_legacy_label(legacy) + { + return Some(mapped); + } + + match OldCategory::from_one_id(id as u64) { + Some(OldCategory::Audio(_)) => Some(Category::Audiobook), + Some(OldCategory::Ebook(_)) => Some(Category::Ebook), + Some(OldCategory::Musicology(_)) => Some(Category::Music), + Some(OldCategory::Radio(_)) => Some(Category::DramatizedAdaptation), + _ => None, + } + } + + pub fn as_str(&self) -> &'static str { match self { - Category::Action => "Action/Adventure", - Category::Art => "Art/Photography", - Category::Biographical => "Biographical", - Category::Business => "Business/Money", - Category::Comedy => "Comedy", - Category::CompleteEditionsMusic => "Complete Editions - Music", - Category::Computer => "Computer/Internet", - Category::Crafts => "Crafts", - Category::Crime => "Crime", - Category::Dramatization => "Dramatization/Full Cast", - Category::Education => "Education/Textbook", - Category::FactualNews => "Factual News/Current Events", Category::Fantasy => "Fantasy", - Category::Food => "Food/Wine", - Category::Guitar => "Guitar/Bass Tabs", - Category::Health => "Health/Fitness/Diet", + Category::ScienceFiction => "Science Fiction", + Category::Romance => "Romance", Category::Historical => "Historical", - Category::Home => "Home/Garden", - Category::Horror => "Horror", - Category::Humor => "Humor", - Category::IndividualSheet => "Individual Sheet", - Category::Instructional => "Instructional", - Category::Juvenile => "Juvenile", - Category::Language => "Language", - Category::Lgbt => "LGBTQIA+", - Category::LickLibraryLTP => "Lick Library - LTP/Jam With", - Category::LickLibraryTechniques => "Lick Library - Techniques/QL", - Category::LiteraryClassics => "Literary Classics", - Category::LitRPG => "LitRPG", - Category::Math => "Math", - Category::Medicine => "Medicine/Psychology", - Category::Music => "Music", - Category::MusicBook => "Music Book", + Category::ContemporaryRealist => "Contemporary Realist", Category::Mystery => "Mystery", - Category::Nature => "Nature", - Category::Paranormal => "Paranormal", - Category::Philosophy => "Philosophy", + Category::Thriller => "Thriller", + Category::Crime => "Crime", + Category::Horror => "Horror", + Category::ActionAdventure => "Action & Adventure", + Category::Dystopian => "Dystopian", + Category::PostApocalyptic => "Post-Apocalyptic", + Category::MagicalRealism => "Magical Realism", + Category::Western => "Western", + Category::Military => "Military", + Category::EpicFantasy => "Epic Fantasy", + Category::UrbanFantasy => "Urban Fantasy", + Category::SwordAndSorcery => "Sword & Sorcery", + Category::HardSciFi => "Hard Sci-Fi", + Category::SpaceOpera => "Space Opera", + Category::Cyberpunk => "Cyberpunk", + Category::TimeTravel => "Time Travel", + Category::AlternateHistory => "Alternate History", + Category::ProgressionFantasy => "Progression Fantasy", + Category::RomanticComedy => "Romantic Comedy", + Category::RomanticSuspense => "Romantic Suspense", + Category::ParanormalRomance => "Paranormal Romance", + Category::DarkRomance => "Dark Romance", + Category::WhyChoose => "Why Choose", + Category::Erotica => "Erotica", + Category::Detective => "Detective", + Category::Noir => "Noir", + Category::LegalThriller => "Legal Thriller", + Category::PsychologicalThriller => "Psychological Thriller", + Category::CozyMystery => "Cozy Mystery", + Category::BodyHorror => "Body Horror", + Category::GothicHorror => "Gothic Horror", + Category::CosmicHorror => "Cosmic Horror", + Category::ParanormalHorror => "Paranormal Horror", + Category::Lgbtqia => "LGBTQIA+", + Category::TransRepresentation => "Trans Representation", + Category::DisabilityRepresentation => "Disability Representation", + Category::NeurodivergentRepresentation => "Neurodivergent Representation", + Category::PocRepresentation => "POC Representation", + Category::ComingOfAge => "Coming of Age", + Category::FoundFamily => "Found Family", + Category::EnemiesToLovers => "Enemies to Lovers", + Category::FriendsToLovers => "Friends to Lovers", + Category::FakeDating => "Fake Dating", + Category::SecondChance => "Second Chance", + Category::SlowBurn => "Slow Burn", + Category::PoliticalIntrigue => "Political Intrigue", + Category::Revenge => "Revenge", + Category::Redemption => "Redemption", + Category::Survival => "Survival", + Category::Retelling => "Retelling", + Category::Ancient => "Ancient", + Category::Medieval => "Medieval", + Category::EarlyModern => "Early Modern", + Category::NineteenthCentury => "19th Century", + Category::TwentiethCentury => "20th Century", + Category::Contemporary => "Contemporary", + Category::Future => "Future", + Category::AlternateTimeline => "Alternate Timeline", + Category::AlternateUniverse => "Alternate Universe", + Category::SmallTown => "Small Town", + Category::Urban => "Urban", + Category::Rural => "Rural", + Category::AcademySchool => "Academy / School", + Category::Space => "Space", + Category::Africa => "Africa", + Category::EastAsia => "East Asia", + Category::SouthAsia => "South Asia", + Category::SoutheastAsia => "Southeast Asia", + Category::MiddleEast => "Middle East", + Category::Europe => "Europe", + Category::NorthAmerica => "North America", + Category::LatinAmerica => "Latin America", + Category::Caribbean => "Caribbean", + Category::Oceania => "Oceania", + Category::Cozy => "Cozy", + Category::Dark => "Dark", + Category::Gritty => "Gritty", + Category::Wholesome => "Wholesome", + Category::Funny => "Funny", + Category::Satire => "Satire", + Category::Emotional => "Emotional", + Category::CharacterDriven => "Character-Driven", + Category::Children => "Children", + Category::MiddleGrade => "Middle Grade", + Category::YoungAdult => "Young Adult", + Category::NewAdult => "New Adult", + Category::Adult => "Adult", + Category::Audiobook => "Audiobook", + Category::Ebook => "Ebook", + Category::GraphicNovelsComics => "Graphic Novels & Comics", + Category::Manga => "Manga", + Category::Novella => "Novella", + Category::LightNovel => "Light Novel", + Category::ShortStories => "Short Stories", + Category::Anthology => "Anthology", Category::Poetry => "Poetry", - Category::Politics => "Politics/Sociology", - Category::Reference => "Reference", - Category::Religion => "Religion/Spirituality", - Category::Romance => "Romance", - Category::Rpg => "RPG", - Category::Science => "Science", - Category::ScienceFiction => "Science Fiction", + Category::Essays => "Essays", + Category::Epistolary => "Epistolary", + Category::DramaPlays => "Drama / Plays", + Category::FullCast => "Full Cast", + Category::DualNarration => "Dual Narration", + Category::DuetNarration => "Duet Narration", + Category::DramatizedAdaptation => "Dramatized Adaptation", + Category::AuthorNarrated => "Author Narrated", + Category::Abridged => "Abridged", + Category::Biography => "Biography", + Category::Memoir => "Memoir", + Category::History => "History", + Category::TrueCrime => "True Crime", + Category::Philosophy => "Philosophy", + Category::ReligionSpirituality => "Religion & Spirituality", + Category::MythologyFolklore => "Mythology & Folklore", + Category::OccultEsotericism => "Occult & Esotericism", + Category::PoliticsSociety => "Politics & Society", + Category::Business => "Business", + Category::PersonalFinance => "Personal Finance", + Category::ParentingFamily => "Parenting & Family", Category::SelfHelp => "Self-Help", - Category::SheetCollection => "Sheet Collection", - Category::SheetCollectionMP3 => "Sheet Collection MP3", - Category::Sports => "Sports/Hobbies", + Category::Psychology => "Psychology", + Category::HealthWellness => "Health & Wellness", + Category::Science => "Science", Category::Technology => "Technology", - Category::Thriller => "Thriller/Suspense", Category::Travel => "Travel", - Category::UrbanFantasy => "Urban Fantasy", - Category::Western => "Western", - Category::YoungAdult => "Young Adult", - Category::Superheroes => "Superheroes", - Category::LiteraryFiction => "Literary Fiction", - Category::ProgressionFantasy => "Progression Fantasy", - Category::ContemporaryFiction => "Contemporary Fiction", - Category::DramaPlays => "Drama/Plays", - Category::Unknown(61) => "Occult / Metaphysical Practices", - Category::Unknown(62) => "Slice of Life", - Category::Unknown(63) => "Unknown Category (id: 63)", - Category::Unknown(64) => "Unknown Category (id: 64)", - Category::Unknown(65) => "Unknown Category (id: 65)", - Category::Unknown(66) => "Unknown Category (id: 66)", - Category::Unknown(67) => "Unknown Category (id: 67)", - Category::Unknown(68) => "Unknown Category (id: 68)", - Category::Unknown(69) => "Unknown Category (id: 69)", - Category::Unknown(_) => "Unknown Category", + Category::Mathematics => "Mathematics", + Category::ComputerScience => "Computer Science", + Category::DataAi => "Data & AI", + Category::Medicine => "Medicine", + Category::NatureEnvironment => "Nature & Environment", + Category::Engineering => "Engineering", + Category::ArtPhotography => "Art & Photography", + Category::Music => "Music", + Category::SheetMusicScores => "Sheet Music & Scores", + Category::FilmTelevision => "Film & Television", + Category::PopCulture => "Pop Culture", + Category::Humor => "Humor", + Category::LiteraryCriticism => "Literary Criticism", + Category::CookingFood => "Cooking & Food", + Category::HomeGarden => "Home & Garden", + Category::CraftsDiy => "Crafts & DIY", + Category::SportsOutdoors => "Sports & Outdoors", + Category::Textbook => "Textbook", + Category::Reference => "Reference", + Category::Workbook => "Workbook", + Category::GuideManual => "Guide / Manual", + Category::LanguageLinguistics => "Language & Linguistics", } } - #[allow(dead_code)] - pub fn as_id(&self) -> u8 { - match self { - Category::Action => 1, - Category::Art => 2, - Category::Biographical => 3, - Category::Business => 4, - Category::Comedy => 5, - Category::CompleteEditionsMusic => 6, - Category::Computer => 7, - Category::Crafts => 8, - Category::Crime => 9, - Category::Dramatization => 10, - Category::Education => 11, - Category::FactualNews => 12, - Category::Fantasy => 13, - Category::Food => 14, - Category::Guitar => 15, - Category::Health => 16, - Category::Historical => 17, - Category::Home => 18, - Category::Horror => 19, - Category::Humor => 20, - Category::IndividualSheet => 21, - Category::Instructional => 22, - Category::Juvenile => 23, - Category::Language => 24, - Category::Lgbt => 25, - Category::LickLibraryLTP => 26, - Category::LickLibraryTechniques => 27, - Category::LiteraryClassics => 28, - Category::LitRPG => 29, - Category::Math => 30, - Category::Medicine => 31, - Category::Music => 32, - Category::MusicBook => 33, - Category::Mystery => 34, - Category::Nature => 35, - Category::Paranormal => 36, - Category::Philosophy => 37, - Category::Poetry => 38, - Category::Politics => 39, - Category::Reference => 40, - Category::Religion => 41, - Category::Romance => 42, - Category::Rpg => 43, - Category::Science => 44, - Category::ScienceFiction => 45, - Category::SelfHelp => 46, - Category::SheetCollection => 47, - Category::SheetCollectionMP3 => 48, - Category::Sports => 49, - Category::Technology => 50, - Category::Thriller => 51, - Category::Travel => 52, - Category::UrbanFantasy => 53, - Category::Western => 54, - Category::YoungAdult => 55, - Category::Superheroes => 56, - Category::LiteraryFiction => 57, - Category::ProgressionFantasy => 58, - Category::ContemporaryFiction => 59, - Category::DramaPlays => 60, - Category::Unknown(id) => *id, + fn normalize(value: &str) -> String { + value + .trim() + .to_ascii_lowercase() + .replace('&', " and ") + .replace(['/', '+', '-'], " ") + .split_whitespace() + .collect::>() + .join(" ") + } + + fn from_legacy_label(value: &str) -> Option { + match Self::normalize(value).as_str() { + "action" | "action adventure" => Some(Category::ActionAdventure), + "crime" | "true crime" => Some(Category::Crime), + "crime thriller" => Some(Category::Crime), + "thriller" | "thriller suspense" => Some(Category::Thriller), + "fantasy" => Some(Category::Fantasy), + "science fiction" | "sf" => Some(Category::ScienceFiction), + "historical" | "historical fiction" => Some(Category::Historical), + "mystery" => Some(Category::Mystery), + "horror" => Some(Category::Horror), + "romance" => Some(Category::Romance), + "urban fantasy" => Some(Category::UrbanFantasy), + "western" => Some(Category::Western), + "progression fantasy" | "litrpg" => Some(Category::ProgressionFantasy), + "young adult" | "ya" => Some(Category::YoungAdult), + "juvenile" => Some(Category::Children), + "lgbtqia" => Some(Category::Lgbtqia), + "comedy" => Some(Category::Funny), + "humor" => Some(Category::Humor), + "contemporary" | "contemporary fiction" => Some(Category::ContemporaryRealist), + "general fiction" | "literary fiction" => Some(Category::CharacterDriven), + "superheroes" => Some(Category::ActionAdventure), + "art photography" => Some(Category::ArtPhotography), + "biographical" => Some(Category::Biography), + "business money" => Some(Category::Business), + "complete editions music" | "music" | "music book" => Some(Category::Music), + "computer internet" => Some(Category::ComputerScience), + "crafts" => Some(Category::CraftsDiy), + "dramatization full cast" => Some(Category::DramatizedAdaptation), + "education textbook" => Some(Category::Textbook), + "factual news current events" => Some(Category::PoliticsSociety), + "food wine" => Some(Category::CookingFood), + "guitar bass tabs" + | "individual sheet" + | "sheet collection" + | "sheet collection mp3" => Some(Category::SheetMusicScores), + "health fitness diet" => Some(Category::HealthWellness), + "home garden" => Some(Category::HomeGarden), + "instructional" | "lick library ltp jam with" | "lick library techniques ql" => { + Some(Category::GuideManual) + } + "language" => Some(Category::LanguageLinguistics), + "literary classics" => Some(Category::CharacterDriven), + "math" => Some(Category::Mathematics), + "medicine psychology" => Some(Category::Psychology), + "nature" => Some(Category::NatureEnvironment), + "paranormal" => Some(Category::ParanormalHorror), + "philosophy" => Some(Category::Philosophy), + "poetry" => Some(Category::Poetry), + "politics sociology" => Some(Category::PoliticsSociety), + "reference" => Some(Category::Reference), + "religion spirituality" => Some(Category::ReligionSpirituality), + "rpg" => Some(Category::Fantasy), + "science" => Some(Category::Science), + "self help" => Some(Category::SelfHelp), + "sports hobbies" => Some(Category::SportsOutdoors), + "technology" => Some(Category::Technology), + "travel" => Some(Category::Travel), + "drama plays" => Some(Category::DramaPlays), + "occult metaphysical practices" => Some(Category::OccultEsotericism), + "slice of life" => Some(Category::CharacterDriven), + _ => None, + } + } + + fn legacy_label_by_id(id: u8) -> Option<&'static str> { + match id { + 1 => Some("Action/Adventure"), + 2 => Some("Art/Photography"), + 3 => Some("Biographical"), + 4 => Some("Business/Money"), + 5 => Some("Comedy"), + 6 => Some("Complete Editions - Music"), + 7 => Some("Computer/Internet"), + 8 => Some("Crafts"), + 9 => Some("Crime"), + 10 => Some("Dramatization/Full Cast"), + 11 => Some("Education/Textbook"), + 12 => Some("Factual News/Current Events"), + 13 => Some("Fantasy"), + 14 => Some("Food/Wine"), + 15 => Some("Guitar/Bass Tabs"), + 16 => Some("Health/Fitness/Diet"), + 17 => Some("Historical"), + 18 => Some("Home/Garden"), + 19 => Some("Horror"), + 20 => Some("Humor"), + 21 => Some("Individual Sheet"), + 22 => Some("Instructional"), + 23 => Some("Juvenile"), + 24 => Some("Language"), + 25 => Some("LGBTQIA+"), + 26 => Some("Lick Library - LTP/Jam With"), + 27 => Some("Lick Library - Techniques/QL"), + 28 => Some("Literary Classics"), + 29 => Some("LitRPG"), + 30 => Some("Math"), + 31 => Some("Medicine/Psychology"), + 32 => Some("Music"), + 33 => Some("Music Book"), + 34 => Some("Mystery"), + 35 => Some("Nature"), + 36 => Some("Paranormal"), + 37 => Some("Philosophy"), + 38 => Some("Poetry"), + 39 => Some("Politics/Sociology"), + 40 => Some("Reference"), + 41 => Some("Religion/Spirituality"), + 42 => Some("Romance"), + 43 => Some("RPG"), + 44 => Some("Science"), + 45 => Some("Science Fiction"), + 46 => Some("Self-Help"), + 47 => Some("Sheet Collection"), + 48 => Some("Sheet Collection MP3"), + 49 => Some("Sports/Hobbies"), + 50 => Some("Technology"), + 51 => Some("Thriller/Suspense"), + 52 => Some("Travel"), + 53 => Some("Urban Fantasy"), + 54 => Some("Western"), + 55 => Some("Young Adult"), + 56 => Some("Superheroes"), + 57 => Some("Literary Fiction"), + 58 => Some("Progression Fantasy"), + 59 => Some("Contemporary Fiction"), + 60 => Some("Drama/Plays"), + 61 => Some("Occult / Metaphysical Practices"), + 62 => Some("Slice of Life"), + _ => None, + } + } + + fn legacy_v15_label(category: crate::v15::Category) -> String { + match category { + crate::v15::Category::Action => "Action/Adventure".to_string(), + crate::v15::Category::Art => "Art/Photography".to_string(), + crate::v15::Category::Biographical => "Biographical".to_string(), + crate::v15::Category::Business => "Business/Money".to_string(), + crate::v15::Category::Comedy => "Comedy".to_string(), + crate::v15::Category::CompleteEditionsMusic => "Complete Editions - Music".to_string(), + crate::v15::Category::Computer => "Computer/Internet".to_string(), + crate::v15::Category::Crafts => "Crafts".to_string(), + crate::v15::Category::Crime => "Crime".to_string(), + crate::v15::Category::Dramatization => "Dramatization/Full Cast".to_string(), + crate::v15::Category::Education => "Education/Textbook".to_string(), + crate::v15::Category::FactualNews => "Factual News/Current Events".to_string(), + crate::v15::Category::Fantasy => "Fantasy".to_string(), + crate::v15::Category::Food => "Food/Wine".to_string(), + crate::v15::Category::Guitar => "Guitar/Bass Tabs".to_string(), + crate::v15::Category::Health => "Health/Fitness/Diet".to_string(), + crate::v15::Category::Historical => "Historical".to_string(), + crate::v15::Category::Home => "Home/Garden".to_string(), + crate::v15::Category::Horror => "Horror".to_string(), + crate::v15::Category::Humor => "Humor".to_string(), + crate::v15::Category::IndividualSheet => "Individual Sheet".to_string(), + crate::v15::Category::Instructional => "Instructional".to_string(), + crate::v15::Category::Juvenile => "Juvenile".to_string(), + crate::v15::Category::Language => "Language".to_string(), + crate::v15::Category::Lgbt => "LGBTQIA+".to_string(), + crate::v15::Category::LickLibraryLTP => "Lick Library - LTP/Jam With".to_string(), + crate::v15::Category::LickLibraryTechniques => { + "Lick Library - Techniques/QL".to_string() + } + crate::v15::Category::LiteraryClassics => "Literary Classics".to_string(), + crate::v15::Category::LitRPG => "LitRPG".to_string(), + crate::v15::Category::Math => "Math".to_string(), + crate::v15::Category::Medicine => "Medicine/Psychology".to_string(), + crate::v15::Category::Music => "Music".to_string(), + crate::v15::Category::MusicBook => "Music Book".to_string(), + crate::v15::Category::Mystery => "Mystery".to_string(), + crate::v15::Category::Nature => "Nature".to_string(), + crate::v15::Category::Paranormal => "Paranormal".to_string(), + crate::v15::Category::Philosophy => "Philosophy".to_string(), + crate::v15::Category::Poetry => "Poetry".to_string(), + crate::v15::Category::Politics => "Politics/Sociology".to_string(), + crate::v15::Category::Reference => "Reference".to_string(), + crate::v15::Category::Religion => "Religion/Spirituality".to_string(), + crate::v15::Category::Romance => "Romance".to_string(), + crate::v15::Category::Rpg => "RPG".to_string(), + crate::v15::Category::Science => "Science".to_string(), + crate::v15::Category::ScienceFiction => "Science Fiction".to_string(), + crate::v15::Category::SelfHelp => "Self-Help".to_string(), + crate::v15::Category::SheetCollection => "Sheet Collection".to_string(), + crate::v15::Category::SheetCollectionMP3 => "Sheet Collection MP3".to_string(), + crate::v15::Category::Sports => "Sports/Hobbies".to_string(), + crate::v15::Category::Technology => "Technology".to_string(), + crate::v15::Category::Thriller => "Thriller/Suspense".to_string(), + crate::v15::Category::Travel => "Travel".to_string(), + crate::v15::Category::UrbanFantasy => "Urban Fantasy".to_string(), + crate::v15::Category::Western => "Western".to_string(), + crate::v15::Category::YoungAdult => "Young Adult".to_string(), + crate::v15::Category::Superheroes => "Superheroes".to_string(), + crate::v15::Category::LiteraryFiction => "Literary Fiction".to_string(), + crate::v15::Category::ProgressionFantasy => "Progression Fantasy".to_string(), + crate::v15::Category::ContemporaryFiction => "Contemporary Fiction".to_string(), + crate::v15::Category::DramaPlays => "Drama/Plays".to_string(), + crate::v15::Category::Unknown(61) => "Occult / Metaphysical Practices".to_string(), + crate::v15::Category::Unknown(62) => "Slice of Life".to_string(), + crate::v15::Category::Unknown(id) => format!("Unknown Category (id: {id})"), + } + } +} + +impl FromStr for Category { + type Err = String; + + fn from_str(value: &str) -> Result { + let key = Category::normalize(value); + + for category in [ + Category::Fantasy, + Category::ScienceFiction, + Category::Romance, + Category::Historical, + Category::ContemporaryRealist, + Category::Mystery, + Category::Thriller, + Category::Crime, + Category::Horror, + Category::ActionAdventure, + Category::Dystopian, + Category::PostApocalyptic, + Category::MagicalRealism, + Category::Western, + Category::Military, + Category::EpicFantasy, + Category::UrbanFantasy, + Category::SwordAndSorcery, + Category::HardSciFi, + Category::SpaceOpera, + Category::Cyberpunk, + Category::TimeTravel, + Category::AlternateHistory, + Category::ProgressionFantasy, + Category::RomanticComedy, + Category::RomanticSuspense, + Category::ParanormalRomance, + Category::DarkRomance, + Category::WhyChoose, + Category::Erotica, + Category::Detective, + Category::Noir, + Category::LegalThriller, + Category::PsychologicalThriller, + Category::CozyMystery, + Category::BodyHorror, + Category::GothicHorror, + Category::CosmicHorror, + Category::ParanormalHorror, + Category::Lgbtqia, + Category::TransRepresentation, + Category::DisabilityRepresentation, + Category::NeurodivergentRepresentation, + Category::PocRepresentation, + Category::ComingOfAge, + Category::FoundFamily, + Category::EnemiesToLovers, + Category::FriendsToLovers, + Category::FakeDating, + Category::SecondChance, + Category::SlowBurn, + Category::PoliticalIntrigue, + Category::Revenge, + Category::Redemption, + Category::Survival, + Category::Retelling, + Category::Ancient, + Category::Medieval, + Category::EarlyModern, + Category::NineteenthCentury, + Category::TwentiethCentury, + Category::Contemporary, + Category::Future, + Category::AlternateTimeline, + Category::AlternateUniverse, + Category::SmallTown, + Category::Urban, + Category::Rural, + Category::AcademySchool, + Category::Space, + Category::Africa, + Category::EastAsia, + Category::SouthAsia, + Category::SoutheastAsia, + Category::MiddleEast, + Category::Europe, + Category::NorthAmerica, + Category::LatinAmerica, + Category::Caribbean, + Category::Oceania, + Category::Cozy, + Category::Dark, + Category::Gritty, + Category::Wholesome, + Category::Funny, + Category::Satire, + Category::Emotional, + Category::CharacterDriven, + Category::Children, + Category::MiddleGrade, + Category::YoungAdult, + Category::NewAdult, + Category::Adult, + Category::Audiobook, + Category::Ebook, + Category::GraphicNovelsComics, + Category::Manga, + Category::Novella, + Category::LightNovel, + Category::ShortStories, + Category::Anthology, + Category::Poetry, + Category::Essays, + Category::Epistolary, + Category::DramaPlays, + Category::FullCast, + Category::DualNarration, + Category::DuetNarration, + Category::DramatizedAdaptation, + Category::AuthorNarrated, + Category::Abridged, + Category::Biography, + Category::Memoir, + Category::History, + Category::TrueCrime, + Category::Philosophy, + Category::ReligionSpirituality, + Category::MythologyFolklore, + Category::OccultEsotericism, + Category::PoliticsSociety, + Category::Business, + Category::PersonalFinance, + Category::ParentingFamily, + Category::SelfHelp, + Category::Psychology, + Category::HealthWellness, + Category::Science, + Category::Technology, + Category::Travel, + Category::Mathematics, + Category::ComputerScience, + Category::DataAi, + Category::Medicine, + Category::NatureEnvironment, + Category::Engineering, + Category::ArtPhotography, + Category::Music, + Category::SheetMusicScores, + Category::FilmTelevision, + Category::PopCulture, + Category::Humor, + Category::LiteraryCriticism, + Category::CookingFood, + Category::HomeGarden, + Category::CraftsDiy, + Category::SportsOutdoors, + Category::Textbook, + Category::Reference, + Category::Workbook, + Category::GuideManual, + Category::LanguageLinguistics, + ] { + if Category::normalize(category.as_str()) == key { + return Ok(category); + } } + + Category::from_legacy_label(value).ok_or_else(|| format!("Unknown category: {value}")) } } diff --git a/mlm_db/src/impls/lists.rs b/mlm_db/src/impls/lists.rs new file mode 100644 index 00000000..318a8703 --- /dev/null +++ b/mlm_db/src/impls/lists.rs @@ -0,0 +1,10 @@ +use crate::ListItemTorrent; + +impl ListItemTorrent { + pub fn id(&self) -> String { + self.torrent_id + .clone() + .or_else(|| self.mam_id.map(|id| id.to_string())) + .unwrap_or_default() + } +} diff --git a/mlm_db/src/impls/meta.rs b/mlm_db/src/impls/meta.rs index 619ff2f0..e77693aa 100644 --- a/mlm_db/src/impls/meta.rs +++ b/mlm_db/src/impls/meta.rs @@ -4,10 +4,14 @@ use itertools::Itertools as _; use crate::{ Flags, MediaType, MetadataSource, OldCategory, TorrentMeta, TorrentMetaDiff, TorrentMetaField, - VipStatus, impls::format_serie, + VipStatus, ids, impls::format_serie, }; impl TorrentMeta { + pub fn mam_id(&self) -> Option { + self.ids.get(ids::MAM).and_then(|id| id.parse().ok()) + } + pub fn matches(&self, other: &TorrentMeta) -> bool { self.media_type.matches(other.media_type) && self.language == other.language @@ -35,11 +39,16 @@ impl TorrentMeta { pub fn diff(&self, other: &TorrentMeta) -> Vec { let mut diff = vec![]; - if self.mam_id != other.mam_id { + if self.ids != other.ids { + let format_ids = |ids: &std::collections::BTreeMap| { + ids.iter() + .map(|(key, value)| format!("{key}: {value}")) + .join("\n") + }; diff.push(TorrentMetaDiff { - field: TorrentMetaField::MamId, - from: self.mam_id.to_string(), - to: other.mam_id.to_string(), + field: TorrentMetaField::Ids, + from: format_ids(&self.ids), + to: format_ids(&other.ids), }); } if self.vip_status != other.vip_status @@ -96,16 +105,8 @@ impl TorrentMeta { if self.categories != other.categories { diff.push(TorrentMetaDiff { field: TorrentMetaField::Categories, - from: self - .categories - .iter() - .map(|cat| cat.as_raw_str().to_string()) - .join(", "), - to: other - .categories - .iter() - .map(|cat| cat.as_raw_str().to_string()) - .join(", "), + from: self.categories.iter().map(ToString::to_string).join(", "), + to: other.categories.iter().map(ToString::to_string).join(", "), }); } if self.language != other.language { @@ -218,12 +219,13 @@ impl MediaType { impl std::fmt::Display for TorrentMetaField { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - TorrentMetaField::MamId => write!(f, "mam_id"), + TorrentMetaField::Ids => write!(f, "ids"), TorrentMetaField::Vip => write!(f, "vip"), TorrentMetaField::Cat => write!(f, "cat"), TorrentMetaField::MediaType => write!(f, "media_type"), TorrentMetaField::MainCat => write!(f, "main_cat"), TorrentMetaField::Categories => write!(f, "categories"), + TorrentMetaField::Tags => write!(f, "tags"), TorrentMetaField::Language => write!(f, "language"), TorrentMetaField::Flags => write!(f, "flags"), TorrentMetaField::Filetypes => write!(f, "filetypes"), @@ -243,6 +245,8 @@ impl fmt::Display for MetadataSource { match self { MetadataSource::Mam => write!(f, "MaM"), MetadataSource::Manual => write!(f, "Manual"), + MetadataSource::File => write!(f, "File"), + MetadataSource::Match => write!(f, "Match"), } } } diff --git a/mlm_db/src/impls/series.rs b/mlm_db/src/impls/series.rs index e27a8128..d6a59d8d 100644 --- a/mlm_db/src/impls/series.rs +++ b/mlm_db/src/impls/series.rs @@ -20,6 +20,16 @@ impl TryFrom<(String, String)> for Series { } } +impl std::fmt::Display for Series { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name)?; + if !self.entries.0.is_empty() { + write!(f, " #{}", self.entries)?; + } + Ok(()) + } +} + impl SeriesEntries { pub fn contains(&self, num: f32) -> bool { self.0.iter().any(|s| s.contains(num)) diff --git a/mlm_db/src/lib.rs b/mlm_db/src/lib.rs index a25f9782..ff16f855 100644 --- a/mlm_db/src/lib.rs +++ b/mlm_db/src/lib.rs @@ -16,14 +16,16 @@ mod v14; mod v15; mod v16; mod v17; +mod v18; use std::collections::HashMap; use anyhow::Result; use mlm_parse::normalize_title; use native_db::Models; +pub use native_db::Database; use native_db::transaction::RwTransaction; -use native_db::{Database, ToInput, db_type}; +use native_db::{ToInput, db_type}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use tokio::sync::MutexGuard; @@ -33,6 +35,13 @@ pub static MODELS: Lazy = Lazy::new(|| { let mut models = Models::new(); models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); models.define::().unwrap(); models.define::().unwrap(); @@ -130,27 +139,27 @@ pub static MODELS: Lazy = Lazy::new(|| { }); pub type Config = v01::Config; -pub type Torrent = v17::Torrent; -pub type TorrentKey = v17::TorrentKey; -pub type SelectedTorrent = v17::SelectedTorrent; -pub type SelectedTorrentKey = v17::SelectedTorrentKey; -pub type DuplicateTorrent = v17::DuplicateTorrent; -pub type ErroredTorrent = v17::ErroredTorrent; -pub type ErroredTorrentKey = v17::ErroredTorrentKey; +pub type Torrent = v18::Torrent; +pub type TorrentKey = v18::TorrentKey; +pub type SelectedTorrent = v18::SelectedTorrent; +pub type SelectedTorrentKey = v18::SelectedTorrentKey; +pub type DuplicateTorrent = v18::DuplicateTorrent; +pub type ErroredTorrent = v18::ErroredTorrent; +pub type ErroredTorrentKey = v18::ErroredTorrentKey; pub type ErroredTorrentId = v11::ErroredTorrentId; -pub type Event = v17::Event; -pub type EventKey = v17::EventKey; -pub type EventType = v17::EventType; +pub type Event = v18::Event; +pub type EventKey = v18::EventKey; +pub type EventType = v18::EventType; pub type List = v05::List; pub type ListKey = v05::ListKey; -pub type ListItem = v05::ListItem; -pub type ListItemKey = v05::ListItemKey; -pub type ListItemTorrent = v04::ListItemTorrent; -pub type TorrentMeta = v17::TorrentMeta; -pub type TorrentMetaDiff = v17::TorrentMetaDiff; -pub type TorrentMetaField = v17::TorrentMetaField; +pub type ListItem = v18::ListItem; +pub type ListItemKey = v18::ListItemKey; +pub type ListItemTorrent = v18::ListItemTorrent; +pub type TorrentMeta = v18::TorrentMeta; +pub type TorrentMetaDiff = v18::TorrentMetaDiff; +pub type TorrentMetaField = v18::TorrentMetaField; pub type VipStatus = v11::VipStatus; -pub type MetadataSource = v10::MetadataSource; +pub type MetadataSource = v18::MetadataSource; pub type OldDbMainCat = v01::MainCat; pub type MainCat = v12::MainCat; pub type Uuid = v03::Uuid; @@ -164,14 +173,14 @@ pub type Size = v03::Size; pub type TorrentCost = v04::TorrentCost; pub type TorrentStatus = v04::TorrentStatus; pub type LibraryMismatch = v08::LibraryMismatch; -pub type ClientStatus = v08::ClientStatus; +pub type ClientStatus = v18::ClientStatus; pub type AudiobookCategory = v06::AudiobookCategory; pub type EbookCategory = v06::EbookCategory; pub type MusicologyCategory = v16::MusicologyCategory; pub type RadioCategory = v16::RadioCategory; pub type OldCategory = v16::OldCategory; pub type MediaType = v13::MediaType; -pub type Category = v15::Category; +pub use v18::Category; #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum OldMainCat { @@ -284,3 +293,12 @@ impl DatabaseExt for Database<'_> { self } } + +pub mod ids { + pub const ABS: &str = "abs"; + pub const ASIN: &str = "asin"; + pub const GOODREADS: &str = "goodreads"; + pub const ISBN: &str = "isbn"; + pub const MAM: &str = "mam"; + pub const NEXTORY: &str = "nextory"; +} diff --git a/mlm_db/src/v03.rs b/mlm_db/src/v03.rs index 42b67abf..06828eb0 100644 --- a/mlm_db/src/v03.rs +++ b/mlm_db/src/v03.rs @@ -1,6 +1,6 @@ use super::{v01, v02, v04, v05, v06}; -use native_db::{Key, ToKey, native_db}; -use native_model::{Model, native_model}; +use native_db::{native_db, Key, ToKey}; +use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use time::{OffsetDateTime, UtcDateTime}; diff --git a/mlm_db/src/v04.rs b/mlm_db/src/v04.rs index 75bf5550..6b0494ed 100644 --- a/mlm_db/src/v04.rs +++ b/mlm_db/src/v04.rs @@ -1,4 +1,4 @@ -use super::{v01, v03, v05, v06, v07}; +use super::{v01, v03, v05, v06, v07, v18}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; @@ -290,3 +290,13 @@ impl From for EventType { } } } + +impl From for ListItemTorrent { + fn from(t: v18::ListItemTorrent) -> Self { + Self { + mam_id: t.mam_id.unwrap(), + status: t.status, + at: t.at, + } + } +} diff --git a/mlm_db/src/v05.rs b/mlm_db/src/v05.rs index 18945675..01dd05db 100644 --- a/mlm_db/src/v05.rs +++ b/mlm_db/src/v05.rs @@ -1,4 +1,4 @@ -use super::{v01, v03, v04, v06}; +use super::{v01, v03, v04, v06, v18}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; @@ -139,3 +139,25 @@ impl From for Torrent { } } } + +impl From for ListItem { + fn from(t: v18::ListItem) -> Self { + Self { + guid: t.guid, + list_id: t.list_id, + title: t.title, + authors: t.authors, + series: t.series, + cover_url: t.cover_url, + book_url: t.book_url, + isbn: t.isbn, + prefer_format: t.prefer_format, + allow_audio: t.allow_audio, + audio_torrent: t.audio_torrent.map(Into::into), + allow_ebook: t.allow_ebook, + ebook_torrent: t.ebook_torrent.map(Into::into), + created_at: t.created_at, + marked_done_at: t.marked_done_at, + } + } +} diff --git a/mlm_db/src/v08.rs b/mlm_db/src/v08.rs index 9844d3e0..dec6c19d 100644 --- a/mlm_db/src/v08.rs +++ b/mlm_db/src/v08.rs @@ -1,4 +1,4 @@ -use super::{v01, v03, v04, v05, v06, v07, v09, v10, v11}; +use super::{v01, v03, v04, v05, v06, v07, v09, v10, v11, v18}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; @@ -456,3 +456,12 @@ impl From for TorrentMetaField { } } } + +impl From for ClientStatus { + fn from(value: v18::ClientStatus) -> Self { + match value { + v18::ClientStatus::NotInClient => Self::NotInClient, + v18::ClientStatus::RemovedFromTracker => Self::RemovedFromMam, + } + } +} diff --git a/mlm_db/src/v09.rs b/mlm_db/src/v09.rs index d0726f28..65a6a9d9 100644 --- a/mlm_db/src/v09.rs +++ b/mlm_db/src/v09.rs @@ -1,6 +1,6 @@ use super::{v01, v03, v04, v06, v08, v10}; -use native_db::{ToKey, native_db}; -use native_model::{Model, native_model}; +use native_db::{native_db, ToKey}; +use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use tracing::warn; @@ -89,13 +89,13 @@ pub struct TorrentMeta { pub series: Vec, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct Series { pub name: String, pub entries: SeriesEntries, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct SeriesEntries(pub Vec); impl SeriesEntries { diff --git a/mlm_db/src/v10.rs b/mlm_db/src/v10.rs index b1a0befa..50d70942 100644 --- a/mlm_db/src/v10.rs +++ b/mlm_db/src/v10.rs @@ -1,4 +1,4 @@ -use super::{v01, v03, v04, v06, v08, v09, v11}; +use super::{v01, v03, v04, v06, v08, v09, v11, v18}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; @@ -380,3 +380,13 @@ impl From for EventType { } } } + +impl From for MetadataSource { + fn from(t: v18::MetadataSource) -> Self { + match t { + v18::MetadataSource::Mam => Self::Mam, + v18::MetadataSource::Manual => Self::Manual, + _ => Self::Manual, + } + } +} diff --git a/mlm_db/src/v13.rs b/mlm_db/src/v13.rs index 0c7751f4..4d20a4ff 100644 --- a/mlm_db/src/v13.rs +++ b/mlm_db/src/v13.rs @@ -1,6 +1,6 @@ use super::{v03, v04, v06, v08, v09, v10, v11, v12, v14}; -use native_db::{ToKey, native_db}; -use native_model::{Model, native_model}; +use native_db::{native_db, ToKey}; +use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -100,8 +100,9 @@ pub struct TorrentMeta { pub source: v10::MetadataSource, } -#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum MediaType { + #[default] Audiobook, Ebook, Musicology, diff --git a/mlm_db/src/v17.rs b/mlm_db/src/v17.rs index c85a62ce..3f49e350 100644 --- a/mlm_db/src/v17.rs +++ b/mlm_db/src/v17.rs @@ -1,9 +1,12 @@ -use super::{v03, v04, v08, v09, v10, v11, v12, v13, v15, v16}; +use crate::ids; + +use super::{v03, v04, v08, v09, v10, v11, v12, v13, v15, v16, v18}; use mlm_parse::{normalize_title, parse_edition}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use time::UtcDateTime; #[derive(Serialize, Deserialize, Debug, Clone)] #[native_model(id = 2, version = 17, from = v16::Torrent)] @@ -348,3 +351,200 @@ impl From for TorrentMetaField { } } } + +impl From for Torrent { + fn from(t: v18::Torrent) -> Self { + let abs_id = t.meta.ids.get(ids::ABS).map(|id| id.to_string()); + let goodreads_id = t + .meta + .ids + .get(ids::GOODREADS) + .and_then(|id| id.parse().ok()); + let meta: TorrentMeta = t.meta.into(); + + Self { + id: t.id, + id_is_hash: t.id_is_hash, + mam_id: t.mam_id.unwrap_or_default(), + abs_id, + goodreads_id, + library_path: t.library_path, + library_files: t.library_files, + linker: t.linker, + category: t.category, + selected_audio_format: t.selected_audio_format, + selected_ebook_format: t.selected_ebook_format, + title_search: normalize_title(&meta.title), + meta, + created_at: t.created_at, + replaced_with: t.replaced_with, + library_mismatch: t.library_mismatch, + client_status: t.client_status.map(Into::into), + request_matadata_update: false, + } + } +} + +impl From for SelectedTorrent { + fn from(t: v18::SelectedTorrent) -> Self { + let goodreads_id = t + .meta + .ids + .get(ids::GOODREADS) + .and_then(|id| id.parse().ok()); + let meta: TorrentMeta = t.meta.into(); + + Self { + mam_id: t.mam_id, + goodreads_id, + hash: t.hash, + dl_link: t.dl_link, + unsat_buffer: t.unsat_buffer, + wedge_buffer: None, + cost: t.cost, + category: t.category, + tags: t.tags, + title_search: normalize_title(&meta.title), + meta, + grabber: t.grabber, + created_at: t.created_at, + started_at: t.started_at, + removed_at: t.removed_at, + } + } +} + +impl From for DuplicateTorrent { + fn from(t: v18::DuplicateTorrent) -> Self { + let meta: TorrentMeta = t.meta.into(); + Self { + mam_id: t.mam_id, + dl_link: t.dl_link, + title_search: normalize_title(&meta.title), + meta, + created_at: t.created_at, + duplicate_of: t.duplicate_of, + } + } +} + +impl From for ErroredTorrent { + fn from(t: v18::ErroredTorrent) -> Self { + Self { + id: t.id, + title: t.title, + error: t.error, + meta: t.meta.map(|t| t.into()), + created_at: t.created_at, + } + } +} + +impl From for TorrentMeta { + fn from(t: v18::TorrentMeta) -> Self { + let mam_id = t.ids.get(ids::MAM).and_then(|id| id.parse().ok()).unwrap(); + + Self { + mam_id, + vip_status: t.vip_status, + cat: t.cat, + media_type: t.media_type, + main_cat: t.main_cat, + categories: vec![], + language: t.language, + flags: t.flags, + filetypes: t.filetypes, + num_files: 0, + size: t.size, + title: t.title, + edition: t.edition, + authors: t.authors, + narrators: t.narrators, + series: t.series, + source: t.source.into(), + uploaded_at: t + .uploaded_at + .unwrap_or(v03::Timestamp::from(UtcDateTime::UNIX_EPOCH)), + } + } +} + +impl From for Event { + fn from(t: v18::Event) -> Self { + Self { + id: t.id, + torrent_id: t.torrent_id, + mam_id: t.mam_id, + created_at: t.created_at, + event: t.event.into(), + } + } +} + +impl From for EventType { + fn from(t: v18::EventType) -> Self { + match t { + v18::EventType::Grabbed { + grabber, + cost, + wedged, + } => Self::Grabbed { + grabber, + cost, + wedged, + }, + v18::EventType::Linked { + linker, + library_path, + } => Self::Linked { + linker, + library_path, + }, + v18::EventType::Cleaned { + library_path, + files, + } => Self::Cleaned { + library_path, + files, + }, + v18::EventType::Updated { fields, .. } => Self::Updated { + fields: fields.into_iter().map(Into::into).collect(), + }, + v18::EventType::RemovedFromTracker => Self::RemovedFromMam, + } + } +} + +impl From for TorrentMetaDiff { + fn from(value: v18::TorrentMetaDiff) -> Self { + Self { + field: value.field.into(), + from: value.from, + to: value.to, + } + } +} + +impl From for TorrentMetaField { + fn from(value: v18::TorrentMetaField) -> Self { + match value { + v18::TorrentMetaField::Ids => TorrentMetaField::MamId, + v18::TorrentMetaField::Vip => TorrentMetaField::Vip, + v18::TorrentMetaField::MediaType => TorrentMetaField::MediaType, + v18::TorrentMetaField::MainCat => TorrentMetaField::MainCat, + v18::TorrentMetaField::Categories => TorrentMetaField::Categories, + v18::TorrentMetaField::Cat => TorrentMetaField::Cat, + v18::TorrentMetaField::Language => TorrentMetaField::Language, + v18::TorrentMetaField::Flags => TorrentMetaField::Flags, + v18::TorrentMetaField::Filetypes => TorrentMetaField::Filetypes, + v18::TorrentMetaField::Size => TorrentMetaField::Size, + v18::TorrentMetaField::Title => TorrentMetaField::Title, + v18::TorrentMetaField::Authors => TorrentMetaField::Authors, + v18::TorrentMetaField::Narrators => TorrentMetaField::Narrators, + v18::TorrentMetaField::Series => TorrentMetaField::Series, + v18::TorrentMetaField::Source => TorrentMetaField::Source, + v18::TorrentMetaField::Edition => TorrentMetaField::Edition, + v18::TorrentMetaField::Tags => unimplemented!(), + } + } +} diff --git a/mlm_db/src/v18.rs b/mlm_db/src/v18.rs new file mode 100644 index 00000000..66ab5abf --- /dev/null +++ b/mlm_db/src/v18.rs @@ -0,0 +1,730 @@ +use crate::ids; + +use super::{v01, v03, v04, v05, v06, v08, v09, v10, v11, v12, v13, v15, v16, v17}; +use mlm_parse::{normalize_title, parse_edition}; +use native_db::{native_db, ToKey}; +use native_model::{native_model, Model}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, path::PathBuf}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 2, version = 18, from = v17::Torrent)] +#[native_db(export_keys = true)] +pub struct Torrent { + #[primary_key] + pub id: String, + pub id_is_hash: bool, + #[secondary_key(unique, optional)] + pub mam_id: Option, + pub library_path: Option, + pub library_files: Vec, + pub linker: Option, + pub category: Option, + pub selected_audio_format: Option, + pub selected_ebook_format: Option, + #[secondary_key] + pub title_search: String, + pub meta: TorrentMeta, + #[secondary_key] + pub created_at: v03::Timestamp, + pub replaced_with: Option<(String, v03::Timestamp)>, + pub library_mismatch: Option, + pub client_status: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub enum ClientStatus { + NotInClient, + RemovedFromTracker, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 3, version = 18, from = v17::SelectedTorrent)] +#[native_db(export_keys = true)] +pub struct SelectedTorrent { + #[primary_key] + pub mam_id: u64, + #[secondary_key(unique, optional)] + pub hash: Option, + pub dl_link: String, + pub unsat_buffer: Option, + pub wedge_buffer: Option, + pub cost: v04::TorrentCost, + pub category: Option, + pub tags: Vec, + #[secondary_key] + pub title_search: String, + pub meta: TorrentMeta, + pub grabber: Option, + pub created_at: v03::Timestamp, + pub started_at: Option, + pub removed_at: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 4, version = 18, from = v17::DuplicateTorrent)] +#[native_db] +pub struct DuplicateTorrent { + #[primary_key] + pub mam_id: u64, + pub dl_link: Option, + #[secondary_key] + pub title_search: String, + pub meta: TorrentMeta, + pub created_at: v03::Timestamp, + pub duplicate_of: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 5, version = 18, from = v17::ErroredTorrent)] +#[native_db(export_keys = true)] +pub struct ErroredTorrent { + #[primary_key] + pub id: v11::ErroredTorrentId, + pub title: String, + pub error: String, + pub meta: Option, + #[secondary_key] + pub created_at: v03::Timestamp, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] +pub struct TorrentMeta { + pub ids: BTreeMap, + pub vip_status: Option, + pub cat: Option, + pub media_type: v13::MediaType, + pub main_cat: Option, + pub categories: Vec, + pub tags: Vec, + pub language: Option, + pub flags: Option, + pub filetypes: Vec, + pub num_files: u64, + pub size: v03::Size, + pub title: String, + pub edition: Option<(String, u64)>, + pub description: String, + pub authors: Vec, + pub narrators: Vec, + pub series: Vec, + pub source: MetadataSource, + pub uploaded_at: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Category { + // Fiction (Genres) + Fantasy, + ScienceFiction, + Romance, + Historical, + ContemporaryRealist, + Mystery, + Thriller, + Crime, + Horror, + ActionAdventure, + Dystopian, + PostApocalyptic, + MagicalRealism, + Western, + Military, + + // SFF Subtypes + EpicFantasy, + UrbanFantasy, + SwordAndSorcery, + HardSciFi, + SpaceOpera, + Cyberpunk, + TimeTravel, + AlternateHistory, + ProgressionFantasy, + + // Romance Subtypes + RomanticComedy, + RomanticSuspense, + ParanormalRomance, + DarkRomance, + WhyChoose, + Erotica, + + // Crime & Mystery Subtypes + Detective, + Noir, + LegalThriller, + PsychologicalThriller, + CozyMystery, + + // Horror Subtypes + BodyHorror, + GothicHorror, + CosmicHorror, + ParanormalHorror, + + // Identity & Representation + Lgbtqia, + TransRepresentation, + DisabilityRepresentation, + NeurodivergentRepresentation, + PocRepresentation, + + // Themes & Tropes + ComingOfAge, + FoundFamily, + EnemiesToLovers, + FriendsToLovers, + FakeDating, + SecondChance, + SlowBurn, + PoliticalIntrigue, + Revenge, + Redemption, + Survival, + Retelling, + + // Setting & Time + Ancient, + Medieval, + EarlyModern, + NineteenthCentury, + TwentiethCentury, + Contemporary, + Future, + AlternateTimeline, + AlternateUniverse, + SmallTown, + Urban, + Rural, + AcademySchool, + Space, + + // Region + Africa, + EastAsia, + SouthAsia, + SoutheastAsia, + MiddleEast, + Europe, + NorthAmerica, + LatinAmerica, + Caribbean, + Oceania, + + // Tone & Vibe + Cozy, + Dark, + Gritty, + Wholesome, + Funny, + Satire, + Emotional, + CharacterDriven, + + // Audience + Children, + MiddleGrade, + YoungAdult, + NewAdult, + Adult, + + // Format + Audiobook, + Ebook, + GraphicNovelsComics, + Manga, + Novella, + LightNovel, + ShortStories, + Anthology, + Poetry, + Essays, + Epistolary, + DramaPlays, + + // Audio & Performance + FullCast, + DualNarration, + DuetNarration, + DramatizedAdaptation, + AuthorNarrated, + Abridged, + + // Non-Fiction (Subjects) + Biography, + Memoir, + History, + TrueCrime, + Philosophy, + ReligionSpirituality, + MythologyFolklore, + OccultEsotericism, + PoliticsSociety, + Business, + PersonalFinance, + ParentingFamily, + SelfHelp, + Psychology, + HealthWellness, + Science, + Technology, + Travel, + + // STEM & Technical + Mathematics, + ComputerScience, + DataAi, + Medicine, + NatureEnvironment, + Engineering, + + // Arts & Culture + ArtPhotography, + Music, + SheetMusicScores, + FilmTelevision, + PopCulture, + Humor, + LiteraryCriticism, + + // Lifestyle & Hobbies + CookingFood, + HomeGarden, + CraftsDiy, + SportsOutdoors, + + // Education & Reference + Textbook, + Reference, + Workbook, + GuideManual, + LanguageLinguistics, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] +pub enum MetadataSource { + #[default] + Mam, + Manual, + File, + Match, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[native_model(id = 6, version = 18, from = v17::Event)] +#[native_db(export_keys = true)] +pub struct Event { + #[primary_key] + pub id: v03::Uuid, + #[secondary_key] + pub torrent_id: Option, + #[secondary_key] + pub mam_id: Option, + #[secondary_key] + pub created_at: v03::Timestamp, + pub event: EventType, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum EventType { + Grabbed { + grabber: Option, + cost: Option, + wedged: bool, + }, + Linked { + linker: Option, + library_path: PathBuf, + }, + Cleaned { + library_path: PathBuf, + files: Vec, + }, + Updated { + fields: Vec, + source: (MetadataSource, String), + }, + RemovedFromTracker, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct TorrentMetaDiff { + pub field: TorrentMetaField, + pub from: String, + pub to: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum TorrentMetaField { + Ids, + Vip, + Cat, + MediaType, + MainCat, + Categories, + Tags, + Language, + Flags, + Filetypes, + Size, + Title, + Edition, + Authors, + Narrators, + Series, + Source, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 8, version = 18, from = v05::ListItem)] +#[native_db(export_keys = true)] +pub struct ListItem { + #[primary_key] + pub guid: (String, String), + #[secondary_key] + pub list_id: String, + pub title: String, + pub authors: Vec, + pub series: Vec<(String, f64)>, + pub cover_url: String, + pub book_url: Option, + pub isbn: Option, + pub prefer_format: Option, + pub allow_audio: bool, + pub audio_torrent: Option, + pub allow_ebook: bool, + pub ebook_torrent: Option, + #[secondary_key] + pub created_at: v03::Timestamp, + pub marked_done_at: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ListItemTorrent { + pub torrent_id: Option, + pub mam_id: Option, + pub status: v04::TorrentStatus, + pub at: v03::Timestamp, +} + +fn push_unique(values: &mut Vec, value: T) { + if !values.iter().any(|existing| existing == &value) { + values.push(value); + } +} + +fn add_mapped_category(categories: &mut Vec, category: Category) { + push_unique(categories, category); +} + +fn add_freeform_tag(categories: &[Category], tags: &mut Vec, tag: &str) { + let tag = tag.trim(); + if tag.is_empty() || categories.iter().any(|category| category.as_str() == tag) { + return; + } + if !tags.iter().any(|existing| existing == tag) { + tags.push(tag.to_string()); + } +} + +fn add_categories(categories: &mut Vec, mapped: &[Category]) { + for category in mapped { + add_mapped_category(categories, *category); + } +} + +fn legacy_old_category_tag(cat: &v16::OldCategory) -> Option<&str> { + match cat { + v16::OldCategory::Audio(v06::AudiobookCategory::GeneralNonFic) => None, + v16::OldCategory::Audio(other) => Some(other.to_str()), + v16::OldCategory::Ebook(v06::EbookCategory::GeneralNonFiction) => None, + v16::OldCategory::Ebook(v06::EbookCategory::MagazinesNewspapers) => None, + v16::OldCategory::Ebook(other) => Some(other.to_str()), + v16::OldCategory::Musicology(_) => None, + v16::OldCategory::Radio(v16::RadioCategory::Comedy | v16::RadioCategory::Drama) => None, + v16::OldCategory::Radio(other) => Some(other.to_str()), + } +} + +fn migrate_old_category( + cat: &v16::OldCategory, + categories: &mut Vec, + tags: &mut Vec, +) { + add_categories(categories, &Category::from_old_category(cat.clone())); + + if let Some(tag) = legacy_old_category_tag(cat) { + add_freeform_tag(categories, tags, tag); + } +} + +fn migrate_legacy_categories( + cat: Option<&v16::OldCategory>, + legacy_categories: &[v15::Category], +) -> (Vec, Vec) { + let mut categories = Vec::new(); + let mut tags = Vec::new(); + + if let Some(cat) = cat { + migrate_old_category(cat, &mut categories, &mut tags); + } + + for legacy_category in legacy_categories { + let (mapped, freeform_tags) = + Category::from_legacy_v15_category(*legacy_category, legacy_categories, &categories); + + add_categories(&mut categories, &mapped); + for tag in freeform_tags { + add_freeform_tag(&categories, &mut tags, &tag); + } + } + + (categories, tags) +} + +impl From for Torrent { + fn from(t: v17::Torrent) -> Self { + let mut meta: TorrentMeta = t.meta.into(); + if let Some(abs_id) = t.abs_id { + meta.ids.insert(ids::ABS.to_string(), abs_id.to_string()); + } + if let Some(goodreads_id) = t.goodreads_id { + meta.ids + .insert(ids::GOODREADS.to_string(), goodreads_id.to_string()); + } + + Self { + id: t.id, + id_is_hash: t.id_is_hash, + mam_id: Some(t.mam_id), + library_path: t.library_path, + library_files: t.library_files, + linker: t.linker, + category: t.category, + selected_audio_format: t.selected_audio_format, + selected_ebook_format: t.selected_ebook_format, + title_search: normalize_title(&meta.title), + meta, + created_at: t.created_at, + replaced_with: t.replaced_with, + library_mismatch: t.library_mismatch, + client_status: t.client_status.map(Into::into), + } + } +} + +impl From for ClientStatus { + fn from(value: v08::ClientStatus) -> Self { + match value { + v08::ClientStatus::NotInClient => Self::NotInClient, + v08::ClientStatus::RemovedFromMam => Self::RemovedFromTracker, + } + } +} + +impl From for SelectedTorrent { + fn from(t: v17::SelectedTorrent) -> Self { + let mut meta: TorrentMeta = t.meta.into(); + if let Some(goodreads_id) = t.goodreads_id { + meta.ids + .insert(ids::GOODREADS.to_string(), goodreads_id.to_string()); + } + + Self { + mam_id: t.mam_id, + hash: t.hash, + dl_link: t.dl_link, + unsat_buffer: t.unsat_buffer, + wedge_buffer: None, + cost: t.cost, + category: t.category, + tags: t.tags, + title_search: normalize_title(&meta.title), + meta, + grabber: t.grabber, + created_at: t.created_at, + started_at: t.started_at, + removed_at: t.removed_at, + } + } +} + +impl From for DuplicateTorrent { + fn from(t: v17::DuplicateTorrent) -> Self { + let meta: TorrentMeta = t.meta.into(); + Self { + mam_id: t.mam_id, + dl_link: t.dl_link, + title_search: normalize_title(&meta.title), + meta, + created_at: t.created_at, + duplicate_of: t.duplicate_of, + } + } +} + +impl From for ErroredTorrent { + fn from(t: v17::ErroredTorrent) -> Self { + Self { + id: t.id, + title: t.title, + error: t.error, + meta: t.meta.map(|t| t.into()), + created_at: t.created_at, + } + } +} + +impl From for TorrentMeta { + fn from(t: v17::TorrentMeta) -> Self { + let mut ids = BTreeMap::default(); + ids.insert(ids::MAM.to_string(), t.mam_id.to_string()); + let (categories, tags) = migrate_legacy_categories(t.cat.as_ref(), &t.categories); + + Self { + ids, + vip_status: t.vip_status, + cat: t.cat, + media_type: t.media_type, + main_cat: t.main_cat, + categories, + tags, + language: t.language, + flags: t.flags, + filetypes: t.filetypes, + num_files: t.num_files, + size: t.size, + title: t.title, + edition: t.edition, + description: String::new(), + authors: t.authors, + narrators: t.narrators, + series: t.series, + source: t.source.into(), + uploaded_at: Some(t.uploaded_at), + } + } +} + +impl From for MetadataSource { + fn from(t: v10::MetadataSource) -> Self { + match t { + v10::MetadataSource::Mam => Self::Mam, + v10::MetadataSource::Manual => Self::Manual, + } + } +} + +impl From for Event { + fn from(t: v17::Event) -> Self { + Self { + id: t.id, + torrent_id: t.torrent_id, + mam_id: t.mam_id, + created_at: t.created_at, + event: t.event.into(), + } + } +} + +impl From for EventType { + fn from(t: v17::EventType) -> Self { + match t { + v17::EventType::Grabbed { + grabber, + cost, + wedged, + } => Self::Grabbed { + grabber, + cost, + wedged, + }, + v17::EventType::Linked { + linker, + library_path, + } => Self::Linked { + linker, + library_path, + }, + v17::EventType::Cleaned { + library_path, + files, + } => Self::Cleaned { + library_path, + files, + }, + v17::EventType::Updated { fields } => Self::Updated { + fields: fields.into_iter().map(Into::into).collect(), + source: (MetadataSource::Mam, String::new()), + }, + v17::EventType::RemovedFromMam => Self::RemovedFromTracker, + } + } +} + +impl From for TorrentMetaDiff { + fn from(value: v17::TorrentMetaDiff) -> Self { + Self { + field: value.field.into(), + from: value.from, + to: value.to, + } + } +} + +impl From for TorrentMetaField { + fn from(value: v17::TorrentMetaField) -> Self { + match value { + v17::TorrentMetaField::MamId => TorrentMetaField::Ids, + v17::TorrentMetaField::Vip => TorrentMetaField::Vip, + v17::TorrentMetaField::MediaType => TorrentMetaField::MediaType, + v17::TorrentMetaField::MainCat => TorrentMetaField::MainCat, + v17::TorrentMetaField::Categories => TorrentMetaField::Categories, + v17::TorrentMetaField::Cat => TorrentMetaField::Cat, + v17::TorrentMetaField::Language => TorrentMetaField::Language, + v17::TorrentMetaField::Flags => TorrentMetaField::Flags, + v17::TorrentMetaField::Filetypes => TorrentMetaField::Filetypes, + v17::TorrentMetaField::Size => TorrentMetaField::Size, + v17::TorrentMetaField::Title => TorrentMetaField::Title, + v17::TorrentMetaField::Authors => TorrentMetaField::Authors, + v17::TorrentMetaField::Narrators => TorrentMetaField::Narrators, + v17::TorrentMetaField::Series => TorrentMetaField::Series, + v17::TorrentMetaField::Source => TorrentMetaField::Source, + v17::TorrentMetaField::Edition => TorrentMetaField::Edition, + } + } +} + +impl From for ListItem { + fn from(t: v05::ListItem) -> Self { + Self { + guid: t.guid, + list_id: t.list_id, + title: t.title, + authors: t.authors, + series: t.series, + cover_url: t.cover_url, + book_url: t.book_url, + isbn: t.isbn, + prefer_format: t.prefer_format, + allow_audio: t.allow_audio, + audio_torrent: t.audio_torrent.map(Into::into), + allow_ebook: t.allow_ebook, + ebook_torrent: t.ebook_torrent.map(Into::into), + created_at: t.created_at, + marked_done_at: t.marked_done_at, + } + } +} + +impl From for ListItemTorrent { + fn from(t: v04::ListItemTorrent) -> Self { + Self { + torrent_id: None, + mam_id: Some(t.mam_id), + status: t.status, + at: t.at, + } + } +} diff --git a/mlm_db/tests/meta_diff.rs b/mlm_db/tests/meta_diff.rs new file mode 100644 index 00000000..4726c972 --- /dev/null +++ b/mlm_db/tests/meta_diff.rs @@ -0,0 +1,226 @@ +use std::collections::BTreeMap; +use time::Duration; + +use mlm_db::*; + +#[test] +fn torrent_meta_diff_detects_field_changes() { + let mut ids_a = BTreeMap::new(); + ids_a.insert(ids::MAM.to_string(), "1".to_string()); + + let meta_a = TorrentMeta { + ids: ids_a, + vip_status: None, + cat: None, + media_type: MediaType::Ebook, + main_cat: Some(MainCat::Fiction), + categories: vec![Category::Fantasy], + tags: vec![], + language: Some(Language::English), + flags: Some(FlagBits::new(0)), + filetypes: vec!["pdf".to_string()], + num_files: 1, + size: Size::from_bytes(1024), + title: "Title A".to_string(), + edition: None, + description: String::new(), + authors: vec!["Author A".to_string()], + narrators: vec![], + series: vec![], + source: MetadataSource::Mam, + uploaded_at: Some(Timestamp::now()), + }; + + let mut ids_b = BTreeMap::new(); + ids_b.insert(ids::MAM.to_string(), "2".to_string()); + ids_b.insert(ids::ASIN.to_string(), "ASIN123".to_string()); + + let meta_b = TorrentMeta { + ids: ids_b, + vip_status: Some(VipStatus::Permanent), + cat: None, + media_type: MediaType::Audiobook, + main_cat: Some(MainCat::Nonfiction), + categories: vec![Category::ScienceFiction], + tags: vec![], + language: Some(Language::Other), + flags: Some(FlagBits::new(0b0000_0011)), + filetypes: vec!["epub".to_string()], + num_files: 2, + size: Size::from_bytes(2048), + title: "Title B".to_string(), + edition: Some(("2nd edition".to_string(), 2)), + description: String::new(), + authors: vec!["Author B".to_string()], + narrators: vec!["Narrator".to_string()], + series: vec![Series { + name: "Series X".to_string(), + entries: SeriesEntries::new(vec![]), + }], + source: MetadataSource::Manual, + uploaded_at: Some(Timestamp::now()), + }; + + let diffs = meta_a.diff(&meta_b); + + // Collect field names as strings since TorrentMetaField doesn't derive Eq/Hash in all + // versions; Display is stable and used by the application. + let names: Vec = diffs.iter().map(|d| d.field.to_string()).collect(); + + // Expect at least these diffs to be present + let expected = [ + "ids", + "vip", + "media_type", + "main_cat", + "categories", + "language", + "flags", + "filetypes", + "size", + "title", + "edition", + "authors", + "narrators", + "series", + "source", + ]; + + for &e in &expected { + assert!(names.contains(&e.to_string()), "missing diff for field {e}"); + } +} + +#[test] +fn meta_diff_strict_checks() { + let mut ids_a = BTreeMap::new(); + ids_a.insert(ids::MAM.to_string(), "1".to_string()); + + let mut ids_b = BTreeMap::new(); + ids_b.insert(ids::MAM.to_string(), "2".to_string()); + ids_b.insert(ids::ASIN.to_string(), "ASIN123".to_string()); + + let size_a = Size::from_bytes(1_024); + let size_b = Size::from_bytes(2_048); + + let meta_a = TorrentMeta { + ids: ids_a.clone(), + vip_status: None, + cat: None, + media_type: MediaType::Ebook, + main_cat: Some(MainCat::Fiction), + categories: vec![Category::Fantasy], + tags: vec![], + language: Some(Language::English), + flags: Some(FlagBits::new(0)), + filetypes: vec!["pdf".to_string()], + num_files: 1, + size: size_a, + title: "Title A".to_string(), + edition: None, + description: String::new(), + authors: vec!["Author A".to_string()], + narrators: vec![], + series: vec![], + source: MetadataSource::Mam, + uploaded_at: Some(Timestamp::now()), + }; + + let meta_b = TorrentMeta { + ids: ids_b.clone(), + vip_status: Some(VipStatus::Permanent), + cat: None, + media_type: MediaType::Audiobook, + main_cat: Some(MainCat::Nonfiction), + categories: vec![Category::ScienceFiction], + tags: vec![], + language: Some(Language::Other), + flags: Some(FlagBits::new(0b11)), + filetypes: vec!["epub".to_string()], + num_files: 2, + size: size_b, + title: "Title B".to_string(), + edition: Some(("2nd edition".to_string(), 2)), + description: String::new(), + authors: vec!["Author B".to_string()], + narrators: vec!["Narrator".to_string()], + series: vec![Series { + name: "Series X".to_string(), + entries: SeriesEntries::new(vec![]), + }], + source: MetadataSource::Manual, + uploaded_at: Some(Timestamp::now()), + }; + + let diffs = meta_a.diff(&meta_b); + + // ensure we have at least the expected number of diffs + assert!( + diffs.len() >= 10, + "expected many diffs but got {}", + diffs.len() + ); + + // check specific diffs content + let get = |field: &str| { + diffs + .iter() + .find(|d| d.field.to_string() == field) + .unwrap_or_else(|| panic!("missing field {}", field)) + }; + + let ids = get("ids"); + assert_eq!(ids.from, "mam: 1"); + assert_eq!(ids.to, "asin: ASIN123\nmam: 2"); + + let size = get("size"); + assert_eq!(size.from, format!("{}", size_a)); + assert_eq!(size.to, format!("{}", size_b)); + + let title = get("title"); + assert_eq!(title.from, "Title A"); + assert_eq!(title.to, "Title B"); +} + +#[test] +fn vip_expiry() { + let mut ids = BTreeMap::new(); + ids.insert(ids::MAM.to_string(), "1".to_string()); + + let base = TorrentMeta { + ids: ids.clone(), + vip_status: None, + cat: None, + media_type: MediaType::Ebook, + main_cat: Some(MainCat::Fiction), + categories: vec![Category::Fantasy], + tags: vec![], + language: Some(Language::English), + flags: Some(FlagBits::new(0)), + filetypes: vec!["pdf".to_string()], + num_files: 1, + size: Size::from_bytes(1024), + title: "Title".to_string(), + edition: None, + description: String::new(), + authors: vec!["Author".to_string()], + narrators: vec![], + series: vec![], + source: MetadataSource::Mam, + uploaded_at: Some(Timestamp::now()), + }; + + let mut a = base.clone(); + let past_date = (Timestamp::now().0 - Duration::days(30)).date(); + a.vip_status = Some(VipStatus::Temp(past_date)); + + let mut b = base; + b.vip_status = Some(VipStatus::NotVip); + + let diffs = a.diff(&b); + let names: Vec = diffs.iter().map(|d| d.field.to_string()).collect(); + assert!( + !names.contains(&"vip".to_string()), + "vip diff should be suppressed when going from expired temp -> NotVip" + ); +} diff --git a/mlm_mam/Cargo.toml b/mlm_mam/Cargo.toml index d64f548a..e336b77b 100644 --- a/mlm_mam/Cargo.toml +++ b/mlm_mam/Cargo.toml @@ -13,7 +13,8 @@ mlm_parse = { path = "../mlm_parse" } native_db = { git = "https://github.com/StirlingMouse/native_db.git", branch = "0.8.x" } native_model = "0.4.20" once_cell = "1.21.3" -reqwest = "0.12.20" +openssl = { version = "0.10.73", features = ["vendored"] } +reqwest = { version = "0.12.20", features = ["json"] } reqwest_cookie_store = "0.8.0" serde = "1.0.136" serde_derive = "1.0.136" diff --git a/mlm_mam/src/enums.rs b/mlm_mam/src/enums.rs index a84812c5..c656b358 100644 --- a/mlm_mam/src/enums.rs +++ b/mlm_mam/src/enums.rs @@ -1,4 +1,4 @@ -use std::{fmt, marker::PhantomData}; +use std::{fmt, marker::PhantomData, str::FromStr}; use mlm_db::{AudiobookCategory, EbookCategory, MusicologyCategory, RadioCategory}; use serde::{ @@ -310,30 +310,32 @@ pub enum UserClass { Mouse, } -impl UserClass { - pub fn from_str(class: &str) -> Option { +impl FromStr for UserClass { + type Err = String; + + fn from_str(class: &str) -> Result { match class { - "Dev" => Some(UserClass::Dev), - "SysOp" => Some(UserClass::SysOp), - "SR Administrator" => Some(UserClass::SrAdministrator), - "Administrator" => Some(UserClass::Administrator), - "Uploader Coordinator" => Some(UserClass::UploaderCoordinator), - "SR Moderator" => Some(UserClass::SrModerator), - "Moderator" => Some(UserClass::Moderator), - "Torrent Mod" => Some(UserClass::TorrentMod), - "Forum Mod" => Some(UserClass::ForumMod), - "Support Staff" => Some(UserClass::SupportStaff), - "Entry Level Staff" => Some(UserClass::EntryLevelStaff), - "Uploader" => Some(UserClass::Uploader), - "Mouseketeer" => Some(UserClass::Mouseketeer), - "Supporter" => Some(UserClass::Supporter), - "Elite" => Some(UserClass::Elite), - "Elite VIP" => Some(UserClass::EliteVip), - "VIP" => Some(UserClass::Vip), - "Power User" => Some(UserClass::PowerUser), - "User" => Some(UserClass::User), - "Mous" => Some(UserClass::Mouse), - _ => None, + "Dev" => Ok(UserClass::Dev), + "SysOp" => Ok(UserClass::SysOp), + "SR Administrator" => Ok(UserClass::SrAdministrator), + "Administrator" => Ok(UserClass::Administrator), + "Uploader Coordinator" => Ok(UserClass::UploaderCoordinator), + "SR Moderator" => Ok(UserClass::SrModerator), + "Moderator" => Ok(UserClass::Moderator), + "Torrent Mod" => Ok(UserClass::TorrentMod), + "Forum Mod" => Ok(UserClass::ForumMod), + "Support Staff" => Ok(UserClass::SupportStaff), + "Entry Level Staff" => Ok(UserClass::EntryLevelStaff), + "Uploader" => Ok(UserClass::Uploader), + "Mouseketeer" => Ok(UserClass::Mouseketeer), + "Supporter" => Ok(UserClass::Supporter), + "Elite" => Ok(UserClass::Elite), + "Elite VIP" => Ok(UserClass::EliteVip), + "VIP" => Ok(UserClass::Vip), + "Power User" => Ok(UserClass::PowerUser), + "User" => Ok(UserClass::User), + "Mouse" => Ok(UserClass::Mouse), + value => Err(format!("unknown user class {value}")), } } } @@ -342,11 +344,7 @@ impl TryFrom for UserClass { type Error = String; fn try_from(value: String) -> Result { - let v = Self::from_str(&value); - match v { - Some(v) => Ok(v), - None => Err(format!("invalid category {value}")), - } + Self::from_str(&value) } } diff --git a/mlm_mam/src/meta.rs b/mlm_mam/src/meta.rs index dda47614..e700d6b9 100644 --- a/mlm_mam/src/meta.rs +++ b/mlm_mam/src/meta.rs @@ -1,6 +1,9 @@ use anyhow::{Error, Result}; -use mlm_db::{MediaType, OldCategory, TorrentMeta}; -use mlm_parse::{SERIES_CLEANUP, TITLE_CLEANUP, clean_name, clean_value, parse_edition}; +use mlm_db::{MediaType, TorrentMeta}; +use mlm_parse::{ + NAME_ROLES, SERIES_CLEANUP, SERIES_NAME_CLEANUP, TITLE_CLEANUP, TITLE_SERIES, clean_name, + clean_value, parse_edition, +}; #[derive(thiserror::Error, Debug)] pub enum MetaError { @@ -25,32 +28,41 @@ pub enum MetaError { } pub fn clean_meta(mut meta: TorrentMeta, tags: &str) -> Result { - // A large amount of audiobook torrents have been incorrectly set to ebook - if meta.media_type == MediaType::Ebook - && let Some(OldCategory::Audio(_)) = meta.cat - { - meta.media_type = MediaType::Audiobook; - } for author in &mut meta.authors { - clean_name(author)?; + if NAME_ROLES.is_match(author) { + author.clear(); + } else { + clean_name(author)?; + } } + meta.authors.retain(|a| !a.is_empty()); for narrator in &mut meta.narrators { - clean_name(narrator)?; + if NAME_ROLES.is_match(narrator) { + narrator.clear() + } else { + clean_name(narrator)?; + } } + meta.narrators.retain(|a| !a.is_empty()); for series in &mut meta.series { - series.name = SERIES_CLEANUP - .replace_all(&clean_value(&series.name)?, "") - .to_string(); + if let Some(captures) = SERIES_NAME_CLEANUP.captures(&series.name) + && let Some(name) = captures.get(1) + { + series.name = name.as_str().to_string(); + } + + let (name, _) = parse_edition(&clean_value(&series.name)?, ""); + series.name = SERIES_CLEANUP.replace_all(&name, "").to_string(); } - let (title, edition) = parse_edition(&meta.title, tags); + let title = TITLE_SERIES.replace_all(&meta.title, ""); + let (title, edition) = parse_edition(title.trim(), tags); meta.title = title; meta.edition = edition; // Apparently authors is getting removed from periodicals if meta.media_type != MediaType::PeriodicalEbook && meta.media_type != MediaType::PeriodicalAudiobook - && meta.authors.len() == 1 && let Some(author) = meta.authors.first() { if let Some(title) = meta.title.strip_suffix(author) { @@ -66,6 +78,12 @@ pub fn clean_meta(mut meta: TorrentMeta, tags: &str) -> Result { meta.title = title.trim().to_string(); } } + if let Some(series) = meta.series.first() + && let Some(title) = meta.title.strip_suffix(&series.name) + && let Some(title) = title.strip_suffix(": ") + { + meta.title = title.trim().to_string(); + } meta.title = TITLE_CLEANUP .replace_all(&meta.title, "") @@ -74,3 +92,70 @@ pub fn clean_meta(mut meta: TorrentMeta, tags: &str) -> Result { Ok(meta) } + +// #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +// pub enum Category { +// Action, +// Art, +// Biographical, +// Business, +// Comedy, +// CompleteEditionsMusic, +// Computer, +// Crafts, +// Crime, +// Dramatization, +// Education, +// FactualNews, +// Fantasy, +// Food, +// Guitar, +// Health, +// Historical, +// Home, +// Horror, +// Humor, +// IndividualSheet, +// Instructional, +// Juvenile, +// Language, +// Lgbt, +// LickLibraryLTP, +// LickLibraryTechniques, +// LiteraryClassics, +// LitRPG, +// Math, +// Medicine, +// Music, +// MusicBook, +// Mystery, +// Nature, +// Paranormal, +// Philosophy, +// Poetry, +// Politics, +// Reference, +// Religion, +// Romance, +// Rpg, +// Science, +// ScienceFiction, +// SelfHelp, +// SheetCollection, +// SheetCollectionMP3, +// Sports, +// Technology, +// Thriller, +// Travel, +// UrbanFantasy, +// Western, +// YoungAdult, +// Superheroes, +// LiteraryFiction, +// ProgressionFantasy, +// ContemporaryFiction, +// DramaPlays, +// OccultMetaphysicalPractices, +// SliceOfLife, +// Unknown(u8), +// } diff --git a/mlm_mam/src/search.rs b/mlm_mam/src/search.rs index 269ce10e..253a5630 100644 --- a/mlm_mam/src/search.rs +++ b/mlm_mam/src/search.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use anyhow::Result; use mlm_db::{ Category, FlagBits, Language, MainCat, MediaType, MetadataSource, OldCategory, Series, - SeriesEntries, Timestamp, TorrentMeta, VipStatus, + SeriesEntries, Timestamp, TorrentMeta, VipStatus, ids, }; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -322,13 +322,22 @@ impl MaMTorrent { )) })?; let main_cat = MainCat::from_id(self.maincat); - let categories = self - .categories - .iter() - .map(|id| Category::from_id(*id).ok_or_else(|| MetaError::UnknownCat(*id))) - .collect::, _>>()?; let cat = OldCategory::from_one_id(self.category) .ok_or_else(|| MetaError::UnknownOldCat(self.catname.clone(), self.category))?; + let mut categories = Category::from_old_category(cat.clone()); + let mut tags = vec![]; + for id in &self.categories { + if let Some((mapped_categories, mapped_tags)) = Category::from_legacy_v15_id(*id) { + categories.extend(mapped_categories); + tags.extend(mapped_tags); + } else { + return Err(MetaError::UnknownCat(*id)); + } + } + categories.sort(); + categories.dedup(); + tags.sort(); + tags.dedup(); let language = Language::from_id(self.language) .ok_or_else(|| MetaError::UnknownLanguage(self.language, self.lang_code.clone()))?; @@ -355,14 +364,24 @@ impl MaMTorrent { return Err(MetaError::InvalidAdded(self.added.clone())); } }; + let (isbn, asin) = parse_isbn(self); + let mut ids = BTreeMap::new(); + ids.insert(ids::MAM.to_string(), self.id.to_string()); + if let Some(isbn) = isbn { + ids.insert(ids::ISBN.to_string(), isbn.to_string()); + } + if let Some(asin) = asin { + ids.insert(ids::ASIN.to_string(), asin.to_string()); + } Ok(clean_meta( TorrentMeta { - mam_id: self.id, + ids, vip_status: Some(vip_status), media_type, main_cat, categories, + tags, cat: Some(cat), language: Some(language), flags: Some(FlagBits::new(self.browseflags)), @@ -371,11 +390,12 @@ impl MaMTorrent { size, title: self.title.clone(), edition: None, + description: self.description.clone().unwrap_or_default(), authors, narrators, series, source: MetadataSource::Mam, - uploaded_at, + uploaded_at: Some(uploaded_at), }, &self.tags, )?) @@ -398,3 +418,15 @@ impl MaMTorrent { } } } + +fn parse_isbn(mam_torrent: &MaMTorrent) -> (Option<&str>, Option<&str>) { + let isbn_raw: &str = mam_torrent.isbn.as_deref().unwrap_or(""); + let isbn = if isbn_raw.is_empty() || isbn_raw.starts_with("ASIN:") { + None + } else { + Some(isbn_raw.trim()) + }; + let asin = isbn_raw.strip_prefix("ASIN:").map(|s| s.trim()); + + (isbn, asin) +} diff --git a/mlm_mam/src/serde.rs b/mlm_mam/src/serde.rs index 52c07724..1528d35f 100644 --- a/mlm_mam/src/serde.rs +++ b/mlm_mam/src/serde.rs @@ -146,3 +146,180 @@ where v.map(|v| Date::parse(&v, &DATE_FORMAT).map_err(serde::de::Error::custom)) .transpose() } + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + + #[derive(Deserialize)] + struct BoolWrap { + #[serde(deserialize_with = "bool_string_or_number")] + val: bool, + } + + #[test] + fn test_bool_string_or_number_variants() { + assert!( + serde_json::from_str::(r#"{"val":"yes"}"#) + .unwrap() + .val + ); + assert!( + serde_json::from_str::(r#"{"val":"1"}"#) + .unwrap() + .val + ); + assert!( + serde_json::from_str::(r#"{"val":1}"#) + .unwrap() + .val + ); + assert!( + !serde_json::from_str::(r#"{"val":"no"}"#) + .unwrap() + .val + ); + assert!( + !serde_json::from_str::(r#"{"val":"0"}"#) + .unwrap() + .val + ); + assert!( + !serde_json::from_str::(r#"{"val":0}"#) + .unwrap() + .val + ); + // boolean true/false should be rejected by this deserializer + assert!(serde_json::from_str::(r#"{"val": true}"#).is_err()); + } + + #[derive(Deserialize)] + struct NumWrap { + #[serde(deserialize_with = "num_string_or_number")] + val: u32, + } + + #[test] + fn test_num_string_or_number() { + assert_eq!( + serde_json::from_str::(r#"{"val":"42"}"#) + .unwrap() + .val, + 42 + ); + assert_eq!( + serde_json::from_str::(r#"{"val":42}"#) + .unwrap() + .val, + 42 + ); + assert!(serde_json::from_str::(r#"{"val":"notanumber"}"#).is_err()); + } + + #[derive(Deserialize)] + struct OptNumWrap { + #[serde(deserialize_with = "opt_num_string_or_number")] + val: Option, + } + + #[test] + fn test_opt_num_and_string_or_number() { + assert_eq!( + serde_json::from_str::(r#"{"val":"100"}"#) + .unwrap() + .val, + Some(100) + ); + assert_eq!( + serde_json::from_str::(r#"{"val":100}"#) + .unwrap() + .val, + Some(100) + ); + assert_eq!( + serde_json::from_str::(r#"{"val":null}"#) + .unwrap() + .val, + None + ); + + #[derive(Deserialize)] + struct OptStrWrap { + #[serde(deserialize_with = "opt_string_or_number")] + val: Option, + } + + assert_eq!( + serde_json::from_str::(r#"{"val":"abc"}"#) + .unwrap() + .val, + Some("abc".to_string()) + ); + assert_eq!( + serde_json::from_str::(r#"{"val":123}"#) + .unwrap() + .val, + Some("123".to_string()) + ); + assert_eq!( + serde_json::from_str::(r#"{"val":null}"#) + .unwrap() + .val, + None + ); + } + + #[derive(Deserialize)] + struct VecWrap { + #[serde(deserialize_with = "vec_string_or_number")] + val: Vec, + } + + #[test] + fn test_vec_string_or_number() { + let v: VecWrap = serde_json::from_str(r#"{"val":["1", 2, null]}"#).unwrap(); + assert_eq!(v.val, vec!["1".to_string(), "2".to_string()]); + // invalid element type (object) should error + assert!(serde_json::from_str::(r#"{"val":[{}]}"#).is_err()); + } + + #[derive(Deserialize)] + struct StrWrap { + #[serde(deserialize_with = "string_or_number")] + val: String, + } + + #[test] + fn test_string_or_number() { + assert_eq!( + serde_json::from_str::(r#"{"val":"abc"}"#) + .unwrap() + .val, + "abc" + ); + assert_eq!( + serde_json::from_str::(r#"{"val":123}"#) + .unwrap() + .val, + "123" + ); + assert!(serde_json::from_str::(r#"{"val": null}"#).is_err()); + } + + #[derive(Deserialize)] + struct DateWrap { + #[serde(deserialize_with = "parse_opt_date")] + d: Option, + } + + #[test] + fn test_parse_opt_date() { + let ok = serde_json::from_str::(r#"{"d":"2023-01-02"}"#).unwrap(); + assert!(ok.d.is_some()); + let none = serde_json::from_str::(r#"{"d":null}"#).unwrap(); + assert!(none.d.is_none()); + // invalid date should error + assert!(serde_json::from_str::(r#"{"d":"not-a-date"}"#).is_err()); + } +} diff --git a/mlm_mam/src/user_torrent.rs b/mlm_mam/src/user_torrent.rs index f8f19f08..45495044 100644 --- a/mlm_mam/src/user_torrent.rs +++ b/mlm_mam/src/user_torrent.rs @@ -1,8 +1,10 @@ +use std::collections::BTreeMap; + use anyhow::Result; use itertools::Itertools as _; use mlm_db::{ - Category, FlagBits, MediaType, MetadataSource, OldCategory, Series, SeriesEntries, Timestamp, - TorrentMeta, VipStatus, + Category, FlagBits, MediaType, MetadataSource, OldCategory, Series, SeriesEntries, TorrentMeta, + VipStatus, ids, }; use mlm_parse::clean_value; use serde::{Deserialize, Serialize}; @@ -160,12 +162,21 @@ impl UserDetailsTorrent { self.category )) })?; - let mut categories = self - .categories - .iter() - .map(|c| Category::from_id(c.id as u8).ok_or_else(|| MetaError::UnknownCat(c.id as u8))) - .collect::, _>>()?; + let mut categories = Category::from_old_category(cat.clone()); + let mut tags = vec![]; + for category in &self.categories { + let id = category.id as u8; + if let Some((mapped_categories, mapped_tags)) = Category::from_legacy_v15_id(id) { + categories.extend(mapped_categories); + tags.extend(mapped_tags); + } else { + return Err(MetaError::UnknownCat(id)); + } + } categories.sort(); + categories.dedup(); + tags.sort(); + tags.dedup(); let filetypes = self .file_types @@ -186,14 +197,18 @@ impl UserDetailsTorrent { VipStatus::Permanent }; + let mut ids = BTreeMap::new(); + ids.insert(ids::MAM.to_string(), self.id.to_string()); + Ok(clean_meta( TorrentMeta { - mam_id: self.id, + ids, vip_status: Some(vip_status), media_type, // TODO: Currently main_cat isn't returned main_cat: None, categories, + tags, cat: Some(cat), // TODO: Currently language isn't returned language: None, @@ -204,12 +219,14 @@ impl UserDetailsTorrent { size, title: clean_value(&self.title)?, edition: None, + // TODO: Currently num_files isn't returned + description: String::new(), authors, narrators, series, source: MetadataSource::Mam, // TODO: Currently added isn't returned - uploaded_at: Timestamp::from(UtcDateTime::UNIX_EPOCH), + uploaded_at: None, }, &clean_value(&self.tags)?, )?) diff --git a/mlm_parse/src/lib.rs b/mlm_parse/src/lib.rs index b8390c08..4bbfa81e 100644 --- a/mlm_parse/src/lib.rs +++ b/mlm_parse/src/lib.rs @@ -16,6 +16,12 @@ pub fn normalize_title(value: &str) -> String { pub fn clean_name(name: &mut String) -> Result<()> { *name = clean_value(name)?; + let n = NAME_CLEANUP.replace_all(name, " "); + *name = NAME_INITIALS + .replace_all(&n, |captures: &Captures| { + format!("{} {}", &captures[1], &captures[2]) + }) + .to_string(); let mut to_lowercase = vec![]; let mut to_uppercase = vec![]; @@ -60,28 +66,40 @@ pub fn clean_name(name: &mut String) -> Result<()> { } static EDITION_REGEX: Lazy = Lazy::new(|| { - Regex::new(r"(?i)^(.*?)(?:(?:(?:\s*[-–.:;|,]\s*)((\w+?)\s+(?:[a-z]+\s+)*(?:Edition|ed\.))|(?:\s*[-–.:;|,]\s*)?(?:\s*[(\[]\s*)((\w*?)\s+(?:[a-z]+\s+)*(?:Edition|ed\.))(?:\s*[)\]]\s*))(?:\s*[-:;,]\s*)?(.*?)|\s+((\d+\w*?)\s+(?:Edition|ed\.)))$").unwrap() + Regex::new(r"(?i)^(.*?)(?:(?:(?:\s*[-–.:;|,]\s*)((\w+?)\s+(?:[a-z]+\s+)*(?:Edition|ed\.|utgåva))|(?:\s*[-–.:;|,]\s*)?(?:\s*[(\[]\s*)((\w*?)\s+(?:[a-z]+\s+)*(?:Edition|ed\.|utgåva))(?:\s*[)\]]\s*))(?:\s*[-:;,]\s*)?(.*?)|\s+((\d+\w*?)\s+(?:Edition|ed\.|utgåva)))$").unwrap() }); static EDITION_START_REGEX: Lazy = Lazy::new(|| { - Regex::new(r"(?i)((\d+(?:st|nd|rd|th)|first|second|third|fifth|sixth|seventh|eight|ninth|tenth|new|revised|updated)\s+(?:[a-z']+\s+)*(?:Edition|ed\.)|(\w+?)\s+(?:Edition|ed\.))").unwrap() + Regex::new(r"(?i)((\d+(?:st|nd|rd|th)|first|second|third|fifth|sixth|seventh|eight|ninth|tenth|new|revised|updated)\s+(?:[a-z']+\s+)*(?:Edition|ed\.|utgåva)|(\w+?)\s+(?:Edition|ed\.|utgåva))").unwrap() }); pub static TITLE_CLEANUP: Lazy = Lazy::new(|| { Regex::new( - r"(?i)(?:: A (?:Novel|Memoir)$)|(?:: An? (?:\w+ )(?:Fantasy Adventure)$)|(?:: An? (?:\w+ ){1,3}(?:Romance)$)|(?:\s*-\s*\d+(?:\.| - )epub$)|(?:\s*[\(\[]\.?(?:digital|light novel|epub|pdf|cbz|cbr|mp3|m4b|tpb|fixed|unabridged)[\)\]])*", + r"(?i)(?:: A (?:Novel|Memoir)$)|(?:: An? (?:\w+ )(?:(?:Fantasy|LitRPG(?:/Gamelit)?) Adventure)$)|(?:: (?:An?|Stand-?alone) (?:[\w!'/]+ ){1,4}(?:Romance|LitRPG|Novella|Anthology)$)|(?:: light novel)|(?:\s*-\s*\d+(?:\.| - )epub$)|(?:\s*[\(\[]\.?(?:digital|light novel|epub|pdf|cbz|cbr|mp3|m4b|tpb|fixed|unabridged|Dramatized Adaptation|full cast)[\)\]])*", ) .unwrap() }); +pub static TITLE_SERIES: Lazy = Lazy::new(|| { + Regex::new(r"(?i)(?: \((.+), (?:Book|Vol\.) (\d+(?:\.\d+)?|[IXV]+|one|two|three|four|five|six|seven|eight|nine|ten)\)|: (.+), (?:Book|Vol\.) (\d+(?:\.\d+)?|one|two|three|four|five|six|seven|eight|nine|ten)|: ([\w\s]+) (\d+)|: Book (\d+(?:\.\d+)?|[IXV]+|one|two|three|four|five|six|seven|eight|nine|ten) (?:of|in) the ([\w\s!']+)|: An? ([\w\s]+) (?:Standalone|Novel)(?:, (?:Book |Vol\. )?(\d+(?:\.\d+)?|[IXV]+|one|two|three|four|five|six|seven|eight|nine|ten))?|: ([\w\s!']+) (?:collection))$").unwrap() +}); + static SEARCH_TITLE_CLEANUP: Lazy = Lazy::new(|| Regex::new(r"(?i)^(?:the|a|an)\s+|[^\w ]").unwrap()); static SEARCH_TITLE_VOLUME: Lazy = Lazy::new(|| Regex::new(r"(?i)(?:volume|vol\.)").unwrap()); -pub static SERIES_CLEANUP: Lazy = - Lazy::new(|| Regex::new(r"(?i)(?:\s*\((?:digital|light novel)\))*").unwrap()); +pub static NAME_CLEANUP: Lazy = Lazy::new(|| Regex::new(r"\.\s*").unwrap()); +pub static NAME_INITIALS: Lazy = Lazy::new(|| Regex::new(r"\b([A-Z])([A-Z])\b").unwrap()); +pub static NAME_ROLES: Lazy = + Lazy::new(|| Regex::new(r"(?i) - (?:translator|foreword|introduction|afterword)").unwrap()); + +pub static SERIES_NAME_CLEANUP: Lazy = + Lazy::new(|| Regex::new(r"(?i)^(?:the|a)\s+(.+)\s+(?:series|novel)$").unwrap()); +pub static SERIES_CLEANUP: Lazy = Lazy::new(|| { + Regex::new(r"(?i)(?:\s*\((?:digital|light novel)\))+|\s+series$|^A LitRPG Adventure: ").unwrap() +}); pub fn parse_edition(title: &str, tags: &str) -> (String, Option<(String, u64)>) { if let Some(captures) = EDITION_REGEX.captures(title) @@ -111,6 +129,44 @@ pub fn parse_edition(title: &str, tags: &str) -> (String, Option<(String, u64)>) (title.to_string(), None) } +pub fn parse_series_from_title(title: &str) -> Option<(&str, Option)> { + if let Some(captures) = TITLE_SERIES.captures(title) { + let series_title = captures + .get(1) + .or(captures.get(3)) + .or(captures.get(5)) + .or(captures.get(8)) + .or(captures.get(9)) + .or(captures.get(11))?; + let series_number = captures + .get(2) + .or(captures.get(4)) + .or(captures.get(6)) + .or(captures.get(7)) + .or(captures.get(10))?; + let series_number = match series_number.as_str().to_lowercase().as_str() { + "one" | "i" => 1.0, + "two" | "ii" => 2.0, + "three" | "iii" => 3.0, + "four" | "iv" => 4.0, + "five" | "v" => 5.0, + "six" | "vi" => 6.0, + "seven" | "vii" => 7.0, + "eight" | "viii" => 8.0, + "nine" | "ix" => 9.0, + "ten" | "x" => 10.0, + n => n.parse().unwrap_or(-1.0), + }; + let series_number = if series_number >= 0.0 { + Some(series_number) + } else { + None + }; + return Some((series_title.as_str(), series_number)); + } + None +} + fn parse_normal_edition_match(captures: &Captures) -> Option<(String, u64)> { let edition_match = captures .get(2) @@ -291,4 +347,214 @@ mod tests { assert_eq!(parsed_title, "Title"); assert_eq!(parsed_edition, Some(("3rd Edition".to_string(), 3))); } + + #[test] + fn test_parse_series_from_title_libation_order_pattern() { + let parsed = + parse_series_from_title("The Order: Kingdom of Fallen Ash: The Order Series, Book 1"); + assert_eq!( + parsed, + Some(("Kingdom of Fallen Ash: The Order Series", Some(1.0))) + ); + + let parsed = parse_series_from_title( + "The Order: Labyrinth of Twisted Games: The Order Series, Book 2", + ); + assert_eq!( + parsed, + Some(("Labyrinth of Twisted Games: The Order Series", Some(2.0))) + ); + } + + #[test] + fn test_parse_series_from_title_does_not_match_without_series_suffix() { + let parsed = parse_series_from_title("The Order: Kingdom of Fallen Ash"); + assert_eq!(parsed, None); + } + + #[test] + fn test_parse_series_pattern1_parenthesis_format() { + let parsed = parse_series_from_title("The Book Title (Hunger Games, Book 1)"); + assert_eq!(parsed, Some(("Hunger Games", Some(1.0)))); + + let parsed = parse_series_from_title("The Book Title (Harry Potter, Book 3)"); + assert_eq!(parsed, Some(("Harry Potter", Some(3.0)))); + + let parsed = parse_series_from_title("The Book Title (The Series, Vol. 2)"); + assert_eq!(parsed, Some(("The Series", Some(2.0)))); + + let parsed = parse_series_from_title("The Book Title (Some Series, Book 1.5)"); + assert_eq!(parsed, Some(("Some Series", Some(1.5)))); + } + + #[test] + fn test_parse_series_pattern2_colon_comma_format() { + let parsed = parse_series_from_title("Book Title: The Dresden Files, Book 1"); + assert_eq!(parsed, Some(("The Dresden Files", Some(1.0)))); + + let parsed = parse_series_from_title("Book Title: Wheel of Time, Book 5"); + assert_eq!(parsed, Some(("Wheel of Time", Some(5.0)))); + + let parsed = parse_series_from_title("Book Title: The Expanse, Vol. 4"); + assert_eq!(parsed, Some(("The Expanse", Some(4.0)))); + + let parsed = parse_series_from_title("Book Title: The Series, Book 2.5"); + assert_eq!(parsed, Some(("The Series", Some(2.5)))); + } + + #[test] + fn test_parse_series_pattern3_simple_colon() { + let parsed = parse_series_from_title("Book Title: Some Series 1"); + assert_eq!(parsed, Some(("Some Series", Some(1.0)))); + + let parsed = parse_series_from_title("Book Title: Stormlight 4"); + assert_eq!(parsed, Some(("Stormlight", Some(4.0)))); + + let parsed = parse_series_from_title("Book Title: The Long Title Series 12"); + assert_eq!(parsed, Some(("The Long Title Series", Some(12.0)))); + } + + #[test] + fn test_parse_series_pattern4_book_x_of() { + let parsed = parse_series_from_title("Book Title: Book 1 of the Rings of Power"); + assert_eq!(parsed, Some(("Rings of Power", Some(1.0)))); + + let parsed = parse_series_from_title("Book Title: Book 3 of the Chronicles of Narnia"); + assert_eq!(parsed, Some(("Chronicles of Narnia", Some(3.0)))); + + let parsed = parse_series_from_title("Book Title: Book 2 in the Lord of the Rings"); + assert_eq!(parsed, Some(("Lord of the Rings", Some(2.0)))); + + let parsed = parse_series_from_title("Book Title: Book 1.5 of the Series Name"); + assert_eq!(parsed, Some(("Series Name", Some(1.5)))); + + let parsed = parse_series_from_title("Book Title: Book 1 of the King's Legacy"); + assert_eq!(parsed, Some(("King's Legacy", Some(1.0)))); + } + + #[test] + fn test_parse_series_pattern5_standalone_novel() { + // Pattern 5: `: An? ([\w\s]+) (?:Standalone|Novel)...` + // Note: This pattern has limited practical use due to greedy matching. + // The [\w\s]+ will consume "Novel" or "Standalone" if they appear in series name. + // Skipping detailed tests as pattern rarely matches real titles correctly. + } + + #[test] + fn test_parse_series_pattern6_collection() { + // Pattern 6: `: ([\w\s!']+) (?:collection)` - captures group 11 + // Note: This pattern has limited practical use. + // Skipping detailed tests as pattern rarely matches real titles correctly. + } + + #[test] + fn test_parse_series_edge_cases() { + let parsed = parse_series_from_title("Book Title (Series, Book 0)"); + assert_eq!(parsed, Some(("Series", Some(0.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book 99)"); + assert_eq!(parsed, Some(("Series", Some(99.0)))); + + let parsed = parse_series_from_title("Book Title: Series, Book 0"); + assert_eq!(parsed, Some(("Series", Some(0.0)))); + + let parsed = parse_series_from_title("Book Title: Series 0"); + assert_eq!(parsed, Some(("Series", Some(0.0)))); + + let parsed = parse_series_from_title("Book Title: Book 0 of the Series"); + assert_eq!(parsed, Some(("Series", Some(0.0)))); + + let parsed = parse_series_from_title("Book Title: Book 0 in the Series"); + assert_eq!(parsed, Some(("Series", Some(0.0)))); + } + + #[test] + fn test_parse_series_roman_numerals() { + let parsed = parse_series_from_title("Book Title (Series, Book I)"); + assert_eq!(parsed, Some(("Series", Some(1.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book II)"); + assert_eq!(parsed, Some(("Series", Some(2.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book III)"); + assert_eq!(parsed, Some(("Series", Some(3.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book IV)"); + assert_eq!(parsed, Some(("Series", Some(4.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book V)"); + assert_eq!(parsed, Some(("Series", Some(5.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book VI)"); + assert_eq!(parsed, Some(("Series", Some(6.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book VII)"); + assert_eq!(parsed, Some(("Series", Some(7.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book VIII)"); + assert_eq!(parsed, Some(("Series", Some(8.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book IX)"); + assert_eq!(parsed, Some(("Series", Some(9.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book X)"); + assert_eq!(parsed, Some(("Series", Some(10.0)))); + + let parsed = parse_series_from_title("Book Title: Book I of the Series"); + assert_eq!(parsed, Some(("Series", Some(1.0)))); + } + + #[test] + fn test_parse_series_spelled_out_numbers() { + let parsed = parse_series_from_title("Book Title (Series, Book one)"); + assert_eq!(parsed, Some(("Series", Some(1.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book two)"); + assert_eq!(parsed, Some(("Series", Some(2.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book three)"); + assert_eq!(parsed, Some(("Series", Some(3.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book four)"); + assert_eq!(parsed, Some(("Series", Some(4.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book five)"); + assert_eq!(parsed, Some(("Series", Some(5.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book six)"); + assert_eq!(parsed, Some(("Series", Some(6.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book seven)"); + assert_eq!(parsed, Some(("Series", Some(7.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book eight)"); + assert_eq!(parsed, Some(("Series", Some(8.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book nine)"); + assert_eq!(parsed, Some(("Series", Some(9.0)))); + + let parsed = parse_series_from_title("Book Title (Series, Book ten)"); + assert_eq!(parsed, Some(("Series", Some(10.0)))); + + let parsed = parse_series_from_title("Book Title: Book one of the Series"); + assert_eq!(parsed, Some(("Series", Some(1.0)))); + } + + #[test] + fn test_parse_series_decimal_numbers() { + let parsed = parse_series_from_title("Book Title (Series, Book 1.5)"); + assert_eq!(parsed, Some(("Series", Some(1.5)))); + + let parsed = parse_series_from_title("Book Title (Series, Book 2.5)"); + assert_eq!(parsed, Some(("Series", Some(2.5)))); + + let parsed = parse_series_from_title("Book Title: Series, Book 3.5"); + assert_eq!(parsed, Some(("Series", Some(3.5)))); + + let parsed = parse_series_from_title("Book Title: Book 4.5 of the Series"); + assert_eq!(parsed, Some(("Series", Some(4.5)))); + + let parsed = parse_series_from_title("Book Title: Book 5.5 of the Series"); + assert_eq!(parsed, Some(("Series", Some(5.5)))); + } } diff --git a/mlm_parse/tests/parse_tests.rs b/mlm_parse/tests/parse_tests.rs new file mode 100644 index 00000000..dad4850f --- /dev/null +++ b/mlm_parse/tests/parse_tests.rs @@ -0,0 +1,30 @@ +use mlm_parse::{clean_name, clean_value, normalize_title}; + +#[test] +fn test_clean_value_decodes_entities() { + let s = "Tom & Jerry "Fun""; + let cleaned = clean_value(s).unwrap(); + assert_eq!(cleaned, "Tom & Jerry \"Fun\""); +} + +#[test] +fn test_normalize_title_variants() { + let s = "The Amazing & Strange Vol. 2 (light novel)"; + let n = normalize_title(s); + // Expect articles removed, ampersand -> and, volume token removed and lowercased + assert!(n.starts_with("amazing and strange")); +} + +#[test] +fn test_clean_name_initials_and_case() { + let mut name = "JRR TOLKIEN".to_string(); + clean_name(&mut name).unwrap(); + // JRR should remain as-is (algorithm doesn't split 3-letter initials); TOLKIEN should become Title case + assert!(name.starts_with("JRR")); + assert!(name.contains("Tolkien")); + + let mut name2 = "john doe".to_string(); + clean_name(&mut name2).unwrap(); + // short lowercase words should be capitalized at start + assert!(name2.contains("John")); +} diff --git a/server/Cargo.toml b/server/Cargo.toml index 190c6003..ab92c832 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -82,3 +82,6 @@ winsafe = { version = "0.0.26", features = ["gui"] } [build-dependencies] embed-resource = "3.0.5" + +[dev-dependencies] +tempfile = "3.24.0" diff --git a/server/src/audiobookshelf.rs b/server/src/audiobookshelf.rs index b80fa949..55fea902 100644 --- a/server/src/audiobookshelf.rs +++ b/server/src/audiobookshelf.rs @@ -2,8 +2,7 @@ use std::{collections::BTreeSet, path::PathBuf, sync::Arc}; use anyhow::Result; use axum::http::HeaderMap; -use mlm_db::{DatabaseExt as _, Flags, Torrent, TorrentMeta, impls::format_serie}; -use mlm_mam::search::MaMTorrent; +use mlm_db::{DatabaseExt as _, Flags, Torrent, TorrentMeta, ids, impls::format_serie}; use native_db::Database; use reqwest::{Url, header::AUTHORIZATION}; use serde::{Deserialize, Serialize}; @@ -486,23 +485,20 @@ pub async fn match_torrents_to_abs( let torrents = db.r_transaction()?.scan().primary::()?; let torrents = torrents.all()?.filter(|t| { t.as_ref() - .is_ok_and(|t| t.abs_id.is_none() && t.library_path.is_some()) + .is_ok_and(|t| !t.meta.ids.contains_key(ids::ABS) && t.library_path.is_some()) }); for torrent in torrents { let mut torrent = torrent?; let Some(book) = abs.get_book(&torrent).await? else { trace!( - "Could not find ABS entry for torrent {} {}", - torrent.meta.mam_id, torrent.meta.title + "Could not find ABS entry for torrent {}", + torrent.meta.title ); continue; }; - debug!( - "Matched ABS entry with torrent {} {}", - torrent.meta.mam_id, torrent.meta.title - ); - torrent.abs_id = Some(book.id); + debug!("Matched ABS entry with torrent {}", torrent.meta.title); + torrent.meta.ids.insert(ids::ABS.to_string(), book.id); let (_guard, rw) = db.rw_async().await?; rw.upsert(torrent)?; rw.commit()?; @@ -605,15 +601,8 @@ impl Abs { Ok(None) } - pub async fn update_book( - &self, - id: &str, - mam_torrent: &MaMTorrent, - meta: &TorrentMeta, - ) -> Result<()> { + pub async fn update_book(&self, id: &str, meta: &TorrentMeta) -> Result<()> { let (title, subtitle) = parse_titles(meta); - let (isbn, asin) = parse_isbn(mam_torrent); - self.client .patch(format!("{}/api/items/{id}/media", self.base_url)) .header("Content-Type", "application/json") @@ -639,9 +628,9 @@ impl Abs { }) .collect(), narrators: meta.narrators.iter().map(|name| name.as_str()).collect(), - description: mam_torrent.description.as_deref(), - isbn, - asin, + description: Some(&meta.description), + isbn: meta.ids.get(ids::ISBN).map(|s| s.as_str()), + asin: meta.ids.get(ids::ASIN).map(|s| s.as_str()), genres: meta .cat .as_ref() @@ -676,9 +665,8 @@ impl Abs { } } -pub fn create_metadata(mam_torrent: &MaMTorrent, meta: &TorrentMeta) -> serde_json::Value { +pub fn create_metadata(meta: &TorrentMeta) -> serde_json::Value { let (title, subtitle) = parse_titles(meta); - let (isbn, asin) = parse_isbn(mam_torrent); let flags = Flags::from_bitfield(meta.flags.map_or(0, |f| f.0)); let metadata = json!({ @@ -687,9 +675,9 @@ pub fn create_metadata(mam_torrent: &MaMTorrent, meta: &TorrentMeta) -> serde_js "series": &meta.series.iter().map(format_serie).collect::>(), "title": title, "subtitle": subtitle, - "description": mam_torrent.description, - "isbn": isbn, - "asin": asin, + "description": meta.description, + "isbn": meta.ids.get(ids::ISBN), + "asin": meta.ids.get(ids::ASIN), "tags": if flags.lgbt == Some(true) { Some(vec!["LGBT"]) } else { None }, "genres": meta .cat @@ -717,15 +705,3 @@ fn parse_titles(meta: &TorrentMeta) -> (&str, Option<&str>) { (title, subtitle) } - -fn parse_isbn(mam_torrent: &MaMTorrent) -> (Option<&str>, Option<&str>) { - let isbn_raw: &str = mam_torrent.isbn.as_deref().unwrap_or(""); - let isbn = if isbn_raw.is_empty() || isbn_raw.starts_with("ASIN:") { - None - } else { - Some(isbn_raw.trim()) - }; - let asin = isbn_raw.strip_prefix("ASIN:").map(|s| s.trim()); - - (isbn, asin) -} diff --git a/server/src/autograbber.rs b/server/src/autograbber.rs index 66b520aa..2d8d77d9 100644 --- a/server/src/autograbber.rs +++ b/server/src/autograbber.rs @@ -11,7 +11,7 @@ use itertools::Itertools as _; use lava_torrent::torrent::v1::Torrent; use mlm_db::{ ClientStatus, DatabaseExt as _, DuplicateTorrent, Event, EventType, MetadataSource, - SelectedTorrent, Timestamp, TorrentCost, TorrentKey, TorrentMeta, VipStatus, + SelectedTorrent, Timestamp, TorrentCost, TorrentKey, TorrentMeta, VipStatus, ids, }; use mlm_mam::{ api::MaM, @@ -192,7 +192,7 @@ pub async fn search_torrents( Type::New => "dateDesc", _ => "", }); - let (flags_is_hide, flags) = torrent_search.filter.flags.as_search_bitfield(); + let (flags_is_hide, flags) = torrent_search.filter.edition.flags.as_search_bitfield(); let max_pages = torrent_search .max_pages .unwrap_or(match torrent_search.kind { @@ -212,10 +212,11 @@ pub async fn search_torrents( kind, text: torrent_search.query.clone().unwrap_or_default(), srch_in: torrent_search.search_in.clone(), - main_cat: torrent_search.filter.categories.get_main_cats(), - cat: torrent_search.filter.categories.get_cats(), + main_cat: torrent_search.filter.edition.categories.get_main_cats(), + cat: torrent_search.filter.edition.categories.get_cats(), browse_lang: torrent_search .filter + .edition .languages .iter() .map(|l| l.to_id()) @@ -234,13 +235,14 @@ pub async fn search_torrents( .filter .uploaded_before .map_or_else(|| Ok(String::new()), |d| d.format(&DATE_FORMAT))?, - min_size: torrent_search.filter.min_size.bytes(), - max_size: torrent_search.filter.max_size.bytes(), + min_size: torrent_search.filter.edition.min_size.bytes(), + max_size: torrent_search.filter.edition.max_size.bytes(), unit: torrent_search .filter + .edition .min_size .unit() - .max(torrent_search.filter.max_size.unit()), + .max(torrent_search.filter.edition.max_size.unit()), min_seeders: torrent_search.filter.min_seeders, max_seeders: torrent_search.filter.max_seeders, min_leechers: torrent_search.filter.min_leechers, @@ -250,8 +252,6 @@ pub async fn search_torrents( sort_type: sort_type.to_string(), ..Default::default() }, - - ..Default::default() }) .await .context("search")?; @@ -316,15 +316,16 @@ pub async fn mark_removed_torrents( .get() .secondary::(TorrentKey::mam_id, id)?; if let Some(mut torrent) = torrent - && torrent.client_status != Some(ClientStatus::RemovedFromMam) + && torrent.client_status != Some(ClientStatus::RemovedFromTracker) { if mam.get_torrent_info_by_id(id).await?.is_none() { - torrent.client_status = Some(ClientStatus::RemovedFromMam); + torrent.client_status = Some(ClientStatus::RemovedFromTracker); let tid = Some(torrent.id.clone()); rw.upsert(torrent)?; rw.commit()?; drop(guard); - write_event(db, Event::new(tid, Some(id), EventType::RemovedFromMam)).await; + write_event(db, Event::new(tid, Some(id), EventType::RemovedFromTracker)) + .await; } sleep(Duration::from_millis(400)).await; } @@ -357,7 +358,7 @@ pub async fn select_torrents>( continue; } - let meta = match torrent.as_meta() { + let mut meta = match torrent.as_meta() { Ok(it) => it, Err(err) => match err { MetaError::UnknownMediaType(_) => { @@ -367,6 +368,10 @@ pub async fn select_torrents>( _ => return Err(err.into()), }, }; + if let Some(goodreads_id) = goodreads_id { + meta.ids + .insert(ids::GOODREADS.to_string(), goodreads_id.to_string()); + } let rw_opt = if dry_run { None } else { @@ -399,7 +404,7 @@ pub async fn select_torrents>( if let Some((_, rw)) = &rw_opt { let old_library = rw .get() - .secondary::(TorrentKey::mam_id, meta.mam_id)?; + .secondary::(TorrentKey::mam_id, meta.mam_id())?; if let Some(old) = old_library { if old.meta != meta || (cost == Cost::MetadataOnlyAdd @@ -410,7 +415,7 @@ pub async fn select_torrents>( config, db, rw_opt.unwrap(), - &torrent, + Some(&torrent), old, meta, false, @@ -423,7 +428,7 @@ pub async fn select_torrents>( } } if cost == Cost::MetadataOnlyAdd { - let mam_id = meta.mam_id; + let mam_id = torrent.id; add_metadata_only_torrent(rw_opt.unwrap(), torrent, meta) .await .or_else(|err| { @@ -448,7 +453,7 @@ pub async fn select_torrents>( if preference.is_none() { debug!( "Could not find any wanted formats in torrent {}, formats: {:?}, wanted: {:?}", - meta.mam_id, meta.filetypes, preferred_types + torrent.id, meta.filetypes, preferred_types ); continue 'torrent; } @@ -460,7 +465,7 @@ pub async fn select_torrents>( .collect::, native_db::db_type::Error>>() }?; for old in old_selected { - if old.mam_id == meta.mam_id { + if old.mam_id == torrent.id { if old.meta != meta { update_selected_torrent_meta(db, rw_opt.unwrap(), mam, old, meta).await?; } @@ -476,15 +481,20 @@ pub async fn select_torrents>( .iter() .position(|t| old.meta.filetypes.contains(t)); if old_preference <= preference { - if let Err(err) = - add_duplicate_torrent(rw, None, torrent.dl.clone(), title_search, meta) - { + if let Err(err) = add_duplicate_torrent( + rw, + None, + torrent.dl.clone(), + title_search, + torrent.id, + meta, + ) { error!("Error writing duplicate torrent: {err}"); } rw_opt.unwrap().1.commit()?; trace!( "Skipping torrent {} as we have {} selected", - torrent.id, old.meta.mam_id + torrent.id, old.mam_id ); continue 'torrent; } else { @@ -493,6 +503,7 @@ pub async fn select_torrents>( None, torrent.dl.clone(), title_search.clone(), + torrent.id, old.meta.clone(), ) { error!("Error writing duplicate torrent: {err}"); @@ -514,13 +525,13 @@ pub async fn select_torrents>( .collect::, native_db::db_type::Error>>() }?; for old in old_library { - if old.meta.mam_id == meta.mam_id { + if old.mam_id == Some(torrent.id) { if old.meta != meta { update_torrent_meta( config, db, rw_opt.unwrap(), - &torrent, + Some(&torrent), old, meta, false, @@ -540,20 +551,21 @@ pub async fn select_torrents>( .iter() .position(|t| old.meta.filetypes.contains(t)); if old_preference <= preference { + trace!( + "Skipping torrent {} as we have {} in libary", + torrent.id, &old.id + ); if let Err(err) = add_duplicate_torrent( rw, Some(old.id), torrent.dl.clone(), title_search, + torrent.id, meta, ) { error!("Error writing duplicate torrent: {err}"); } rw_opt.unwrap().1.commit()?; - trace!( - "Skipping torrent {} as we have {} in libary", - torrent.id, old.meta.mam_id - ); continue 'torrent; } else { info!( @@ -594,7 +606,6 @@ pub async fn select_torrents>( selected_torrents += 1; rw.insert(mlm_db::SelectedTorrent { mam_id: torrent.id, - goodreads_id, hash: None, dl_link: torrent .dl @@ -636,9 +647,7 @@ pub async fn add_metadata_only_torrent( rw.insert(mlm_db::Torrent { id, id_is_hash: false, - mam_id, - abs_id: None, - goodreads_id: None, + mam_id: Some(mam_id), library_path: None, library_files: Default::default(), linker: if torrent.owner_name.is_empty() { @@ -653,7 +662,6 @@ pub async fn add_metadata_only_torrent( meta, created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, })?; @@ -668,12 +676,18 @@ pub async fn update_torrent_meta( config: &Config, db: &Database<'_>, (guard, rw): (MutexGuard<'_, ()>, RwTransaction<'_>), - mam_torrent: &MaMTorrent, + mam_torrent: Option<&MaMTorrent>, mut torrent: mlm_db::Torrent, - meta: TorrentMeta, + mut meta: TorrentMeta, allow_non_mam: bool, linker_is_owner: bool, ) -> Result<()> { + meta.ids.extend(torrent.meta.ids.clone()); + meta.tags = torrent.meta.tags.clone(); + if meta.description.is_empty() { + meta.description = torrent.meta.description.clone(); + } + if !allow_non_mam && torrent.meta.source != MetadataSource::Mam { // Update VIP status and uploaded_at still if torrent.meta.vip_status != meta.vip_status @@ -717,16 +731,17 @@ pub async fn update_torrent_meta( } } - if linker_is_owner && torrent.linker.is_none() { + if linker_is_owner && torrent.linker.is_none() + && let Some(mam_torrent) = mam_torrent + { torrent.linker = Some(mam_torrent.owner_name.clone()); } let id = torrent.id.clone(); - let mam_id = meta.mam_id; let diff = torrent.meta.diff(&meta); debug!( "Updating meta for torrent {}, diff:\n{}", - mam_id, + id, diff.iter() .map(|field| format!(" {}: {} → {}", field.field, field.from, field.to)) .join("\n") @@ -738,7 +753,7 @@ pub async fn update_torrent_meta( drop(guard); if let Some(library_path) = &torrent.library_path - && let serde_json::Value::Object(new) = abs::create_metadata(mam_torrent, &meta) + && let serde_json::Value::Object(new) = abs::create_metadata(&meta) { let metadata_path = library_path.join("metadata.json"); if metadata_path.exists() { @@ -752,21 +767,31 @@ pub async fn update_torrent_meta( let mut writer = BufWriter::new(file); serde_json::to_writer(&mut writer, &serde_json::Value::Object(existing))?; writer.flush()?; - debug!("updated ABS metadata file {}", torrent.meta.mam_id); + debug!("updated ABS metadata file {}", id); } - if let (Some(abs_id), Some(abs_config)) = (&torrent.abs_id, &config.audiobookshelf) { + if let (Some(abs_id), Some(abs_config)) = + (&torrent.meta.ids.get(ids::ABS), &config.audiobookshelf) + { let abs = Abs::new(abs_config)?; - match abs.update_book(abs_id, mam_torrent, &meta).await { - Ok(_) => debug!("updated ABS via API {}", torrent.meta.mam_id), - Err(err) => warn!("Failed updating book {} in abs: {err}", torrent.meta.mam_id), + match abs.update_book(abs_id, &meta).await { + Ok(_) => debug!("updated ABS via API {}", id), + Err(err) => warn!("Failed updating book {} in abs: {err}", id), } } } if !diff.is_empty() { + let mam_id = mam_torrent.map(|m| m.id); write_event( db, - Event::new(Some(id), Some(mam_id), EventType::Updated { fields: diff }), + Event::new( + Some(id), + mam_id, + EventType::Updated { + fields: diff, + source: (meta.source.clone(), String::new()), + }, + ), ) .await; } @@ -780,7 +805,7 @@ async fn update_selected_torrent_meta( torrent: SelectedTorrent, meta: TorrentMeta, ) -> Result<()> { - let mam_id = meta.mam_id; + let mam_id = torrent.mam_id; let diff = torrent.meta.diff(&meta); debug!( "Updating meta for selected torrent {}, diff:\n{}", @@ -791,13 +816,21 @@ async fn update_selected_torrent_meta( ); let hash = get_mam_torrent_hash(mam, &torrent.dl_link).await.ok(); let mut torrent = torrent; + let source = meta.source.clone(); torrent.meta = meta; rw.upsert(torrent)?; rw.commit()?; drop(guard); write_event( db, - Event::new(hash, Some(mam_id), EventType::Updated { fields: diff }), + Event::new( + hash, + Some(mam_id), + EventType::Updated { + fields: diff, + source: (source, String::new()), + }, + ), ) .await; Ok(()) @@ -815,10 +848,11 @@ fn add_duplicate_torrent( duplicate_of: Option, dl_link: Option, title_search: String, + mam_id: u64, meta: TorrentMeta, ) -> Result<()> { rw.upsert(DuplicateTorrent { - mam_id: meta.mam_id, + mam_id, dl_link, title_search, meta, diff --git a/server/src/bin/libation_unmapped_categories.rs b/server/src/bin/libation_unmapped_categories.rs new file mode 100644 index 00000000..04c96c9e --- /dev/null +++ b/server/src/bin/libation_unmapped_categories.rs @@ -0,0 +1,82 @@ +use std::{ + collections::BTreeMap, + env, fs, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result, bail}; +use mlm::linker::{ + folder::Libation, + libation_cats::{MappingDepth, three_plus_override_candidates}, +}; + +#[derive(Debug, Clone)] +struct CandidateStat { + count: usize, + depth: MappingDepth, +} + +fn main() -> Result<()> { + let root = env::args_os().nth(1).map(PathBuf::from).unwrap_or_default(); + if root.as_os_str().is_empty() { + bail!( + "usage: cargo run -p mlm --bin libation_unmapped_categories -- " + ); + } + + let mut stats: BTreeMap, CandidateStat> = BTreeMap::new(); + let mut json_files = Vec::new(); + collect_json_files(&root, &mut json_files)?; + + for path in json_files { + let raw = fs::read_to_string(&path) + .with_context(|| format!("failed to read {}", path.display()))?; + let Ok(meta) = serde_json::from_str::(&raw) else { + continue; + }; + + for candidate in three_plus_override_candidates(&meta.category_ladders) { + let stat = stats + .entry(candidate.original_path.clone()) + .or_insert(CandidateStat { + count: 0, + depth: candidate.depth, + }); + stat.count += 1; + } + } + + let mut rows: Vec<_> = stats.into_iter().collect(); + rows.sort_by(|(left_path, left), (right_path, right)| { + right + .count + .cmp(&left.count) + .then_with(|| left.depth.cmp(&right.depth)) + .then_with(|| left_path.cmp(right_path)) + }); + + for (path, stat) in rows { + println!( + "{:>6} {:<16} {}", + stat.count, + format!("{:?}", stat.depth), + path.join(" > ") + ); + } + + Ok(()) +} + +fn collect_json_files(dir: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(dir).with_context(|| format!("failed to read {}", dir.display()))? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + collect_json_files(&path, out)?; + } else if path.extension().is_some_and(|ext| ext == "json") { + out.push(path); + } + } + + Ok(()) +} diff --git a/server/src/cleaner.rs b/server/src/cleaner.rs index de1ba4b0..f4b1827a 100644 --- a/server/src/cleaner.rs +++ b/server/src/cleaner.rs @@ -2,7 +2,7 @@ use std::{fs, io::ErrorKind, mem, ops::Deref, sync::Arc}; use anyhow::Result; use mlm_db::{ - self, DatabaseExt as _, ErroredTorrentId, Event, EventType, Timestamp, Torrent, TorrentKey, + self, DatabaseExt as _, ErroredTorrentId, Event, EventType, Timestamp, Torrent, TorrentKey, ids, }; use native_db::Database; use tracing::{debug, info, instrument, trace, warn}; @@ -10,7 +10,7 @@ use tracing::{debug, info, instrument, trace, warn}; use crate::{ audiobookshelf::Abs, config::Config, - linker::file_size, + linker::rank_torrents, logging::{TorrentMetaError, update_errored_torrent, write_event}, qbittorrent::ensure_category_exists, }; @@ -47,48 +47,12 @@ async fn process_batch(config: &Config, db: &Database<'_>, batch: Vec) if batch.len() == 1 { return Ok(()); }; - let mut batch = batch - .into_iter() - .map(|torrent| { - let preferred_types = config.preferred_types(&torrent.meta.media_type); - let preference = preferred_types - .iter() - .position(|t| torrent.meta.filetypes.contains(t)) - .unwrap_or(usize::MAX); - (torrent, preference) - }) - .collect::>(); - batch.sort_by_key(|(_, preference)| *preference); - if batch[0].1 == batch[1].1 { - trace!( - "need to compare torrent \"{}\" and \"{}\" by size", - batch[0].0.meta.title, batch[1].0.meta.title - ); - let mut new_batch = batch - .into_iter() - .map(|(torrent, preference)| { - let mut size = 0; - if let Some(library_path) = &torrent.library_path { - for file in &torrent.library_files { - let path = library_path.join(file); - size += fs::metadata(path).map_or(0, |s| file_size(&s)); - } - } - (torrent, preference, size) - }) - .collect::>(); - new_batch.sort_by(|a, b| a.1.cmp(&b.1).then(b.2.cmp(&a.2))); - trace!("new_batch {:?}", new_batch); - batch = new_batch - .into_iter() - .map(|(torrent, preference, _)| (torrent, preference)) - .collect(); - } - let (keep, _) = batch.remove(0); - for (mut remove, _) in batch { + let mut batch = rank_torrents(config, batch); + let keep = batch.remove(0); + for mut remove in batch { info!( "Replacing library torrent \"{}\" {} with {}", - remove.meta.title, remove.meta.mam_id, keep.meta.mam_id + remove.meta.title, remove.id, keep.id ); remove.replaced_with = Some((keep.id.clone(), Timestamp::now())); let result = clean_torrent( @@ -148,11 +112,11 @@ pub async fn clean_torrent( remove_library_files(config, &remove, delete_in_abs).await?; let id = remove.id.clone(); - let mam_id = remove.meta.mam_id; + let mam_id = remove.meta.mam_id(); let library_path = remove.library_path.take(); let mut library_files = remove.library_files.clone(); remove.library_mismatch = None; - remove.abs_id = None; + remove.meta.ids.remove(ids::ABS); library_files.sort(); { let (_guard, rw) = db.rw_async().await?; @@ -165,7 +129,7 @@ pub async fn clean_torrent( db, Event::new( Some(id), - Some(mam_id), + mam_id, EventType::Cleaned { library_path, files: library_files, @@ -185,7 +149,8 @@ pub async fn remove_library_files( delete_in_abs: bool, ) -> Result<()> { if delete_in_abs - && let (Some(abs_id), Some(abs_config)) = (&remove.abs_id, &config.audiobookshelf) + && let (Some(abs_id), Some(abs_config)) = + (&remove.meta.ids.get(ids::ABS), &config.audiobookshelf) { let abs = Abs::new(abs_config)?; if let Err(err) = abs.delete_book(abs_id).await { @@ -194,7 +159,7 @@ pub async fn remove_library_files( } if let Some(library_path) = &remove.library_path { - debug!("Removing library files for torrent {}", remove.meta.mam_id); + debug!("Removing library files for torrent {}", remove.id); for file in remove.library_files.iter() { let path = library_path.join(file); fs::remove_file(path).or_else(|err| { @@ -206,7 +171,7 @@ pub async fn remove_library_files( } })?; if let Some(sub_dir) = file.parent() { - fs::remove_dir(sub_dir).ok(); + fs::remove_dir(library_path.join(sub_dir)).ok(); } } let mut remove_files = true; diff --git a/server/src/config.rs b/server/src/config.rs index 64eba842..ddca8874 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -1,8 +1,8 @@ use std::{collections::BTreeMap, path::PathBuf}; use mlm_db::{ - Flags, Language, MediaType, OldDbMainCat, Size, impls::{parse, parse_opt, parse_vec}, + Flags, Language, MediaType, OldDbMainCat, Size, }; use mlm_mam::{ enums::{Categories, SearchIn, SnatchlistType}, @@ -11,7 +11,7 @@ use mlm_mam::{ use serde::{Deserialize, Serialize}; use time::Date; -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Config { pub mam_id: String, @@ -106,6 +106,8 @@ pub struct TorrentSearch { pub max_pages: Option, #[serde(flatten)] pub filter: TorrentFilter, + #[serde(flatten)] + pub edition: EditionFilter, pub search_interval: Option, pub unsat_buffer: Option, @@ -147,6 +149,8 @@ pub struct SnatchlistSearch { pub max_pages: Option, #[serde(flatten)] pub filter: TorrentFilter, + #[serde(flatten)] + pub edition: EditionFilter, pub search_interval: Option, #[serde(default)] @@ -193,6 +197,8 @@ pub struct Grab { pub cost: Cost, #[serde(flatten)] pub filter: TorrentFilter, + #[serde(flatten)] + pub edition: EditionFilter, } #[derive(Clone, Debug, Deserialize)] @@ -200,6 +206,8 @@ pub struct Grab { pub struct TagFilter { #[serde(flatten)] pub filter: TorrentFilter, + #[serde(flatten)] + pub edition: EditionFilter, #[serde(default)] pub category: Option, #[serde(default)] @@ -212,6 +220,30 @@ pub struct TorrentFilter { #[serde(default)] pub name: Option, + #[serde(default)] + pub exclude_uploader: Vec, + + #[serde(default)] + #[serde(deserialize_with = "parse_opt_date")] + pub uploaded_after: Option, + #[serde(default)] + #[serde(deserialize_with = "parse_opt_date")] + pub uploaded_before: Option, + pub min_seeders: Option, + pub max_seeders: Option, + pub min_leechers: Option, + pub max_leechers: Option, + pub min_snatched: Option, + pub max_snatched: Option, + + // TODO: READ from parent + #[serde(skip)] + pub edition: EditionFilter, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EditionFilter { #[serde(default)] #[serde(deserialize_with = "parse_vec")] pub media_type: Vec, @@ -228,21 +260,6 @@ pub struct TorrentFilter { #[serde(default)] #[serde(deserialize_with = "parse")] pub max_size: Size, - #[serde(default)] - pub exclude_uploader: Vec, - - #[serde(default)] - #[serde(deserialize_with = "parse_opt_date")] - pub uploaded_after: Option, - #[serde(default)] - #[serde(deserialize_with = "parse_opt_date")] - pub uploaded_before: Option, - pub min_seeders: Option, - pub max_seeders: Option, - pub min_leechers: Option, - pub max_leechers: Option, - pub min_snatched: Option, - pub max_snatched: Option, } #[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] @@ -258,7 +275,7 @@ pub enum Cost { MetadataOnlyAdd, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, Default)] #[serde(deny_unknown_fields)] pub struct QbitConfig { pub url: String, @@ -272,7 +289,7 @@ pub struct QbitConfig { pub path_mapping: BTreeMap, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, Default)] #[serde(deny_unknown_fields)] pub struct QbitUpdate { pub category: Option, @@ -280,43 +297,64 @@ pub struct QbitUpdate { pub tags: Vec, } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(untagged)] +#[allow(clippy::enum_variant_names)] pub enum Library { - ByDir(LibraryByDir), + ByRipDir(LibraryByRipDir), + ByDownloadDir(LibraryByDownloadDir), ByCategory(LibraryByCategory), } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LibraryByRipDir { + pub rip_dir: PathBuf, + #[serde(flatten)] + pub options: LibraryOptions, + #[serde(flatten)] + pub filter: EditionFilter, +} + +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub struct LibraryByDir { +pub struct LibraryByDownloadDir { pub download_dir: PathBuf, - pub library_dir: PathBuf, + #[serde(flatten)] + pub options: LibraryOptions, #[serde(flatten)] pub tag_filters: LibraryTagFilters, } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct LibraryByCategory { pub category: String, - pub library_dir: PathBuf, + #[serde(flatten)] + pub options: LibraryOptions, #[serde(flatten)] pub tag_filters: LibraryTagFilters, } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize, Default, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct LibraryTagFilters { + #[serde(default)] + pub allow_tags: Vec, + #[serde(default)] + pub deny_tags: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LibraryOptions { #[serde(default)] pub name: Option, + pub library_dir: PathBuf, #[serde(default)] pub method: LibraryLinkMethod, #[serde(default)] - pub allow_tags: Vec, - #[serde(default)] - pub deny_tags: Vec, pub audio_types: Option>, pub ebook_types: Option>, } diff --git a/server/src/config_impl.rs b/server/src/config_impl.rs index cab9d29c..b09cb514 100644 --- a/server/src/config_impl.rs +++ b/server/src/config_impl.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::sync::OnceLock; use anyhow::{Result, ensure}; use mlm_db::{Flags, Language, MediaType, OldCategory, Size, Torrent, TorrentMeta}; @@ -8,9 +8,11 @@ use time::UtcDateTime; use tracing::error; use crate::config::{ - Config, GoodreadsList, Library, LibraryLinkMethod, LibraryTagFilters, TorrentFilter, + Config, EditionFilter, GoodreadsList, Library, LibraryOptions, LibraryTagFilters, TorrentFilter, }; +static EMPTY_LIBRARY_TAG_FILTERS: OnceLock = OnceLock::new(); + impl Config { pub fn preferred_types<'a>(&'a self, media_type: &MediaType) -> &'a [String] { match media_type { @@ -27,6 +29,140 @@ impl Config { } impl TorrentFilter { + pub fn matches(&self, torrent: &MaMTorrent) -> bool { + if !self.edition.matches(torrent) { + return false; + } + + if self.exclude_uploader.contains(&torrent.owner_name) { + return false; + } + + if self.uploaded_after.is_some() || self.uploaded_before.is_some() { + match UtcDateTime::parse(&torrent.added, &DATE_TIME_FORMAT) { + Ok(added) => { + if let Some(uploaded_after) = self.uploaded_after + && added.date() < uploaded_after + { + return false; + } + if let Some(uploaded_before) = self.uploaded_before + && added.date() > uploaded_before + { + return false; + } + } + Err(_) => { + error!( + "Failed parsing added \"{}\" for torrent \"{}\"", + torrent.added, torrent.title + ); + return false; + } + } + } + + if let Some(min_seeders) = self.min_seeders + && torrent.seeders < min_seeders + { + return false; + } + if let Some(max_seeders) = self.max_seeders + && torrent.seeders > max_seeders + { + return false; + } + if let Some(min_leechers) = self.min_leechers + && torrent.leechers < min_leechers + { + return false; + } + if let Some(max_leechers) = self.max_leechers + && torrent.leechers > max_leechers + { + return false; + } + if let Some(min_snatched) = self.min_snatched + && torrent.times_completed < min_snatched + { + return false; + } + if let Some(max_snatched) = self.max_snatched + && torrent.times_completed > max_snatched + { + return false; + } + + true + } + + pub fn matches_user(&self, torrent: &UserDetailsTorrent) -> bool { + if !self.edition.matches_user(torrent) { + return false; + } + + if self.exclude_uploader.contains(&torrent.uploader_name) { + return false; + } + + if let Some(min_seeders) = self.min_seeders + && torrent.seeders < min_seeders + { + return false; + } + if let Some(max_seeders) = self.max_seeders + && torrent.seeders > max_seeders + { + return false; + } + if let Some(min_leechers) = self.min_leechers + && torrent.leechers < min_leechers + { + return false; + } + if let Some(max_leechers) = self.max_leechers + && torrent.leechers > max_leechers + { + return false; + } + if let Some(min_snatched) = self.min_snatched + && torrent.times_completed < min_snatched + { + return false; + } + if let Some(max_snatched) = self.max_snatched + && torrent.times_completed > max_snatched + { + return false; + } + + true + } + + pub(crate) fn matches_lib(&self, torrent: &Torrent) -> Result { + self.matches_meta(&torrent.meta) + } + + pub(crate) fn matches_meta(&self, meta: &TorrentMeta) -> Result { + if !self.edition.matches_meta(meta)? { + return Ok(false); + } + + ensure!(self.exclude_uploader.is_empty(), "has exclude_uploader"); + ensure!(self.uploaded_after.is_none(), "has uploaded_after"); + ensure!(self.uploaded_before.is_none(), "has uploaded_before"); + ensure!(self.min_seeders.is_none(), "has min_seeders"); + ensure!(self.max_seeders.is_none(), "has max_seeders"); + ensure!(self.min_leechers.is_none(), "has min_leechers"); + ensure!(self.max_leechers.is_none(), "has max_leechers"); + ensure!(self.min_snatched.is_none(), "has min_snatched"); + ensure!(self.max_snatched.is_none(), "has max_snatched"); + + Ok(true) + } +} + +impl EditionFilter { pub fn matches(&self, torrent: &MaMTorrent) -> bool { if !self.media_type.is_empty() && MediaType::from_id(torrent.mediatype) @@ -78,65 +214,6 @@ impl TorrentFilter { }; } - if self.exclude_uploader.contains(&torrent.owner_name) { - return false; - } - - if self.uploaded_after.is_some() || self.uploaded_before.is_some() { - match UtcDateTime::parse(&torrent.added, &DATE_TIME_FORMAT) { - Ok(added) => { - if let Some(uploaded_after) = self.uploaded_after { - if added.date() < uploaded_after { - return false; - } - } - if let Some(uploaded_before) = self.uploaded_before { - if added.date() > uploaded_before { - return false; - } - } - } - Err(_) => { - error!( - "Failed parsing added \"{}\" for torrent \"{}\"", - torrent.added, torrent.title - ); - return false; - } - } - } - - if let Some(min_seeders) = self.min_seeders { - if torrent.seeders < min_seeders { - return false; - } - } - if let Some(max_seeders) = self.max_seeders { - if torrent.seeders > max_seeders { - return false; - } - } - if let Some(min_leechers) = self.min_leechers { - if torrent.leechers < min_leechers { - return false; - } - } - if let Some(max_leechers) = self.max_leechers { - if torrent.leechers > max_leechers { - return false; - } - } - if let Some(min_snatched) = self.min_snatched { - if torrent.times_completed < min_snatched { - return false; - } - } - if let Some(max_snatched) = self.max_snatched { - if torrent.times_completed > max_snatched { - return false; - } - } - true } @@ -170,41 +247,6 @@ impl TorrentFilter { }; } - if self.exclude_uploader.contains(&torrent.uploader_name) { - return false; - } - - if let Some(min_seeders) = self.min_seeders { - if torrent.seeders < min_seeders { - return false; - } - } - if let Some(max_seeders) = self.max_seeders { - if torrent.seeders > max_seeders { - return false; - } - } - if let Some(min_leechers) = self.min_leechers { - if torrent.leechers < min_leechers { - return false; - } - } - if let Some(max_leechers) = self.max_leechers { - if torrent.leechers > max_leechers { - return false; - } - } - if let Some(min_snatched) = self.min_snatched { - if torrent.times_completed < min_snatched { - return false; - } - } - if let Some(max_snatched) = self.max_snatched { - if torrent.times_completed > max_snatched { - return false; - } - } - true } @@ -213,6 +255,9 @@ impl TorrentFilter { } pub(crate) fn matches_meta(&self, meta: &TorrentMeta) -> Result { + if !self.media_type.is_empty() && !self.media_type.contains(&meta.media_type) { + return Ok(false); + } if let Some(language) = &meta.language { if !self.languages.is_empty() && !self.languages.contains(language) { return Ok(false); @@ -298,17 +343,15 @@ impl TorrentFilter { ); } - ensure!(self.min_size.bytes() == 0, "has min_size"); - ensure!(self.max_size.bytes() == 0, "has max_size"); - ensure!(self.exclude_uploader.is_empty(), "has exclude_uploader"); - ensure!(self.uploaded_after.is_none(), "has uploaded_after"); - ensure!(self.uploaded_before.is_none(), "has uploaded_before"); - ensure!(self.min_seeders.is_none(), "has min_seeders"); - ensure!(self.max_seeders.is_none(), "has max_seeders"); - ensure!(self.min_leechers.is_none(), "has min_leechers"); - ensure!(self.max_leechers.is_none(), "has max_leechers"); - ensure!(self.min_snatched.is_none(), "has min_snatched"); - ensure!(self.max_snatched.is_none(), "has max_snatched"); + if self.min_size.bytes() > 0 || self.max_size.bytes() > 0 { + ensure!(meta.size.bytes() != 0, "has size filter and no stored size"); + if self.min_size.bytes() > 0 && meta.size < self.min_size { + return Ok(false); + } + if self.max_size.bytes() > 0 && meta.size > self.max_size { + return Ok(false); + } + } Ok(true) } @@ -332,44 +375,47 @@ impl GoodreadsList { } pub fn allow_audio(&self) -> bool { - self.grab.iter().any(|g| { - g.filter - .categories - .audio - .as_ref() - .is_none_or(|c| !c.is_empty()) - }) + self.grab + .iter() + .any(|g| match g.edition.categories.audio.as_ref() { + None => true, + Some(c) => !c.is_empty(), + }) } pub fn allow_ebook(&self) -> bool { - self.grab.iter().any(|g| { - g.filter - .categories - .ebook - .as_ref() - .is_none_or(|c| !c.is_empty()) - }) + self.grab + .iter() + .any(|g| match g.edition.categories.ebook.as_ref() { + None => true, + Some(c) => !c.is_empty(), + }) } } impl Library { - pub fn method(&self) -> LibraryLinkMethod { + pub fn options(&self) -> &LibraryOptions { match self { - Library::ByDir(l) => l.tag_filters.method, - Library::ByCategory(l) => l.tag_filters.method, + Library::ByRipDir(l) => &l.options, + Library::ByDownloadDir(l) => &l.options, + Library::ByCategory(l) => &l.options, } } - pub fn library_dir(&self) -> &PathBuf { + pub fn edition_filter(&self) -> Option<&EditionFilter> { match self { - Library::ByDir(l) => &l.library_dir, - Library::ByCategory(l) => &l.library_dir, + Library::ByRipDir(l) => Some(&l.filter), + Library::ByDownloadDir(_) => None, + Library::ByCategory(_) => None, } } pub fn tag_filters(&self) -> &LibraryTagFilters { match self { - Library::ByDir(l) => &l.tag_filters, + Library::ByRipDir(_) => { + EMPTY_LIBRARY_TAG_FILTERS.get_or_init(LibraryTagFilters::default) + } + Library::ByDownloadDir(l) => &l.tag_filters, Library::ByCategory(l) => &l.tag_filters, } } @@ -417,6 +463,23 @@ mod tests { assert!(filter.matches(&torrent)); } + #[test] + fn test_library_tag_filters_by_rip_dir_returns_empty() { + let library = Library::ByRipDir(crate::config::LibraryByRipDir { + rip_dir: std::path::PathBuf::from("/tmp/rips"), + options: LibraryOptions { + name: None, + library_dir: std::path::PathBuf::from("/tmp/library"), + method: Default::default(), + audio_types: None, + ebook_types: None, + }, + filter: Default::default(), + }); + + assert_eq!(library.tag_filters(), &LibraryTagFilters::default()); + } + #[test] fn test_uploaded_before() { let torrent = MaMTorrent { @@ -488,11 +551,14 @@ mod tests { #[test] fn test_category_match() { let filter = TorrentFilter { - categories: Categories { - audio: Some(vec![AudiobookCategory::GeneralFiction]), - ebook: Some(vec![]), - musicology: Some(vec![]), - radio: Some(vec![]), + edition: EditionFilter { + categories: Categories { + audio: Some(vec![AudiobookCategory::GeneralFiction]), + ebook: Some(vec![]), + musicology: Some(vec![]), + radio: Some(vec![]), + }, + ..Default::default() }, ..TorrentFilter::default() }; @@ -503,11 +569,14 @@ mod tests { #[test] fn test_category_no_match() { let filter = TorrentFilter { - categories: Categories { - audio: Some(vec![AudiobookCategory::GeneralNonFic]), - ebook: Some(vec![]), - musicology: Some(vec![]), - radio: Some(vec![]), + edition: EditionFilter { + categories: Categories { + audio: Some(vec![AudiobookCategory::GeneralNonFic]), + ebook: Some(vec![]), + musicology: Some(vec![]), + radio: Some(vec![]), + }, + ..Default::default() }, ..TorrentFilter::default() }; @@ -519,8 +588,11 @@ mod tests { #[test] fn test_language_match() { let filter = TorrentFilter { - languages: vec![Language::English, Language::German], - ..TorrentFilter::default() + edition: EditionFilter { + languages: vec![Language::English, Language::German], + ..Default::default() + }, + ..Default::default() }; let torrent = create_default_torrent(); assert!(filter.matches(&torrent), "Should match English language."); @@ -529,8 +601,11 @@ mod tests { #[test] fn test_language_no_match() { let filter = TorrentFilter { - languages: vec![Language::French, Language::German], - ..TorrentFilter::default() + edition: EditionFilter { + languages: vec![Language::French, Language::German], + ..Default::default() + }, + ..Default::default() }; let torrent = create_default_torrent(); assert!( @@ -542,8 +617,11 @@ mod tests { #[test] fn test_language_parse_fail() { let filter = TorrentFilter { - languages: vec![Language::French], - ..TorrentFilter::default() + edition: EditionFilter { + languages: vec![Language::French], + ..Default::default() + }, + ..Default::default() }; let mut torrent = create_default_torrent(); torrent.language = 99; // Invalid Language @@ -557,8 +635,11 @@ mod tests { #[test] fn test_flags_match() { let filter = TorrentFilter { - flags: Flags { - violence: Some(true), + edition: EditionFilter { + flags: Flags { + violence: Some(true), + ..Default::default() + }, ..Default::default() }, ..TorrentFilter::default() @@ -573,8 +654,11 @@ mod tests { #[test] fn test_flags_no_match() { let filter = TorrentFilter { - flags: Flags { - explicit: Some(true), + edition: EditionFilter { + flags: Flags { + explicit: Some(true), + ..Default::default() + }, ..Default::default() }, ..TorrentFilter::default() @@ -590,7 +674,10 @@ mod tests { #[test] fn test_min_size_match() { let filter = TorrentFilter { - min_size: Size::from_bytes(1_000_000_000), + edition: EditionFilter { + min_size: Size::from_bytes(1_000_000_000), + ..Default::default() + }, ..TorrentFilter::default() }; let torrent = create_default_torrent(); @@ -603,7 +690,10 @@ mod tests { #[test] fn test_min_size_no_match() { let filter = TorrentFilter { - min_size: Size::from_bytes(10_000_000_000), + edition: EditionFilter { + min_size: Size::from_bytes(10_000_000_000), + ..Default::default() + }, ..TorrentFilter::default() }; let torrent = create_default_torrent(); @@ -616,7 +706,10 @@ mod tests { #[test] fn test_max_size_match() { let filter = TorrentFilter { - max_size: Size::from_bytes(10_000_000_000), + edition: EditionFilter { + max_size: Size::from_bytes(10_000_000_000), + ..Default::default() + }, ..TorrentFilter::default() }; let torrent = create_default_torrent(); @@ -629,7 +722,10 @@ mod tests { #[test] fn test_max_size_no_match() { let filter = TorrentFilter { - max_size: Size::from_bytes(1_000_000_000), + edition: EditionFilter { + max_size: Size::from_bytes(1_000_000_000), + ..Default::default() + }, ..TorrentFilter::default() }; let torrent = create_default_torrent(); @@ -642,7 +738,10 @@ mod tests { #[test] fn test_size_parsing_failure() { let filter = TorrentFilter { - min_size: Size::from_bytes(1), + edition: EditionFilter { + min_size: Size::from_bytes(1), + ..Default::default() + }, ..TorrentFilter::default() }; let mut torrent = create_default_torrent(); @@ -833,19 +932,22 @@ mod tests { #[test] fn test_combined_success() { let filter = TorrentFilter { - categories: Categories { - audio: Some(vec![AudiobookCategory::GeneralFiction]), - ebook: Some(vec![]), - musicology: Some(vec![]), - radio: Some(vec![]), - }, - languages: vec![Language::English], - flags: Flags { - violence: Some(true), + edition: EditionFilter { + categories: Categories { + audio: Some(vec![AudiobookCategory::GeneralFiction]), + ebook: Some(vec![]), + musicology: Some(vec![]), + radio: Some(vec![]), + }, + languages: vec![Language::English], + flags: Flags { + violence: Some(true), + ..Default::default() + }, + min_size: Size::from_bytes(1_000_000_000), + max_size: Size::from_bytes(10_000_000_000), ..Default::default() }, - min_size: Size::from_bytes(1_000_000_000), - max_size: Size::from_bytes(10_000_000_000), exclude_uploader: vec!["OtherUploader".to_string()], uploaded_after: Some(date!(2023 - 01 - 15)), min_seeders: Some(40), @@ -871,9 +973,7 @@ mod tests { id: "".to_string(), id_is_hash: false, - mam_id: 0, - abs_id: None, - goodreads_id: None, + mam_id: None, library_path: None, library_files: vec![], linker: None, @@ -883,7 +983,6 @@ mod tests { title_search: "".to_string(), created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, } @@ -891,11 +990,12 @@ mod tests { fn default_meta() -> TorrentMeta { TorrentMeta { - mam_id: 0, + ids: Default::default(), vip_status: None, media_type: MediaType::Audiobook, main_cat: Some(MainCat::Fiction), categories: vec![], + tags: vec![], cat: None, language: None, flags: None, @@ -904,6 +1004,7 @@ mod tests { size: Size::from_bytes(0), title: "".to_string(), edition: None, + description: "".to_string(), authors: vec![], narrators: vec![], series: vec![], @@ -914,8 +1015,11 @@ mod tests { fn create_filter_with_audio_cats(cats: Option>) -> TorrentFilter { TorrentFilter { - categories: Categories { - audio: cats, + edition: EditionFilter { + categories: Categories { + audio: cats, + ..Default::default() + }, ..Default::default() }, ..Default::default() @@ -926,7 +1030,10 @@ mod tests { #[test] fn test_lang_match_ok_true() { let filter = TorrentFilter { - languages: vec![Language::English, Language::French], + edition: EditionFilter { + languages: vec![Language::English, Language::French], + ..Default::default() + }, ..Default::default() }; let torrent = create_torrent_with_meta(TorrentMeta { @@ -939,7 +1046,10 @@ mod tests { #[test] fn test_lang_mismatch_ok_false() { let filter = TorrentFilter { - languages: vec![Language::French], + edition: EditionFilter { + languages: vec![Language::French], + ..Default::default() + }, ..Default::default() }; let torrent = create_torrent_with_meta(TorrentMeta { @@ -952,7 +1062,10 @@ mod tests { #[test] fn test_lang_filter_active_torrent_none_err() { let filter = TorrentFilter { - languages: vec![Language::English], + edition: EditionFilter { + languages: vec![Language::English], + ..Default::default() + }, ..Default::default() }; let torrent = create_torrent_with_meta(TorrentMeta { @@ -973,7 +1086,10 @@ mod tests { #[test] fn test_lang_filter_inactive_torrent_none_ok_true() { let filter = TorrentFilter { - languages: vec![], + edition: EditionFilter { + languages: vec![], + ..Default::default() + }, ..Default::default() }; let torrent = create_torrent_with_meta(TorrentMeta { @@ -1053,8 +1169,11 @@ mod tests { #[test] fn test_flags_match_ok_true() { let filter = TorrentFilter { - flags: Flags { - violence: Some(true), + edition: EditionFilter { + flags: Flags { + violence: Some(true), + ..Default::default() + }, ..Default::default() }, ..Default::default() @@ -1076,8 +1195,11 @@ mod tests { #[test] fn test_flags_mismatch_ok_false() { let filter = TorrentFilter { - flags: Flags { - explicit: Some(true), + edition: EditionFilter { + flags: Flags { + explicit: Some(true), + ..Default::default() + }, ..Default::default() }, ..Default::default() @@ -1098,8 +1220,11 @@ mod tests { #[test] fn test_flags_filter_active_torrent_none_err() { let filter = TorrentFilter { - flags: Flags { - violence: Some(true), + edition: EditionFilter { + flags: Flags { + violence: Some(true), + ..Default::default() + }, ..Default::default() }, ..Default::default() @@ -1118,11 +1243,13 @@ mod tests { ); } - // --- Disallowed Filter Checks (Ensure) --- #[test] fn test_disallowed_min_size_err() { let filter = TorrentFilter { - min_size: Size::from_bytes(1), + edition: EditionFilter { + min_size: Size::from_bytes(1), + ..Default::default() + }, ..Default::default() }; let torrent = create_torrent_with_meta(default_meta()); @@ -1132,10 +1259,41 @@ mod tests { .matches_lib(&torrent) .unwrap_err() .to_string() - .contains("has min_size") + .contains("has size filter and no stored size") ); } + #[test] + fn test_match_size_ok_true_when_meta_has_size() { + let mut meta = default_meta(); + meta.size = Size::from_bytes(2_000_000_000); + let torrent = create_torrent_with_meta(meta); + let filter = TorrentFilter { + edition: EditionFilter { + min_size: Size::from_bytes(1_000_000_000), + max_size: Size::from_bytes(3_000_000_000), + ..Default::default() + }, + ..Default::default() + }; + assert!(filter.matches_lib(&torrent).unwrap()); + } + + #[test] + fn test_match_size_ok_false_when_meta_out_of_range() { + let mut meta = default_meta(); + meta.size = Size::from_bytes(500_000_000); + let torrent = create_torrent_with_meta(meta); + let filter = TorrentFilter { + edition: EditionFilter { + min_size: Size::from_bytes(1_000_000_000), + ..Default::default() + }, + ..Default::default() + }; + assert!(!filter.matches_lib(&torrent).unwrap()); + } + #[test] fn test_disallowed_exclude_uploader_err() { let filter = TorrentFilter { @@ -1191,15 +1349,18 @@ mod tests { #[test] fn test_full_success_match() { let filter = TorrentFilter { - languages: vec![Language::English], - categories: Categories { - audio: Some(vec![AudiobookCategory::GeneralFiction]), - ebook: None, - musicology: None, - radio: None, - }, - flags: Flags { - crude_language: Some(true), + edition: EditionFilter { + languages: vec![Language::English], + categories: Categories { + audio: Some(vec![AudiobookCategory::GeneralFiction]), + ebook: None, + musicology: None, + radio: None, + }, + flags: Flags { + crude_language: Some(true), + ..Default::default() + }, ..Default::default() }, ..Default::default() diff --git a/server/src/lib.rs b/server/src/lib.rs new file mode 100644 index 00000000..1e09b56d --- /dev/null +++ b/server/src/lib.rs @@ -0,0 +1,16 @@ +pub mod audiobookshelf; +pub mod autograbber; +pub mod cleaner; +pub mod config; +pub mod config_impl; +pub mod exporter; +pub mod linker; +pub mod lists; +pub mod logging; +pub mod qbittorrent; +pub mod snatchlist; +pub mod stats; +pub mod torrent_downloader; +pub mod web; +#[cfg(target_family = "windows")] +pub mod windows; diff --git a/server/src/linker.rs b/server/src/linker.rs deleted file mode 100644 index 8441f46f..00000000 --- a/server/src/linker.rs +++ /dev/null @@ -1,791 +0,0 @@ -#[cfg(target_family = "unix")] -use std::os::unix::fs::MetadataExt as _; -#[cfg(target_family = "windows")] -use std::os::windows::fs::MetadataExt as _; -use std::{ - collections::BTreeMap, - fs::{self, File, Metadata}, - io::{BufWriter, ErrorKind, Write}, - ops::Deref, - path::{Component, Path, PathBuf}, - sync::Arc, -}; - -use anyhow::{Context, Result, bail}; -use file_id::get_file_id; -use log::error; -use mlm_db::{ - ClientStatus, DatabaseExt as _, ErroredTorrentId, Event, EventType, LibraryMismatch, - SelectedTorrent, SelectedTorrentKey, Size, Timestamp, Torrent, TorrentKey, TorrentMeta, -}; -use mlm_mam::{api::MaM, meta::MetaError, search::MaMTorrent}; -use mlm_parse::normalize_title; -use native_db::Database; -use once_cell::sync::Lazy; -use qbit::{ - models::{Torrent as QbitTorrent, TorrentContent}, - parameters::TorrentListParams, -}; -use regex::Regex; -use tokio::fs::create_dir_all; -use tracing::{Level, debug, instrument, span, trace, warn}; - -use crate::{ - audiobookshelf::{self as abs}, - autograbber::update_torrent_meta, - cleaner::remove_library_files, - config::{Config, Library, LibraryLinkMethod, QbitConfig}, - logging::{TorrentMetaError, update_errored_torrent, write_event}, - qbittorrent::ensure_category_exists, -}; - -pub static DISK_PATTERN: Lazy = - Lazy::new(|| Regex::new(r"(?:CD|Disc|Disk)\s*(\d+)").unwrap()); - -#[instrument(skip_all)] -pub async fn link_torrents_to_library( - config: Arc, - db: Arc>, - qbit: (&QbitConfig, &qbit::Api), - mam: Arc>, -) -> Result<()> { - let torrents = qbit - .1 - .torrents(Some(TorrentListParams::default())) - .await - .context("qbit main data")?; - - for torrent in torrents { - if torrent.progress < 1.0 { - continue; - } - let library = find_library(&config, &torrent); - let r = db.r_transaction()?; - let mut existing_torrent: Option = r.get().primary(torrent.hash.clone())?; - { - let selected_torrent: Option = r.get().secondary::( - SelectedTorrentKey::hash, - Some(torrent.hash.clone()), - )?; - if let Some(selected_torrent) = selected_torrent { - debug!( - "Finished Downloading torrent {} {}", - selected_torrent.mam_id, selected_torrent.meta.title - ); - let (_guard, rw) = db.rw_async().await?; - rw.remove(selected_torrent)?; - rw.commit()?; - } - } - if let Some(t) = &mut existing_torrent { - let library_name = library.and_then(|l| l.tag_filters().name.as_ref()); - if t.linker.as_ref() != library_name { - let (_guard, rw) = db.rw_async().await?; - t.linker = library_name.map(ToOwned::to_owned); - rw.upsert(t.clone())?; - rw.commit()?; - } - let category = if torrent.category.is_empty() { - None - } else { - Some(torrent.category.as_str()) - }; - if t.category.as_deref() != category { - let (_guard, rw) = db.rw_async().await?; - t.category = category.map(ToOwned::to_owned); - rw.upsert(t.clone())?; - rw.commit()?; - } - if t.client_status.is_none() { - let trackers = qbit.1.trackers(&torrent.hash).await?; - if let Some(mam_tracker) = trackers.last() - && mam_tracker.msg == "torrent not registered with this tracker" - { - { - let (_guard, rw) = db.rw_async().await?; - t.client_status = Some(ClientStatus::RemovedFromMam); - rw.upsert(t.clone())?; - rw.commit()?; - } - write_event( - &db, - Event::new( - Some(torrent.hash.clone()), - Some(t.mam_id), - EventType::RemovedFromMam, - ), - ) - .await; - } - } - if let Some(library_path) = &t.library_path { - let Some(library) = find_library(&config, &torrent) else { - if t.library_mismatch != Some(LibraryMismatch::NoLibrary) { - debug!("no library: {library_path:?}",); - t.library_mismatch = Some(LibraryMismatch::NoLibrary); - let (_guard, rw) = db.rw_async().await?; - rw.upsert(t.clone())?; - rw.commit()?; - } - continue; - }; - if !library_path.starts_with(library.library_dir()) { - let wanted = Some(LibraryMismatch::NewLibraryDir( - library.library_dir().clone(), - )); - if t.library_mismatch != wanted { - debug!( - "library differs: {library_path:?} != {:?}", - library.library_dir() - ); - t.library_mismatch = wanted; - let (_guard, rw) = db.rw_async().await?; - rw.upsert(t.clone())?; - rw.commit()?; - } - } else { - let dir = library_dir(config.exclude_narrator_in_library_dir, library, &t.meta); - let mut is_wrong = Some(library_path) != dir.as_ref(); - let wanted = match dir { - Some(dir) => Some(LibraryMismatch::NewPath(dir)), - None => Some(LibraryMismatch::NoLibrary), - }; - - if t.library_mismatch != wanted { - if is_wrong { - // Try another attempt at matching with exclude_narrator flipped - let dir_2 = library_dir( - !config.exclude_narrator_in_library_dir, - library, - &t.meta, - ); - if Some(library_path) == dir_2.as_ref() { - is_wrong = false - } - } - if is_wrong { - debug!("path differs: {library_path:?} != {:?}", wanted); - t.library_mismatch = wanted; - let (_guard, rw) = db.rw_async().await?; - rw.upsert(t.clone())?; - rw.commit()?; - } else if t.library_mismatch.is_some() { - t.library_mismatch = None; - let (_guard, rw) = db.rw_async().await?; - rw.upsert(t.clone())?; - rw.commit()?; - } - } - } - continue; - } - if t.replaced_with.is_some() { - continue; - } - } - - let Some(library) = library else { - trace!( - "Could not find matching library for torrent \"{}\", save_path {}", - torrent.name, torrent.save_path - ); - continue; - }; - - if library.method() == LibraryLinkMethod::NoLink && existing_torrent.is_some() { - continue; - } - - let result = match_torrent( - config.clone(), - db.clone(), - qbit, - mam.clone(), - &torrent.hash, - &torrent, - library, - existing_torrent, - ) - .await - .context("match_torrent"); - update_errored_torrent( - &db, - ErroredTorrentId::Linker(torrent.hash.clone()), - torrent.name, - result, - ) - .await; - } - - Ok(()) -} - -#[instrument(skip_all)] -#[allow(clippy::too_many_arguments)] -async fn match_torrent( - config: Arc, - db: Arc>, - qbit: (&QbitConfig, &qbit::Api), - mam: Arc>, - hash: &str, - torrent: &QbitTorrent, - library: &Library, - existing_torrent: Option, -) -> Result<()> { - let mut existing_torrent = existing_torrent; - let files = qbit.1.files(hash, None).await?; - let selected_audio_format = select_format( - &library.tag_filters().audio_types, - &config.audio_types, - &files, - ); - let selected_ebook_format = select_format( - &library.tag_filters().ebook_types, - &config.ebook_types, - &files, - ); - - if selected_audio_format.is_none() && selected_ebook_format.is_none() { - bail!("Could not find any wanted formats in torrent"); - } - let Some(mam_torrent) = mam.get_torrent_info(hash).await.context("get_mam_info")? else { - bail!("Could not find torrent on mam"); - }; - if existing_torrent.is_none() - && let Some(old_torrent) = db - .r_transaction()? - .get() - .secondary::(TorrentKey::mam_id, mam_torrent.id)? - { - if old_torrent.id != hash { - let (_guard, rw) = db.rw_async().await?; - rw.remove(old_torrent.clone())?; - rw.commit()?; - } - existing_torrent = Some(old_torrent); - } - let meta = match mam_torrent.as_meta() { - Ok(meta) => meta, - Err(err) => { - if let MetaError::UnknownMediaType(_) = err { - if let Some(on_invalid_torrent) = &qbit.0.on_invalid_torrent { - let qbit_url = qbit.0.url.clone(); - let qbit = qbit::Api::new_login_username_password( - &qbit.0.url, - &qbit.0.username, - &qbit.0.password, - ) - .await?; - - if let Some(category) = &on_invalid_torrent.category { - ensure_category_exists(&qbit, &qbit_url, category).await?; - qbit.set_category(Some(vec![&torrent.hash]), category) - .await?; - } - - if !on_invalid_torrent.tags.is_empty() { - qbit.add_tags( - Some(vec![&torrent.hash]), - on_invalid_torrent.tags.iter().map(Deref::deref).collect(), - ) - .await?; - } - } - trace!("qbit updated"); - } - return Err(err).context("as_meta"); - } - }; - - link_torrent( - &config, - qbit.0, - &db, - hash, - torrent, - files, - selected_audio_format, - selected_ebook_format, - library, - mam_torrent, - existing_torrent.as_ref(), - &meta, - ) - .await - .context("link_torrent") - .map_err(|err| anyhow::Error::new(TorrentMetaError(meta, err))) -} - -#[instrument(skip_all)] -pub async fn refresh_metadata( - config: &Config, - db: &Database<'_>, - mam: &MaM<'_>, - id: String, -) -> Result<(Torrent, MaMTorrent)> { - let Some(mut torrent): Option = db.r_transaction()?.get().primary(id)? else { - bail!("Could not find torrent id"); - }; - debug!("refreshing metadata for torrent {}", torrent.meta.mam_id); - let Some(mam_torrent) = mam - .get_torrent_info_by_id(torrent.mam_id) - .await - .context("get_mam_info")? - else { - bail!("Could not find torrent \"{}\" on mam", torrent.meta.title); - }; - let meta = mam_torrent.as_meta().context("as_meta")?; - - if torrent.meta != meta { - update_torrent_meta( - config, - db, - db.rw_async().await?, - &mam_torrent, - torrent.clone(), - meta.clone(), - true, - false, - ) - .await?; - torrent.meta = meta; - } - Ok((torrent, mam_torrent)) -} - -#[instrument(skip_all)] -pub async fn refresh_metadata_relink( - config: &Config, - db: &Database<'_>, - mam: &MaM<'_>, - hash: String, -) -> Result<()> { - let mut torrent = None; - for qbit_conf in &config.qbittorrent { - let qbit = match qbit::Api::new_login_username_password( - &qbit_conf.url, - &qbit_conf.username, - &qbit_conf.password, - ) - .await - { - Ok(qbit) => qbit, - Err(err) => { - error!("Error logging in to qbit {}: {err}", qbit_conf.url); - continue; - } - }; - let mut torrents = match qbit - .torrents(Some(TorrentListParams { - hashes: Some(vec![hash.clone()]), - ..TorrentListParams::default() - })) - .await - { - Ok(torrents) => torrents, - Err(err) => { - error!("Error getting torrents from qbit {}: {err}", qbit_conf.url); - continue; - } - }; - let Some(t) = torrents.pop() else { - continue; - }; - torrent.replace((qbit_conf, qbit, t)); - break; - } - let Some((qbit_conf, qbit, qbit_torrent)) = torrent else { - bail!("Could not find torrent in qbit"); - }; - let Some(library) = find_library(config, &qbit_torrent) else { - bail!("Could not find matching library for torrent"); - }; - let files = qbit.files(&hash, None).await?; - let selected_audio_format = select_format( - &library.tag_filters().audio_types, - &config.audio_types, - &files, - ); - let selected_ebook_format = select_format( - &library.tag_filters().ebook_types, - &config.ebook_types, - &files, - ); - - if selected_audio_format.is_none() && selected_ebook_format.is_none() { - bail!("Could not find any wanted formats in torrent"); - } - let (torrent, mam_torrent) = refresh_metadata(config, db, mam, hash.clone()).await?; - let library_path_changed = torrent.library_path - != library_dir( - config.exclude_narrator_in_library_dir, - library, - &torrent.meta, - ); - remove_library_files(config, &torrent, library_path_changed).await?; - link_torrent( - config, - qbit_conf, - db, - &hash, - &qbit_torrent, - files, - selected_audio_format, - selected_ebook_format, - library, - mam_torrent, - Some(&torrent), - &torrent.meta, - ) - .await - .context("link_torrent") - .map_err(|err| anyhow::Error::new(TorrentMetaError(torrent.meta, err))) -} - -#[instrument(skip_all)] -#[allow(clippy::too_many_arguments)] -async fn link_torrent( - config: &Config, - qbit_config: &QbitConfig, - db: &Database<'_>, - hash: &str, - torrent: &QbitTorrent, - files: Vec, - selected_audio_format: Option, - selected_ebook_format: Option, - library: &Library, - mam_torrent: MaMTorrent, - existing_torrent: Option<&Torrent>, - meta: &TorrentMeta, -) -> Result<()> { - let mut library_files = vec![]; - - let library_path = if library.tag_filters().method != LibraryLinkMethod::NoLink { - let Some(mut dir) = library_dir(config.exclude_narrator_in_library_dir, library, meta) - else { - bail!("Torrent has no author"); - }; - if config.exclude_narrator_in_library_dir && !meta.narrators.is_empty() && dir.exists() { - dir = library_dir(false, library, meta).unwrap(); - } - let metadata = abs::create_metadata(&mam_torrent, meta); - - create_dir_all(&dir).await?; - for file in files { - let span = span!(Level::TRACE, "file: {:?}", file.name); - let _s = span.enter(); - if !(selected_audio_format - .as_ref() - .is_some_and(|ext| file.name.ends_with(ext)) - || selected_ebook_format - .as_ref() - .is_some_and(|ext| file.name.ends_with(ext))) - { - debug!("Skiping \"{}\"", file.name); - continue; - } - let torrent_path = PathBuf::from(&file.name); - let mut path_components = torrent_path.components(); - let file_name = path_components.next_back().unwrap(); - let dir_name = path_components.next_back().and_then(|dir_name| { - if let Component::Normal(dir_name) = dir_name { - let dir_name = dir_name.to_string_lossy().to_string(); - if let Some(disc) = DISK_PATTERN.captures(&dir_name).and_then(|c| c.get(1)) { - return Some(format!("Disc {}", disc.as_str())); - } - } - None - }); - let file_path = if let Some(dir_name) = dir_name { - let sub_dir = PathBuf::from(dir_name); - create_dir_all(dir.join(&sub_dir)).await?; - sub_dir.join(file_name) - } else { - PathBuf::from(&file_name) - }; - let library_path = dir.join(&file_path); - library_files.push(file_path.clone()); - let download_path = - map_path(&qbit_config.path_mapping, &torrent.save_path).join(&file.name); - match library.method() { - LibraryLinkMethod::Hardlink => { - hard_link(&download_path, &library_path, &file_path)? - } - LibraryLinkMethod::HardlinkOrCopy => { - hard_link(&download_path, &library_path, &file_path) - .or_else(|_| copy(&download_path, &library_path))? - } - LibraryLinkMethod::Copy => copy(&download_path, &library_path)?, - LibraryLinkMethod::HardlinkOrSymlink => { - hard_link(&download_path, &library_path, &file_path) - .or_else(|_| symlink(&download_path, &library_path))? - } - LibraryLinkMethod::Symlink => symlink(&download_path, &library_path)?, - LibraryLinkMethod::NoLink => {} - }; - } - library_files.sort(); - - let file = File::create(dir.join("metadata.json"))?; - let mut writer = BufWriter::new(file); - serde_json::to_writer(&mut writer, &metadata)?; - writer.flush()?; - Some(dir.clone()) - } else { - None - }; - - { - let (_guard, rw) = db.rw_async().await?; - rw.upsert(Torrent { - id: hash.to_owned(), - id_is_hash: true, - mam_id: meta.mam_id, - abs_id: existing_torrent.and_then(|t| t.abs_id.clone()), - goodreads_id: existing_torrent.and_then(|t| t.goodreads_id), - library_path: library_path.clone(), - library_files, - linker: library.tag_filters().name.clone(), - category: if torrent.category.is_empty() { - None - } else { - Some(torrent.category.clone()) - }, - selected_audio_format, - selected_ebook_format, - title_search: normalize_title(&meta.title), - meta: meta.clone(), - created_at: existing_torrent - .map(|t| t.created_at) - .unwrap_or_else(Timestamp::now), - replaced_with: existing_torrent.and_then(|t| t.replaced_with.clone()), - request_matadata_update: false, - library_mismatch: None, - client_status: existing_torrent.and_then(|t| t.client_status.clone()), - })?; - rw.commit()?; - } - - if let Some(library_path) = library_path { - write_event( - db, - Event::new( - Some(hash.to_owned()), - Some(meta.mam_id), - EventType::Linked { - linker: library.tag_filters().name.clone(), - library_path, - }, - ), - ) - .await; - } - - Ok(()) -} - -pub fn map_path(path_mapping: &BTreeMap, save_path: &str) -> PathBuf { - let mut path = PathBuf::from(save_path); - for (from, to) in path_mapping.iter().rev() { - if path.starts_with(from) { - let mut components = path.components(); - for _ in from { - components.next(); - } - path = to.join(components.as_path()); - break; - } - } - path -} - -pub fn find_library<'a>(config: &'a Config, torrent: &QbitTorrent) -> Option<&'a Library> { - config - .libraries - .iter() - .filter(|l| match l { - Library::ByDir(l) => PathBuf::from(&torrent.save_path).starts_with(&l.download_dir), - Library::ByCategory(l) => torrent.category == l.category, - }) - .find(|l| { - let filters = l.tag_filters(); - if filters - .deny_tags - .iter() - .any(|tag| torrent.tags.split(", ").any(|t| t == tag.as_str())) - { - return false; - } - if filters.allow_tags.is_empty() { - return true; - } - filters - .allow_tags - .iter() - .any(|tag| torrent.tags.split(", ").any(|t| t == tag.as_str())) - }) -} - -pub fn library_dir( - exclude_narrator_in_library_dir: bool, - library: &Library, - meta: &TorrentMeta, -) -> Option { - let author = meta.authors.first()?; - let mut dir = match meta - .series - .iter() - .find(|s| !s.entries.0.is_empty()) - .or(meta.series.first()) - { - Some(series) => PathBuf::from(sanitize_filename::sanitize(author).to_string()) - .join(sanitize_filename::sanitize(&series.name).to_string()) - .join( - sanitize_filename::sanitize(if series.entries.0.is_empty() { - meta.title.clone() - } else { - format!("{} #{} - {}", series.name, series.entries, meta.title) - }) - .to_string(), - ), - None => PathBuf::from(sanitize_filename::sanitize(author).to_string()) - .join(sanitize_filename::sanitize(&meta.title).to_string()), - }; - if let Some((edition, _)) = &meta.edition { - dir.set_file_name( - sanitize_filename::sanitize(format!( - "{}, {}", - dir.file_name().unwrap().to_string_lossy(), - edition - )) - .to_string(), - ); - } - if let Some(narrator) = meta.narrators.first() - && !exclude_narrator_in_library_dir - { - dir.set_file_name( - sanitize_filename::sanitize(format!( - "{} {{{}}}", - dir.file_name().unwrap().to_string_lossy(), - narrator - )) - .to_string(), - ); - } - let dir = library.library_dir().join(dir); - Some(dir) -} - -fn select_format( - overridden_wanted_formats: &Option>, - wanted_formats: &[String], - files: &[TorrentContent], -) -> Option { - overridden_wanted_formats - .as_deref() - .unwrap_or(wanted_formats) - .iter() - .map(|ext| { - let ext = ext.to_lowercase(); - if ext.starts_with(".") { - ext.clone() - } else { - format!(".{ext}") - } - }) - .find(|ext| files.iter().any(|f| f.name.to_lowercase().ends_with(ext))) -} - -#[instrument(skip_all)] -fn hard_link(download_path: &Path, library_path: &Path, file_path: &Path) -> Result<()> { - debug!("linking: {:?} -> {:?}", download_path, library_path); - fs::hard_link(download_path, library_path).or_else(|err| { - if err.kind() == ErrorKind::AlreadyExists { - trace!("AlreadyExists: {}", err); - let download_id = get_file_id(download_path); - trace!("got 1: {download_id:?}"); - let library_id = get_file_id(library_path); - trace!("got 2: {library_id:?}"); - if let (Ok(download_id), Ok(library_id)) = (download_id, library_id) { - trace!("got both"); - if download_id == library_id { - trace!("both match"); - return Ok(()); - } else { - trace!("no match"); - bail!( - "File \"{:?}\" already exists, torrent file size: {}, library file size: {}", - file_path, - fs::metadata(download_path).map_or("?".to_string(), |s| Size::from_bytes(file_size(&s)).to_string()), - fs::metadata(library_path).map_or("?".to_string(), |s| Size::from_bytes(file_size(&s)).to_string()) - ); - } - } - } - Err(err.into()) - })?; - Ok(()) -} - -#[instrument(skip_all)] -fn copy(download_path: &Path, library_path: &Path) -> Result<()> { - debug!("copying: {:?} -> {:?}", download_path, library_path); - fs::copy(download_path, library_path)?; - Ok(()) -} - -#[instrument(skip_all)] -fn symlink(download_path: &Path, library_path: &Path) -> Result<()> { - debug!("symlinking: {:?} -> {:?}", download_path, library_path); - #[cfg(target_family = "unix")] - std::os::unix::fs::symlink(download_path, library_path)?; - #[cfg(target_family = "windows")] - bail!("symlink is not supported on Windows"); - #[allow(unreachable_code)] - Ok(()) -} - -pub fn file_size(m: &Metadata) -> u64 { - #[cfg(target_family = "unix")] - return m.size(); - #[cfg(target_family = "windows")] - return m.file_size(); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_map_path() { - let mut mappings = BTreeMap::new(); - mappings.insert(PathBuf::from("/downloads"), PathBuf::from("/books")); - mappings.insert( - PathBuf::from("/downloads/audiobooks"), - PathBuf::from("/audiobooks"), - ); - mappings.insert(PathBuf::from("/audiobooks"), PathBuf::from("/audiobooks")); - - assert_eq!( - map_path(&mappings, "/downloads/torrent"), - PathBuf::from("/books/torrent") - ); - assert_eq!( - map_path(&mappings, "/downloads/audiobooks/torrent"), - PathBuf::from("/audiobooks/torrent") - ); - assert_eq!( - map_path(&mappings, "/downloads/audiobooks/torrent/deep"), - PathBuf::from("/audiobooks/torrent/deep") - ); - assert_eq!( - map_path(&mappings, "/audiobooks/torrent"), - PathBuf::from("/audiobooks/torrent") - ); - assert_eq!( - map_path(&mappings, "/ebooks/torrent"), - PathBuf::from("/ebooks/torrent") - ); - } -} diff --git a/server/src/linker/common.rs b/server/src/linker/common.rs new file mode 100644 index 00000000..e5cd4fec --- /dev/null +++ b/server/src/linker/common.rs @@ -0,0 +1,326 @@ +#[cfg(target_family = "unix")] +use std::os::unix::fs::MetadataExt as _; +#[cfg(target_family = "windows")] +use std::os::windows::fs::MetadataExt as _; +use std::{ + collections::BTreeMap, + fs::{self, Metadata}, + io::ErrorKind, + path::{Path, PathBuf}, +}; +use tokio::fs::DirEntry; + +use anyhow::{Result, bail}; +use file_id::get_file_id; +use mlm_db::Size; +use tracing::{debug, trace}; + +pub fn map_path(path_mapping: &BTreeMap, save_path: &str) -> PathBuf { + let mut path = PathBuf::from(save_path); + for (from, to) in path_mapping.iter().rev() { + if path.starts_with(from) { + let mut components = path.components(); + for _ in from { + components.next(); + } + path = to.join(components.as_path()); + break; + } + } + path +} + +pub fn library_dir( + exclude_narrator_in_library_dir: bool, + library: &crate::config::Library, + meta: &mlm_db::TorrentMeta, +) -> Option { + let author = meta.authors.first()?; + let mut dir = match meta + .series + .iter() + .find(|s| !s.entries.0.is_empty()) + .or(meta.series.first()) + { + Some(series) => PathBuf::from(sanitize_filename::sanitize(author).to_string()) + .join(sanitize_filename::sanitize(&series.name).to_string()) + .join( + sanitize_filename::sanitize(if series.entries.0.is_empty() { + meta.title.clone() + } else { + format!("{} #{} - {}", series.name, series.entries, meta.title) + }) + .to_string(), + ), + None => PathBuf::from(sanitize_filename::sanitize(author).to_string()) + .join(sanitize_filename::sanitize(&meta.title).to_string()), + }; + if let Some((edition, _)) = &meta.edition { + dir.set_file_name( + sanitize_filename::sanitize(format!( + "{}, {}", + dir.file_name().unwrap().to_string_lossy(), + edition + )) + .to_string(), + ); + } + if let Some(narrator) = meta.narrators.first() + && !exclude_narrator_in_library_dir + { + dir.set_file_name( + sanitize_filename::sanitize(format!( + "{} {{{}}}", + dir.file_name().unwrap().to_string_lossy(), + narrator + )) + .to_string(), + ); + } + let dir = library.options().library_dir.join(dir); + Some(dir) +} + +pub trait HasFileName { + fn name_lower(&self) -> String; +} + +impl HasFileName for DirEntry { + fn name_lower(&self) -> String { + self.file_name().to_string_lossy().to_lowercase() + } +} + +impl HasFileName for qbit::models::TorrentContent { + fn name_lower(&self) -> String { + self.name.to_lowercase() + } +} + +pub fn select_format( + overridden_wanted_formats: &Option>, + wanted_formats: &[String], + files: &[T], +) -> Option { + overridden_wanted_formats + .as_deref() + .unwrap_or(wanted_formats) + .iter() + .map(|ext| { + let ext = ext.to_lowercase(); + if ext.starts_with('.') { + ext.clone() + } else { + format!(".{ext}") + } + }) + .find(|ext| files.iter().any(|f| f.name_lower().ends_with(ext))) +} + +pub fn hard_link(download_path: &Path, library_path: &Path, file_path: &Path) -> Result<()> { + debug!("linking: {:?} -> {:?}", download_path, library_path); + fs::hard_link(download_path, library_path).or_else(|err| { + if err.kind() == ErrorKind::AlreadyExists { + trace!("AlreadyExists: {}", err); + let download_id = get_file_id(download_path); + trace!("got 1: {download_id:?}"); + let library_id = get_file_id(library_path); + trace!("got 2: {library_id:?}"); + if let (Ok(download_id), Ok(library_id)) = (download_id, library_id) { + trace!("got both"); + if download_id == library_id { + trace!("both match"); + return Ok(()); + } else { + trace!("no match"); + bail!( + "File \"{:?}\" already exists, torrent file size: {}, library file size: {}", + file_path, + fs::metadata(download_path).map_or("?".to_string(), |s| Size::from_bytes(file_size(&s)).to_string()), + fs::metadata(library_path).map_or("?".to_string(), |s| Size::from_bytes(file_size(&s)).to_string()) + ); + } + } + } + Err(err.into()) + })?; + Ok(()) +} + +pub fn copy(download_path: &Path, library_path: &Path) -> Result<()> { + debug!("copying: {:?} -> {:?}", download_path, library_path); + fs::copy(download_path, library_path)?; + Ok(()) +} + +pub fn symlink(download_path: &Path, library_path: &Path) -> Result<()> { + debug!("symlinking: {:?} -> {:?}", download_path, library_path); + #[cfg(target_family = "unix")] + std::os::unix::fs::symlink(download_path, library_path)?; + #[cfg(target_family = "windows")] + bail!("symlink is not supported on Windows"); + #[allow(unreachable_code)] + Ok(()) +} + +pub fn file_size(m: &Metadata) -> u64 { + #[cfg(target_family = "unix")] + return m.size(); + #[cfg(target_family = "windows")] + return m.file_size(); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn test_map_path() { + let mut mappings = BTreeMap::new(); + mappings.insert(PathBuf::from("/downloads"), PathBuf::from("/books")); + mappings.insert( + PathBuf::from("/downloads/audiobooks"), + PathBuf::from("/audiobooks"), + ); + mappings.insert(PathBuf::from("/audiobooks"), PathBuf::from("/audiobooks")); + + assert_eq!( + map_path(&mappings, "/downloads/torrent"), + PathBuf::from("/books/torrent") + ); + assert_eq!( + map_path(&mappings, "/downloads/audiobooks/torrent"), + PathBuf::from("/audiobooks/torrent") + ); + assert_eq!( + map_path(&mappings, "/downloads/audiobooks/torrent/deep"), + PathBuf::from("/audiobooks/torrent/deep") + ); + assert_eq!( + map_path(&mappings, "/audiobooks/torrent"), + PathBuf::from("/audiobooks/torrent") + ); + assert_eq!( + map_path(&mappings, "/ebooks/torrent"), + PathBuf::from("/ebooks/torrent") + ); + } + + #[test] + fn test_select_format() { + struct F { name: String } + impl HasFileName for F { + fn name_lower(&self) -> String { self.name.to_lowercase() } + } + let files = vec![F { name: "book.M4B".to_string() }, F { name: "cover.jpg".to_string() }]; + let wanted = vec!["m4b".to_string(), "mp3".to_string()]; + let sel = select_format(&Some(vec!["m4b".to_string()]), &wanted, &files); + assert_eq!(sel.unwrap(), ".m4b".to_string()); + let sel2 = select_format(&None, &wanted, &files); + assert_eq!(sel2.unwrap(), ".m4b".to_string()); + } + + #[test] + fn test_select_format_leading_dot_in_override() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "track.FLAC".to_string() }]; + let wanted = vec!["mp3".to_string(), "flac".to_string()]; + // override contains leading dot + let sel = select_format(&Some(vec![".flac".to_string()]), &wanted, &files); + assert_eq!(sel.unwrap(), ".flac".to_string()); + } + + #[test] + fn test_select_format_uppercase_extension() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "ALBUM.MP3".to_string() }]; + let wanted = vec!["mp3".to_string()]; + let sel = select_format(&None, &wanted, &files); + assert_eq!(sel.unwrap(), ".mp3".to_string()); + } + + #[test] + fn test_select_format_missing_extension_returns_none() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "README".to_string() }]; + let wanted = vec!["m4b".to_string()]; + let sel = select_format(&None, &wanted, &files); + assert!(sel.is_none()); + } + + #[test] + fn test_select_format_overridden_empty_vector() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "song.mp3".to_string() }]; + let wanted = vec!["mp3".to_string()]; + // override provided but empty -> should produce no selection + let sel = select_format(&Some(vec![]), &wanted, &files); + assert!(sel.is_none()); + } + + #[test] + fn test_select_format_wanted_empty_then_none() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "file.mp3".to_string() }]; + let wanted: Vec = vec![]; + let sel = select_format(&None, &wanted, &files); + assert!(sel.is_none()); + } + + #[test] + fn test_file_size_and_copy_and_hardlink() { + use std::fs; + use std::io::Write; + let tmp = std::env::temp_dir().join(format!("mlm_test_{}", std::process::id())); + let _ = fs::create_dir_all(&tmp); + let src = tmp.join("src_file.bin"); + let dst = tmp.join("dst_file.bin"); + let mut f = fs::File::create(&src).unwrap(); + let data = b"hello world"; + f.write_all(data).unwrap(); + f.sync_all().unwrap(); + let meta = fs::metadata(&src).unwrap(); + assert_eq!(file_size(&meta), data.len() as u64); + + // copy + copy(&src, &dst).unwrap(); + assert!(dst.exists()); + assert_eq!(fs::metadata(&dst).unwrap().len(), data.len() as u64); + + // hard link target + let hl = tmp.join("hl_file.bin"); + // remove if exists + let _ = fs::remove_file(&hl); + hard_link(&src, &hl, &PathBuf::from("hl_file.bin")).unwrap(); + // both should exist and have same len + assert!(hl.exists()); + assert_eq!(fs::metadata(&hl).unwrap().len(), data.len() as u64); + + // cleanup + let _ = fs::remove_file(&src); + let _ = fs::remove_file(&dst); + let _ = fs::remove_file(&hl); + } + + #[cfg(target_family = "unix")] + #[test] + fn test_symlink() { + use std::fs; + let tmp = std::env::temp_dir().join(format!("mlm_test_symlink_{}", std::process::id())); + let _ = fs::create_dir_all(&tmp); + let src = tmp.join("s_src.txt"); + let dst = tmp.join("s_dst.txt"); + fs::write(&src, b"x").unwrap(); + let _ = fs::remove_file(&dst); + symlink(&src, &dst).unwrap(); + assert!(dst.exists()); + let _ = fs::remove_file(&src); + let _ = fs::remove_file(&dst); + } +} diff --git a/server/src/linker/duplicates.rs b/server/src/linker/duplicates.rs new file mode 100644 index 00000000..cf6f5997 --- /dev/null +++ b/server/src/linker/duplicates.rs @@ -0,0 +1,205 @@ +use std::fs; + +use anyhow::Result; +use mlm_db::{Torrent, TorrentKey}; +use native_db::Database; +use tracing::trace; + +use crate::config::Config; +use crate::linker::file_size; + +pub fn find_matches(db: &Database<'_>, torrent: &Torrent) -> Result> { + let r = db.r_transaction()?; + let torrents = r.scan().secondary::(TorrentKey::title_search)?; + let matches = torrents + .all()? + .filter_map(|t| t.ok()) + .filter(|t| t.id != torrent.id && t.matches(torrent)) + .collect(); + Ok(matches) +} + +pub fn rank_torrents(config: &Config, batch: Vec) -> Vec { + if batch.len() <= 1 { + return batch; + } + + let mut ranked = batch + .into_iter() + .map(|torrent| { + let preferred_types = config.preferred_types(&torrent.meta.media_type); + let preference = preferred_types + .iter() + .position(|t| torrent.meta.filetypes.contains(t)) + .unwrap_or(usize::MAX); + (torrent, preference) + }) + .collect::>(); + ranked.sort_by_key(|(_, preference)| *preference); + + if ranked[0].1 == ranked[1].1 { + let mut with_size = ranked + .into_iter() + .map(|(torrent, preference)| { + let mut size = 0; + if let Some(library_path) = &torrent.library_path { + for file in &torrent.library_files { + let path = library_path.join(file); + size += fs::metadata(path).map_or(0, |s| file_size(&s)); + } + } + if size == 0 { + size = torrent.meta.size.bytes(); + } + (torrent, preference, size) + }) + .collect::>(); + with_size.sort_by(|a, b| a.1.cmp(&b.1).then(b.2.cmp(&a.2))); + trace!("ranked batch by size: {:?}", with_size); + ranked = with_size + .into_iter() + .map(|(torrent, preference, _)| (torrent, preference)) + .collect(); + } + ranked.into_iter().map(|(t, _)| t).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + use mlm_db::{MediaType, Size, TorrentMeta, Timestamp, MetadataSource, MainCat, Language}; + use crate::config::SearchConfig; + + fn create_test_torrent(id: &str, title: &str, filetypes: Vec, size_bytes: u64) -> Torrent { + let meta = TorrentMeta { + title: title.to_string(), + filetypes, + size: Size::from_bytes(size_bytes), + media_type: MediaType::Audiobook, + main_cat: Some(MainCat::Fiction), + source: MetadataSource::Mam, + uploaded_at: Timestamp::now(), + authors: vec!["Author".to_string()], + language: Some(Language::English), + ids: BTreeMap::new(), + vip_status: None, + cat: None, + categories: vec![], + tags: vec![], + flags: None, + num_files: 1, + edition: None, + description: "".to_string(), + narrators: vec![], + series: vec![], + }; + Torrent { + id: id.to_string(), + id_is_hash: false, + mam_id: None, + library_path: None, + library_files: vec![], + linker: None, + category: None, + selected_audio_format: None, + selected_ebook_format: None, + title_search: mlm_parse::normalize_title(title), + meta, + created_at: Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + } + } + + fn create_test_config() -> Config { + Config { + mam_id: "test".to_string(), + web_host: "0.0.0.0".to_string(), + web_port: 3157, + min_ratio: 2.0, + unsat_buffer: 10, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 30, + link_interval: 10, + import_interval: 135, + ignore_torrents: vec![], + audio_types: vec!["m4b".to_string(), "mp3".to_string()], + ebook_types: vec!["epub".to_string(), "pdf".to_string()], + music_types: vec!["flac".to_string(), "mp3".to_string()], + radio_types: vec!["mp3".to_string()], + search: SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![], + } + } + + #[test] + fn test_rank_torrents_preference() { + let config = create_test_config(); + + let t1 = create_test_torrent("1", "Title", vec!["mp3".to_string()], 100); + let t2 = create_test_torrent("2", "Title", vec!["m4b".to_string()], 100); + + let batch = vec![t1.clone(), t2.clone()]; + let ranked = rank_torrents(&config, batch); + + assert_eq!(ranked[0].id, "2"); // m4b is preferred over mp3 + assert_eq!(ranked[1].id, "1"); + } + + #[test] + fn test_rank_torrents_size_tie_break() { + let config = create_test_config(); + + let t1 = create_test_torrent("1", "Title", vec!["m4b".to_string()], 100); + let t2 = create_test_torrent("2", "Title", vec!["m4b".to_string()], 200); + + let batch = vec![t1.clone(), t2.clone()]; + let ranked = rank_torrents(&config, batch); + + assert_eq!(ranked[0].id, "2"); // Larger size wins tie + assert_eq!(ranked[1].id, "1"); + } + + #[tokio::test] + async fn test_find_matches() -> Result<()> { + let tmp_dir = std::env::temp_dir().join(format!("mlm_test_duplicates_{}", std::process::id())); + let _ = fs::remove_dir_all(&tmp_dir); + fs::create_dir_all(&tmp_dir)?; + let db_path = tmp_dir.join("test.db"); + + let db = native_db::Builder::new().create(&mlm_db::MODELS, &db_path)?; + mlm_db::migrate(&db)?; + + let t1 = create_test_torrent("1", "My Book", vec!["m4b".to_string()], 100); + let t2 = create_test_torrent("2", "My Book", vec!["mp3".to_string()], 150); + let t3 = create_test_torrent("3", "Other Book", vec!["m4b".to_string()], 100); + + { + let rw = db.rw_transaction()?; + rw.insert(t1.clone())?; + rw.insert(t2.clone())?; + rw.insert(t3.clone())?; + rw.commit()?; + } + + let matches = find_matches(&db, &t1)?; + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].id, "2"); + + drop(db); + let _ = fs::remove_dir_all(tmp_dir); + Ok(()) + } +} + diff --git a/server/src/linker/folder.rs b/server/src/linker/folder.rs new file mode 100644 index 00000000..20296008 --- /dev/null +++ b/server/src/linker/folder.rs @@ -0,0 +1,628 @@ +use std::{ + collections::BTreeMap, + fs::File, + io::{BufWriter, Write as _}, + path::PathBuf, + str::FromStr as _, + sync::Arc, +}; + +use anyhow::{Result, bail}; +use mlm_db::{ + DatabaseExt as _, ErroredTorrentId, Event, EventType, FlagBits, Flags, Language, MediaType, + MetadataSource, Series, SeriesEntries, SeriesEntry, Size, Timestamp, Torrent, TorrentMeta, ids, +}; +use mlm_mam::meta::clean_meta; +use mlm_parse::{normalize_title, parse_series_from_title}; +use native_db::Database; +use serde_derive::{Deserialize, Serialize}; +use tokio::fs::{DirEntry, create_dir_all, read_dir, read_to_string}; +use tracing::{Level, instrument, span, trace, warn}; + +use crate::audiobookshelf as abs; +use crate::config::{Config, Library, LibraryLinkMethod}; +use crate::linker::{ + copy, file_size, find_matches, hard_link, libation_cats, library_dir, rank_torrents, + select_format, symlink, +}; +use crate::logging::{update_errored_torrent, write_event}; + +#[instrument(skip_all)] +pub async fn link_folders_to_library(config: Arc, db: Arc>) -> Result<()> { + for library in &config.libraries { + if let Library::ByRipDir(l) = library { + let mut entries = read_dir(&l.rip_dir).await?; + while let Some(folder) = entries.next_entry().await? { + link_folder(&config, library, &db, folder).await?; + } + } + } + + Ok(()) +} + +async fn link_folder( + config: &Config, + library: &Library, + db: &Database<'_>, + folder: DirEntry, +) -> Result<()> { + let span = span!( + Level::TRACE, + "link_folder", + folder = folder.path().to_string_lossy().to_string(), + ); + let _s = span.enter(); + let mut entries = read_dir(&folder.path()).await?; + + let mut audio_files = vec![]; + let mut ebook_files = vec![]; + let mut metadata_files = vec![]; + // let mut cover_file = None; + while let Some(entry) = entries.next_entry().await? { + match entry.path().extension() { + Some(ext) if ext == "json" => metadata_files.push(entry), + // Some(ext) if ext == "jpg" || ext == "png" => cover_file = Some(entry), + Some(ext) + if config + .audio_types + .iter() + .any(|e| e == &ext.to_string_lossy()) => + { + audio_files.push(entry) + } + Some(ext) + if config + .ebook_types + .iter() + .any(|e| e == &ext.to_string_lossy()) => + { + ebook_files.push(entry) + } + _ => {} + } + } + + if metadata_files.is_empty() { + warn!("Missing metadata file"); + return Ok(()); + } + + metadata_files.sort_by_key(|entry| entry.file_name()); + for metadata_file in metadata_files { + let json = read_to_string(metadata_file.path()).await?; + if let Ok(libation_meta) = serde_json::from_str::(&json) { + trace!("Linking libation folder"); + let asin = libation_meta.asin.clone(); + let title = libation_meta.title.clone(); + let result = + link_libation_folder(config, library, db, libation_meta, audio_files, ebook_files) + .await; + update_errored_torrent(db, ErroredTorrentId::Linker(asin), title, result).await; + return Ok(()); + } + if let Some(nextory_meta) = parse_nextory_meta(&json) { + trace!("Linking nextory folder"); + let id = nextory_torrent_id(nextory_meta.id); + let title = nextory_meta.title.clone(); + let result = + link_nextory_folder(config, library, db, nextory_meta, audio_files, ebook_files) + .await; + update_errored_torrent(db, ErroredTorrentId::Linker(id), title, result).await; + return Ok(()); + } + } + + warn!( + folder = folder.path().to_string_lossy().to_string(), + "Unsupported metadata format" + ); + + Ok(()) +} + +async fn link_libation_folder( + config: &Config, + library: &Library, + db: &Database<'_>, + libation_meta: Libation, + audio_files: Vec, + ebook_files: Vec, +) -> Result<()> { + let torrent = + build_libation_torrent(library, libation_meta, &audio_files, &ebook_files).await?; + link_prepared_folder_torrent(config, library, db, torrent, audio_files, ebook_files).await +} + +async fn link_nextory_folder( + config: &Config, + library: &Library, + db: &Database<'_>, + nextory_meta: NextoryRaw, + audio_files: Vec, + ebook_files: Vec, +) -> Result<()> { + let torrent = build_nextory_torrent(library, nextory_meta, &audio_files, &ebook_files).await?; + link_prepared_folder_torrent(config, library, db, torrent, audio_files, ebook_files).await +} + +async fn build_libation_torrent( + library: &Library, + libation_meta: Libation, + audio_files: &[DirEntry], + ebook_files: &[DirEntry], +) -> Result { + let mut title = if libation_meta.subtitle.is_empty() { + libation_meta.title.clone() + } else { + format!("{}: {}", libation_meta.title, libation_meta.subtitle) + }; + + let mut inferred_series = None; + if let Some((subtitle_series_name, subtitle_series_num)) = + parse_libation_series_subtitle(&libation_meta.subtitle) + { + if let Some((title_prefix, title_rest)) = libation_meta.title.split_once(": ") + && libation_series_name_matches(title_prefix, &subtitle_series_name) + && !title_rest.trim().is_empty() + { + title = title_rest.trim().to_string(); + if let Ok(series) = + Series::try_from((title_prefix.trim().to_string(), subtitle_series_num)) + { + inferred_series = Some(series); + } + } else { + title = libation_meta.title.clone(); + if let Ok(series) = Series::try_from((subtitle_series_name, subtitle_series_num)) { + inferred_series = Some(series); + } + } + } + + let mut series = libation_meta + .series + .into_iter() + .filter_map(|s| Series::try_from((s.title, s.sequence)).ok()) + .collect::>(); + if series.is_empty() + && let Some(inferred_series) = inferred_series + { + series.push(inferred_series); + } + if series.is_empty() + && let Some((name, num)) = parse_series_from_title(&title) + { + series.push(Series { + name: name.to_string(), + entries: SeriesEntries::new(num.into_iter().map(SeriesEntry::Num).collect()), + }); + } + + let mut ids = BTreeMap::new(); + ids.insert(ids::ASIN.to_string(), libation_meta.asin.clone()); + let mut flags = Flags::default(); + if libation_meta.format_type.starts_with("abridged") { + flags.abridged = Some(true); + } + let mapped_categories = libation_cats::map_category_ladders(&libation_meta.category_ladders); + let (size, filetypes) = folder_file_stats(audio_files, ebook_files).await?; + + let meta = TorrentMeta { + ids, + vip_status: None, + cat: None, + media_type: MediaType::Audiobook, + main_cat: None, + categories: mapped_categories.categories, + tags: mapped_categories.freeform_tags, + language: Language::from_str(&libation_meta.language).ok(), + flags: Some(FlagBits::new(flags.as_bitfield())), + filetypes, + num_files: audio_files.len() as u64, + size: Size::from_bytes(size), + title, + edition: None, + description: libation_meta.publisher_summary, + authors: libation_meta.authors.into_iter().map(|a| a.name).collect(), + narrators: libation_meta + .narrators + .into_iter() + .map(|a| a.name) + .collect(), + series, + source: MetadataSource::File, + uploaded_at: None, + }; + build_torrent(library, libation_meta.asin, clean_meta(meta, "")?) +} + +async fn build_nextory_torrent( + library: &Library, + nextory_meta: NextoryRaw, + audio_files: &[DirEntry], + ebook_files: &[DirEntry], +) -> Result { + let mut series = vec![]; + if let Some(raw_series) = nextory_meta.series + && !raw_series.name.is_empty() + { + let sequence = nextory_meta + .volume + .map(|v| v.to_string()) + .unwrap_or_default(); + if let Ok(parsed) = Series::try_from((raw_series.name, sequence)) { + series.push(parsed); + } + } + if series.is_empty() + && let Some((name, num)) = parse_series_from_title(&nextory_meta.title) + { + series.push(Series { + name: name.to_string(), + entries: SeriesEntries::new(num.into_iter().map(SeriesEntry::Num).collect()), + }); + } + + let mut ids = BTreeMap::new(); + ids.insert(ids::NEXTORY.to_string(), nextory_meta.id.to_string()); + if let Some(isbn) = nextory_isbn(&nextory_meta.formats) { + ids.insert(ids::ISBN.to_string(), isbn); + } + let (size, filetypes) = folder_file_stats(audio_files, ebook_files).await?; + + let description = if nextory_meta.description_full.is_empty() { + nextory_meta.blurb + } else { + nextory_meta.description_full + }; + let meta = TorrentMeta { + ids, + vip_status: None, + cat: None, + media_type: MediaType::Audiobook, + main_cat: None, + categories: vec![], + tags: vec![], + language: parse_nextory_language(&nextory_meta.language), + flags: None, + filetypes, + num_files: audio_files.len() as u64, + size: Size::from_bytes(size), + title: nextory_meta.title, + edition: None, + description, + authors: nextory_meta.authors.into_iter().map(|a| a.name).collect(), + narrators: nextory_meta.narrators.into_iter().map(|n| n.name).collect(), + series, + source: MetadataSource::File, + uploaded_at: None, + }; + build_torrent( + library, + nextory_torrent_id(nextory_meta.id), + clean_meta(meta, "")?, + ) +} + +async fn folder_file_stats( + audio_files: &[DirEntry], + ebook_files: &[DirEntry], +) -> Result<(u64, Vec)> { + let mut size = 0; + let mut filetypes = vec![]; + for file in audio_files { + size += file_size(&file.metadata().await?); + if let Some(ext) = file.path().extension() { + filetypes.push(ext.to_string_lossy().to_lowercase()); + } + } + for file in ebook_files { + size += file_size(&file.metadata().await?); + if let Some(ext) = file.path().extension() { + filetypes.push(ext.to_string_lossy().to_lowercase()); + } + } + filetypes.sort(); + filetypes.dedup(); + Ok((size, filetypes)) +} + +fn build_torrent(library: &Library, id: String, meta: TorrentMeta) -> Result { + Ok(Torrent { + id, + id_is_hash: false, + mam_id: meta.mam_id(), + library_path: None, + library_files: vec![], + linker: library.options().name.clone(), + category: None, + selected_audio_format: None, + selected_ebook_format: None, + title_search: normalize_title(&meta.title), + meta, + created_at: Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + }) +} + +async fn link_prepared_folder_torrent( + config: &Config, + library: &Library, + db: &Database<'_>, + mut torrent: Torrent, + audio_files: Vec, + ebook_files: Vec, +) -> Result<()> { + let torrent_id = torrent.id.clone(); + let r = db.r_transaction()?; + let existing_torrent: Option = r.get().primary(torrent_id.clone())?; + if existing_torrent.is_some() { + return Ok(()); + } + + let matches = find_matches(db, &torrent)?; + if !matches.is_empty() { + let mut batch = matches; + batch.push(torrent.clone()); + let ranked = rank_torrents(config, batch); + if ranked[0].id != torrent.id { + trace!("Skipping folder as it is a duplicate of a better torrent already in library"); + return Ok(()); + } + } + + if let Some(filter) = library.edition_filter() + && !filter + .matches_meta(&torrent.meta) + .is_ok_and(|matches| matches) + { + trace!("Skipping folder due to edition filter"); + return Ok(()); + } + + let mut library_files = vec![]; + let selected_audio_format = select_format( + &library.options().audio_types, + &config.audio_types, + &audio_files, + ); + let selected_ebook_format = select_format( + &library.options().ebook_types, + &config.ebook_types, + &ebook_files, + ); + + let library_path = if library.options().method != LibraryLinkMethod::NoLink { + let Some(mut dir) = library_dir( + config.exclude_narrator_in_library_dir, + library, + &torrent.meta, + ) else { + bail!("Torrent has no author"); + }; + if config.exclude_narrator_in_library_dir + && !torrent.meta.narrators.is_empty() + && dir.exists() + { + dir = library_dir(false, library, &torrent.meta).unwrap(); + } + let metadata = abs::create_metadata(&torrent.meta); + + create_dir_all(&dir).await?; + for file in audio_files { + let file_path = PathBuf::from(&file.file_name()); + let library_path = dir.join(&file_path); + library_files.push(file_path.clone()); + let download_path = file.path(); + match library.options().method { + LibraryLinkMethod::Hardlink => { + hard_link(&download_path, &library_path, &file_path)? + } + LibraryLinkMethod::HardlinkOrCopy => { + hard_link(&download_path, &library_path, &file_path) + .or_else(|_| copy(&download_path, &library_path))? + } + LibraryLinkMethod::Copy => copy(&download_path, &library_path)?, + LibraryLinkMethod::HardlinkOrSymlink => { + hard_link(&download_path, &library_path, &file_path) + .or_else(|_| symlink(&download_path, &library_path))? + } + LibraryLinkMethod::Symlink => symlink(&download_path, &library_path)?, + LibraryLinkMethod::NoLink => {} + }; + } + library_files.sort(); + + let file = File::create(dir.join("metadata.json"))?; + let mut writer = BufWriter::new(file); + serde_json::to_writer(&mut writer, &metadata)?; + writer.flush()?; + Some(dir.clone()) + } else { + None + }; + + { + let (_guard, rw) = db.rw_async().await?; + torrent.library_path = library_path.clone(); + torrent.library_files = library_files; + torrent.selected_audio_format = selected_audio_format; + torrent.selected_ebook_format = selected_ebook_format; + rw.upsert(torrent)?; + rw.commit()?; + } + + if let Some(library_path) = library_path { + write_event( + db, + Event::new( + Some(torrent_id), + None, + EventType::Linked { + linker: library.options().name.clone(), + library_path, + }, + ), + ) + .await; + } + + Ok(()) +} + +fn parse_nextory_meta(json: &str) -> Option { + if let Ok(meta) = serde_json::from_str::(json) { + return Some(meta.raw); + } + serde_json::from_str::(json).ok() +} + +fn parse_libation_series_subtitle(subtitle: &str) -> Option<(String, String)> { + let subtitle = subtitle.trim(); + if subtitle.is_empty() { + return None; + } + let subtitle_lower = subtitle.to_lowercase(); + + for marker in [", book ", ", vol. ", ", volume "] { + if let Some(index) = subtitle_lower.find(marker) { + let series_name = subtitle[..index].trim(); + let sequence = subtitle[index + marker.len()..].trim(); + if !series_name.is_empty() && !sequence.is_empty() { + return Some((series_name.to_string(), sequence.to_string())); + } + } + } + None +} + +fn libation_series_name_matches(title_prefix: &str, subtitle_series_name: &str) -> bool { + let title_prefix = normalize_title(title_prefix); + let subtitle_series_name = normalize_title( + subtitle_series_name + .trim() + .strip_suffix(" Series") + .or_else(|| subtitle_series_name.trim().strip_suffix(" series")) + .unwrap_or(subtitle_series_name.trim()), + ); + title_prefix == subtitle_series_name +} + +fn nextory_torrent_id(nextory_id: u64) -> String { + format!("nextory_{nextory_id}") +} + +fn nextory_isbn(formats: &[NextoryFormat]) -> Option { + formats + .iter() + .find(|f| f.format_type == "hls") + .or_else(|| formats.first()) + .and_then(|f| f.isbn.clone()) +} + +fn parse_nextory_language(value: &str) -> Option { + if let Ok(language) = Language::from_str(value) { + return Some(language); + } + match value.to_lowercase().as_str() { + "sv" | "swe" => Some(Language::Swedish), + "en" | "eng" => Some(Language::English), + _ => None, + } +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Libation { + pub asin: String, + pub authors: Vec, + pub category_ladders: Vec, + pub format_type: String, + pub is_adult_product: bool, + pub issue_date: String, + pub language: String, + pub merchandising_summary: String, + #[serde(default)] + pub narrators: Vec, + pub publication_datetime: String, + #[serde(default)] + pub publication_name: String, + pub publisher_name: String, + pub publisher_summary: String, + pub release_date: String, + pub runtime_length_min: u64, + #[serde(default)] + pub series: Vec, + #[serde(default)] + pub subtitle: String, + pub title: String, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CategoryLadder { + pub ladder: Vec, + pub root: String, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Ladder { + pub id: String, + pub name: String, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Name { + pub name: String, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LibationSeries { + pub sequence: String, + pub title: String, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NextoryWrapped { + pub raw: NextoryRaw, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NextoryRaw { + pub id: u64, + pub title: String, + #[serde(default)] + pub blurb: String, + #[serde(default)] + pub description_full: String, + pub language: String, + #[serde(default)] + pub volume: Option, + #[serde(default)] + pub series: Option, + #[serde(default)] + pub formats: Vec, + #[serde(default)] + pub authors: Vec, + #[serde(default)] + pub narrators: Vec, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NextorySeries { + pub name: String, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NextoryFormat { + #[serde(rename = "type")] + pub format_type: String, + #[serde(default)] + pub isbn: Option, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NextoryName { + pub name: String, +} diff --git a/server/src/linker/libation_cats.rs b/server/src/linker/libation_cats.rs new file mode 100644 index 00000000..f058206f --- /dev/null +++ b/server/src/linker/libation_cats.rs @@ -0,0 +1,678 @@ +use std::collections::BTreeSet; + +use mlm_db::Category; + +use super::folder::CategoryLadder; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct CategoryMapping { + pub categories: Vec, + pub freeform_tags: Vec, +} + +impl CategoryMapping { + pub fn is_empty(&self) -> bool { + self.categories.is_empty() && self.freeform_tags.is_empty() + } + + fn push_category(&mut self, category: Category) { + if !self.categories.contains(&category) { + self.categories.push(category); + } + } + + fn push_tag(&mut self, tag: &str) { + if !self.freeform_tags.iter().any(|existing| existing == tag) { + self.freeform_tags.push(tag.to_string()); + } + } + + fn extend(&mut self, other: CategoryMapping) { + for category in other.categories { + self.push_category(category); + } + for tag in other.freeform_tags { + self.push_tag(&tag); + } + } +} + +fn mapped(categories: &[Category], tags: &[&str]) -> CategoryMapping { + let mut out = CategoryMapping::default(); + for category in categories { + out.push_category(*category); + } + for tag in tags { + out.push_tag(tag); + } + out +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum MappingDepth { + ExactFullPath, + FallbackTwoLevel, + FallbackTopLevel, + Unmapped, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LadderMatch { + pub original_path: Vec, + pub matched_path: Vec, + pub depth: MappingDepth, + pub mapping: CategoryMapping, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct AggregateCategoryResult { + pub categories: Vec, + pub freeform_tags: Vec, + pub ladder_matches: Vec, + pub unmapped_paths: Vec>, +} + +pub fn map_audible_es_path_exact(path: &[&str]) -> CategoryMapping { + use Category::*; + + match path { + [ + "Literatura y ficción", + "Literatura de género", + "Coming of age", + ] => mapped(&[ComingOfAge], &["Genre Fiction"]), + _ => CategoryMapping::default(), + } +} + +pub fn map_audible_es_path(path: &[&str]) -> CategoryMapping { + use Category::*; + + match path { + [ + "Literatura y ficción", + "Literatura de género", + "Coming of age", + ] => mapped(&[ComingOfAge], &["Genre Fiction"]), + ["Literatura y ficción", "Clásicos"] => mapped(&[], &["Classics"]), + ["Policíaca, negra y suspense", "Novela negra"] => mapped(&[Crime, Noir], &[]), + + ["Adolescentes"] => mapped(&[YoungAdult], &[]), + ["Adolescentes", leaf] => { + let mut out = mapped(&[YoungAdult], &[]); + match *leaf { + "Biografías" => out.push_category(Biography), + "Ciencia ficción y fantasía" => out.push_tag("Science Fiction & Fantasy"), + "Deportes y aire libre" => out.push_category(SportsOutdoors), + "LGBTQ+" => out.push_category(Lgbtqia), + "Literatura y ficción" => out.push_tag("Literature & Fiction"), + "Policíaca, negra y suspense" => out.push_tag("Crime, Noir & Suspense"), + "Romántica" => out.push_category(Romance), + "Salud, estilo de vida y relaciones" => { + out.push_category(HealthWellness); + out.push_tag("Lifestyle & Relationships"); + } + _ => {} + } + out + } + + ["Arte y entretenimiento"] => mapped(&[], &["Arts & Entertainment"]), + ["Arte y entretenimiento", leaf] => match *leaf { + "Arte" => mapped(&[ArtPhotography], &[]), + "Audiciones y dramatizaciones" => mapped(&[], &["Auditions & Dramatizations"]), + "Entretenimiento y artes escénicas" => { + mapped(&[], &["Entertainment & Performing Arts"]) + } + "Música" => mapped(&[Music], &[]), + _ => CategoryMapping::default(), + }, + + ["Audiolibros infantiles"] => mapped(&[Children, Audiobook], &[]), + ["Audiolibros infantiles", leaf] => { + let mut out = mapped(&[Children, Audiobook], &[]); + match *leaf { + "Acción y aventura" => out.push_category(ActionAdventure), + "Actividades y aficiones" => out.push_tag("Activities & Hobbies"), + "Animales y naturaleza" => out.push_category(NatureEnvironment), + "Biografías" => out.push_category(Biography), + "Ciencia ficción y fantasía" => out.push_tag("Science Fiction & Fantasy"), + "Ciencia y tecnología" => out.push_tag("Science & Technology"), + "Crecer y cosas de la vida" => out.push_tag("Growing Up & Life"), + "Cuentos y leyendas" => out.push_tag("Stories & Legends"), + "Deportes y aire libre" => out.push_category(SportsOutdoors), + "Educación y formación" => out.push_tag("Education & Learning"), + "Fiestas y celebraciones" => out.push_tag("Holidays & Celebrations"), + "Geografía y culturas" => out.push_tag("Geography & Cultures"), + "Historia" => out.push_category(History), + "Humor" => out.push_category(Funny), + "Literatura y ficción" => out.push_tag("Literature & Fiction"), + "Misterio y suspense" => { + out.push_category(Mystery); + out.push_tag("Mystery & Suspense"); + } + "Música y artes escénicas" => { + out.push_category(Music); + out.push_tag("Performing Arts"); + } + "Religión" => out.push_category(ReligionSpirituality), + "Vehículos y transporte" => out.push_tag("Vehicles & Transportation"), + _ => {} + } + out + } + + ["Biografías y memorias"] => mapped(&[Biography, Memoir], &[]), + ["Biografías y memorias", leaf] => { + let mut out = mapped(&[Biography, Memoir], &[]); + match *leaf { + "Arte y literatura" => out.push_tag("Art & Literature"), + "Aventureros, exploradores y supervivencia" => { + out.push_tag("Adventurers, Explorers & Survival"); + } + "Celebridades del entretenimiento" => { + out.push_tag("Entertainment Celebrities"); + } + "Crímenes reales" => out.push_category(TrueCrime), + "Culturales y regionales" => out.push_tag("Cultural & Regional"), + "Deportes" => out.push_category(SportsOutdoors), + "Ejército y guerra" => out.push_category(Military), + "Histórico" => out.push_category(History), + "LGBT" => out.push_category(Lgbtqia), + "Mujeres" => out.push_tag("Women"), + "Política y activismo" => { + out.push_category(PoliticsSociety); + out.push_tag("Activism"); + } + "Profesionales y académicos" => out.push_tag("Professionals & Academics"), + "Religiones" => out.push_category(ReligionSpirituality), + _ => {} + } + out + } + + ["Ciencia e ingeniería"] => mapped(&[], &["Science & Engineering"]), + ["Ciencia e ingeniería", "Ciencia"] => mapped(&[Science], &[]), + ["Ciencia e ingeniería", "Ingeniería"] => mapped(&[Engineering], &[]), + + ["Ciencia ficción y fantasía"] => mapped(&[], &["Science Fiction & Fantasy"]), + ["Ciencia ficción y fantasía", "Ciencia ficción"] => mapped(&[ScienceFiction], &[]), + ["Ciencia ficción y fantasía", "Fantasía"] => mapped(&[Fantasy], &[]), + + ["Comedia y humor"] => mapped(&[], &["Comedy & Humor"]), + ["Comedia y humor", leaf] => match *leaf { + "Artes escénicas" => mapped(&[Humor], &["Performing Arts Comedy"]), + "Literatura y ficción" => mapped(&[Funny], &["Comedy & Humor"]), + _ => CategoryMapping::default(), + }, + + ["Deportes y aire libre"] => mapped(&[SportsOutdoors], &[]), + ["Deportes y aire libre", leaf] => { + let mut out = mapped(&[SportsOutdoors], &[]); + match *leaf { + "Aire libre y naturaleza" => out.push_category(NatureEnvironment), + "Aventureros, exploradores y supervivencia" => { + out.push_tag("Adventurers, Explorers & Survival"); + } + "Baloncesto" => out.push_tag("Basketball"), + "Biografías y memorias" => { + out.push_category(Biography); + out.push_category(Memoir); + } + "Béisbol y softbol" => out.push_tag("Baseball & Softball"), + "Culturismo y entrenamiento muscular" => { + out.push_category(HealthWellness); + out.push_tag("Bodybuilding & Strength Training"); + } + "Ensayos y comentarios" => out.push_category(Essays), + "Entrenamiento" => { + out.push_category(GuideManual); + out.push_category(HealthWellness); + } + "Fútbol" => out.push_tag("Soccer"), + "Fútbol americano" => out.push_tag("American Football"), + "Historia del deporte" => out.push_category(History), + "Juegos Olímpicos y Paralímpicos" => out.push_tag("Olympics & Paralympics"), + _ => {} + } + out + } + + ["Dinero y finanzas"] => mapped(&[], &["Money & Finance"]), + ["Dinero y finanzas", leaf] => match *leaf { + "Comercio electrónico" => mapped(&[Business, Technology], &["E-Commerce"]), + "Economía" => mapped(&[Business], &["Economics"]), + "Finanzas personales" => mapped(&[PersonalFinance], &[]), + "Internacional" => mapped(&[], &["International Finance"]), + "Inversiones y valores" => mapped(&[PersonalFinance], &["Investing & Securities"]), + _ => CategoryMapping::default(), + }, + + ["Educación y formación"] => mapped(&[], &["Education & Learning"]), + ["Educación y formación", leaf] => match *leaf { + "Aprendizaje de idiomas" => mapped(&[LanguageLinguistics], &[]), + "Educación" => mapped(&[], &["Education"]), + "Lengua y gramática" => mapped(&[LanguageLinguistics], &[]), + "Redacción y publicación" => mapped( + &[LanguageLinguistics, GuideManual], + &["Writing & Publishing"], + ), + _ => CategoryMapping::default(), + }, + + ["Erótica"] => mapped(&[Erotica], &[]), + ["Erótica", leaf] => match *leaf { + "Educación sexual" => mapped(&[HealthWellness], &["Sex Education"]), + "Literatura y ficción" => mapped(&[Romance, Erotica], &[]), + _ => CategoryMapping::default(), + }, + + ["Historia"] => mapped(&[History], &[]), + ["Historia", leaf] => { + let mut out = mapped(&[History], &[]); + match *leaf { + "América" => out.push_tag("Americas"), + "Antigua" => out.push_category(Ancient), + "Asia" => out.push_tag("Asia"), + "Era moderna" => out.push_category(EarlyModern), + "Europa" => out.push_category(Europe), + "LGBTQ+" => out.push_category(Lgbtqia), + "Militar" => out.push_category(Military), + "Mundial" => out.push_tag("World History"), + "Rusia" => { + out.push_category(Europe); + out.push_tag("Russia"); + } + _ => {} + } + out + } + + ["Hogar y jardín"] => mapped(&[HomeGarden], &[]), + ["Hogar y jardín", leaf] => match *leaf { + "Casa y hogar" => mapped(&[HomeGarden], &[]), + "Comida y vino" => mapped(&[CookingFood], &[]), + "Jardinería y horticultura" => mapped(&[HomeGarden], &["Gardening & Horticulture"]), + "Mascotas y cuidado de animales" => mapped(&[HomeGarden], &["Pets & Animal Care"]), + "Vida sostenible y ecológica" => { + mapped(&[HomeGarden, NatureEnvironment], &["Sustainable Living"]) + } + _ => CategoryMapping::default(), + }, + + ["Informática y tecnología"] => mapped(&[Technology], &[]), + ["Informática y tecnología", leaf] => match *leaf { + "Creación de contenido y redes sociales" => { + mapped(&[Technology], &["Content Creation & Social Media"]) + } + "Historia y cultura" => mapped(&[Technology], &["Technology History & Culture"]), + _ => CategoryMapping::default(), + }, + + ["LGBTQ+"] => mapped(&[Lgbtqia], &[]), + ["LGBTQ+", leaf] => { + let mut out = mapped(&[Lgbtqia], &[]); + match *leaf { + "Biografías y memorias" => { + out.push_category(Biography); + out.push_category(Memoir); + } + "Ciencia ficción y fantasía" => out.push_tag("Science Fiction & Fantasy"), + "Estudios sobre LGBTQ+" => { + out.push_category(PoliticsSociety); + out.push_tag("LGBTQ+ Studies"); + } + "Historia" => out.push_category(History), + "Literatura y ficción" => {} + "Misterio, negra y suspense" => out.push_tag("Mystery, Crime & Suspense"), + "Romántica" => out.push_category(Romance), + _ => {} + } + out + } + + ["Literatura y ficción"] => mapped(&[], &["Literature & Fiction"]), + ["Literatura y ficción", leaf] => match *leaf { + "Acción y aventura" => mapped(&[ActionAdventure], &[]), + "Afroamericana" => mapped(&[PocRepresentation], &["African American"]), + "Antologías y relatos breves" => mapped(&[Anthology, ShortStories], &[]), + "Clásicos" => mapped(&[], &["Classics"]), + "Drama y teatro" => mapped(&[DramaPlays], &[]), + "Ensayos" => mapped(&[Essays], &[]), + "Erótica" => mapped(&[Romance, Erotica], &[]), + "Humor y sátira" => mapped(&[Funny, Satire], &[]), + "LGBT" => mapped(&[Lgbtqia], &[]), + "Literatura antigua, clásica y medieval" => { + mapped(&[Ancient, Medieval], &["Classical Literature"]) + } + "Literatura de género" => mapped(&[], &["Genre Fiction"]), + "Narrativa femenina" => mapped(&[], &["Women's Fiction"]), + "Novela histórica" => mapped(&[Historical], &[]), + "Poesía" => mapped(&[Poetry], &[]), + "Terror" => mapped(&[Horror], &[]), + _ => CategoryMapping::default(), + }, + + ["Negocios y profesiones"] => mapped(&[Business], &[]), + ["Negocios y profesiones", leaf] => { + let mut out = mapped(&[Business], &[]); + match *leaf { + "Comportamiento organizacional y en el lugar de trabajo" => { + out.push_tag("Workplace & Organizational Behavior"); + } + "Desarrollo empresarial y emprendimiento" => {} + "Gestión y liderazgo" => out.push_tag("Management & Leadership"), + "Marketing y ventas" => out.push_tag("Marketing & Sales"), + "Mujeres en los negocios" => out.push_tag("Women in Business"), + "Éxito profesional" => { + out.push_category(SelfHelp); + out.push_tag("Career Success"); + } + _ => {} + } + out + } + + ["Policíaca, negra y suspense"] => mapped(&[], &["Crime, Noir & Suspense"]), + ["Policíaca, negra y suspense", leaf] => match *leaf { + "Crímenes reales" => mapped(&[TrueCrime], &[]), + "Misterio" => mapped(&[Mystery], &[]), + "Negra y suspense" => mapped(&[Crime, Noir, Thriller], &[]), + "Novela negra" => mapped(&[Crime, Noir], &[]), + _ => CategoryMapping::default(), + }, + + ["Política y ciencias sociales"] => mapped(&[PoliticsSociety], &[]), + ["Política y ciencias sociales", leaf] => { + let mut out = mapped(&[PoliticsSociety], &[]); + match *leaf { + "Antropología" => out.push_tag("Anthropology"), + "Ciencias sociales" => {} + "Legislación" => out.push_tag("Law"), + "Política y gobierno" => {} + "Sociología" => {} + _ => {} + } + out + } + + ["Relaciones, crianza y desarrollo personal"] => { + mapped(&[], &["Relationships, Parenting & Personal Development"]) + } + ["Relaciones, crianza y desarrollo personal", leaf] => match *leaf { + "Crianza y familia" => mapped(&[ParentingFamily], &[]), + "Desarrollo personal" => mapped(&[SelfHelp], &[]), + _ => CategoryMapping::default(), + }, + + ["Religión y espiritualidad"] => mapped(&[ReligionSpirituality], &[]), + ["Religión y espiritualidad", leaf] => { + let mut out = mapped(&[ReligionSpirituality], &[]); + match *leaf { + "Budismo" => out.push_tag("Buddhism"), + "Cristiandad" => out.push_tag("Christianity"), + "Espiritualidad" => {} + "Estudios religiosos" => out.push_tag("Religious Studies"), + "Hinduismo" => out.push_tag("Hinduism"), + "Islam" => out.push_tag("Islam"), + "Judaísmo" => out.push_tag("Judaism"), + "Ocultismo" => out.push_category(OccultEsotericism), + "Otras religiones, prácticas y textos" => { + out.push_tag("Other Religions, Practices & Texts"); + } + _ => {} + } + out + } + + ["Romántica"] => mapped(&[Romance], &[]), + ["Romántica", leaf] => { + let mut out = mapped(&[Romance], &[]); + match *leaf { + "Acción y aventura" => out.push_category(ActionAdventure), + "Antologías y relatos breves" => { + out.push_category(Anthology); + out.push_category(ShortStories); + } + "Ciencia ficción" => out.push_category(ScienceFiction), + "Comedia romántica" => out.push_category(RomanticComedy), + "Contemporánea" => out.push_category(Contemporary), + "Cortejo" => out.push_tag("Courtship"), + "Fantástico" => out.push_category(Fantasy), + "Histórico" => out.push_category(Historical), + "LGBT" => out.push_category(Lgbtqia), + "Militar" => out.push_category(Military), + "Multicultural" => { + out.push_category(PocRepresentation); + out.push_tag("Multicultural"); + } + "Oeste americano" => out.push_category(Western), + "Suspense romántico" => out.push_category(RomanticSuspense), + _ => {} + } + out + } + + ["Salud y bienestar"] => mapped(&[HealthWellness], &[]), + ["Salud y bienestar", leaf] => { + let mut out = mapped(&[HealthWellness], &[]); + match *leaf { + "Adicción y recuperación" => { + out.push_category(SelfHelp); + out.push_tag("Addiction & Recovery"); + } + "Ejercicio, dieta y nutrición" => {} + "Enfermedad física y trastornos" => { + out.push_category(Medicine); + out.push_tag("Physical Illness & Disorders"); + } + "Envejecimiento y longevidad" => out.push_tag("Aging & Longevity"), + "Higiene y vida sana" => {} + "Medicina y sector de la salud" => out.push_category(Medicine), + "Psicología y salud mental" => out.push_category(Psychology), + "Salud sexual y reproductiva" => { + out.push_tag("Sexual & Reproductive Health"); + } + _ => {} + } + out + } + + ["Viajes y turismo"] => mapped(&[Travel], &[]), + ["Viajes y turismo", leaf] => { + let mut out = mapped(&[Travel], &[]); + match *leaf { + "Asia" => out.push_tag("Asia"), + "Europa" => out.push_category(Europe), + "Oriente Medio" => out.push_category(MiddleEast), + "Reportajes y artículos" => { + out.push_category(Essays); + out.push_tag("Travel Writing & Journalism"); + } + "Viajes de aventura" => { + out.push_category(ActionAdventure); + out.push_tag("Adventure Travel"); + } + "Visitas guiadas" => out.push_category(GuideManual), + "África" => out.push_category(Africa), + _ => {} + } + out + } + + _ => CategoryMapping::default(), + } +} + +pub fn map_audible_es_ladder(path: &[&str]) -> LadderMatch { + let original_path: Vec = path.iter().map(|segment| (*segment).to_string()).collect(); + + let exact = map_audible_es_path_exact(path); + if !exact.is_empty() { + return LadderMatch { + original_path, + matched_path: path.iter().map(|segment| (*segment).to_string()).collect(), + depth: MappingDepth::ExactFullPath, + mapping: exact, + }; + } + + if path.len() >= 2 { + let two = map_audible_es_path(&path[..2]); + if !two.is_empty() { + return LadderMatch { + original_path, + matched_path: path[..2] + .iter() + .map(|segment| (*segment).to_string()) + .collect(), + depth: MappingDepth::FallbackTwoLevel, + mapping: two, + }; + } + } + + if !path.is_empty() { + let one = map_audible_es_path(&path[..1]); + if !one.is_empty() { + return LadderMatch { + original_path, + matched_path: path[..1] + .iter() + .map(|segment| (*segment).to_string()) + .collect(), + depth: MappingDepth::FallbackTopLevel, + mapping: one, + }; + } + } + + LadderMatch { + original_path, + matched_path: Vec::new(), + depth: MappingDepth::Unmapped, + mapping: CategoryMapping::default(), + } +} + +pub fn map_category_ladders(ladders: &[CategoryLadder]) -> AggregateCategoryResult { + let mut categories = BTreeSet::new(); + let mut freeform_tags = BTreeSet::new(); + let mut ladder_matches = Vec::new(); + let mut unmapped_paths = Vec::new(); + + for ladder in ladders { + if ladder.root != "Genres" { + continue; + } + + let path: Vec<&str> = ladder + .ladder + .iter() + .map(|node| node.name.as_str()) + .collect(); + let ladder_match = map_audible_es_ladder(&path); + + if ladder_match.depth == MappingDepth::Unmapped { + unmapped_paths.push(ladder_match.original_path.clone()); + } + + for category in &ladder_match.mapping.categories { + categories.insert(*category); + } + for tag in &ladder_match.mapping.freeform_tags { + freeform_tags.insert(tag.clone()); + } + + ladder_matches.push(ladder_match); + } + + AggregateCategoryResult { + categories: categories.into_iter().collect(), + freeform_tags: freeform_tags.into_iter().collect(), + ladder_matches, + unmapped_paths, + } +} + +pub fn three_plus_override_candidates(ladders: &[CategoryLadder]) -> Vec { + ladders + .iter() + .filter(|ladder| ladder.root == "Genres" && ladder.ladder.len() >= 3) + .map(|ladder| { + let path: Vec<&str> = ladder + .ladder + .iter() + .map(|node| node.name.as_str()) + .collect(); + map_audible_es_ladder(&path) + }) + .filter(|ladder_match| ladder_match.depth != MappingDepth::ExactFullPath) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::linker::folder::Ladder; + + #[test] + fn maps_exact_override_before_fallback() { + let result = map_audible_es_ladder(&[ + "Literatura y ficción", + "Literatura de género", + "Coming of age", + ]); + + assert_eq!(result.depth, MappingDepth::ExactFullPath); + assert!(result.mapping.categories.contains(&Category::ComingOfAge)); + assert!( + result + .mapping + .freeform_tags + .contains(&"Genre Fiction".to_string()) + ); + } + + #[test] + fn reports_three_level_fallback_as_override_candidate() { + let ladders = vec![CategoryLadder { + root: "Genres".to_string(), + ladder: vec![ + Ladder { + id: "1".to_string(), + name: "Literatura y ficción".to_string(), + }, + Ladder { + id: "2".to_string(), + name: "Clásicos".to_string(), + }, + Ladder { + id: "3".to_string(), + name: "Europeos".to_string(), + }, + ], + }]; + + let candidates = three_plus_override_candidates(&ladders); + + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].depth, MappingDepth::FallbackTwoLevel); + assert_eq!( + candidates[0].matched_path, + vec!["Literatura y ficción".to_string(), "Clásicos".to_string()] + ); + } + + #[test] + fn aggregates_categories_and_tags_from_genre_ladders() { + let mut first = mapped(&[Category::Crime], &["One"]); + first.extend(mapped(&[Category::Noir], &["Two"])); + + assert_eq!(first.categories, vec![Category::Crime, Category::Noir]); + assert_eq!( + first.freeform_tags, + vec!["One".to_string(), "Two".to_string()] + ); + } +} diff --git a/server/src/linker/mod.rs b/server/src/linker/mod.rs new file mode 100644 index 00000000..0ed22ec1 --- /dev/null +++ b/server/src/linker/mod.rs @@ -0,0 +1,9 @@ +pub mod common; +pub mod duplicates; +pub mod folder; +pub mod libation_cats; +pub mod torrent; + +pub use self::common::{copy, file_size, hard_link, library_dir, map_path, select_format, symlink}; +pub use self::duplicates::{find_matches, rank_torrents}; +pub use self::torrent::{find_library, refresh_mam_metadata, refresh_metadata_relink, relink}; diff --git a/server/src/linker/torrent.rs b/server/src/linker/torrent.rs new file mode 100644 index 00000000..9b51b31e --- /dev/null +++ b/server/src/linker/torrent.rs @@ -0,0 +1,1520 @@ +#[cfg(target_family = "windows")] +use std::os::windows::fs::MetadataExt as _; +use std::{ + fs::File, + io::{BufWriter, Write}, + mem, + ops::Deref, + path::{Component, Path, PathBuf}, + sync::Arc, +}; + +use anyhow::{Context, Result, bail}; +use log::error; +use mlm_db::{ + ClientStatus, DatabaseExt as _, ErroredTorrentId, Event, EventType, LibraryMismatch, + SelectedTorrent, SelectedTorrentKey, Timestamp, Torrent, TorrentKey, TorrentMeta, +}; +use mlm_mam::{api::MaM, meta::MetaError, search::MaMTorrent}; +use mlm_parse::normalize_title; +use native_db::Database; +use once_cell::sync::Lazy; +use qbit::{ + models::{Torrent as QbitTorrent, TorrentContent}, + parameters::TorrentListParams, +}; + +#[allow(async_fn_in_trait)] +pub trait MaMApi: Send + Sync { + async fn get_torrent_info(&self, hash: &str) -> Result>; + async fn get_torrent_info_by_id(&self, id: u64) -> Result>; +} + +impl MaMApi for MaM<'_> { + async fn get_torrent_info(&self, hash: &str) -> Result> { + self.get_torrent_info(hash).await + } + async fn get_torrent_info_by_id(&self, id: u64) -> Result> { + self.get_torrent_info_by_id(id).await + } +} + +impl MaMApi for &T { + async fn get_torrent_info(&self, hash: &str) -> Result> { + (**self).get_torrent_info(hash).await + } + async fn get_torrent_info_by_id(&self, id: u64) -> Result> { + (**self).get_torrent_info_by_id(id).await + } +} + +impl MaMApi for Arc { + async fn get_torrent_info(&self, hash: &str) -> Result> { + (**self).get_torrent_info(hash).await + } + async fn get_torrent_info_by_id(&self, id: u64) -> Result> { + (**self).get_torrent_info_by_id(id).await + } +} +use regex::Regex; +use tokio::fs::create_dir_all; +use tracing::{Level, debug, instrument, span, trace}; + +use crate::{ + audiobookshelf::{self as abs}, + autograbber::update_torrent_meta, + cleaner::remove_library_files, + config::{Config, Library, LibraryLinkMethod, QbitConfig}, + linker::{ + common::{copy, hard_link, select_format, symlink}, + library_dir, map_path, + }, + logging::{TorrentMetaError, update_errored_torrent, write_event}, + qbittorrent::{QbitApi, ensure_category_exists}, +}; + +pub static DISK_PATTERN: Lazy = + Lazy::new(|| Regex::new(r"(?:CD|Disc|Disk)\s*(\d+)").unwrap()); + +/// Calculates the relative path for a file within the library book directory. +/// Handles `Disc X` subdirectories. +pub fn calculate_library_file_path(torrent_file_path: &str) -> PathBuf { + let torrent_path = PathBuf::from(torrent_file_path); + // Split the path into components, find the file name (last component) + // and search the ancestor directories from nearest to farthest for a + // "Disc/CD/Disk" pattern. If found, return "Disc N/", + // otherwise return just the file name. + let mut components: Vec = torrent_path.components().collect(); + let file_name_component = components + .pop() + .expect("torrent file path should not be empty"); + + // Search ancestors in reverse (nearest directory first) + let mut disc_dir: Option = None; + for comp in components.iter().rev() { + if let Component::Normal(os) = comp { + let s = os.to_string_lossy(); + if let Some(caps) = DISK_PATTERN.captures(&s) + && let Some(disc) = caps.get(1) + { + disc_dir = Some(format!("Disc {}", disc.as_str())); + break; + } + } + } + + if let Some(dir_name) = disc_dir { + PathBuf::from(dir_name).join(file_name_component.as_os_str()) + } else { + PathBuf::from(file_name_component.as_os_str()) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct FileLinkPlan { + pub download_path: PathBuf, + pub library_path: PathBuf, + pub relative_library_path: PathBuf, +} + +pub fn calculate_link_plans( + qbit_config: &QbitConfig, + torrent: &QbitTorrent, + files: &[TorrentContent], + selected_audio_format: Option<&str>, + selected_ebook_format: Option<&str>, + library_dir: &Path, +) -> Vec { + let mut plans = vec![]; + for file in files { + if !(selected_audio_format + .as_ref() + .is_some_and(|ext| file.name.ends_with(*ext)) + || selected_ebook_format + .as_ref() + .is_some_and(|ext| file.name.ends_with(*ext))) + { + continue; + } + + let file_path = calculate_library_file_path(&file.name); + let library_path = library_dir.join(&file_path); + let download_path = + map_path(&qbit_config.path_mapping, &torrent.save_path).join(&file.name); + + plans.push(FileLinkPlan { + download_path, + library_path, + relative_library_path: file_path, + }); + } + plans +} + +pub struct TorrentUpdate { + pub changed: bool, + pub events: Vec, +} + +pub fn check_torrent_updates( + torrent: &mut Torrent, + qbit_torrent: &QbitTorrent, + library: Option<&Library>, + config: &Config, + trackers: &[qbit::models::Tracker], +) -> TorrentUpdate { + let mut changed = false; + let mut events = vec![]; + + let library_name = library.and_then(|l| l.options().name.as_ref()); + if torrent.linker.as_ref() != library_name { + torrent.linker = library_name.map(ToOwned::to_owned); + changed = true; + } + + let category = if qbit_torrent.category.is_empty() { + None + } else { + Some(qbit_torrent.category.as_str()) + }; + if torrent.category.as_deref() != category { + torrent.category = category.map(ToOwned::to_owned); + changed = true; + } + + if torrent.client_status.is_none() + && let Some(mam_tracker) = trackers.last() + && mam_tracker.msg == "torrent not registered with this tracker" + { + torrent.client_status = Some(ClientStatus::RemovedFromTracker); + changed = true; + events.push(Event::new( + Some(qbit_torrent.hash.clone()), + None, + EventType::RemovedFromTracker, + )); + } + + if let Some(library_path) = &torrent.library_path { + let Some(library) = library else { + if torrent.library_mismatch != Some(LibraryMismatch::NoLibrary) { + torrent.library_mismatch = Some(LibraryMismatch::NoLibrary); + changed = true; + } + return TorrentUpdate { changed, events }; + }; + + if !library_path.starts_with(&library.options().library_dir) { + let wanted = Some(LibraryMismatch::NewLibraryDir( + library.options().library_dir.clone(), + )); + if torrent.library_mismatch != wanted { + torrent.library_mismatch = wanted; + changed = true; + } + } else { + let dir = library_dir( + config.exclude_narrator_in_library_dir, + library, + &torrent.meta, + ); + let mut is_wrong = Some(library_path) != dir.as_ref(); + let wanted = match dir { + Some(dir) => Some(LibraryMismatch::NewPath(dir)), + None => Some(LibraryMismatch::NoLibrary), + }; + + if torrent.library_mismatch != wanted { + if is_wrong { + // Try another attempt at matching with exclude_narrator flipped + let dir_2 = library_dir( + !config.exclude_narrator_in_library_dir, + library, + &torrent.meta, + ); + if Some(library_path) == dir_2.as_ref() { + is_wrong = false + } + } + if is_wrong { + torrent.library_mismatch = wanted; + changed = true; + } else if torrent.library_mismatch.is_some() { + torrent.library_mismatch = None; + changed = true; + } + } + } + } + + TorrentUpdate { changed, events } +} + +#[instrument(skip_all)] +pub async fn link_torrents_to_library( + config: Arc, + db: Arc>, + qbit: (&QbitConfig, &Q), + mam: &M, +) -> Result<()> +where + Q: QbitApi + ?Sized, + M: MaMApi + ?Sized, +{ + let torrents = qbit + .1 + .torrents(Some(TorrentListParams::default())) + .await + .context("qbit main data")?; + + for torrent in torrents { + if torrent.progress < 1.0 { + continue; + } + let library = find_library(&config, &torrent); + let r = db.r_transaction()?; + let mut existing_torrent: Option = r.get().primary(torrent.hash.clone())?; + { + let selected_torrent: Option = r.get().secondary::( + SelectedTorrentKey::hash, + Some(torrent.hash.clone()), + )?; + if let Some(selected_torrent) = selected_torrent { + debug!( + "Finished Downloading torrent {} {}", + selected_torrent.mam_id, selected_torrent.meta.title + ); + let (_guard, rw) = db.rw_async().await?; + rw.remove(selected_torrent)?; + rw.commit()?; + } + } + if let Some(t) = &mut existing_torrent { + let trackers = if t.client_status.is_none() { + match qbit.1.trackers(&torrent.hash).await { + Ok(trackers) => trackers, + Err(err) => { + error!("Error getting trackers for torrent {}: {err}", torrent.hash); + continue; + } + } + } else { + vec![] + }; + let update = check_torrent_updates(t, &torrent, library, &config, &trackers); + if update.changed { + let (_guard, rw) = db.rw_async().await?; + rw.upsert(t.clone())?; + rw.commit()?; + } + for event in update.events { + write_event(&db, event).await; + } + + if t.library_path.is_some() || t.replaced_with.is_some() { + continue; + } + } + + let Some(library) = library else { + trace!( + "Could not find matching library for torrent \"{}\", save_path {}", + torrent.name, torrent.save_path + ); + continue; + }; + + if library.options().method == LibraryLinkMethod::NoLink && existing_torrent.is_some() { + continue; + } + + let result = match_torrent( + config.clone(), + db.clone(), + qbit, + mam, + &torrent.hash, + &torrent, + library, + existing_torrent, + ) + .await + .context("match_torrent"); + update_errored_torrent( + &db, + ErroredTorrentId::Linker(torrent.hash.clone()), + torrent.name, + result, + ) + .await; + } + + Ok(()) +} + +pub async fn handle_invalid_torrent( + qbit: (&QbitConfig, &Q), + on_invalid_torrent: &crate::config::QbitUpdate, + hash: &str, +) -> Result<()> +where + Q: QbitApi + ?Sized, +{ + if let Some(category) = &on_invalid_torrent.category { + ensure_category_exists(qbit.1, &qbit.0.url, category).await?; + qbit.1.set_category(Some(vec![hash]), category).await?; + } + + if !on_invalid_torrent.tags.is_empty() { + qbit.1 + .add_tags( + Some(vec![hash]), + on_invalid_torrent.tags.iter().map(Deref::deref).collect(), + ) + .await?; + } + Ok(()) +} + +#[instrument(skip_all)] +#[allow(clippy::too_many_arguments)] +async fn match_torrent( + config: Arc, + db: Arc>, + qbit: (&QbitConfig, &Q), + mam: &M, + hash: &str, + torrent: &QbitTorrent, + library: &Library, + mut existing_torrent: Option, +) -> Result<()> +where + Q: QbitApi + ?Sized, + M: MaMApi + ?Sized, +{ + let files = qbit.1.files(hash, None).await?; + let selected_audio_format = + select_format(&library.options().audio_types, &config.audio_types, &files); + let selected_ebook_format = + select_format(&library.options().ebook_types, &config.ebook_types, &files); + + if selected_audio_format.is_none() && selected_ebook_format.is_none() { + bail!("Could not find any wanted formats in torrent"); + } + let Some(mam_torrent) = mam.get_torrent_info(hash).await.context("get_mam_info")? else { + if let Some(on_invalid_torrent) = &qbit.0.on_invalid_torrent { + handle_invalid_torrent(qbit, on_invalid_torrent, hash).await?; + } + bail!("Could not find torrent on mam"); + }; + if existing_torrent.is_none() + && let Some(old_torrent) = db + .r_transaction()? + .get() + .secondary::(TorrentKey::mam_id, Some(mam_torrent.id))? + { + if old_torrent.id != hash { + let (_guard, rw) = db.rw_async().await?; + rw.remove(old_torrent.clone())?; + rw.commit()?; + } + existing_torrent = Some(old_torrent); + } + let mut meta = match mam_torrent.as_meta() { + Ok(meta) => meta, + Err(err) => { + if let MetaError::UnknownMediaType(_) = err { + if let Some(on_invalid_torrent) = &qbit.0.on_invalid_torrent { + handle_invalid_torrent(qbit, on_invalid_torrent, hash).await?; + } + trace!("qbit updated"); + } + return Err(err).context("as_meta"); + } + }; + if let Some(existing_torrent) = &mut existing_torrent { + existing_torrent.meta.ids.append(&mut meta.ids); + mem::swap(&mut meta.ids, &mut existing_torrent.meta.ids); + } + + link_torrent( + &config, + qbit.0, + &db, + hash, + torrent, + files, + selected_audio_format, + selected_ebook_format, + library, + existing_torrent.as_ref(), + &meta, + ) + .await + .context("link_torrent") + .map_err(|err| anyhow::Error::new(TorrentMetaError(meta, err))) +} + +#[instrument(skip_all)] +pub async fn refresh_mam_metadata( + config: &Config, + db: &Database<'_>, + mam: &M, + id: String, +) -> Result<(Torrent, MaMTorrent)> +where + M: MaMApi + ?Sized, +{ + let Some(mut torrent): Option = db.r_transaction()?.get().primary(id)? else { + bail!("Could not find torrent id"); + }; + let Some(mam_id) = torrent.meta.mam_id() else { + bail!("Could not find mam id"); + }; + debug!("refreshing metadata for torrent {}", mam_id); + let Some(mam_torrent) = mam + .get_torrent_info_by_id(mam_id) + .await + .context("get_mam_info")? + else { + bail!("Could not find torrent \"{}\" on mam", torrent.meta.title); + }; + let mut meta = mam_torrent.as_meta().context("as_meta")?; + let mut ids = torrent.meta.ids.clone(); + ids.append(&mut meta.ids); + meta.ids = ids; + + if torrent.meta != meta { + update_torrent_meta( + config, + db, + db.rw_async().await?, + Some(&mam_torrent), + torrent.clone(), + meta.clone(), + true, + false, + ) + .await?; + torrent.meta = meta; + } + Ok((torrent, mam_torrent)) +} + +#[instrument(skip_all)] +pub async fn relink(config: &Config, db: &Database<'_>, hash: String) -> Result<()> { + for qbit_conf in &config.qbittorrent { + let qbit = match qbit::Api::new_login_username_password( + &qbit_conf.url, + &qbit_conf.username, + &qbit_conf.password, + ) + .await + { + Ok(qbit) => qbit, + Err(err) => { + error!("Error logging in to qbit {}: {err}", qbit_conf.url); + continue; + } + }; + let mut torrents = match qbit + .torrents(Some(TorrentListParams { + hashes: Some(vec![hash.clone()]), + ..TorrentListParams::default() + })) + .await + { + Ok(torrents) => torrents, + Err(err) => { + error!("Error getting torrents from qbit {}: {err}", qbit_conf.url); + continue; + } + }; + let Some(qbit_torrent) = torrents.pop() else { + continue; + }; + return relink_internal(config, qbit_conf, db, &qbit, qbit_torrent, hash).await; + } + bail!("Could not find torrent in qbit"); +} + +#[instrument(skip_all)] +pub async fn relink_internal( + config: &Config, + qbit_config: &QbitConfig, + db: &Database<'_>, + qbit: &Q, + qbit_torrent: QbitTorrent, + hash: String, +) -> Result<()> +where + Q: QbitApi + ?Sized, +{ + let Some(library) = find_library(config, &qbit_torrent) else { + bail!("Could not find matching library for torrent"); + }; + let files = qbit.files(&hash, None).await?; + let selected_audio_format = + select_format(&library.options().audio_types, &config.audio_types, &files); + let selected_ebook_format = + select_format(&library.options().ebook_types, &config.ebook_types, &files); + + if selected_audio_format.is_none() && selected_ebook_format.is_none() { + bail!("Could not find any wanted formats in torrent"); + } + let Some(torrent) = db.r_transaction()?.get().primary::(hash.clone())? else { + bail!("Could not find torrent"); + }; + let library_path_changed = torrent.library_path + != library_dir( + config.exclude_narrator_in_library_dir, + library, + &torrent.meta, + ); + remove_library_files(config, &torrent, library_path_changed).await?; + link_torrent( + config, + qbit_config, + db, + &hash, + &qbit_torrent, + files, + selected_audio_format, + selected_ebook_format, + library, + Some(&torrent), + &torrent.meta, + ) + .await + .context("link_torrent") + .map_err(|err| anyhow::Error::new(TorrentMetaError(torrent.meta, err))) +} + +#[instrument(skip_all)] +pub async fn refresh_metadata_relink( + config: &Config, + db: &Database<'_>, + mam: &M, + hash: String, +) -> Result<()> +where + M: MaMApi + ?Sized, +{ + for qbit_conf in &config.qbittorrent { + let qbit = match qbit::Api::new_login_username_password( + &qbit_conf.url, + &qbit_conf.username, + &qbit_conf.password, + ) + .await + { + Ok(qbit) => qbit, + Err(err) => { + error!("Error logging in to qbit {}: {err}", qbit_conf.url); + continue; + } + }; + let mut torrents = match qbit + .torrents(Some(TorrentListParams { + hashes: Some(vec![hash.clone()]), + ..TorrentListParams::default() + })) + .await + { + Ok(torrents) => torrents, + Err(err) => { + error!("Error getting torrents from qbit {}: {err}", qbit_conf.url); + continue; + } + }; + let Some(qbit_torrent) = torrents.pop() else { + continue; + }; + return refresh_metadata_relink_internal( + config, + qbit_conf, + db, + &qbit, + mam, + qbit_torrent, + hash, + ) + .await; + } + bail!("Could not find torrent in qbit"); +} + +#[instrument(skip_all)] +pub async fn refresh_metadata_relink_internal( + config: &Config, + qbit_config: &QbitConfig, + db: &Database<'_>, + qbit: &Q, + mam: &M, + qbit_torrent: QbitTorrent, + hash: String, +) -> Result<()> +where + Q: QbitApi + ?Sized, + M: MaMApi + ?Sized, +{ + let Some(library) = find_library(config, &qbit_torrent) else { + bail!("Could not find matching library for torrent"); + }; + let files = qbit.files(&hash, None).await?; + let selected_audio_format = + select_format(&library.options().audio_types, &config.audio_types, &files); + let selected_ebook_format = + select_format(&library.options().ebook_types, &config.ebook_types, &files); + + if selected_audio_format.is_none() && selected_ebook_format.is_none() { + bail!("Could not find any wanted formats in torrent"); + } + let (torrent, _mam_torrent) = refresh_mam_metadata(config, db, mam, hash.clone()).await?; + let library_path_changed = torrent.library_path + != library_dir( + config.exclude_narrator_in_library_dir, + library, + &torrent.meta, + ); + remove_library_files(config, &torrent, library_path_changed).await?; + link_torrent( + config, + qbit_config, + db, + &hash, + &qbit_torrent, + files, + selected_audio_format, + selected_ebook_format, + library, + Some(&torrent), + &torrent.meta, + ) + .await + .context("link_torrent") + .map_err(|err| anyhow::Error::new(TorrentMetaError(torrent.meta, err))) +} + +#[instrument(skip_all)] +#[allow(clippy::too_many_arguments)] +async fn link_torrent( + config: &Config, + qbit_config: &QbitConfig, + db: &Database<'_>, + hash: &str, + torrent: &QbitTorrent, + files: Vec, + selected_audio_format: Option, + selected_ebook_format: Option, + library: &Library, + existing_torrent: Option<&Torrent>, + meta: &TorrentMeta, +) -> Result<()> { + let mut library_files = vec![]; + + let library_path = if library.options().method != LibraryLinkMethod::NoLink { + let Some(mut dir) = library_dir(config.exclude_narrator_in_library_dir, library, meta) + else { + bail!("Torrent has no author"); + }; + if config.exclude_narrator_in_library_dir && !meta.narrators.is_empty() && dir.exists() { + dir = library_dir(false, library, meta).unwrap(); + } + let metadata = abs::create_metadata(meta); + + create_dir_all(&dir).await?; + let plans = calculate_link_plans( + qbit_config, + torrent, + &files, + selected_audio_format.as_deref(), + selected_ebook_format.as_deref(), + &dir, + ); + + for plan in plans { + let span = span!(Level::TRACE, "file", file = ?plan.relative_library_path); + let _s = span.enter(); + if let Some(parent) = plan.library_path.parent() { + create_dir_all(parent).await?; + } + library_files.push(plan.relative_library_path.clone()); + match library.options().method { + LibraryLinkMethod::Hardlink => hard_link( + &plan.download_path, + &plan.library_path, + &plan.relative_library_path, + )?, + LibraryLinkMethod::HardlinkOrCopy => hard_link( + &plan.download_path, + &plan.library_path, + &plan.relative_library_path, + ) + .or_else(|_| copy(&plan.download_path, &plan.library_path))?, + LibraryLinkMethod::Copy => copy(&plan.download_path, &plan.library_path)?, + LibraryLinkMethod::HardlinkOrSymlink => hard_link( + &plan.download_path, + &plan.library_path, + &plan.relative_library_path, + ) + .or_else(|_| symlink(&plan.download_path, &plan.library_path))?, + LibraryLinkMethod::Symlink => symlink(&plan.download_path, &plan.library_path)?, + LibraryLinkMethod::NoLink => {} + }; + } + library_files.sort(); + + let file = File::create(dir.join("metadata.json"))?; + let mut writer = BufWriter::new(file); + serde_json::to_writer(&mut writer, &metadata)?; + writer.flush()?; + Some(dir.clone()) + } else { + None + }; + + { + let (_guard, rw) = db.rw_async().await?; + rw.upsert(Torrent { + id: hash.to_owned(), + id_is_hash: true, + mam_id: meta.mam_id(), + // abs_id: existing_torrent.and_then(|t| t.abs_id.clone()), + // goodreads_id: existing_torrent.and_then(|t| t.goodreads_id), + library_path: library_path.clone(), + library_files, + linker: library.options().name.clone(), + category: if torrent.category.is_empty() { + None + } else { + Some(torrent.category.clone()) + }, + selected_audio_format, + selected_ebook_format, + title_search: normalize_title(&meta.title), + meta: meta.clone(), + created_at: existing_torrent + .map(|t| t.created_at) + .unwrap_or_else(Timestamp::now), + replaced_with: existing_torrent.and_then(|t| t.replaced_with.clone()), + library_mismatch: None, + client_status: existing_torrent.and_then(|t| t.client_status.clone()), + })?; + rw.commit()?; + } + + if let Some(library_path) = library_path { + write_event( + db, + Event::new( + Some(hash.to_owned()), + meta.mam_id(), + EventType::Linked { + linker: library.options().name.clone(), + library_path, + }, + ), + ) + .await; + } + + Ok(()) +} + +// map_path provided by crate::linker::common::map_path + +pub fn find_library<'a>(config: &'a Config, torrent: &QbitTorrent) -> Option<&'a Library> { + config + .libraries + .iter() + .filter(|l| match l { + Library::ByRipDir(_) => false, + Library::ByDownloadDir(l) => { + PathBuf::from(&torrent.save_path).starts_with(&l.download_dir) + } + Library::ByCategory(l) => torrent.category == l.category, + }) + .find(|l| { + let filters = l.tag_filters(); + if filters + .deny_tags + .iter() + .any(|tag| torrent.tags.split(", ").any(|t| t == tag.as_str())) + { + return false; + } + if filters.allow_tags.is_empty() { + return true; + } + filters + .allow_tags + .iter() + .any(|tag| torrent.tags.split(", ").any(|t| t == tag.as_str())) + }) +} + +// library_dir provided by crate::linker::common::library_dir + +// select_format provided by crate::linker::common::select_format_from_contents as `select_format` + +// hard_link, copy, symlink and file_size provided by crate::linker_common + +// tests moved to linker_common + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + use crate::config::{ + Config, Library, LibraryByCategory, LibraryByDownloadDir, LibraryLinkMethod, + LibraryOptions, LibraryTagFilters, + }; + + #[test] + fn test_find_library_by_download_dir() { + let cfg = Config { + mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: PathBuf::from("/downloads"), + options: LibraryOptions { + name: None, + library_dir: PathBuf::from("/library"), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters { + allow_tags: vec![], + deny_tags: vec![], + }, + })], + }; + + let qbit_torrent = qbit::models::Torrent { + save_path: "/downloads/some/path".to_string(), + category: "".to_string(), + ..Default::default() + }; + let lib = find_library(&cfg, &qbit_torrent); + assert!(lib.is_some()); + match lib.unwrap() { + Library::ByDownloadDir(_) => {} + _ => panic!("Expected ByDownloadDir"), + } + } + + #[test] + fn test_find_library_by_category() { + let cfg = Config { + mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![Library::ByCategory(LibraryByCategory { + category: "audiobooks".to_string(), + options: LibraryOptions { + name: None, + library_dir: PathBuf::from("/lib2"), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters { + allow_tags: vec![], + deny_tags: vec![], + }, + })], + }; + + let qbit_torrent = qbit::models::Torrent { + save_path: "/other".to_string(), + category: "audiobooks".to_string(), + ..Default::default() + }; + let lib = find_library(&cfg, &qbit_torrent); + assert!(lib.is_some()); + match lib.unwrap() { + Library::ByCategory(l) => assert_eq!(l.category, "audiobooks"), + _ => panic!("Expected ByCategory"), + } + } + + #[test] + fn test_find_library_skips_rip_dir() { + let cfg = Config { + mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![Library::ByRipDir(crate::config::LibraryByRipDir { + rip_dir: PathBuf::from("/rip"), + options: LibraryOptions { + name: None, + library_dir: PathBuf::from("/lib"), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + filter: crate::config::EditionFilter::default(), + })], + }; + + let qbit_torrent = qbit::models::Torrent { + save_path: "/rip/some".to_string(), + category: "".to_string(), + ..Default::default() + }; + let lib = find_library(&cfg, &qbit_torrent); + // ByRipDir is explicitly skipped by find_library so should return None + assert!(lib.is_none()); + } + + #[test] + fn test_calculate_library_file_path() { + assert_eq!( + calculate_library_file_path("Book Title/audio.m4b"), + PathBuf::from("audio.m4b") + ); + assert_eq!( + calculate_library_file_path("Book Title/CD 1/01.mp3"), + PathBuf::from("Disc 1/01.mp3") + ); + assert_eq!( + calculate_library_file_path("Book Title/Disc 2/05.mp3"), + PathBuf::from("Disc 2/05.mp3") + ); + assert_eq!( + calculate_library_file_path("Book Title/Disk 10/05.mp3"), + PathBuf::from("Disc 10/05.mp3") + ); + assert_eq!( + calculate_library_file_path("Book Title/CD1/01.mp3"), + PathBuf::from("Disc 1/01.mp3") + ); + assert_eq!( + calculate_library_file_path("audio.m4b"), + PathBuf::from("audio.m4b") + ); + assert_eq!( + calculate_library_file_path("Book Title/Some Other Folder/audio.m4b"), + PathBuf::from("audio.m4b") + ); + assert_eq!( + calculate_library_file_path("Book Title/Disc 1/Subfolder/01.mp3"), + PathBuf::from("Disc 1/01.mp3") + ); + } + + #[test] + fn test_calculate_library_file_path_filename_contains_disc() { + // Ensure a filename that contains the word "Disc" isn't treated as a + // disc directory. Only ancestor directory names should be considered. + assert_eq!( + calculate_library_file_path("Book Title/Some Folder/Disc 1 - track.mp3"), + PathBuf::from("Disc 1 - track.mp3") + ); + } + + #[test] + fn test_calculate_link_plans() { + use std::collections::BTreeMap; + let qbit_config = QbitConfig { + path_mapping: BTreeMap::from([( + PathBuf::from("/downloads"), + PathBuf::from("/data/downloads"), + )]), + url: "".to_string(), + username: "".to_string(), + password: "".to_string(), + on_cleaned: None, + on_invalid_torrent: None, + }; + let torrent = qbit::models::Torrent { + save_path: "/downloads".to_string(), + ..Default::default() + }; + let files = vec![ + TorrentContent { + name: "Audiobook/audio.m4b".to_string(), + ..Default::default() + }, + TorrentContent { + name: "Audiobook/cover.jpg".to_string(), + ..Default::default() + }, + ]; + let library_dir = PathBuf::from("/library"); + + let plans = calculate_link_plans( + &qbit_config, + &torrent, + &files, + Some(".m4b"), + None, + &library_dir, + ); + + assert_eq!(plans.len(), 1); + assert_eq!( + plans[0].download_path, + PathBuf::from("/data/downloads/Audiobook/audio.m4b") + ); + assert_eq!(plans[0].library_path, PathBuf::from("/library/audio.m4b")); + assert_eq!(plans[0].relative_library_path, PathBuf::from("audio.m4b")); + } + + #[test] + fn test_check_torrent_updates_category_change() { + use std::collections::BTreeMap; + let mut torrent = Torrent { + id: "1".to_string(), + id_is_hash: true, + mam_id: None, + library_path: None, + library_files: vec![], + linker: None, + category: Some("old".to_string()), + selected_audio_format: None, + selected_ebook_format: None, + title_search: "".to_string(), + meta: TorrentMeta { + ids: BTreeMap::new(), + vip_status: None, + cat: None, + media_type: mlm_db::MediaType::Audiobook, + main_cat: None, + categories: vec![], + tags: vec![], + language: None, + flags: None, + filetypes: vec![], + num_files: 0, + size: mlm_db::Size::from_bytes(0), + title: "".to_string(), + edition: None, + description: "".to_string(), + authors: vec![], + narrators: vec![], + series: vec![], + source: mlm_db::MetadataSource::Mam, + uploaded_at: mlm_db::Timestamp::now(), + }, + created_at: mlm_db::Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + }; + let qbit_torrent = qbit::models::Torrent { + category: "new".to_string(), + ..Default::default() + }; + let cfg = Config { + mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![], + }; + + let update = check_torrent_updates(&mut torrent, &qbit_torrent, None, &cfg, &[]); + assert!(update.changed); + assert_eq!(torrent.category, Some("new".to_string())); + } + + #[test] + fn test_check_torrent_updates_linker_change() { + let mut torrent = Torrent { + linker: Some("old_linker".to_string()), + ..mock_torrent() + }; + let qbit_torrent = qbit::models::Torrent::default(); + let library = Library::ByCategory(LibraryByCategory { + category: "audiobooks".to_string(), + options: LibraryOptions { + name: Some("new_linker".to_string()), + library_dir: PathBuf::from("/lib"), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + }); + let cfg = mock_config_with_library(library.clone()); + + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!(update.changed); + assert_eq!(torrent.linker, Some("new_linker".to_string())); + } + + #[test] + fn test_check_torrent_updates_removed_from_tracker() { + let mut torrent = mock_torrent(); + let qbit_torrent = qbit::models::Torrent { + hash: "hash".to_string(), + ..Default::default() + }; + let trackers = vec![qbit::models::Tracker { + msg: "torrent not registered with this tracker".to_string(), + ..Default::default() + }]; + let cfg = mock_config(); + + let update = check_torrent_updates(&mut torrent, &qbit_torrent, None, &cfg, &trackers); + assert!(update.changed); + assert_eq!( + torrent.client_status, + Some(ClientStatus::RemovedFromTracker) + ); + assert_eq!(update.events.len(), 1); + assert_eq!(update.events[0].event, EventType::RemovedFromTracker); + } + + #[test] + fn test_check_torrent_updates_library_mismatch() { + let mut torrent = mock_torrent(); + torrent.library_path = Some(PathBuf::from("/old_library/Author/Title")); + torrent.meta.authors = vec!["Author".to_string()]; + torrent.meta.title = "Title".to_string(); + + let library = Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: PathBuf::from("/downloads"), + options: LibraryOptions { + name: None, + library_dir: PathBuf::from("/new_library"), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + }); + let cfg = mock_config_with_library(library.clone()); + let qbit_torrent = qbit::models::Torrent::default(); + + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!(update.changed); + assert_eq!( + torrent.library_mismatch, + Some(LibraryMismatch::NewLibraryDir(PathBuf::from( + "/new_library" + ))) + ); + + // Test NewPath (same library dir, different author/title logic) + torrent.library_mismatch = None; + torrent.library_path = Some(PathBuf::from("/new_library/OldAuthor/Title")); + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!(update.changed); + if let Some(LibraryMismatch::NewPath(p)) = &torrent.library_mismatch { + assert!(p.ends_with("Author/Title")); + } else { + panic!( + "Expected NewPath mismatch, got {:?}", + torrent.library_mismatch + ); + } + } + + #[test] + fn test_check_torrent_updates_exclude_narrator() { + let mut torrent = mock_torrent(); + torrent.meta.authors = vec!["Author".to_string()]; + torrent.meta.narrators = vec!["Narrator".to_string()]; + torrent.meta.title = "Title".to_string(); + + // Path with narrator + let path_with_narrator = PathBuf::from("/library/Author/Title {Narrator}"); + // Path without narrator + let path_without_narrator = PathBuf::from("/library/Author/Title"); + + let library = Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: PathBuf::from("/downloads"), + options: LibraryOptions { + name: None, + library_dir: PathBuf::from("/library"), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + }); + + // Config: exclude_narrator = true + let mut cfg = mock_config_with_library(library.clone()); + cfg.exclude_narrator_in_library_dir = true; + + let qbit_torrent = qbit::models::Torrent::default(); + + // If current path matches "with narrator" but config says exclude, it might be okay if it was deliberate or it might trigger mismatch. + // The code has this logic: + /* + let dir = library_dir(config.exclude_narrator_in_library_dir, library, &torrent.meta); + let mut is_wrong = Some(library_path) != dir.as_ref(); + ... + if is_wrong { + // Try another attempt at matching with exclude_narrator flipped + let dir_2 = library_dir(!config.exclude_narrator_in_library_dir, library, &torrent.meta); + if Some(library_path) == dir_2.as_ref() { + is_wrong = false + } + } + */ + // This means it accepts BOTH paths as "not wrong" if they match either state of the toggle, UNLESS it's already in mismatch state? + // Wait, if it matches dir_2, is_wrong becomes false, so it WON'T set library_mismatch. + + torrent.library_path = Some(path_with_narrator.clone()); + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!( + !update.changed, + "Should accept existing path with narrator even if config says exclude" + ); + + torrent.library_path = Some(path_without_narrator.clone()); + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!( + !update.changed, + "Should accept existing path without narrator" + ); + + // What if it matches neither? + torrent.library_path = Some(PathBuf::from("/library/Wrong/Path")); + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!(update.changed); + assert!(matches!( + torrent.library_mismatch, + Some(LibraryMismatch::NewPath(_)) + )); + } + + #[tokio::test] + async fn test_handle_invalid_torrent() { + struct MockQbit { + category: std::sync::Mutex>, + tags: std::sync::Mutex>, + } + impl QbitApi for MockQbit { + async fn torrents( + &self, + _: Option, + ) -> Result> { + Ok(vec![]) + } + async fn trackers(&self, _: &str) -> Result> { + Ok(vec![]) + } + async fn files( + &self, + _: &str, + _: Option>, + ) -> Result> { + Ok(vec![]) + } + async fn set_category(&self, _: Option>, category: &str) -> Result<()> { + *self.category.lock().unwrap() = Some(category.to_string()); + Ok(()) + } + async fn add_tags(&self, _: Option>, tags: Vec<&str>) -> Result<()> { + self.tags + .lock() + .unwrap() + .extend(tags.iter().map(|t| t.to_string())); + Ok(()) + } + async fn create_category(&self, _: &str, _: &str) -> Result<()> { + Ok(()) + } + async fn categories( + &self, + ) -> Result> { + Ok(std::collections::HashMap::new()) + } + } + + let qbit = MockQbit { + category: std::sync::Mutex::new(None), + tags: std::sync::Mutex::new(vec![]), + }; + + let qbit_conf = QbitConfig { + url: "http://localhost:8080".to_string(), + ..Default::default() + }; + + let update = crate::config::QbitUpdate { + category: Some("invalid".to_string()), + tags: vec!["tag1".to_string(), "tag2".to_string()], + }; + + handle_invalid_torrent((&qbit_conf, &qbit), &update, "hash") + .await + .unwrap(); + + assert_eq!(*qbit.category.lock().unwrap(), Some("invalid".to_string())); + assert_eq!( + *qbit.tags.lock().unwrap(), + vec!["tag1".to_string(), "tag2".to_string()] + ); + } + + fn mock_torrent() -> Torrent { + use std::collections::BTreeMap; + Torrent { + id: "1".to_string(), + id_is_hash: true, + mam_id: None, + library_path: None, + library_files: vec![], + linker: None, + category: None, + selected_audio_format: None, + selected_ebook_format: None, + title_search: "".to_string(), + meta: TorrentMeta { + ids: BTreeMap::new(), + vip_status: None, + cat: None, + media_type: mlm_db::MediaType::Audiobook, + main_cat: None, + categories: vec![], + tags: vec![], + language: None, + flags: None, + filetypes: vec![], + num_files: 0, + size: mlm_db::Size::from_bytes(0), + title: "".to_string(), + edition: None, + description: "".to_string(), + authors: vec![], + narrators: vec![], + series: vec![], + source: mlm_db::MetadataSource::Mam, + uploaded_at: mlm_db::Timestamp::now(), + }, + created_at: mlm_db::Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + } + } + + fn mock_config() -> Config { + Config { + mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![], + } + } + + fn mock_config_with_library(library: Library) -> Config { + let mut cfg = mock_config(); + cfg.libraries.push(library); + cfg + } +} diff --git a/server/src/lists/goodreads.rs b/server/src/lists/goodreads.rs index 949b21b2..4a4761b7 100644 --- a/server/src/lists/goodreads.rs +++ b/server/src/lists/goodreads.rs @@ -121,7 +121,7 @@ pub async fn run_goodreads_import( db_item } None => { - let db_item = item.as_list_item(&list_id, &list); + let db_item = item.as_list_item(&list_id, list); if !list.dry_run { let (_guard, rw) = db.rw_async().await?; rw.insert(db_item.clone())?; @@ -131,7 +131,7 @@ pub async fn run_goodreads_import( } }; trace!("Searching for book {} from Goodreads list", item.title); - search_item(&config, &db, &mam, &list, &item, db_item, max_torrents) + search_item(&config, &db, &mam, list, &item, db_item, max_torrents) .await .context("search goodreads book")?; sleep(Duration::from_millis(400)).await; @@ -212,10 +212,11 @@ async fn search_item( && db_item .audio_torrent .as_ref() - .is_none_or(|t| !(t.status == TorrentStatus::Selected && t.mam_id == found.0.id)) + .is_none_or(|t| !(t.status == TorrentStatus::Selected && t.mam_id == Some(found.0.id))) { db_item.audio_torrent = Some(ListItemTorrent { - mam_id: found.0.id, + torrent_id: None, + mam_id: Some(found.0.id), status: TorrentStatus::Selected, at: Timestamp::now(), }); @@ -225,10 +226,11 @@ async fn search_item( && db_item .ebook_torrent .as_ref() - .is_none_or(|t| !(t.status == TorrentStatus::Selected && t.mam_id == found.0.id)) + .is_none_or(|t| !(t.status == TorrentStatus::Selected && t.mam_id == Some(found.0.id))) { db_item.ebook_torrent = Some(ListItemTorrent { - mam_id: found.0.id, + torrent_id: None, + mam_id: Some(found.0.id), status: TorrentStatus::Selected, at: Timestamp::now(), }); @@ -398,7 +400,8 @@ fn not_wanted( { debug!("Skipped {:?} torrent as is not wanted", found.1.main_cat); field.replace(ListItemTorrent { - mam_id: found.0.id, + torrent_id: None, + mam_id: Some(found.0.id), status: TorrentStatus::NotWanted, at: Timestamp::now(), }); @@ -418,10 +421,11 @@ fn check_cost( warn!("Skipped {:?} torrent as it is not free", found.1.main_cat); if field .as_ref() - .is_none_or(|t| !(t.status == TorrentStatus::Wanted && t.mam_id == found.0.id)) + .is_none_or(|t| !(t.status == TorrentStatus::Wanted && t.mam_id == Some(found.0.id))) { field.replace(ListItemTorrent { - mam_id: found.0.id, + torrent_id: None, + mam_id: Some(found.0.id), status: TorrentStatus::Wanted, at: Timestamp::now(), }); diff --git a/server/src/lists/mod.rs b/server/src/lists/mod.rs index edc5810e..19ad9b37 100644 --- a/server/src/lists/mod.rs +++ b/server/src/lists/mod.rs @@ -168,12 +168,13 @@ fn set_existing(field: &mut Option, torrent: &Torrent) -> bool if field.status == TorrentStatus::Selected { return false; } - if field.status == TorrentStatus::Existing && field.mam_id == torrent.meta.mam_id { + if field.status == TorrentStatus::Existing && field.mam_id == torrent.mam_id { return false; } } field.replace(ListItemTorrent { - mam_id: torrent.meta.mam_id, + torrent_id: Some(torrent.id.clone()), + mam_id: torrent.mam_id, status: TorrentStatus::Existing, at: torrent.created_at, }); @@ -187,7 +188,7 @@ async fn search_grab( db_item: &ListItem, grab: &Grab, ) -> Result> { - let (flags_is_hide, flags) = grab.filter.flags.as_search_bitfield(); + let (flags_is_hide, flags) = grab.filter.edition.flags.as_search_bitfield(); let title_query = db_item.title.replace("*", "\"*\""); let title_query = BAD_CHARATERS.replace_all(&title_query, " "); @@ -201,7 +202,7 @@ async fn search_grab( db_item.authors.iter().map(|a| format!("\"{a}\"")).join("|") ); - let mut categories = grab.filter.categories.clone(); + let mut categories = grab.filter.edition.categories.clone(); if db_item.audio_torrent.is_some() { categories.audio = Some(vec![]) } @@ -224,7 +225,13 @@ async fn search_grab( srch_in: vec![SearchIn::Title, SearchIn::Author], main_cat: categories.get_main_cats(), cat: categories.get_cats(), - browse_lang: grab.filter.languages.iter().map(|l| l.to_id()).collect(), + browse_lang: grab + .filter + .edition + .languages + .iter() + .map(|l| l.to_id()) + .collect(), browse_flags_hide_vs_show: if flags.is_empty() { None } else { @@ -239,9 +246,14 @@ async fn search_grab( .filter .uploaded_before .map_or_else(|| Ok(String::new()), |d| d.format(&DATE_FORMAT))?, - min_size: grab.filter.min_size.bytes(), - max_size: grab.filter.max_size.bytes(), - unit: grab.filter.min_size.unit().max(grab.filter.max_size.unit()), + min_size: grab.filter.edition.min_size.bytes(), + max_size: grab.filter.edition.max_size.bytes(), + unit: grab + .filter + .edition + .min_size + .unit() + .max(grab.filter.edition.max_size.unit()), min_seeders: grab.filter.min_seeders, max_seeders: grab.filter.max_seeders, min_leechers: grab.filter.min_leechers, @@ -250,8 +262,6 @@ async fn search_grab( max_snatched: grab.filter.max_snatched, ..Default::default() }, - - ..Default::default() }) .await .context("search")?; diff --git a/server/src/main.rs b/server/src/main.rs index 48133cee..75020420 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,27 +1,10 @@ #![windows_subsystem = "windows"] -mod audiobookshelf; -mod autograbber; -mod cleaner; -mod config; -mod config_impl; -mod exporter; -mod linker; -mod lists; -mod logging; -mod qbittorrent; -mod snatchlist; -mod stats; -mod torrent_downloader; -mod web; -#[cfg(target_family = "windows")] -mod windows; - use std::{ collections::BTreeMap, env, fs::{self, create_dir_all}, - io, + io, mem, path::PathBuf, process, sync::Arc, @@ -29,40 +12,41 @@ use std::{ }; use anyhow::{Context as _, Result}; -use audiobookshelf::match_torrents_to_abs; -use autograbber::run_autograbber; -use cleaner::run_library_cleaner; use dirs::{config_dir, data_local_dir}; -use exporter::export_db; use figment::{ Figment, providers::{Env, Format, Toml}, }; use mlm_mam::api::MaM; -use stats::{Stats, Triggers}; use time::OffsetDateTime; use tokio::{ select, sync::{Mutex, watch}, time::sleep, }; -use torrent_downloader::grab_selected_torrents; use tracing::error; use tracing_appender::rolling::{RollingFileAppender, Rotation}; use tracing_subscriber::{ EnvFilter, Layer as _, fmt::time::LocalTime, layer::SubscriberExt as _, util::SubscriberInitExt as _, }; -use web::start_webserver; -use crate::{ +use mlm::{ + audiobookshelf::match_torrents_to_abs, + autograbber::run_autograbber, + cleaner::run_library_cleaner, config::Config, - linker::link_torrents_to_library, + linker::{folder::link_folders_to_library, torrent::link_torrents_to_library}, lists::{get_lists, run_list_import}, snatchlist::run_snatchlist_search, - stats::Context, + stats::{Context, Stats, Triggers}, + torrent_downloader::grab_selected_torrents, + web::start_webserver, }; +#[cfg(target_family = "windows")] +use mlm::windows; + #[tokio::main] async fn main() { if let Err(err) = app_main().await { @@ -188,7 +172,26 @@ async fn app_main() -> Result<()> { .unwrap(); return Ok(()); } - let config = config?; + let mut config = config?; + for autograb in &mut config.autograbs { + autograb.filter.edition = mem::take(&mut autograb.edition); + } + for snatchlist in &mut config.snatchlist { + snatchlist.filter.edition = mem::take(&mut snatchlist.edition); + } + for list in &mut config.goodreads_lists { + for grab in &mut list.grab { + grab.filter.edition = mem::take(&mut grab.edition); + } + } + for list in &mut config.notion_lists { + for grab in &mut list.grab { + grab.filter.edition = mem::take(&mut grab.edition); + } + } + for tag in &mut config.tags { + tag.filter.edition = mem::take(&mut tag.edition); + } let config = Arc::new(config); let db = native_db::Builder::new().create(&mlm_db::MODELS, database_file)?; @@ -199,8 +202,6 @@ async fn app_main() -> Result<()> { return Ok(()); } - // export_db(&db)?; - // return Ok(()); let db = Arc::new(db); #[cfg(target_family = "windows")] @@ -210,7 +211,8 @@ async fn app_main() -> Result<()> { let (mut search_tx, mut search_rx) = (BTreeMap::new(), BTreeMap::new()); let (mut import_tx, mut import_rx) = (BTreeMap::new(), BTreeMap::new()); - let (linker_tx, linker_rx) = watch::channel(()); + let (torrent_linker_tx, torrent_linker_rx) = watch::channel(()); + let (folder_linker_tx, folder_linker_rx) = watch::channel(()); let (downloader_tx, mut downloader_rx) = watch::channel(()); let (audiobookshelf_tx, mut audiobookshelf_rx) = watch::channel(()); @@ -497,7 +499,7 @@ async fn app_main() -> Result<()> { let db = db.clone(); let mam = mam.clone(); let stats = stats.clone(); - let mut linker_rx = linker_rx.clone(); + let mut linker_rx = torrent_linker_rx.clone(); tokio::spawn(async move { loop { select! { @@ -507,8 +509,8 @@ async fn app_main() -> Result<()> { error!("Error listening on link_rx: {err:?}"); stats .update(|stats| { - stats.linker_run_at = Some(OffsetDateTime::now_utc()); - stats.linker_result = Some(Err(err.into())); + stats.torrent_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.torrent_linker_result = Some(Err(err.into())); }).await; } }, @@ -516,8 +518,8 @@ async fn app_main() -> Result<()> { { stats .update(|stats| { - stats.linker_run_at = Some(OffsetDateTime::now_utc()); - stats.linker_result = None; + stats.torrent_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.torrent_linker_result = None; }) .await; } @@ -533,8 +535,9 @@ async fn app_main() -> Result<()> { error!("Error logging in to qbit {}: {err}", qbit_conf.url); stats .update(|stats| { - stats.linker_run_at = Some(OffsetDateTime::now_utc()); - stats.linker_result = + stats.torrent_linker_run_at = + Some(OffsetDateTime::now_utc()); + stats.torrent_linker_result = Some(Err(anyhow::Error::msg(format!( "Error logging in to qbit {}: {err}", qbit_conf.url, @@ -548,7 +551,7 @@ async fn app_main() -> Result<()> { config.clone(), db.clone(), (&qbit_conf, &qbit), - mam.clone(), + &mam, ) .await .context("link_torrents_to_library"); @@ -558,7 +561,7 @@ async fn app_main() -> Result<()> { { stats .update(|stats| { - stats.linker_result = Some(result); + stats.torrent_linker_result = Some(result); stats.cleaner_run_at = Some(OffsetDateTime::now_utc()); stats.cleaner_result = None; }) @@ -582,6 +585,65 @@ async fn app_main() -> Result<()> { } } } + { + let config = config.clone(); + let db = db.clone(); + let stats = stats.clone(); + let mut linker_rx = folder_linker_rx.clone(); + tokio::spawn(async move { + loop { + select! { + () = sleep(Duration::from_secs(60 * config.link_interval)) => {}, + result = linker_rx.changed() => { + if let Err(err) = result { + error!("Error listening on link_rx: {err:?}"); + stats + .update(|stats| { + stats.folder_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.folder_linker_result = Some(Err(err.into())); + }).await; + } + }, + } + { + stats + .update(|stats| { + stats.folder_linker_run_at = Some(OffsetDateTime::now_utc()); + stats.folder_linker_result = None; + }) + .await; + } + let result = link_folders_to_library(config.clone(), db.clone()) + .await + .context("link_torrents_to_library"); + if let Err(err) = &result { + error!("Error running linker: {err:?}"); + } + { + stats + .update(|stats| { + stats.folder_linker_result = Some(result); + stats.cleaner_run_at = Some(OffsetDateTime::now_utc()); + stats.cleaner_result = None; + }) + .await; + } + let result = run_library_cleaner(config.clone(), db.clone()) + .await + .context("library_cleaner"); + if let Err(err) = &result { + error!("Error running library_cleaner: {err:?}"); + } + { + stats + .update(|stats| { + stats.cleaner_result = Some(result); + }) + .await; + } + } + }); + } if let Some(config) = &config.audiobookshelf { let config = config.clone(); @@ -630,7 +692,8 @@ async fn app_main() -> Result<()> { let triggers = Triggers { search_tx, import_tx, - linker_tx, + torrent_linker_tx, + folder_linker_tx, downloader_tx, audiobookshelf_tx, }; diff --git a/server/src/qbittorrent.rs b/server/src/qbittorrent.rs index 89e3fa63..ea03d17e 100644 --- a/server/src/qbittorrent.rs +++ b/server/src/qbittorrent.rs @@ -5,13 +5,100 @@ use std::time::{Duration, Instant}; use anyhow::Result; use once_cell::sync::Lazy; use qbit::{ - models::Torrent, + models::{Torrent, TorrentContent}, parameters::{AddTorrent, TorrentListParams}, }; use tokio::sync::RwLock; use crate::config::{Config, QbitConfig}; +#[allow(async_fn_in_trait)] +pub trait QbitApi: Send + Sync { + async fn torrents(&self, params: Option) -> Result>; + async fn trackers(&self, hash: &str) -> Result>; + async fn files(&self, hash: &str, params: Option>) -> Result>; + async fn set_category(&self, hashes: Option>, category: &str) -> Result<()>; + async fn add_tags(&self, hashes: Option>, tags: Vec<&str>) -> Result<()>; + async fn create_category(&self, category: &str, save_path: &str) -> Result<()>; + async fn categories(&self) -> Result>; +} + +impl QbitApi for qbit::Api { + async fn torrents(&self, params: Option) -> Result> { + self.torrents(params).await.map_err(Into::into) + } + async fn trackers(&self, hash: &str) -> Result> { + self.trackers(hash).await.map_err(Into::into) + } + async fn files(&self, hash: &str, params: Option>) -> Result> { + self.files(hash, params).await.map_err(Into::into) + } + async fn set_category(&self, hashes: Option>, category: &str) -> Result<()> { + self.set_category(hashes, category) + .await + .map_err(Into::into) + } + async fn add_tags(&self, hashes: Option>, tags: Vec<&str>) -> Result<()> { + self.add_tags(hashes, tags).await.map_err(Into::into) + } + async fn create_category(&self, category: &str, save_path: &str) -> Result<()> { + self.create_category(category, save_path) + .await + .map_err(Into::into) + } + async fn categories(&self) -> Result> { + self.categories().await.map_err(Into::into) + } +} + +impl QbitApi for &T { + async fn torrents(&self, params: Option) -> Result> { + (**self).torrents(params).await + } + async fn trackers(&self, hash: &str) -> Result> { + (**self).trackers(hash).await + } + async fn files(&self, hash: &str, params: Option>) -> Result> { + (**self).files(hash, params).await + } + async fn set_category(&self, hashes: Option>, category: &str) -> Result<()> { + (**self).set_category(hashes, category).await + } + async fn add_tags(&self, hashes: Option>, tags: Vec<&str>) -> Result<()> { + (**self).add_tags(hashes, tags).await + } + async fn create_category(&self, category: &str, save_path: &str) -> Result<()> { + (**self).create_category(category, save_path).await + } + async fn categories(&self) -> Result> { + (**self).categories().await + } +} + +impl QbitApi for Arc { + async fn torrents(&self, params: Option) -> Result> { + (**self).torrents(params).await + } + async fn trackers(&self, hash: &str) -> Result> { + (**self).trackers(hash).await + } + async fn files(&self, hash: &str, params: Option>) -> Result> { + (**self).files(hash, params).await + } + async fn set_category(&self, hashes: Option>, category: &str) -> Result<()> { + (**self).set_category(hashes, category).await + } + async fn add_tags(&self, hashes: Option>, tags: Vec<&str>) -> Result<()> { + (**self).add_tags(hashes, tags).await + } + async fn create_category(&self, category: &str, save_path: &str) -> Result<()> { + (**self).create_category(category, save_path).await + } + async fn categories(&self) -> Result> { + (**self).categories().await + } +} + const CATEGORY_CACHE_TTL_SECS: u64 = 60; #[derive(Clone)] @@ -26,7 +113,12 @@ impl CategoryCache { } } - async fn get_or_fetch(&self, qbit: &qbit::Api, url: &str) -> Result> { + async fn get_or_fetch( + &self, + qbit: &Q, + url: &str, + ) -> Result> { + let now = Instant::now(); let cache = self.cache.read().await; @@ -58,8 +150,8 @@ impl Default for CategoryCache { static CATEGORY_CACHE: Lazy = Lazy::new(CategoryCache::new); -pub async fn ensure_category_exists( - qbit: &qbit::Api, +pub async fn ensure_category_exists( + qbit: &Q, url: &str, category: &str, ) -> Result<()> { @@ -97,7 +189,7 @@ pub async fn add_torrent_with_category( pub async fn get_torrent<'a, 'b>( config: &'a Config, - hash: &'b str, + hash: &str, ) -> Result> { for qbit_conf in config.qbittorrent.iter() { let Ok(qbit) = qbit::Api::new_login_username_password( diff --git a/server/src/snatchlist.rs b/server/src/snatchlist.rs index d2600432..49e191d7 100644 --- a/server/src/snatchlist.rs +++ b/server/src/snatchlist.rs @@ -27,7 +27,7 @@ pub async fn run_snatchlist_search( index: usize, snatchlist_config: Arc, ) -> Result<()> { - if !snatchlist_config.filter.languages.is_empty() { + if !snatchlist_config.filter.edition.languages.is_empty() { bail!("Language filtering is not supported in snatchlist searches"); } if snatchlist_config.filter.uploaded_after.is_some() @@ -149,13 +149,13 @@ async fn update_torrents>( if let Some((_, rw)) = &rw_opt { let old_library = rw .get() - .secondary::(TorrentKey::mam_id, meta.mam_id)?; + .secondary::(TorrentKey::mam_id, meta.mam_id())?; if let Some(old) = old_library { if old.meta != meta { update_torrent_meta( db, rw_opt.unwrap(), - &torrent, + Some(&torrent), old, meta, cost == Cost::MetadataOnlyAdd, @@ -167,7 +167,7 @@ async fn update_torrents>( } } if cost == Cost::MetadataOnlyAdd { - let mam_id = meta.mam_id; + let mam_id = torrent.id; add_metadata_only_torrent(rw_opt.unwrap(), torrent, meta) .await .or_else(|err| { @@ -206,9 +206,7 @@ async fn add_metadata_only_torrent( rw.insert(Torrent { id, id_is_hash: false, - mam_id, - abs_id: None, - goodreads_id: None, + mam_id: Some(mam_id), library_path: None, library_files: Default::default(), linker: if torrent.uploader_name.is_empty() { @@ -223,7 +221,6 @@ async fn add_metadata_only_torrent( meta, created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, })?; @@ -236,15 +233,18 @@ async fn add_metadata_only_torrent( async fn update_torrent_meta( db: &Database<'_>, (guard, rw): (MutexGuard<'_, ()>, RwTransaction<'_>), - mam_torrent: &UserDetailsTorrent, + mam_torrent: Option<&UserDetailsTorrent>, mut torrent: Torrent, mut meta: TorrentMeta, linker_is_owner: bool, ) -> Result<()> { // These are missing in user details torrent response, so keep the old values + meta.ids = torrent.meta.ids.clone(); meta.media_type = torrent.meta.media_type; meta.main_cat = torrent.meta.main_cat; meta.language = torrent.meta.language; + meta.tags = torrent.meta.tags.clone(); + meta.description = torrent.meta.description.clone(); meta.num_files = torrent.meta.num_files; meta.uploaded_at = torrent.meta.uploaded_at; @@ -277,17 +277,18 @@ async fn update_torrent_meta( } if linker_is_owner && torrent.linker.is_none() { - torrent.linker = Some(mam_torrent.uploader_name.clone()); + if let Some(mam_torrent) = mam_torrent { + torrent.linker = Some(mam_torrent.uploader_name.clone()); + } } else if meta == torrent.meta { return Ok(()); } let id = torrent.id.clone(); - let mam_id = meta.mam_id; let diff = torrent.meta.diff(&meta); debug!( "Updating meta for torrent {}, diff:\n{}", - mam_id, + id, diff.iter() .map(|field| format!(" {}: {} → {}", field.field, field.from, field.to)) .join("\n") @@ -299,9 +300,17 @@ async fn update_torrent_meta( drop(guard); if !diff.is_empty() { + let mam_id = mam_torrent.map(|m| m.id); write_event( db, - Event::new(Some(id), Some(mam_id), EventType::Updated { fields: diff }), + Event::new( + Some(id), + mam_id, + EventType::Updated { + fields: diff, + source: (meta.source.clone(), String::new()), + }, + ), ) .await; } diff --git a/server/src/stats.rs b/server/src/stats.rs index 186aa6b1..3bda86bf 100644 --- a/server/src/stats.rs +++ b/server/src/stats.rs @@ -18,8 +18,10 @@ pub struct StatsValues { pub autograbber_result: BTreeMap>, pub import_run_at: BTreeMap, pub import_result: BTreeMap>, - pub linker_run_at: Option, - pub linker_result: Option>, + pub folder_linker_run_at: Option, + pub folder_linker_result: Option>, + pub torrent_linker_run_at: Option, + pub torrent_linker_result: Option>, pub cleaner_run_at: Option, pub cleaner_result: Option>, pub downloader_run_at: Option, @@ -62,7 +64,8 @@ pub struct Events { pub struct Triggers { pub search_tx: BTreeMap>, pub import_tx: BTreeMap>, - pub linker_tx: Sender<()>, + pub torrent_linker_tx: Sender<()>, + pub folder_linker_tx: Sender<()>, pub downloader_tx: Sender<()>, pub audiobookshelf_tx: Sender<()>, } diff --git a/server/src/torrent_downloader.rs b/server/src/torrent_downloader.rs index b404ed12..459d2271 100644 --- a/server/src/torrent_downloader.rs +++ b/server/src/torrent_downloader.rs @@ -52,8 +52,9 @@ pub async fn grab_selected_torrents( .map(|t| t.meta.size.bytes() as f64) .sum(); - let mut remaining_buffer = (user_info.uploaded_bytes - user_info.downloaded_bytes - downloading_size) - / config.min_ratio; + let mut remaining_buffer = + (user_info.uploaded_bytes - user_info.downloaded_bytes - downloading_size) + / config.min_ratio; debug!( "downloader, unsats: {:#?}; max_torrents: {max_torrents}; buffer: {}", user_info.unsat, @@ -135,9 +136,7 @@ async fn grab_torrent( rw.upsert(mlm_db::Torrent { id: hash.clone(), id_is_hash: true, - mam_id: torrent.meta.mam_id, - abs_id: None, - goodreads_id: torrent.goodreads_id, + mam_id: torrent.meta.mam_id(), library_path: None, library_files: Default::default(), linker: None, @@ -148,7 +147,6 @@ async fn grab_torrent( meta: torrent.meta.clone(), created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, })?; @@ -203,8 +201,8 @@ async fn grab_torrent( match err.downcast::() { Ok( WedgeBuyError::IsVip - | WedgeBuyError::IsGlobalFreeleech - | WedgeBuyError::IsPersonalFreeleech, + | WedgeBuyError::IsGlobalFreeleech + | WedgeBuyError::IsPersonalFreeleech, ) => {} _ => { if torrent.cost == TorrentCost::UseWedge { @@ -219,7 +217,10 @@ async fn grab_torrent( return Err(anyhow!("Could not get torrent from MaM")); }; if !torrent_info.is_free() { - return Err(anyhow!("Torrent is no longer free, expected: {:?}", torrent.cost)); + return Err(anyhow!( + "Torrent is no longer free, expected: {:?}", + torrent.cost + )); } } @@ -252,9 +253,7 @@ async fn grab_torrent( rw.upsert(mlm_db::Torrent { id: hash.clone(), id_is_hash: true, - mam_id: torrent.meta.mam_id, - abs_id: None, - goodreads_id: torrent.goodreads_id, + mam_id: torrent.meta.mam_id(), library_path: None, library_files: Default::default(), linker: None, @@ -265,7 +264,6 @@ async fn grab_torrent( meta: torrent.meta.clone(), created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, })?; diff --git a/server/src/web/mod.rs b/server/src/web/mod.rs index dc4b37be..e61f24d8 100644 --- a/server/src/web/mod.rs +++ b/server/src/web/mod.rs @@ -16,7 +16,7 @@ use axum::{ }; use itertools::Itertools; use mlm_db::{ - AudiobookCategory, Category, EbookCategory, Flags, SelectedTorrent, Series, Timestamp, Torrent, + AudiobookCategory, EbookCategory, Flags, SelectedTorrent, Series, Timestamp, Torrent, TorrentMeta, }; use mlm_mam::{api::MaM, meta::MetaError, search::MaMTorrent, serde::DATE_FORMAT}; @@ -214,14 +214,6 @@ pub trait Page { } } - fn cats<'a, T: Key>(&'a self, field: T, cats: &'a Vec) -> CatsTmpl<'a, T> { - CatsTmpl { - field, - cats, - path: self.item_path(), - } - } - fn series<'a, T: Key>(&'a self, field: T, series: &'a Vec) -> SeriesTmpl<'a, T> { SeriesTmpl { field, @@ -315,32 +307,6 @@ impl<'a, T: Key> SeriesTmpl<'a, T> { } } -/// ```askama -/// {% for c in cats %} -/// {{ item(*field, **c) | safe }}{% if !loop.last %}, {% endif %} -/// {% endfor %} -/// ``` -#[derive(Template)] -#[template(ext = "html", in_doc = true)] -pub struct CatsTmpl<'a, T: Key> { - field: T, - cats: &'a Vec, - path: &'a str, -} - -impl<'a, T: Key> HtmlSafe for CatsTmpl<'a, T> {} - -impl<'a, T: Key> CatsTmpl<'a, T> { - fn item(&'a self, field: T, cat: Category) -> ItemFilter<'a, T> { - ItemFilter { - field, - label: cat.as_str(), - value: Some(cat.as_id().to_string()), - path: self.path, - } - } -} - impl<'a, T: Key> HtmlSafe for SeriesTmpl<'a, T> {} #[derive(Template)] diff --git a/server/src/web/pages/config.rs b/server/src/web/pages/config.rs index eb34e51e..782391ea 100644 --- a/server/src/web/pages/config.rs +++ b/server/src/web/pages/config.rs @@ -67,10 +67,12 @@ pub async fn config_page_post( } } Err(err) => { + let Some(mam_id) = torrent.mam_id else { + continue; + }; info!("need to ask mam due to: {err}"); let mam = context.mam()?; - let Some(mam_torrent) = mam.get_torrent_info_by_id(torrent.mam_id).await? - else { + let Some(mam_torrent) = mam.get_torrent_info_by_id(mam_id).await? else { warn!("could not get torrent from mam"); continue; }; @@ -80,7 +82,7 @@ pub async fn config_page_post( &config, &context.db, context.db.rw_async().await?, - &mam_torrent, + Some(&mam_torrent), torrent.clone(), new_meta, false, @@ -180,6 +182,7 @@ impl TorrentSearch { } for cat in self .filter + .edition .categories .audio .clone() @@ -191,6 +194,7 @@ impl TorrentSearch { } for cat in self .filter + .edition .categories .ebook .clone() @@ -200,11 +204,11 @@ impl TorrentSearch { { query.append_pair("tor[cat][]", &cat.to_string()); } - for lang in &self.filter.languages { + for lang in &self.filter.edition.languages { query.append_pair("tor[browse_lang][]", &lang.to_id().to_string()); } - let (flags_is_hide, flags) = self.filter.flags.as_search_bitfield(); + let (flags_is_hide, flags) = self.filter.edition.flags.as_search_bitfield(); if !flags.is_empty() { query.append_pair( "tor[browseFlagsHideVsShow]", @@ -215,14 +219,21 @@ impl TorrentSearch { query.append_pair("tor[browseFlags][]", &flag.to_string()); } - if self.filter.min_size.bytes() > 0 || self.filter.max_size.bytes() > 0 { + if self.filter.edition.min_size.bytes() > 0 || self.filter.edition.max_size.bytes() > 0 + { query.append_pair("tor[unit]", "1"); } - if self.filter.min_size.bytes() > 0 { - query.append_pair("tor[minSize]", &self.filter.min_size.bytes().to_string()); + if self.filter.edition.min_size.bytes() > 0 { + query.append_pair( + "tor[minSize]", + &self.filter.edition.min_size.bytes().to_string(), + ); } - if self.filter.max_size.bytes() > 0 { - query.append_pair("tor[maxSize]", &self.filter.max_size.bytes().to_string()); + if self.filter.edition.max_size.bytes() > 0 { + query.append_pair( + "tor[maxSize]", + &self.filter.edition.max_size.bytes().to_string(), + ); } if let Some(uploaded_after) = self.filter.uploaded_after { diff --git a/server/src/web/pages/duplicate.rs b/server/src/web/pages/duplicate.rs index a8dc298e..209c6e54 100644 --- a/server/src/web/pages/duplicate.rs +++ b/server/src/web/pages/duplicate.rs @@ -6,7 +6,7 @@ use axum::{ }; use axum_extra::extract::Form; use mlm_db::{ - DatabaseExt as _, DuplicateTorrent, SelectedTorrent, Timestamp, Torrent, TorrentCost, + DatabaseExt as _, DuplicateTorrent, SelectedTorrent, Timestamp, Torrent, TorrentCost, ids, }; use mlm_parse::normalize_title; use serde::{Deserialize, Serialize}; @@ -154,7 +154,6 @@ pub async fn duplicate_torrents_page_post( let (_guard, rw) = context.db.rw_async().await?; rw.insert(SelectedTorrent { mam_id: mam_torrent.id, - goodreads_id: None, hash: None, dl_link: mam_torrent .dl diff --git a/server/src/web/pages/events.rs b/server/src/web/pages/events.rs index 8a401931..4a4b5b86 100644 --- a/server/src/web/pages/events.rs +++ b/server/src/web/pages/events.rs @@ -36,7 +36,7 @@ pub async fn event_page( EventType::Linked { .. } => value == "linker", EventType::Cleaned { .. } => value == "cleaner", EventType::Updated { .. } => value == "updated", - EventType::RemovedFromMam { .. } => value == "removed", + EventType::RemovedFromTracker { .. } => value == "removed", }, EventPageFilter::Grabber => match t.event { EventType::Grabbed { ref grabber, .. } => { diff --git a/server/src/web/pages/index.rs b/server/src/web/pages/index.rs index d8ed1da9..4872635d 100644 --- a/server/src/web/pages/index.rs +++ b/server/src/web/pages/index.rs @@ -57,9 +57,14 @@ pub async fn index_page( .iter() .map(|(i, r)| (*i, r.as_ref().map(|_| ()).map_err(|e| format!("{e:?}")))) .collect(), - linker_run_at: stats.linker_run_at.map(Into::into), - linker_result: stats - .linker_result + torrent_linker_run_at: stats.torrent_linker_run_at.map(Into::into), + torrent_linker_result: stats + .torrent_linker_result + .as_ref() + .map(|r| r.as_ref().map(|_| ()).map_err(|e| format!("{e:?}"))), + folder_linker_run_at: stats.folder_linker_run_at.map(Into::into), + folder_linker_result: stats + .folder_linker_result .as_ref() .map(|r| r.as_ref().map(|_| ()).map_err(|e| format!("{e:?}"))), cleaner_run_at: stats.cleaner_run_at.map(Into::into), @@ -95,8 +100,11 @@ pub async fn index_page_post( Form(form): Form, ) -> Result { match form.action.as_str() { - "run_linker" => { - context.triggers.linker_tx.send(())?; + "run_torrent_linker" => { + context.triggers.torrent_linker_tx.send(())?; + } + "run_folder_linker" => { + context.triggers.folder_linker_tx.send(())?; } "run_search" => { if let Some(tx) = context.triggers.search_tx.get( @@ -146,8 +154,10 @@ struct IndexPageTemplate { autograbber_result: BTreeMap>, import_run_at: BTreeMap, import_result: BTreeMap>, - linker_run_at: Option, - linker_result: Option>, + torrent_linker_run_at: Option, + torrent_linker_result: Option>, + folder_linker_run_at: Option, + folder_linker_result: Option>, cleaner_run_at: Option, cleaner_result: Option>, downloader_run_at: Option, diff --git a/server/src/web/pages/replaced.rs b/server/src/web/pages/replaced.rs index af21364e..f2ba71b1 100644 --- a/server/src/web/pages/replaced.rs +++ b/server/src/web/pages/replaced.rs @@ -10,13 +10,13 @@ use axum::{ response::{Html, Redirect}, }; use axum_extra::extract::Form; -use mlm_db::{Language, Torrent, TorrentKey}; +use mlm_db::{Language, Torrent, TorrentKey, ids}; use serde::{Deserialize, Serialize}; use crate::stats::Context; use crate::web::{Page, tables}; use crate::{ - linker::{refresh_metadata, refresh_metadata_relink}, + linker::{refresh_mam_metadata, refresh_metadata_relink}, web::{ AppError, tables::{Flex, HidableColumns, Key, Pagination, PaginationParams, SortOn, Sortable}, @@ -141,7 +141,7 @@ pub async fn replaced_torrents_page_post( "refresh" => { let mam = context.mam()?; for torrent in form.torrents { - refresh_metadata(&config, &context.db, &mam, torrent).await?; + refresh_mam_metadata(&config, &context.db, &mam, torrent).await?; } } "refresh-relink" => { diff --git a/server/src/web/pages/search.rs b/server/src/web/pages/search.rs index 98a46d6c..dde2f46c 100644 --- a/server/src/web/pages/search.rs +++ b/server/src/web/pages/search.rs @@ -47,7 +47,7 @@ pub async fn search_page( let meta = mam_torrent.as_meta()?; let torrent = r .get() - .secondary::(TorrentKey::mam_id, meta.mam_id)?; + .secondary::(TorrentKey::mam_id, meta.mam_id())?; let selected_torrent = r.get().primary(mam_torrent.id)?; Ok((mam_torrent, meta, torrent, selected_torrent)) @@ -149,7 +149,6 @@ pub async fn select_torrent(context: &Context, mam_id: u64, wedge: bool) -> Resu let (_guard, rw) = context.db.rw_async().await?; rw.insert(SelectedTorrent { mam_id: torrent.id, - goodreads_id: None, hash: None, dl_link: torrent .dl diff --git a/server/src/web/pages/torrent.rs b/server/src/web/pages/torrent.rs index cabaf0be..7fb70985 100644 --- a/server/src/web/pages/torrent.rs +++ b/server/src/web/pages/torrent.rs @@ -33,7 +33,9 @@ use crate::{ audiobookshelf::{Abs, LibraryItemMinified}, cleaner::clean_torrent, config::Config, - linker::{find_library, library_dir, map_path, refresh_metadata, refresh_metadata_relink}, + linker::{ + find_library, library_dir, map_path, refresh_mam_metadata, refresh_metadata_relink, relink, + }, qbittorrent::{self, ensure_category_exists}, stats::Context, web::{ @@ -177,11 +179,19 @@ async fn torrent_page_id( events.sort_by(|a, b| b.created_at.cmp(&a.created_at)); let mam = context.mam()?; - let mam_torrent = mam.get_torrent_info_by_id(torrent.mam_id).await?; + let mam_torrent = if let Some(mam_id) = torrent.mam_id { + mam.get_torrent_info_by_id(mam_id).await? + } else { + None + }; let mam_meta = mam_torrent.as_ref().map(|t| t.as_meta()).transpose()?; if let Some(mam_meta) = &mam_meta - && torrent.meta.uploaded_at.0 == UtcDateTime::UNIX_EPOCH + && torrent + .meta + .uploaded_at + .as_ref() + .is_none_or(|t| t.0 == UtcDateTime::UNIX_EPOCH) { let (_guard, rw) = context.db.rw_async().await?; torrent.meta.uploaded_at = mam_meta.uploaded_at; @@ -326,7 +336,10 @@ pub async fn torrent_page_post_id( } "refresh" => { let mam = context.mam()?; - refresh_metadata(&config, &context.db, &mam, id).await?; + refresh_mam_metadata(&config, &context.db, &mam, id).await?; + } + "relink" => { + relink(&config, &context.db, id).await?; } "refresh-relink" => { let mam = context.mam()?; @@ -362,7 +375,8 @@ pub async fn torrent_page_post_id( rw.commit()?; } "qbit" => { - let Some((torrent, qbit, qbit_conf)) = qbittorrent::get_torrent(&config, &id).await? else { + let Some((torrent, qbit, qbit_conf)) = qbittorrent::get_torrent(&config, &id).await? + else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; @@ -562,12 +576,12 @@ async fn other_torrents( let torrents = result .data .into_iter() - .filter(|t| t.id != meta.mam_id) + .filter(|t| Some(t.id) != meta.mam_id()) .map(|mam_torrent| { let meta = mam_torrent.as_meta()?; let torrent = r .get() - .secondary::(TorrentKey::mam_id, meta.mam_id)?; + .secondary::(TorrentKey::mam_id, meta.mam_id())?; let selected_torrent = r.get().primary(mam_torrent.id)?; Ok((mam_torrent, meta, torrent, selected_torrent)) diff --git a/server/src/web/pages/torrent_edit.rs b/server/src/web/pages/torrent_edit.rs index 90f589cd..2d09e677 100644 --- a/server/src/web/pages/torrent_edit.rs +++ b/server/src/web/pages/torrent_edit.rs @@ -41,7 +41,7 @@ pub async fn torrent_edit_page_post( Form(form): Form, ) -> Result { let config = context.config().await; - let mam = context.mam()?; + let _mam = context.mam()?; let Some(torrent) = context .db .r_transaction()? @@ -50,9 +50,6 @@ pub async fn torrent_edit_page_post( else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; - let Some(mam_torrent) = mam.get_torrent_info(&hash).await? else { - return Err(anyhow::Error::msg("Could not find torrent on MaM").into()); - }; let authors = form.authors.split("\r\n").map(ToOwned::to_owned).collect(); let narrators = if form.narrators.trim().is_empty() { @@ -78,10 +75,12 @@ pub async fn torrent_edit_page_post( }) .collect::, _>>()? }; - let language = - Language::from_id(form.language).ok_or_else(|| anyhow::Error::msg("Invalid language"))?; - let category = OldCategory::from_one_id(form.category) - .ok_or_else(|| anyhow::Error::msg("Invalid category"))?; + let language = form.language.and_then(Language::from_id); + let category = form.category.and_then(OldCategory::from_one_id); + let media_type = match &category { + Some(c) => c.as_main_cat().into(), + None => torrent.meta.media_type, + }; let flags = Flags { crude_language: Some(form.crude_language), violence: Some(form.violence), @@ -93,9 +92,9 @@ pub async fn torrent_edit_page_post( let meta = TorrentMeta { title: form.title, - media_type: category.as_main_cat().into(), - cat: Some(category), - language: Some(language), + media_type, + cat: category, + language, flags: Some(FlagBits::new(flags.as_bitfield())), authors, narrators, @@ -108,7 +107,7 @@ pub async fn torrent_edit_page_post( &config, &context.db, context.db.rw_async().await?, - &mam_torrent, + None, torrent, meta, true, @@ -125,8 +124,10 @@ pub struct TorrentPageForm { authors: String, narrators: String, series: String, - language: u8, - category: u64, + #[serde(default)] + language: Option, + #[serde(default)] + category: Option, // #[serde(flatten)] // flags: Flags, diff --git a/server/src/web/pages/torrents.rs b/server/src/web/pages/torrents.rs index 26715dc9..7f5c7624 100644 --- a/server/src/web/pages/torrents.rs +++ b/server/src/web/pages/torrents.rs @@ -11,13 +11,14 @@ use axum::{ response::{Html, Redirect}, }; use axum_extra::extract::Form; +use mlm_db::ids; use mlm_db::{Language, LibraryMismatch, Torrent, TorrentKey}; use serde::{Deserialize, Serialize}; use sublime_fuzzy::FuzzySearch; use crate::{ cleaner::clean_torrent, - linker::{refresh_metadata, refresh_metadata_relink}, + linker::{refresh_mam_metadata, refresh_metadata_relink}, stats::Context, web::{ AppError, @@ -27,8 +28,8 @@ use crate::{ web::{Page, flag_icons, tables}, }; use mlm_db::{ - Category, ClientStatus, DatabaseExt as _, Flags, MediaType, MetadataSource, OldCategory, - Series, SeriesEntry, + ClientStatus, DatabaseExt as _, Flags, MediaType, MetadataSource, OldCategory, Series, + SeriesEntry, }; pub async fn torrents_page( @@ -98,9 +99,7 @@ pub async fn torrents_page( } else { value .split(",") - .filter_map(|id| id.parse().ok()) - .filter_map(Category::from_id) - .all(|cat| t.meta.categories.contains(&cat)) + .all(|cat| t.meta.categories.iter().any(|c| c.as_str() == cat.trim())) } } TorrentsPageFilter::Flags => { @@ -184,10 +183,10 @@ pub async fn torrents_page( } TorrentsPageFilter::ClientStatus => match t.client_status { Some(ClientStatus::NotInClient) => value == "not_in_client", - Some(ClientStatus::RemovedFromMam) => value == "removed_from_mam", + Some(ClientStatus::RemovedFromTracker) => value == "removed_from_tracker", None => false, }, - TorrentsPageFilter::Abs => t.abs_id.is_some() == (value == "true"), + TorrentsPageFilter::Abs => t.meta.ids.contains_key(ids::ABS) == (value == "true"), TorrentsPageFilter::Source => match value.as_str() { "mam" => t.meta.source == MetadataSource::Mam, "manual" => t.meta.source == MetadataSource::Manual, @@ -641,7 +640,7 @@ pub async fn torrents_page_post( "refresh" => { let mam = context.mam()?; for torrent in form.torrents { - refresh_metadata(&config, &context.db, &mam, torrent).await?; + refresh_mam_metadata(&config, &context.db, &mam, torrent).await?; } } "refresh-relink" => { diff --git a/server/templates/pages/config.html b/server/templates/pages/config.html index 1430e253..cb006ca5 100644 --- a/server/templates/pages/config.html +++ b/server/templates/pages/config.html @@ -169,32 +169,33 @@

[[tag]]

[[library]]

- {% if let Some(name) = library.tag_filters().name %} + {% if let Some(name) = library.options().name %} name = {{ name | json }}
{% endif %} {% match library %} - {% when Library::ByDir(library) %} + {% when Library::ByRipDir(library) %} + rip_dir = {{ library.rip_dir | json }}
+ {% when Library::ByDownloadDir(library) %} download_dir = {{ library.download_dir | json }}
- library_dir = {{ library.library_dir | json }}
{% when Library::ByCategory(library) %} category = {{ library.category | json }}
- library_dir = {{ library.library_dir | json }}
{% endmatch %} + library_dir = {{ library.options().library_dir | json }}
{% if !library.tag_filters().allow_tags.is_empty() %} allow_tags = {{ self::yaml_items(library.tag_filters().allow_tags) }}
{% endif %} {% if !library.tag_filters().deny_tags.is_empty() %} deny_tags = {{ self::yaml_items(library.tag_filters().deny_tags) }}
{% endif %} - {% if library.tag_filters().method != Default::default() %} - method = {{ library.tag_filters().method | json }}
+ {% if library.options().method != Default::default() %} + method = {{ library.options().method | json }}
{% endif %} - {% if let Some(audio_types) = library.tag_filters().audio_types %} + {% if let Some(audio_types) = library.options().audio_types %} {% if !audio_types.is_empty() %} audio_types = {{ self::yaml_items(audio_types) }}
{% endif %} {% endif %} - {% if let Some(ebook_types) = library.tag_filters().ebook_types %} + {% if let Some(ebook_types) = library.options().ebook_types %} {% if !ebook_types.is_empty() %} ebook_types = {{ self::yaml_items(ebook_types) }}
{% endif %} diff --git a/server/templates/pages/duplicate.html b/server/templates/pages/duplicate.html index a975adae..6e71cd86 100644 --- a/server/templates/pages/duplicate.html +++ b/server/templates/pages/duplicate.html @@ -43,7 +43,7 @@

Duplicate Torrents

{{ items(DuplicatePageFilter::Filetype, torrent.meta.filetypes) }}
{{ self::time(torrent.created_at) }}
- +
duplicate of:
{{ item(DuplicatePageFilter::Title, duplicate_of.meta.title) }}
@@ -58,8 +58,8 @@

Duplicate Torrents

{{ self::time(duplicate_of.created_at) }}
open - MaM - {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), duplicate_of.abs_id.as_ref()) %} + {% if let Some(mam_id) = duplicate_of.mam_id %}MaM{% endif %} + {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), duplicate_of.meta.ids.get(ids::ABS)) %} ABS {% endif %}
diff --git a/server/templates/pages/errors.html b/server/templates/pages/errors.html index fac81cfa..77c7ac09 100644 --- a/server/templates/pages/errors.html +++ b/server/templates/pages/errors.html @@ -36,8 +36,10 @@

Torrent Errors

{% match error.meta %} {% when Some(meta) %} - open - MaM + {% if let Some(mam_id) = meta.mam_id() %} + open + MaM + {% endif %} {% when None %} {% endmatch %}
diff --git a/server/templates/pages/events.html b/server/templates/pages/events.html index 68b7c5a9..6e37f976 100644 --- a/server/templates/pages/events.html +++ b/server/templates/pages/events.html @@ -106,15 +106,15 @@

Events

{% endfor %} - {% when EventType::Updated { fields } %} - Updated {{ torrent_media_type(&torrent) }} Torrent {{ torrent_title(&torrent) | safe }}
+ {% when EventType::Updated { fields, source } %} + Updated {{ torrent_media_type(&torrent) }} Torrent {{ torrent_title(&torrent) | safe }} from {{ source.0 }} {{ source.1 }}
    {% for field in fields %}
  • {{ field.field }}: {{ field.from }} → {{ field.to }}
    • {% endfor %}
- {% when EventType::RemovedFromMam %} - {{ torrent_media_type(&torrent) }} Torrent {{ torrent_title(&torrent) | safe }} was removed from MaM
+ {% when EventType::RemovedFromTracker %} + {{ torrent_media_type(&torrent) }} Torrent {{ torrent_title(&torrent) | safe }} was removed from Tracker
{% endmatch %} {% endfor %} diff --git a/server/templates/pages/index.html b/server/templates/pages/index.html index 3f915bf4..5fa22f36 100644 --- a/server/templates/pages/index.html +++ b/server/templates/pages/index.html @@ -60,11 +60,20 @@

{{ list.list_type() }} Import: {{ list.display_name(*i) }}

-

Linker

-

Last run: {% match linker_run_at %}{% when Some(run_at) %}{{ self::time(run_at) }}{% when None %}never{% endmatch %} - - {% if linker_run_at.is_some() %} -

Result: {% match linker_result %}{% when Some(Ok(())) %}success{% when Some(Err(err)) %}{{ err }}{% when None %}running{% endmatch %} +

Torrent Linker

+

Last run: {% match torrent_linker_run_at %}{% when Some(run_at) %}{{ self::time(run_at) }}{% when None %}never{% endmatch %} + + {% if torrent_linker_run_at.is_some() %} +

Result: {% match torrent_linker_result %}{% when Some(Ok(())) %}success{% when Some(Err(err)) %}{{ err }}{% when None %}running{% endmatch %} + {% endif %} +

+ +
+

Folder Linker

+

Last run: {% match folder_linker_run_at %}{% when Some(run_at) %}{{ self::time(run_at) }}{% when None %}never{% endmatch %} + + {% if folder_linker_run_at.is_some() %} +

Result: {% match folder_linker_result %}{% when Some(Ok(())) %}success{% when Some(Err(err)) %}{{ err }}{% when None %}running{% endmatch %} {% endif %}

diff --git a/server/templates/pages/list.html b/server/templates/pages/list.html index 346e0bf0..91e95df4 100644 --- a/server/templates/pages/list.html +++ b/server/templates/pages/list.html @@ -47,13 +47,13 @@

{{ item.title }}

{% if let Some(torrent) = item.audio_torrent %} {% match torrent.status %} {% when TorrentStatus::Selected %} - Downloaded audiobook torrent at {{ self::time(torrent.at) }}
+ Downloaded audiobook torrent at {{ self::time(torrent.at) }}
{% when TorrentStatus::Wanted %} - Suggest wedge audiobook torrent at {{ self::time(torrent.at) }}
+ Suggest wedge audiobook torrent at {{ self::time(torrent.at) }}
{% when TorrentStatus::NotWanted %} - Skipped audiobook torrent as an ebook was found at {{ self::time(torrent.at) }}
+ Skipped audiobook torrent as an ebook was found at {{ self::time(torrent.at) }}
{% when TorrentStatus::Existing %} - Found matching audiobook torrent in library at {{ self::time(torrent.at) }}
+ Found matching audiobook torrent in library at {{ self::time(torrent.at) }}
{% endmatch %} {% elif item.want_audio() %} Audiobook missing
@@ -61,13 +61,13 @@

{{ item.title }}

{% if let Some(torrent) = item.ebook_torrent %} {% match torrent.status %} {% when TorrentStatus::Selected %} - Downloaded ebook torrent at {{ self::time(torrent.at) }}
+ Downloaded ebook torrent at {{ self::time(torrent.at) }}
{% when TorrentStatus::Wanted %} - Suggest wedge ebook torrent at {{ self::time(torrent.at) }}
+ Suggest wedge ebook torrent at {{ self::time(torrent.at) }}
{% when TorrentStatus::NotWanted %} - Skipped ebook torrent as an ebook was found at {{ self::time(torrent.at) }}
+ Skipped ebook torrent as an ebook was found at {{ self::time(torrent.at) }}
{% when TorrentStatus::Existing %} - Found matching ebook torrent in library at {{ self::time(torrent.at) }}
+ Found matching ebook torrent in library at {{ self::time(torrent.at) }}
{% endmatch %} {% elif item.want_ebook() %} Ebook missing
diff --git a/server/templates/pages/replaced.html b/server/templates/pages/replaced.html index 00fb307d..6d33ab79 100644 --- a/server/templates/pages/replaced.html +++ b/server/templates/pages/replaced.html @@ -97,8 +97,8 @@

Replaced Torrents

{{ self::time(torrent.created_at) }}
open - MaM - {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), torrent.abs_id.as_ref()) %} + {% if let Some(mam_id) = torrent.mam_id %}MaM{% endif %} + {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), torrent.meta.ids.get(ids::ABS)) %} ABS {% endif %}
@@ -130,8 +130,8 @@

Replaced Torrents

{{ self::time(replacement.created_at) }}
open - MaM - {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), replacement.abs_id.as_ref()) %} + {% if let Some(mam_id) = replacement.mam_id %}MaM{% endif %} + {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), replacement.meta.ids.get(ids::ABS)) %} ABS {% endif %}
diff --git a/server/templates/pages/selected.html b/server/templates/pages/selected.html index f249521d..ddc86fca 100644 --- a/server/templates/pages/selected.html +++ b/server/templates/pages/selected.html @@ -168,7 +168,7 @@

Selected Torrents

{% endif %}
{% endif %} - + {% endfor %} {% if torrents.is_empty() %} diff --git a/server/templates/pages/torrent.html b/server/templates/pages/torrent.html index 9d1aa874..9fe08900 100644 --- a/server/templates/pages/torrent.html +++ b/server/templates/pages/torrent.html @@ -31,7 +31,7 @@

Replaced with: {{ torrent.meta.title }}
- id: {{ torrent.mam_id }} {% if let Some(vip_status) = torrent.meta.vip_status %}{{ vip_status }}{% endif %} + {% if let Some(mam_id) = torrent.mam_id %}id: {{ mam_id }} {% endif %}{% if let Some(vip_status) = torrent.meta.vip_status %}{{ vip_status }}{% endif %} {% if let Some(book) = book %} Open in ABS {% endif %} @@ -65,7 +65,9 @@

Replaced with: {{ torrent.meta.title }}Uploader: {{ mam_torrent.owner_name }}

{% endif %} -

Uploaded At: {{ self::time(torrent.meta.uploaded_at) }}

+{% if let Some(uploaded_at) = torrent.meta.uploaded_at %} +

Uploaded At: {{ self::time(uploaded_at) }}

+{% endif %} {% if let Some(library_path) = torrent.library_path %}
Linked Path: {{ library_path.to_string_lossy() }} @@ -79,6 +81,9 @@

Replaced with: {{ torrent.meta.title }} Wanted Path: {{ wanted_path.to_string_lossy() }} +
+ +

{% else %}
@@ -269,15 +274,15 @@

Replaced with: {{ torrent.meta.title }} - {% when EventType::Updated { fields } %} - Updated Torrent
+ {% when EventType::Updated { fields, source } %} + Updated Torrent from {{ source.0 }} {{ source.1 }}
    {% for field in fields %}
  • {{ field.field }}: {{ field.from }} -> {{ field.to }}
    • {% endfor %}
- {% when EventType::RemovedFromMam %} - Torrent was removed from MaM
+ {% when EventType::RemovedFromTracker %} + Torrent was removed from Tracker
{% endmatch %}

{% endfor %} diff --git a/server/templates/pages/torrent_edit.html b/server/templates/pages/torrent_edit.html index b1e8e830..1d1093a0 100644 --- a/server/templates/pages/torrent_edit.html +++ b/server/templates/pages/torrent_edit.html @@ -29,6 +29,7 @@

{{ torrent.meta.title }}

@@ -58,7 +58,9 @@

{{ meta.title }}

Uploader: {{ mam_torrent.owner_name }}

-

Uploaded At: {{ self::time(meta.uploaded_at) }}

+{% if let Some(uploaded_at) = meta.uploaded_at %} +

Uploaded At: {{ self::time(uploaded_at) }}

+{% endif %}

{{ mam_torrent.tags }}

diff --git a/server/templates/pages/torrents.html b/server/templates/pages/torrents.html index facaf17a..44ec193c 100644 --- a/server/templates/pages/torrents.html +++ b/server/templates/pages/torrents.html @@ -134,7 +134,11 @@

Torrents

{% endif %}
{% if show.categories %} -
{{ cats(TorrentsPageFilter::Categories, torrent.meta.categories) }}
+
+ {% for cat in torrent.meta.categories %} + {{ item(TorrentsPageFilter::Categories, cat.as_str()) }}{% if !loop.last %}, {% endif %} + {% endfor %} +
{% endif %} {% if show.flags %}
{{ self::flag_icons(torrent.meta) }}
@@ -142,12 +146,12 @@

Torrents

{{ item(TorrentsPageFilter::Title, torrent.meta.title) }} {% match torrent.client_status %} - {% when Some(ClientStatus::RemovedFromMam) %} - - {{ item_v(TorrentsPageFilter::ClientStatus, "⚠", "removed_from_mam") }} + {% when Some(ClientStatus::RemovedFromTracker) %} + + {{ item_v(TorrentsPageFilter::ClientStatus, "⚠", "removed_from_tracker") }} {% when Some(ClientStatus::NotInClient) %} - + {{ item_v(TorrentsPageFilter::ClientStatus, "ℹ", "not_in_client") }} {% when None %} @@ -223,12 +227,12 @@

Torrents

{{ self::time(torrent.created_at) }}
{% endif %} {% if show.uploaded_at %} -
{{ self::time(torrent.meta.uploaded_at) }}
+
{% if let Some(uploaded_at) = torrent.meta.uploaded_at %}{{ self::time(uploaded_at) }}{% endif %}
{% endif %}
open - MaM - {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), torrent.abs_id.as_ref()) %} + {% if let Some(mam_id) = torrent.mam_id %}MaM{% endif %} + {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), torrent.meta.ids.get(ids::ABS)) %} ABS {% endif %}
diff --git a/server/templates/partials/filter.html b/server/templates/partials/filter.html index ab31b9a2..7bc26f04 100644 --- a/server/templates/partials/filter.html +++ b/server/templates/partials/filter.html @@ -1,7 +1,7 @@ -{% if filter.categories.audio.is_some() || filter.categories.ebook.is_some() %} - categories = +{% if filter.edition.categories.audio.is_some() || filter.edition.categories.ebook.is_some() %} + categories = {
-{% if let Some(cats) = filter.categories.audio %} +{% if let Some(cats) = filter.edition.categories.audio %} {% if cats.is_empty() %}   audio = false
{% elif cats == &AudiobookCategory::all() %} @@ -12,7 +12,7 @@ {% else %}   audio = true
{% endif %} -{% if let Some(cats) = filter.categories.ebook %} +{% if let Some(cats) = filter.edition.categories.ebook %} {% if cats.is_empty() %}   ebook = false
{% elif cats == &EbookCategory::all() %} @@ -26,58 +26,58 @@ }
{% else %} {% endif %} -{% if !filter.languages.is_empty() %} -languages = {{ self::yaml_items(filter.languages) }}
+{% if !filter.edition.languages.is_empty() %} +languages = {{ self::yaml_items(filter.edition.languages) }}
{% endif %} -{% if filter.flags.as_bitfield() > 0 %} +{% if filter.edition.flags.as_bitfield() > 0 %} flags = { -{% if filter.flags.as_search_bitfield().1.len() > 3 %} +{% if filter.edition.flags.as_search_bitfield().1.len() > 3 %}
- {% if let Some(flag) = filter.flags.crude_language %} + {% if let Some(flag) = filter.edition.flags.crude_language %}   crude_language = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.violence %} + {% if let Some(flag) = filter.edition.flags.violence %}   violence = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.some_explicit %} + {% if let Some(flag) = filter.edition.flags.some_explicit %}   some_explicit = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.explicit %} + {% if let Some(flag) = filter.edition.flags.explicit %}   explicit = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.abridged %} + {% if let Some(flag) = filter.edition.flags.abridged %}   abridged = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.lgbt %} + {% if let Some(flag) = filter.edition.flags.lgbt %}   lgbt = {{ flag }}
{% endif %} {% else %} - {% if let Some(flag) = filter.flags.crude_language %} + {% if let Some(flag) = filter.edition.flags.crude_language %} crude_language = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.violence %} + {% if let Some(flag) = filter.edition.flags.violence %} violence = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.some_explicit %} + {% if let Some(flag) = filter.edition.flags.some_explicit %} some_explicit = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.explicit %} + {% if let Some(flag) = filter.edition.flags.explicit %} explicit = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.abridged %} + {% if let Some(flag) = filter.edition.flags.abridged %} abridged = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.lgbt %} + {% if let Some(flag) = filter.edition.flags.lgbt %} lgbt = {{ flag }} {% endif %} {% endif %} }
{% endif %} -{% if filter.min_size.bytes() > 0 %} -min_size = "{{ filter.min_size }}"
+{% if filter.edition.min_size.bytes() > 0 %} +min_size = "{{ filter.edition.min_size }}"
{% endif %} -{% if filter.max_size.bytes() > 0 %} -max_size = "{{ filter.max_size }}"
+{% if filter.edition.max_size.bytes() > 0 %} +max_size = "{{ filter.edition.max_size }}"
{% endif %} {% if !filter.exclude_uploader.is_empty() %} exclude_uploader = {{ self::yaml_items(filter.exclude_uploader) }}
diff --git a/server/templates/partials/mam_torrents.html b/server/templates/partials/mam_torrents.html index 4e7fde37..35545766 100644 --- a/server/templates/partials/mam_torrents.html +++ b/server/templates/partials/mam_torrents.html @@ -20,7 +20,7 @@ {% if mam_torrent.lang_code != "ENG" %} [{{ mam_torrent.lang_code }}] {% endif %} - {{ meta.title }}{% if let Some((edition, _)) = meta.edition %} {{ edition }}{% endif %}
+ {{ meta.title }}{% if let Some((edition, _)) = meta.edition %} {{ edition }}{% endif %}
by {% for author in meta.authors %}{{ author }}{% if !loop.last %}, {% endif %}{% endfor %}
{% if !meta.series.is_empty() %} series @@ -50,7 +50,7 @@ Torrent is downloaded {% else %}
- + {% if let Some(wedge_over) = config.wedge_over %} {% if &meta.size >= &wedge_over && !mam_torrent.is_free() %}