From cd79f7b1a9379aaaff8b39080c2d8be4229fd6ae Mon Sep 17 00:00:00 2001 From: Stirling Mouse <181794392+StirlingMouse@users.noreply.github.com> Date: Sat, 24 Jan 2026 20:37:19 +0100 Subject: [PATCH 01/14] Start to decouple from MaM --- mlm_db/src/impls.rs | 1 + mlm_db/src/impls/lists.rs | 10 + mlm_db/src/impls/meta.rs | 36 +- mlm_db/src/lib.rs | 52 ++- mlm_db/src/v04.rs | 12 +- mlm_db/src/v05.rs | 24 +- mlm_db/src/v08.rs | 11 +- mlm_db/src/v10.rs | 12 +- mlm_db/src/v17.rs | 202 ++++++++- mlm_db/src/v18.rs | 463 ++++++++++++++++++++ mlm_mam/src/enums.rs | 56 ++- mlm_mam/src/meta.rs | 67 +++ mlm_mam/src/search.rs | 34 +- mlm_mam/src/user_torrent.rs | 22 +- server/src/audiobookshelf.rs | 52 +-- server/src/autograbber.rs | 103 +++-- server/src/cleaner.rs | 15 +- server/src/config_impl.rs | 121 +++-- server/src/linker.rs | 119 ++++- server/src/lists/goodreads.rs | 22 +- server/src/lists/mod.rs | 7 +- server/src/qbittorrent.rs | 2 +- server/src/snatchlist.rs | 23 +- server/src/torrent_downloader.rs | 24 +- server/src/web/mod.rs | 36 +- server/src/web/pages/config.rs | 6 +- server/src/web/pages/duplicate.rs | 3 +- server/src/web/pages/events.rs | 2 +- server/src/web/pages/replaced.rs | 6 +- server/src/web/pages/search.rs | 3 +- server/src/web/pages/torrent.rs | 22 +- server/src/web/pages/torrents.rs | 17 +- server/templates/pages/duplicate.html | 6 +- server/templates/pages/errors.html | 6 +- server/templates/pages/events.html | 8 +- server/templates/pages/list.html | 16 +- server/templates/pages/replaced.html | 8 +- server/templates/pages/selected.html | 2 +- server/templates/pages/torrent.html | 13 +- server/templates/pages/torrent_mam.html | 2 +- server/templates/pages/torrents.html | 14 +- server/templates/partials/mam_torrents.html | 4 +- 42 files changed, 1295 insertions(+), 369 deletions(-) create mode 100644 mlm_db/src/impls/lists.rs create mode 100644 mlm_db/src/v18.rs diff --git a/mlm_db/src/impls.rs b/mlm_db/src/impls.rs index 433ea964..98576cdb 100644 --- a/mlm_db/src/impls.rs +++ b/mlm_db/src/impls.rs @@ -1,6 +1,7 @@ pub mod categories; pub mod flags; pub mod language; +pub mod lists; pub mod meta; pub mod old_categories; pub mod series; diff --git a/mlm_db/src/impls/lists.rs b/mlm_db/src/impls/lists.rs new file mode 100644 index 00000000..318a8703 --- /dev/null +++ b/mlm_db/src/impls/lists.rs @@ -0,0 +1,10 @@ +use crate::ListItemTorrent; + +impl ListItemTorrent { + pub fn id(&self) -> String { + self.torrent_id + .clone() + .or_else(|| self.mam_id.map(|id| id.to_string())) + .unwrap_or_default() + } +} diff --git a/mlm_db/src/impls/meta.rs b/mlm_db/src/impls/meta.rs index 619ff2f0..091c3a78 100644 --- a/mlm_db/src/impls/meta.rs +++ b/mlm_db/src/impls/meta.rs @@ -4,10 +4,14 @@ use itertools::Itertools as _; use crate::{ Flags, MediaType, MetadataSource, OldCategory, TorrentMeta, TorrentMetaDiff, TorrentMetaField, - VipStatus, impls::format_serie, + VipStatus, ids, impls::format_serie, }; impl TorrentMeta { + pub fn mam_id(&self) -> Option { + self.ids.get(ids::MAM).and_then(|id| id.parse().ok()) + } + pub fn matches(&self, other: &TorrentMeta) -> bool { self.media_type.matches(other.media_type) && self.language == other.language @@ -35,11 +39,16 @@ impl TorrentMeta { pub fn diff(&self, other: &TorrentMeta) -> Vec { let mut diff = vec![]; - if self.mam_id != other.mam_id { + if self.ids != other.ids { + let format_ids = |ids: &std::collections::BTreeMap| { + ids.iter() + .map(|(key, value)| format!("{key}: {value}")) + .join("\n") + }; diff.push(TorrentMetaDiff { - field: TorrentMetaField::MamId, - from: self.mam_id.to_string(), - to: other.mam_id.to_string(), + field: TorrentMetaField::Ids, + from: format_ids(&self.ids), + to: format_ids(&other.ids), }); } if self.vip_status != other.vip_status @@ -96,16 +105,8 @@ impl TorrentMeta { if self.categories != other.categories { diff.push(TorrentMetaDiff { field: TorrentMetaField::Categories, - from: self - .categories - .iter() - .map(|cat| cat.as_raw_str().to_string()) - .join(", "), - to: other - .categories - .iter() - .map(|cat| cat.as_raw_str().to_string()) - .join(", "), + from: self.categories.join(", "), + to: other.categories.join(", "), }); } if self.language != other.language { @@ -218,12 +219,13 @@ impl MediaType { impl std::fmt::Display for TorrentMetaField { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - TorrentMetaField::MamId => write!(f, "mam_id"), + TorrentMetaField::Ids => write!(f, "ids"), TorrentMetaField::Vip => write!(f, "vip"), TorrentMetaField::Cat => write!(f, "cat"), TorrentMetaField::MediaType => write!(f, "media_type"), TorrentMetaField::MainCat => write!(f, "main_cat"), TorrentMetaField::Categories => write!(f, "categories"), + TorrentMetaField::Tags => write!(f, "tags"), TorrentMetaField::Language => write!(f, "language"), TorrentMetaField::Flags => write!(f, "flags"), TorrentMetaField::Filetypes => write!(f, "filetypes"), @@ -243,6 +245,8 @@ impl fmt::Display for MetadataSource { match self { MetadataSource::Mam => write!(f, "MaM"), MetadataSource::Manual => write!(f, "Manual"), + MetadataSource::File => write!(f, "File"), + MetadataSource::Match => write!(f, "Match"), } } } diff --git a/mlm_db/src/lib.rs b/mlm_db/src/lib.rs index a25f9782..cb039033 100644 --- a/mlm_db/src/lib.rs +++ b/mlm_db/src/lib.rs @@ -16,6 +16,7 @@ mod v14; mod v15; mod v16; mod v17; +mod v18; use std::collections::HashMap; @@ -33,6 +34,13 @@ pub static MODELS: Lazy = Lazy::new(|| { let mut models = Models::new(); models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); + models.define::().unwrap(); models.define::().unwrap(); models.define::().unwrap(); @@ -130,27 +138,27 @@ pub static MODELS: Lazy = Lazy::new(|| { }); pub type Config = v01::Config; -pub type Torrent = v17::Torrent; -pub type TorrentKey = v17::TorrentKey; -pub type SelectedTorrent = v17::SelectedTorrent; -pub type SelectedTorrentKey = v17::SelectedTorrentKey; -pub type DuplicateTorrent = v17::DuplicateTorrent; -pub type ErroredTorrent = v17::ErroredTorrent; -pub type ErroredTorrentKey = v17::ErroredTorrentKey; +pub type Torrent = v18::Torrent; +pub type TorrentKey = v18::TorrentKey; +pub type SelectedTorrent = v18::SelectedTorrent; +pub type SelectedTorrentKey = v18::SelectedTorrentKey; +pub type DuplicateTorrent = v18::DuplicateTorrent; +pub type ErroredTorrent = v18::ErroredTorrent; +pub type ErroredTorrentKey = v18::ErroredTorrentKey; pub type ErroredTorrentId = v11::ErroredTorrentId; -pub type Event = v17::Event; -pub type EventKey = v17::EventKey; -pub type EventType = v17::EventType; +pub type Event = v18::Event; +pub type EventKey = v18::EventKey; +pub type EventType = v18::EventType; pub type List = v05::List; pub type ListKey = v05::ListKey; -pub type ListItem = v05::ListItem; -pub type ListItemKey = v05::ListItemKey; -pub type ListItemTorrent = v04::ListItemTorrent; -pub type TorrentMeta = v17::TorrentMeta; -pub type TorrentMetaDiff = v17::TorrentMetaDiff; -pub type TorrentMetaField = v17::TorrentMetaField; +pub type ListItem = v18::ListItem; +pub type ListItemKey = v18::ListItemKey; +pub type ListItemTorrent = v18::ListItemTorrent; +pub type TorrentMeta = v18::TorrentMeta; +pub type TorrentMetaDiff = v18::TorrentMetaDiff; +pub type TorrentMetaField = v18::TorrentMetaField; pub type VipStatus = v11::VipStatus; -pub type MetadataSource = v10::MetadataSource; +pub type MetadataSource = v18::MetadataSource; pub type OldDbMainCat = v01::MainCat; pub type MainCat = v12::MainCat; pub type Uuid = v03::Uuid; @@ -164,7 +172,7 @@ pub type Size = v03::Size; pub type TorrentCost = v04::TorrentCost; pub type TorrentStatus = v04::TorrentStatus; pub type LibraryMismatch = v08::LibraryMismatch; -pub type ClientStatus = v08::ClientStatus; +pub type ClientStatus = v18::ClientStatus; pub type AudiobookCategory = v06::AudiobookCategory; pub type EbookCategory = v06::EbookCategory; pub type MusicologyCategory = v16::MusicologyCategory; @@ -284,3 +292,11 @@ impl DatabaseExt for Database<'_> { self } } + +pub mod ids { + pub const ABS: &str = "abs"; + pub const ASIN: &str = "asin"; + pub const GOODREADS: &str = "goodreads"; + pub const ISBN: &str = "isbn"; + pub const MAM: &str = "mam"; +} diff --git a/mlm_db/src/v04.rs b/mlm_db/src/v04.rs index 75bf5550..6b0494ed 100644 --- a/mlm_db/src/v04.rs +++ b/mlm_db/src/v04.rs @@ -1,4 +1,4 @@ -use super::{v01, v03, v05, v06, v07}; +use super::{v01, v03, v05, v06, v07, v18}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; @@ -290,3 +290,13 @@ impl From for EventType { } } } + +impl From for ListItemTorrent { + fn from(t: v18::ListItemTorrent) -> Self { + Self { + mam_id: t.mam_id.unwrap(), + status: t.status, + at: t.at, + } + } +} diff --git a/mlm_db/src/v05.rs b/mlm_db/src/v05.rs index 18945675..01dd05db 100644 --- a/mlm_db/src/v05.rs +++ b/mlm_db/src/v05.rs @@ -1,4 +1,4 @@ -use super::{v01, v03, v04, v06}; +use super::{v01, v03, v04, v06, v18}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; @@ -139,3 +139,25 @@ impl From for Torrent { } } } + +impl From for ListItem { + fn from(t: v18::ListItem) -> Self { + Self { + guid: t.guid, + list_id: t.list_id, + title: t.title, + authors: t.authors, + series: t.series, + cover_url: t.cover_url, + book_url: t.book_url, + isbn: t.isbn, + prefer_format: t.prefer_format, + allow_audio: t.allow_audio, + audio_torrent: t.audio_torrent.map(Into::into), + allow_ebook: t.allow_ebook, + ebook_torrent: t.ebook_torrent.map(Into::into), + created_at: t.created_at, + marked_done_at: t.marked_done_at, + } + } +} diff --git a/mlm_db/src/v08.rs b/mlm_db/src/v08.rs index 9844d3e0..dec6c19d 100644 --- a/mlm_db/src/v08.rs +++ b/mlm_db/src/v08.rs @@ -1,4 +1,4 @@ -use super::{v01, v03, v04, v05, v06, v07, v09, v10, v11}; +use super::{v01, v03, v04, v05, v06, v07, v09, v10, v11, v18}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; @@ -456,3 +456,12 @@ impl From for TorrentMetaField { } } } + +impl From for ClientStatus { + fn from(value: v18::ClientStatus) -> Self { + match value { + v18::ClientStatus::NotInClient => Self::NotInClient, + v18::ClientStatus::RemovedFromTracker => Self::RemovedFromMam, + } + } +} diff --git a/mlm_db/src/v10.rs b/mlm_db/src/v10.rs index b1a0befa..50d70942 100644 --- a/mlm_db/src/v10.rs +++ b/mlm_db/src/v10.rs @@ -1,4 +1,4 @@ -use super::{v01, v03, v04, v06, v08, v09, v11}; +use super::{v01, v03, v04, v06, v08, v09, v11, v18}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; @@ -380,3 +380,13 @@ impl From for EventType { } } } + +impl From for MetadataSource { + fn from(t: v18::MetadataSource) -> Self { + match t { + v18::MetadataSource::Mam => Self::Mam, + v18::MetadataSource::Manual => Self::Manual, + _ => Self::Manual, + } + } +} diff --git a/mlm_db/src/v17.rs b/mlm_db/src/v17.rs index c85a62ce..3f49e350 100644 --- a/mlm_db/src/v17.rs +++ b/mlm_db/src/v17.rs @@ -1,9 +1,12 @@ -use super::{v03, v04, v08, v09, v10, v11, v12, v13, v15, v16}; +use crate::ids; + +use super::{v03, v04, v08, v09, v10, v11, v12, v13, v15, v16, v18}; use mlm_parse::{normalize_title, parse_edition}; use native_db::{ToKey, native_db}; use native_model::{Model, native_model}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use time::UtcDateTime; #[derive(Serialize, Deserialize, Debug, Clone)] #[native_model(id = 2, version = 17, from = v16::Torrent)] @@ -348,3 +351,200 @@ impl From for TorrentMetaField { } } } + +impl From for Torrent { + fn from(t: v18::Torrent) -> Self { + let abs_id = t.meta.ids.get(ids::ABS).map(|id| id.to_string()); + let goodreads_id = t + .meta + .ids + .get(ids::GOODREADS) + .and_then(|id| id.parse().ok()); + let meta: TorrentMeta = t.meta.into(); + + Self { + id: t.id, + id_is_hash: t.id_is_hash, + mam_id: t.mam_id.unwrap_or_default(), + abs_id, + goodreads_id, + library_path: t.library_path, + library_files: t.library_files, + linker: t.linker, + category: t.category, + selected_audio_format: t.selected_audio_format, + selected_ebook_format: t.selected_ebook_format, + title_search: normalize_title(&meta.title), + meta, + created_at: t.created_at, + replaced_with: t.replaced_with, + library_mismatch: t.library_mismatch, + client_status: t.client_status.map(Into::into), + request_matadata_update: false, + } + } +} + +impl From for SelectedTorrent { + fn from(t: v18::SelectedTorrent) -> Self { + let goodreads_id = t + .meta + .ids + .get(ids::GOODREADS) + .and_then(|id| id.parse().ok()); + let meta: TorrentMeta = t.meta.into(); + + Self { + mam_id: t.mam_id, + goodreads_id, + hash: t.hash, + dl_link: t.dl_link, + unsat_buffer: t.unsat_buffer, + wedge_buffer: None, + cost: t.cost, + category: t.category, + tags: t.tags, + title_search: normalize_title(&meta.title), + meta, + grabber: t.grabber, + created_at: t.created_at, + started_at: t.started_at, + removed_at: t.removed_at, + } + } +} + +impl From for DuplicateTorrent { + fn from(t: v18::DuplicateTorrent) -> Self { + let meta: TorrentMeta = t.meta.into(); + Self { + mam_id: t.mam_id, + dl_link: t.dl_link, + title_search: normalize_title(&meta.title), + meta, + created_at: t.created_at, + duplicate_of: t.duplicate_of, + } + } +} + +impl From for ErroredTorrent { + fn from(t: v18::ErroredTorrent) -> Self { + Self { + id: t.id, + title: t.title, + error: t.error, + meta: t.meta.map(|t| t.into()), + created_at: t.created_at, + } + } +} + +impl From for TorrentMeta { + fn from(t: v18::TorrentMeta) -> Self { + let mam_id = t.ids.get(ids::MAM).and_then(|id| id.parse().ok()).unwrap(); + + Self { + mam_id, + vip_status: t.vip_status, + cat: t.cat, + media_type: t.media_type, + main_cat: t.main_cat, + categories: vec![], + language: t.language, + flags: t.flags, + filetypes: t.filetypes, + num_files: 0, + size: t.size, + title: t.title, + edition: t.edition, + authors: t.authors, + narrators: t.narrators, + series: t.series, + source: t.source.into(), + uploaded_at: t + .uploaded_at + .unwrap_or(v03::Timestamp::from(UtcDateTime::UNIX_EPOCH)), + } + } +} + +impl From for Event { + fn from(t: v18::Event) -> Self { + Self { + id: t.id, + torrent_id: t.torrent_id, + mam_id: t.mam_id, + created_at: t.created_at, + event: t.event.into(), + } + } +} + +impl From for EventType { + fn from(t: v18::EventType) -> Self { + match t { + v18::EventType::Grabbed { + grabber, + cost, + wedged, + } => Self::Grabbed { + grabber, + cost, + wedged, + }, + v18::EventType::Linked { + linker, + library_path, + } => Self::Linked { + linker, + library_path, + }, + v18::EventType::Cleaned { + library_path, + files, + } => Self::Cleaned { + library_path, + files, + }, + v18::EventType::Updated { fields, .. } => Self::Updated { + fields: fields.into_iter().map(Into::into).collect(), + }, + v18::EventType::RemovedFromTracker => Self::RemovedFromMam, + } + } +} + +impl From for TorrentMetaDiff { + fn from(value: v18::TorrentMetaDiff) -> Self { + Self { + field: value.field.into(), + from: value.from, + to: value.to, + } + } +} + +impl From for TorrentMetaField { + fn from(value: v18::TorrentMetaField) -> Self { + match value { + v18::TorrentMetaField::Ids => TorrentMetaField::MamId, + v18::TorrentMetaField::Vip => TorrentMetaField::Vip, + v18::TorrentMetaField::MediaType => TorrentMetaField::MediaType, + v18::TorrentMetaField::MainCat => TorrentMetaField::MainCat, + v18::TorrentMetaField::Categories => TorrentMetaField::Categories, + v18::TorrentMetaField::Cat => TorrentMetaField::Cat, + v18::TorrentMetaField::Language => TorrentMetaField::Language, + v18::TorrentMetaField::Flags => TorrentMetaField::Flags, + v18::TorrentMetaField::Filetypes => TorrentMetaField::Filetypes, + v18::TorrentMetaField::Size => TorrentMetaField::Size, + v18::TorrentMetaField::Title => TorrentMetaField::Title, + v18::TorrentMetaField::Authors => TorrentMetaField::Authors, + v18::TorrentMetaField::Narrators => TorrentMetaField::Narrators, + v18::TorrentMetaField::Series => TorrentMetaField::Series, + v18::TorrentMetaField::Source => TorrentMetaField::Source, + v18::TorrentMetaField::Edition => TorrentMetaField::Edition, + v18::TorrentMetaField::Tags => unimplemented!(), + } + } +} diff --git a/mlm_db/src/v18.rs b/mlm_db/src/v18.rs new file mode 100644 index 00000000..dc2a1955 --- /dev/null +++ b/mlm_db/src/v18.rs @@ -0,0 +1,463 @@ +use crate::ids; + +use super::{v01, v03, v04, v05, v08, v09, v10, v11, v12, v13, v16, v17}; +use mlm_parse::{normalize_title, parse_edition}; +use native_db::{ToKey, native_db}; +use native_model::{Model, native_model}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, path::PathBuf}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 2, version = 18, from = v17::Torrent)] +#[native_db(export_keys = true)] +pub struct Torrent { + #[primary_key] + pub id: String, + pub id_is_hash: bool, + #[secondary_key(unique, optional)] + pub mam_id: Option, + pub library_path: Option, + pub library_files: Vec, + pub linker: Option, + pub category: Option, + pub selected_audio_format: Option, + pub selected_ebook_format: Option, + #[secondary_key] + pub title_search: String, + pub meta: TorrentMeta, + #[secondary_key] + pub created_at: v03::Timestamp, + pub replaced_with: Option<(String, v03::Timestamp)>, + pub library_mismatch: Option, + pub client_status: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub enum ClientStatus { + NotInClient, + RemovedFromTracker, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 3, version = 18, from = v17::SelectedTorrent)] +#[native_db(export_keys = true)] +pub struct SelectedTorrent { + #[primary_key] + pub mam_id: u64, + #[secondary_key(unique, optional)] + pub hash: Option, + pub dl_link: String, + pub unsat_buffer: Option, + pub wedge_buffer: Option, + pub cost: v04::TorrentCost, + pub category: Option, + pub tags: Vec, + #[secondary_key] + pub title_search: String, + pub meta: TorrentMeta, + pub grabber: Option, + pub created_at: v03::Timestamp, + pub started_at: Option, + pub removed_at: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 4, version = 18, from = v17::DuplicateTorrent)] +#[native_db] +pub struct DuplicateTorrent { + #[primary_key] + pub mam_id: u64, + pub dl_link: Option, + #[secondary_key] + pub title_search: String, + pub meta: TorrentMeta, + pub created_at: v03::Timestamp, + pub duplicate_of: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 5, version = 18, from = v17::ErroredTorrent)] +#[native_db(export_keys = true)] +pub struct ErroredTorrent { + #[primary_key] + pub id: v11::ErroredTorrentId, + pub title: String, + pub error: String, + pub meta: Option, + #[secondary_key] + pub created_at: v03::Timestamp, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct TorrentMeta { + pub ids: BTreeMap, + pub vip_status: Option, + pub cat: Option, + pub media_type: v13::MediaType, + pub main_cat: Option, + pub categories: Vec, + pub tags: Vec, + pub language: Option, + pub flags: Option, + pub filetypes: Vec, + pub num_files: u64, + pub size: v03::Size, + pub title: String, + pub edition: Option<(String, u64)>, + pub description: String, + pub authors: Vec, + pub narrators: Vec, + pub series: Vec, + pub source: MetadataSource, + pub uploaded_at: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum MetadataSource { + Mam, + Manual, + File, + Match, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 6, version = 18, from = v17::Event)] +#[native_db(export_keys = true)] +pub struct Event { + #[primary_key] + pub id: v03::Uuid, + #[secondary_key] + pub torrent_id: Option, + #[secondary_key] + pub mam_id: Option, + #[secondary_key] + pub created_at: v03::Timestamp, + pub event: EventType, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum EventType { + Grabbed { + grabber: Option, + cost: Option, + wedged: bool, + }, + Linked { + linker: Option, + library_path: PathBuf, + }, + Cleaned { + library_path: PathBuf, + files: Vec, + }, + Updated { + fields: Vec, + source: (MetadataSource, String), + }, + RemovedFromTracker, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TorrentMetaDiff { + pub field: TorrentMetaField, + pub from: String, + pub to: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum TorrentMetaField { + Ids, + Vip, + Cat, + MediaType, + MainCat, + Categories, + Tags, + Language, + Flags, + Filetypes, + Size, + Title, + Edition, + Authors, + Narrators, + Series, + Source, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[native_model(id = 8, version = 18, from = v05::ListItem)] +#[native_db(export_keys = true)] +pub struct ListItem { + #[primary_key] + pub guid: (String, String), + #[secondary_key] + pub list_id: String, + pub title: String, + pub authors: Vec, + pub series: Vec<(String, f64)>, + pub cover_url: String, + pub book_url: Option, + pub isbn: Option, + pub prefer_format: Option, + pub allow_audio: bool, + pub audio_torrent: Option, + pub allow_ebook: bool, + pub ebook_torrent: Option, + #[secondary_key] + pub created_at: v03::Timestamp, + pub marked_done_at: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ListItemTorrent { + pub torrent_id: Option, + pub mam_id: Option, + pub status: v04::TorrentStatus, + pub at: v03::Timestamp, +} + +impl From for Torrent { + fn from(t: v17::Torrent) -> Self { + let mut meta: TorrentMeta = t.meta.into(); + if let Some(abs_id) = t.abs_id { + meta.ids.insert(ids::ABS.to_string(), abs_id.to_string()); + } + if let Some(goodreads_id) = t.goodreads_id { + meta.ids + .insert(ids::GOODREADS.to_string(), goodreads_id.to_string()); + } + + Self { + id: t.id, + id_is_hash: t.id_is_hash, + mam_id: Some(t.mam_id), + library_path: t.library_path, + library_files: t.library_files, + linker: t.linker, + category: t.category, + selected_audio_format: t.selected_audio_format, + selected_ebook_format: t.selected_ebook_format, + title_search: normalize_title(&meta.title), + meta, + created_at: t.created_at, + replaced_with: t.replaced_with, + library_mismatch: t.library_mismatch, + client_status: t.client_status.map(Into::into), + } + } +} + +impl From for ClientStatus { + fn from(value: v08::ClientStatus) -> Self { + match value { + v08::ClientStatus::NotInClient => Self::NotInClient, + v08::ClientStatus::RemovedFromMam => Self::RemovedFromTracker, + } + } +} + +impl From for SelectedTorrent { + fn from(t: v17::SelectedTorrent) -> Self { + let mut meta: TorrentMeta = t.meta.into(); + if let Some(goodreads_id) = t.goodreads_id { + meta.ids + .insert(ids::GOODREADS.to_string(), goodreads_id.to_string()); + } + + Self { + mam_id: t.mam_id, + hash: t.hash, + dl_link: t.dl_link, + unsat_buffer: t.unsat_buffer, + wedge_buffer: None, + cost: t.cost, + category: t.category, + tags: t.tags, + title_search: normalize_title(&meta.title), + meta, + grabber: t.grabber, + created_at: t.created_at, + started_at: t.started_at, + removed_at: t.removed_at, + } + } +} + +impl From for DuplicateTorrent { + fn from(t: v17::DuplicateTorrent) -> Self { + let meta: TorrentMeta = t.meta.into(); + Self { + mam_id: t.mam_id, + dl_link: t.dl_link, + title_search: normalize_title(&meta.title), + meta, + created_at: t.created_at, + duplicate_of: t.duplicate_of, + } + } +} + +impl From for ErroredTorrent { + fn from(t: v17::ErroredTorrent) -> Self { + Self { + id: t.id, + title: t.title, + error: t.error, + meta: t.meta.map(|t| t.into()), + created_at: t.created_at, + } + } +} + +impl From for TorrentMeta { + fn from(t: v17::TorrentMeta) -> Self { + let mut ids = BTreeMap::default(); + ids.insert(ids::MAM.to_string(), t.mam_id.to_string()); + + Self { + ids, + vip_status: t.vip_status, + cat: t.cat, + media_type: t.media_type, + main_cat: t.main_cat, + categories: t.categories.iter().map(ToString::to_string).collect(), + tags: vec![], + language: t.language, + flags: t.flags, + filetypes: t.filetypes, + num_files: t.num_files, + size: t.size, + title: t.title, + edition: t.edition, + description: String::new(), + authors: t.authors, + narrators: t.narrators, + series: t.series, + source: t.source.into(), + uploaded_at: Some(t.uploaded_at), + } + } +} + +impl From for MetadataSource { + fn from(t: v10::MetadataSource) -> Self { + match t { + v10::MetadataSource::Mam => Self::Mam, + v10::MetadataSource::Manual => Self::Manual, + } + } +} + +impl From for Event { + fn from(t: v17::Event) -> Self { + Self { + id: t.id, + torrent_id: t.torrent_id, + mam_id: t.mam_id, + created_at: t.created_at, + event: t.event.into(), + } + } +} + +impl From for EventType { + fn from(t: v17::EventType) -> Self { + match t { + v17::EventType::Grabbed { + grabber, + cost, + wedged, + } => Self::Grabbed { + grabber, + cost, + wedged, + }, + v17::EventType::Linked { + linker, + library_path, + } => Self::Linked { + linker, + library_path, + }, + v17::EventType::Cleaned { + library_path, + files, + } => Self::Cleaned { + library_path, + files, + }, + v17::EventType::Updated { fields } => Self::Updated { + fields: fields.into_iter().map(Into::into).collect(), + source: (MetadataSource::Mam, String::new()), + }, + v17::EventType::RemovedFromMam => Self::RemovedFromTracker, + } + } +} + +impl From for TorrentMetaDiff { + fn from(value: v17::TorrentMetaDiff) -> Self { + Self { + field: value.field.into(), + from: value.from, + to: value.to, + } + } +} + +impl From for TorrentMetaField { + fn from(value: v17::TorrentMetaField) -> Self { + match value { + v17::TorrentMetaField::MamId => TorrentMetaField::Ids, + v17::TorrentMetaField::Vip => TorrentMetaField::Vip, + v17::TorrentMetaField::MediaType => TorrentMetaField::MediaType, + v17::TorrentMetaField::MainCat => TorrentMetaField::MainCat, + v17::TorrentMetaField::Categories => TorrentMetaField::Categories, + v17::TorrentMetaField::Cat => TorrentMetaField::Cat, + v17::TorrentMetaField::Language => TorrentMetaField::Language, + v17::TorrentMetaField::Flags => TorrentMetaField::Flags, + v17::TorrentMetaField::Filetypes => TorrentMetaField::Filetypes, + v17::TorrentMetaField::Size => TorrentMetaField::Size, + v17::TorrentMetaField::Title => TorrentMetaField::Title, + v17::TorrentMetaField::Authors => TorrentMetaField::Authors, + v17::TorrentMetaField::Narrators => TorrentMetaField::Narrators, + v17::TorrentMetaField::Series => TorrentMetaField::Series, + v17::TorrentMetaField::Source => TorrentMetaField::Source, + v17::TorrentMetaField::Edition => TorrentMetaField::Edition, + } + } +} + +impl From for ListItem { + fn from(t: v05::ListItem) -> Self { + Self { + guid: t.guid, + list_id: t.list_id, + title: t.title, + authors: t.authors, + series: t.series, + cover_url: t.cover_url, + book_url: t.book_url, + isbn: t.isbn, + prefer_format: t.prefer_format, + allow_audio: t.allow_audio, + audio_torrent: t.audio_torrent.map(Into::into), + allow_ebook: t.allow_ebook, + ebook_torrent: t.ebook_torrent.map(Into::into), + created_at: t.created_at, + marked_done_at: t.marked_done_at, + } + } +} + +impl From for ListItemTorrent { + fn from(t: v04::ListItemTorrent) -> Self { + Self { + torrent_id: None, + mam_id: Some(t.mam_id), + status: t.status, + at: t.at, + } + } +} diff --git a/mlm_mam/src/enums.rs b/mlm_mam/src/enums.rs index a84812c5..c656b358 100644 --- a/mlm_mam/src/enums.rs +++ b/mlm_mam/src/enums.rs @@ -1,4 +1,4 @@ -use std::{fmt, marker::PhantomData}; +use std::{fmt, marker::PhantomData, str::FromStr}; use mlm_db::{AudiobookCategory, EbookCategory, MusicologyCategory, RadioCategory}; use serde::{ @@ -310,30 +310,32 @@ pub enum UserClass { Mouse, } -impl UserClass { - pub fn from_str(class: &str) -> Option { +impl FromStr for UserClass { + type Err = String; + + fn from_str(class: &str) -> Result { match class { - "Dev" => Some(UserClass::Dev), - "SysOp" => Some(UserClass::SysOp), - "SR Administrator" => Some(UserClass::SrAdministrator), - "Administrator" => Some(UserClass::Administrator), - "Uploader Coordinator" => Some(UserClass::UploaderCoordinator), - "SR Moderator" => Some(UserClass::SrModerator), - "Moderator" => Some(UserClass::Moderator), - "Torrent Mod" => Some(UserClass::TorrentMod), - "Forum Mod" => Some(UserClass::ForumMod), - "Support Staff" => Some(UserClass::SupportStaff), - "Entry Level Staff" => Some(UserClass::EntryLevelStaff), - "Uploader" => Some(UserClass::Uploader), - "Mouseketeer" => Some(UserClass::Mouseketeer), - "Supporter" => Some(UserClass::Supporter), - "Elite" => Some(UserClass::Elite), - "Elite VIP" => Some(UserClass::EliteVip), - "VIP" => Some(UserClass::Vip), - "Power User" => Some(UserClass::PowerUser), - "User" => Some(UserClass::User), - "Mous" => Some(UserClass::Mouse), - _ => None, + "Dev" => Ok(UserClass::Dev), + "SysOp" => Ok(UserClass::SysOp), + "SR Administrator" => Ok(UserClass::SrAdministrator), + "Administrator" => Ok(UserClass::Administrator), + "Uploader Coordinator" => Ok(UserClass::UploaderCoordinator), + "SR Moderator" => Ok(UserClass::SrModerator), + "Moderator" => Ok(UserClass::Moderator), + "Torrent Mod" => Ok(UserClass::TorrentMod), + "Forum Mod" => Ok(UserClass::ForumMod), + "Support Staff" => Ok(UserClass::SupportStaff), + "Entry Level Staff" => Ok(UserClass::EntryLevelStaff), + "Uploader" => Ok(UserClass::Uploader), + "Mouseketeer" => Ok(UserClass::Mouseketeer), + "Supporter" => Ok(UserClass::Supporter), + "Elite" => Ok(UserClass::Elite), + "Elite VIP" => Ok(UserClass::EliteVip), + "VIP" => Ok(UserClass::Vip), + "Power User" => Ok(UserClass::PowerUser), + "User" => Ok(UserClass::User), + "Mouse" => Ok(UserClass::Mouse), + value => Err(format!("unknown user class {value}")), } } } @@ -342,11 +344,7 @@ impl TryFrom for UserClass { type Error = String; fn try_from(value: String) -> Result { - let v = Self::from_str(&value); - match v { - Some(v) => Ok(v), - None => Err(format!("invalid category {value}")), - } + Self::from_str(&value) } } diff --git a/mlm_mam/src/meta.rs b/mlm_mam/src/meta.rs index dda47614..bd90c752 100644 --- a/mlm_mam/src/meta.rs +++ b/mlm_mam/src/meta.rs @@ -74,3 +74,70 @@ pub fn clean_meta(mut meta: TorrentMeta, tags: &str) -> Result { Ok(meta) } + +// #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +// pub enum Category { +// Action, +// Art, +// Biographical, +// Business, +// Comedy, +// CompleteEditionsMusic, +// Computer, +// Crafts, +// Crime, +// Dramatization, +// Education, +// FactualNews, +// Fantasy, +// Food, +// Guitar, +// Health, +// Historical, +// Home, +// Horror, +// Humor, +// IndividualSheet, +// Instructional, +// Juvenile, +// Language, +// Lgbt, +// LickLibraryLTP, +// LickLibraryTechniques, +// LiteraryClassics, +// LitRPG, +// Math, +// Medicine, +// Music, +// MusicBook, +// Mystery, +// Nature, +// Paranormal, +// Philosophy, +// Poetry, +// Politics, +// Reference, +// Religion, +// Romance, +// Rpg, +// Science, +// ScienceFiction, +// SelfHelp, +// SheetCollection, +// SheetCollectionMP3, +// Sports, +// Technology, +// Thriller, +// Travel, +// UrbanFantasy, +// Western, +// YoungAdult, +// Superheroes, +// LiteraryFiction, +// ProgressionFantasy, +// ContemporaryFiction, +// DramaPlays, +// OccultMetaphysicalPractices, +// SliceOfLife, +// Unknown(u8), +// } diff --git a/mlm_mam/src/search.rs b/mlm_mam/src/search.rs index 269ce10e..869efd0c 100644 --- a/mlm_mam/src/search.rs +++ b/mlm_mam/src/search.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use anyhow::Result; use mlm_db::{ Category, FlagBits, Language, MainCat, MediaType, MetadataSource, OldCategory, Series, - SeriesEntries, Timestamp, TorrentMeta, VipStatus, + SeriesEntries, Timestamp, TorrentMeta, VipStatus, ids, }; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -325,7 +325,11 @@ impl MaMTorrent { let categories = self .categories .iter() - .map(|id| Category::from_id(*id).ok_or_else(|| MetaError::UnknownCat(*id))) + .map(|id| { + Category::from_id(*id) + .ok_or_else(|| MetaError::UnknownCat(*id)) + .map(|cat| cat.to_string()) + }) .collect::, _>>()?; let cat = OldCategory::from_one_id(self.category) .ok_or_else(|| MetaError::UnknownOldCat(self.catname.clone(), self.category))?; @@ -355,10 +359,19 @@ impl MaMTorrent { return Err(MetaError::InvalidAdded(self.added.clone())); } }; + let (isbn, asin) = parse_isbn(self); + let mut ids = BTreeMap::new(); + ids.insert(ids::MAM.to_string(), self.id.to_string()); + if let Some(isbn) = isbn { + ids.insert(ids::ISBN.to_string(), isbn.to_string()); + } + if let Some(asin) = asin { + ids.insert(ids::ASIN.to_string(), asin.to_string()); + } Ok(clean_meta( TorrentMeta { - mam_id: self.id, + ids, vip_status: Some(vip_status), media_type, main_cat, @@ -371,11 +384,12 @@ impl MaMTorrent { size, title: self.title.clone(), edition: None, + description: self.description.clone().unwrap_or_default(), authors, narrators, series, source: MetadataSource::Mam, - uploaded_at, + uploaded_at: Some(uploaded_at), }, &self.tags, )?) @@ -398,3 +412,15 @@ impl MaMTorrent { } } } + +fn parse_isbn(mam_torrent: &MaMTorrent) -> (Option<&str>, Option<&str>) { + let isbn_raw: &str = mam_torrent.isbn.as_deref().unwrap_or(""); + let isbn = if isbn_raw.is_empty() || isbn_raw.starts_with("ASIN:") { + None + } else { + Some(isbn_raw.trim()) + }; + let asin = isbn_raw.strip_prefix("ASIN:").map(|s| s.trim()); + + (isbn, asin) +} diff --git a/mlm_mam/src/user_torrent.rs b/mlm_mam/src/user_torrent.rs index f8f19f08..772588b3 100644 --- a/mlm_mam/src/user_torrent.rs +++ b/mlm_mam/src/user_torrent.rs @@ -1,8 +1,10 @@ +use std::collections::BTreeMap; + use anyhow::Result; use itertools::Itertools as _; use mlm_db::{ - Category, FlagBits, MediaType, MetadataSource, OldCategory, Series, SeriesEntries, Timestamp, - TorrentMeta, VipStatus, + Category, FlagBits, MediaType, MetadataSource, OldCategory, Series, SeriesEntries, TorrentMeta, + VipStatus, ids, }; use mlm_parse::clean_value; use serde::{Deserialize, Serialize}; @@ -163,7 +165,11 @@ impl UserDetailsTorrent { let mut categories = self .categories .iter() - .map(|c| Category::from_id(c.id as u8).ok_or_else(|| MetaError::UnknownCat(c.id as u8))) + .map(|c| { + Category::from_id(c.id as u8) + .ok_or_else(|| MetaError::UnknownCat(c.id as u8)) + .map(|cat| cat.to_string()) + }) .collect::, _>>()?; categories.sort(); @@ -186,14 +192,18 @@ impl UserDetailsTorrent { VipStatus::Permanent }; + let mut ids = BTreeMap::new(); + ids.insert(ids::MAM.to_string(), self.id.to_string()); + Ok(clean_meta( TorrentMeta { - mam_id: self.id, + ids, vip_status: Some(vip_status), media_type, // TODO: Currently main_cat isn't returned main_cat: None, categories, + tags: vec![], cat: Some(cat), // TODO: Currently language isn't returned language: None, @@ -204,12 +214,14 @@ impl UserDetailsTorrent { size, title: clean_value(&self.title)?, edition: None, + // TODO: Currently num_files isn't returned + description: String::new(), authors, narrators, series, source: MetadataSource::Mam, // TODO: Currently added isn't returned - uploaded_at: Timestamp::from(UtcDateTime::UNIX_EPOCH), + uploaded_at: None, }, &clean_value(&self.tags)?, )?) diff --git a/server/src/audiobookshelf.rs b/server/src/audiobookshelf.rs index b80fa949..55fea902 100644 --- a/server/src/audiobookshelf.rs +++ b/server/src/audiobookshelf.rs @@ -2,8 +2,7 @@ use std::{collections::BTreeSet, path::PathBuf, sync::Arc}; use anyhow::Result; use axum::http::HeaderMap; -use mlm_db::{DatabaseExt as _, Flags, Torrent, TorrentMeta, impls::format_serie}; -use mlm_mam::search::MaMTorrent; +use mlm_db::{DatabaseExt as _, Flags, Torrent, TorrentMeta, ids, impls::format_serie}; use native_db::Database; use reqwest::{Url, header::AUTHORIZATION}; use serde::{Deserialize, Serialize}; @@ -486,23 +485,20 @@ pub async fn match_torrents_to_abs( let torrents = db.r_transaction()?.scan().primary::()?; let torrents = torrents.all()?.filter(|t| { t.as_ref() - .is_ok_and(|t| t.abs_id.is_none() && t.library_path.is_some()) + .is_ok_and(|t| !t.meta.ids.contains_key(ids::ABS) && t.library_path.is_some()) }); for torrent in torrents { let mut torrent = torrent?; let Some(book) = abs.get_book(&torrent).await? else { trace!( - "Could not find ABS entry for torrent {} {}", - torrent.meta.mam_id, torrent.meta.title + "Could not find ABS entry for torrent {}", + torrent.meta.title ); continue; }; - debug!( - "Matched ABS entry with torrent {} {}", - torrent.meta.mam_id, torrent.meta.title - ); - torrent.abs_id = Some(book.id); + debug!("Matched ABS entry with torrent {}", torrent.meta.title); + torrent.meta.ids.insert(ids::ABS.to_string(), book.id); let (_guard, rw) = db.rw_async().await?; rw.upsert(torrent)?; rw.commit()?; @@ -605,15 +601,8 @@ impl Abs { Ok(None) } - pub async fn update_book( - &self, - id: &str, - mam_torrent: &MaMTorrent, - meta: &TorrentMeta, - ) -> Result<()> { + pub async fn update_book(&self, id: &str, meta: &TorrentMeta) -> Result<()> { let (title, subtitle) = parse_titles(meta); - let (isbn, asin) = parse_isbn(mam_torrent); - self.client .patch(format!("{}/api/items/{id}/media", self.base_url)) .header("Content-Type", "application/json") @@ -639,9 +628,9 @@ impl Abs { }) .collect(), narrators: meta.narrators.iter().map(|name| name.as_str()).collect(), - description: mam_torrent.description.as_deref(), - isbn, - asin, + description: Some(&meta.description), + isbn: meta.ids.get(ids::ISBN).map(|s| s.as_str()), + asin: meta.ids.get(ids::ASIN).map(|s| s.as_str()), genres: meta .cat .as_ref() @@ -676,9 +665,8 @@ impl Abs { } } -pub fn create_metadata(mam_torrent: &MaMTorrent, meta: &TorrentMeta) -> serde_json::Value { +pub fn create_metadata(meta: &TorrentMeta) -> serde_json::Value { let (title, subtitle) = parse_titles(meta); - let (isbn, asin) = parse_isbn(mam_torrent); let flags = Flags::from_bitfield(meta.flags.map_or(0, |f| f.0)); let metadata = json!({ @@ -687,9 +675,9 @@ pub fn create_metadata(mam_torrent: &MaMTorrent, meta: &TorrentMeta) -> serde_js "series": &meta.series.iter().map(format_serie).collect::>(), "title": title, "subtitle": subtitle, - "description": mam_torrent.description, - "isbn": isbn, - "asin": asin, + "description": meta.description, + "isbn": meta.ids.get(ids::ISBN), + "asin": meta.ids.get(ids::ASIN), "tags": if flags.lgbt == Some(true) { Some(vec!["LGBT"]) } else { None }, "genres": meta .cat @@ -717,15 +705,3 @@ fn parse_titles(meta: &TorrentMeta) -> (&str, Option<&str>) { (title, subtitle) } - -fn parse_isbn(mam_torrent: &MaMTorrent) -> (Option<&str>, Option<&str>) { - let isbn_raw: &str = mam_torrent.isbn.as_deref().unwrap_or(""); - let isbn = if isbn_raw.is_empty() || isbn_raw.starts_with("ASIN:") { - None - } else { - Some(isbn_raw.trim()) - }; - let asin = isbn_raw.strip_prefix("ASIN:").map(|s| s.trim()); - - (isbn, asin) -} diff --git a/server/src/autograbber.rs b/server/src/autograbber.rs index 66b520aa..8b94d0cd 100644 --- a/server/src/autograbber.rs +++ b/server/src/autograbber.rs @@ -11,7 +11,7 @@ use itertools::Itertools as _; use lava_torrent::torrent::v1::Torrent; use mlm_db::{ ClientStatus, DatabaseExt as _, DuplicateTorrent, Event, EventType, MetadataSource, - SelectedTorrent, Timestamp, TorrentCost, TorrentKey, TorrentMeta, VipStatus, + SelectedTorrent, Timestamp, TorrentCost, TorrentKey, TorrentMeta, VipStatus, ids, }; use mlm_mam::{ api::MaM, @@ -250,8 +250,6 @@ pub async fn search_torrents( sort_type: sort_type.to_string(), ..Default::default() }, - - ..Default::default() }) .await .context("search")?; @@ -316,15 +314,16 @@ pub async fn mark_removed_torrents( .get() .secondary::(TorrentKey::mam_id, id)?; if let Some(mut torrent) = torrent - && torrent.client_status != Some(ClientStatus::RemovedFromMam) + && torrent.client_status != Some(ClientStatus::RemovedFromTracker) { if mam.get_torrent_info_by_id(id).await?.is_none() { - torrent.client_status = Some(ClientStatus::RemovedFromMam); + torrent.client_status = Some(ClientStatus::RemovedFromTracker); let tid = Some(torrent.id.clone()); rw.upsert(torrent)?; rw.commit()?; drop(guard); - write_event(db, Event::new(tid, Some(id), EventType::RemovedFromMam)).await; + write_event(db, Event::new(tid, Some(id), EventType::RemovedFromTracker)) + .await; } sleep(Duration::from_millis(400)).await; } @@ -357,7 +356,7 @@ pub async fn select_torrents>( continue; } - let meta = match torrent.as_meta() { + let mut meta = match torrent.as_meta() { Ok(it) => it, Err(err) => match err { MetaError::UnknownMediaType(_) => { @@ -367,6 +366,10 @@ pub async fn select_torrents>( _ => return Err(err.into()), }, }; + if let Some(goodreads_id) = goodreads_id { + meta.ids + .insert(ids::GOODREADS.to_string(), goodreads_id.to_string()); + } let rw_opt = if dry_run { None } else { @@ -399,7 +402,7 @@ pub async fn select_torrents>( if let Some((_, rw)) = &rw_opt { let old_library = rw .get() - .secondary::(TorrentKey::mam_id, meta.mam_id)?; + .secondary::(TorrentKey::mam_id, meta.mam_id())?; if let Some(old) = old_library { if old.meta != meta || (cost == Cost::MetadataOnlyAdd @@ -423,7 +426,7 @@ pub async fn select_torrents>( } } if cost == Cost::MetadataOnlyAdd { - let mam_id = meta.mam_id; + let mam_id = torrent.id; add_metadata_only_torrent(rw_opt.unwrap(), torrent, meta) .await .or_else(|err| { @@ -448,7 +451,7 @@ pub async fn select_torrents>( if preference.is_none() { debug!( "Could not find any wanted formats in torrent {}, formats: {:?}, wanted: {:?}", - meta.mam_id, meta.filetypes, preferred_types + torrent.id, meta.filetypes, preferred_types ); continue 'torrent; } @@ -460,7 +463,7 @@ pub async fn select_torrents>( .collect::, native_db::db_type::Error>>() }?; for old in old_selected { - if old.mam_id == meta.mam_id { + if old.mam_id == torrent.id { if old.meta != meta { update_selected_torrent_meta(db, rw_opt.unwrap(), mam, old, meta).await?; } @@ -476,15 +479,20 @@ pub async fn select_torrents>( .iter() .position(|t| old.meta.filetypes.contains(t)); if old_preference <= preference { - if let Err(err) = - add_duplicate_torrent(rw, None, torrent.dl.clone(), title_search, meta) - { + if let Err(err) = add_duplicate_torrent( + rw, + None, + torrent.dl.clone(), + title_search, + torrent.id, + meta, + ) { error!("Error writing duplicate torrent: {err}"); } rw_opt.unwrap().1.commit()?; trace!( "Skipping torrent {} as we have {} selected", - torrent.id, old.meta.mam_id + torrent.id, old.mam_id ); continue 'torrent; } else { @@ -493,6 +501,7 @@ pub async fn select_torrents>( None, torrent.dl.clone(), title_search.clone(), + torrent.id, old.meta.clone(), ) { error!("Error writing duplicate torrent: {err}"); @@ -514,7 +523,7 @@ pub async fn select_torrents>( .collect::, native_db::db_type::Error>>() }?; for old in old_library { - if old.meta.mam_id == meta.mam_id { + if old.mam_id == Some(torrent.id) { if old.meta != meta { update_torrent_meta( config, @@ -540,20 +549,21 @@ pub async fn select_torrents>( .iter() .position(|t| old.meta.filetypes.contains(t)); if old_preference <= preference { + trace!( + "Skipping torrent {} as we have {} in libary", + torrent.id, &old.id + ); if let Err(err) = add_duplicate_torrent( rw, Some(old.id), torrent.dl.clone(), title_search, + torrent.id, meta, ) { error!("Error writing duplicate torrent: {err}"); } rw_opt.unwrap().1.commit()?; - trace!( - "Skipping torrent {} as we have {} in libary", - torrent.id, old.meta.mam_id - ); continue 'torrent; } else { info!( @@ -594,7 +604,6 @@ pub async fn select_torrents>( selected_torrents += 1; rw.insert(mlm_db::SelectedTorrent { mam_id: torrent.id, - goodreads_id, hash: None, dl_link: torrent .dl @@ -636,9 +645,7 @@ pub async fn add_metadata_only_torrent( rw.insert(mlm_db::Torrent { id, id_is_hash: false, - mam_id, - abs_id: None, - goodreads_id: None, + mam_id: Some(mam_id), library_path: None, library_files: Default::default(), linker: if torrent.owner_name.is_empty() { @@ -653,7 +660,6 @@ pub async fn add_metadata_only_torrent( meta, created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, })?; @@ -670,10 +676,16 @@ pub async fn update_torrent_meta( (guard, rw): (MutexGuard<'_, ()>, RwTransaction<'_>), mam_torrent: &MaMTorrent, mut torrent: mlm_db::Torrent, - meta: TorrentMeta, + mut meta: TorrentMeta, allow_non_mam: bool, linker_is_owner: bool, ) -> Result<()> { + meta.ids.extend(torrent.meta.ids.clone()); + meta.tags = torrent.meta.tags.clone(); + if meta.description.is_empty() { + meta.description = torrent.meta.description.clone(); + } + if !allow_non_mam && torrent.meta.source != MetadataSource::Mam { // Update VIP status and uploaded_at still if torrent.meta.vip_status != meta.vip_status @@ -722,7 +734,7 @@ pub async fn update_torrent_meta( } let id = torrent.id.clone(); - let mam_id = meta.mam_id; + let mam_id = mam_torrent.id; let diff = torrent.meta.diff(&meta); debug!( "Updating meta for torrent {}, diff:\n{}", @@ -738,7 +750,7 @@ pub async fn update_torrent_meta( drop(guard); if let Some(library_path) = &torrent.library_path - && let serde_json::Value::Object(new) = abs::create_metadata(mam_torrent, &meta) + && let serde_json::Value::Object(new) = abs::create_metadata(&meta) { let metadata_path = library_path.join("metadata.json"); if metadata_path.exists() { @@ -752,13 +764,15 @@ pub async fn update_torrent_meta( let mut writer = BufWriter::new(file); serde_json::to_writer(&mut writer, &serde_json::Value::Object(existing))?; writer.flush()?; - debug!("updated ABS metadata file {}", torrent.meta.mam_id); + debug!("updated ABS metadata file {}", mam_torrent.id); } - if let (Some(abs_id), Some(abs_config)) = (&torrent.abs_id, &config.audiobookshelf) { + if let (Some(abs_id), Some(abs_config)) = + (&torrent.meta.ids.get(ids::ABS), &config.audiobookshelf) + { let abs = Abs::new(abs_config)?; - match abs.update_book(abs_id, mam_torrent, &meta).await { - Ok(_) => debug!("updated ABS via API {}", torrent.meta.mam_id), - Err(err) => warn!("Failed updating book {} in abs: {err}", torrent.meta.mam_id), + match abs.update_book(abs_id, &meta).await { + Ok(_) => debug!("updated ABS via API {}", mam_torrent.id), + Err(err) => warn!("Failed updating book {} in abs: {err}", mam_torrent.id), } } } @@ -766,7 +780,14 @@ pub async fn update_torrent_meta( if !diff.is_empty() { write_event( db, - Event::new(Some(id), Some(mam_id), EventType::Updated { fields: diff }), + Event::new( + Some(id), + Some(mam_id), + EventType::Updated { + fields: diff, + source: (MetadataSource::Mam, String::new()), + }, + ), ) .await; } @@ -780,7 +801,7 @@ async fn update_selected_torrent_meta( torrent: SelectedTorrent, meta: TorrentMeta, ) -> Result<()> { - let mam_id = meta.mam_id; + let mam_id = torrent.mam_id; let diff = torrent.meta.diff(&meta); debug!( "Updating meta for selected torrent {}, diff:\n{}", @@ -797,7 +818,14 @@ async fn update_selected_torrent_meta( drop(guard); write_event( db, - Event::new(hash, Some(mam_id), EventType::Updated { fields: diff }), + Event::new( + hash, + Some(mam_id), + EventType::Updated { + fields: diff, + source: (MetadataSource::Mam, String::new()), + }, + ), ) .await; Ok(()) @@ -815,10 +843,11 @@ fn add_duplicate_torrent( duplicate_of: Option, dl_link: Option, title_search: String, + mam_id: u64, meta: TorrentMeta, ) -> Result<()> { rw.upsert(DuplicateTorrent { - mam_id: meta.mam_id, + mam_id, dl_link, title_search, meta, diff --git a/server/src/cleaner.rs b/server/src/cleaner.rs index de1ba4b0..b719d46b 100644 --- a/server/src/cleaner.rs +++ b/server/src/cleaner.rs @@ -2,7 +2,7 @@ use std::{fs, io::ErrorKind, mem, ops::Deref, sync::Arc}; use anyhow::Result; use mlm_db::{ - self, DatabaseExt as _, ErroredTorrentId, Event, EventType, Timestamp, Torrent, TorrentKey, + self, DatabaseExt as _, ErroredTorrentId, Event, EventType, Timestamp, Torrent, TorrentKey, ids, }; use native_db::Database; use tracing::{debug, info, instrument, trace, warn}; @@ -88,7 +88,7 @@ async fn process_batch(config: &Config, db: &Database<'_>, batch: Vec) for (mut remove, _) in batch { info!( "Replacing library torrent \"{}\" {} with {}", - remove.meta.title, remove.meta.mam_id, keep.meta.mam_id + remove.meta.title, remove.id, keep.id ); remove.replaced_with = Some((keep.id.clone(), Timestamp::now())); let result = clean_torrent( @@ -148,11 +148,11 @@ pub async fn clean_torrent( remove_library_files(config, &remove, delete_in_abs).await?; let id = remove.id.clone(); - let mam_id = remove.meta.mam_id; + let mam_id = remove.meta.mam_id(); let library_path = remove.library_path.take(); let mut library_files = remove.library_files.clone(); remove.library_mismatch = None; - remove.abs_id = None; + remove.meta.ids.remove(ids::ABS); library_files.sort(); { let (_guard, rw) = db.rw_async().await?; @@ -165,7 +165,7 @@ pub async fn clean_torrent( db, Event::new( Some(id), - Some(mam_id), + mam_id, EventType::Cleaned { library_path, files: library_files, @@ -185,7 +185,8 @@ pub async fn remove_library_files( delete_in_abs: bool, ) -> Result<()> { if delete_in_abs - && let (Some(abs_id), Some(abs_config)) = (&remove.abs_id, &config.audiobookshelf) + && let (Some(abs_id), Some(abs_config)) = + (&remove.meta.ids.get(ids::ABS), &config.audiobookshelf) { let abs = Abs::new(abs_config)?; if let Err(err) = abs.delete_book(abs_id).await { @@ -194,7 +195,7 @@ pub async fn remove_library_files( } if let Some(library_path) = &remove.library_path { - debug!("Removing library files for torrent {}", remove.meta.mam_id); + debug!("Removing library files for torrent {}", remove.id); for file in remove.library_files.iter() { let path = library_path.join(file); fs::remove_file(path).or_else(|err| { diff --git a/server/src/config_impl.rs b/server/src/config_impl.rs index cab9d29c..74a2ef3b 100644 --- a/server/src/config_impl.rs +++ b/server/src/config_impl.rs @@ -85,15 +85,15 @@ impl TorrentFilter { if self.uploaded_after.is_some() || self.uploaded_before.is_some() { match UtcDateTime::parse(&torrent.added, &DATE_TIME_FORMAT) { Ok(added) => { - if let Some(uploaded_after) = self.uploaded_after { - if added.date() < uploaded_after { - return false; - } + if let Some(uploaded_after) = self.uploaded_after + && added.date() < uploaded_after + { + return false; } - if let Some(uploaded_before) = self.uploaded_before { - if added.date() > uploaded_before { - return false; - } + if let Some(uploaded_before) = self.uploaded_before + && added.date() > uploaded_before + { + return false; } } Err(_) => { @@ -106,35 +106,35 @@ impl TorrentFilter { } } - if let Some(min_seeders) = self.min_seeders { - if torrent.seeders < min_seeders { - return false; - } + if let Some(min_seeders) = self.min_seeders + && torrent.seeders < min_seeders + { + return false; } - if let Some(max_seeders) = self.max_seeders { - if torrent.seeders > max_seeders { - return false; - } + if let Some(max_seeders) = self.max_seeders + && torrent.seeders > max_seeders + { + return false; } - if let Some(min_leechers) = self.min_leechers { - if torrent.leechers < min_leechers { - return false; - } + if let Some(min_leechers) = self.min_leechers + && torrent.leechers < min_leechers + { + return false; } - if let Some(max_leechers) = self.max_leechers { - if torrent.leechers > max_leechers { - return false; - } + if let Some(max_leechers) = self.max_leechers + && torrent.leechers > max_leechers + { + return false; } - if let Some(min_snatched) = self.min_snatched { - if torrent.times_completed < min_snatched { - return false; - } + if let Some(min_snatched) = self.min_snatched + && torrent.times_completed < min_snatched + { + return false; } - if let Some(max_snatched) = self.max_snatched { - if torrent.times_completed > max_snatched { - return false; - } + if let Some(max_snatched) = self.max_snatched + && torrent.times_completed > max_snatched + { + return false; } true @@ -174,35 +174,35 @@ impl TorrentFilter { return false; } - if let Some(min_seeders) = self.min_seeders { - if torrent.seeders < min_seeders { - return false; - } + if let Some(min_seeders) = self.min_seeders + && torrent.seeders < min_seeders + { + return false; } - if let Some(max_seeders) = self.max_seeders { - if torrent.seeders > max_seeders { - return false; - } + if let Some(max_seeders) = self.max_seeders + && torrent.seeders > max_seeders + { + return false; } - if let Some(min_leechers) = self.min_leechers { - if torrent.leechers < min_leechers { - return false; - } + if let Some(min_leechers) = self.min_leechers + && torrent.leechers < min_leechers + { + return false; } - if let Some(max_leechers) = self.max_leechers { - if torrent.leechers > max_leechers { - return false; - } + if let Some(max_leechers) = self.max_leechers + && torrent.leechers > max_leechers + { + return false; } - if let Some(min_snatched) = self.min_snatched { - if torrent.times_completed < min_snatched { - return false; - } + if let Some(min_snatched) = self.min_snatched + && torrent.times_completed < min_snatched + { + return false; } - if let Some(max_snatched) = self.max_snatched { - if torrent.times_completed > max_snatched { - return false; - } + if let Some(max_snatched) = self.max_snatched + && torrent.times_completed > max_snatched + { + return false; } true @@ -871,9 +871,7 @@ mod tests { id: "".to_string(), id_is_hash: false, - mam_id: 0, - abs_id: None, - goodreads_id: None, + mam_id: None, library_path: None, library_files: vec![], linker: None, @@ -883,7 +881,6 @@ mod tests { title_search: "".to_string(), created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, } @@ -891,11 +888,12 @@ mod tests { fn default_meta() -> TorrentMeta { TorrentMeta { - mam_id: 0, + ids: Default::default(), vip_status: None, media_type: MediaType::Audiobook, main_cat: Some(MainCat::Fiction), categories: vec![], + tags: vec![], cat: None, language: None, flags: None, @@ -904,6 +902,7 @@ mod tests { size: Size::from_bytes(0), title: "".to_string(), edition: None, + description: "".to_string(), authors: vec![], narrators: vec![], series: vec![], diff --git a/server/src/linker.rs b/server/src/linker.rs index 8441f46f..7a459c9a 100644 --- a/server/src/linker.rs +++ b/server/src/linker.rs @@ -103,7 +103,7 @@ pub async fn link_torrents_to_library( { { let (_guard, rw) = db.rw_async().await?; - t.client_status = Some(ClientStatus::RemovedFromMam); + t.client_status = Some(ClientStatus::RemovedFromTracker); rw.upsert(t.clone())?; rw.commit()?; } @@ -111,8 +111,8 @@ pub async fn link_torrents_to_library( &db, Event::new( Some(torrent.hash.clone()), - Some(t.mam_id), - EventType::RemovedFromMam, + None, + EventType::RemovedFromTracker, ), ) .await; @@ -255,7 +255,7 @@ async fn match_torrent( && let Some(old_torrent) = db .r_transaction()? .get() - .secondary::(TorrentKey::mam_id, mam_torrent.id)? + .secondary::(TorrentKey::mam_id, Some(mam_torrent.id))? { if old_torrent.id != hash { let (_guard, rw) = db.rw_async().await?; @@ -307,7 +307,6 @@ async fn match_torrent( selected_audio_format, selected_ebook_format, library, - mam_torrent, existing_torrent.as_ref(), &meta, ) @@ -317,7 +316,7 @@ async fn match_torrent( } #[instrument(skip_all)] -pub async fn refresh_metadata( +pub async fn refresh_mam_metadata( config: &Config, db: &Database<'_>, mam: &MaM<'_>, @@ -326,9 +325,12 @@ pub async fn refresh_metadata( let Some(mut torrent): Option = db.r_transaction()?.get().primary(id)? else { bail!("Could not find torrent id"); }; - debug!("refreshing metadata for torrent {}", torrent.meta.mam_id); + let Some(mam_id) = torrent.meta.mam_id() else { + bail!("Could not find mam id"); + }; + debug!("refreshing metadata for torrent {}", mam_id); let Some(mam_torrent) = mam - .get_torrent_info_by_id(torrent.mam_id) + .get_torrent_info_by_id(mam_id) .await .context("get_mam_info")? else { @@ -353,6 +355,91 @@ pub async fn refresh_metadata( Ok((torrent, mam_torrent)) } +#[instrument(skip_all)] +pub async fn relink(config: &Config, db: &Database<'_>, hash: String) -> Result<()> { + let mut torrent = None; + for qbit_conf in &config.qbittorrent { + let qbit = match qbit::Api::new_login_username_password( + &qbit_conf.url, + &qbit_conf.username, + &qbit_conf.password, + ) + .await + { + Ok(qbit) => qbit, + Err(err) => { + error!("Error logging in to qbit {}: {err}", qbit_conf.url); + continue; + } + }; + let mut torrents = match qbit + .torrents(Some(TorrentListParams { + hashes: Some(vec![hash.clone()]), + ..TorrentListParams::default() + })) + .await + { + Ok(torrents) => torrents, + Err(err) => { + error!("Error getting torrents from qbit {}: {err}", qbit_conf.url); + continue; + } + }; + let Some(t) = torrents.pop() else { + continue; + }; + torrent.replace((qbit_conf, qbit, t)); + break; + } + let Some((qbit_conf, qbit, qbit_torrent)) = torrent else { + bail!("Could not find torrent in qbit"); + }; + let Some(library) = find_library(config, &qbit_torrent) else { + bail!("Could not find matching library for torrent"); + }; + let files = qbit.files(&hash, None).await?; + let selected_audio_format = select_format( + &library.tag_filters().audio_types, + &config.audio_types, + &files, + ); + let selected_ebook_format = select_format( + &library.tag_filters().ebook_types, + &config.ebook_types, + &files, + ); + + if selected_audio_format.is_none() && selected_ebook_format.is_none() { + bail!("Could not find any wanted formats in torrent"); + } + let Some(torrent) = db.r_transaction()?.get().primary::(hash.clone())? else { + bail!("Could not find torrent"); + }; + let library_path_changed = torrent.library_path + != library_dir( + config.exclude_narrator_in_library_dir, + library, + &torrent.meta, + ); + remove_library_files(config, &torrent, library_path_changed).await?; + link_torrent( + config, + qbit_conf, + db, + &hash, + &qbit_torrent, + files, + selected_audio_format, + selected_ebook_format, + library, + Some(&torrent), + &torrent.meta, + ) + .await + .context("link_torrent") + .map_err(|err| anyhow::Error::new(TorrentMetaError(torrent.meta, err))) +} + #[instrument(skip_all)] pub async fn refresh_metadata_relink( config: &Config, @@ -415,7 +502,7 @@ pub async fn refresh_metadata_relink( if selected_audio_format.is_none() && selected_ebook_format.is_none() { bail!("Could not find any wanted formats in torrent"); } - let (torrent, mam_torrent) = refresh_metadata(config, db, mam, hash.clone()).await?; + let (torrent, _mam_torrent) = refresh_mam_metadata(config, db, mam, hash.clone()).await?; let library_path_changed = torrent.library_path != library_dir( config.exclude_narrator_in_library_dir, @@ -433,7 +520,6 @@ pub async fn refresh_metadata_relink( selected_audio_format, selected_ebook_format, library, - mam_torrent, Some(&torrent), &torrent.meta, ) @@ -454,7 +540,7 @@ async fn link_torrent( selected_audio_format: Option, selected_ebook_format: Option, library: &Library, - mam_torrent: MaMTorrent, + // mam_torrent: MaMTorrent, existing_torrent: Option<&Torrent>, meta: &TorrentMeta, ) -> Result<()> { @@ -468,7 +554,7 @@ async fn link_torrent( if config.exclude_narrator_in_library_dir && !meta.narrators.is_empty() && dir.exists() { dir = library_dir(false, library, meta).unwrap(); } - let metadata = abs::create_metadata(&mam_torrent, meta); + let metadata = abs::create_metadata(meta); create_dir_all(&dir).await?; for file in files { @@ -540,9 +626,9 @@ async fn link_torrent( rw.upsert(Torrent { id: hash.to_owned(), id_is_hash: true, - mam_id: meta.mam_id, - abs_id: existing_torrent.and_then(|t| t.abs_id.clone()), - goodreads_id: existing_torrent.and_then(|t| t.goodreads_id), + mam_id: meta.mam_id(), + // abs_id: existing_torrent.and_then(|t| t.abs_id.clone()), + // goodreads_id: existing_torrent.and_then(|t| t.goodreads_id), library_path: library_path.clone(), library_files, linker: library.tag_filters().name.clone(), @@ -559,7 +645,6 @@ async fn link_torrent( .map(|t| t.created_at) .unwrap_or_else(Timestamp::now), replaced_with: existing_torrent.and_then(|t| t.replaced_with.clone()), - request_matadata_update: false, library_mismatch: None, client_status: existing_torrent.and_then(|t| t.client_status.clone()), })?; @@ -571,7 +656,7 @@ async fn link_torrent( db, Event::new( Some(hash.to_owned()), - Some(meta.mam_id), + meta.mam_id(), EventType::Linked { linker: library.tag_filters().name.clone(), library_path, diff --git a/server/src/lists/goodreads.rs b/server/src/lists/goodreads.rs index 949b21b2..4a4761b7 100644 --- a/server/src/lists/goodreads.rs +++ b/server/src/lists/goodreads.rs @@ -121,7 +121,7 @@ pub async fn run_goodreads_import( db_item } None => { - let db_item = item.as_list_item(&list_id, &list); + let db_item = item.as_list_item(&list_id, list); if !list.dry_run { let (_guard, rw) = db.rw_async().await?; rw.insert(db_item.clone())?; @@ -131,7 +131,7 @@ pub async fn run_goodreads_import( } }; trace!("Searching for book {} from Goodreads list", item.title); - search_item(&config, &db, &mam, &list, &item, db_item, max_torrents) + search_item(&config, &db, &mam, list, &item, db_item, max_torrents) .await .context("search goodreads book")?; sleep(Duration::from_millis(400)).await; @@ -212,10 +212,11 @@ async fn search_item( && db_item .audio_torrent .as_ref() - .is_none_or(|t| !(t.status == TorrentStatus::Selected && t.mam_id == found.0.id)) + .is_none_or(|t| !(t.status == TorrentStatus::Selected && t.mam_id == Some(found.0.id))) { db_item.audio_torrent = Some(ListItemTorrent { - mam_id: found.0.id, + torrent_id: None, + mam_id: Some(found.0.id), status: TorrentStatus::Selected, at: Timestamp::now(), }); @@ -225,10 +226,11 @@ async fn search_item( && db_item .ebook_torrent .as_ref() - .is_none_or(|t| !(t.status == TorrentStatus::Selected && t.mam_id == found.0.id)) + .is_none_or(|t| !(t.status == TorrentStatus::Selected && t.mam_id == Some(found.0.id))) { db_item.ebook_torrent = Some(ListItemTorrent { - mam_id: found.0.id, + torrent_id: None, + mam_id: Some(found.0.id), status: TorrentStatus::Selected, at: Timestamp::now(), }); @@ -398,7 +400,8 @@ fn not_wanted( { debug!("Skipped {:?} torrent as is not wanted", found.1.main_cat); field.replace(ListItemTorrent { - mam_id: found.0.id, + torrent_id: None, + mam_id: Some(found.0.id), status: TorrentStatus::NotWanted, at: Timestamp::now(), }); @@ -418,10 +421,11 @@ fn check_cost( warn!("Skipped {:?} torrent as it is not free", found.1.main_cat); if field .as_ref() - .is_none_or(|t| !(t.status == TorrentStatus::Wanted && t.mam_id == found.0.id)) + .is_none_or(|t| !(t.status == TorrentStatus::Wanted && t.mam_id == Some(found.0.id))) { field.replace(ListItemTorrent { - mam_id: found.0.id, + torrent_id: None, + mam_id: Some(found.0.id), status: TorrentStatus::Wanted, at: Timestamp::now(), }); diff --git a/server/src/lists/mod.rs b/server/src/lists/mod.rs index edc5810e..34de776e 100644 --- a/server/src/lists/mod.rs +++ b/server/src/lists/mod.rs @@ -168,12 +168,13 @@ fn set_existing(field: &mut Option, torrent: &Torrent) -> bool if field.status == TorrentStatus::Selected { return false; } - if field.status == TorrentStatus::Existing && field.mam_id == torrent.meta.mam_id { + if field.status == TorrentStatus::Existing && field.mam_id == torrent.mam_id { return false; } } field.replace(ListItemTorrent { - mam_id: torrent.meta.mam_id, + torrent_id: Some(torrent.id.clone()), + mam_id: torrent.mam_id, status: TorrentStatus::Existing, at: torrent.created_at, }); @@ -250,8 +251,6 @@ async fn search_grab( max_snatched: grab.filter.max_snatched, ..Default::default() }, - - ..Default::default() }) .await .context("search")?; diff --git a/server/src/qbittorrent.rs b/server/src/qbittorrent.rs index 89e3fa63..bb3808ad 100644 --- a/server/src/qbittorrent.rs +++ b/server/src/qbittorrent.rs @@ -97,7 +97,7 @@ pub async fn add_torrent_with_category( pub async fn get_torrent<'a, 'b>( config: &'a Config, - hash: &'b str, + hash: &str, ) -> Result> { for qbit_conf in config.qbittorrent.iter() { let Ok(qbit) = qbit::Api::new_login_username_password( diff --git a/server/src/snatchlist.rs b/server/src/snatchlist.rs index d2600432..4bd2ca18 100644 --- a/server/src/snatchlist.rs +++ b/server/src/snatchlist.rs @@ -149,7 +149,7 @@ async fn update_torrents>( if let Some((_, rw)) = &rw_opt { let old_library = rw .get() - .secondary::(TorrentKey::mam_id, meta.mam_id)?; + .secondary::(TorrentKey::mam_id, meta.mam_id())?; if let Some(old) = old_library { if old.meta != meta { update_torrent_meta( @@ -167,7 +167,7 @@ async fn update_torrents>( } } if cost == Cost::MetadataOnlyAdd { - let mam_id = meta.mam_id; + let mam_id = torrent.id; add_metadata_only_torrent(rw_opt.unwrap(), torrent, meta) .await .or_else(|err| { @@ -206,9 +206,7 @@ async fn add_metadata_only_torrent( rw.insert(Torrent { id, id_is_hash: false, - mam_id, - abs_id: None, - goodreads_id: None, + mam_id: Some(mam_id), library_path: None, library_files: Default::default(), linker: if torrent.uploader_name.is_empty() { @@ -223,7 +221,6 @@ async fn add_metadata_only_torrent( meta, created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, })?; @@ -242,9 +239,12 @@ async fn update_torrent_meta( linker_is_owner: bool, ) -> Result<()> { // These are missing in user details torrent response, so keep the old values + meta.ids = torrent.meta.ids.clone(); meta.media_type = torrent.meta.media_type; meta.main_cat = torrent.meta.main_cat; meta.language = torrent.meta.language; + meta.tags = torrent.meta.tags.clone(); + meta.description = torrent.meta.description.clone(); meta.num_files = torrent.meta.num_files; meta.uploaded_at = torrent.meta.uploaded_at; @@ -283,7 +283,7 @@ async fn update_torrent_meta( } let id = torrent.id.clone(); - let mam_id = meta.mam_id; + let mam_id = mam_torrent.id; let diff = torrent.meta.diff(&meta); debug!( "Updating meta for torrent {}, diff:\n{}", @@ -301,7 +301,14 @@ async fn update_torrent_meta( if !diff.is_empty() { write_event( db, - Event::new(Some(id), Some(mam_id), EventType::Updated { fields: diff }), + Event::new( + Some(id), + Some(mam_id), + EventType::Updated { + fields: diff, + source: (MetadataSource::Mam, String::new()), + }, + ), ) .await; } diff --git a/server/src/torrent_downloader.rs b/server/src/torrent_downloader.rs index b404ed12..459d2271 100644 --- a/server/src/torrent_downloader.rs +++ b/server/src/torrent_downloader.rs @@ -52,8 +52,9 @@ pub async fn grab_selected_torrents( .map(|t| t.meta.size.bytes() as f64) .sum(); - let mut remaining_buffer = (user_info.uploaded_bytes - user_info.downloaded_bytes - downloading_size) - / config.min_ratio; + let mut remaining_buffer = + (user_info.uploaded_bytes - user_info.downloaded_bytes - downloading_size) + / config.min_ratio; debug!( "downloader, unsats: {:#?}; max_torrents: {max_torrents}; buffer: {}", user_info.unsat, @@ -135,9 +136,7 @@ async fn grab_torrent( rw.upsert(mlm_db::Torrent { id: hash.clone(), id_is_hash: true, - mam_id: torrent.meta.mam_id, - abs_id: None, - goodreads_id: torrent.goodreads_id, + mam_id: torrent.meta.mam_id(), library_path: None, library_files: Default::default(), linker: None, @@ -148,7 +147,6 @@ async fn grab_torrent( meta: torrent.meta.clone(), created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, })?; @@ -203,8 +201,8 @@ async fn grab_torrent( match err.downcast::() { Ok( WedgeBuyError::IsVip - | WedgeBuyError::IsGlobalFreeleech - | WedgeBuyError::IsPersonalFreeleech, + | WedgeBuyError::IsGlobalFreeleech + | WedgeBuyError::IsPersonalFreeleech, ) => {} _ => { if torrent.cost == TorrentCost::UseWedge { @@ -219,7 +217,10 @@ async fn grab_torrent( return Err(anyhow!("Could not get torrent from MaM")); }; if !torrent_info.is_free() { - return Err(anyhow!("Torrent is no longer free, expected: {:?}", torrent.cost)); + return Err(anyhow!( + "Torrent is no longer free, expected: {:?}", + torrent.cost + )); } } @@ -252,9 +253,7 @@ async fn grab_torrent( rw.upsert(mlm_db::Torrent { id: hash.clone(), id_is_hash: true, - mam_id: torrent.meta.mam_id, - abs_id: None, - goodreads_id: torrent.goodreads_id, + mam_id: torrent.meta.mam_id(), library_path: None, library_files: Default::default(), linker: None, @@ -265,7 +264,6 @@ async fn grab_torrent( meta: torrent.meta.clone(), created_at: Timestamp::now(), replaced_with: None, - request_matadata_update: false, library_mismatch: None, client_status: None, })?; diff --git a/server/src/web/mod.rs b/server/src/web/mod.rs index dc4b37be..e61f24d8 100644 --- a/server/src/web/mod.rs +++ b/server/src/web/mod.rs @@ -16,7 +16,7 @@ use axum::{ }; use itertools::Itertools; use mlm_db::{ - AudiobookCategory, Category, EbookCategory, Flags, SelectedTorrent, Series, Timestamp, Torrent, + AudiobookCategory, EbookCategory, Flags, SelectedTorrent, Series, Timestamp, Torrent, TorrentMeta, }; use mlm_mam::{api::MaM, meta::MetaError, search::MaMTorrent, serde::DATE_FORMAT}; @@ -214,14 +214,6 @@ pub trait Page { } } - fn cats<'a, T: Key>(&'a self, field: T, cats: &'a Vec) -> CatsTmpl<'a, T> { - CatsTmpl { - field, - cats, - path: self.item_path(), - } - } - fn series<'a, T: Key>(&'a self, field: T, series: &'a Vec) -> SeriesTmpl<'a, T> { SeriesTmpl { field, @@ -315,32 +307,6 @@ impl<'a, T: Key> SeriesTmpl<'a, T> { } } -/// ```askama -/// {% for c in cats %} -/// {{ item(*field, **c) | safe }}{% if !loop.last %}, {% endif %} -/// {% endfor %} -/// ``` -#[derive(Template)] -#[template(ext = "html", in_doc = true)] -pub struct CatsTmpl<'a, T: Key> { - field: T, - cats: &'a Vec, - path: &'a str, -} - -impl<'a, T: Key> HtmlSafe for CatsTmpl<'a, T> {} - -impl<'a, T: Key> CatsTmpl<'a, T> { - fn item(&'a self, field: T, cat: Category) -> ItemFilter<'a, T> { - ItemFilter { - field, - label: cat.as_str(), - value: Some(cat.as_id().to_string()), - path: self.path, - } - } -} - impl<'a, T: Key> HtmlSafe for SeriesTmpl<'a, T> {} #[derive(Template)] diff --git a/server/src/web/pages/config.rs b/server/src/web/pages/config.rs index eb34e51e..d8a4f862 100644 --- a/server/src/web/pages/config.rs +++ b/server/src/web/pages/config.rs @@ -67,10 +67,12 @@ pub async fn config_page_post( } } Err(err) => { + let Some(mam_id) = torrent.mam_id else { + continue; + }; info!("need to ask mam due to: {err}"); let mam = context.mam()?; - let Some(mam_torrent) = mam.get_torrent_info_by_id(torrent.mam_id).await? - else { + let Some(mam_torrent) = mam.get_torrent_info_by_id(mam_id).await? else { warn!("could not get torrent from mam"); continue; }; diff --git a/server/src/web/pages/duplicate.rs b/server/src/web/pages/duplicate.rs index a8dc298e..209c6e54 100644 --- a/server/src/web/pages/duplicate.rs +++ b/server/src/web/pages/duplicate.rs @@ -6,7 +6,7 @@ use axum::{ }; use axum_extra::extract::Form; use mlm_db::{ - DatabaseExt as _, DuplicateTorrent, SelectedTorrent, Timestamp, Torrent, TorrentCost, + DatabaseExt as _, DuplicateTorrent, SelectedTorrent, Timestamp, Torrent, TorrentCost, ids, }; use mlm_parse::normalize_title; use serde::{Deserialize, Serialize}; @@ -154,7 +154,6 @@ pub async fn duplicate_torrents_page_post( let (_guard, rw) = context.db.rw_async().await?; rw.insert(SelectedTorrent { mam_id: mam_torrent.id, - goodreads_id: None, hash: None, dl_link: mam_torrent .dl diff --git a/server/src/web/pages/events.rs b/server/src/web/pages/events.rs index 8a401931..4a4b5b86 100644 --- a/server/src/web/pages/events.rs +++ b/server/src/web/pages/events.rs @@ -36,7 +36,7 @@ pub async fn event_page( EventType::Linked { .. } => value == "linker", EventType::Cleaned { .. } => value == "cleaner", EventType::Updated { .. } => value == "updated", - EventType::RemovedFromMam { .. } => value == "removed", + EventType::RemovedFromTracker { .. } => value == "removed", }, EventPageFilter::Grabber => match t.event { EventType::Grabbed { ref grabber, .. } => { diff --git a/server/src/web/pages/replaced.rs b/server/src/web/pages/replaced.rs index af21364e..f2ba71b1 100644 --- a/server/src/web/pages/replaced.rs +++ b/server/src/web/pages/replaced.rs @@ -10,13 +10,13 @@ use axum::{ response::{Html, Redirect}, }; use axum_extra::extract::Form; -use mlm_db::{Language, Torrent, TorrentKey}; +use mlm_db::{Language, Torrent, TorrentKey, ids}; use serde::{Deserialize, Serialize}; use crate::stats::Context; use crate::web::{Page, tables}; use crate::{ - linker::{refresh_metadata, refresh_metadata_relink}, + linker::{refresh_mam_metadata, refresh_metadata_relink}, web::{ AppError, tables::{Flex, HidableColumns, Key, Pagination, PaginationParams, SortOn, Sortable}, @@ -141,7 +141,7 @@ pub async fn replaced_torrents_page_post( "refresh" => { let mam = context.mam()?; for torrent in form.torrents { - refresh_metadata(&config, &context.db, &mam, torrent).await?; + refresh_mam_metadata(&config, &context.db, &mam, torrent).await?; } } "refresh-relink" => { diff --git a/server/src/web/pages/search.rs b/server/src/web/pages/search.rs index 98a46d6c..dde2f46c 100644 --- a/server/src/web/pages/search.rs +++ b/server/src/web/pages/search.rs @@ -47,7 +47,7 @@ pub async fn search_page( let meta = mam_torrent.as_meta()?; let torrent = r .get() - .secondary::(TorrentKey::mam_id, meta.mam_id)?; + .secondary::(TorrentKey::mam_id, meta.mam_id())?; let selected_torrent = r.get().primary(mam_torrent.id)?; Ok((mam_torrent, meta, torrent, selected_torrent)) @@ -149,7 +149,6 @@ pub async fn select_torrent(context: &Context, mam_id: u64, wedge: bool) -> Resu let (_guard, rw) = context.db.rw_async().await?; rw.insert(SelectedTorrent { mam_id: torrent.id, - goodreads_id: None, hash: None, dl_link: torrent .dl diff --git a/server/src/web/pages/torrent.rs b/server/src/web/pages/torrent.rs index cabaf0be..5dc7bd90 100644 --- a/server/src/web/pages/torrent.rs +++ b/server/src/web/pages/torrent.rs @@ -33,7 +33,9 @@ use crate::{ audiobookshelf::{Abs, LibraryItemMinified}, cleaner::clean_torrent, config::Config, - linker::{find_library, library_dir, map_path, refresh_metadata, refresh_metadata_relink}, + linker::{ + find_library, library_dir, map_path, refresh_mam_metadata, refresh_metadata_relink, relink, + }, qbittorrent::{self, ensure_category_exists}, stats::Context, web::{ @@ -177,7 +179,11 @@ async fn torrent_page_id( events.sort_by(|a, b| b.created_at.cmp(&a.created_at)); let mam = context.mam()?; - let mam_torrent = mam.get_torrent_info_by_id(torrent.mam_id).await?; + let mam_torrent = if let Some(mam_id) = torrent.mam_id { + mam.get_torrent_info_by_id(mam_id).await? + } else { + None + }; let mam_meta = mam_torrent.as_ref().map(|t| t.as_meta()).transpose()?; if let Some(mam_meta) = &mam_meta @@ -326,7 +332,10 @@ pub async fn torrent_page_post_id( } "refresh" => { let mam = context.mam()?; - refresh_metadata(&config, &context.db, &mam, id).await?; + refresh_mam_metadata(&config, &context.db, &mam, id).await?; + } + "relink" => { + relink(&config, &context.db, id).await?; } "refresh-relink" => { let mam = context.mam()?; @@ -362,7 +371,8 @@ pub async fn torrent_page_post_id( rw.commit()?; } "qbit" => { - let Some((torrent, qbit, qbit_conf)) = qbittorrent::get_torrent(&config, &id).await? else { + let Some((torrent, qbit, qbit_conf)) = qbittorrent::get_torrent(&config, &id).await? + else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; @@ -562,12 +572,12 @@ async fn other_torrents( let torrents = result .data .into_iter() - .filter(|t| t.id != meta.mam_id) + .filter(|t| Some(t.id) != meta.mam_id()) .map(|mam_torrent| { let meta = mam_torrent.as_meta()?; let torrent = r .get() - .secondary::(TorrentKey::mam_id, meta.mam_id)?; + .secondary::(TorrentKey::mam_id, meta.mam_id())?; let selected_torrent = r.get().primary(mam_torrent.id)?; Ok((mam_torrent, meta, torrent, selected_torrent)) diff --git a/server/src/web/pages/torrents.rs b/server/src/web/pages/torrents.rs index 26715dc9..7f5c7624 100644 --- a/server/src/web/pages/torrents.rs +++ b/server/src/web/pages/torrents.rs @@ -11,13 +11,14 @@ use axum::{ response::{Html, Redirect}, }; use axum_extra::extract::Form; +use mlm_db::ids; use mlm_db::{Language, LibraryMismatch, Torrent, TorrentKey}; use serde::{Deserialize, Serialize}; use sublime_fuzzy::FuzzySearch; use crate::{ cleaner::clean_torrent, - linker::{refresh_metadata, refresh_metadata_relink}, + linker::{refresh_mam_metadata, refresh_metadata_relink}, stats::Context, web::{ AppError, @@ -27,8 +28,8 @@ use crate::{ web::{Page, flag_icons, tables}, }; use mlm_db::{ - Category, ClientStatus, DatabaseExt as _, Flags, MediaType, MetadataSource, OldCategory, - Series, SeriesEntry, + ClientStatus, DatabaseExt as _, Flags, MediaType, MetadataSource, OldCategory, Series, + SeriesEntry, }; pub async fn torrents_page( @@ -98,9 +99,7 @@ pub async fn torrents_page( } else { value .split(",") - .filter_map(|id| id.parse().ok()) - .filter_map(Category::from_id) - .all(|cat| t.meta.categories.contains(&cat)) + .all(|cat| t.meta.categories.iter().any(|c| c.as_str() == cat.trim())) } } TorrentsPageFilter::Flags => { @@ -184,10 +183,10 @@ pub async fn torrents_page( } TorrentsPageFilter::ClientStatus => match t.client_status { Some(ClientStatus::NotInClient) => value == "not_in_client", - Some(ClientStatus::RemovedFromMam) => value == "removed_from_mam", + Some(ClientStatus::RemovedFromTracker) => value == "removed_from_tracker", None => false, }, - TorrentsPageFilter::Abs => t.abs_id.is_some() == (value == "true"), + TorrentsPageFilter::Abs => t.meta.ids.contains_key(ids::ABS) == (value == "true"), TorrentsPageFilter::Source => match value.as_str() { "mam" => t.meta.source == MetadataSource::Mam, "manual" => t.meta.source == MetadataSource::Manual, @@ -641,7 +640,7 @@ pub async fn torrents_page_post( "refresh" => { let mam = context.mam()?; for torrent in form.torrents { - refresh_metadata(&config, &context.db, &mam, torrent).await?; + refresh_mam_metadata(&config, &context.db, &mam, torrent).await?; } } "refresh-relink" => { diff --git a/server/templates/pages/duplicate.html b/server/templates/pages/duplicate.html index a975adae..6e71cd86 100644 --- a/server/templates/pages/duplicate.html +++ b/server/templates/pages/duplicate.html @@ -43,7 +43,7 @@

Duplicate Torrents

{{ items(DuplicatePageFilter::Filetype, torrent.meta.filetypes) }}
{{ self::time(torrent.created_at) }}
- +
duplicate of:
{{ item(DuplicatePageFilter::Title, duplicate_of.meta.title) }}
@@ -58,8 +58,8 @@

Duplicate Torrents

{{ self::time(duplicate_of.created_at) }}
open - MaM - {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), duplicate_of.abs_id.as_ref()) %} + {% if let Some(mam_id) = duplicate_of.mam_id %}MaM{% endif %} + {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), duplicate_of.meta.ids.get(ids::ABS)) %} ABS {% endif %}
diff --git a/server/templates/pages/errors.html b/server/templates/pages/errors.html index fac81cfa..77c7ac09 100644 --- a/server/templates/pages/errors.html +++ b/server/templates/pages/errors.html @@ -36,8 +36,10 @@

Torrent Errors

{% match error.meta %} {% when Some(meta) %} - open - MaM + {% if let Some(mam_id) = meta.mam_id() %} + open + MaM + {% endif %} {% when None %} {% endmatch %}
diff --git a/server/templates/pages/events.html b/server/templates/pages/events.html index 68b7c5a9..6e37f976 100644 --- a/server/templates/pages/events.html +++ b/server/templates/pages/events.html @@ -106,15 +106,15 @@

Events

{% endfor %} - {% when EventType::Updated { fields } %} - Updated {{ torrent_media_type(&torrent) }} Torrent {{ torrent_title(&torrent) | safe }}
+ {% when EventType::Updated { fields, source } %} + Updated {{ torrent_media_type(&torrent) }} Torrent {{ torrent_title(&torrent) | safe }} from {{ source.0 }} {{ source.1 }}
    {% for field in fields %}
  • {{ field.field }}: {{ field.from }} → {{ field.to }}
    • {% endfor %}
- {% when EventType::RemovedFromMam %} - {{ torrent_media_type(&torrent) }} Torrent {{ torrent_title(&torrent) | safe }} was removed from MaM
+ {% when EventType::RemovedFromTracker %} + {{ torrent_media_type(&torrent) }} Torrent {{ torrent_title(&torrent) | safe }} was removed from Tracker
{% endmatch %} {% endfor %} diff --git a/server/templates/pages/list.html b/server/templates/pages/list.html index 346e0bf0..91e95df4 100644 --- a/server/templates/pages/list.html +++ b/server/templates/pages/list.html @@ -47,13 +47,13 @@

{{ item.title }}

{% if let Some(torrent) = item.audio_torrent %} {% match torrent.status %} {% when TorrentStatus::Selected %} - Downloaded audiobook torrent at {{ self::time(torrent.at) }}
+ Downloaded audiobook torrent at {{ self::time(torrent.at) }}
{% when TorrentStatus::Wanted %} - Suggest wedge audiobook torrent at {{ self::time(torrent.at) }}
+ Suggest wedge audiobook torrent at {{ self::time(torrent.at) }}
{% when TorrentStatus::NotWanted %} - Skipped audiobook torrent as an ebook was found at {{ self::time(torrent.at) }}
+ Skipped audiobook torrent as an ebook was found at {{ self::time(torrent.at) }}
{% when TorrentStatus::Existing %} - Found matching audiobook torrent in library at {{ self::time(torrent.at) }}
+ Found matching audiobook torrent in library at {{ self::time(torrent.at) }}
{% endmatch %} {% elif item.want_audio() %} Audiobook missing
@@ -61,13 +61,13 @@

{{ item.title }}

{% if let Some(torrent) = item.ebook_torrent %} {% match torrent.status %} {% when TorrentStatus::Selected %} - Downloaded ebook torrent at {{ self::time(torrent.at) }}
+ Downloaded ebook torrent at {{ self::time(torrent.at) }}
{% when TorrentStatus::Wanted %} - Suggest wedge ebook torrent at {{ self::time(torrent.at) }}
+ Suggest wedge ebook torrent at {{ self::time(torrent.at) }}
{% when TorrentStatus::NotWanted %} - Skipped ebook torrent as an ebook was found at {{ self::time(torrent.at) }}
+ Skipped ebook torrent as an ebook was found at {{ self::time(torrent.at) }}
{% when TorrentStatus::Existing %} - Found matching ebook torrent in library at {{ self::time(torrent.at) }}
+ Found matching ebook torrent in library at {{ self::time(torrent.at) }}
{% endmatch %} {% elif item.want_ebook() %} Ebook missing
diff --git a/server/templates/pages/replaced.html b/server/templates/pages/replaced.html index 00fb307d..6d33ab79 100644 --- a/server/templates/pages/replaced.html +++ b/server/templates/pages/replaced.html @@ -97,8 +97,8 @@

Replaced Torrents

{{ self::time(torrent.created_at) }}
open - MaM - {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), torrent.abs_id.as_ref()) %} + {% if let Some(mam_id) = torrent.mam_id %}MaM{% endif %} + {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), torrent.meta.ids.get(ids::ABS)) %} ABS {% endif %}
@@ -130,8 +130,8 @@

Replaced Torrents

{{ self::time(replacement.created_at) }}
open - MaM - {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), replacement.abs_id.as_ref()) %} + {% if let Some(mam_id) = replacement.mam_id %}MaM{% endif %} + {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), replacement.meta.ids.get(ids::ABS)) %} ABS {% endif %}
diff --git a/server/templates/pages/selected.html b/server/templates/pages/selected.html index f249521d..ddc86fca 100644 --- a/server/templates/pages/selected.html +++ b/server/templates/pages/selected.html @@ -168,7 +168,7 @@

Selected Torrents

{% endif %} {% endif %} - + {% endfor %} {% if torrents.is_empty() %} diff --git a/server/templates/pages/torrent.html b/server/templates/pages/torrent.html index 9d1aa874..2cfbcd0f 100644 --- a/server/templates/pages/torrent.html +++ b/server/templates/pages/torrent.html @@ -31,7 +31,7 @@

Replaced with: {{ torrent.meta.title }} {% else %}
@@ -269,15 +272,15 @@

Replaced with: {{ torrent.meta.title }} - {% when EventType::Updated { fields } %} - Updated Torrent
+ {% when EventType::Updated { fields, source } %} + Updated Torrent from {{ source.0 }} {{ source.1 }}
    {% for field in fields %}
  • {{ field.field }}: {{ field.from }} -> {{ field.to }}
    • {% endfor %}
- {% when EventType::RemovedFromMam %} - Torrent was removed from MaM
+ {% when EventType::RemovedFromTracker %} + Torrent was removed from Tracker
{% endmatch %}

{% endfor %} diff --git a/server/templates/pages/torrent_mam.html b/server/templates/pages/torrent_mam.html index 3324938d..b485b854 100644 --- a/server/templates/pages/torrent_mam.html +++ b/server/templates/pages/torrent_mam.html @@ -20,7 +20,7 @@

{{ meta.title }}

{% endif %}
diff --git a/server/templates/pages/torrents.html b/server/templates/pages/torrents.html index facaf17a..eb09f799 100644 --- a/server/templates/pages/torrents.html +++ b/server/templates/pages/torrents.html @@ -134,7 +134,7 @@

Torrents

{% endif %}
{% if show.categories %} -
{{ cats(TorrentsPageFilter::Categories, torrent.meta.categories) }}
+
{{ items(TorrentsPageFilter::Categories, torrent.meta.categories) }}
{% endif %} {% if show.flags %}
{{ self::flag_icons(torrent.meta) }}
@@ -142,12 +142,12 @@

Torrents

{{ item(TorrentsPageFilter::Title, torrent.meta.title) }} {% match torrent.client_status %} - {% when Some(ClientStatus::RemovedFromMam) %} - - {{ item_v(TorrentsPageFilter::ClientStatus, "⚠", "removed_from_mam") }} + {% when Some(ClientStatus::RemovedFromTracker) %} + + {{ item_v(TorrentsPageFilter::ClientStatus, "⚠", "removed_from_tracker") }} {% when Some(ClientStatus::NotInClient) %} - + {{ item_v(TorrentsPageFilter::ClientStatus, "ℹ", "not_in_client") }} {% when None %} @@ -227,8 +227,8 @@

Torrents

{% endif %}
open - MaM - {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), torrent.abs_id.as_ref()) %} + {% if let Some(mam_id) = torrent.mam_id %}MaM{% endif %} + {% if let (Some(abs_url), Some(abs_id)) = (abs_url.as_ref(), torrent.meta.ids.get(ids::ABS)) %} ABS {% endif %}
diff --git a/server/templates/partials/mam_torrents.html b/server/templates/partials/mam_torrents.html index 4e7fde37..35545766 100644 --- a/server/templates/partials/mam_torrents.html +++ b/server/templates/partials/mam_torrents.html @@ -20,7 +20,7 @@ {% if mam_torrent.lang_code != "ENG" %} [{{ mam_torrent.lang_code }}] {% endif %} - {{ meta.title }}{% if let Some((edition, _)) = meta.edition %} {{ edition }}{% endif %}
+ {{ meta.title }}{% if let Some((edition, _)) = meta.edition %} {{ edition }}{% endif %}
by {% for author in meta.authors %}{{ author }}{% if !loop.last %}, {% endif %}{% endfor %}
{% if !meta.series.is_empty() %} series @@ -50,7 +50,7 @@ Torrent is downloaded {% else %} - + {% if let Some(wedge_over) = config.wedge_over %} {% if &meta.size >= &wedge_over && !mam_torrent.is_free() %} - {% if linker_run_at.is_some() %} -

Result: {% match linker_result %}{% when Some(Ok(())) %}success{% when Some(Err(err)) %}{{ err }}{% when None %}running{% endmatch %} +

Torrent Linker

+

Last run: {% match torrent_linker_run_at %}{% when Some(run_at) %}{{ self::time(run_at) }}{% when None %}never{% endmatch %} + + {% if torrent_linker_run_at.is_some() %} +

Result: {% match torrent_linker_result %}{% when Some(Ok(())) %}success{% when Some(Err(err)) %}{{ err }}{% when None %}running{% endmatch %} + {% endif %} + + +

+

Folder Linker

+

Last run: {% match folder_linker_run_at %}{% when Some(run_at) %}{{ self::time(run_at) }}{% when None %}never{% endmatch %} + + {% if folder_linker_run_at.is_some() %} +

Result: {% match folder_linker_result %}{% when Some(Ok(())) %}success{% when Some(Err(err)) %}{{ err }}{% when None %}running{% endmatch %} {% endif %}

diff --git a/server/templates/partials/filter.html b/server/templates/partials/filter.html index ab31b9a2..7bc26f04 100644 --- a/server/templates/partials/filter.html +++ b/server/templates/partials/filter.html @@ -1,7 +1,7 @@ -{% if filter.categories.audio.is_some() || filter.categories.ebook.is_some() %} - categories = +{% if filter.edition.categories.audio.is_some() || filter.edition.categories.ebook.is_some() %} + categories = {
-{% if let Some(cats) = filter.categories.audio %} +{% if let Some(cats) = filter.edition.categories.audio %} {% if cats.is_empty() %}   audio = false
{% elif cats == &AudiobookCategory::all() %} @@ -12,7 +12,7 @@ {% else %}   audio = true
{% endif %} -{% if let Some(cats) = filter.categories.ebook %} +{% if let Some(cats) = filter.edition.categories.ebook %} {% if cats.is_empty() %}   ebook = false
{% elif cats == &EbookCategory::all() %} @@ -26,58 +26,58 @@ }
{% else %} {% endif %} -{% if !filter.languages.is_empty() %} -languages = {{ self::yaml_items(filter.languages) }}
+{% if !filter.edition.languages.is_empty() %} +languages = {{ self::yaml_items(filter.edition.languages) }}
{% endif %} -{% if filter.flags.as_bitfield() > 0 %} +{% if filter.edition.flags.as_bitfield() > 0 %} flags = { -{% if filter.flags.as_search_bitfield().1.len() > 3 %} +{% if filter.edition.flags.as_search_bitfield().1.len() > 3 %}
- {% if let Some(flag) = filter.flags.crude_language %} + {% if let Some(flag) = filter.edition.flags.crude_language %}   crude_language = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.violence %} + {% if let Some(flag) = filter.edition.flags.violence %}   violence = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.some_explicit %} + {% if let Some(flag) = filter.edition.flags.some_explicit %}   some_explicit = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.explicit %} + {% if let Some(flag) = filter.edition.flags.explicit %}   explicit = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.abridged %} + {% if let Some(flag) = filter.edition.flags.abridged %}   abridged = {{ flag }}
{% endif %} - {% if let Some(flag) = filter.flags.lgbt %} + {% if let Some(flag) = filter.edition.flags.lgbt %}   lgbt = {{ flag }}
{% endif %} {% else %} - {% if let Some(flag) = filter.flags.crude_language %} + {% if let Some(flag) = filter.edition.flags.crude_language %} crude_language = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.violence %} + {% if let Some(flag) = filter.edition.flags.violence %} violence = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.some_explicit %} + {% if let Some(flag) = filter.edition.flags.some_explicit %} some_explicit = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.explicit %} + {% if let Some(flag) = filter.edition.flags.explicit %} explicit = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.abridged %} + {% if let Some(flag) = filter.edition.flags.abridged %} abridged = {{ flag }} {% endif %} - {% if let Some(flag) = filter.flags.lgbt %} + {% if let Some(flag) = filter.edition.flags.lgbt %} lgbt = {{ flag }} {% endif %} {% endif %} }
{% endif %} -{% if filter.min_size.bytes() > 0 %} -min_size = "{{ filter.min_size }}"
+{% if filter.edition.min_size.bytes() > 0 %} +min_size = "{{ filter.edition.min_size }}"
{% endif %} -{% if filter.max_size.bytes() > 0 %} -max_size = "{{ filter.max_size }}"
+{% if filter.edition.max_size.bytes() > 0 %} +max_size = "{{ filter.edition.max_size }}"
{% endif %} {% if !filter.exclude_uploader.is_empty() %} exclude_uploader = {{ self::yaml_items(filter.exclude_uploader) }}
From 7416be1d98f5b88302e388560f704e6592c4c34b Mon Sep 17 00:00:00 2001 From: Stirling Mouse <181794392+StirlingMouse@users.noreply.github.com> Date: Sat, 7 Feb 2026 09:40:54 +0100 Subject: [PATCH 04/14] Check for duplicates while linking folders --- server/src/cleaner.rs | 44 +------ server/src/linker/duplicates.rs | 205 ++++++++++++++++++++++++++++++++ server/src/linker/folder.rs | 58 +++++---- server/src/linker/mod.rs | 4 +- 4 files changed, 249 insertions(+), 62 deletions(-) create mode 100644 server/src/linker/duplicates.rs diff --git a/server/src/cleaner.rs b/server/src/cleaner.rs index b719d46b..3b7402c0 100644 --- a/server/src/cleaner.rs +++ b/server/src/cleaner.rs @@ -10,7 +10,7 @@ use tracing::{debug, info, instrument, trace, warn}; use crate::{ audiobookshelf::Abs, config::Config, - linker::file_size, + linker::rank_torrents, logging::{TorrentMetaError, update_errored_torrent, write_event}, qbittorrent::ensure_category_exists, }; @@ -47,45 +47,9 @@ async fn process_batch(config: &Config, db: &Database<'_>, batch: Vec) if batch.len() == 1 { return Ok(()); }; - let mut batch = batch - .into_iter() - .map(|torrent| { - let preferred_types = config.preferred_types(&torrent.meta.media_type); - let preference = preferred_types - .iter() - .position(|t| torrent.meta.filetypes.contains(t)) - .unwrap_or(usize::MAX); - (torrent, preference) - }) - .collect::>(); - batch.sort_by_key(|(_, preference)| *preference); - if batch[0].1 == batch[1].1 { - trace!( - "need to compare torrent \"{}\" and \"{}\" by size", - batch[0].0.meta.title, batch[1].0.meta.title - ); - let mut new_batch = batch - .into_iter() - .map(|(torrent, preference)| { - let mut size = 0; - if let Some(library_path) = &torrent.library_path { - for file in &torrent.library_files { - let path = library_path.join(file); - size += fs::metadata(path).map_or(0, |s| file_size(&s)); - } - } - (torrent, preference, size) - }) - .collect::>(); - new_batch.sort_by(|a, b| a.1.cmp(&b.1).then(b.2.cmp(&a.2))); - trace!("new_batch {:?}", new_batch); - batch = new_batch - .into_iter() - .map(|(torrent, preference, _)| (torrent, preference)) - .collect(); - } - let (keep, _) = batch.remove(0); - for (mut remove, _) in batch { + let mut batch = rank_torrents(config, batch); + let keep = batch.remove(0); + for mut remove in batch { info!( "Replacing library torrent \"{}\" {} with {}", remove.meta.title, remove.id, keep.id diff --git a/server/src/linker/duplicates.rs b/server/src/linker/duplicates.rs new file mode 100644 index 00000000..cf6f5997 --- /dev/null +++ b/server/src/linker/duplicates.rs @@ -0,0 +1,205 @@ +use std::fs; + +use anyhow::Result; +use mlm_db::{Torrent, TorrentKey}; +use native_db::Database; +use tracing::trace; + +use crate::config::Config; +use crate::linker::file_size; + +pub fn find_matches(db: &Database<'_>, torrent: &Torrent) -> Result> { + let r = db.r_transaction()?; + let torrents = r.scan().secondary::(TorrentKey::title_search)?; + let matches = torrents + .all()? + .filter_map(|t| t.ok()) + .filter(|t| t.id != torrent.id && t.matches(torrent)) + .collect(); + Ok(matches) +} + +pub fn rank_torrents(config: &Config, batch: Vec) -> Vec { + if batch.len() <= 1 { + return batch; + } + + let mut ranked = batch + .into_iter() + .map(|torrent| { + let preferred_types = config.preferred_types(&torrent.meta.media_type); + let preference = preferred_types + .iter() + .position(|t| torrent.meta.filetypes.contains(t)) + .unwrap_or(usize::MAX); + (torrent, preference) + }) + .collect::>(); + ranked.sort_by_key(|(_, preference)| *preference); + + if ranked[0].1 == ranked[1].1 { + let mut with_size = ranked + .into_iter() + .map(|(torrent, preference)| { + let mut size = 0; + if let Some(library_path) = &torrent.library_path { + for file in &torrent.library_files { + let path = library_path.join(file); + size += fs::metadata(path).map_or(0, |s| file_size(&s)); + } + } + if size == 0 { + size = torrent.meta.size.bytes(); + } + (torrent, preference, size) + }) + .collect::>(); + with_size.sort_by(|a, b| a.1.cmp(&b.1).then(b.2.cmp(&a.2))); + trace!("ranked batch by size: {:?}", with_size); + ranked = with_size + .into_iter() + .map(|(torrent, preference, _)| (torrent, preference)) + .collect(); + } + ranked.into_iter().map(|(t, _)| t).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + use mlm_db::{MediaType, Size, TorrentMeta, Timestamp, MetadataSource, MainCat, Language}; + use crate::config::SearchConfig; + + fn create_test_torrent(id: &str, title: &str, filetypes: Vec, size_bytes: u64) -> Torrent { + let meta = TorrentMeta { + title: title.to_string(), + filetypes, + size: Size::from_bytes(size_bytes), + media_type: MediaType::Audiobook, + main_cat: Some(MainCat::Fiction), + source: MetadataSource::Mam, + uploaded_at: Timestamp::now(), + authors: vec!["Author".to_string()], + language: Some(Language::English), + ids: BTreeMap::new(), + vip_status: None, + cat: None, + categories: vec![], + tags: vec![], + flags: None, + num_files: 1, + edition: None, + description: "".to_string(), + narrators: vec![], + series: vec![], + }; + Torrent { + id: id.to_string(), + id_is_hash: false, + mam_id: None, + library_path: None, + library_files: vec![], + linker: None, + category: None, + selected_audio_format: None, + selected_ebook_format: None, + title_search: mlm_parse::normalize_title(title), + meta, + created_at: Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + } + } + + fn create_test_config() -> Config { + Config { + mam_id: "test".to_string(), + web_host: "0.0.0.0".to_string(), + web_port: 3157, + min_ratio: 2.0, + unsat_buffer: 10, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 30, + link_interval: 10, + import_interval: 135, + ignore_torrents: vec![], + audio_types: vec!["m4b".to_string(), "mp3".to_string()], + ebook_types: vec!["epub".to_string(), "pdf".to_string()], + music_types: vec!["flac".to_string(), "mp3".to_string()], + radio_types: vec!["mp3".to_string()], + search: SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![], + } + } + + #[test] + fn test_rank_torrents_preference() { + let config = create_test_config(); + + let t1 = create_test_torrent("1", "Title", vec!["mp3".to_string()], 100); + let t2 = create_test_torrent("2", "Title", vec!["m4b".to_string()], 100); + + let batch = vec![t1.clone(), t2.clone()]; + let ranked = rank_torrents(&config, batch); + + assert_eq!(ranked[0].id, "2"); // m4b is preferred over mp3 + assert_eq!(ranked[1].id, "1"); + } + + #[test] + fn test_rank_torrents_size_tie_break() { + let config = create_test_config(); + + let t1 = create_test_torrent("1", "Title", vec!["m4b".to_string()], 100); + let t2 = create_test_torrent("2", "Title", vec!["m4b".to_string()], 200); + + let batch = vec![t1.clone(), t2.clone()]; + let ranked = rank_torrents(&config, batch); + + assert_eq!(ranked[0].id, "2"); // Larger size wins tie + assert_eq!(ranked[1].id, "1"); + } + + #[tokio::test] + async fn test_find_matches() -> Result<()> { + let tmp_dir = std::env::temp_dir().join(format!("mlm_test_duplicates_{}", std::process::id())); + let _ = fs::remove_dir_all(&tmp_dir); + fs::create_dir_all(&tmp_dir)?; + let db_path = tmp_dir.join("test.db"); + + let db = native_db::Builder::new().create(&mlm_db::MODELS, &db_path)?; + mlm_db::migrate(&db)?; + + let t1 = create_test_torrent("1", "My Book", vec!["m4b".to_string()], 100); + let t2 = create_test_torrent("2", "My Book", vec!["mp3".to_string()], 150); + let t3 = create_test_torrent("3", "Other Book", vec!["m4b".to_string()], 100); + + { + let rw = db.rw_transaction()?; + rw.insert(t1.clone())?; + rw.insert(t2.clone())?; + rw.insert(t3.clone())?; + rw.commit()?; + } + + let matches = find_matches(&db, &t1)?; + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].id, "2"); + + drop(db); + let _ = fs::remove_dir_all(tmp_dir); + Ok(()) + } +} + diff --git a/server/src/linker/folder.rs b/server/src/linker/folder.rs index 174bf579..57d70073 100644 --- a/server/src/linker/folder.rs +++ b/server/src/linker/folder.rs @@ -22,7 +22,9 @@ use tracing::{Level, instrument, span, trace, warn}; use crate::audiobookshelf as abs; use crate::config::{Config, Library, LibraryLinkMethod}; -use crate::linker::common::{copy, file_size, hard_link, library_dir, select_format, symlink}; +use crate::linker::{ + copy, file_size, find_matches, hard_link, library_dir, rank_torrents, select_format, symlink, +}; use crate::logging::{update_errored_torrent, write_event}; #[instrument(skip_all)] @@ -176,6 +178,35 @@ async fn link_libation_folder( }; let meta = clean_meta(meta, "")?; + let mut torrent = Torrent { + id: libation_meta.asin.clone(), + id_is_hash: false, + mam_id: meta.mam_id(), + library_path: None, + library_files: vec![], + linker: library.options().name.clone(), + category: None, + selected_audio_format: None, + selected_ebook_format: None, + title_search: normalize_title(&meta.title), + meta: meta.clone(), + created_at: Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + }; + + let matches = find_matches(db, &torrent)?; + if !matches.is_empty() { + let mut batch = matches; + batch.push(torrent.clone()); + let ranked = rank_torrents(config, batch); + if ranked[0].id != torrent.id { + trace!("Skipping folder as it is a duplicate of a better torrent already in library"); + return Ok(()); + } + } + if let Some(filter) = library.edition_filter() && !filter.matches_meta(&meta).is_ok_and(|matches| matches) { @@ -241,26 +272,11 @@ async fn link_libation_folder( { let (_guard, rw) = db.rw_async().await?; - rw.upsert(Torrent { - id: libation_meta.asin.clone(), - id_is_hash: false, - mam_id: meta.mam_id(), - library_path: library_path.clone(), - library_files, - linker: library.options().name.clone(), - category: None, - selected_audio_format, - selected_ebook_format, - title_search: normalize_title(&meta.title), - meta: meta.clone(), - // created_at: existing_torrent - // .map(|t| t.created_at) - // .unwrap_or_else(Timestamp::now), - created_at: Timestamp::now(), - replaced_with: None, - library_mismatch: None, - client_status: None, - })?; + torrent.library_path = library_path.clone(); + torrent.library_files = library_files; + torrent.selected_audio_format = selected_audio_format; + torrent.selected_ebook_format = selected_ebook_format; + rw.upsert(torrent)?; rw.commit()?; } diff --git a/server/src/linker/mod.rs b/server/src/linker/mod.rs index e5a1f0f6..922147ee 100644 --- a/server/src/linker/mod.rs +++ b/server/src/linker/mod.rs @@ -1,6 +1,8 @@ pub mod common; +pub mod duplicates; pub mod folder; pub mod torrent; -pub use self::common::{file_size, library_dir, map_path}; +pub use self::common::{copy, file_size, hard_link, library_dir, map_path, select_format, symlink}; +pub use self::duplicates::{find_matches, rank_torrents}; pub use self::torrent::{find_library, refresh_mam_metadata, refresh_metadata_relink, relink}; From d1c2a26276ca18b1388ac7c225bd027460ec8ec7 Mon Sep 17 00:00:00 2001 From: Stirling Mouse <181794392+StirlingMouse@users.noreply.github.com> Date: Sat, 7 Feb 2026 09:52:50 +0100 Subject: [PATCH 05/14] Add size checking in filters --- server/src/config_impl.rs | 46 ++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/server/src/config_impl.rs b/server/src/config_impl.rs index a6510c09..247a17c3 100644 --- a/server/src/config_impl.rs +++ b/server/src/config_impl.rs @@ -340,9 +340,15 @@ impl EditionFilter { ); } - // TODO: Match size - ensure!(self.min_size.bytes() == 0, "has min_size"); - ensure!(self.max_size.bytes() == 0, "has max_size"); + if self.min_size.bytes() > 0 || self.max_size.bytes() > 0 { + ensure!(meta.size.bytes() != 0, "has size filter and no stored size"); + if self.min_size.bytes() > 0 && meta.size < self.min_size { + return Ok(false); + } + if self.max_size.bytes() > 0 && meta.size > self.max_size { + return Ok(false); + } + } Ok(true) } @@ -1238,7 +1244,6 @@ mod tests { ); } - // --- Disallowed Filter Checks (Ensure) --- #[test] fn test_disallowed_min_size_err() { let filter = TorrentFilter { @@ -1255,10 +1260,41 @@ mod tests { .matches_lib(&torrent) .unwrap_err() .to_string() - .contains("has min_size") + .contains("has size filter and no stored size") ); } + #[test] + fn test_match_size_ok_true_when_meta_has_size() { + let mut meta = default_meta(); + meta.size = Size::from_bytes(2_000_000_000); + let torrent = create_torrent_with_meta(meta); + let filter = TorrentFilter { + edition: EditionFilter { + min_size: Size::from_bytes(1_000_000_000), + max_size: Size::from_bytes(3_000_000_000), + ..Default::default() + }, + ..Default::default() + }; + assert!(filter.matches_lib(&torrent).unwrap()); + } + + #[test] + fn test_match_size_ok_false_when_meta_out_of_range() { + let mut meta = default_meta(); + meta.size = Size::from_bytes(500_000_000); + let torrent = create_torrent_with_meta(meta); + let filter = TorrentFilter { + edition: EditionFilter { + min_size: Size::from_bytes(1_000_000_000), + ..Default::default() + }, + ..Default::default() + }; + assert!(!filter.matches_lib(&torrent).unwrap()); + } + #[test] fn test_disallowed_exclude_uploader_err() { let filter = TorrentFilter { From 7eb69560c460788b5f1e983dd6fa836585a6902c Mon Sep 17 00:00:00 2001 From: Stirling Mouse <181794392+StirlingMouse@users.noreply.github.com> Date: Sat, 7 Feb 2026 10:19:49 +0100 Subject: [PATCH 06/14] Add initial integration tests --- Cargo.lock | 21 ++-- mlm_db/src/lib.rs | 3 +- server/Cargo.toml | 3 + server/src/config_impl.rs | 3 + server/src/lib.rs | 16 +++ server/src/main.rs | 38 ++----- server/tests/cleaner_test.rs | 66 +++++++++++ server/tests/common/mod.rs | 212 +++++++++++++++++++++++++++++++++++ server/tests/linker_test.rs | 133 ++++++++++++++++++++++ 9 files changed, 456 insertions(+), 39 deletions(-) create mode 100644 server/src/lib.rs create mode 100644 server/tests/cleaner_test.rs create mode 100644 server/tests/common/mod.rs create mode 100644 server/tests/linker_test.rs diff --git a/Cargo.lock b/Cargo.lock index a5507c96..3291f795 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1396,9 +1396,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libredox" @@ -1412,9 +1412,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" @@ -1587,6 +1587,7 @@ dependencies = [ "serde_derive", "serde_json", "sublime_fuzzy", + "tempfile", "thiserror 2.0.17", "time", "tokio", @@ -2367,15 +2368,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.7" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ "bitflags 2.9.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2835,15 +2836,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.20.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] diff --git a/mlm_db/src/lib.rs b/mlm_db/src/lib.rs index cb039033..bccaedea 100644 --- a/mlm_db/src/lib.rs +++ b/mlm_db/src/lib.rs @@ -23,8 +23,9 @@ use std::collections::HashMap; use anyhow::Result; use mlm_parse::normalize_title; use native_db::Models; +pub use native_db::Database; use native_db::transaction::RwTransaction; -use native_db::{Database, ToInput, db_type}; +use native_db::{ToInput, db_type}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use tokio::sync::MutexGuard; diff --git a/server/Cargo.toml b/server/Cargo.toml index 190c6003..ab92c832 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -82,3 +82,6 @@ winsafe = { version = "0.0.26", features = ["gui"] } [build-dependencies] embed-resource = "3.0.5" + +[dev-dependencies] +tempfile = "3.24.0" diff --git a/server/src/config_impl.rs b/server/src/config_impl.rs index 247a17c3..3dd2abb3 100644 --- a/server/src/config_impl.rs +++ b/server/src/config_impl.rs @@ -255,6 +255,9 @@ impl EditionFilter { } pub(crate) fn matches_meta(&self, meta: &TorrentMeta) -> Result { + if !self.media_type.is_empty() && !self.media_type.contains(&meta.media_type) { + return Ok(false); + } if let Some(language) = &meta.language { if !self.languages.is_empty() && !self.languages.contains(language) { return Ok(false); diff --git a/server/src/lib.rs b/server/src/lib.rs new file mode 100644 index 00000000..1e09b56d --- /dev/null +++ b/server/src/lib.rs @@ -0,0 +1,16 @@ +pub mod audiobookshelf; +pub mod autograbber; +pub mod cleaner; +pub mod config; +pub mod config_impl; +pub mod exporter; +pub mod linker; +pub mod lists; +pub mod logging; +pub mod qbittorrent; +pub mod snatchlist; +pub mod stats; +pub mod torrent_downloader; +pub mod web; +#[cfg(target_family = "windows")] +pub mod windows; diff --git a/server/src/main.rs b/server/src/main.rs index 989133b6..1b40f3ad 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,22 +1,5 @@ #![windows_subsystem = "windows"] -mod audiobookshelf; -mod autograbber; -mod cleaner; -mod config; -mod config_impl; -mod exporter; -mod linker; -mod lists; -mod logging; -mod qbittorrent; -mod snatchlist; -mod stats; -mod torrent_downloader; -mod web; -#[cfg(target_family = "windows")] -mod windows; - use std::{ collections::BTreeMap, env, @@ -29,40 +12,41 @@ use std::{ }; use anyhow::{Context as _, Result}; -use audiobookshelf::match_torrents_to_abs; -use autograbber::run_autograbber; -use cleaner::run_library_cleaner; use dirs::{config_dir, data_local_dir}; -use exporter::export_db; use figment::{ Figment, providers::{Env, Format, Toml}, }; use mlm_mam::api::MaM; -use stats::{Stats, Triggers}; use time::OffsetDateTime; use tokio::{ select, sync::{Mutex, watch}, time::sleep, }; -use torrent_downloader::grab_selected_torrents; use tracing::error; use tracing_appender::rolling::{RollingFileAppender, Rotation}; use tracing_subscriber::{ EnvFilter, Layer as _, fmt::time::LocalTime, layer::SubscriberExt as _, util::SubscriberInitExt as _, }; -use web::start_webserver; -use crate::{ +use mlm::{ + audiobookshelf::match_torrents_to_abs, + autograbber::run_autograbber, + cleaner::run_library_cleaner, config::Config, linker::{folder::link_folders_to_library, torrent::link_torrents_to_library}, lists::{get_lists, run_list_import}, snatchlist::run_snatchlist_search, - stats::Context, + stats::{Context, Stats, Triggers}, + torrent_downloader::grab_selected_torrents, + web::start_webserver, }; +#[cfg(target_family = "windows")] +use mlm::windows; + #[tokio::main] async fn main() { if let Err(err) = app_main().await { @@ -218,8 +202,6 @@ async fn app_main() -> Result<()> { return Ok(()); } - // export_db(&db)?; - // return Ok(()); let db = Arc::new(db); #[cfg(target_family = "windows")] diff --git a/server/tests/cleaner_test.rs b/server/tests/cleaner_test.rs new file mode 100644 index 00000000..ca47d127 --- /dev/null +++ b/server/tests/cleaner_test.rs @@ -0,0 +1,66 @@ +mod common; + +use common::{TestDb, MockFs, mock_config, MockTorrentBuilder}; +use mlm::cleaner::run_library_cleaner; +use std::sync::Arc; +use mlm_db::{Torrent, DatabaseExt}; + +#[tokio::test] +async fn test_run_library_cleaner() -> anyhow::Result<()> { + let test_db = TestDb::new()?; + let mock_fs = MockFs::new()?; + let config = Arc::new(mock_config(mock_fs.rip_dir.clone(), mock_fs.library_dir.clone())); + + // Create two versions of the same book + let lib_path1 = mock_fs.library_dir.join("Author 1").join("Book 1 (v1)"); + let lib_path2 = mock_fs.library_dir.join("Author 1").join("Book 1 (v2)"); + std::fs::create_dir_all(&lib_path1)?; + std::fs::create_dir_all(&lib_path2)?; + std::fs::write(lib_path1.join("file.m4b"), "v1")?; + std::fs::write(lib_path2.join("file.m4b"), "v2 is longer than v1")?; + + let mut t1 = MockTorrentBuilder::new("ID1", "Book 1") + .with_library_path(lib_path1.clone()) + .with_size(100) + .with_author("Author 1") + .with_language(mlm_db::Language::English) + .build(); + t1.library_files = vec!["file.m4b".into()]; + + let mut t2 = MockTorrentBuilder::new("ID2", "Book 1") + .with_library_path(lib_path2.clone()) + .with_size(200) // Better version because it's larger + .with_author("Author 1") + .with_language(mlm_db::Language::English) + .build(); + t2.library_files = vec!["file.m4b".into()]; + + { + let (_guard, rw) = test_db.db.rw_async().await?; + rw.insert(t1)?; + rw.insert(t2)?; + rw.commit()?; + } + + run_library_cleaner(config.clone(), test_db.db.clone()).await?; + + let r = test_db.db.r_transaction()?; + let t1_after: Torrent = r.get().primary("ID1".to_string())?.unwrap(); + let t2_after: Torrent = r.get().primary("ID2".to_string())?.unwrap(); + + // t1 should be replaced_with t2 + assert!(t1_after.replaced_with.is_some(), "t1 should be replaced"); + assert_eq!(t1_after.replaced_with.unwrap().0, "ID2"); + assert!(t1_after.library_path.is_none(), "t1 library path should be cleared"); + + // t2 should still be there + assert!(t2_after.replaced_with.is_none(), "t2 should not be replaced"); + assert!(t2_after.library_path.is_some(), "t2 library path should still be set"); + + // Files for t1 should be deleted + assert!(!lib_path1.exists(), "t1 files should be deleted"); + // Files for t2 should still exist + assert!(lib_path2.exists(), "t2 files should still exist"); + + Ok(()) +} diff --git a/server/tests/common/mod.rs b/server/tests/common/mod.rs new file mode 100644 index 00000000..f36a564f --- /dev/null +++ b/server/tests/common/mod.rs @@ -0,0 +1,212 @@ +use anyhow::Result; +use mlm::config::{Config, Library, LibraryByRipDir, LibraryLinkMethod, LibraryOptions}; +use mlm_db::{ + migrate, Database, MainCat, MediaType, MetadataSource, Size, Timestamp, Torrent, TorrentMeta, + MODELS, +}; +use native_db::Builder; +use std::path::PathBuf; +use std::sync::Arc; +use tempfile::TempDir; + +pub struct TestDb { + pub db: Arc>, + #[allow(dead_code)] + temp_dir: TempDir, +} + +impl TestDb { + pub fn new() -> Result { + let temp_dir = tempfile::tempdir()?; + let db_path = temp_dir.path().join("test.db"); + let db = Builder::new().create(&MODELS, db_path)?; + migrate(&db)?; + Ok(Self { + db: Arc::new(db), + temp_dir, + }) + } +} + +pub struct MockTorrentBuilder { + torrent: Torrent, +} + +impl MockTorrentBuilder { + pub fn new(id: &str, title: &str) -> Self { + Self { + torrent: Torrent { + id: id.to_string(), + id_is_hash: false, + mam_id: None, + library_path: None, + library_files: vec![], + linker: None, + category: None, + selected_audio_format: None, + selected_ebook_format: None, + title_search: mlm_parse::normalize_title(title), + meta: TorrentMeta { + ids: Default::default(), + vip_status: None, + cat: None, + media_type: MediaType::Audiobook, + main_cat: Some(MainCat::Fiction), + categories: vec![], + tags: vec![], + language: None, + flags: None, + filetypes: vec!["m4b".to_string()], + num_files: 1, + size: Size::from_bytes(0), + title: title.to_string(), + edition: None, + description: "".to_string(), + authors: vec![], + narrators: vec![], + series: vec![], + source: MetadataSource::Mam, + uploaded_at: Some(Timestamp::now()), + }, + created_at: Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + }, + } + } + + pub fn with_library_path(mut self, path: PathBuf) -> Self { + self.torrent.library_path = Some(path); + self + } + + pub fn with_mam_id(mut self, mam_id: u64) -> Self { + self.torrent.mam_id = Some(mam_id); + self.torrent + .meta + .ids + .insert(mlm_db::ids::MAM.to_string(), mam_id.to_string()); + self + } + + pub fn with_size(mut self, size_bytes: u64) -> Self { + self.torrent.meta.size = Size::from_bytes(size_bytes); + self + } + + pub fn with_author(mut self, author: &str) -> Self { + self.torrent.meta.authors.push(author.to_string()); + self + } + + pub fn with_language(mut self, language: mlm_db::Language) -> Self { + self.torrent.meta.language = Some(language); + self + } + + pub fn build(self) -> Torrent { + self.torrent + } +} + +pub struct MockFs { + #[allow(dead_code)] + pub root: TempDir, + pub rip_dir: PathBuf, + pub library_dir: PathBuf, +} + +impl MockFs { + pub fn new() -> Result { + let root = tempfile::tempdir()?; + let rip_dir = root.path().join("rip"); + let library_dir = root.path().join("library"); + std::fs::create_dir_all(&rip_dir)?; + std::fs::create_dir_all(&library_dir)?; + Ok(Self { + root, + rip_dir, + library_dir, + }) + } + + pub fn create_libation_folder( + &self, + asin: &str, + title: &str, + authors: Vec<&str>, + ) -> Result { + let folder_path = self.rip_dir.join(asin); + std::fs::create_dir_all(&folder_path)?; + + let libation_meta = serde_json::json!({ + "asin": asin, + "title": title, + "subtitle": "", + "authors": authors.into_iter().map(|a| serde_json::json!({"name": a})).collect::>(), + "narrators": [], + "series": [], + "language": "English", + "format_type": "unabridged", + "publisher_summary": "Test summary", + "merchandising_summary": "Test merchandising summary", + "category_ladders": [], + "is_adult_product": false, + "issue_date": "2023-01-01", + "publication_datetime": "2023-01-01T00:00:00Z", + "publication_name": "Test Publisher", + "publisher_name": "Test Publisher", + "release_date": "2023-01-01", + "runtime_length_min": 60, + }); + + let meta_path = folder_path.join(format!("{}.json", asin)); + std::fs::write(meta_path, serde_json::to_string(&libation_meta)?)?; + + let audio_path = folder_path.join(format!("{}.m4b", asin)); + std::fs::write(audio_path, "fake audio data")?; + + Ok(folder_path) + } +} + +pub fn mock_config(rip_dir: PathBuf, library_dir: PathBuf) -> Config { + Config { + mam_id: "test".to_string(), + web_host: "127.0.0.1".to_string(), + web_port: 3157, + min_ratio: 2.0, + unsat_buffer: 10, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 30, + link_interval: 10, + import_interval: 135, + ignore_torrents: vec![], + audio_types: vec!["m4b".to_string()], + ebook_types: vec!["epub".to_string()], + music_types: vec!["mp3".to_string()], + radio_types: vec!["mp3".to_string()], + search: Default::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![Library::ByRipDir(LibraryByRipDir { + rip_dir, + options: LibraryOptions { + name: Some("test_library".to_string()), + library_dir, + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + filter: Default::default(), + })], + } +} diff --git a/server/tests/linker_test.rs b/server/tests/linker_test.rs new file mode 100644 index 00000000..a861c49e --- /dev/null +++ b/server/tests/linker_test.rs @@ -0,0 +1,133 @@ +mod common; + +use common::{TestDb, MockFs, mock_config}; +use mlm::linker::folder::link_folders_to_library; +use std::sync::Arc; +use mlm_db::{Torrent, DatabaseExt}; + +#[tokio::test] +async fn test_link_folders_to_library() -> anyhow::Result<()> { + let test_db = TestDb::new()?; + let mock_fs = MockFs::new()?; + let config = Arc::new(mock_config(mock_fs.rip_dir.clone(), mock_fs.library_dir.clone())); + + mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; + + link_folders_to_library(config.clone(), test_db.db.clone()).await?; + + let r = test_db.db.r_transaction()?; + let torrent: Option = r.get().primary("B00TEST1".to_string())?; + assert!(torrent.is_some()); + let torrent = torrent.unwrap(); + assert_eq!(torrent.meta.title, "Test Book 1"); + assert_eq!(torrent.meta.authors, vec!["Author 1"]); + assert!(torrent.library_path.is_some()); + + // Check if files were created in library + let expected_dir = mock_fs.library_dir.join("Author 1").join("Test Book 1"); + assert!(expected_dir.exists()); + assert!(expected_dir.join("B00TEST1.m4b").exists()); + assert!(expected_dir.join("metadata.json").exists()); + + Ok(()) +} + +#[tokio::test] +async fn test_link_folders_to_library_duplicate_skipping() -> anyhow::Result<()> { + let test_db = TestDb::new()?; + let mock_fs = MockFs::new()?; + let config = Arc::new(mock_config(mock_fs.rip_dir.clone(), mock_fs.library_dir.clone())); + + // Create a better version already in the DB + let existing = common::MockTorrentBuilder::new("MAM123", "Test Book 1") + .with_mam_id(123) + .with_size(1000) // 1000 bytes + .with_author("Author 1") + .with_language(mlm_db::Language::English) + .build(); + + { + let (_guard, rw) = test_db.db.rw_async().await?; + rw.insert(existing)?; + rw.commit()?; + } + + // Create a libation folder with the same title but smaller size (worse version) + // Libation folder files will have small size "fake audio data" = 15 bytes + mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; + + link_folders_to_library(config.clone(), test_db.db.clone()).await?; + + let r = test_db.db.r_transaction()?; + let torrent: Option = r.get().primary("B00TEST1".to_string())?; + // It should NOT be in the DB because it was skipped as a duplicate of a better version + assert!(torrent.is_none()); + + Ok(()) +} + +#[tokio::test] +async fn test_link_folders_to_library_filter_size_too_small() -> anyhow::Result<()> { + let test_db = TestDb::new()?; + let mock_fs = MockFs::new()?; + let mut config = mock_config(mock_fs.rip_dir.clone(), mock_fs.library_dir.clone()); + + if let mlm::config::Library::ByRipDir(ref mut l) = config.libraries[0] { + l.filter.min_size = mlm_db::Size::from_bytes(100); // Libation folder is 15 bytes + } + let config = Arc::new(config); + + mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; + + link_folders_to_library(config.clone(), test_db.db.clone()).await?; + + let r = test_db.db.r_transaction()?; + let torrent: Option = r.get().primary("B00TEST1".to_string())?; + assert!(torrent.is_none(), "Should have been skipped due to size filter"); + + Ok(()) +} + +#[tokio::test] +async fn test_link_folders_to_library_filter_media_type_mismatch() -> anyhow::Result<()> { + let test_db = TestDb::new()?; + let mock_fs = MockFs::new()?; + let mut config = mock_config(mock_fs.rip_dir.clone(), mock_fs.library_dir.clone()); + + if let mlm::config::Library::ByRipDir(ref mut l) = config.libraries[0] { + l.filter.media_type = vec![mlm_db::MediaType::Ebook]; // Libation is Audiobook + } + let config = Arc::new(config); + + mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; + + link_folders_to_library(config.clone(), test_db.db.clone()).await?; + + let r = test_db.db.r_transaction()?; + let torrent: Option = r.get().primary("B00TEST1".to_string())?; + assert!(torrent.is_none(), "Should have been skipped due to media type filter"); + + Ok(()) +} + +#[tokio::test] +async fn test_link_folders_to_library_filter_language_mismatch() -> anyhow::Result<()> { + let test_db = TestDb::new()?; + let mock_fs = MockFs::new()?; + let mut config = mock_config(mock_fs.rip_dir.clone(), mock_fs.library_dir.clone()); + + if let mlm::config::Library::ByRipDir(ref mut l) = config.libraries[0] { + l.filter.languages = vec![mlm_db::Language::German]; // Libation is English + } + let config = Arc::new(config); + + mock_fs.create_libation_folder("B00TEST1", "Test Book 1", vec!["Author 1"])?; + + link_folders_to_library(config.clone(), test_db.db.clone()).await?; + + let r = test_db.db.r_transaction()?; + let torrent: Option = r.get().primary("B00TEST1".to_string())?; + assert!(torrent.is_none(), "Should have been skipped due to language filter"); + + Ok(()) +} From 477b87712a0e52843d65db548fe422e821e5a5da Mon Sep 17 00:00:00 2001 From: Stirling Mouse <181794392+StirlingMouse@users.noreply.github.com> Date: Sat, 7 Feb 2026 10:47:07 +0100 Subject: [PATCH 07/14] Add more unit tests tests: extend select_format unit tests for edge cases (leading dot, uppercase, missing ext, empty overrides) --- Cargo.lock | 1 + mlm_db/tests/meta_diff.rs | 226 +++++++++++++++++++++++++++++++++ mlm_mam/Cargo.toml | 3 +- mlm_mam/src/serde.rs | 177 ++++++++++++++++++++++++++ mlm_parse/tests/parse_tests.rs | 30 +++++ server/src/config_impl.rs | 28 ++-- server/src/linker/common.rs | 52 ++++++++ server/tests/config_impl.rs | 170 +++++++++++++++++++++++++ 8 files changed, 670 insertions(+), 17 deletions(-) create mode 100644 mlm_db/tests/meta_diff.rs create mode 100644 mlm_parse/tests/parse_tests.rs create mode 100644 server/tests/config_impl.rs diff --git a/Cargo.lock b/Cargo.lock index 3291f795..19cea592 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1641,6 +1641,7 @@ dependencies = [ "native_db", "native_model", "once_cell", + "openssl", "reqwest", "reqwest_cookie_store", "serde", diff --git a/mlm_db/tests/meta_diff.rs b/mlm_db/tests/meta_diff.rs new file mode 100644 index 00000000..cab0cb8d --- /dev/null +++ b/mlm_db/tests/meta_diff.rs @@ -0,0 +1,226 @@ +use std::collections::BTreeMap; +use time::Duration; + +use mlm_db::*; + +#[test] +fn torrent_meta_diff_detects_field_changes() { + let mut ids_a = BTreeMap::new(); + ids_a.insert(ids::MAM.to_string(), "1".to_string()); + + let meta_a = TorrentMeta { + ids: ids_a, + vip_status: None, + cat: None, + media_type: MediaType::Ebook, + main_cat: Some(MainCat::Fiction), + categories: vec!["CatA".to_string()], + tags: vec![], + language: Some(Language::English), + flags: Some(FlagBits::new(0)), + filetypes: vec!["pdf".to_string()], + num_files: 1, + size: Size::from_bytes(1024), + title: "Title A".to_string(), + edition: None, + description: String::new(), + authors: vec!["Author A".to_string()], + narrators: vec![], + series: vec![], + source: MetadataSource::Mam, + uploaded_at: Some(Timestamp::now()), + }; + + let mut ids_b = BTreeMap::new(); + ids_b.insert(ids::MAM.to_string(), "2".to_string()); + ids_b.insert(ids::ASIN.to_string(), "ASIN123".to_string()); + + let meta_b = TorrentMeta { + ids: ids_b, + vip_status: Some(VipStatus::Permanent), + cat: None, + media_type: MediaType::Audiobook, + main_cat: Some(MainCat::Nonfiction), + categories: vec!["CatB".to_string()], + tags: vec![], + language: Some(Language::Other), + flags: Some(FlagBits::new(0b0000_0011)), + filetypes: vec!["epub".to_string()], + num_files: 2, + size: Size::from_bytes(2048), + title: "Title B".to_string(), + edition: Some(("2nd edition".to_string(), 2)), + description: String::new(), + authors: vec!["Author B".to_string()], + narrators: vec!["Narrator".to_string()], + series: vec![Series { + name: "Series X".to_string(), + entries: SeriesEntries::new(vec![]), + }], + source: MetadataSource::Manual, + uploaded_at: Some(Timestamp::now()), + }; + + let diffs = meta_a.diff(&meta_b); + + // Collect field names as strings since TorrentMetaField doesn't derive Eq/Hash in all + // versions; Display is stable and used by the application. + let names: Vec = diffs.iter().map(|d| d.field.to_string()).collect(); + + // Expect at least these diffs to be present + let expected = [ + "ids", + "vip", + "media_type", + "main_cat", + "categories", + "language", + "flags", + "filetypes", + "size", + "title", + "edition", + "authors", + "narrators", + "series", + "source", + ]; + + for &e in &expected { + assert!(names.contains(&e.to_string()), "missing diff for field {e}"); + } +} + +#[test] +fn meta_diff_strict_checks() { + let mut ids_a = BTreeMap::new(); + ids_a.insert(ids::MAM.to_string(), "1".to_string()); + + let mut ids_b = BTreeMap::new(); + ids_b.insert(ids::MAM.to_string(), "2".to_string()); + ids_b.insert(ids::ASIN.to_string(), "ASIN123".to_string()); + + let size_a = Size::from_bytes(1_024); + let size_b = Size::from_bytes(2_048); + + let meta_a = TorrentMeta { + ids: ids_a.clone(), + vip_status: None, + cat: None, + media_type: MediaType::Ebook, + main_cat: Some(MainCat::Fiction), + categories: vec!["CatA".to_string()], + tags: vec![], + language: Some(Language::English), + flags: Some(FlagBits::new(0)), + filetypes: vec!["pdf".to_string()], + num_files: 1, + size: size_a, + title: "Title A".to_string(), + edition: None, + description: String::new(), + authors: vec!["Author A".to_string()], + narrators: vec![], + series: vec![], + source: MetadataSource::Mam, + uploaded_at: Some(Timestamp::now()), + }; + + let meta_b = TorrentMeta { + ids: ids_b.clone(), + vip_status: Some(VipStatus::Permanent), + cat: None, + media_type: MediaType::Audiobook, + main_cat: Some(MainCat::Nonfiction), + categories: vec!["CatB".to_string()], + tags: vec![], + language: Some(Language::Other), + flags: Some(FlagBits::new(0b11)), + filetypes: vec!["epub".to_string()], + num_files: 2, + size: size_b, + title: "Title B".to_string(), + edition: Some(("2nd edition".to_string(), 2)), + description: String::new(), + authors: vec!["Author B".to_string()], + narrators: vec!["Narrator".to_string()], + series: vec![Series { + name: "Series X".to_string(), + entries: SeriesEntries::new(vec![]), + }], + source: MetadataSource::Manual, + uploaded_at: Some(Timestamp::now()), + }; + + let diffs = meta_a.diff(&meta_b); + + // ensure we have at least the expected number of diffs + assert!( + diffs.len() >= 10, + "expected many diffs but got {}", + diffs.len() + ); + + // check specific diffs content + let get = |field: &str| { + diffs + .iter() + .find(|d| d.field.to_string() == field) + .unwrap_or_else(|| panic!("missing field {}", field)) + }; + + let ids = get("ids"); + assert_eq!(ids.from, "mam: 1"); + assert_eq!(ids.to, "asin: ASIN123\nmam: 2"); + + let size = get("size"); + assert_eq!(size.from, format!("{}", size_a)); + assert_eq!(size.to, format!("{}", size_b)); + + let title = get("title"); + assert_eq!(title.from, "Title A"); + assert_eq!(title.to, "Title B"); +} + +#[test] +fn vip_expiry() { + let mut ids = BTreeMap::new(); + ids.insert(ids::MAM.to_string(), "1".to_string()); + + let base = TorrentMeta { + ids: ids.clone(), + vip_status: None, + cat: None, + media_type: MediaType::Ebook, + main_cat: Some(MainCat::Fiction), + categories: vec!["Cat".to_string()], + tags: vec![], + language: Some(Language::English), + flags: Some(FlagBits::new(0)), + filetypes: vec!["pdf".to_string()], + num_files: 1, + size: Size::from_bytes(1024), + title: "Title".to_string(), + edition: None, + description: String::new(), + authors: vec!["Author".to_string()], + narrators: vec![], + series: vec![], + source: MetadataSource::Mam, + uploaded_at: Some(Timestamp::now()), + }; + + let mut a = base.clone(); + let past_date = (Timestamp::now().0 - Duration::days(30)).date(); + a.vip_status = Some(VipStatus::Temp(past_date)); + + let mut b = base; + b.vip_status = Some(VipStatus::NotVip); + + let diffs = a.diff(&b); + let names: Vec = diffs.iter().map(|d| d.field.to_string()).collect(); + assert!( + !names.contains(&"vip".to_string()), + "vip diff should be suppressed when going from expired temp -> NotVip" + ); +} diff --git a/mlm_mam/Cargo.toml b/mlm_mam/Cargo.toml index d64f548a..e336b77b 100644 --- a/mlm_mam/Cargo.toml +++ b/mlm_mam/Cargo.toml @@ -13,7 +13,8 @@ mlm_parse = { path = "../mlm_parse" } native_db = { git = "https://github.com/StirlingMouse/native_db.git", branch = "0.8.x" } native_model = "0.4.20" once_cell = "1.21.3" -reqwest = "0.12.20" +openssl = { version = "0.10.73", features = ["vendored"] } +reqwest = { version = "0.12.20", features = ["json"] } reqwest_cookie_store = "0.8.0" serde = "1.0.136" serde_derive = "1.0.136" diff --git a/mlm_mam/src/serde.rs b/mlm_mam/src/serde.rs index 52c07724..1528d35f 100644 --- a/mlm_mam/src/serde.rs +++ b/mlm_mam/src/serde.rs @@ -146,3 +146,180 @@ where v.map(|v| Date::parse(&v, &DATE_FORMAT).map_err(serde::de::Error::custom)) .transpose() } + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + + #[derive(Deserialize)] + struct BoolWrap { + #[serde(deserialize_with = "bool_string_or_number")] + val: bool, + } + + #[test] + fn test_bool_string_or_number_variants() { + assert!( + serde_json::from_str::(r#"{"val":"yes"}"#) + .unwrap() + .val + ); + assert!( + serde_json::from_str::(r#"{"val":"1"}"#) + .unwrap() + .val + ); + assert!( + serde_json::from_str::(r#"{"val":1}"#) + .unwrap() + .val + ); + assert!( + !serde_json::from_str::(r#"{"val":"no"}"#) + .unwrap() + .val + ); + assert!( + !serde_json::from_str::(r#"{"val":"0"}"#) + .unwrap() + .val + ); + assert!( + !serde_json::from_str::(r#"{"val":0}"#) + .unwrap() + .val + ); + // boolean true/false should be rejected by this deserializer + assert!(serde_json::from_str::(r#"{"val": true}"#).is_err()); + } + + #[derive(Deserialize)] + struct NumWrap { + #[serde(deserialize_with = "num_string_or_number")] + val: u32, + } + + #[test] + fn test_num_string_or_number() { + assert_eq!( + serde_json::from_str::(r#"{"val":"42"}"#) + .unwrap() + .val, + 42 + ); + assert_eq!( + serde_json::from_str::(r#"{"val":42}"#) + .unwrap() + .val, + 42 + ); + assert!(serde_json::from_str::(r#"{"val":"notanumber"}"#).is_err()); + } + + #[derive(Deserialize)] + struct OptNumWrap { + #[serde(deserialize_with = "opt_num_string_or_number")] + val: Option, + } + + #[test] + fn test_opt_num_and_string_or_number() { + assert_eq!( + serde_json::from_str::(r#"{"val":"100"}"#) + .unwrap() + .val, + Some(100) + ); + assert_eq!( + serde_json::from_str::(r#"{"val":100}"#) + .unwrap() + .val, + Some(100) + ); + assert_eq!( + serde_json::from_str::(r#"{"val":null}"#) + .unwrap() + .val, + None + ); + + #[derive(Deserialize)] + struct OptStrWrap { + #[serde(deserialize_with = "opt_string_or_number")] + val: Option, + } + + assert_eq!( + serde_json::from_str::(r#"{"val":"abc"}"#) + .unwrap() + .val, + Some("abc".to_string()) + ); + assert_eq!( + serde_json::from_str::(r#"{"val":123}"#) + .unwrap() + .val, + Some("123".to_string()) + ); + assert_eq!( + serde_json::from_str::(r#"{"val":null}"#) + .unwrap() + .val, + None + ); + } + + #[derive(Deserialize)] + struct VecWrap { + #[serde(deserialize_with = "vec_string_or_number")] + val: Vec, + } + + #[test] + fn test_vec_string_or_number() { + let v: VecWrap = serde_json::from_str(r#"{"val":["1", 2, null]}"#).unwrap(); + assert_eq!(v.val, vec!["1".to_string(), "2".to_string()]); + // invalid element type (object) should error + assert!(serde_json::from_str::(r#"{"val":[{}]}"#).is_err()); + } + + #[derive(Deserialize)] + struct StrWrap { + #[serde(deserialize_with = "string_or_number")] + val: String, + } + + #[test] + fn test_string_or_number() { + assert_eq!( + serde_json::from_str::(r#"{"val":"abc"}"#) + .unwrap() + .val, + "abc" + ); + assert_eq!( + serde_json::from_str::(r#"{"val":123}"#) + .unwrap() + .val, + "123" + ); + assert!(serde_json::from_str::(r#"{"val": null}"#).is_err()); + } + + #[derive(Deserialize)] + struct DateWrap { + #[serde(deserialize_with = "parse_opt_date")] + d: Option, + } + + #[test] + fn test_parse_opt_date() { + let ok = serde_json::from_str::(r#"{"d":"2023-01-02"}"#).unwrap(); + assert!(ok.d.is_some()); + let none = serde_json::from_str::(r#"{"d":null}"#).unwrap(); + assert!(none.d.is_none()); + // invalid date should error + assert!(serde_json::from_str::(r#"{"d":"not-a-date"}"#).is_err()); + } +} diff --git a/mlm_parse/tests/parse_tests.rs b/mlm_parse/tests/parse_tests.rs new file mode 100644 index 00000000..dad4850f --- /dev/null +++ b/mlm_parse/tests/parse_tests.rs @@ -0,0 +1,30 @@ +use mlm_parse::{clean_name, clean_value, normalize_title}; + +#[test] +fn test_clean_value_decodes_entities() { + let s = "Tom & Jerry "Fun""; + let cleaned = clean_value(s).unwrap(); + assert_eq!(cleaned, "Tom & Jerry \"Fun\""); +} + +#[test] +fn test_normalize_title_variants() { + let s = "The Amazing & Strange Vol. 2 (light novel)"; + let n = normalize_title(s); + // Expect articles removed, ampersand -> and, volume token removed and lowercased + assert!(n.starts_with("amazing and strange")); +} + +#[test] +fn test_clean_name_initials_and_case() { + let mut name = "JRR TOLKIEN".to_string(); + clean_name(&mut name).unwrap(); + // JRR should remain as-is (algorithm doesn't split 3-letter initials); TOLKIEN should become Title case + assert!(name.starts_with("JRR")); + assert!(name.contains("Tolkien")); + + let mut name2 = "john doe".to_string(); + clean_name(&mut name2).unwrap(); + // short lowercase words should be capitalized at start + assert!(name2.contains("John")); +} diff --git a/server/src/config_impl.rs b/server/src/config_impl.rs index 3dd2abb3..b09cb514 100644 --- a/server/src/config_impl.rs +++ b/server/src/config_impl.rs @@ -375,25 +375,21 @@ impl GoodreadsList { } pub fn allow_audio(&self) -> bool { - self.grab.iter().any(|g| { - g.filter - .edition - .categories - .audio - .as_ref() - .is_none_or(|c| !c.is_empty()) - }) + self.grab + .iter() + .any(|g| match g.edition.categories.audio.as_ref() { + None => true, + Some(c) => !c.is_empty(), + }) } pub fn allow_ebook(&self) -> bool { - self.grab.iter().any(|g| { - g.filter - .edition - .categories - .ebook - .as_ref() - .is_none_or(|c| !c.is_empty()) - }) + self.grab + .iter() + .any(|g| match g.edition.categories.ebook.as_ref() { + None => true, + Some(c) => !c.is_empty(), + }) } } diff --git a/server/src/linker/common.rs b/server/src/linker/common.rs index 296257a2..e5cd4fec 100644 --- a/server/src/linker/common.rs +++ b/server/src/linker/common.rs @@ -221,6 +221,58 @@ mod tests { assert_eq!(sel2.unwrap(), ".m4b".to_string()); } + #[test] + fn test_select_format_leading_dot_in_override() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "track.FLAC".to_string() }]; + let wanted = vec!["mp3".to_string(), "flac".to_string()]; + // override contains leading dot + let sel = select_format(&Some(vec![".flac".to_string()]), &wanted, &files); + assert_eq!(sel.unwrap(), ".flac".to_string()); + } + + #[test] + fn test_select_format_uppercase_extension() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "ALBUM.MP3".to_string() }]; + let wanted = vec!["mp3".to_string()]; + let sel = select_format(&None, &wanted, &files); + assert_eq!(sel.unwrap(), ".mp3".to_string()); + } + + #[test] + fn test_select_format_missing_extension_returns_none() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "README".to_string() }]; + let wanted = vec!["m4b".to_string()]; + let sel = select_format(&None, &wanted, &files); + assert!(sel.is_none()); + } + + #[test] + fn test_select_format_overridden_empty_vector() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "song.mp3".to_string() }]; + let wanted = vec!["mp3".to_string()]; + // override provided but empty -> should produce no selection + let sel = select_format(&Some(vec![]), &wanted, &files); + assert!(sel.is_none()); + } + + #[test] + fn test_select_format_wanted_empty_then_none() { + struct F { name: String } + impl HasFileName for F { fn name_lower(&self) -> String { self.name.to_lowercase() } } + let files = vec![F { name: "file.mp3".to_string() }]; + let wanted: Vec = vec![]; + let sel = select_format(&None, &wanted, &files); + assert!(sel.is_none()); + } + #[test] fn test_file_size_and_copy_and_hardlink() { use std::fs; diff --git a/server/tests/config_impl.rs b/server/tests/config_impl.rs new file mode 100644 index 00000000..f88a7bea --- /dev/null +++ b/server/tests/config_impl.rs @@ -0,0 +1,170 @@ +use mlm::config::{Cost, EditionFilter, GoodreadsList, Grab, TorrentFilter}; +use mlm_db::AudiobookCategory; +use mlm_mam::enums::Categories; + +#[test] +fn test_list_id_with_shelf_in_query() { + let list = GoodreadsList { + url: "https://goodreads.com/user/12345?foo=bar&shelf=to-read".to_string(), + name: None, + prefer_format: None, + grab: vec![], + search_interval: None, + unsat_buffer: None, + wedge_buffer: None, + dry_run: false, + }; + + let id = list.list_id().expect("should parse list id"); + assert_eq!(id, "12345:to-read"); +} + +#[test] +fn test_list_id_without_shelf() { + let list = GoodreadsList { + url: "https://goodreads.com/user/67890".to_string(), + name: None, + prefer_format: None, + grab: vec![], + search_interval: None, + unsat_buffer: None, + wedge_buffer: None, + dry_run: false, + }; + + let id = list.list_id().expect("should parse list id"); + assert_eq!(id, "67890:"); +} + +#[test] +fn test_list_id_shelf_first_param() { + let list = GoodreadsList { + url: "https://goodreads.com/user/abcde?shelf=owned&x=1".to_string(), + name: None, + prefer_format: None, + grab: vec![], + search_interval: None, + unsat_buffer: None, + wedge_buffer: None, + dry_run: false, + }; + + let id = list.list_id().expect("should parse list id"); + assert_eq!(id, "abcde:owned"); +} + +#[test] +fn test_allow_audio_true_when_none_or_nonempty() { + // audio = None -> allowed + let g1 = GoodreadsList { + url: "https://goodreads.com/user/1".to_string(), + name: None, + prefer_format: None, + grab: vec![Grab { + cost: Cost::default(), + filter: TorrentFilter::default(), + edition: EditionFilter::default(), + }], + search_interval: None, + unsat_buffer: None, + wedge_buffer: None, + dry_run: false, + }; + assert!(g1.allow_audio()); + + // audio = Some(non-empty) -> allowed + let cats = Categories { + audio: Some(vec![AudiobookCategory::GeneralFiction]), + ..Default::default() + }; + let g2 = GoodreadsList { + url: "https://goodreads.com/user/2".to_string(), + name: None, + prefer_format: None, + grab: vec![Grab { + cost: Cost::default(), + filter: TorrentFilter::default(), + edition: EditionFilter { + categories: cats, + ..Default::default() + }, + }], + search_interval: None, + unsat_buffer: None, + wedge_buffer: None, + dry_run: false, + }; + assert!(g2.allow_audio()); +} + +#[test] +fn test_allow_audio_false_when_all_grabs_empty_audio() { + let cats = Categories { + audio: Some(vec![]), + ..Default::default() + }; + + let list = GoodreadsList { + url: "https://goodreads.com/user/3".to_string(), + name: None, + prefer_format: None, + grab: vec![Grab { + cost: Cost::default(), + filter: TorrentFilter::default(), + edition: EditionFilter { + categories: cats, + ..Default::default() + }, + }], + search_interval: None, + unsat_buffer: None, + wedge_buffer: None, + dry_run: false, + }; + + assert!(!list.allow_audio()); +} + +#[test] +fn test_allow_ebook_behaviour() { + // ebook = None -> allowed + let g1 = GoodreadsList { + url: "https://goodreads.com/user/4".to_string(), + name: None, + prefer_format: None, + grab: vec![Grab { + cost: Cost::default(), + filter: TorrentFilter::default(), + edition: EditionFilter::default(), + }], + search_interval: None, + unsat_buffer: None, + wedge_buffer: None, + dry_run: false, + }; + assert!(g1.allow_ebook()); + + // ebook = Some(empty) -> disallowed + let cats = Categories { + ebook: Some(vec![]), + ..Default::default() + }; + let g2 = GoodreadsList { + url: "https://goodreads.com/user/5".to_string(), + name: None, + prefer_format: None, + grab: vec![Grab { + cost: Cost::default(), + filter: TorrentFilter::default(), + edition: EditionFilter { + categories: cats, + ..Default::default() + }, + }], + search_interval: None, + unsat_buffer: None, + wedge_buffer: None, + dry_run: false, + }; + assert!(!g2.allow_ebook()); +} From 8f84e0f2a4480fadb0aac4aee6eb8a936dd25ffc Mon Sep 17 00:00:00 2001 From: Stirling Mouse <181794392+StirlingMouse@users.noreply.github.com> Date: Sat, 7 Feb 2026 11:54:31 +0100 Subject: [PATCH 08/14] Add integration tests for link torrent --- mlm_db/src/v03.rs | 4 +- mlm_db/src/v09.rs | 8 +- mlm_db/src/v13.rs | 7 +- mlm_db/src/v18.rs | 17 +- server/src/cleaner.rs | 2 +- server/src/config.rs | 20 +- server/src/linker/torrent.rs | 1041 +++++++++++++---- server/src/main.rs | 2 +- server/src/qbittorrent.rs | 100 +- server/tests/common/mod.rs | 3 + .../{linker_test.rs => linker_folder_test.rs} | 0 server/tests/linker_torrent_test.rs | 642 ++++++++++ 12 files changed, 1617 insertions(+), 229 deletions(-) rename server/tests/{linker_test.rs => linker_folder_test.rs} (100%) create mode 100644 server/tests/linker_torrent_test.rs diff --git a/mlm_db/src/v03.rs b/mlm_db/src/v03.rs index 42b67abf..06828eb0 100644 --- a/mlm_db/src/v03.rs +++ b/mlm_db/src/v03.rs @@ -1,6 +1,6 @@ use super::{v01, v02, v04, v05, v06}; -use native_db::{Key, ToKey, native_db}; -use native_model::{Model, native_model}; +use native_db::{native_db, Key, ToKey}; +use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use time::{OffsetDateTime, UtcDateTime}; diff --git a/mlm_db/src/v09.rs b/mlm_db/src/v09.rs index d0726f28..65a6a9d9 100644 --- a/mlm_db/src/v09.rs +++ b/mlm_db/src/v09.rs @@ -1,6 +1,6 @@ use super::{v01, v03, v04, v06, v08, v10}; -use native_db::{ToKey, native_db}; -use native_model::{Model, native_model}; +use native_db::{native_db, ToKey}; +use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use tracing::warn; @@ -89,13 +89,13 @@ pub struct TorrentMeta { pub series: Vec, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct Series { pub name: String, pub entries: SeriesEntries, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct SeriesEntries(pub Vec); impl SeriesEntries { diff --git a/mlm_db/src/v13.rs b/mlm_db/src/v13.rs index 0c7751f4..4d20a4ff 100644 --- a/mlm_db/src/v13.rs +++ b/mlm_db/src/v13.rs @@ -1,6 +1,6 @@ use super::{v03, v04, v06, v08, v09, v10, v11, v12, v14}; -use native_db::{ToKey, native_db}; -use native_model::{Model, native_model}; +use native_db::{native_db, ToKey}; +use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -100,8 +100,9 @@ pub struct TorrentMeta { pub source: v10::MetadataSource, } -#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum MediaType { + #[default] Audiobook, Ebook, Musicology, diff --git a/mlm_db/src/v18.rs b/mlm_db/src/v18.rs index dc2a1955..e92a0549 100644 --- a/mlm_db/src/v18.rs +++ b/mlm_db/src/v18.rs @@ -2,8 +2,8 @@ use crate::ids; use super::{v01, v03, v04, v05, v08, v09, v10, v11, v12, v13, v16, v17}; use mlm_parse::{normalize_title, parse_edition}; -use native_db::{ToKey, native_db}; -use native_model::{Model, native_model}; +use native_db::{native_db, ToKey}; +use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, path::PathBuf}; @@ -88,7 +88,7 @@ pub struct ErroredTorrent { pub created_at: v03::Timestamp, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] pub struct TorrentMeta { pub ids: BTreeMap, pub vip_status: Option, @@ -112,15 +112,16 @@ pub struct TorrentMeta { pub uploaded_at: Option, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] pub enum MetadataSource { + #[default] Mam, Manual, File, Match, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[native_model(id = 6, version = 18, from = v17::Event)] #[native_db(export_keys = true)] pub struct Event { @@ -135,7 +136,7 @@ pub struct Event { pub event: EventType, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum EventType { Grabbed { grabber: Option, @@ -157,14 +158,14 @@ pub enum EventType { RemovedFromTracker, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct TorrentMetaDiff { pub field: TorrentMetaField, pub from: String, pub to: String, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum TorrentMetaField { Ids, Vip, diff --git a/server/src/cleaner.rs b/server/src/cleaner.rs index 3b7402c0..f4b1827a 100644 --- a/server/src/cleaner.rs +++ b/server/src/cleaner.rs @@ -171,7 +171,7 @@ pub async fn remove_library_files( } })?; if let Some(sub_dir) = file.parent() { - fs::remove_dir(sub_dir).ok(); + fs::remove_dir(library_path.join(sub_dir)).ok(); } } let mut remove_files = true; diff --git a/server/src/config.rs b/server/src/config.rs index 90991684..ddca8874 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -1,8 +1,8 @@ use std::{collections::BTreeMap, path::PathBuf}; use mlm_db::{ - Flags, Language, MediaType, OldDbMainCat, Size, impls::{parse, parse_opt, parse_vec}, + Flags, Language, MediaType, OldDbMainCat, Size, }; use mlm_mam::{ enums::{Categories, SearchIn, SnatchlistType}, @@ -11,7 +11,7 @@ use mlm_mam::{ use serde::{Deserialize, Serialize}; use time::Date; -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Config { pub mam_id: String, @@ -275,7 +275,7 @@ pub enum Cost { MetadataOnlyAdd, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, Default)] #[serde(deny_unknown_fields)] pub struct QbitConfig { pub url: String, @@ -289,7 +289,7 @@ pub struct QbitConfig { pub path_mapping: BTreeMap, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, Default)] #[serde(deny_unknown_fields)] pub struct QbitUpdate { pub category: Option, @@ -297,7 +297,7 @@ pub struct QbitUpdate { pub tags: Vec, } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(untagged)] #[allow(clippy::enum_variant_names)] pub enum Library { @@ -306,7 +306,7 @@ pub enum Library { ByCategory(LibraryByCategory), } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct LibraryByRipDir { pub rip_dir: PathBuf, @@ -316,7 +316,7 @@ pub struct LibraryByRipDir { pub filter: EditionFilter, } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct LibraryByDownloadDir { pub download_dir: PathBuf, @@ -326,7 +326,7 @@ pub struct LibraryByDownloadDir { pub tag_filters: LibraryTagFilters, } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct LibraryByCategory { pub category: String, @@ -336,7 +336,7 @@ pub struct LibraryByCategory { pub tag_filters: LibraryTagFilters, } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize, Default, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct LibraryTagFilters { #[serde(default)] @@ -345,7 +345,7 @@ pub struct LibraryTagFilters { pub deny_tags: Vec, } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct LibraryOptions { #[serde(default)] diff --git a/server/src/linker/torrent.rs b/server/src/linker/torrent.rs index 82b89671..acae36a9 100644 --- a/server/src/linker/torrent.rs +++ b/server/src/linker/torrent.rs @@ -5,7 +5,7 @@ use std::{ io::{BufWriter, Write}, mem, ops::Deref, - path::{Component, PathBuf}, + path::{Component, Path, PathBuf}, sync::Arc, }; @@ -23,6 +23,39 @@ use qbit::{ models::{Torrent as QbitTorrent, TorrentContent}, parameters::TorrentListParams, }; + +#[allow(async_fn_in_trait)] +pub trait MaMApi: Send + Sync { + async fn get_torrent_info(&self, hash: &str) -> Result>; + async fn get_torrent_info_by_id(&self, id: u64) -> Result>; +} + +impl MaMApi for MaM<'_> { + async fn get_torrent_info(&self, hash: &str) -> Result> { + self.get_torrent_info(hash).await + } + async fn get_torrent_info_by_id(&self, id: u64) -> Result> { + self.get_torrent_info_by_id(id).await + } +} + +impl MaMApi for &T { + async fn get_torrent_info(&self, hash: &str) -> Result> { + (**self).get_torrent_info(hash).await + } + async fn get_torrent_info_by_id(&self, id: u64) -> Result> { + (**self).get_torrent_info_by_id(id).await + } +} + +impl MaMApi for Arc { + async fn get_torrent_info(&self, hash: &str) -> Result> { + (**self).get_torrent_info(hash).await + } + async fn get_torrent_info_by_id(&self, id: u64) -> Result> { + (**self).get_torrent_info_by_id(id).await + } +} use regex::Regex; use tokio::fs::create_dir_all; use tracing::{Level, debug, instrument, span, trace}; @@ -37,19 +70,197 @@ use crate::{ library_dir, map_path, }, logging::{TorrentMetaError, update_errored_torrent, write_event}, - qbittorrent::ensure_category_exists, + qbittorrent::{QbitApi, ensure_category_exists}, }; pub static DISK_PATTERN: Lazy = Lazy::new(|| Regex::new(r"(?:CD|Disc|Disk)\s*(\d+)").unwrap()); +/// Calculates the relative path for a file within the library book directory. +/// Handles `Disc X` subdirectories. +pub fn calculate_library_file_path(torrent_file_path: &str) -> PathBuf { + let torrent_path = PathBuf::from(torrent_file_path); + // Split the path into components, find the file name (last component) + // and search the ancestor directories from nearest to farthest for a + // "Disc/CD/Disk" pattern. If found, return "Disc N/", + // otherwise return just the file name. + let mut components: Vec = torrent_path.components().collect(); + let file_name_component = components + .pop() + .expect("torrent file path should not be empty"); + + // Search ancestors in reverse (nearest directory first) + let mut disc_dir: Option = None; + for comp in components.iter().rev() { + if let Component::Normal(os) = comp { + let s = os.to_string_lossy(); + if let Some(caps) = DISK_PATTERN.captures(&s) + && let Some(disc) = caps.get(1) + { + disc_dir = Some(format!("Disc {}", disc.as_str())); + break; + } + } + } + + if let Some(dir_name) = disc_dir { + PathBuf::from(dir_name).join(file_name_component.as_os_str()) + } else { + PathBuf::from(file_name_component.as_os_str()) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct FileLinkPlan { + pub download_path: PathBuf, + pub library_path: PathBuf, + pub relative_library_path: PathBuf, +} + +pub fn calculate_link_plans( + qbit_config: &QbitConfig, + torrent: &QbitTorrent, + files: &[TorrentContent], + selected_audio_format: Option<&str>, + selected_ebook_format: Option<&str>, + library_dir: &Path, +) -> Vec { + let mut plans = vec![]; + for file in files { + if !(selected_audio_format + .as_ref() + .is_some_and(|ext| file.name.ends_with(*ext)) + || selected_ebook_format + .as_ref() + .is_some_and(|ext| file.name.ends_with(*ext))) + { + continue; + } + + let file_path = calculate_library_file_path(&file.name); + let library_path = library_dir.join(&file_path); + let download_path = + map_path(&qbit_config.path_mapping, &torrent.save_path).join(&file.name); + + plans.push(FileLinkPlan { + download_path, + library_path, + relative_library_path: file_path, + }); + } + plans +} + +pub struct TorrentUpdate { + pub changed: bool, + pub events: Vec, +} + +pub fn check_torrent_updates( + torrent: &mut Torrent, + qbit_torrent: &QbitTorrent, + library: Option<&Library>, + config: &Config, + trackers: &[qbit::models::Tracker], +) -> TorrentUpdate { + let mut changed = false; + let mut events = vec![]; + + let library_name = library.and_then(|l| l.options().name.as_ref()); + if torrent.linker.as_ref() != library_name { + torrent.linker = library_name.map(ToOwned::to_owned); + changed = true; + } + + let category = if qbit_torrent.category.is_empty() { + None + } else { + Some(qbit_torrent.category.as_str()) + }; + if torrent.category.as_deref() != category { + torrent.category = category.map(ToOwned::to_owned); + changed = true; + } + + if torrent.client_status.is_none() + && let Some(mam_tracker) = trackers.last() + && mam_tracker.msg == "torrent not registered with this tracker" + { + torrent.client_status = Some(ClientStatus::RemovedFromTracker); + changed = true; + events.push(Event::new( + Some(qbit_torrent.hash.clone()), + None, + EventType::RemovedFromTracker, + )); + } + + if let Some(library_path) = &torrent.library_path { + let Some(library) = library else { + if torrent.library_mismatch != Some(LibraryMismatch::NoLibrary) { + torrent.library_mismatch = Some(LibraryMismatch::NoLibrary); + changed = true; + } + return TorrentUpdate { changed, events }; + }; + + if !library_path.starts_with(&library.options().library_dir) { + let wanted = Some(LibraryMismatch::NewLibraryDir( + library.options().library_dir.clone(), + )); + if torrent.library_mismatch != wanted { + torrent.library_mismatch = wanted; + changed = true; + } + } else { + let dir = library_dir( + config.exclude_narrator_in_library_dir, + library, + &torrent.meta, + ); + let mut is_wrong = Some(library_path) != dir.as_ref(); + let wanted = match dir { + Some(dir) => Some(LibraryMismatch::NewPath(dir)), + None => Some(LibraryMismatch::NoLibrary), + }; + + if torrent.library_mismatch != wanted { + if is_wrong { + // Try another attempt at matching with exclude_narrator flipped + let dir_2 = library_dir( + !config.exclude_narrator_in_library_dir, + library, + &torrent.meta, + ); + if Some(library_path) == dir_2.as_ref() { + is_wrong = false + } + } + if is_wrong { + torrent.library_mismatch = wanted; + changed = true; + } else if torrent.library_mismatch.is_some() { + torrent.library_mismatch = None; + changed = true; + } + } + } + } + + TorrentUpdate { changed, events } +} + #[instrument(skip_all)] -pub async fn link_torrents_to_library( +pub async fn link_torrents_to_library( config: Arc, db: Arc>, - qbit: (&QbitConfig, &qbit::Api), - mam: Arc>, -) -> Result<()> { + qbit: (&QbitConfig, &Q), + mam: &M, +) -> Result<()> +where + Q: QbitApi + ?Sized, + M: MaMApi + ?Sized, +{ let torrents = qbit .1 .torrents(Some(TorrentListParams::default())) @@ -79,108 +290,28 @@ pub async fn link_torrents_to_library( } } if let Some(t) = &mut existing_torrent { - let library_name = library.and_then(|l| l.options().name.as_ref()); - if t.linker.as_ref() != library_name { - let (_guard, rw) = db.rw_async().await?; - t.linker = library_name.map(ToOwned::to_owned); - rw.upsert(t.clone())?; - rw.commit()?; - } - let category = if torrent.category.is_empty() { - None + let trackers = if t.client_status.is_none() { + match qbit.1.trackers(&torrent.hash).await { + Ok(trackers) => trackers, + Err(err) => { + error!("Error getting trackers for torrent {}: {err}", torrent.hash); + continue; + } + } } else { - Some(torrent.category.as_str()) + vec![] }; - if t.category.as_deref() != category { + let update = check_torrent_updates(t, &torrent, library, &config, &trackers); + if update.changed { let (_guard, rw) = db.rw_async().await?; - t.category = category.map(ToOwned::to_owned); rw.upsert(t.clone())?; rw.commit()?; } - if t.client_status.is_none() { - let trackers = qbit.1.trackers(&torrent.hash).await?; - if let Some(mam_tracker) = trackers.last() - && mam_tracker.msg == "torrent not registered with this tracker" - { - { - let (_guard, rw) = db.rw_async().await?; - t.client_status = Some(ClientStatus::RemovedFromTracker); - rw.upsert(t.clone())?; - rw.commit()?; - } - write_event( - &db, - Event::new( - Some(torrent.hash.clone()), - None, - EventType::RemovedFromTracker, - ), - ) - .await; - } - } - if let Some(library_path) = &t.library_path { - let Some(library) = find_library(&config, &torrent) else { - if t.library_mismatch != Some(LibraryMismatch::NoLibrary) { - debug!("no library: {library_path:?}",); - t.library_mismatch = Some(LibraryMismatch::NoLibrary); - let (_guard, rw) = db.rw_async().await?; - rw.upsert(t.clone())?; - rw.commit()?; - } - continue; - }; - if !library_path.starts_with(&library.options().library_dir) { - let wanted = Some(LibraryMismatch::NewLibraryDir( - library.options().library_dir.clone(), - )); - if t.library_mismatch != wanted { - debug!( - "library differs: {library_path:?} != {:?}", - library.options().library_dir - ); - t.library_mismatch = wanted; - let (_guard, rw) = db.rw_async().await?; - rw.upsert(t.clone())?; - rw.commit()?; - } - } else { - let dir = library_dir(config.exclude_narrator_in_library_dir, library, &t.meta); - let mut is_wrong = Some(library_path) != dir.as_ref(); - let wanted = match dir { - Some(dir) => Some(LibraryMismatch::NewPath(dir)), - None => Some(LibraryMismatch::NoLibrary), - }; - - if t.library_mismatch != wanted { - if is_wrong { - // Try another attempt at matching with exclude_narrator flipped - let dir_2 = library_dir( - !config.exclude_narrator_in_library_dir, - library, - &t.meta, - ); - if Some(library_path) == dir_2.as_ref() { - is_wrong = false - } - } - if is_wrong { - debug!("path differs: {library_path:?} != {:?}", wanted); - t.library_mismatch = wanted; - let (_guard, rw) = db.rw_async().await?; - rw.upsert(t.clone())?; - rw.commit()?; - } else if t.library_mismatch.is_some() { - t.library_mismatch = None; - let (_guard, rw) = db.rw_async().await?; - rw.upsert(t.clone())?; - rw.commit()?; - } - } - } - continue; + for event in update.events { + write_event(&db, event).await; } - if t.replaced_with.is_some() { + + if t.library_path.is_some() || t.replaced_with.is_some() { continue; } } @@ -201,7 +332,7 @@ pub async fn link_torrents_to_library( config.clone(), db.clone(), qbit, - mam.clone(), + mam, &torrent.hash, &torrent, library, @@ -221,18 +352,46 @@ pub async fn link_torrents_to_library( Ok(()) } +pub async fn handle_invalid_torrent( + qbit: (&QbitConfig, &Q), + on_invalid_torrent: &crate::config::QbitUpdate, + hash: &str, +) -> Result<()> +where + Q: QbitApi + ?Sized, +{ + if let Some(category) = &on_invalid_torrent.category { + ensure_category_exists(qbit.1, &qbit.0.url, category).await?; + qbit.1.set_category(Some(vec![hash]), category).await?; + } + + if !on_invalid_torrent.tags.is_empty() { + qbit.1 + .add_tags( + Some(vec![hash]), + on_invalid_torrent.tags.iter().map(Deref::deref).collect(), + ) + .await?; + } + Ok(()) +} + #[instrument(skip_all)] #[allow(clippy::too_many_arguments)] -async fn match_torrent( +async fn match_torrent( config: Arc, db: Arc>, - qbit: (&QbitConfig, &qbit::Api), - mam: Arc>, + qbit: (&QbitConfig, &Q), + mam: &M, hash: &str, torrent: &QbitTorrent, library: &Library, mut existing_torrent: Option, -) -> Result<()> { +) -> Result<()> +where + Q: QbitApi + ?Sized, + M: MaMApi + ?Sized, +{ let files = qbit.1.files(hash, None).await?; let selected_audio_format = select_format(&library.options().audio_types, &config.audio_types, &files); @@ -243,6 +402,9 @@ async fn match_torrent( bail!("Could not find any wanted formats in torrent"); } let Some(mam_torrent) = mam.get_torrent_info(hash).await.context("get_mam_info")? else { + if let Some(on_invalid_torrent) = &qbit.0.on_invalid_torrent { + handle_invalid_torrent(qbit, on_invalid_torrent, hash).await?; + } bail!("Could not find torrent on mam"); }; if existing_torrent.is_none() @@ -263,27 +425,7 @@ async fn match_torrent( Err(err) => { if let MetaError::UnknownMediaType(_) = err { if let Some(on_invalid_torrent) = &qbit.0.on_invalid_torrent { - let qbit_url = qbit.0.url.clone(); - let qbit = qbit::Api::new_login_username_password( - &qbit.0.url, - &qbit.0.username, - &qbit.0.password, - ) - .await?; - - if let Some(category) = &on_invalid_torrent.category { - ensure_category_exists(&qbit, &qbit_url, category).await?; - qbit.set_category(Some(vec![&torrent.hash]), category) - .await?; - } - - if !on_invalid_torrent.tags.is_empty() { - qbit.add_tags( - Some(vec![&torrent.hash]), - on_invalid_torrent.tags.iter().map(Deref::deref).collect(), - ) - .await?; - } + handle_invalid_torrent(qbit, on_invalid_torrent, hash).await?; } trace!("qbit updated"); } @@ -314,12 +456,15 @@ async fn match_torrent( } #[instrument(skip_all)] -pub async fn refresh_mam_metadata( +pub async fn refresh_mam_metadata( config: &Config, db: &Database<'_>, - mam: &MaM<'_>, + mam: &M, id: String, -) -> Result<(Torrent, MaMTorrent)> { +) -> Result<(Torrent, MaMTorrent)> +where + M: MaMApi + ?Sized, +{ let Some(mut torrent): Option = db.r_transaction()?.get().primary(id)? else { bail!("Could not find torrent id"); }; @@ -358,7 +503,6 @@ pub async fn refresh_mam_metadata( #[instrument(skip_all)] pub async fn relink(config: &Config, db: &Database<'_>, hash: String) -> Result<()> { - let mut torrent = None; for qbit_conf in &config.qbittorrent { let qbit = match qbit::Api::new_login_username_password( &qbit_conf.url, @@ -386,15 +530,26 @@ pub async fn relink(config: &Config, db: &Database<'_>, hash: String) -> Result< continue; } }; - let Some(t) = torrents.pop() else { + let Some(qbit_torrent) = torrents.pop() else { continue; }; - torrent.replace((qbit_conf, qbit, t)); - break; + return relink_internal(config, qbit_conf, db, &qbit, qbit_torrent, hash).await; } - let Some((qbit_conf, qbit, qbit_torrent)) = torrent else { - bail!("Could not find torrent in qbit"); - }; + bail!("Could not find torrent in qbit"); +} + +#[instrument(skip_all)] +pub async fn relink_internal( + config: &Config, + qbit_config: &QbitConfig, + db: &Database<'_>, + qbit: &Q, + qbit_torrent: QbitTorrent, + hash: String, +) -> Result<()> +where + Q: QbitApi + ?Sized, +{ let Some(library) = find_library(config, &qbit_torrent) else { bail!("Could not find matching library for torrent"); }; @@ -419,7 +574,7 @@ pub async fn relink(config: &Config, db: &Database<'_>, hash: String) -> Result< remove_library_files(config, &torrent, library_path_changed).await?; link_torrent( config, - qbit_conf, + qbit_config, db, &hash, &qbit_torrent, @@ -436,13 +591,15 @@ pub async fn relink(config: &Config, db: &Database<'_>, hash: String) -> Result< } #[instrument(skip_all)] -pub async fn refresh_metadata_relink( +pub async fn refresh_metadata_relink( config: &Config, db: &Database<'_>, - mam: &MaM<'_>, + mam: &M, hash: String, -) -> Result<()> { - let mut torrent = None; +) -> Result<()> +where + M: MaMApi + ?Sized, +{ for qbit_conf in &config.qbittorrent { let qbit = match qbit::Api::new_login_username_password( &qbit_conf.url, @@ -470,15 +627,37 @@ pub async fn refresh_metadata_relink( continue; } }; - let Some(t) = torrents.pop() else { + let Some(qbit_torrent) = torrents.pop() else { continue; }; - torrent.replace((qbit_conf, qbit, t)); - break; + return refresh_metadata_relink_internal( + config, + qbit_conf, + db, + &qbit, + mam, + qbit_torrent, + hash, + ) + .await; } - let Some((qbit_conf, qbit, qbit_torrent)) = torrent else { - bail!("Could not find torrent in qbit"); - }; + bail!("Could not find torrent in qbit"); +} + +#[instrument(skip_all)] +pub async fn refresh_metadata_relink_internal( + config: &Config, + qbit_config: &QbitConfig, + db: &Database<'_>, + qbit: &Q, + mam: &M, + qbit_torrent: QbitTorrent, + hash: String, +) -> Result<()> +where + Q: QbitApi + ?Sized, + M: MaMApi + ?Sized, +{ let Some(library) = find_library(config, &qbit_torrent) else { bail!("Could not find matching library for torrent"); }; @@ -501,7 +680,7 @@ pub async fn refresh_metadata_relink( remove_library_files(config, &torrent, library_path_changed).await?; link_torrent( config, - qbit_conf, + qbit_config, db, &hash, &qbit_torrent, @@ -545,56 +724,42 @@ async fn link_torrent( let metadata = abs::create_metadata(meta); create_dir_all(&dir).await?; - for file in files { - let span = span!(Level::TRACE, "file: {:?}", file.name); + let plans = calculate_link_plans( + qbit_config, + torrent, + &files, + selected_audio_format.as_deref(), + selected_ebook_format.as_deref(), + &dir, + ); + + for plan in plans { + let span = span!(Level::TRACE, "file", file = ?plan.relative_library_path); let _s = span.enter(); - if !(selected_audio_format - .as_ref() - .is_some_and(|ext| file.name.ends_with(ext)) - || selected_ebook_format - .as_ref() - .is_some_and(|ext| file.name.ends_with(ext))) - { - debug!("Skiping \"{}\"", file.name); - continue; + if let Some(parent) = plan.library_path.parent() { + create_dir_all(parent).await?; } - let torrent_path = PathBuf::from(&file.name); - let mut path_components = torrent_path.components(); - let file_name = path_components.next_back().unwrap(); - let dir_name = path_components.next_back().and_then(|dir_name| { - if let Component::Normal(dir_name) = dir_name { - let dir_name = dir_name.to_string_lossy().to_string(); - if let Some(disc) = DISK_PATTERN.captures(&dir_name).and_then(|c| c.get(1)) { - return Some(format!("Disc {}", disc.as_str())); - } - } - None - }); - let file_path = if let Some(dir_name) = dir_name { - let sub_dir = PathBuf::from(dir_name); - create_dir_all(dir.join(&sub_dir)).await?; - sub_dir.join(file_name) - } else { - PathBuf::from(&file_name) - }; - let library_path = dir.join(&file_path); - library_files.push(file_path.clone()); - let download_path = - map_path(&qbit_config.path_mapping, &torrent.save_path).join(&file.name); + library_files.push(plan.relative_library_path.clone()); match library.options().method { - LibraryLinkMethod::Hardlink => { - hard_link(&download_path, &library_path, &file_path)? - } - LibraryLinkMethod::HardlinkOrCopy => { - hard_link(&download_path, &library_path, &file_path) - .or_else(|_| copy(&download_path, &library_path))? - } - LibraryLinkMethod::Copy => copy(&download_path, &library_path)?, - LibraryLinkMethod::HardlinkOrSymlink => { - hard_link(&download_path, &library_path, &file_path) - .or_else(|_| symlink(&download_path, &library_path))? - } - LibraryLinkMethod::Symlink => symlink(&download_path, &library_path)?, + LibraryLinkMethod::Hardlink => hard_link( + &plan.download_path, + &plan.library_path, + &plan.relative_library_path, + )?, + LibraryLinkMethod::HardlinkOrCopy => hard_link( + &plan.download_path, + &plan.library_path, + &plan.relative_library_path, + ) + .or_else(|_| copy(&plan.download_path, &plan.library_path))?, + LibraryLinkMethod::Copy => copy(&plan.download_path, &plan.library_path)?, + LibraryLinkMethod::HardlinkOrSymlink => hard_link( + &plan.download_path, + &plan.library_path, + &plan.relative_library_path, + ) + .or_else(|_| symlink(&plan.download_path, &plan.library_path))?, + LibraryLinkMethod::Symlink => symlink(&plan.download_path, &plan.library_path)?, LibraryLinkMethod::NoLink => {} }; } @@ -868,4 +1033,488 @@ mod tests { // ByRipDir is explicitly skipped by find_library so should return None assert!(lib.is_none()); } + + #[test] + fn test_calculate_library_file_path() { + assert_eq!( + calculate_library_file_path("Book Title/audio.m4b"), + PathBuf::from("audio.m4b") + ); + assert_eq!( + calculate_library_file_path("Book Title/CD 1/01.mp3"), + PathBuf::from("Disc 1/01.mp3") + ); + assert_eq!( + calculate_library_file_path("Book Title/Disc 2/05.mp3"), + PathBuf::from("Disc 2/05.mp3") + ); + assert_eq!( + calculate_library_file_path("Book Title/Disk 10/05.mp3"), + PathBuf::from("Disc 10/05.mp3") + ); + assert_eq!( + calculate_library_file_path("Book Title/CD1/01.mp3"), + PathBuf::from("Disc 1/01.mp3") + ); + assert_eq!( + calculate_library_file_path("audio.m4b"), + PathBuf::from("audio.m4b") + ); + assert_eq!( + calculate_library_file_path("Book Title/Some Other Folder/audio.m4b"), + PathBuf::from("audio.m4b") + ); + assert_eq!( + calculate_library_file_path("Book Title/Disc 1/Subfolder/01.mp3"), + PathBuf::from("Disc 1/01.mp3") + ); + } + + #[test] + fn test_calculate_library_file_path_filename_contains_disc() { + // Ensure a filename that contains the word "Disc" isn't treated as a + // disc directory. Only ancestor directory names should be considered. + assert_eq!( + calculate_library_file_path("Book Title/Some Folder/Disc 1 - track.mp3"), + PathBuf::from("Disc 1 - track.mp3") + ); + } + + #[test] + fn test_calculate_link_plans() { + use std::collections::BTreeMap; + let qbit_config = QbitConfig { + path_mapping: BTreeMap::from([( + PathBuf::from("/downloads"), + PathBuf::from("/data/downloads"), + )]), + url: "".to_string(), + username: "".to_string(), + password: "".to_string(), + on_cleaned: None, + on_invalid_torrent: None, + }; + let torrent = qbit::models::Torrent { + save_path: "/downloads".to_string(), + ..Default::default() + }; + let files = vec![ + TorrentContent { + name: "Audiobook/audio.m4b".to_string(), + ..Default::default() + }, + TorrentContent { + name: "Audiobook/cover.jpg".to_string(), + ..Default::default() + }, + ]; + let library_dir = PathBuf::from("/library"); + + let plans = calculate_link_plans( + &qbit_config, + &torrent, + &files, + Some(".m4b"), + None, + &library_dir, + ); + + assert_eq!(plans.len(), 1); + assert_eq!( + plans[0].download_path, + PathBuf::from("/data/downloads/Audiobook/audio.m4b") + ); + assert_eq!(plans[0].library_path, PathBuf::from("/library/audio.m4b")); + assert_eq!(plans[0].relative_library_path, PathBuf::from("audio.m4b")); + } + + #[test] + fn test_check_torrent_updates_category_change() { + use std::collections::BTreeMap; + let mut torrent = Torrent { + id: "1".to_string(), + id_is_hash: true, + mam_id: None, + library_path: None, + library_files: vec![], + linker: None, + category: Some("old".to_string()), + selected_audio_format: None, + selected_ebook_format: None, + title_search: "".to_string(), + meta: TorrentMeta { + ids: BTreeMap::new(), + vip_status: None, + cat: None, + media_type: mlm_db::MediaType::Audiobook, + main_cat: None, + categories: vec![], + tags: vec![], + language: None, + flags: None, + filetypes: vec![], + num_files: 0, + size: mlm_db::Size::from_bytes(0), + title: "".to_string(), + edition: None, + description: "".to_string(), + authors: vec![], + narrators: vec![], + series: vec![], + source: mlm_db::MetadataSource::Mam, + uploaded_at: mlm_db::Timestamp::now(), + }, + created_at: mlm_db::Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + }; + let qbit_torrent = qbit::models::Torrent { + category: "new".to_string(), + ..Default::default() + }; + let cfg = Config { + mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![], + }; + + let update = check_torrent_updates(&mut torrent, &qbit_torrent, None, &cfg, &[]); + assert!(update.changed); + assert_eq!(torrent.category, Some("new".to_string())); + } + + #[test] + fn test_check_torrent_updates_linker_change() { + let mut torrent = Torrent { + linker: Some("old_linker".to_string()), + ..mock_torrent() + }; + let qbit_torrent = qbit::models::Torrent::default(); + let library = Library::ByCategory(LibraryByCategory { + category: "audiobooks".to_string(), + options: LibraryOptions { + name: Some("new_linker".to_string()), + library_dir: PathBuf::from("/lib"), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + }); + let cfg = mock_config_with_library(library.clone()); + + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!(update.changed); + assert_eq!(torrent.linker, Some("new_linker".to_string())); + } + + #[test] + fn test_check_torrent_updates_removed_from_tracker() { + let mut torrent = mock_torrent(); + let qbit_torrent = qbit::models::Torrent { + hash: "hash".to_string(), + ..Default::default() + }; + let trackers = vec![qbit::models::Tracker { + msg: "torrent not registered with this tracker".to_string(), + ..Default::default() + }]; + let cfg = mock_config(); + + let update = check_torrent_updates(&mut torrent, &qbit_torrent, None, &cfg, &trackers); + assert!(update.changed); + assert_eq!( + torrent.client_status, + Some(ClientStatus::RemovedFromTracker) + ); + assert_eq!(update.events.len(), 1); + assert_eq!(update.events[0].event, EventType::RemovedFromTracker); + } + + #[test] + fn test_check_torrent_updates_library_mismatch() { + let mut torrent = mock_torrent(); + torrent.library_path = Some(PathBuf::from("/old_library/Author/Title")); + torrent.meta.authors = vec!["Author".to_string()]; + torrent.meta.title = "Title".to_string(); + + let library = Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: PathBuf::from("/downloads"), + options: LibraryOptions { + name: None, + library_dir: PathBuf::from("/new_library"), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + }); + let cfg = mock_config_with_library(library.clone()); + let qbit_torrent = qbit::models::Torrent::default(); + + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!(update.changed); + assert_eq!( + torrent.library_mismatch, + Some(LibraryMismatch::NewLibraryDir(PathBuf::from( + "/new_library" + ))) + ); + + // Test NewPath (same library dir, different author/title logic) + torrent.library_mismatch = None; + torrent.library_path = Some(PathBuf::from("/new_library/OldAuthor/Title")); + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!(update.changed); + if let Some(LibraryMismatch::NewPath(p)) = &torrent.library_mismatch { + assert!(p.ends_with("Author/Title")); + } else { + panic!( + "Expected NewPath mismatch, got {:?}", + torrent.library_mismatch + ); + } + } + + #[test] + fn test_check_torrent_updates_exclude_narrator() { + let mut torrent = mock_torrent(); + torrent.meta.authors = vec!["Author".to_string()]; + torrent.meta.narrators = vec!["Narrator".to_string()]; + torrent.meta.title = "Title".to_string(); + + // Path with narrator + let path_with_narrator = PathBuf::from("/library/Author/Title {Narrator}"); + // Path without narrator + let path_without_narrator = PathBuf::from("/library/Author/Title"); + + let library = Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: PathBuf::from("/downloads"), + options: LibraryOptions { + name: None, + library_dir: PathBuf::from("/library"), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + }); + + // Config: exclude_narrator = true + let mut cfg = mock_config_with_library(library.clone()); + cfg.exclude_narrator_in_library_dir = true; + + let qbit_torrent = qbit::models::Torrent::default(); + + // If current path matches "with narrator" but config says exclude, it might be okay if it was deliberate or it might trigger mismatch. + // The code has this logic: + /* + let dir = library_dir(config.exclude_narrator_in_library_dir, library, &torrent.meta); + let mut is_wrong = Some(library_path) != dir.as_ref(); + ... + if is_wrong { + // Try another attempt at matching with exclude_narrator flipped + let dir_2 = library_dir(!config.exclude_narrator_in_library_dir, library, &torrent.meta); + if Some(library_path) == dir_2.as_ref() { + is_wrong = false + } + } + */ + // This means it accepts BOTH paths as "not wrong" if they match either state of the toggle, UNLESS it's already in mismatch state? + // Wait, if it matches dir_2, is_wrong becomes false, so it WON'T set library_mismatch. + + torrent.library_path = Some(path_with_narrator.clone()); + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!( + !update.changed, + "Should accept existing path with narrator even if config says exclude" + ); + + torrent.library_path = Some(path_without_narrator.clone()); + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!( + !update.changed, + "Should accept existing path without narrator" + ); + + // What if it matches neither? + torrent.library_path = Some(PathBuf::from("/library/Wrong/Path")); + let update = check_torrent_updates(&mut torrent, &qbit_torrent, Some(&library), &cfg, &[]); + assert!(update.changed); + assert!(matches!( + torrent.library_mismatch, + Some(LibraryMismatch::NewPath(_)) + )); + } + + #[tokio::test] + async fn test_handle_invalid_torrent() { + struct MockQbit { + category: std::sync::Mutex>, + tags: std::sync::Mutex>, + } + impl QbitApi for MockQbit { + async fn torrents( + &self, + _: Option, + ) -> Result> { + Ok(vec![]) + } + async fn trackers(&self, _: &str) -> Result> { + Ok(vec![]) + } + async fn files( + &self, + _: &str, + _: Option>, + ) -> Result> { + Ok(vec![]) + } + async fn set_category(&self, _: Option>, category: &str) -> Result<()> { + *self.category.lock().unwrap() = Some(category.to_string()); + Ok(()) + } + async fn add_tags(&self, _: Option>, tags: Vec<&str>) -> Result<()> { + self.tags + .lock() + .unwrap() + .extend(tags.iter().map(|t| t.to_string())); + Ok(()) + } + async fn create_category(&self, _: &str, _: &str) -> Result<()> { + Ok(()) + } + async fn categories( + &self, + ) -> Result> { + Ok(std::collections::HashMap::new()) + } + } + + let qbit = MockQbit { + category: std::sync::Mutex::new(None), + tags: std::sync::Mutex::new(vec![]), + }; + + let qbit_conf = QbitConfig { + url: "http://localhost:8080".to_string(), + ..Default::default() + }; + + let update = crate::config::QbitUpdate { + category: Some("invalid".to_string()), + tags: vec!["tag1".to_string(), "tag2".to_string()], + }; + + handle_invalid_torrent((&qbit_conf, &qbit), &update, "hash") + .await + .unwrap(); + + assert_eq!(*qbit.category.lock().unwrap(), Some("invalid".to_string())); + assert_eq!( + *qbit.tags.lock().unwrap(), + vec!["tag1".to_string(), "tag2".to_string()] + ); + } + + fn mock_torrent() -> Torrent { + use std::collections::BTreeMap; + Torrent { + id: "1".to_string(), + id_is_hash: true, + mam_id: None, + library_path: None, + library_files: vec![], + linker: None, + category: None, + selected_audio_format: None, + selected_ebook_format: None, + title_search: "".to_string(), + meta: TorrentMeta { + ids: BTreeMap::new(), + vip_status: None, + cat: None, + media_type: mlm_db::MediaType::Audiobook, + main_cat: None, + categories: vec![], + tags: vec![], + language: None, + flags: None, + filetypes: vec![], + num_files: 0, + size: mlm_db::Size::from_bytes(0), + title: "".to_string(), + edition: None, + description: "".to_string(), + authors: vec![], + narrators: vec![], + series: vec![], + source: mlm_db::MetadataSource::Mam, + uploaded_at: mlm_db::Timestamp::now(), + }, + created_at: mlm_db::Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + } + } + + fn mock_config() -> Config { + Config { + mam_id: "m".to_string(), + web_host: "".to_string(), + web_port: 0, + min_ratio: 0.0, + unsat_buffer: 0, + wedge_buffer: 0, + add_torrents_stopped: false, + exclude_narrator_in_library_dir: false, + search_interval: 0, + link_interval: 0, + import_interval: 0, + ignore_torrents: vec![], + audio_types: vec![], + ebook_types: vec![], + music_types: vec![], + radio_types: vec![], + search: crate::config::SearchConfig::default(), + audiobookshelf: None, + autograbs: vec![], + snatchlist: vec![], + goodreads_lists: vec![], + notion_lists: vec![], + tags: vec![], + qbittorrent: vec![], + libraries: vec![], + } + } + + fn mock_config_with_library(library: Library) -> Config { + let mut cfg = mock_config(); + cfg.libraries.push(library); + cfg + } } diff --git a/server/src/main.rs b/server/src/main.rs index 1b40f3ad..75020420 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -551,7 +551,7 @@ async fn app_main() -> Result<()> { config.clone(), db.clone(), (&qbit_conf, &qbit), - mam.clone(), + &mam, ) .await .context("link_torrents_to_library"); diff --git a/server/src/qbittorrent.rs b/server/src/qbittorrent.rs index bb3808ad..ea03d17e 100644 --- a/server/src/qbittorrent.rs +++ b/server/src/qbittorrent.rs @@ -5,13 +5,100 @@ use std::time::{Duration, Instant}; use anyhow::Result; use once_cell::sync::Lazy; use qbit::{ - models::Torrent, + models::{Torrent, TorrentContent}, parameters::{AddTorrent, TorrentListParams}, }; use tokio::sync::RwLock; use crate::config::{Config, QbitConfig}; +#[allow(async_fn_in_trait)] +pub trait QbitApi: Send + Sync { + async fn torrents(&self, params: Option) -> Result>; + async fn trackers(&self, hash: &str) -> Result>; + async fn files(&self, hash: &str, params: Option>) -> Result>; + async fn set_category(&self, hashes: Option>, category: &str) -> Result<()>; + async fn add_tags(&self, hashes: Option>, tags: Vec<&str>) -> Result<()>; + async fn create_category(&self, category: &str, save_path: &str) -> Result<()>; + async fn categories(&self) -> Result>; +} + +impl QbitApi for qbit::Api { + async fn torrents(&self, params: Option) -> Result> { + self.torrents(params).await.map_err(Into::into) + } + async fn trackers(&self, hash: &str) -> Result> { + self.trackers(hash).await.map_err(Into::into) + } + async fn files(&self, hash: &str, params: Option>) -> Result> { + self.files(hash, params).await.map_err(Into::into) + } + async fn set_category(&self, hashes: Option>, category: &str) -> Result<()> { + self.set_category(hashes, category) + .await + .map_err(Into::into) + } + async fn add_tags(&self, hashes: Option>, tags: Vec<&str>) -> Result<()> { + self.add_tags(hashes, tags).await.map_err(Into::into) + } + async fn create_category(&self, category: &str, save_path: &str) -> Result<()> { + self.create_category(category, save_path) + .await + .map_err(Into::into) + } + async fn categories(&self) -> Result> { + self.categories().await.map_err(Into::into) + } +} + +impl QbitApi for &T { + async fn torrents(&self, params: Option) -> Result> { + (**self).torrents(params).await + } + async fn trackers(&self, hash: &str) -> Result> { + (**self).trackers(hash).await + } + async fn files(&self, hash: &str, params: Option>) -> Result> { + (**self).files(hash, params).await + } + async fn set_category(&self, hashes: Option>, category: &str) -> Result<()> { + (**self).set_category(hashes, category).await + } + async fn add_tags(&self, hashes: Option>, tags: Vec<&str>) -> Result<()> { + (**self).add_tags(hashes, tags).await + } + async fn create_category(&self, category: &str, save_path: &str) -> Result<()> { + (**self).create_category(category, save_path).await + } + async fn categories(&self) -> Result> { + (**self).categories().await + } +} + +impl QbitApi for Arc { + async fn torrents(&self, params: Option) -> Result> { + (**self).torrents(params).await + } + async fn trackers(&self, hash: &str) -> Result> { + (**self).trackers(hash).await + } + async fn files(&self, hash: &str, params: Option>) -> Result> { + (**self).files(hash, params).await + } + async fn set_category(&self, hashes: Option>, category: &str) -> Result<()> { + (**self).set_category(hashes, category).await + } + async fn add_tags(&self, hashes: Option>, tags: Vec<&str>) -> Result<()> { + (**self).add_tags(hashes, tags).await + } + async fn create_category(&self, category: &str, save_path: &str) -> Result<()> { + (**self).create_category(category, save_path).await + } + async fn categories(&self) -> Result> { + (**self).categories().await + } +} + const CATEGORY_CACHE_TTL_SECS: u64 = 60; #[derive(Clone)] @@ -26,7 +113,12 @@ impl CategoryCache { } } - async fn get_or_fetch(&self, qbit: &qbit::Api, url: &str) -> Result> { + async fn get_or_fetch( + &self, + qbit: &Q, + url: &str, + ) -> Result> { + let now = Instant::now(); let cache = self.cache.read().await; @@ -58,8 +150,8 @@ impl Default for CategoryCache { static CATEGORY_CACHE: Lazy = Lazy::new(CategoryCache::new); -pub async fn ensure_category_exists( - qbit: &qbit::Api, +pub async fn ensure_category_exists( + qbit: &Q, url: &str, category: &str, ) -> Result<()> { diff --git a/server/tests/common/mod.rs b/server/tests/common/mod.rs index f36a564f..69f69e62 100644 --- a/server/tests/common/mod.rs +++ b/server/tests/common/mod.rs @@ -28,10 +28,12 @@ impl TestDb { } } +#[allow(dead_code)] pub struct MockTorrentBuilder { torrent: Torrent, } +#[allow(dead_code)] impl MockTorrentBuilder { pub fn new(id: &str, title: &str) -> Self { Self { @@ -131,6 +133,7 @@ impl MockFs { }) } + #[allow(dead_code)] pub fn create_libation_folder( &self, asin: &str, diff --git a/server/tests/linker_test.rs b/server/tests/linker_folder_test.rs similarity index 100% rename from server/tests/linker_test.rs rename to server/tests/linker_folder_test.rs diff --git a/server/tests/linker_torrent_test.rs b/server/tests/linker_torrent_test.rs new file mode 100644 index 00000000..eb053760 --- /dev/null +++ b/server/tests/linker_torrent_test.rs @@ -0,0 +1,642 @@ +mod common; + +use anyhow::Result; +use common::{MockFs, TestDb, mock_config}; +use mlm_db::DatabaseExt as _; +use mlm::config::{ + Library, LibraryByDownloadDir, LibraryLinkMethod, LibraryOptions, LibraryTagFilters, QbitConfig, +}; +use mlm::linker::torrent::{MaMApi, link_torrents_to_library}; +use mlm::qbittorrent::QbitApi; +use mlm_mam::search::MaMTorrent; +use qbit::models::{Torrent as QbitTorrent, TorrentContent, Tracker}; +use qbit::parameters::TorrentListParams; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +struct MockQbit { + torrents: Vec, + files: HashMap>, +} + +impl QbitApi for MockQbit { + async fn torrents(&self, _params: Option) -> Result> { + Ok(self.torrents.clone()) + } + async fn trackers(&self, _hash: &str) -> Result> { + Ok(vec![]) + } + async fn files(&self, hash: &str, _params: Option>) -> Result> { + Ok(self.files.get(hash).cloned().unwrap_or_default()) + } + async fn set_category(&self, _hashes: Option>, _category: &str) -> Result<()> { + Ok(()) + } + async fn add_tags(&self, _hashes: Option>, _tags: Vec<&str>) -> Result<()> { + Ok(()) + } + async fn create_category(&self, _category: &str, _save_path: &str) -> Result<()> { + Ok(()) + } + async fn categories(&self) -> Result> { + Ok(HashMap::new()) + } +} + +struct MockMaM { + torrents: HashMap, +} + +impl MaMApi for MockMaM { + async fn get_torrent_info(&self, hash: &str) -> Result> { + Ok(self.torrents.get(hash).cloned()) + } + async fn get_torrent_info_by_id(&self, id: u64) -> Result> { + Ok(self.torrents.values().find(|t| t.id == id).cloned()) + } +} + +fn mock_meta(title: &str, author: &str) -> mlm_db::TorrentMeta { + mlm_db::TorrentMeta { + ids: BTreeMap::new(), + vip_status: None, + cat: None, + media_type: mlm_db::MediaType::Audiobook, + main_cat: None, + categories: vec![], + tags: vec![], + language: None, + flags: None, + filetypes: vec![], + num_files: 0, + size: mlm_db::Size::from_bytes(0), + title: title.to_string(), + edition: None, + description: "".to_string(), + authors: vec![author.to_string()], + narrators: vec![], + series: vec![], + source: mlm_db::MetadataSource::Mam, + uploaded_at: mlm_db::Timestamp::now(), + } +} + +#[tokio::test] +async fn test_link_torrent_audiobook() -> anyhow::Result<()> { + // Setup test DB and filesystem + let db = TestDb::new()?; + let fs = MockFs::new()?; + + // Create a mock torrent directory in rip folder + let torrent_hash = "1234567890abcdef1234567890abcdef12345678"; + let torrent_name = "Test Audiobook"; + let torrent_dir = fs.rip_dir.join(torrent_name); + std::fs::create_dir_all(&torrent_dir)?; + std::fs::write(torrent_dir.join("audio.m4b"), "fake audio content")?; + + let mut config = mock_config(fs.rip_dir.clone(), fs.library_dir.clone()); + config.qbittorrent.push(QbitConfig { + url: "http://localhost:8080".to_string(), + username: "admin".to_string(), + password: "adminadmin".to_string(), + path_mapping: BTreeMap::new(), + on_cleaned: None, + on_invalid_torrent: None, + }); + config.libraries = vec![Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: fs.rip_dir.clone(), + options: LibraryOptions { + name: Some("test_library".to_string()), + library_dir: fs.library_dir.clone(), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + })]; + let config = Arc::new(config); + + // Setup mock Qbit + let qbit_torrent = QbitTorrent { + hash: torrent_hash.to_string(), + name: torrent_name.to_string(), + save_path: fs.rip_dir.to_string_lossy().to_string(), + progress: 1.0, + ..Default::default() + }; + let qbit_content = TorrentContent { + name: format!("{}/audio.m4b", torrent_name), + size: 100, + ..Default::default() + }; + let mock_qbit = MockQbit { + torrents: vec![qbit_torrent], + files: HashMap::from([(torrent_hash.to_string(), vec![qbit_content])]), + }; + + // Setup mock MaM + let mut mam_torrent = MaMTorrent { + id: 1, + title: "Test Title".to_string(), + added: "2024-01-01 12:00:00".to_string(), + size: "100 B".to_string(), + mediatype: 1, // Audiobook + maincat: 1, // Fiction + catname: "General Fiction".to_string(), + category: 42, // General Fiction in AudiobookCategory + language: 1, // English + lang_code: "en".to_string(), + numfiles: 1, + filetype: "m4b".to_string(), + ..Default::default() + }; + mam_torrent.author_info.insert(1, "Test Author".to_string()); + + let mock_mam = MockMaM { + torrents: HashMap::from([(torrent_hash.to_string(), mam_torrent)]), + }; + + // Run the linker + let qbit_config = config.qbittorrent.first().unwrap(); + link_torrents_to_library( + config.clone(), + db.db.clone(), + (qbit_config, &mock_qbit), + &mock_mam, + ) + .await?; + + // Verify files in library + let library_book_dir = fs.library_dir.join("Test Author").join("Test Title"); + assert!( + library_book_dir.exists(), + "Library book directory should exist at {:?}", + library_book_dir + ); + assert!( + library_book_dir.join("audio.m4b").exists(), + "Audio file should exist in library" + ); + assert!( + library_book_dir.join("metadata.json").exists(), + "Metadata file should exist in library" + ); + + // Verify DB entry + let r = db.db.r_transaction()?; + let torrent: Option = r.get().primary(torrent_hash.to_string())?; + assert!(torrent.is_some(), "Torrent should be in DB"); + let torrent = torrent.unwrap(); + assert_eq!(torrent.meta.title, "Test Title"); + assert_eq!(torrent.meta.authors, vec!["Test Author"]); + + Ok(()) +} + +#[tokio::test] +async fn test_skip_incomplete_torrent() -> anyhow::Result<()> { + let db = TestDb::new()?; + let fs = MockFs::new()?; + let mut config = mock_config(fs.rip_dir.clone(), fs.library_dir.clone()); + config.qbittorrent.push(QbitConfig { + url: "http://localhost:8080".to_string(), + username: "admin".to_string(), + password: "adminadmin".to_string(), + path_mapping: BTreeMap::new(), + on_cleaned: None, + on_invalid_torrent: None, + }); + let config = Arc::new(config); + + let mock_qbit = MockQbit { + torrents: vec![QbitTorrent { + hash: "incomplete".to_string(), + progress: 0.5, + ..Default::default() + }], + files: HashMap::new(), + }; + let mock_mam = MockMaM { + torrents: HashMap::new(), + }; + + let qbit_config = config.qbittorrent.first().unwrap(); + link_torrents_to_library( + config.clone(), + db.db.clone(), + (qbit_config, &mock_qbit), + &mock_mam, + ) + .await?; + + let r = db.db.r_transaction()?; + let torrents: Vec = r + .scan() + .primary::()? + .all()? + .collect::, _>>()?; + assert!(torrents.is_empty(), "Incomplete torrent should be skipped"); + Ok(()) +} + +#[tokio::test] +async fn test_remove_selected_torrent() -> anyhow::Result<()> { + let db = TestDb::new()?; + let fs = MockFs::new()?; + + let torrent_hash = "1234567890abcdef1234567890abcdef12345678"; + + // Add SelectedTorrent to DB + { + let (_guard, rw) = db.db.rw_async().await?; + rw.insert(mlm_db::SelectedTorrent { + mam_id: 1, + hash: Some(torrent_hash.to_string()), + dl_link: "http://example.com/dl".to_string(), + unsat_buffer: None, + wedge_buffer: None, + cost: mlm_db::TorrentCost::GlobalFreeleech, + category: None, + tags: vec![], + title_search: "test title".to_string(), + meta: mlm_db::TorrentMeta { + ids: BTreeMap::from([(mlm_db::ids::MAM.to_string(), "1".to_string())]), + media_type: mlm_db::MediaType::Audiobook, + title: "Test Title".to_string(), + authors: vec!["Test Author".to_string()], + source: mlm_db::MetadataSource::Mam, + uploaded_at: Some(mlm_db::Timestamp::now()), + ..Default::default() + }, + grabber: None, + created_at: mlm_db::Timestamp::now(), + started_at: None, + removed_at: None, + })?; + rw.commit()?; + } + + let mut config = mock_config(fs.rip_dir.clone(), fs.library_dir.clone()); + config.qbittorrent.push(QbitConfig { + url: "".to_string(), + username: "".to_string(), + password: "".to_string(), + path_mapping: BTreeMap::new(), + on_cleaned: None, + on_invalid_torrent: None, + }); + config.libraries = vec![Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: fs.rip_dir.clone(), + options: LibraryOptions { + name: Some("test".to_string()), + library_dir: fs.library_dir.clone(), + method: LibraryLinkMethod::NoLink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + })]; + let config = Arc::new(config); + + let mock_qbit = MockQbit { + torrents: vec![QbitTorrent { + hash: torrent_hash.to_string(), + progress: 1.0, + save_path: fs.rip_dir.to_string_lossy().to_string(), + ..Default::default() + }], + files: HashMap::from([(torrent_hash.to_string(), vec![])]), + }; + let mock_mam = MockMaM { + torrents: HashMap::new(), + }; + + let qbit_config = config.qbittorrent.first().unwrap(); + let _ = link_torrents_to_library( + config.clone(), + db.db.clone(), + (qbit_config, &mock_qbit), + &mock_mam, + ) + .await; + + let r = db.db.r_transaction()?; + let selected: Option = r.get().primary(1u64)?; + assert!( + selected.is_none(), + "SelectedTorrent should be removed from DB" + ); + Ok(()) +} + +#[tokio::test] +async fn test_link_torrent_ebook() -> anyhow::Result<()> { + let db = TestDb::new()?; + let fs = MockFs::new()?; + + let torrent_hash = "abcdef1234567890abcdef1234567890abcdef12"; + let torrent_name = "Test Ebook"; + let torrent_dir = fs.rip_dir.join(torrent_name); + std::fs::create_dir_all(&torrent_dir)?; + std::fs::write(torrent_dir.join("book.epub"), "fake epub content")?; + + let mut config = mock_config(fs.rip_dir.clone(), fs.library_dir.clone()); + config.qbittorrent.push(QbitConfig { + url: "http://localhost:8080".to_string(), + username: "admin".to_string(), + password: "adminadmin".to_string(), + path_mapping: BTreeMap::new(), + on_cleaned: None, + on_invalid_torrent: None, + }); + config.libraries = vec![Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: fs.rip_dir.clone(), + options: LibraryOptions { + name: Some("test_library".to_string()), + library_dir: fs.library_dir.clone(), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + })]; + let config = Arc::new(config); + + let qbit_torrent = QbitTorrent { + hash: torrent_hash.to_string(), + name: torrent_name.to_string(), + save_path: fs.rip_dir.to_string_lossy().to_string(), + progress: 1.0, + ..Default::default() + }; + let qbit_content = TorrentContent { + name: format!("{}/book.epub", torrent_name), + size: 200, + ..Default::default() + }; + let mock_qbit = MockQbit { + torrents: vec![qbit_torrent], + files: HashMap::from([(torrent_hash.to_string(), vec![qbit_content])]), + }; + + let mut mam_torrent = MaMTorrent { + id: 2, + title: "Ebook Title".to_string(), + added: "2024-01-02 12:00:00".to_string(), + size: "200 B".to_string(), + mediatype: 2, // Ebook + category: 46, + language: 1, + lang_code: "en".to_string(), + ..Default::default() + }; + mam_torrent.author_info.insert(2, "Ebook Author".to_string()); + + let mock_mam = MockMaM { + torrents: HashMap::from([(torrent_hash.to_string(), mam_torrent)]), + }; + + let qbit_config = config.qbittorrent.first().unwrap(); + link_torrents_to_library( + config.clone(), + db.db.clone(), + (qbit_config, &mock_qbit), + &mock_mam, + ) + .await?; + + let library_book_dir = fs.library_dir.join("Ebook Author").join("Ebook Title"); + assert!(library_book_dir.exists()); + assert!(library_book_dir.join("book.epub").exists()); + assert!(library_book_dir.join("metadata.json").exists()); + + Ok(()) +} + +#[tokio::test] +async fn test_relink() -> anyhow::Result<()> { + let db = TestDb::new()?; + let fs = MockFs::new()?; + + let torrent_hash = "relink_hash"; + let torrent_name = "Relink Torrent"; + let torrent_dir = fs.rip_dir.join(torrent_name); + std::fs::create_dir_all(&torrent_dir)?; + std::fs::write(torrent_dir.join("audio.m4b"), "audio")?; + + let mut config = mock_config(fs.rip_dir.clone(), fs.library_dir.clone()); + let qbit_config = QbitConfig { + url: "".to_string(), + username: "".to_string(), + password: "".to_string(), + path_mapping: BTreeMap::new(), + on_cleaned: None, + on_invalid_torrent: None, + }; + config.qbittorrent.push(qbit_config.clone()); + config.libraries = vec![Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: fs.rip_dir.clone(), + options: LibraryOptions { + name: Some("test".to_string()), + library_dir: fs.library_dir.clone(), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + })]; + let config = Arc::new(config); + + let old_library_path = fs.library_dir.join("Old Author").join("Title"); + std::fs::create_dir_all(&old_library_path)?; + std::fs::write(old_library_path.join("audio.m4b"), "old audio")?; + + { + let (_guard, rw) = db.db.rw_async().await?; + rw.insert(mlm_db::Torrent { + id: torrent_hash.to_string(), + id_is_hash: true, + mam_id: Some(1), + library_path: Some(old_library_path.clone()), + library_files: vec![std::path::PathBuf::from("audio.m4b")], + linker: Some("test".to_string()), + category: None, + selected_audio_format: Some(".m4b".to_string()), + selected_ebook_format: None, + title_search: "title".to_string(), + meta: mlm_db::TorrentMeta { + ids: BTreeMap::from([(mlm_db::ids::MAM.to_string(), "1".to_string())]), + title: "Title".to_string(), + authors: vec!["New Author".to_string()], + media_type: mlm_db::MediaType::Audiobook, + source: mlm_db::MetadataSource::Mam, + uploaded_at: Some(mlm_db::Timestamp::now()), + ..Default::default() + }, + created_at: mlm_db::Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + })?; + rw.commit()?; + } + + let mock_qbit = MockQbit { + torrents: vec![], + files: HashMap::from([( + torrent_hash.to_string(), + vec![TorrentContent { + name: format!("{}/audio.m4b", torrent_name), + ..Default::default() + }], + )]), + }; + + let qbit_torrent = QbitTorrent { + hash: torrent_hash.to_string(), + save_path: fs.rip_dir.to_string_lossy().to_string(), + ..Default::default() + }; + + mlm::linker::torrent::relink_internal( + &config, + &qbit_config, + &db.db, + &mock_qbit, + qbit_torrent, + torrent_hash.to_string(), + ) + .await?; + + assert!(!old_library_path.exists()); + let new_library_path = fs.library_dir.join("New Author").join("Title"); + assert!(new_library_path.exists()); + assert!(new_library_path.join("audio.m4b").exists()); + + let r = db.db.r_transaction()?; + let torrent: mlm_db::Torrent = r.get().primary(torrent_hash.to_string())?.unwrap(); + assert_eq!(torrent.library_path, Some(new_library_path)); + + Ok(()) +} + +#[tokio::test] +async fn test_refresh_metadata_relink() -> anyhow::Result<()> { + let db = TestDb::new()?; + let fs = MockFs::new()?; + + let torrent_hash = "refresh_relink_hash"; + let torrent_name = "Refresh Relink Torrent"; + let torrent_dir = fs.rip_dir.join(torrent_name); + std::fs::create_dir_all(&torrent_dir)?; + std::fs::write(torrent_dir.join("audio.m4b"), "audio")?; + + let mut config = mock_config(fs.rip_dir.clone(), fs.library_dir.clone()); + let qbit_config = QbitConfig { + url: "".to_string(), + username: "".to_string(), + password: "".to_string(), + path_mapping: BTreeMap::new(), + on_cleaned: None, + on_invalid_torrent: None, + }; + config.qbittorrent.push(qbit_config.clone()); + config.libraries = vec![Library::ByDownloadDir(LibraryByDownloadDir { + download_dir: fs.rip_dir.clone(), + options: LibraryOptions { + name: Some("test".to_string()), + library_dir: fs.library_dir.clone(), + method: LibraryLinkMethod::Hardlink, + audio_types: None, + ebook_types: None, + }, + tag_filters: LibraryTagFilters::default(), + })]; + let config = Arc::new(config); + + { + let (_guard, rw) = db.db.rw_async().await?; + rw.insert(mlm_db::Torrent { + id: torrent_hash.to_string(), + id_is_hash: true, + mam_id: Some(2), + library_path: Some(fs.library_dir.join("Old Author").join("Title")), + library_files: vec![std::path::PathBuf::from("audio.m4b")], + linker: Some("test".to_string()), + category: None, + selected_audio_format: Some(".m4b".to_string()), + selected_ebook_format: None, + title_search: "title".to_string(), + meta: mlm_db::TorrentMeta { + ids: BTreeMap::from([(mlm_db::ids::MAM.to_string(), "2".to_string())]), + title: "Title".to_string(), + authors: vec!["Old Author".to_string()], + media_type: mlm_db::MediaType::Audiobook, + source: mlm_db::MetadataSource::Mam, + uploaded_at: Some(mlm_db::Timestamp::now()), + ..Default::default() + }, + created_at: mlm_db::Timestamp::now(), + replaced_with: None, + library_mismatch: None, + client_status: None, + })?; + rw.commit()?; + } + + let mock_qbit = MockQbit { + torrents: vec![], + files: HashMap::from([( + torrent_hash.to_string(), + vec![TorrentContent { + name: format!("{}/audio.m4b", torrent_name), + ..Default::default() + }], + )]), + }; + + let mut mam_torrent = MaMTorrent { + id: 2, + title: "Title".to_string(), + added: "2024-01-01 12:00:00".to_string(), + size: "100 B".to_string(), + mediatype: 1, // Audiobook + category: 42, + language: 1, + lang_code: "en".to_string(), + ..Default::default() + }; + mam_torrent.author_info.insert(2, "Refreshed Author".to_string()); + + let mock_mam = MockMaM { + torrents: HashMap::from([(torrent_hash.to_string(), mam_torrent)]), + }; + + let qbit_torrent = QbitTorrent { + hash: torrent_hash.to_string(), + save_path: fs.rip_dir.to_string_lossy().to_string(), + ..Default::default() + }; + + mlm::linker::torrent::refresh_metadata_relink_internal( + &config, + &qbit_config, + &db.db, + &mock_qbit, + &mock_mam, + qbit_torrent, + torrent_hash.to_string(), + ) + .await?; + + let new_library_path = fs.library_dir.join("Refreshed Author").join("Title"); + assert!(new_library_path.exists()); + assert!(new_library_path.join("audio.m4b").exists()); + + let r = db.db.r_transaction()?; + let torrent: mlm_db::Torrent = r.get().primary(torrent_hash.to_string())?.unwrap(); + assert_eq!(torrent.meta.authors, vec!["Refreshed Author"]); + assert_eq!(torrent.library_path, Some(new_library_path)); + + Ok(()) +} From cad99f6ef24e48c8b703877d157c5f28ca27f108 Mon Sep 17 00:00:00 2001 From: Stirling Mouse <181794392+StirlingMouse@users.noreply.github.com> Date: Sat, 14 Feb 2026 11:07:50 +0100 Subject: [PATCH 09/14] Support editing non-mam torrents --- server/src/autograbber.rs | 27 ++++++++++++---------- server/src/linker/torrent.rs | 2 +- server/src/snatchlist.rs | 16 +++++++------ server/src/web/pages/config.rs | 2 +- server/src/web/pages/torrent_edit.rs | 29 ++++++++++++------------ server/templates/pages/torrent_edit.html | 2 ++ 6 files changed, 43 insertions(+), 35 deletions(-) diff --git a/server/src/autograbber.rs b/server/src/autograbber.rs index 28868398..2d8d77d9 100644 --- a/server/src/autograbber.rs +++ b/server/src/autograbber.rs @@ -415,7 +415,7 @@ pub async fn select_torrents>( config, db, rw_opt.unwrap(), - &torrent, + Some(&torrent), old, meta, false, @@ -531,7 +531,7 @@ pub async fn select_torrents>( config, db, rw_opt.unwrap(), - &torrent, + Some(&torrent), old, meta, false, @@ -676,7 +676,7 @@ pub async fn update_torrent_meta( config: &Config, db: &Database<'_>, (guard, rw): (MutexGuard<'_, ()>, RwTransaction<'_>), - mam_torrent: &MaMTorrent, + mam_torrent: Option<&MaMTorrent>, mut torrent: mlm_db::Torrent, mut meta: TorrentMeta, allow_non_mam: bool, @@ -731,16 +731,17 @@ pub async fn update_torrent_meta( } } - if linker_is_owner && torrent.linker.is_none() { + if linker_is_owner && torrent.linker.is_none() + && let Some(mam_torrent) = mam_torrent + { torrent.linker = Some(mam_torrent.owner_name.clone()); } let id = torrent.id.clone(); - let mam_id = mam_torrent.id; let diff = torrent.meta.diff(&meta); debug!( "Updating meta for torrent {}, diff:\n{}", - mam_id, + id, diff.iter() .map(|field| format!(" {}: {} → {}", field.field, field.from, field.to)) .join("\n") @@ -766,28 +767,29 @@ pub async fn update_torrent_meta( let mut writer = BufWriter::new(file); serde_json::to_writer(&mut writer, &serde_json::Value::Object(existing))?; writer.flush()?; - debug!("updated ABS metadata file {}", mam_torrent.id); + debug!("updated ABS metadata file {}", id); } if let (Some(abs_id), Some(abs_config)) = (&torrent.meta.ids.get(ids::ABS), &config.audiobookshelf) { let abs = Abs::new(abs_config)?; match abs.update_book(abs_id, &meta).await { - Ok(_) => debug!("updated ABS via API {}", mam_torrent.id), - Err(err) => warn!("Failed updating book {} in abs: {err}", mam_torrent.id), + Ok(_) => debug!("updated ABS via API {}", id), + Err(err) => warn!("Failed updating book {} in abs: {err}", id), } } } if !diff.is_empty() { + let mam_id = mam_torrent.map(|m| m.id); write_event( db, Event::new( Some(id), - Some(mam_id), + mam_id, EventType::Updated { fields: diff, - source: (MetadataSource::Mam, String::new()), + source: (meta.source.clone(), String::new()), }, ), ) @@ -814,6 +816,7 @@ async fn update_selected_torrent_meta( ); let hash = get_mam_torrent_hash(mam, &torrent.dl_link).await.ok(); let mut torrent = torrent; + let source = meta.source.clone(); torrent.meta = meta; rw.upsert(torrent)?; rw.commit()?; @@ -825,7 +828,7 @@ async fn update_selected_torrent_meta( Some(mam_id), EventType::Updated { fields: diff, - source: (MetadataSource::Mam, String::new()), + source: (source, String::new()), }, ), ) diff --git a/server/src/linker/torrent.rs b/server/src/linker/torrent.rs index acae36a9..9b51b31e 100644 --- a/server/src/linker/torrent.rs +++ b/server/src/linker/torrent.rs @@ -489,7 +489,7 @@ where config, db, db.rw_async().await?, - &mam_torrent, + Some(&mam_torrent), torrent.clone(), meta.clone(), true, diff --git a/server/src/snatchlist.rs b/server/src/snatchlist.rs index 95ba8575..49e191d7 100644 --- a/server/src/snatchlist.rs +++ b/server/src/snatchlist.rs @@ -155,7 +155,7 @@ async fn update_torrents>( update_torrent_meta( db, rw_opt.unwrap(), - &torrent, + Some(&torrent), old, meta, cost == Cost::MetadataOnlyAdd, @@ -233,7 +233,7 @@ async fn add_metadata_only_torrent( async fn update_torrent_meta( db: &Database<'_>, (guard, rw): (MutexGuard<'_, ()>, RwTransaction<'_>), - mam_torrent: &UserDetailsTorrent, + mam_torrent: Option<&UserDetailsTorrent>, mut torrent: Torrent, mut meta: TorrentMeta, linker_is_owner: bool, @@ -277,17 +277,18 @@ async fn update_torrent_meta( } if linker_is_owner && torrent.linker.is_none() { - torrent.linker = Some(mam_torrent.uploader_name.clone()); + if let Some(mam_torrent) = mam_torrent { + torrent.linker = Some(mam_torrent.uploader_name.clone()); + } } else if meta == torrent.meta { return Ok(()); } let id = torrent.id.clone(); - let mam_id = mam_torrent.id; let diff = torrent.meta.diff(&meta); debug!( "Updating meta for torrent {}, diff:\n{}", - mam_id, + id, diff.iter() .map(|field| format!(" {}: {} → {}", field.field, field.from, field.to)) .join("\n") @@ -299,14 +300,15 @@ async fn update_torrent_meta( drop(guard); if !diff.is_empty() { + let mam_id = mam_torrent.map(|m| m.id); write_event( db, Event::new( Some(id), - Some(mam_id), + mam_id, EventType::Updated { fields: diff, - source: (MetadataSource::Mam, String::new()), + source: (meta.source.clone(), String::new()), }, ), ) diff --git a/server/src/web/pages/config.rs b/server/src/web/pages/config.rs index eecf60c0..782391ea 100644 --- a/server/src/web/pages/config.rs +++ b/server/src/web/pages/config.rs @@ -82,7 +82,7 @@ pub async fn config_page_post( &config, &context.db, context.db.rw_async().await?, - &mam_torrent, + Some(&mam_torrent), torrent.clone(), new_meta, false, diff --git a/server/src/web/pages/torrent_edit.rs b/server/src/web/pages/torrent_edit.rs index 90f589cd..2d09e677 100644 --- a/server/src/web/pages/torrent_edit.rs +++ b/server/src/web/pages/torrent_edit.rs @@ -41,7 +41,7 @@ pub async fn torrent_edit_page_post( Form(form): Form, ) -> Result { let config = context.config().await; - let mam = context.mam()?; + let _mam = context.mam()?; let Some(torrent) = context .db .r_transaction()? @@ -50,9 +50,6 @@ pub async fn torrent_edit_page_post( else { return Err(anyhow::Error::msg("Could not find torrent").into()); }; - let Some(mam_torrent) = mam.get_torrent_info(&hash).await? else { - return Err(anyhow::Error::msg("Could not find torrent on MaM").into()); - }; let authors = form.authors.split("\r\n").map(ToOwned::to_owned).collect(); let narrators = if form.narrators.trim().is_empty() { @@ -78,10 +75,12 @@ pub async fn torrent_edit_page_post( }) .collect::, _>>()? }; - let language = - Language::from_id(form.language).ok_or_else(|| anyhow::Error::msg("Invalid language"))?; - let category = OldCategory::from_one_id(form.category) - .ok_or_else(|| anyhow::Error::msg("Invalid category"))?; + let language = form.language.and_then(Language::from_id); + let category = form.category.and_then(OldCategory::from_one_id); + let media_type = match &category { + Some(c) => c.as_main_cat().into(), + None => torrent.meta.media_type, + }; let flags = Flags { crude_language: Some(form.crude_language), violence: Some(form.violence), @@ -93,9 +92,9 @@ pub async fn torrent_edit_page_post( let meta = TorrentMeta { title: form.title, - media_type: category.as_main_cat().into(), - cat: Some(category), - language: Some(language), + media_type, + cat: category, + language, flags: Some(FlagBits::new(flags.as_bitfield())), authors, narrators, @@ -108,7 +107,7 @@ pub async fn torrent_edit_page_post( &config, &context.db, context.db.rw_async().await?, - &mam_torrent, + None, torrent, meta, true, @@ -125,8 +124,10 @@ pub struct TorrentPageForm { authors: String, narrators: String, series: String, - language: u8, - category: u64, + #[serde(default)] + language: Option, + #[serde(default)] + category: Option, // #[serde(flatten)] // flags: Flags, diff --git a/server/templates/pages/torrent_edit.html b/server/templates/pages/torrent_edit.html index b1e8e830..1d1093a0 100644 --- a/server/templates/pages/torrent_edit.html +++ b/server/templates/pages/torrent_edit.html @@ -29,6 +29,7 @@

{{ torrent.meta.title }}