Skip to content

fix: Parsing IQDB correctly #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lyssieth merged 3 commits into from
Oct 20, 2025
Merged

fix: Parsing IQDB correctly #9

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bebbaf9
fix: Parsing IQDB correctly
Nachtalb Oct 19, 2025
d12c4ab
refactor: Simpler IQDB processing workflow
Nachtalb Oct 19, 2025
c7b0831
refactor: Fix IQDB urls on the fly
Nachtalb Oct 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 17 additions & 75 deletions src/source/iqdb.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,31 +57,11 @@ impl Source for Iqdb {

let html = scraper::Html::parse_document(&text);

let pages = html.select(sel!("#pages > div")).collect::<Vec<_>>();

let best_match = if pages.len() > 2 {
Self::harvest_best_match(&pages[0])
} else {
None
};

let mut items = Vec::new();

if let Some(best_match) = best_match {
items.push(best_match);
}

for page in pages.into_iter().skip(2) {
let page = Self::harvest_page(&page);

items.extend(page);
}

for item in &mut items {
if item.link.starts_with("//") {
item.link = format!("https:{}", item.link);
}
}
let items: Vec<Item> = html
.select(sel!("#pages > div"))
.skip(1)
.filter_map(Self::harvest_page)
.collect();

Ok(Output {
original_url: url.to_string(),
Expand All @@ -95,66 +75,28 @@ impl Source for Iqdb {
}

impl Iqdb {
fn harvest_page(page: &ElementRef) -> Option<Item> {
let dom = page;

fn harvest_page(page: ElementRef) -> Option<Item> {
debug!("selecting .image a");
let link = dom.select(sel!(".image a")).next()?;
let link = page.select(sel!(".image a")).next()?;

debug!("grabbing href");
let url = link.value().attr("href")?;

debug!("collecting trs");
let score = dom.select(sel!("tr")).collect::<Vec<_>>();

if score.len() != 5 {
return Some(Item {
link: url.to_string(),
similarity: -1.0,
});
}

debug!("grabbing score");
let score = score[3];
debug!("grabbing td");
let td = score.select(sel!("td")).next()?;

let score = td.text().collect::<String>();
let score = score.split_once('%')?.0.parse::<f32>().ok()? / 100.0;

Some(Item {
link: url.to_string(),
similarity: score,
})
}

fn harvest_best_match(pages: &ElementRef) -> Option<Item> {
debug!("selecting .image a");
let link = pages.select(sel!(".image a")).next()?;

debug!("grabbing href");
let url = link.value().attr("href")?;

debug!("collecting trs");
let score = pages.select(sel!("tr")).collect::<Vec<_>>();

if score.len() != 5 {
return Some(Item {
link: url.to_string(),
similarity: -1.0,
});
}
debug!("fix broken url if needed");
let url = if url.starts_with("//") {
format!("https:{url}")
} else {
url.to_string()
};

debug!("grabbing score");
let score = score[3];
debug!("grabbing td");
let td = score.select(sel!("td")).next()?;
let score = page.select(sel!("tr:last-child > td")).next()?;

let score = td.text().collect::<String>();
debug!("parsing score");
let score = score.text().collect::<String>();
let score = score.split_once('%')?.0.parse::<f32>().ok()? / 100.0;

Some(Item {
link: url.to_string(),
link: url,
similarity: score,
})
}
Expand Down
Loading