Skip to content

Yahoo cheerio migration, dead code cleanup, enriched share endpoint #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Pyronewbic merged 1 commit into from
May 12, 2026
+161 −154
Merged

Yahoo cheerio migration, dead code cleanup, enriched share endpoint #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import { gradeImage } from "./lib/grading/grading.js";
import { parseListingLanguagesFromInput, filterByCondition, detectCondition, flagPriceOutliers } from "./lib/search/filters.js";
import { buildEbaySearchQuery } from "./lib/search/listingQuery.js";
import { EBAY_CATEGORY_TCG_SINGLE_CARDS_US } from "./lib/search/ebayCategories.js";
import { getRedisStatus, sha256 } from "./lib/data/redis-cache.js";
import { saveGradeLog, getGradeLogs, saveDrop, getDrops, getDrop, saveWebhook, getWebhooks, deleteWebhook, getFirestoreStatus, saveAlert, getActiveAlerts, updateAlert, getAlertsByEmail, saveErrorLog, getErrorLogs, clearErrorLogs } from "./lib/data/firestore.js";
import { getDemoSearchResult, listDemoCards, findDemoByNumber } from "./lib/data/demo.js";
import { createApiKey, listApiKeys, getApiKey, updateApiKey, deleteApiKey, rotateApiKey, validateApiKey } from "./lib/data/api-keys.js";
Expand Down Expand Up @@ -435,13 +434,13 @@ app.delete("/api/errors", ownerOnly, async (req, res) => {

// GET /api/health
app.get("/api/health", async (req, res) => {
const [redisStatus, firestoreStatus] = await Promise.all([getRedisStatus(), getFirestoreStatus()]);
const firestoreStatus = await getFirestoreStatus();
let ebayUsage = null;
try { ebayUsage = await getEbayUsageToday(); } catch {}
res.json({
status: "ok",
uptime: Math.floor(process.uptime()),
redis: redisStatus,
redis: { connected: false, status: "not configured" },
firestore: firestoreStatus,
ebay: { configured: !!(clientId && clientSecret), usageToday: ebayUsage, dailyCap: DAILY_CAP },
});
Expand Down Expand Up @@ -685,13 +684,30 @@ app.get("/api/card/share/:setCode/:number", async (req, res) => {

const identity = card || parseCardIdentity(searchQuery);
if (identity.setCode) identity.setName = SET_NAME_MAP[identity.setCode] || identity.setCode;
if (search?.query) {
const fromQuery = parseCardIdentity(search.query);
if (!identity.rarity && fromQuery.rarity) identity.rarity = fromQuery.rarity;
if ((!identity.name || identity.name === identity.setName) && fromQuery.name) identity.name = fromQuery.name;
}
if (identity.name && identity.setName && identity.name.includes(identity.setName)) {
identity.name = identity.name.replace(identity.setName, "").replace(/\s+/g, " ").trim();
}

const active = search ? Object.values(search.activeByCountry || {}).flat() : [];
const lowestPrice = active.length ? Math.min(...active.map(i => i.totalCost || i.price)) : null;
const lowestSource = lowestPrice && active.length
? active.find(i => (i.totalCost || i.price) === lowestPrice)
: null;

const sourceMap = {};
for (const item of active) {
const src = item.itemWebUrl?.includes("magi") ? "magi" : item.itemWebUrl?.includes("yahoo") ? "yahoo" : item.itemWebUrl?.includes("snkrdunk") ? "snkrdunk" : "ebay";
if (!sourceMap[src]) sourceMap[src] = { count: 0, lowest: Infinity };
sourceMap[src].count++;
const p = item.totalCost || item.price;
if (p < sourceMap[src].lowest) sourceMap[src].lowest = p;
}

res.json({
cardId,
identity,
Expand All @@ -700,10 +716,11 @@ app.get("/api/card/share/:setCode/:number", async (req, res) => {
source: lowestSource?.itemWebUrl?.includes("magi") ? "magi" : lowestSource?.itemWebUrl?.includes("yahoo") ? "yahoo" : lowestSource?.itemWebUrl?.includes("snkrdunk") ? "snkrdunk" : "ebay",
listingCount: active.length,
currency: active[0]?.priceCurrency || "USD",
bySources: sourceMap,
},
psaSignal: search?.psaSignal || null,
priceHistory: priceData,
searchQuery,
searchQuery: search?.query || searchQuery,
});
} catch (e) {
logError("card-share", e.message, req.originalUrl, req.requestId);
Expand Down
7 changes: 0 additions & 7 deletions lib/data/card-identity.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,10 +337,3 @@ export async function findCardByQuery(query) {
}
}

export async function updateCardField(cardId, field, value) {
const fs = getDb();
if (!fs) return;
try {
await fs.collection(COLLECTION).doc(cardId).update({ [field]: value });
} catch {}
}
283 changes: 140 additions & 143 deletions lib/sources/yahooauctions.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import { chromium } from "playwright";
import * as cheerio from "cheerio";
import { translateToJapanese, fetchJPYRate } from "./magi.js";

const YAHOO_SEARCH = "https://auctions.yahoo.co.jp/search/search";
const YAHOO_CLOSED = "https://auctions.yahoo.co.jp/closedsearch/closedsearch";
const POKEMON_TCG_CAT = "2084241343";
const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36";

function searchUrl(keyword, { sort = "cbids", order = "a", n = 50 } = {}) {
const p = new URLSearchParams({
Expand Down Expand Up @@ -33,88 +34,92 @@ function gradeFromTitle(title) {
return null;
}

async function scrapeActive(browser, keyword, limit) {
const page = await browser.newPage();
try {
await page.goto(searchUrl(keyword), { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(1500);

const items = await page.evaluate(({ lim, extractPriceFn }) => {
function getPrice(el) {
const bonus = el.querySelector(".Product__bonus");
const link = el.querySelector(".Product__titleLink");
const buynow = parseInt(bonus?.getAttribute("data-auction-buynowprice"), 10);
if (buynow > 0) return buynow;
const current = parseInt(bonus?.getAttribute("data-auction-price") || link?.getAttribute("data-auction-price"), 10);
if (current > 0) return current;
const priceEl = el.querySelector(".Product__priceValue");
if (priceEl) {
const m = priceEl.textContent?.replace(/[^0-9]/g, "");
if (m) return parseInt(m, 10) || 0;
async function scrapeActive(keyword, limit) {
const url = searchUrl(keyword);
const res = await fetch(url, {
headers: { "User-Agent": UA, Accept: "text/html" },
signal: AbortSignal.timeout(10000),
});
if (!res.ok) return [];
const html = await res.text();
const $ = cheerio.load(html);
const results = [];

$(".Product__detail").each((_, el) => {
if (results.length >= limit) return false;
const $el = $(el);
const link = $el.find(".Product__titleLink");
const bonus = $el.find(".Product__bonus");
if (!link.length) return;

const title = link.attr("data-auction-title") || link.text().trim() || "";
if (!title) return;

const auctionId = bonus.attr("data-auction-id") || link.attr("data-auction-id") || "";

let price = 0;
const buynow = parseInt(bonus.attr("data-auction-buynowprice"), 10);
if (buynow > 0) {
price = buynow;
} else {
const current = parseInt(bonus.attr("data-auction-price") || link.attr("data-auction-price"), 10);
if (current > 0) {
price = current;
} else {
const priceEl = $el.find(".Product__priceValue");
if (priceEl.length) {
const m = priceEl.text()?.replace(/[^0-9]/g, "");
if (m) price = parseInt(m, 10) || 0;
}
return 0;
}
}

const results = [];
const els = document.querySelectorAll(".Product__detail");
for (const el of els) {
if (results.length >= lim) break;
const link = el.querySelector(".Product__titleLink");
const bonus = el.querySelector(".Product__bonus");
if (!link) continue;
const title = link.getAttribute("data-auction-title") || link.textContent?.trim() || "";
const auctionId = bonus?.getAttribute("data-auction-id") || link.getAttribute("data-auction-id") || "";
const price = getPrice(el);
const img = link.getAttribute("data-auction-img") || "";
const href = link.href || "";
if (!title) continue;
results.push({ title, auctionId, price, img, href });
}
return results;
}, { lim: limit });
const img = link.attr("data-auction-img") || "";
const href = link.attr("href") || "";

results.push({ title, auctionId, price, img, href });
});

return items;
} finally {
await page.close();
}
return results;
}

async function scrapeSold(browser, keyword, limit) {
const page = await browser.newPage();
try {
await page.goto(closedUrl(keyword), { waitUntil: "networkidle", timeout: 45000 });
await page.waitForTimeout(2000);

const items = await page.evaluate((lim) => {
const seen = new Set();
const results = [];
const links = document.querySelectorAll('a[href*="/jp/auction/"]');
for (const a of links) {
if (results.length >= lim) break;
const title = a.textContent?.trim();
if (!title || title.length < 5) continue;
const href = a.href || "";
const auctionId = href.split("/auction/").pop() || "";
if (seen.has(auctionId)) continue;
seen.add(auctionId);
const clParams = a.getAttribute("data-cl-params") || "";
const pm = clParams.match(/etc:p=([0-9]+)/);
let price = pm ? parseInt(pm[1], 10) : 0;
if (!price) {
const container = a.closest("[class*='sc-']") || a.parentElement;
const text = container?.textContent || "";
const ym = text.match(/([0-9,]+)\s*円/);
if (ym) price = parseInt(ym[1].replace(/,/g, ""), 10) || 0;
}
results.push({ title, auctionId, price, href });
}
return results;
}, limit);
async function scrapeSold(keyword, limit) {
const url = closedUrl(keyword);
const res = await fetch(url, {
headers: { "User-Agent": UA, Accept: "text/html" },
signal: AbortSignal.timeout(10000),
});
if (!res.ok) return [];
const html = await res.text();
const $ = cheerio.load(html);
const seen = new Set();
const results = [];

$('a[href*="/jp/auction/"]').each((_, el) => {
if (results.length >= limit) return false;
const $a = $(el);
const title = $a.text().trim();
if (!title || title.length < 5) return;

const href = $a.attr("href") || "";
const auctionId = href.split("/auction/").pop() || "";
if (seen.has(auctionId)) return;
seen.add(auctionId);

const clParams = $a.attr("data-cl-params") || "";
const pm = clParams.match(/etc:p=([0-9]+)/);
let price = pm ? parseInt(pm[1], 10) : 0;
if (!price) {
const container = $a.parent();
const text = container.text() || "";
const ym = text.match(/([0-9,]+)\s*円/);
if (ym) price = parseInt(ym[1].replace(/,/g, ""), 10) || 0;
}

results.push({ title, auctionId, price, href });
});

return items;
} finally {
await page.close();
}
return results;
}

export async function searchYahooAuctions(card, config, { log = console.log } = {}) {
Expand All @@ -132,74 +137,66 @@ export async function searchYahooAuctions(card, config, { log = console.log } =

log(` yahoo q: ${query}`);

const [jpyPerUsd, browser] = await Promise.all([
const [jpyPerUsd, activeRaw, soldRaw] = await Promise.all([
fetchJPYRate(),
chromium.launch({ headless: true }),
scrapeActive(query, resultsPerCard),
scrapeSold(query, soldListingsLimit),
]);

try {
const [activeRaw, soldRaw] = await Promise.all([
scrapeActive(browser, query, resultsPerCard),
scrapeSold(browser, query, soldListingsLimit),
]);

const toUSD = (jpy) => (jpy != null && jpy > 0 ? Math.round((jpy / jpyPerUsd) * 100) / 100 : null);

const auctionUrl = (item) =>
item.href || `https://page.auctions.yahoo.co.jp/jp/auction/${item.auctionId}`;

const active = activeRaw
.map((raw) => ({
itemId: raw.auctionId,
itemWebUrl: auctionUrl(raw),
title: raw.title,
price: toUSD(raw.price),
priceCurrency: "USD",
priceJPY: raw.price,
shippingLabel: "—",
totalCost: toUSD(raw.price),
listingGradeLabel: gradeFromTitle(raw.title),
shippingToBuyer: Object.fromEntries(
deliveryCountries.map((c) => [c, { eligible: null }]),
),
grade: null,
}))
.filter((r) => listingFormat !== "raw" || r.listingGradeLabel == null);

const sold = soldRaw
.map((raw) => ({
itemId: raw.auctionId,
itemWebUrl: auctionUrl(raw),
title: raw.title,
price: toUSD(raw.price),
currency: "USD",
priceJPY: raw.price,
endedDate: "—",
listingGradeLabel: gradeFromTitle(raw.title),
}))
.filter((r) => listingFormat !== "raw" || r.listingGradeLabel == null);

log(` yahoo: ${active.length} active, ${sold.length} sold (¥${Math.round(jpyPerUsd)}/USD)`);

const listingDesc =
listingFormat === "slab" && slab
? `Yahoo Auctions JP — ${slab.provider} ${slab.grade} (¥${Math.round(jpyPerUsd)}/USD)`
: `Yahoo Auctions JP (¥${Math.round(jpyPerUsd)}/USD)`;

return {
query: card,
ebaySearchQuery: query,
listingFormat,
listingDescription: listingDesc,
slab: listingFormat === "slab" ? { ...slab } : null,
lang: "jp",
activeByCountry: Object.fromEntries(deliveryCountries.map((c) => [c, active])),
sold,
gradingLabel: "yahoo listing",
counts: { activeTotal: active.length, sold: sold.length },
source: "yahoo",
};
} finally {
await browser.close();
}
const toUSD = (jpy) => (jpy != null && jpy > 0 ? Math.round((jpy / jpyPerUsd) * 100) / 100 : null);

const auctionUrl = (item) =>
item.href || `https://page.auctions.yahoo.co.jp/jp/auction/${item.auctionId}`;

const active = activeRaw
.map((raw) => ({
itemId: raw.auctionId,
itemWebUrl: auctionUrl(raw),
title: raw.title,
price: toUSD(raw.price),
priceCurrency: "USD",
priceJPY: raw.price,
shippingLabel: "—",
totalCost: toUSD(raw.price),
listingGradeLabel: gradeFromTitle(raw.title),
shippingToBuyer: Object.fromEntries(
deliveryCountries.map((c) => [c, { eligible: null }]),
),
grade: null,
}))
.filter((r) => listingFormat !== "raw" || r.listingGradeLabel == null);

const sold = soldRaw
.map((raw) => ({
itemId: raw.auctionId,
itemWebUrl: auctionUrl(raw),
title: raw.title,
price: toUSD(raw.price),
currency: "USD",
priceJPY: raw.price,
endedDate: "—",
listingGradeLabel: gradeFromTitle(raw.title),
}))
.filter((r) => listingFormat !== "raw" || r.listingGradeLabel == null);

log(` yahoo: ${active.length} active, ${sold.length} sold (¥${Math.round(jpyPerUsd)}/USD)`);

const listingDesc =
listingFormat === "slab" && slab
? `Yahoo Auctions JP — ${slab.provider} ${slab.grade} (¥${Math.round(jpyPerUsd)}/USD)`
: `Yahoo Auctions JP (¥${Math.round(jpyPerUsd)}/USD)`;

return {
query: card,
ebaySearchQuery: query,
listingFormat,
listingDescription: listingDesc,
slab: listingFormat === "slab" ? { ...slab } : null,
lang: "jp",
activeByCountry: Object.fromEntries(deliveryCountries.map((c) => [c, active])),
sold,
gradingLabel: "yahoo listing",
counts: { activeTotal: active.length, sold: sold.length },
source: "yahoo",
};
}