diff --git a/api.js b/api.js index 5502be4..ded1486 100644 --- a/api.js +++ b/api.js @@ -15,7 +15,6 @@ import { gradeImage } from "./lib/grading/grading.js"; import { parseListingLanguagesFromInput, filterByCondition, detectCondition, flagPriceOutliers } from "./lib/search/filters.js"; import { buildEbaySearchQuery } from "./lib/search/listingQuery.js"; import { EBAY_CATEGORY_TCG_SINGLE_CARDS_US } from "./lib/search/ebayCategories.js"; -import { getRedisStatus, sha256 } from "./lib/data/redis-cache.js"; import { saveGradeLog, getGradeLogs, saveDrop, getDrops, getDrop, saveWebhook, getWebhooks, deleteWebhook, getFirestoreStatus, saveAlert, getActiveAlerts, updateAlert, getAlertsByEmail, saveErrorLog, getErrorLogs, clearErrorLogs } from "./lib/data/firestore.js"; import { getDemoSearchResult, listDemoCards, findDemoByNumber } from "./lib/data/demo.js"; import { createApiKey, listApiKeys, getApiKey, updateApiKey, deleteApiKey, rotateApiKey, validateApiKey } from "./lib/data/api-keys.js"; @@ -435,13 +434,13 @@ app.delete("/api/errors", ownerOnly, async (req, res) => { // GET /api/health app.get("/api/health", async (req, res) => { - const [redisStatus, firestoreStatus] = await Promise.all([getRedisStatus(), getFirestoreStatus()]); + const firestoreStatus = await getFirestoreStatus(); let ebayUsage = null; try { ebayUsage = await getEbayUsageToday(); } catch {} res.json({ status: "ok", uptime: Math.floor(process.uptime()), - redis: redisStatus, + redis: { connected: false, status: "not configured" }, firestore: firestoreStatus, ebay: { configured: !!(clientId && clientSecret), usageToday: ebayUsage, dailyCap: DAILY_CAP }, }); @@ -685,6 +684,14 @@ app.get("/api/card/share/:setCode/:number", async (req, res) => { const identity = card || parseCardIdentity(searchQuery); if (identity.setCode) identity.setName = SET_NAME_MAP[identity.setCode] || identity.setCode; + if (search?.query) { + const fromQuery = parseCardIdentity(search.query); + if (!identity.rarity && fromQuery.rarity) identity.rarity = fromQuery.rarity; + if ((!identity.name || identity.name === identity.setName) && fromQuery.name) identity.name = fromQuery.name; + } + if (identity.name && identity.setName && identity.name.includes(identity.setName)) { + identity.name = identity.name.replace(identity.setName, "").replace(/\s+/g, " ").trim(); + } const active = search ? Object.values(search.activeByCountry || {}).flat() : []; const lowestPrice = active.length ? Math.min(...active.map(i => i.totalCost || i.price)) : null; @@ -692,6 +699,15 @@ app.get("/api/card/share/:setCode/:number", async (req, res) => { ? active.find(i => (i.totalCost || i.price) === lowestPrice) : null; + const sourceMap = {}; + for (const item of active) { + const src = item.itemWebUrl?.includes("magi") ? "magi" : item.itemWebUrl?.includes("yahoo") ? "yahoo" : item.itemWebUrl?.includes("snkrdunk") ? "snkrdunk" : "ebay"; + if (!sourceMap[src]) sourceMap[src] = { count: 0, lowest: Infinity }; + sourceMap[src].count++; + const p = item.totalCost || item.price; + if (p < sourceMap[src].lowest) sourceMap[src].lowest = p; + } + res.json({ cardId, identity, @@ -700,10 +716,11 @@ app.get("/api/card/share/:setCode/:number", async (req, res) => { source: lowestSource?.itemWebUrl?.includes("magi") ? "magi" : lowestSource?.itemWebUrl?.includes("yahoo") ? "yahoo" : lowestSource?.itemWebUrl?.includes("snkrdunk") ? "snkrdunk" : "ebay", listingCount: active.length, currency: active[0]?.priceCurrency || "USD", + bySources: sourceMap, }, psaSignal: search?.psaSignal || null, priceHistory: priceData, - searchQuery, + searchQuery: search?.query || searchQuery, }); } catch (e) { logError("card-share", e.message, req.originalUrl, req.requestId); diff --git a/lib/data/card-identity.js b/lib/data/card-identity.js index d66e399..3f6f623 100644 --- a/lib/data/card-identity.js +++ b/lib/data/card-identity.js @@ -337,10 +337,3 @@ export async function findCardByQuery(query) { } } -export async function updateCardField(cardId, field, value) { - const fs = getDb(); - if (!fs) return; - try { - await fs.collection(COLLECTION).doc(cardId).update({ [field]: value }); - } catch {} -} diff --git a/lib/sources/yahooauctions.js b/lib/sources/yahooauctions.js index bfab294..49e1cc2 100644 --- a/lib/sources/yahooauctions.js +++ b/lib/sources/yahooauctions.js @@ -1,9 +1,10 @@ -import { chromium } from "playwright"; +import * as cheerio from "cheerio"; import { translateToJapanese, fetchJPYRate } from "./magi.js"; const YAHOO_SEARCH = "https://auctions.yahoo.co.jp/search/search"; const YAHOO_CLOSED = "https://auctions.yahoo.co.jp/closedsearch/closedsearch"; const POKEMON_TCG_CAT = "2084241343"; +const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"; function searchUrl(keyword, { sort = "cbids", order = "a", n = 50 } = {}) { const p = new URLSearchParams({ @@ -33,88 +34,92 @@ function gradeFromTitle(title) { return null; } -async function scrapeActive(browser, keyword, limit) { - const page = await browser.newPage(); - try { - await page.goto(searchUrl(keyword), { waitUntil: "domcontentloaded", timeout: 30000 }); - await page.waitForTimeout(1500); - - const items = await page.evaluate(({ lim, extractPriceFn }) => { - function getPrice(el) { - const bonus = el.querySelector(".Product__bonus"); - const link = el.querySelector(".Product__titleLink"); - const buynow = parseInt(bonus?.getAttribute("data-auction-buynowprice"), 10); - if (buynow > 0) return buynow; - const current = parseInt(bonus?.getAttribute("data-auction-price") || link?.getAttribute("data-auction-price"), 10); - if (current > 0) return current; - const priceEl = el.querySelector(".Product__priceValue"); - if (priceEl) { - const m = priceEl.textContent?.replace(/[^0-9]/g, ""); - if (m) return parseInt(m, 10) || 0; +async function scrapeActive(keyword, limit) { + const url = searchUrl(keyword); + const res = await fetch(url, { + headers: { "User-Agent": UA, Accept: "text/html" }, + signal: AbortSignal.timeout(10000), + }); + if (!res.ok) return []; + const html = await res.text(); + const $ = cheerio.load(html); + const results = []; + + $(".Product__detail").each((_, el) => { + if (results.length >= limit) return false; + const $el = $(el); + const link = $el.find(".Product__titleLink"); + const bonus = $el.find(".Product__bonus"); + if (!link.length) return; + + const title = link.attr("data-auction-title") || link.text().trim() || ""; + if (!title) return; + + const auctionId = bonus.attr("data-auction-id") || link.attr("data-auction-id") || ""; + + let price = 0; + const buynow = parseInt(bonus.attr("data-auction-buynowprice"), 10); + if (buynow > 0) { + price = buynow; + } else { + const current = parseInt(bonus.attr("data-auction-price") || link.attr("data-auction-price"), 10); + if (current > 0) { + price = current; + } else { + const priceEl = $el.find(".Product__priceValue"); + if (priceEl.length) { + const m = priceEl.text()?.replace(/[^0-9]/g, ""); + if (m) price = parseInt(m, 10) || 0; } - return 0; } + } - const results = []; - const els = document.querySelectorAll(".Product__detail"); - for (const el of els) { - if (results.length >= lim) break; - const link = el.querySelector(".Product__titleLink"); - const bonus = el.querySelector(".Product__bonus"); - if (!link) continue; - const title = link.getAttribute("data-auction-title") || link.textContent?.trim() || ""; - const auctionId = bonus?.getAttribute("data-auction-id") || link.getAttribute("data-auction-id") || ""; - const price = getPrice(el); - const img = link.getAttribute("data-auction-img") || ""; - const href = link.href || ""; - if (!title) continue; - results.push({ title, auctionId, price, img, href }); - } - return results; - }, { lim: limit }); + const img = link.attr("data-auction-img") || ""; + const href = link.attr("href") || ""; + + results.push({ title, auctionId, price, img, href }); + }); - return items; - } finally { - await page.close(); - } + return results; } -async function scrapeSold(browser, keyword, limit) { - const page = await browser.newPage(); - try { - await page.goto(closedUrl(keyword), { waitUntil: "networkidle", timeout: 45000 }); - await page.waitForTimeout(2000); - - const items = await page.evaluate((lim) => { - const seen = new Set(); - const results = []; - const links = document.querySelectorAll('a[href*="/jp/auction/"]'); - for (const a of links) { - if (results.length >= lim) break; - const title = a.textContent?.trim(); - if (!title || title.length < 5) continue; - const href = a.href || ""; - const auctionId = href.split("/auction/").pop() || ""; - if (seen.has(auctionId)) continue; - seen.add(auctionId); - const clParams = a.getAttribute("data-cl-params") || ""; - const pm = clParams.match(/etc:p=([0-9]+)/); - let price = pm ? parseInt(pm[1], 10) : 0; - if (!price) { - const container = a.closest("[class*='sc-']") || a.parentElement; - const text = container?.textContent || ""; - const ym = text.match(/([0-9,]+)\s*円/); - if (ym) price = parseInt(ym[1].replace(/,/g, ""), 10) || 0; - } - results.push({ title, auctionId, price, href }); - } - return results; - }, limit); +async function scrapeSold(keyword, limit) { + const url = closedUrl(keyword); + const res = await fetch(url, { + headers: { "User-Agent": UA, Accept: "text/html" }, + signal: AbortSignal.timeout(10000), + }); + if (!res.ok) return []; + const html = await res.text(); + const $ = cheerio.load(html); + const seen = new Set(); + const results = []; + + $('a[href*="/jp/auction/"]').each((_, el) => { + if (results.length >= limit) return false; + const $a = $(el); + const title = $a.text().trim(); + if (!title || title.length < 5) return; + + const href = $a.attr("href") || ""; + const auctionId = href.split("/auction/").pop() || ""; + if (seen.has(auctionId)) return; + seen.add(auctionId); + + const clParams = $a.attr("data-cl-params") || ""; + const pm = clParams.match(/etc:p=([0-9]+)/); + let price = pm ? parseInt(pm[1], 10) : 0; + if (!price) { + const container = $a.parent(); + const text = container.text() || ""; + const ym = text.match(/([0-9,]+)\s*円/); + if (ym) price = parseInt(ym[1].replace(/,/g, ""), 10) || 0; + } + + results.push({ title, auctionId, price, href }); + }); - return items; - } finally { - await page.close(); - } + return results; } export async function searchYahooAuctions(card, config, { log = console.log } = {}) { @@ -132,74 +137,66 @@ export async function searchYahooAuctions(card, config, { log = console.log } = log(` yahoo q: ${query}`); - const [jpyPerUsd, browser] = await Promise.all([ + const [jpyPerUsd, activeRaw, soldRaw] = await Promise.all([ fetchJPYRate(), - chromium.launch({ headless: true }), + scrapeActive(query, resultsPerCard), + scrapeSold(query, soldListingsLimit), ]); - try { - const [activeRaw, soldRaw] = await Promise.all([ - scrapeActive(browser, query, resultsPerCard), - scrapeSold(browser, query, soldListingsLimit), - ]); - - const toUSD = (jpy) => (jpy != null && jpy > 0 ? Math.round((jpy / jpyPerUsd) * 100) / 100 : null); - - const auctionUrl = (item) => - item.href || `https://page.auctions.yahoo.co.jp/jp/auction/${item.auctionId}`; - - const active = activeRaw - .map((raw) => ({ - itemId: raw.auctionId, - itemWebUrl: auctionUrl(raw), - title: raw.title, - price: toUSD(raw.price), - priceCurrency: "USD", - priceJPY: raw.price, - shippingLabel: "—", - totalCost: toUSD(raw.price), - listingGradeLabel: gradeFromTitle(raw.title), - shippingToBuyer: Object.fromEntries( - deliveryCountries.map((c) => [c, { eligible: null }]), - ), - grade: null, - })) - .filter((r) => listingFormat !== "raw" || r.listingGradeLabel == null); - - const sold = soldRaw - .map((raw) => ({ - itemId: raw.auctionId, - itemWebUrl: auctionUrl(raw), - title: raw.title, - price: toUSD(raw.price), - currency: "USD", - priceJPY: raw.price, - endedDate: "—", - listingGradeLabel: gradeFromTitle(raw.title), - })) - .filter((r) => listingFormat !== "raw" || r.listingGradeLabel == null); - - log(` yahoo: ${active.length} active, ${sold.length} sold (¥${Math.round(jpyPerUsd)}/USD)`); - - const listingDesc = - listingFormat === "slab" && slab - ? `Yahoo Auctions JP — ${slab.provider} ${slab.grade} (¥${Math.round(jpyPerUsd)}/USD)` - : `Yahoo Auctions JP (¥${Math.round(jpyPerUsd)}/USD)`; - - return { - query: card, - ebaySearchQuery: query, - listingFormat, - listingDescription: listingDesc, - slab: listingFormat === "slab" ? { ...slab } : null, - lang: "jp", - activeByCountry: Object.fromEntries(deliveryCountries.map((c) => [c, active])), - sold, - gradingLabel: "yahoo listing", - counts: { activeTotal: active.length, sold: sold.length }, - source: "yahoo", - }; - } finally { - await browser.close(); - } + const toUSD = (jpy) => (jpy != null && jpy > 0 ? Math.round((jpy / jpyPerUsd) * 100) / 100 : null); + + const auctionUrl = (item) => + item.href || `https://page.auctions.yahoo.co.jp/jp/auction/${item.auctionId}`; + + const active = activeRaw + .map((raw) => ({ + itemId: raw.auctionId, + itemWebUrl: auctionUrl(raw), + title: raw.title, + price: toUSD(raw.price), + priceCurrency: "USD", + priceJPY: raw.price, + shippingLabel: "—", + totalCost: toUSD(raw.price), + listingGradeLabel: gradeFromTitle(raw.title), + shippingToBuyer: Object.fromEntries( + deliveryCountries.map((c) => [c, { eligible: null }]), + ), + grade: null, + })) + .filter((r) => listingFormat !== "raw" || r.listingGradeLabel == null); + + const sold = soldRaw + .map((raw) => ({ + itemId: raw.auctionId, + itemWebUrl: auctionUrl(raw), + title: raw.title, + price: toUSD(raw.price), + currency: "USD", + priceJPY: raw.price, + endedDate: "—", + listingGradeLabel: gradeFromTitle(raw.title), + })) + .filter((r) => listingFormat !== "raw" || r.listingGradeLabel == null); + + log(` yahoo: ${active.length} active, ${sold.length} sold (¥${Math.round(jpyPerUsd)}/USD)`); + + const listingDesc = + listingFormat === "slab" && slab + ? `Yahoo Auctions JP — ${slab.provider} ${slab.grade} (¥${Math.round(jpyPerUsd)}/USD)` + : `Yahoo Auctions JP (¥${Math.round(jpyPerUsd)}/USD)`; + + return { + query: card, + ebaySearchQuery: query, + listingFormat, + listingDescription: listingDesc, + slab: listingFormat === "slab" ? { ...slab } : null, + lang: "jp", + activeByCountry: Object.fromEntries(deliveryCountries.map((c) => [c, active])), + sold, + gradingLabel: "yahoo listing", + counts: { activeTotal: active.length, sold: sold.length }, + source: "yahoo", + }; }