From 08b03b55f390501c706f8fc4ff85d1cd08b55ba4 Mon Sep 17 00:00:00 2001 From: nelson Date: Thu, 30 Apr 2026 19:15:46 +0100 Subject: [PATCH] fix: add home timeline MCP access --- src/mcp/local-tools.js | 105 +++++++++++++++++++++++++++++++++- src/mcp/server.js | 11 ++++ src/scrapers/twitter/index.js | 37 ++++++++---- 3 files changed, 139 insertions(+), 14 deletions(-) diff --git a/src/mcp/local-tools.js b/src/mcp/local-tools.js index 7ad47e4a..77699d09 100644 --- a/src/mcp/local-tools.js +++ b/src/mcp/local-tools.js @@ -61,6 +61,11 @@ async function ensureBrowser() { return { browser, page }; } +export async function getPage() { + const { page } = await ensureBrowser(); + return page; +} + /** * Close browser (called by server.js on SIGINT/SIGTERM) */ @@ -691,13 +696,107 @@ export async function x_auto_like({ keywords = [], maxLikes = 20 }) { export async function x_get_trends({ category, limit = 30 }) { const { page: pg } = await ensureBrowser(); - return scrapeTrending(pg, { limit }); + return scrapeTrending(pg, { category: category || 'trending', limit }); } export async function x_get_explore({ category, limit = 30 }) { const { page: pg } = await ensureBrowser(); // Explore and trending share the same underlying page data - return scrapeTrending(pg, { limit }); + return scrapeTrending(pg, { category: category || 'trending', limit }); +} + +export async function x_get_home_timeline({ timeline = 'for_you', limit = 25 }) { + const { page: pg } = await ensureBrowser(); + + const normalizedTimeline = timeline === 'following' ? 'following' : 'for_you'; + const targetLabel = normalizedTimeline === 'following' ? 'Following' : 'For you'; + + await pg.goto('https://x.com/home', { waitUntil: 'networkidle2', timeout: 30000 }); + await randomDelay(2000, 3000); + await pg.waitForSelector('article[data-testid="tweet"]', { timeout: 30000 }).catch(() => {}); + + await pg.evaluate(async (label) => { + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + const tabs = Array.from(document.querySelectorAll('[data-testid="ScrollSnap-List"] a, [role="tab"]')); + const wanted = tabs.find((tab) => (tab.textContent || '').trim().toLowerCase().includes(label.toLowerCase())); + if (wanted && wanted.getAttribute('aria-selected') !== 'true') { + wanted.click(); + await sleep(2000); + } + }, targetLabel); + + const posts = new Map(); + let emptyScrolls = 0; + + while (posts.size < limit && emptyScrolls < 5) { + const batch = await pg.evaluate(() => { + const parseMetric = (value) => { + if (!value) return 0; + const cleaned = String(value).replace(/,/g, '').trim(); + const match = cleaned.match(/([\d.]+)\s*([KMB]?)/i); + if (!match) return 0; + const num = parseFloat(match[1]); + const suffix = (match[2] || '').toUpperCase(); + const mult = suffix === 'K' ? 1e3 : suffix === 'M' ? 1e6 : suffix === 'B' ? 1e9 : 1; + return Math.round(num * mult); + }; + + return Array.from(document.querySelectorAll('article[data-testid="tweet"]')).map((tweet) => { + const text = tweet.querySelector('[data-testid="tweetText"]')?.textContent?.trim() || ''; + const timeEl = tweet.querySelector('time'); + const statusLink = tweet.querySelector('a[href*="/status/"]'); + const userLinks = Array.from(tweet.querySelectorAll('[data-testid="User-Name"] a[href^="/"]')); + + let username = ''; + let displayName = ''; + for (const link of userLinks) { + const href = link.getAttribute('href') || ''; + if (!href.startsWith('/') || href.includes('/status/')) continue; + if (!username) username = href.replace(/^\//, ''); + if (!displayName) displayName = link.textContent?.trim() || ''; + } + + const replyAria = tweet.querySelector('[data-testid="reply"]')?.getAttribute('aria-label') || ''; + const retweetAria = tweet.querySelector('[data-testid="retweet"]')?.getAttribute('aria-label') || ''; + const likeAria = tweet.querySelector('[data-testid="like"], [data-testid="unlike"]')?.getAttribute('aria-label') || ''; + const viewText = tweet.querySelector('[data-testid="app-text-transition-container"]')?.textContent?.trim() || ''; + const quotedText = Array.from(tweet.querySelectorAll('[data-testid="tweetText"]')).slice(1).map((el) => el.textContent?.trim()).filter(Boolean); + + return { + id: statusLink?.getAttribute('href') || `${username}:${text.slice(0, 80)}`, + username, + displayName, + text, + timestamp: timeEl?.getAttribute('datetime') || '', + url: statusLink ? `https://x.com${statusLink.getAttribute('href')}` : '', + replies: parseMetric(replyAria), + retweets: parseMetric(retweetAria), + likes: parseMetric(likeAria), + views: parseMetric(viewText), + quotedText, + isRepost: !!tweet.querySelector('[data-testid="socialContext"]'), + }; + }).filter((item) => item.id && item.text); + }); + + const before = posts.size; + for (const item of batch) { + if (!posts.has(item.id)) posts.set(item.id, item); + if (posts.size >= limit) break; + } + + emptyScrolls = posts.size === before ? emptyScrolls + 1 : 0; + if (posts.size >= limit) break; + + await pg.evaluate(() => window.scrollBy(0, window.innerHeight * 1.5)); + await sleep(1800); + } + + return { + timeline: normalizedTimeline, + count: Math.min(posts.size, limit), + posts: Array.from(posts.values()).slice(0, limit), + }; } // ============================================================================ @@ -1374,6 +1473,7 @@ export const toolMap = { // Discovery x_get_trends, x_get_explore, + x_get_home_timeline, // Notifications x_get_notifications, x_mute_user, @@ -1413,6 +1513,7 @@ export const toolMap = { x_client_get_followers, x_client_get_trends, // Utility (not an MCP tool, used by server.js cleanup) + getPage, closeBrowser, }; diff --git a/src/mcp/server.js b/src/mcp/server.js index 1f467775..87dbc1aa 100755 --- a/src/mcp/server.js +++ b/src/mcp/server.js @@ -487,6 +487,17 @@ const TOOLS = [ }, }, }, + { + name: 'x_get_home_timeline', + description: 'Read posts from your home timeline on X/Twitter. Supports For You or Following.', + inputSchema: { + type: 'object', + properties: { + timeline: { type: 'string', enum: ['for_you', 'following'], description: 'Which home timeline to read (default: for_you)' }, + limit: { type: 'number', description: 'Maximum posts to collect (default: 25)' }, + }, + }, + }, // ====== Notifications ====== { name: 'x_get_notifications', diff --git a/src/scrapers/twitter/index.js b/src/scrapers/twitter/index.js index ded588d5..996245b3 100644 --- a/src/scrapers/twitter/index.js +++ b/src/scrapers/twitter/index.js @@ -768,27 +768,40 @@ export async function scrapeNotifications(page, options = {}) { * Scrape trending topics from the Explore page */ export async function scrapeTrending(page, options = {}) { - const { limit = 30 } = options; - - await page.goto('https://x.com/explore/tabs/trending', { waitUntil: 'networkidle2' }); + const { category = 'trending', limit = 30 } = options; + const tab = ({ + trending: 'trending', + news: 'news', + sports: 'sports', + entertainment: 'entertainment', + for_you: 'foryou', + 'for-you': 'foryou', + })[category] || 'trending'; + + await page.goto(`https://x.com/explore/tabs/${tab}`, { waitUntil: 'networkidle2' }); await randomDelay(2000, 3000); - + for (let i = 0; i < 3; i++) { await page.evaluate(() => window.scrollBy(0, window.innerHeight)); await sleep(1500); } - + const trends = await page.$$eval('[data-testid="trend"]', (els) => els.map((el) => { const spans = el.querySelectorAll('span'); - const texts = Array.from(spans).map(s => s.innerText).filter(Boolean); - const category = texts[0] || ''; - const topic = texts.find(t => t.startsWith('#') || t.length > 3) || texts[1] || ''; - const posts = texts.find(t => /posts|tweets/i.test(t)) || ''; - return { category, topic, posts, platform: 'twitter' }; - }) + const texts = Array.from(spans).map((s) => s.innerText.trim()).filter(Boolean); + const posts = texts.find((t) => /posts|tweets/i.test(t)) || ''; + const rank = texts.find((t) => /^\d+$/.test(t)) || ''; + const context = texts.find((t) => /trending|news|sports|entertainment|for you/i.test(t)) || ''; + const topic = texts.find((t, idx) => idx > 1 && !/^\d+$/.test(t) && !/posts|tweets/i.test(t) && !/trending|news|sports|entertainment|for you/i.test(t)) + || texts.find((t) => t.startsWith('#')) + || texts[2] + || texts[1] + || ''; + return { rank, category: context, topic, posts, platform: 'twitter' }; + }).filter((item) => item.topic) ); - + return trends.slice(0, limit); }