From 091b00ef3435fa8515af00da0acdf1cb62cb8b23 Mon Sep 17 00:00:00 2001 From: Jonas Date: Tue, 10 Jun 2025 11:34:49 +1200 Subject: [PATCH 1/5] derjogi Fix: Respect 'following' timeline mode on timeline population #19 --- README.md | 20 +++++++++++++++++--- src/base.ts | 10 +++++++++- src/environment.ts | 3 +++ src/timeline.ts | 2 +- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index af56303..ee2313b 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Create or edit `.env` file in your project root: ```bash # Required Twitter API v2 Credentials TWITTER_API_KEY= # Your Twitter API Key -TWITTER_API_SECRET_KEY= # Your Twitter API Secret Key +TWITTER_API_SECRET_KEY= # Your Twitter API Secret Key TWITTER_ACCESS_TOKEN= # Your Access Token TWITTER_ACCESS_TOKEN_SECRET= # Your Access Token Secret @@ -55,9 +55,10 @@ TWITTER_AUTO_RESPOND_REPLIES=true # Automatically respond to replies TWITTER_MAX_INTERACTIONS_PER_RUN=10 # Maximum interactions processed per cycle # Timeline Algorithm Configuration +TWITTER_TIMELINE_MODE= # Timeline type: "following" or "foryou" (default: foryou) TWITTER_TIMELINE_ALGORITHM=weighted # Algorithm: "weighted" or "latest" TWITTER_TIMELINE_USER_BASED_WEIGHT=3 # Weight for user-based scoring -TWITTER_TIMELINE_TIME_BASED_WEIGHT=2 # Weight for time-based scoring +TWITTER_TIMELINE_TIME_BASED_WEIGHT=2 # Weight for time-based scoring TWITTER_TIMELINE_RELEVANCE_WEIGHT=5 # Weight for relevance scoring # Advanced Settings @@ -114,6 +115,7 @@ When `TWITTER_POST_ENABLE=true`, the client automatically generates and posts tw ### Timeline Monitoring The client monitors and processes the Twitter timeline: +- **Timeline Mode**: Choose between "following" (users you follow) or "foryou" (algorithmic feed) - **Weighted Algorithm**: Scores tweets based on user relationships, time, and relevance - **Latest Algorithm**: Processes tweets in chronological order - Configurable interaction limits and intervals @@ -179,6 +181,18 @@ TWITTER_POST_INTERVAL_MAX=180 TWITTER_POST_INTERVAL_VARIANCE=0.2 ``` +### Timeline Mode Configuration + +**Following Mode** (`TWITTER_TIMELINE_MODE=following`): +- Shows only tweets from accounts you follow +- More focused, targeted content +- Smaller tweet volume + +**For You Mode** (`TWITTER_TIMELINE_MODE=foryou` or empty): +- Shows algorithmic timeline with recommendations +- Includes trending topics and suggested content +- Higher tweet volume + ## Development ### Testing @@ -187,7 +201,7 @@ TWITTER_POST_INTERVAL_VARIANCE=0.2 # Run tests bun test -# Run with debug logging +# Run with debug logging DEBUG=eliza:* bun start # Test without posting diff --git a/src/base.ts b/src/base.ts index 4a579d4..bcc5ca1 100644 --- a/src/base.ts +++ b/src/base.ts @@ -15,6 +15,7 @@ import { type Tweet, } from "./client/index"; import { TwitterInteractionPayload } from "./types"; +import { TIMELINE_TYPE } from "./timeline"; interface TwitterUser { id_str: string; @@ -639,7 +640,14 @@ export class ClientBase { } } - const timeline = await this.fetchHomeTimeline(cachedTimeline ? 10 : 50); + // Check if user wants 'following' timeline vs 'for you' algorithmic feed + const following = + (this.state?.TWITTER_TIMELINE_MODE || + this.runtime.getSetting("TWITTER_TIMELINE_MODE")) + === TIMELINE_TYPE.Following; + + // Fetch timeline: 10 if cached, 20 for following, 50 for algorithmic feed + const timeline = await this.fetchHomeTimeline(cachedTimeline ? 10 : following ? 20 : 50, following); // Get the most recent 20 mentions and interactions const mentionsAndInteractions = await this.fetchSearchTweets( diff --git a/src/environment.ts b/src/environment.ts index 55ee738..6b0cda1 100644 --- a/src/environment.ts +++ b/src/environment.ts @@ -13,6 +13,7 @@ export const twitterEnvSchema = z.object({ TWITTER_ACCESS_TOKEN: z.string().optional(), TWITTER_ACCESS_TOKEN_SECRET: z.string().optional(), TWITTER_TARGET_USERS: z.string().default(""), + TWITTER_TIMELINE_MODE: z.string().refine(val => !val || val === 'following' || val === 'foryou', 'Timeline mode must be either "following", "foryou", or empty').default(""), TWITTER_RETRY_LIMIT: z.string().default("5"), TWITTER_POLL_INTERVAL: z.string().default("120"), TWITTER_SEARCH_ENABLE: z.string().default("true"), @@ -153,6 +154,7 @@ export async function validateTwitterConfig( TWITTER_INTERACTION_INTERVAL_MAX: String( safeParseInt(getConfig("TWITTER_INTERACTION_INTERVAL_MAX"), 30), ), + TWITTER_TIMELINE_MODE: getConfig("TWITTER_TIMELINE_MODE") || "", TWITTER_TIMELINE_ALGORITHM: (getConfig("TWITTER_TIMELINE_ALGORITHM") === "latest" ? "latest" @@ -314,6 +316,7 @@ function getDefaultConfig(): TwitterConfig { getConfig("TWITTER_INTERACTION_INTERVAL_MIN") || "15", TWITTER_INTERACTION_INTERVAL_MAX: getConfig("TWITTER_INTERACTION_INTERVAL_MAX") || "30", + TWITTER_TIMELINE_MODE: getConfig("TWITTER_TIMELINE_MODE") || "", TWITTER_TIMELINE_ALGORITHM: (getConfig("TWITTER_TIMELINE_ALGORITHM") === "latest" ? "latest" diff --git a/src/timeline.ts b/src/timeline.ts index ebb6b1c..42ab199 100644 --- a/src/timeline.ts +++ b/src/timeline.ts @@ -21,7 +21,7 @@ import { import { sendTweet, parseActionResponseFromText } from "./utils"; import { ActionResponse } from "./types"; -enum TIMELINE_TYPE { +export enum TIMELINE_TYPE { ForYou = "foryou", Following = "following", } From a51eef51e82f61fa8d319ea39f45961f94744588 Mon Sep 17 00:00:00 2001 From: Jonas Date: Wed, 11 Jun 2025 15:45:18 +1200 Subject: [PATCH 2/5] derjogi Fix Tweet Parsing #23 * Often tweets are nested within an additional top-level 'tweet', which resulted in all values being 'undefined' * Better 'text': For re-tweets, the full_text is often truncated, but the original tweet has the full text, so prefer taking that * General: Use the original tweet (if available) as fallback for 'undefined' values --- src/timeline.ts | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/timeline.ts b/src/timeline.ts index 42ab199..de83b81 100644 --- a/src/timeline.ts +++ b/src/timeline.ts @@ -69,33 +69,47 @@ export class TwitterTimelineClient { : await this.twitterClient.fetchHomeTimeline(count, []); return homeTimeline - .map((tweet) => ({ - id: tweet.rest_id, - name: tweet.core?.user_results?.result?.legacy?.name, - username: tweet.core?.user_results?.result?.legacy?.screen_name, - text: tweet.legacy?.full_text, - inReplyToStatusId: tweet.legacy?.in_reply_to_status_id_str, - timestamp: new Date(tweet.legacy?.created_at).getTime() / 1000, - userId: tweet.legacy?.user_id_str, - conversationId: tweet.legacy?.conversation_id_str, - permanentUrl: `https://twitter.com/${tweet.core?.user_results?.result?.legacy?.screen_name}/status/${tweet.rest_id}`, - hashtags: tweet.legacy?.entities?.hashtags || [], - mentions: tweet.legacy?.entities?.user_mentions || [], - photos: - tweet.legacy?.entities?.media + .map((_tweet) => { + // First, check whether the tweet is nested within another top-level 'tweet' (which it appears is sometimes(?) the case): + let tweet = _tweet.tweet || _tweet; + // If the tweet is a retweet, get the original tweet, that often has a more complete text: + const orig = tweet.legacy?.retweeted_status_result?.result; + const timelineTweet = { + id: getValue(tweet, orig, "rest_id"), + name: getValue(tweet, orig, "core.user_results.result.legacy.name"), + username: getValue(tweet, orig, "core.user_results.result.legacy.screen_name"), + text: getValue(orig, tweet, "legacy.full_text"), + inReplyToStatusId: getValue(tweet, orig, "legacy.in_reply_to_status_id_str"), + timestamp: new Date(getValue(tweet, orig, "legacy.created_at")).getTime() / 1000, + userId: getValue(tweet, orig, "legacy.user_id_str"), + conversationId: getValue(tweet, orig, "legacy.conversation_id_str"), + permanentUrl: `https://twitter.com/${getValue(tweet, orig, "core.user_results.result.legacy.screen_name")}/status/${getValue(tweet, orig, "rest_id")}`, + hashtags: getValue(tweet, orig, "legacy.entities.hashtags") || [], + mentions: getValue(tweet, orig, "legacy.entities.user_mentions") || [], + photos: getValue(tweet, orig, "legacy.entities.media") ?.filter((media) => media.type === "photo") .map((media) => ({ id: media.id_str, url: media.media_url_https, // Store media_url_https as url alt_text: media.alt_text, })) || [], + thread: getValue(tweet, orig, "thread") || [], + urls: getValue(tweet, orig, "legacy.entities.urls") || [], + videos: getValue(tweet, orig, "legacy.entities.media")?.filter( + (media) => media.type === "video" thread: tweet.thread || [], urls: tweet.legacy?.entities?.urls || [], videos: tweet.legacy?.entities?.media?.filter( (media) => media.type === "video", ) || [], - })) + } + if (!timelineTweet.id || !timelineTweet.name || !timelineTweet.username) { + logger.debug("Missing tweet data:", timelineTweet); + logger.debug("Original tweet data:", tweet); + } + return timelineTweet; + }) .filter((tweet) => tweet.username !== twitterUsername); // do not perform action on self-tweets } From 9aa83c441fe25f31453f9ba27af9fb44895046c8 Mon Sep 17 00:00:00 2001 From: Jonas Date: Wed, 11 Jun 2025 15:46:16 +1200 Subject: [PATCH 3/5] Limit processed tweets on top of what we receive from twitter --- package.json | 7 +++++++ src/timeline.ts | 23 ++++++++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 971dcec..1d82b9b 100644 --- a/package.json +++ b/package.json @@ -242,6 +242,13 @@ "required": false, "default": true, "sensitive": false + }, + "TWITTER_MAX_ACTIONS_BATCH_SIZE": { + "type": "number", + "description": "Maximum number of tweets to process and possibly interact with in a single batch.", + "required": false, + "default": 20, + "sensitive": false } } } diff --git a/src/timeline.ts b/src/timeline.ts index de83b81..91efd56 100644 --- a/src/timeline.ts +++ b/src/timeline.ts @@ -139,23 +139,32 @@ export class TwitterTimelineClient { } async handleTimeline() { - console.log("Start Hanldeling Twitter Timeline"); + console.log("Start Handling Twitter Timeline"); const tweets = await this.getTimeline(20); - const maxActionsPerCycle = 20; + const maxActionsPerCycle = this.state?.TWITTER_MAX_ACTIONS_BATCH_SIZE + || this.runtime.getSetting("TWITTER_MAX_ACTIONS_BATCH_SIZE") + || 20; const tweetDecisions = []; + let count = 0; + for (const tweet of tweets) { try { const tweetId = this.createTweetId(this.runtime, tweet); // Skip if we've already processed this tweet const memory = await this.runtime.getMemoryById(tweetId); if (memory) { - console.log(`Already processed tweet ID: ${tweet.id}`); + logger.log(`Already processed tweet ID: ${tweet.id}`); continue; } - - const roomId = createUniqueUuid(this.runtime, tweet.conversationId); - + count++; + if (count > maxActionsPerCycle) { + // Count in getTimeline(#) sometimes doesn't seem to have an effect and it might retrieve more than we want. + // Stop processing if we've reached the max number of tweets to process. + break; + } + logger.debug(`Processing tweet #${count} of ${Math.min(tweets.length, maxActionsPerCycle)})`); + const message = this.formMessage(this.runtime, tweet); let state = await this.runtime.composeState(message); @@ -193,7 +202,7 @@ Choose any combination of [LIKE], [RETWEET], [QUOTE], and [REPLY] that are appro tweet: tweet, actionResponse: actions, tweetState: state, - roomId: roomId, + roomId: message.roomId, }); } catch (error) { logger.error(`Error processing tweet ${tweet.id}:`, error); From 7dc677f7bfdc4bf481848d1eba54a0bf97c7c041 Mon Sep 17 00:00:00 2001 From: gnomonprime Date: Sat, 28 Jun 2025 22:28:06 +0930 Subject: [PATCH 4/5] add missing getValue function from #23 --- src/timeline.ts | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/timeline.ts b/src/timeline.ts index 91efd56..a4557de 100644 --- a/src/timeline.ts +++ b/src/timeline.ts @@ -68,6 +68,32 @@ export class TwitterTimelineClient { ? await this.twitterClient.fetchFollowingTimeline(count, []) : await this.twitterClient.fetchHomeTimeline(count, []); + function getValue(primary: any, fallback: any, path: string): T | undefined { + const pathParts = path.split('.'); + + for (const obj of [primary, fallback]) { + if (obj == null) continue; + + let current = obj; + let isValid = true; + + for (const part of pathParts) { + if (current == null || typeof current !== 'object') { + isValid = false; + break; + } + + current = current[part]; + } + + if (isValid && current != null) { + return current as T; + } + } + + return undefined; + } + return homeTimeline .map((_tweet) => { // First, check whether the tweet is nested within another top-level 'tweet' (which it appears is sometimes(?) the case): @@ -97,11 +123,6 @@ export class TwitterTimelineClient { urls: getValue(tweet, orig, "legacy.entities.urls") || [], videos: getValue(tweet, orig, "legacy.entities.media")?.filter( (media) => media.type === "video" - thread: tweet.thread || [], - urls: tweet.legacy?.entities?.urls || [], - videos: - tweet.legacy?.entities?.media?.filter( - (media) => media.type === "video", ) || [], } if (!timelineTweet.id || !timelineTweet.name || !timelineTweet.username) { @@ -164,7 +185,7 @@ export class TwitterTimelineClient { break; } logger.debug(`Processing tweet #${count} of ${Math.min(tweets.length, maxActionsPerCycle)})`); - + const message = this.formMessage(this.runtime, tweet); let state = await this.runtime.composeState(message); From 0572eb787f6ec0dc90968015e05349d2c1e393c7 Mon Sep 17 00:00:00 2001 From: gnomonprime Date: Sat, 28 Jun 2025 22:31:47 +0930 Subject: [PATCH 5/5] derjogi Limit processed tweets on top of what we receive from twitter #24 --- README.md | 6 ++++-- package.json | 7 +++++++ src/environment.ts | 6 ++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee2313b..ff6c2a2 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ TWITTER_INTERACTION_INTERVAL_MAX=30 # Maximum interval between interactions ( TWITTER_INTERACTION_INTERVAL_VARIANCE=0.3 # Random variance for interaction intervals TWITTER_AUTO_RESPOND_MENTIONS=true # Automatically respond to mentions TWITTER_AUTO_RESPOND_REPLIES=true # Automatically respond to replies -TWITTER_MAX_INTERACTIONS_PER_RUN=10 # Maximum interactions processed per cycle +TWITTER_MAX_INTERACTIONS_PER_RUN=10 # Maximum mentions/replies processed per cycle +TWITTER_MAX_ACTIONS_BATCH_SIZE=20 # Maximum timeline tweets to analyze for actions per cycle # Timeline Algorithm Configuration TWITTER_TIMELINE_MODE= # Timeline type: "following" or "foryou" (default: foryou) @@ -225,7 +226,8 @@ TWITTER_DRY_RUN=true bun start - Verify `TWITTER_SEARCH_ENABLE=true` - Check `TWITTER_TARGET_USERS` configuration - Ensure the timeline contains relevant content -- Review `TWITTER_MAX_INTERACTIONS_PER_RUN` setting +- Review `TWITTER_MAX_INTERACTIONS_PER_RUN` setting for mention/reply processing +- Review `TWITTER_MAX_ACTIONS_BATCH_SIZE` setting for timeline action processing #### Timeline Issues - Try switching between "weighted" and "latest" algorithms diff --git a/package.json b/package.json index 1d82b9b..1109036 100644 --- a/package.json +++ b/package.json @@ -166,6 +166,13 @@ "default": 0.3, "sensitive": false }, + "TWITTER_TIMELINE_MODE": { + "type": "string", + "description": "Timeline type: 'following' (users you follow) or 'foryou' (algorithmic feed). Empty defaults to 'foryou'.", + "required": false, + "default": "", + "sensitive": false + }, "TWITTER_TIMELINE_ALGORITHM": { "type": "string", "description": "Timeline processing algorithm: 'weighted' (default) or 'latest'.", diff --git a/src/environment.ts b/src/environment.ts index 6b0cda1..6e982d0 100644 --- a/src/environment.ts +++ b/src/environment.ts @@ -32,6 +32,7 @@ export const twitterEnvSchema = z.object({ TWITTER_TIMELINE_RELEVANCE_WEIGHT: z.string().default("5"), TWITTER_MAX_TWEET_LENGTH: z.string().default("4000"), TWITTER_MAX_INTERACTIONS_PER_RUN: z.string().default("10"), + TWITTER_MAX_ACTIONS_BATCH_SIZE: z.string().default("20"), TWITTER_DM_ONLY: z.string().default("false"), TWITTER_ENABLE_ACTION_PROCESSING: z.string().default("false"), TWITTER_ACTION_INTERVAL: z.string().default("240"), @@ -174,6 +175,9 @@ export async function validateTwitterConfig( TWITTER_MAX_INTERACTIONS_PER_RUN: String( safeParseInt(getConfig("TWITTER_MAX_INTERACTIONS_PER_RUN"), 10), ), + TWITTER_MAX_ACTIONS_BATCH_SIZE: String( + safeParseInt(getConfig("TWITTER_MAX_ACTIONS_BATCH_SIZE"), 20), + ), TWITTER_DM_ONLY: String( getConfig("TWITTER_DM_ONLY")?.toLowerCase() === "true", ), @@ -330,6 +334,8 @@ function getDefaultConfig(): TwitterConfig { TWITTER_MAX_TWEET_LENGTH: getConfig("TWITTER_MAX_TWEET_LENGTH") || "4000", TWITTER_MAX_INTERACTIONS_PER_RUN: getConfig("TWITTER_MAX_INTERACTIONS_PER_RUN") || "10", + TWITTER_MAX_ACTIONS_BATCH_SIZE: + getConfig("TWITTER_MAX_ACTIONS_BATCH_SIZE") || "20", TWITTER_DM_ONLY: getConfig("TWITTER_DM_ONLY") || "false", TWITTER_ENABLE_ACTION_PROCESSING: getConfig("TWITTER_ENABLE_ACTION_PROCESSING") || "false",