derjogi fixes

README.md

@@ -28,7 +28,7 @@ Create or edit `.env` file in your project root:
 ```bash
 # Required Twitter API v2 Credentials
 TWITTER_API_KEY=                    # Your Twitter API Key
-TWITTER_API_SECRET_KEY=             # Your Twitter API Secret Key  
+TWITTER_API_SECRET_KEY=             # Your Twitter API Secret Key
 TWITTER_ACCESS_TOKEN=               # Your Access Token
 TWITTER_ACCESS_TOKEN_SECRET=        # Your Access Token Secret
 
@@ -52,12 +52,14 @@ TWITTER_INTERACTION_INTERVAL_MAX=30    # Maximum interval between interactions (
 TWITTER_INTERACTION_INTERVAL_VARIANCE=0.3  # Random variance for interaction intervals
 TWITTER_AUTO_RESPOND_MENTIONS=true     # Automatically respond to mentions
 TWITTER_AUTO_RESPOND_REPLIES=true      # Automatically respond to replies
-TWITTER_MAX_INTERACTIONS_PER_RUN=10    # Maximum interactions processed per cycle
+TWITTER_MAX_INTERACTIONS_PER_RUN=10    # Maximum mentions/replies processed per cycle
+TWITTER_MAX_ACTIONS_BATCH_SIZE=20      # Maximum timeline tweets to analyze for actions per cycle
 
 # Timeline Algorithm Configuration
+TWITTER_TIMELINE_MODE=             # Timeline type: "following" or "foryou" (default: foryou)
 TWITTER_TIMELINE_ALGORITHM=weighted    # Algorithm: "weighted" or "latest"
 TWITTER_TIMELINE_USER_BASED_WEIGHT=3   # Weight for user-based scoring
-TWITTER_TIMELINE_TIME_BASED_WEIGHT=2   # Weight for time-based scoring  
+TWITTER_TIMELINE_TIME_BASED_WEIGHT=2   # Weight for time-based scoring
 TWITTER_TIMELINE_RELEVANCE_WEIGHT=5    # Weight for relevance scoring
 
 # Advanced Settings
@@ -114,6 +116,7 @@ When `TWITTER_POST_ENABLE=true`, the client automatically generates and posts tw
 ### Timeline Monitoring
 
 The client monitors and processes the Twitter timeline:
+- **Timeline Mode**: Choose between "following" (users you follow) or "foryou" (algorithmic feed)
 - **Weighted Algorithm**: Scores tweets based on user relationships, time, and relevance
 - **Latest Algorithm**: Processes tweets in chronological order
 - Configurable interaction limits and intervals
@@ -179,6 +182,18 @@ TWITTER_POST_INTERVAL_MAX=180
 TWITTER_POST_INTERVAL_VARIANCE=0.2
 ```
 
+### Timeline Mode Configuration
+
+**Following Mode** (`TWITTER_TIMELINE_MODE=following`):
+- Shows only tweets from accounts you follow
+- More focused, targeted content
+- Smaller tweet volume
+
+**For You Mode** (`TWITTER_TIMELINE_MODE=foryou` or empty):
+- Shows algorithmic timeline with recommendations
+- Includes trending topics and suggested content
+- Higher tweet volume
+
 ## Development
 
 ### Testing
@@ -187,7 +202,7 @@ TWITTER_POST_INTERVAL_VARIANCE=0.2
 # Run tests
 bun test
 
-# Run with debug logging  
+# Run with debug logging
 DEBUG=eliza:* bun start
 
 # Test without posting
@@ -211,7 +226,8 @@ TWITTER_DRY_RUN=true bun start
 - Verify `TWITTER_SEARCH_ENABLE=true`
 - Check `TWITTER_TARGET_USERS` configuration
 - Ensure the timeline contains relevant content
-- Review `TWITTER_MAX_INTERACTIONS_PER_RUN` setting
+- Review `TWITTER_MAX_INTERACTIONS_PER_RUN` setting for mention/reply processing
+- Review `TWITTER_MAX_ACTIONS_BATCH_SIZE` setting for timeline action processing
 
 #### Timeline Issues
 - Try switching between "weighted" and "latest" algorithms

package.json

@@ -166,6 +166,13 @@
         "default": 0.3,
         "sensitive": false
       },
+      "TWITTER_TIMELINE_MODE": {
+        "type": "string",
+        "description": "Timeline type: 'following' (users you follow) or 'foryou' (algorithmic feed). Empty defaults to 'foryou'.",
+        "required": false,
+        "default": "",
+        "sensitive": false
+      },
       "TWITTER_TIMELINE_ALGORITHM": {
         "type": "string",
         "description": "Timeline processing algorithm: 'weighted' (default) or 'latest'.",
@@ -242,6 +249,13 @@
         "required": false,
         "default": true,
         "sensitive": false
+      },
+      "TWITTER_MAX_ACTIONS_BATCH_SIZE": {
+        "type": "number",
+        "description": "Maximum number of tweets to process and possibly interact with in a single batch.",
+        "required": false,
+        "default": 20,
+        "sensitive": false
       }
     }
   }

src/base.ts

@@ -15,6 +15,7 @@ import {
   type Tweet,
 } from "./client/index";
 import { TwitterInteractionPayload } from "./types";
+import { TIMELINE_TYPE } from "./timeline";
 
 interface TwitterUser {
   id_str: string;
@@ -639,7 +640,14 @@ export class ClientBase {
       }
     }
 
-    const timeline = await this.fetchHomeTimeline(cachedTimeline ? 10 : 50);
+    // Check if user wants 'following' timeline vs 'for you' algorithmic feed
+    const following =
+      (this.state?.TWITTER_TIMELINE_MODE ||
+       this.runtime.getSetting("TWITTER_TIMELINE_MODE"))
+      === TIMELINE_TYPE.Following;
+
+    // Fetch timeline: 10 if cached, 20 for following, 50 for algorithmic feed
+    const timeline = await this.fetchHomeTimeline(cachedTimeline ? 10 : following ? 20 : 50, following);
 
     // Get the most recent 20 mentions and interactions
     const mentionsAndInteractions = await this.fetchSearchTweets(

src/environment.ts

@@ -13,6 +13,7 @@ export const twitterEnvSchema = z.object({
   TWITTER_ACCESS_TOKEN: z.string().optional(),
   TWITTER_ACCESS_TOKEN_SECRET: z.string().optional(),
   TWITTER_TARGET_USERS: z.string().default(""),
+  TWITTER_TIMELINE_MODE: z.string().refine(val => !val || val === 'following' || val === 'foryou', 'Timeline mode must be either "following", "foryou", or empty').default(""),
   TWITTER_RETRY_LIMIT: z.string().default("5"),
   TWITTER_POLL_INTERVAL: z.string().default("120"),
   TWITTER_SEARCH_ENABLE: z.string().default("true"),
@@ -31,6 +32,7 @@ export const twitterEnvSchema = z.object({
   TWITTER_TIMELINE_RELEVANCE_WEIGHT: z.string().default("5"),
   TWITTER_MAX_TWEET_LENGTH: z.string().default("4000"),
   TWITTER_MAX_INTERACTIONS_PER_RUN: z.string().default("10"),
+  TWITTER_MAX_ACTIONS_BATCH_SIZE: z.string().default("20"),
   TWITTER_DM_ONLY: z.string().default("false"),
   TWITTER_ENABLE_ACTION_PROCESSING: z.string().default("false"),
   TWITTER_ACTION_INTERVAL: z.string().default("240"),
@@ -153,6 +155,7 @@ export async function validateTwitterConfig(
       TWITTER_INTERACTION_INTERVAL_MAX: String(
         safeParseInt(getConfig("TWITTER_INTERACTION_INTERVAL_MAX"), 30),
       ),
+      TWITTER_TIMELINE_MODE: getConfig("TWITTER_TIMELINE_MODE") || "",
       TWITTER_TIMELINE_ALGORITHM: (getConfig("TWITTER_TIMELINE_ALGORITHM") ===
       "latest"
         ? "latest"
@@ -172,6 +175,9 @@ export async function validateTwitterConfig(
       TWITTER_MAX_INTERACTIONS_PER_RUN: String(
         safeParseInt(getConfig("TWITTER_MAX_INTERACTIONS_PER_RUN"), 10),
       ),
+      TWITTER_MAX_ACTIONS_BATCH_SIZE: String(
+        safeParseInt(getConfig("TWITTER_MAX_ACTIONS_BATCH_SIZE"), 20),
+      ),
       TWITTER_DM_ONLY: String(
         getConfig("TWITTER_DM_ONLY")?.toLowerCase() === "true",
       ),
@@ -314,6 +320,7 @@ function getDefaultConfig(): TwitterConfig {
       getConfig("TWITTER_INTERACTION_INTERVAL_MIN") || "15",
     TWITTER_INTERACTION_INTERVAL_MAX:
       getConfig("TWITTER_INTERACTION_INTERVAL_MAX") || "30",
+    TWITTER_TIMELINE_MODE: getConfig("TWITTER_TIMELINE_MODE") || "",
     TWITTER_TIMELINE_ALGORITHM: (getConfig("TWITTER_TIMELINE_ALGORITHM") ===
     "latest"
       ? "latest"
@@ -327,6 +334,8 @@ function getDefaultConfig(): TwitterConfig {
     TWITTER_MAX_TWEET_LENGTH: getConfig("TWITTER_MAX_TWEET_LENGTH") || "4000",
     TWITTER_MAX_INTERACTIONS_PER_RUN:
       getConfig("TWITTER_MAX_INTERACTIONS_PER_RUN") || "10",
+    TWITTER_MAX_ACTIONS_BATCH_SIZE:
+      getConfig("TWITTER_MAX_ACTIONS_BATCH_SIZE") || "20",
     TWITTER_DM_ONLY: getConfig("TWITTER_DM_ONLY") || "false",
     TWITTER_ENABLE_ACTION_PROCESSING:
       getConfig("TWITTER_ENABLE_ACTION_PROCESSING") || "false",

src/timeline.ts

@@ -21,7 +21,7 @@ import {
 import { sendTweet, parseActionResponseFromText } from "./utils";
 import { ActionResponse } from "./types";
 
-enum TIMELINE_TYPE {
+export enum TIMELINE_TYPE {
   ForYou = "foryou",
   Following = "following",
 }
@@ -68,34 +68,69 @@ export class TwitterTimelineClient {
         ? await this.twitterClient.fetchFollowingTimeline(count, [])
         : await this.twitterClient.fetchHomeTimeline(count, []);
 
+    function getValue<T = any>(primary: any, fallback: any, path: string): T | undefined {
+       const pathParts = path.split('.');
+
+       for (const obj of [primary, fallback]) {
+         if (obj == null) continue;
+
+         let current = obj;
+         let isValid = true;
+
+         for (const part of pathParts) {
+           if (current == null || typeof current !== 'object') {
+             isValid = false;
+             break;
+           }
+
+           current = current[part];
+         }
+
+         if (isValid && current != null) {
+           return current as T;
+         }
+       }
+
+       return undefined;
+    }
+
     return homeTimeline
-      .map((tweet) => ({
-        id: tweet.rest_id,
-        name: tweet.core?.user_results?.result?.legacy?.name,
-        username: tweet.core?.user_results?.result?.legacy?.screen_name,
-        text: tweet.legacy?.full_text,
-        inReplyToStatusId: tweet.legacy?.in_reply_to_status_id_str,
-        timestamp: new Date(tweet.legacy?.created_at).getTime() / 1000,
-        userId: tweet.legacy?.user_id_str,
-        conversationId: tweet.legacy?.conversation_id_str,
-        permanentUrl: `https://twitter.com/${tweet.core?.user_results?.result?.legacy?.screen_name}/status/${tweet.rest_id}`,
-        hashtags: tweet.legacy?.entities?.hashtags || [],
-        mentions: tweet.legacy?.entities?.user_mentions || [],
-        photos:
-          tweet.legacy?.entities?.media
+      .map((_tweet) => {
+        // First, check whether the tweet is nested within another top-level 'tweet' (which it appears is sometimes(?) the case):
+        let tweet = _tweet.tweet || _tweet;
+        // If the tweet is a retweet, get the original tweet, that often has a more complete text:
+        const orig = tweet.legacy?.retweeted_status_result?.result;
+        const timelineTweet = {
+          id: getValue(tweet, orig, "rest_id"),
+          name: getValue(tweet, orig, "core.user_results.result.legacy.name"),
+          username: getValue(tweet, orig, "core.user_results.result.legacy.screen_name"),
+          text: getValue(orig, tweet, "legacy.full_text"),
+          inReplyToStatusId: getValue(tweet, orig, "legacy.in_reply_to_status_id_str"),
+          timestamp: new Date(getValue(tweet, orig, "legacy.created_at")).getTime() / 1000,
+          userId: getValue(tweet, orig, "legacy.user_id_str"),
+          conversationId: getValue(tweet, orig, "legacy.conversation_id_str"),
+          permanentUrl: `https://twitter.com/${getValue(tweet, orig, "core.user_results.result.legacy.screen_name")}/status/${getValue(tweet, orig, "rest_id")}`,
+          hashtags: getValue(tweet, orig, "legacy.entities.hashtags") || [],
+          mentions: getValue(tweet, orig, "legacy.entities.user_mentions") || [],
+          photos: getValue(tweet, orig, "legacy.entities.media")
             ?.filter((media) => media.type === "photo")
             .map((media) => ({
               id: media.id_str,
               url: media.media_url_https, // Store media_url_https as url
               alt_text: media.alt_text,
             })) || [],
-        thread: tweet.thread || [],
-        urls: tweet.legacy?.entities?.urls || [],
-        videos:
-          tweet.legacy?.entities?.media?.filter(
-            (media) => media.type === "video",
+          thread: getValue(tweet, orig, "thread") || [],
+          urls: getValue(tweet, orig, "legacy.entities.urls") || [],
+          videos: getValue(tweet, orig, "legacy.entities.media")?.filter(
+            (media) => media.type === "video"
           ) || [],
-      }))
+        }
+        if (!timelineTweet.id || !timelineTweet.name || !timelineTweet.username) {
+          logger.debug("Missing tweet data:", timelineTweet);
+          logger.debug("Original tweet data:", tweet);
+        }
+        return timelineTweet;
+      })
       .filter((tweet) => tweet.username !== twitterUsername); // do not perform action on self-tweets
   }
 
@@ -125,22 +160,31 @@ export class TwitterTimelineClient {
   }
 
   async handleTimeline() {
-    console.log("Start Hanldeling Twitter Timeline");
+    console.log("Start Handling Twitter Timeline");
 
     const tweets = await this.getTimeline(20);
-    const maxActionsPerCycle = 20;
+    const maxActionsPerCycle = this.state?.TWITTER_MAX_ACTIONS_BATCH_SIZE
+      || this.runtime.getSetting("TWITTER_MAX_ACTIONS_BATCH_SIZE")
+      || 20;
     const tweetDecisions = [];
+    let count = 0;
+
     for (const tweet of tweets) {
       try {
         const tweetId = this.createTweetId(this.runtime, tweet);
         // Skip if we've already processed this tweet
         const memory = await this.runtime.getMemoryById(tweetId);
         if (memory) {
-          console.log(`Already processed tweet ID: ${tweet.id}`);
+          logger.log(`Already processed tweet ID: ${tweet.id}`);
           continue;
         }
-
-        const roomId = createUniqueUuid(this.runtime, tweet.conversationId);
+        count++;
+        if (count > maxActionsPerCycle) {
+          // Count in getTimeline(#) sometimes doesn't seem to have an effect and it might retrieve more than we want.
+          // Stop processing if we've reached the max number of tweets to process.
+          break;
+        }
+        logger.debug(`Processing tweet #${count} of ${Math.min(tweets.length, maxActionsPerCycle)})`);
 
         const message = this.formMessage(this.runtime, tweet);
 
@@ -179,7 +223,7 @@ Choose any combination of [LIKE], [RETWEET], [QUOTE], and [REPLY] that are appro
           tweet: tweet,
           actionResponse: actions,
           tweetState: state,
-          roomId: roomId,
+          roomId: message.roomId,
         });
       } catch (error) {
         logger.error(`Error processing tweet ${tweet.id}:`, error);