diff --git a/.agents/skills/x/SKILL.md b/.agents/skills/x/SKILL.md new file mode 100644 index 0000000..48fdf01 --- /dev/null +++ b/.agents/skills/x/SKILL.md @@ -0,0 +1,172 @@ +--- +name: X +description: Use when building applications that interact with X (formerly Twitter) data and functionality. Reach for this skill when agents need to search posts, manage user interactions, stream real-time data, handle authentication, work with API endpoints, manage rate limits, or integrate X data into applications. +metadata: + mintlify-proj: x + version: "1.0" +--- + +# X API Skill Reference + +## Product summary + +The X API provides programmatic access to X's public conversation through REST endpoints. Agents use it to read posts, publish content, manage users, search archives, stream real-time data, and analyze trends. The API uses pay-per-usage pricing with flexible authentication methods (OAuth 1.0a, OAuth 2.0, Bearer tokens). Key files: Developer Console at console.x.com for app credentials; API base URL is `https://api.x.com/2/`. Official SDKs available for Python (`xdk`) and TypeScript (`@xdevplatform/xdk`). Primary docs: https://docs.x.com/x-api/introduction + +## When to use + +Reach for this skill when: +- Building applications that read, search, or publish posts +- Integrating user authentication (OAuth flows) +- Streaming real-time posts or managing filtered rules +- Looking up user profiles, followers, or relationships +- Managing direct messages, lists, bookmarks, or spaces +- Handling API authentication, rate limits, or error responses +- Paginating through large result sets +- Requesting specific data fields or related objects via expansions +- Debugging API errors (401, 403, 429, etc.) +- Choosing between recent search (7 days) vs. full-archive search + +## Quick reference + +### Authentication methods + +| Method | Use case | Scope | +|:-------|:---------|:------| +| **Bearer Token** | App-only, public data | Read-only, no user context | +| **OAuth 1.0a** | User-context requests, actions on behalf of user | Read, write, DMs (3 levels) | +| **OAuth 2.0** | Modern user-context, fine-grained scopes | Recommended for new projects | +| **Basic Auth** | Enterprise APIs only | Server-to-server | + +### Common endpoints + +| Resource | Endpoint | Method | Use | +|:---------|:---------|:-------|:----| +| User lookup | `/2/users/by/username/:username` | GET | Get user by username | +| Post lookup | `/2/tweets/:id` | GET | Get post by ID | +| Recent search | `/2/tweets/search/recent` | GET | Search last 7 days | +| Full-archive search | `/2/tweets/search/all` | GET | Search all posts (paid) | +| Filtered stream | `/2/tweets/search/stream` | GET | Real-time posts matching rules | +| Create post | `/2/tweets` | POST | Publish a post | +| User timeline | `/2/users/:id/tweets` | GET | Get user's posts | +| Followers | `/2/users/:id/followers` | GET | Get user's followers | + +### Field parameters + +Request additional data with field parameters: + +```bash +# Post fields +?tweet.fields=created_at,public_metrics,lang,author_id + +# User fields +?user.fields=created_at,description,public_metrics,verified + +# Media fields +?media.fields=url,preview_image_url,alt_text + +# Expansions (include related objects) +?expansions=author_id,referenced_tweets.id +``` + +### Rate limit headers + +Every response includes: +- `x-rate-limit-limit` — max requests in window +- `x-rate-limit-remaining` — requests left +- `x-rate-limit-reset` — Unix timestamp when window resets + +### Response structure + +```json +{ + "data": { /* primary result */ }, + "includes": { /* related objects from expansions */ }, + "meta": { /* pagination, result count */ } +} +``` + +## Decision guidance + +| Scenario | Choose | Why | +|:---------|:-------|:----| +| **Search recent vs. full-archive** | Recent if <7 days old | Recent is free; full-archive requires paid access | +| **Bearer token vs. OAuth** | Bearer for public data only | OAuth needed for user-context actions (post, like, DM) | +| **Filtered stream vs. polling** | Filtered stream for real-time | Stream is efficient; polling wastes rate limits | +| **Pagination vs. since_id** | Pagination for backfill | since_id for incremental updates (fewer API calls) | +| **Fields vs. expansions** | Fields for object's own data | Expansions for related objects (author, media, etc.) | + +## Workflow + +1. **Set up credentials** + - Create app at console.x.com + - Generate Bearer Token (app-only) or OAuth credentials (user-context) + - Store securely; never commit to code + +2. **Choose authentication method** + - Public data only? Use Bearer Token + - Need user actions? Use OAuth 1.0a or 2.0 + - Check endpoint docs for required auth type + +3. **Build the request** + - Identify endpoint (search, lookup, manage, stream) + - Add required parameters (query, user_id, etc.) + - Request fields and expansions for needed data + - Set max_results and pagination_token if needed + +4. **Handle the response** + - Check HTTP status code (200 = success, 4xx/5xx = error) + - Parse `data` field for primary results + - Check `includes` for related objects + - Use `meta.next_token` for pagination + +5. **Manage rate limits** + - Monitor `x-rate-limit-remaining` header + - Implement exponential backoff for 429 errors + - Cache responses when possible + - Use streaming instead of polling for real-time data + +6. **Verify and test** + - Test with cURL or Postman first + - Check error responses match expected format + - Verify pagination works for large result sets + - Confirm fields/expansions return expected data + +## Common gotchas + +- **Missing Bearer Token**: Ensure token is in `Authorization: Bearer $TOKEN` header, not as query parameter +- **Forgetting expansions**: Requesting `author_id` field without `expansions=author_id` returns only the ID, not author details +- **Rate limit confusion**: Rate limits and billing are separate; you can hit rate limits without incurring costs +- **Protected accounts**: Posts from protected accounts only visible with user authorization; returns 403 otherwise +- **Pagination tokens expire**: Don't store tokens long-term; regenerate if needed +- **Query length limits**: Recent search = 512 chars, full-archive = 1,024 chars (4,096 for Enterprise) +- **Stream rules limit**: Filtered stream allows max 1,000 rules; hitting cap returns error +- **Deleted posts return 404**: Don't assume post exists; handle 404 gracefully +- **Partial errors in batch requests**: 200 response may include both `data` and `errors` array; check both +- **OAuth 2.0 callback URLs**: Must match exactly in Developer Console (including trailing slashes); use `http://127.0.0.1` for local dev, not `localhost` + +## Verification checklist + +Before submitting work: + +- [ ] Authentication credentials are correct and not hardcoded +- [ ] Endpoint URL matches API docs (base: `https://api.x.com/2/`) +- [ ] Required parameters are included (query, user_id, etc.) +- [ ] Field parameters use correct syntax (`tweet.fields=`, `user.fields=`, etc.) +- [ ] Expansions are paired with field parameters for related objects +- [ ] Error handling checks HTTP status code and `errors` array +- [ ] Rate limit headers are monitored (implement backoff for 429) +- [ ] Pagination logic handles `next_token` correctly +- [ ] Response parsing handles both `data` and `includes` objects +- [ ] OAuth tokens are stored securely (env vars, not code) +- [ ] Tested with actual API (not just mock data) + +## Resources + +- **Comprehensive navigation**: https://docs.x.com/llms.txt — Full page-by-page listing for agent reference +- **Getting started**: https://docs.x.com/x-api/introduction — Overview and quick start +- **Authentication guide**: https://docs.x.com/fundamentals/authentication/overview — All auth methods explained +- **Rate limits reference**: https://docs.x.com/x-api/fundamentals/rate-limits — Per-endpoint limits and handling + +--- + +> For additional documentation and navigation, see: https://docs.x.com/llms.txt \ No newline at end of file diff --git a/README.md b/README.md index 7bb797d..8b796ac 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,8 @@ If you already have an access token (e.g., from another OAuth flow or the X Deve ```toml [x] access_token = "your_access_token_here" +# Optional: provide refresh_token to enable automatic token refresh +refresh_token = "your_refresh_token_here" ``` #### Option B: OAuth 2.0 PKCE Flow (Interactive) diff --git a/src/x2raindrop_cli/cli.py b/src/x2raindrop_cli/cli.py index 3ec5444..85f625a 100644 --- a/src/x2raindrop_cli/cli.py +++ b/src/x2raindrop_cli/cli.py @@ -6,6 +6,7 @@ from __future__ import annotations +from datetime import datetime from pathlib import Path from typing import Annotated @@ -100,7 +101,18 @@ def _get_x_token(settings: Settings) -> OAuth2Token | None: if settings.x.has_direct_token(): direct_token = settings.x.get_direct_token() if direct_token: - logger.debug("Using direct access token") + logger.debug("Using direct token from config/env") + if settings.x.access_token and settings.x.refresh_token: + # Allow pairing a refresh token with a direct access token. + # The access token will be refreshed automatically by XClient when needed. + return OAuth2Token( + access_token=settings.x.access_token, + refresh_token=settings.x.refresh_token, + token_type="bearer", + # Mark expired so refresh happens on first request when configured. + expires_at=datetime.now(), + scope="", + ) return OAuth2Token.from_access_token(direct_token) # Fall back to PKCE flow if client_id is configured @@ -262,7 +274,11 @@ def sync( console.print() # Initialize clients - x_client = XClient(token) + x_client = XClient( + token, + refresh_client_id=settings.x.client_id, + refresh_client_secret=settings.x.client_secret, + ) raindrop_client = RaindropClient(settings.raindrop.token) state = SyncState(settings.sync.state_path) diff --git a/src/x2raindrop_cli/config.py b/src/x2raindrop_cli/config.py index cb34135..ed83a59 100644 --- a/src/x2raindrop_cli/config.py +++ b/src/x2raindrop_cli/config.py @@ -70,6 +70,7 @@ class XSettings(BaseSettings): token_path: Path to store the OAuth2 tokens. scopes: OAuth2 scopes to request. access_token: Direct access token (alternative to PKCE flow). + refresh_token: Refresh token for direct access token refresh (optional). bearer_token: App-only bearer token (limited functionality, read-only). """ @@ -100,6 +101,10 @@ class XSettings(BaseSettings): None, description="Direct OAuth2 access token (skips browser login)", ) + refresh_token: str | None = Field( + None, + description="Direct OAuth2 refresh token (enables automatic access token refresh)", + ) bearer_token: str | None = Field( None, description="App-only bearer token (limited functionality)", @@ -305,6 +310,8 @@ def create_default_config(path: Path | None = None) -> Path: # Option 1: Direct access token (simplest, no browser login needed) # Get this from X Developer Portal or existing OAuth flow "access_token": "", + # Optional: provide refresh token to auto-refresh access token + "refresh_token": "", # Option 2: OAuth PKCE flow (interactive browser login) # Set client_id to enable `x2raindrop x login` "client_id": "", diff --git a/src/x2raindrop_cli/x/client.py b/src/x2raindrop_cli/x/client.py index 124fc5b..eeb20ad 100644 --- a/src/x2raindrop_cli/x/client.py +++ b/src/x2raindrop_cli/x/client.py @@ -16,7 +16,7 @@ from xdk import Client as XdkClient from x2raindrop_cli.models import BookmarkItem -from x2raindrop_cli.x.auth_pkce import OAuth2Token +from x2raindrop_cli.x.auth_pkce import OAuth2Token, refresh_access_token if TYPE_CHECKING: pass @@ -71,18 +71,29 @@ class XClient: number of API calls performed in this process for visibility. """ - def __init__(self, token: OAuth2Token) -> None: + def __init__( + self, + token: OAuth2Token, + *, + refresh_client_id: str | None = None, + refresh_client_secret: str | None = None, + ) -> None: """Initialize the client with an OAuth2 token. Args: token: OAuth2 token for authentication. + refresh_client_id: OAuth2 client ID used to refresh access tokens (optional). + refresh_client_secret: OAuth2 client secret used to refresh access tokens (optional). """ self.token = token + self._refresh_client_id = refresh_client_id + self._refresh_client_secret = refresh_client_secret self._user_id: str | None = None self._request_count: int = 0 - self._x_client = XdkClient( - access_token=token.access_token, - ) + self._x_client = XdkClient(access_token=token.access_token) + + # Best-effort early refresh if token is already expired. + self._ensure_fresh_token() @property def request_count(self) -> int: @@ -101,6 +112,35 @@ def close(self) -> None: """Close the underlying XDK session.""" self._x_client.session.close() + def _ensure_fresh_token(self) -> None: + """Refresh the access token if expired and refresh is configured.""" + if not self.token.is_expired(): + return + if not self.token.refresh_token: + return + if not self._refresh_client_id: + logger.warning( + "Token expired but refresh client_id is missing; cannot refresh automatically" + ) + return + + try: + refreshed = refresh_access_token( + self.token.refresh_token, + self._refresh_client_id, + self._refresh_client_secret, + ) + except Exception as e: + logger.warning("Automatic token refresh failed", error=str(e)) + return + + # Swap token + underlying client so future calls use the fresh access token. + self.token = refreshed + with contextlib.suppress(Exception): + self._x_client.session.close() + self._x_client = XdkClient(access_token=self.token.access_token) + logger.info("Access token refreshed for XDK client") + def get_authenticated_user_id(self) -> str: """Get the authenticated user's ID. @@ -110,6 +150,7 @@ def get_authenticated_user_id(self) -> str: if self._user_id is not None: return self._user_id + self._ensure_fresh_token() self._request_count += 1 response = self._x_client.users.get_me() self._user_id = self._extract_id_from_me_response(response) @@ -138,6 +179,7 @@ def get_bookmarks(self, max_results: int | None = None) -> Iterator[BookmarkItem BookmarkItem for each bookmark. """ + self._ensure_fresh_token() user_id = self.get_authenticated_user_id() total_fetched = 0 page_count = 0 @@ -320,6 +362,7 @@ def delete_bookmark(self, tweet_id: str) -> bool: Returns: True if successful. """ + self._ensure_fresh_token() user_id = self.get_authenticated_user_id() self._request_count += 1 diff --git a/tests/test_config.py b/tests/test_config.py index 16f48a5..1b2a9c1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -104,6 +104,18 @@ def test_direct_access_token(self, monkeypatch: MonkeyPatch) -> None: assert settings.get_direct_token() == "test_access_token" assert settings.can_use_pkce_flow() is False + def test_direct_access_token_with_refresh_token(self, monkeypatch: MonkeyPatch) -> None: + """Test direct access token can include refresh token.""" + monkeypatch.setenv("X_ACCESS_TOKEN", "test_access_token") + monkeypatch.setenv("X_REFRESH_TOKEN", "test_refresh_token") + + settings = XSettings() + + assert settings.access_token == "test_access_token" + assert settings.refresh_token == "test_refresh_token" + assert settings.has_direct_token() is True + assert settings.get_direct_token() == "test_access_token" + def test_pkce_flow_with_client_id(self, monkeypatch: MonkeyPatch) -> None: """Test PKCE flow requires client_id.""" monkeypatch.setenv("X_CLIENT_ID", "test_id")