diff --git a/packages/omni/.gitignore b/packages/omni/.gitignore index e0038ba..4f0e4a9 100644 --- a/packages/omni/.gitignore +++ b/packages/omni/.gitignore @@ -14,3 +14,6 @@ pnpm-debug.log* .DS_Store .vscode *.local +examples/twitter/ +settings.local.json + diff --git a/packages/omni/deploy_omni.sh b/packages/omni/deploy_omni.sh index c9942a5..61bba28 100755 --- a/packages/omni/deploy_omni.sh +++ b/packages/omni/deploy_omni.sh @@ -2,8 +2,11 @@ set -eo pipefail +TOOLBOX_DIR="${TOOLBOX_DIR:-/Users/koen/workspace/toolbox}" -cd /Users/koen/workspace/toolbox/packages/omni +echo "Deploying from $TOOLBOX_DIR/packages/omni to toolbox-prod" + +cd "$TOOLBOX_DIR/packages/omni" ssh toolbox-prod "rm -rf /home/azureuser/omni" zip -r - . -x "**.venv/*" -x ".git/*" -x "data/*" -x "node_modules/*" | ssh toolbox-prod "cat > /home/azureuser/archive.zip && rm -rf /home/azureuser/omni && unzip /home/azureuser/archive.zip -d /home/azureuser/omni" @@ -16,7 +19,8 @@ export PATH=/home/azureuser/.local/bin:/home/azureuser/.nvm/versions/node/v20.19 cd /home/azureuser/omni uv venv source .venv/bin/activate -uv pip install -e . || true +# uv cache clean +uv pip install --refresh --prerelease allow -e . || true npm install npm install -g serve " @@ -41,6 +45,7 @@ export PATH=/home/azureuser/.local/bin:/home/azureuser/.nvm/versions/node/v20.19 export VITE_API_BASE_URL=http://20.224.153.50:8000 export USE_ANTHROPIC=True export ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY +export VITE_POSTHOG_API_KEY=$POSTHOG_API_KEY cd /home/azureuser/omni source .venv/bin/activate diff --git a/packages/omni/examples/twitter_account_exists.py b/packages/omni/examples/twitter_account_exists.py index b767820..4a26de4 100644 --- a/packages/omni/examples/twitter_account_exists.py +++ b/packages/omni/examples/twitter_account_exists.py @@ -1,46 +1,24 @@ -import requests +import time +import urllib +from datetime import datetime +from omni.twitter import account_exists, get_guest_token -def account_exists(handle): - handle_url_encoded = handle.replace("@", "") +# warmup +cookie_value, expires = get_guest_token() +datetime_expires = datetime.fromtimestamp(expires) - url = f"https://api.x.com/graphql/96tVxbPqMZDoYB5pmzezKA/UserByScreenName?variables=%7B%22screen_name%22%3A%22{handle_url_encoded}%22%2C%22withGrokTranslatedBio%22%3Afalse%7D&features=%7B%22hidden_profile_subscriptions_enabled%22%3Atrue%2C%22payments_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_is_identity_verified_enabled%22%3Atrue%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22responsive_web_twitter_article_notes_tab_enabled%22%3Atrue%2C%22subscriptions_feature_can_gift_premium%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Atrue%7D" +cookie_decoded = urllib.parse.unquote(cookie_value) - headers = { - "accept": "*/*", - "accept-language": "en-GB,en;q=0.9", - "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", - "cache-control": "no-cache", - "content-type": "application/json", - "origin": "https://x.com", - "pragma": "no-cache", - "priority": "u=1, i", - "referer": "https://x.com/", - "sec-ch-ua": '"Chromium";v="140", "Not=A?Brand";v="24", "Brave";v="140"', - "sec-ch-ua-mobile": "?0", - "sec-ch-ua-platform": '"macOS"', - "sec-fetch-dest": "empty", - "sec-fetch-mode": "cors", - "sec-fetch-site": "same-site", - "sec-gpc": "1", - "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36", - "x-client-transaction-id": "zcVrDRSIRWct0CXT5dceAT2N5xwliK+Nxkijb3TPCA16/oa44aGUkRBDP/wJm87H9A+kSckr/RdWjvO/n/ktx/La3vPEzg", - "x-guest-token": "1970803204867682327", - "x-twitter-active-user": "yes", - "x-twitter-client-language": "en-GB", - "x-xp-forwarded-for": "38cb822faf802d12f9a727d427f9b1014fbd26331085ff5627a9b0c468630c7eff0cd0d43e6195e3c79c43661bb7fa5314711c287e2035c395de5f855be30bd6c0d91b6b891f2c765f883796cc5c76b1dea0bbc49193e4a139ec4168499daefc00426b798829b6163aecf027ed47418510a1b6f2b8add52e87efa04524909db5e1debc9e3f8c0d66891eab7d3e80824a73a706d7bec9a0bd1428c1a8c56c8f02c0e972668556ae9c9fa4fd5d93b76b61980a3551c2c5de4dc05858e1c855af1889944cbd44da5650237bd6c5c45888c36d080a5cea283729af8d71ff3be1dbf723e97b0511225ab55cfb735d9553fc4f911c674079dba0b791c359c05fd1adc66e", - } +time_start = time.time() +cookie_value, expires = get_guest_token() +handle = "elonmusk" +# handle = "XXXXXFQWEGWRBFSD" +exists = account_exists(handle, cookie_value) - response = requests.get(url, headers=headers) - if response.json().get("data") == {}: - return False - else: - return True - - -handle = "femke_plantinga" -exists = account_exists(handle) +time_end = time.time() +print(f"Time taken: {time_end - time_start} seconds") if exists: print(f"Account {handle} exists") else: diff --git a/packages/omni/examples/twitter_test.py b/packages/omni/examples/twitter_test.py new file mode 100644 index 0000000..1fecec5 --- /dev/null +++ b/packages/omni/examples/twitter_test.py @@ -0,0 +1,111 @@ +import asyncio +import json +from http.cookiejar import Cookie +from pathlib import Path + +import browser_cookie3 + +# Keep all cookies that might be relevant for authentication +x_cookies = [] +for c in browser_cookie3.brave(domain_name="x.com"): + if c.domain == ".x.com" and c.name in ["auth_token"]: + x_cookies.append(c) + print("Found x auth_token") + +# print(x_cookies) + +OUTPUT_DIR = Path(".").resolve().parent.parent.parent / "data" + + +def cookie_to_playwright_cookie(cookie: Cookie) -> dict: + cookie_dict = { + "name": cookie.name, + "value": cookie.value, + "domain": cookie.domain.lstrip("."), # Playwright expects no leading dot + "path": cookie.path, + "secure": bool(cookie.secure), + "httpOnly": getattr(cookie, "httponly", False) + or c.get_nonstandard_attr("HttpOnly", False), + } + # Optional: expires + if cookie.expires is not None: + cookie_dict["expires"] = cookie.expires + return cookie_dict + + +def is_home_latest_timeline_url(url): + if "graphql" in url and ("HomeLatestTimeline" in url or "HomeTimeline" in url): + return True + return False + + +async def on_request(request): + if is_home_latest_timeline_url(request.url): + # INSERT_YOUR_CODE + response = await request.response() + if response is not None: + try: + json_data = await response.json() + from datetime import datetime + + dt_str = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"home_latest_timeline_{dt_str}.json" + output_path = OUTPUT_DIR / filename + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(json_data, f) + print(f"Saved JSON to {filename}") + except Exception as e: + print(f"Failed to get JSON from response: {e}") + + print("API call to HomeLatestTimeline") + + +async def playwright_login_with_cookies(): + from playwright.async_api import async_playwright + + # Use the first cookie found (auth_token) for x.com + cookies = [] + for c in x_cookies: + cookies.append(cookie_to_playwright_cookie(c)) + + async with async_playwright() as p: + browser = await p.chromium.launch(headless=False) + context = await browser.new_context() + # Set cookies before navigating + await context.add_cookies(cookies) + # Print all cookies currently in the context + cookies_in_context = await context.cookies() + print("All cookies in context after setting:") + for cookie in cookies_in_context: + print(cookie) + page = await context.new_page() + await asyncio.sleep(3) + page.on("request", on_request) + + await page.goto("https://x.com") + + # INSERT_YOUR_CODE + # Print all spans content with class "r-poiln3" + await asyncio.sleep(3) + + # INSERT_YOUR_CODE + # Find the span (button) that says "Following" and click it + following_span = await page.query_selector("span:has-text('Following')") + if following_span: + await following_span.click() + print("Clicked the 'Following' button.") + else: + print("Could not find the 'Following' button.") + + # spans = await page.query_selector_all("span.r-poiln3") + # print("Contents of all elements:") + # for span in spans: + # text = await span.text_content() + # print(text) + await asyncio.sleep(10000) + await browser.close() + + +# Run the async function +asyncio.run(playwright_login_with_cookies()) diff --git a/packages/omni/frontend/App.vue b/packages/omni/frontend/App.vue index e9837ec..39a7ff5 100644 --- a/packages/omni/frontend/App.vue +++ b/packages/omni/frontend/App.vue @@ -24,7 +24,7 @@
- +
@@ -60,7 +60,7 @@