From 3a8bafc33aee20a3ed10303eb115cb1a8dff2ee7 Mon Sep 17 00:00:00 2001 From: John Cole <29712567+johnisanerd@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:35:21 -0500 Subject: [PATCH] youtube-apify-transcript: switch to a transcripts Actor that isn't failing runs pintostudio/youtube-transcript-scraper failed 67,188 of 530,937 runs in the last 30 days. Point the fetch at johnvc/YoutubeTranscripts, map its timestamped cues onto the skill's start/dur/text shape, keep cached pintostudio rows readable, and drop the stale per-video cost lines. Adds offline tests for the input builder and the normalizer. --- .../youtube-apify-transcript/CHANGELOG.md | 5 + .../skills/youtube-apify-transcript/README.md | 8 +- .../skills/youtube-apify-transcript/SKILL.md | 23 ++- .../youtube-apify-transcript/package.json | 4 +- .../scripts/fetch_transcript.py | 135 +++++++++++++++--- .../scripts/tests/test_fetch_transcript.py | 94 ++++++++++++ 6 files changed, 227 insertions(+), 42 deletions(-) create mode 100644 .claude/skills/youtube-apify-transcript/scripts/tests/test_fetch_transcript.py diff --git a/.claude/skills/youtube-apify-transcript/CHANGELOG.md b/.claude/skills/youtube-apify-transcript/CHANGELOG.md index e7b128bb..1f35373b 100644 --- a/.claude/skills/youtube-apify-transcript/CHANGELOG.md +++ b/.claude/skills/youtube-apify-transcript/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [1.1.3] - 2026-11-02 + +- Point the Apify fetch at a live transcripts Actor and map timestamped cues onto the skill's start/dur/text shape +- Keep cached pintostudio rows readable; drop stale per-video cost lines + ## [1.0.7] - 2026-02-04 - Privacy cleanup: removed hardcoded paths and personal info from docs diff --git a/.claude/skills/youtube-apify-transcript/README.md b/.claude/skills/youtube-apify-transcript/README.md index 3da8e911..dc574a2c 100644 --- a/.claude/skills/youtube-apify-transcript/README.md +++ b/.claude/skills/youtube-apify-transcript/README.md @@ -14,9 +14,7 @@ Fetch YouTube video transcripts from anywhere — even cloud servers where YouTu ## 💰 Free Tier -APIFY offers **$5/month free credits** — that's approximately **714 videos per month** at $0.007 each! - -No credit card required. [Sign up here](https://apify.com/) +APIFY offers a monthly free-credit allowance. No credit card required. [Sign up here](https://apify.com/) ## 🚀 Quick Start @@ -34,9 +32,9 @@ See [SKILL.md](skills/youtube-apify-transcript/SKILL.md) for full documentation, ## 🔗 Links -- [APIFY Free Tier](https://apify.com/pricing) - $5/month free +- [APIFY Free Tier](https://apify.com/pricing) - [Get API Key](https://console.apify.com/account/integrations) -- [YouTube Transcripts Actor](https://apify.com/karamelo/youtube-transcripts) +- [YouTube Transcripts Actor](https://apify.com/johnvc/YoutubeTranscripts) ## ⚙️ Requirements diff --git a/.claude/skills/youtube-apify-transcript/SKILL.md b/.claude/skills/youtube-apify-transcript/SKILL.md index b70aacbc..a9cf1953 100644 --- a/.claude/skills/youtube-apify-transcript/SKILL.md +++ b/.claude/skills/youtube-apify-transcript/SKILL.md @@ -1,6 +1,6 @@ --- name: youtube-apify-transcript -description: Fetch YouTube transcripts via APIFY API. Works from cloud IPs (Hetzner, AWS, etc.) by bypassing YouTube's bot detection. Free tier includes $5/month credits (~714 videos). No credit card required. +description: Fetch YouTube transcripts via APIFY API. Works from cloud IPs (Hetzner, AWS, etc.) by bypassing YouTube's bot detection. Uses a live community Actor. No credit card required. tags: [research] --- @@ -14,20 +14,15 @@ YouTube blocks transcript requests from cloud IPs (AWS, GCP, etc.). APIFY runs t ## Free Tier -- **$5/month free credits** (~714 videos) +- Apify platform credits apply - No credit card required - Perfect for personal use -## Cost - -- **$0.007 per video** (less than 1 cent!) -- Track usage at: https://console.apify.com/billing - ## Links - [APIFY Pricing](https://apify.com/pricing) - [Get API Key](https://console.apify.com/account/integrations) -- [YouTube Transcript Scraper Actor](https://apify.com/pintostudio/youtube-transcript-scraper) +- [YouTube Transcripts Actor](https://apify.com/johnvc/YoutubeTranscripts) ## Setup @@ -73,13 +68,13 @@ python3 scripts/fetch_transcript.py "URL" --lang de ### Caching (saves money!) -Transcripts are cached locally by default. Repeat requests for the same video cost $0. +Transcripts are cached locally by default. Repeat requests for the same video skip a new Actor run. ```bash -# First request: fetches from APIFY ($0.007) +# First request: fetches from APIFY python3 scripts/fetch_transcript.py "URL" -# Second request: uses cache (FREE!) +# Second request: uses cache python3 scripts/fetch_transcript.py "URL" # Output: [cached] Transcript for: VIDEO_ID @@ -116,17 +111,17 @@ python3 scripts/fetch_transcript.py --batch urls.txt --json --output all_transcr ## APIFY Actor Input -The script sends the following input to `pintostudio/youtube-transcript-scraper`: +The script sends the following input to the live YouTube transcripts Actor: ```json { - "videoUrl": "https://www.youtube.com/watch?v=VIDEO_ID" + "youtube_url": "https://www.youtube.com/watch?v=VIDEO_ID" } ``` **Output fields:** -Each result contains a `data` array of transcript segments: +Each result is normalized to a `segments` array of transcript cues: | Field | Type | Description | |---------|--------|------------------------------------| diff --git a/.claude/skills/youtube-apify-transcript/package.json b/.claude/skills/youtube-apify-transcript/package.json index f1b1f14d..0b51ba0a 100644 --- a/.claude/skills/youtube-apify-transcript/package.json +++ b/.claude/skills/youtube-apify-transcript/package.json @@ -1,7 +1,7 @@ { "name": "@openclaw/youtube-apify-transcript", - "version": "1.1.2", - "description": "Fetch YouTube transcripts via APIFY API. Works from cloud IPs (Hetzner, AWS, etc.) by bypassing YouTube's bot detection. Free tier includes $5/month credits (~714 videos).", + "version": "1.1.3", + "description": "Fetch YouTube transcripts via APIFY API. Works from cloud IPs (Hetzner, AWS, etc.) by bypassing YouTube's bot detection. Uses a live community Actor.", "author": "robbyczgw-cla", "license": "MIT", "keywords": [ diff --git a/.claude/skills/youtube-apify-transcript/scripts/fetch_transcript.py b/.claude/skills/youtube-apify-transcript/scripts/fetch_transcript.py index 4d0a1e52..392e197f 100644 --- a/.claude/skills/youtube-apify-transcript/scripts/fetch_transcript.py +++ b/.claude/skills/youtube-apify-transcript/scripts/fetch_transcript.py @@ -28,11 +28,10 @@ try: import requests except ImportError: - print("Error: 'requests' library not installed.", file=sys.stderr) - sys.exit(1) + requests = None -APIFY_ACTOR_ID = "pintostudio~youtube-transcript-scraper" +APIFY_ACTOR_ID = "johnvc~YoutubeTranscripts" GOOSEWORKS_API_BASE = os.environ.get("GOOSEWORKS_API_BASE", "https://api.gooseworks.ai") GOOSEWORKS_API_KEY = os.environ.get("GOOSEWORKS_API_KEY") @@ -45,6 +44,9 @@ def get_api_token(): """Get API token, preferring GooseWorks proxy key.""" + if requests is None: + print("Error: 'requests' library not installed.", file=sys.stderr) + sys.exit(1) token = GOOSEWORKS_API_KEY or os.environ.get("APIFY_API_TOKEN") if not token: print("Error: Set GOOSEWORKS_API_KEY or APIFY_API_TOKEN env var.", file=sys.stderr) @@ -52,6 +54,103 @@ def get_api_token(): return token +def _as_float(value, default=0.0): + """Coerce a timestamp or duration field to float. + + Args: + value: Raw numeric, string, or missing field from an Actor row. + default: Fallback when the value cannot be parsed. + + Returns: + A float suitable for the skill's start/dur JSON fields. + """ + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _normalize_segment(segment): + """Map one transcript cue onto the skill's start/dur/text shape. + + Args: + segment: A string cue or a dict from johnvc timestamped rows + (start/duration) or cached pintostudio rows (start/dur). + + Returns: + Dict with start, dur, and text. + """ + if isinstance(segment, str): + return {"start": 0.0, "dur": 0.0, "text": segment} + if not isinstance(segment, dict): + return {"start": 0.0, "dur": 0.0, "text": ""} + duration = segment.get("dur", segment.get("duration", 0)) + return { + "start": _as_float(segment.get("start", 0)), + "dur": _as_float(duration), + "text": segment.get("text", "") or "", + } + + +def build_actor_input(video_url, language=None): + """Build johnvc/YoutubeTranscripts run input. + + Args: + video_url: Canonical YouTube watch URL. + language: Optional ISO 639-1 code from --lang. + + Returns: + Actor input dict with youtube_url and optional languages. + """ + payload = {"youtube_url": video_url} + if language: + payload["languages"] = [language] + return payload + + +def normalize_transcript_result(result): + """Normalize an Apify dataset item to title/language/segments. + + Accepts live johnvc/YoutubeTranscripts rows (timestamped + duration), + cached pintostudio rows (data + dur), and already-normalized cache + files that store segments. + + Args: + result: One dataset item or a cached transcript dict. + + Returns: + Dict with title, language, and segments (start/dur/text). + + Raises: + ValueError: When the Actor row reports a failed fetch. + """ + if not isinstance(result, dict): + return {"title": "Unknown", "language": "unknown", "segments": []} + + if result.get("success") is False or result.get("error") is True: + message = result.get("error_message") or "transcript fetch failed" + if isinstance(message, bool): + message = "transcript fetch failed" + raise ValueError(str(message)) + + if isinstance(result.get("segments"), list): + raw_segments = result["segments"] + elif isinstance(result.get("timestamped"), list): + raw_segments = result["timestamped"] + elif isinstance(result.get("data"), list): + raw_segments = result["data"] + else: + raw_segments = [] + + language = result.get("language") or result.get("language_code") or "unknown" + title = result.get("title") or "Unknown" + return { + "title": title, + "language": language, + "segments": [_normalize_segment(item) for item in raw_segments], + } + + def extract_video_id(url): """Extract YouTube video ID from various URL formats.""" # Handle youtu.be short links @@ -201,12 +300,7 @@ def run_apify_actor(video_url, api_token, language=None): # Start the actor run run_url = f"{APIFY_API_BASE}/acts/{APIFY_ACTOR_ID}/runs" - input_data = { - "videoUrl": video_url - } - - if language: - input_data["preferredLanguage"] = language + input_data = build_actor_input(video_url, language) headers = { "Content-Type": "application/json" @@ -285,14 +379,18 @@ def run_apify_actor(video_url, api_token, language=None): sys.exit(1) result = results[0] + try: + normalized = normalize_transcript_result(result) + except ValueError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + + if not normalized["segments"]: + print("Error: No transcript found for this video.", file=sys.stderr) + print("The video might not have captions available.", file=sys.stderr) + sys.exit(1) - # Normalize output: extract segments from the `data` field - segments = result.get("data", []) - return { - "title": result.get("title", "Unknown"), - "language": result.get("language", "unknown"), - "segments": segments - } + return normalized except requests.exceptions.RequestException as e: print(f"Error fetching results: {e}", file=sys.stderr) @@ -433,8 +531,6 @@ def process_batch(batch_file, api_token, use_cache, language, output_json): # Print summary print(f"\n{'='*40}", file=sys.stderr) print(f"Batch complete: {fetched} fetched, {cached} cached, {failed} failed", file=sys.stderr) - if fetched > 0: - print(f"[Cost: ~${fetched * 0.007:.3f} for {fetched} API call(s)]", file=sys.stderr) return results @@ -571,9 +667,6 @@ def main(): else: print(output) - if not from_cache: - print("\n[Cost: ~$0.007 per video]", file=sys.stderr) - if __name__ == "__main__": main() diff --git a/.claude/skills/youtube-apify-transcript/scripts/tests/test_fetch_transcript.py b/.claude/skills/youtube-apify-transcript/scripts/tests/test_fetch_transcript.py new file mode 100644 index 00000000..0dd803c7 --- /dev/null +++ b/.claude/skills/youtube-apify-transcript/scripts/tests/test_fetch_transcript.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Offline adapter tests for youtube-apify-transcript normalize helpers.""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPT_DIR)) + +from fetch_transcript import build_actor_input, normalize_transcript_result + + +class BuildActorInputTests(unittest.TestCase): + """Cover johnvc/YoutubeTranscripts run input.""" + + def test_url_only(self) -> None: + """A watch URL becomes youtube_url with no language list.""" + payload = build_actor_input("https://www.youtube.com/watch?v=dQw4w9WgXcQ") + self.assertEqual( + payload, + {"youtube_url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"}, + ) + + def test_language_list(self) -> None: + """--lang becomes a one-item languages array.""" + payload = build_actor_input( + "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + language="de", + ) + self.assertEqual(payload["languages"], ["de"]) + + +class NormalizeTranscriptResultTests(unittest.TestCase): + """Cover live johnvc rows, cached pintostudio rows, and failures.""" + + def test_johnvc_timestamped_duration_maps_to_dur(self) -> None: + """Live Actor rows keep the skill's start/dur/text field names.""" + normalized = normalize_transcript_result( + { + "success": True, + "error": False, + "title": "Example", + "language": "en", + "timestamped": [ + {"start": 0.0, "duration": 2.5, "text": "Hello"}, + {"start": 2.5, "duration": 3.0, "text": "world"}, + ], + } + ) + self.assertEqual(normalized["title"], "Example") + self.assertEqual(normalized["language"], "en") + self.assertEqual( + normalized["segments"], + [ + {"start": 0.0, "dur": 2.5, "text": "Hello"}, + {"start": 2.5, "dur": 3.0, "text": "world"}, + ], + ) + + def test_cached_pintostudio_data_rows_still_parse(self) -> None: + """Older cache files that stored data[] with dur keep working.""" + normalized = normalize_transcript_result( + { + "title": "Cached", + "language": "en", + "data": [ + {"start": "1.0", "dur": "1.5", "text": "cached cue"}, + ], + } + ) + self.assertEqual( + normalized["segments"], + [{"start": 1.0, "dur": 1.5, "text": "cached cue"}], + ) + + def test_failed_actor_row_raises(self) -> None: + """A success=false dataset item becomes a ValueError.""" + with self.assertRaises(ValueError) as caught: + normalize_transcript_result( + { + "success": False, + "error": True, + "error_message": "captions unavailable", + "title": "Missing", + } + ) + self.assertIn("captions unavailable", str(caught.exception)) + + +if __name__ == "__main__": + unittest.main()