diff --git a/tools/i18n/ARCHITECTURE.md b/tools/i18n/ARCHITECTURE.md new file mode 100644 index 0000000000..7b19cd0103 --- /dev/null +++ b/tools/i18n/ARCHITECTURE.md @@ -0,0 +1,155 @@ +# WLED i18n Architecture + +## Overview + +Two-repository architecture separating **build toolchain** (core repo) from **translation files** (community repo). + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Core Repo (WLED/tools/i18n/) │ +│ ├── extract.py # Extract translatable strings from HTML │ +│ ├── build.py # Apply translations at build time │ +│ └── locales/ # Locale configuration │ +└─────────────────────────────────────────────────────────────┘ + ↓ calls +┌─────────────────────────────────────────────────────────────┐ +│ Translation Repo (WLED-translations//) │ +│ ├── static.json # Layer 1: Static HTML (429 entries) │ +│ ├── js.json # Layer 2: JS strings (45 entries) │ +│ ├── effects.json # Layer 3: Effect names (216 entries) │ +│ ├── palettes.json # Layer 4: Palette names (72 entries) │ +│ └── metadata.json # Version, coverage, maintainer │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## Four-Layer Translation Architecture + +| Layer | Content | File | Implementation | Coverage | +|-------|---------|------|----------------|----------| +| **L1** | Static HTML | `static.json` | Regex replacement in HTML text | 429 strings | +| **L2** | JS strings | `js.json` | Replace JS string literals | 45 strings | +| **L3** | Effect names | `effects.json` | C++ PROGMEM `#undef` + redefine | 216/216 (100%) | +| **L4** | Palette names | `palettes.json` | C++ PROGMEM array replacement | 72/72 (100%) | + +--- + +## Build Flow + +### PlatformIO Configuration + +```ini +# platformio_override.ini +[env:esp32dev_zh_CN] +extends = env:esp32dev +custom_usermods = https://github.com/foxlesbiao/WLED-translations +build_flags = ${env:esp32dev.build_flags} -D WLED_LOCALE=zh_CN +extra_scripts = pre:tools/i18n/build.py +``` + +PlatformIO automatically clones the translation repo to `.pio/libdeps/`. + +### Build Steps + +1. PlatformIO clones `WLED-translations` to `.pio/libdeps//WLED-translations/` +2. `build.py` auto-detects translations in `.pio/libdeps/` +3. Applies L1/L2 translations via regex replacement +4. Generates `i18n_effects.h` / `i18n_palettes.h` for L3/L4 (PROGMEM replacement) +5. Output to `build/i18n//` + +--- + +## How Dynamic Content Works + +The key insight: **PROGMEM replacement happens at compile time**, so JSON endpoints return translated strings automatically. + +``` +Browser ESP32 Firmware + │ │ + ├─ GET /json/palettes ─────────→│ PROGMEM array was replaced at compile time + │ ← {"0":"默认","1":"* 随机循环",...} ↓ + │ │ palettes.json → i18n_palettes.h + ├─ GET /json/effects ─────────→│ #undef _data_FX_MODE_STATIC + │ ← {"0":"常亮","1":"闪烁",...}│ #define _data_FX_MODE_STATIC "常亮" +``` + +No firmware code changes needed. The C++ PROGMEM strings are the single source of truth. + +--- + +## Grammar and Word Order + +WLED UI uses **short labels**, not full sentences: + +| Pattern | Example | i18n Impact | +|---------|---------|-------------| +| Single word | "Brightness", "Speed" | ✅ No issue | +| Label + value | "255 segments" | ✅ Works ("255 个段") | +| Full sentences | Almost none | ✅ N/A | +| Plural forms | Not used | ✅ N/A | +| Date formats | Not used in UI | ✅ N/A | + +The architecture **intentionally avoids** complex i18n patterns (ICU MessageFormat, plural rules) because WLED's UI doesn't need them. + +--- + +## What's NOT Translated (By Design) + +| Content | Reason | +|---------|--------| +| User-defined preset names | Belongs to user | +| Usermod settings pages | Dynamic HTML from firmware, varies by hardware | +| System info (IP, memory) | Universal data | +| Effect slider tooltips | Generated from mode data arrays | + +--- + +## Repository Structure + +### Core repo (WLED) + +``` +tools/i18n/ +├── extract.py # String extraction tool +├── build.py # Build-time translation applicator +├── ARCHITECTURE.md # This file +├── README.md # Usage documentation +└── locales/ + └── en_template.json # English template for translators +``` + +### Translation repo (WLED-translations) + +``` +/ +├── static.json # Layer 1: Static HTML text +├── js.json # Layer 2: JavaScript strings +├── effects.json # Layer 3: Effect names (PROGMEM) +├── palettes.json # Layer 4: Palette names (PROGMEM) +└── metadata.json # {"version":"1.0","coverage":"100%","maintainer":"..."} +en_template/ # English template for translators +``` + +--- + +## Adding a New Language + +1. Fork `WLED-translations` +2. Copy `en_template/` to `/` +3. Translate JSON files +4. Submit PR to translation repo + +No changes to WLED core needed. + +--- + +## Coverage Summary (zh_CN) + +| Layer | Content | Count | Status | +|-------|---------|-------|--------| +| L1 | Static HTML | 429 | ✅ Complete | +| L2 | JS strings | 45 | ✅ Complete | +| L3 | Effect names | 216/216 | ✅ 100% | +| L4 | Palette names | 72/72 | ✅ 100% | +| **Total** | | **762** | **100%** | diff --git a/tools/i18n/README.md b/tools/i18n/README.md new file mode 100644 index 0000000000..18cf37833d --- /dev/null +++ b/tools/i18n/README.md @@ -0,0 +1,187 @@ +# WLED i18n Toolchain + +Build-time internationalization for WLED Web UI. Translates HTML/JS strings at compile time — replaces English text, does not add to it. + +## How It Works + +``` +English HTM files (wled00/data/) + ↓ + extract.py → en_template.json + ↓ + Translator creates locale repo (WLED-translations) + ↓ + build.py → Translated HTM files + ↓ + npm run build → html_*.h / js_*.h (C headers) + ↓ + pio run → Firmware with translated UI +``` + +## Quick Start (User) + +Add to `platformio_override.ini`: + +```ini +[env:esp32dev_zh_CN] +extends = env:esp32dev +custom_usermods = https://github.com/foxlesbiao/WLED-translations +build_flags = ${env:esp32dev.build_flags} -D WLED_LOCALE=zh_CN +extra_scripts = pre:tools/i18n/build.py +``` + +Then: `pio run -e esp32dev_zh_CN` + +PlatformIO automatically clones the translations repo to `.pio/libdeps/`. The build script finds translations there automatically. + +## Quick Start (Translator) + +Translations live in a separate repo: [WLED-translations](https://github.com/foxlesbiao/WLED-translations) + +``` +WLED-translations/ +├── library.json # PlatformIO dependency manifest +├── zh_CN/ +│ ├── static.json # Layer 1: static HTML (429 entries) +│ ├── js.json # Layer 2: JS strings (716 entries) +│ ├── effects.json # Layer 3: effect names (216 entries) +│ ├── palettes.json # Layer 4: palette names (72 entries) +│ └── metadata.json +├── de_DE/ +│ └── ... +└── en_template/ # English template (generated by extract.py) +``` + +### Adding a new language + +```bash +# 1. Clone translations repo +git clone https://github.com/foxlesbiao/WLED-translations +cd WLED-translations + +# 2. Generate English template (from WLED source) +python3 /path/to/WLED/tools/i18n/extract.py --stats +cp -r /path/to/WLED/tools/i18n/locales/* en_template/ + +# 3. Create your locale +mkdir de_DE +cp en_template/*.json de_DE/ + +# 4. Fill in "translation" fields in each JSON file +# 5. Commit and push +``` + +## Quick Start (Developer) + +### Extract strings (generate template) + +```bash +python3 tools/i18n/extract.py --stats +# Output: tools/i18n/locales/en_template.json +``` + +### Build translated firmware + +```bash +# Build translated HTM files +python3 tools/i18n/build.py --locale zh_CN \ + --translations-dir /path/to/WLED-translations/zh_CN \ + --output-dir build/i18n/zh_CN + +# Validate translations +python3 tools/i18n/build.py --locale zh_CN --validate + +# Build web UI headers +npm ci && npm run build + +# Build firmware +pio run -e esp32dev +``` + +### Version updates (diff) + +When WLED releases a new version, compare templates to find changes: + +```bash +# Generate old and new templates +python3 tools/i18n/extract.py --stats # on old version +cp tools/i18n/locales/en_template.json en_template_old.json + +python3 tools/i18n/extract.py --stats # on new version +cp tools/i18n/locales/en_template.json en_template_new.json + +# Compare +python3 tools/i18n/diff.py --old en_template_old.json --new en_template_new.json +``` + +Output shows added/removed/modified strings. Translators update only changed entries. + +## Translation Search Order + +`build.py` searches for translations in this order: + +1. `--translations-dir` (explicit path) +2. `.pio/libdeps/*/WLED-translations//` (PlatformIO out-of-tree) +3. `tools/i18n/locales/.json` (local fallback) + +## Coverage + +| Layer | Content | Method | Count | +|-------|---------|--------|-------| +| 1. Static HTML | Labels, buttons, placeholders | DOM text matching | 429 | +| 2. JS strings | `alert()`, `innerHTML`, `innerText` | Script block regex | 716 | +| 3. Effect names | FX names in `FX.cpp` | PROGMEM replacement | 216 | +| 4. Palette names | Palette names in `FX_fcn.cpp` | PROGMEM replacement | 72 | + +## Known Limitations + +### Cannot translate (technical) + +- **Dynamic runtime text** — OTA update errors, Info page content, usermod settings, Pin Info page +- **External tools** — PixelForge add-ons (always English, downloaded on-the-fly) +- **JavaScript template literals** — strings with `${...}` interpolation +- **C++ server-side strings** — ~12 strings in `xml.cpp` need `#ifdef WLED_LOCALE_*` + +### Language-specific issues (acknowledged) + +- Word order differences (e.g., "X of Y" patterns) +- Number formats (decimal point vs comma) +- Grammar rules (singular/plural, countable/uncountable) +- Date formats + +These are known limitations. The tool handles short labels and UI fragments, not full sentences. + +## Layer 3/4: Effect & Palette Names + +Effect names (216) and palette names (72) are translated via C++ PROGMEM replacement: + +```c +// locale_effects.h (auto-generated) +#pragma once +#ifdef WLED_LOCALE +#undef _data_FX_MODE_STATIC +static const char _data_FX_MODE_STATIC[] PROGMEM = "常亮"; +// ... +#endif +``` + +The `.h` files are generated in the translation repo. Users copy them to their local build. + +## Architecture + +``` +WLED (core repo) +└── tools/i18n/ + ├── extract.py # Extract strings from HTML/JS + ├── build.py # Apply translations (pre-build script) + ├── diff.py # Compare template versions + └── README.md + +WLED-translations (community repo) +├── zh_CN/ +│ ├── static.json # Layer 1: HTML text +│ ├── js.json # Layer 2: JS strings +│ ├── effects.json # Layer 3: effect names +│ └── palettes.json # Layer 4: palette names +└── en_template/ # English template +``` diff --git a/tools/i18n/auto_translate.py b/tools/i18n/auto_translate.py new file mode 100644 index 0000000000..b413c6d099 --- /dev/null +++ b/tools/i18n/auto_translate.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""WLED Auto-Translation Script + +Automatically translates newly added WLED strings using LLM. +Works with the diff output to translate only new/modified strings. + +Usage: + python3 auto_translate.py --diff-file diff.json --locale zh_CN + python3 auto_translate.py --translations-path ~/WLED-translations --locale zh_CN +""" + +import sys +import argparse +import tempfile +from pathlib import Path +from typing import Dict, List, Optional +from common import load_json, save_json + + +# Translation rules for WLED +TRANSLATION_RULES = """ +## WLED翻译规则 + +### 必须翻译 +- 按钮文字（Back→返回、Save→保存、Scan→扫描） +- 标题（WiFi Settings→WiFi 设置） +- 标签和表单文字 +- 错误消息和提示 +- 状态信息 + +### 不可改动 +- HTML标签和属性 +- JavaScript代码逻辑 +- CSS样式 +- 技术术语（GPIO、JSON、RGB、WiFi、NTP、OTA、DMX） +- 变量名、函数名 +- URL和API端点 + +### 翻译风格 +- 简洁技术中文 +- 按钮用动词 +- 标题用名词短语 +- 保留英文技术术语 +- 中文全角标点（：，。） + +### WLED特有术语 +- Preset → 预设 +- Segment → 段 +- Palette → 调色板 +- Effect → 特效 +- LED → LED（不翻译） +- Brightness → 亮度 +- Color → 颜色 +- Speed → 速度 +- Intensity → 强度 +""" + + +def load_diff(diff_file: Path) -> Dict: + """Load diff output from diff.py.""" + return load_json(diff_file) + + +def load_template(template_path: Path, filename: str) -> Dict: + """Load a template JSON file. Returns empty dict if not found.""" + filepath = template_path / filename + if not filepath.exists(): + return {} + return load_json(filepath) + + +def save_template(template_path: Path, filename: str, data: Dict): + """Save template JSON file.""" + save_json(template_path / filename, data) + + +def extract_string_info(key: str, template_data: Dict) -> Optional[Dict]: + """Extract string information from template.""" + if key not in template_data: + return None + + entry = template_data[key] + if isinstance(entry, dict): + return { + 'key': key, + 'en': entry.get('en', ''), + 'translation': entry.get('translation', ''), + 'file': entry.get('file', ''), + 'path': entry.get('path', '') + } + else: + return { + 'key': key, + 'en': str(entry), + 'translation': '', + 'file': '', + 'path': '' + } + + +def create_translation_prompt(strings: List[Dict], locale: str) -> str: + """Create prompt for LLM translation.""" + locale_name = {'zh_CN': '简体中文', 'zh_TW': '繁体中文', 'ja': '日语', 'ko': '韩语'}.get(locale, locale) + prompt = f"""你是一个专业的WLED固件Web UI翻译器。请将以下英文字符串翻译成{locale_name}。 + +{TRANSLATION_RULES} + +## 需要翻译的字符串 + +""" + + for i, s in enumerate(strings, 1): + prompt += f"### {i}. Key: `{s['key']}`\n" + prompt += f"英文: `{s['en']}`\n" + if s['file']: + prompt += f"文件: {s['file']}\n" + if s['path']: + prompt += f"路径: {s['path']}\n" + prompt += "\n" + + prompt += """## 输出格式 + +请以JSON格式返回翻译结果，格式如下： +```json +{ + "translations": [ + { + "key": "原始key", + "en": "英文原文", + "translation": "中文翻译" + } + ] +} +``` + +只返回JSON，不要其他内容。""" + + return prompt + + + +def apply_translations(translations_path: Path, locale: str, translations) -> bool: + """Apply translations to locale files.""" + locale_dir = translations_path / locale + + # Validate input type + if isinstance(translations, dict): + # If top-level is a dict with 'translations' key, extract the list + if 'translations' in translations: + translations = translations['translations'] + else: + print("Error: Translation file is a dict but missing 'translations' key") + return False + elif not isinstance(translations, list): + print(f"Error: Translation file must be a list or dict, got {type(translations).__name__}") + return False + + # Load existing translations + static_trans = load_template(locale_dir, 'static.json') + js_trans = load_template(locale_dir, 'js.json') + effects_trans = load_template(locale_dir, 'effects.json') + palettes_trans = load_template(locale_dir, 'palettes.json') + + # Build lookup index for faster searching + # Maps en text -> list of (file_type, key) to handle duplicate en texts + en_lookup: dict[str, list[tuple[str, str]]] = {} + file_map = { + 'static': static_trans, + 'js': js_trans, + 'effects': effects_trans, + 'palettes': palettes_trans, + } + for file_type, file_data in file_map.items(): + for k, v in file_data.items(): + if isinstance(v, dict) and v.get('en'): + en_text = v['en'] + if en_text not in en_lookup: + en_lookup[en_text] = [] + en_lookup[en_text].append((file_type, k)) + + updated_count = 0 + + for t in translations: + if not isinstance(t, dict) or 'key' not in t: + continue + key = t['key'] + translation = t.get('translation', '') + + if not translation: + continue + + # Determine which file this belongs to + applied = False + for ft, fd in file_map.items(): + if key in fd: + fd[key]['translation'] = translation + updated_count += 1 + applied = True + break + if not applied: + # Try to find by en text using lookup index + en_text = t.get('en', '') + if en_text and en_text in en_lookup: + # Apply translation to ALL matching keys (same en text may appear in multiple places) + for file_type, lookup_key in en_lookup[en_text]: + target = file_map[file_type] + if lookup_key in target: + target[lookup_key]['translation'] = translation + updated_count += 1 + + # Save updated translations + if updated_count > 0: + save_template(locale_dir, 'static.json', static_trans) + save_template(locale_dir, 'js.json', js_trans) + save_template(locale_dir, 'effects.json', effects_trans) + save_template(locale_dir, 'palettes.json', palettes_trans) + print(f"✓ Applied {updated_count} translations") + else: + print("No translations to apply") + return True + + +def main(): + parser = argparse.ArgumentParser( + description='Auto-translate WLED strings') + parser.add_argument('--diff-file', + help='Path to diff.json from diff.py (required unless --apply)') + parser.add_argument('--translations-path', required=True, + help='Path to WLED-translations repository') + parser.add_argument('--locale', default='zh_CN', + help='Target locale (default: zh_CN)') + parser.add_argument('--apply', + help='Path to translation results JSON to apply') + parser.add_argument('--prompt-only', action='store_true', + help='Only generate translation prompt, do not translate') + args = parser.parse_args() + + translations_path = Path(args.translations_path) + + # If applying existing translations + if args.apply: + apply_path = Path(args.apply) + if not apply_path.exists(): + print(f"Error: Translation file not found: {apply_path}") + sys.exit(1) + translations = load_json(apply_path) + if not apply_translations(translations_path, args.locale, translations): + sys.exit(1) + return + + # Load diff if provided + added_keys = [] + if args.diff_file: + diff = load_diff(Path(args.diff_file)) + added_keys = diff.get('added', []) + + if not added_keys: + print("No new strings to translate") + return + + print(f"Found {len(added_keys)} new strings to translate") + else: + print("Error: --diff-file is required (unless using --apply)") + sys.exit(1) + + # Load template to get string details + template_path = translations_path / 'en_template_new' + if not template_path.exists(): + template_path = translations_path / 'en_template' + + # Collect strings to translate + strings_to_translate = [] + + # Load all template files + for filename in ['static.json', 'js.json', 'effects.json', 'palettes.json']: + template_data = load_template(template_path, filename) + + for key in added_keys: + info = extract_string_info(key, template_data) + if info and info['en']: + strings_to_translate.append(info) + + if not strings_to_translate: + print("No strings found in templates") + return + + # Generate translation prompt + prompt = create_translation_prompt(strings_to_translate, args.locale) + + if args.prompt_only: + print(prompt) + return + + # Save prompt and instructions + prompt_file = Path(tempfile.gettempdir()) / 'wled_translation_prompt.txt' + with open(prompt_file, 'w', encoding='utf-8') as f: + f.write(prompt) + + print(f"✓ Translation prompt saved to: {prompt_file}") + print(f"✓ {len(strings_to_translate)} strings ready for translation") + print("\nNext steps:") + print("1. Copy the prompt and send to LLM") + print("2. Save LLM response to: /tmp/wled_translation_result.json") + print("3. Run: python3 auto_translate.py --translations-path ~/WLED-translations --apply /tmp/wled_translation_result.json") + + +if __name__ == '__main__': + main() diff --git a/tools/i18n/build.py b/tools/i18n/build.py new file mode 100644 index 0000000000..08f2139ee8 --- /dev/null +++ b/tools/i18n/build.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +""" +WLED i18n Build Script (v4 - out-of-tree translation support) + +Generates translated HTML/JS files from English source + locale JSON. +Translations are loaded from external repos (WLED-translations) via +PlatformIO's custom_usermods mechanism, or from a local directory. + +Usage: + python3 build.py --locale zh_CN --source-dir wled00/data --output-dir build/i18n/zh_CN + +PlatformIO integration (out-of-tree): + # In platformio_override.ini: + # [env:esp32dev_zh_CN] + # extends = env:esp32dev + # custom_usermods = https://github.com/foxlesbiao/WLED-translations + # build_flags = ${env:esp32dev.build_flags} -D WLED_LOCALE=zh_CN + # extra_scripts = pre:tools/i18n/build.py +""" + +import json +import os +import re +import sys +import tempfile +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_DIR = SCRIPT_DIR.parent.parent # WLED root +DATA_DIR = PROJECT_DIR / "wled00" / "data" + +LOCALE_LANG = { + 'zh_CN': 'zh', + 'zh_TW': 'zh-TW', + 'de_DE': 'de', + 'fr_FR': 'fr', + 'ja_JP': 'ja', + 'ko_KR': 'ko', + 'es_ES': 'es', + 'pt_BR': 'pt-BR', + 'ru_RU': 'ru', +} + + +def find_translations_dir(locale, translations_dir=None): + """Locate the translations directory for a locale. + + Search order: + 1. Explicit --translations-dir argument + 2. PlatformIO libdeps (out-of-tree usermod: .pio/libdeps//WLED-translations/) + 3. Local fallback (tools/i18n/locales/) + """ + # 1. Explicit path + if translations_dir: + p = Path(translations_dir) + if p.exists(): + return p + print(f"Warning: --translations-dir not found: {p}", file=sys.stderr) + + # 2. PlatformIO libdeps (out-of-tree usermod) + pio_libdeps = PROJECT_DIR / ".pio" / "libdeps" + if pio_libdeps.exists(): + for env_dir in pio_libdeps.iterdir(): + if not env_dir.is_dir(): + continue + candidate = env_dir / "WLED-translations" + if candidate.exists() and (candidate / locale).exists(): + return candidate / locale + for subdir in env_dir.iterdir(): + if not subdir.is_dir(): + continue + lib_json = subdir / "library.json" + if lib_json.exists(): + try: + with open(lib_json) as f: + meta = json.load(f) + if meta.get("name") == "WLED-translations": + if (subdir / locale).exists(): + return subdir / locale + except (json.JSONDecodeError, KeyError): + pass + + # 3. Local fallback + local = SCRIPT_DIR / "locales" + if (local / f"{locale}.json").exists(): + return local + + return None + + +def load_translations(locale, translations_dir=None): + """Load translation JSON for the given locale, keyed by file.""" + tdir = find_translations_dir(locale, translations_dir) + if tdir is None: + print(f"Error: No translations found for locale '{locale}'.", file=sys.stderr) + print(f" Searched:", file=sys.stderr) + print(f" --translations-dir (if provided)", file=sys.stderr) + print(f" .pio/libdeps/*/WLED-translations/{locale}/", file=sys.stderr) + print(f" tools/i18n/locales/{locale}.json", file=sys.stderr) + sys.exit(1) + + tdir = Path(tdir) + merged = {} + + if tdir.is_file(): + files_to_load = [tdir] + elif tdir.is_dir(): + files_to_load = sorted(tdir.glob("*.json")) + single = tdir / f"{locale}.json" + if single.exists() and single not in files_to_load: + files_to_load = [single] + else: + print(f"Error: Translations path is neither file nor directory: {tdir}", file=sys.stderr) + sys.exit(1) + + for jf in files_to_load: + if jf.name == "metadata.json": + continue + with open(jf, 'r', encoding='utf-8') as f: + data = json.load(f) + for fname, entries in data.items(): + if fname not in merged: + merged[fname] = {} + for key, entry in entries.items(): + trans = entry.get('translation', '').strip() + if trans: + merged[fname][key] = { + 'original': entry.get('en', ''), + 'translation': trans, + } + + if not merged: + print(f"Warning: No translations loaded for {locale} from {tdir}", file=sys.stderr) + + return merged + + +def split_script_blocks(content): + """Split content into (non_script, script) segments for safe processing.""" + segments = [] + pattern = re.compile(r'(]*>)(.*?)()', re.DOTALL | re.IGNORECASE) + last_end = 0 + + for match in pattern.finditer(content): + before = content[last_end:match.start()] + if before: + segments.append((before, False)) + segments.append((match.group(0), True)) + last_end = match.end() + + after = content[last_end:] + if after: + segments.append((after, False)) + + return segments + + +def replace_html_text(content, original, translated): + """Replace HTML text content using exact string matching.""" + escaped = re.escape(original) + total = 0 + + p1 = re.compile(r'(>)\s*(' + escaped + r')\s*(' + translated + r'\g<3>', content) + total += n + + p2 = re.compile(r'()\s*(' + escaped + r')\s*(' + translated + r'\g<3>', content) + total += n + + p3 = re.compile(r'^(\s*)(' + escaped + r')(\s*)$', re.MULTILINE) + content, n = p3.subn(r'\g<1>' + translated + r'\g<3>', content) + total += n + + return content, total + + +def replace_html_attr(content, attr, original, translated): + """Replace HTML attribute value using exact string matching.""" + pattern = re.compile( + r'(' + re.escape(attr) + r'\s*=\s*")(' + re.escape(original) + r')(")', + re.IGNORECASE + ) + new_content, count = pattern.subn(r'\g<1>' + translated + r'\g<3>', content) + return new_content, count + + +def replace_js_in_block(script_block, original, translated): + """Replace a JS string literal within a single block.""" + for quote in ['"', "'", '`']: + escaped = re.escape(original) + pattern = re.compile( + r'([' + quote + r'])(' + escaped + r')([' + quote + r'])' + ) + new_block, count = pattern.subn( + r'\g<1>' + translated + r'\g<3>', + script_block, count=1 + ) + if count > 0: + return new_block, count + + return script_block, 0 + + +def apply_translations(content, file_key, translations, lang_code): + """Apply all translations to a file's content.""" + total = 0 + + content = re.sub( + r'(]*lang\s*=\s*")([^"]+)(")', + r'\g<1>' + lang_code + r'\g<3>', + content, count=1 + ) + + segments = split_script_blocks(content) + file_translations = translations.get(file_key, {}) + new_segments = [] + + for segment_text, is_script in segments: + if is_script: + for key, entry in file_translations.items(): + if key.startswith('js:'): + segment_text, count = replace_js_in_block( + segment_text, entry['original'], entry['translation'] + ) + total += count + else: + for key, entry in file_translations.items(): + if key.startswith('html:'): + parts = key.split(':') + attr_name = parts[-1] + + if attr_name == 'text': + segment_text, count = replace_html_text( + segment_text, entry['original'], entry['translation'] + ) + total += count + elif attr_name in ('placeholder', 'title', 'alt', 'aria-label'): + segment_text, count = replace_html_attr( + segment_text, attr_name, entry['original'], entry['translation'] + ) + total += count + + new_segments.append(segment_text) + + return ''.join(new_segments), total + + +def build_locale(locale, source_dir=None, output_dir=None, translations_dir=None): + """Build translated HTM files for a given locale.""" + file_translations = load_translations(locale, translations_dir) + if not file_translations: + print(f"Warning: No translations found for {locale}") + return 0 + + lang_code = LOCALE_LANG.get(locale, locale.split('_')[0]) + src_dir = Path(source_dir) if source_dir else DATA_DIR + + htm_files = sorted(src_dir.glob('*.htm')) + if not htm_files: + print(f"Error: No .htm files found in {src_dir}", file=sys.stderr) + return 0 + + if output_dir: + out_dir = Path(output_dir) + else: + out_dir = Path(tempfile.mkdtemp(prefix=f'wled_i18n_{locale}_')) + print(f"[i18n] No --output-dir specified, using temp: {out_dir}") + + out_dir.mkdir(parents=True, exist_ok=True) + + if out_dir.resolve() == src_dir.resolve(): + print(f"WARNING: Output dir equals source dir ({src_dir}).", file=sys.stderr) + print(f" English source files will be overwritten!", file=sys.stderr) + print(f" Pass --output-dir to a different location.", file=sys.stderr) + + total_applied = 0 + + for filepath in htm_files: + file_key = filepath.name + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + + content, applied = apply_translations(content, file_key, file_translations, lang_code) + + out_path = out_dir / file_key + with open(out_path, 'w', encoding='utf-8') as f: + f.write(content) + + status = f"{applied} translations" if applied else "no changes" + print(f" {file_key}: {status}") + total_applied += applied + + print(f"\nTotal: {total_applied} translations applied across {len(htm_files)} files") + print(f"Output: {out_dir}") + return total_applied + + +def validate_translations(locale, translations_dir=None): + """Validate translation completeness against English source files.""" + file_translations = load_translations(locale, translations_dir) + if not file_translations: + print(f"No translations for {locale}") + return False + + total_keys = 0 + total_translated = 0 + + for fname, entries in file_translations.items(): + translated = sum(1 for e in entries.values() if e.get('translation', '').strip()) + total = len(entries) + total_keys += total + total_translated += translated + status = "OK" if translated == total else f"{total - translated} missing" + print(f" {fname}: {translated}/{total} ({status})") + + pct = (total_translated / total_keys * 100) if total_keys else 0 + print(f"\nTotal: {total_translated}/{total_keys} ({pct:.1f}%)") + return total_translated == total_keys + + +def main(): + import argparse + parser = argparse.ArgumentParser(description='Build translated WLED Web UI files') + parser.add_argument('--locale', required=True, help='Locale code (e.g. zh_CN)') + parser.add_argument('--source-dir', default=None, help='Source directory (default: wled00/data/)') + parser.add_argument('--output-dir', default=None, help='Output directory (default: temp dir)') + parser.add_argument('--translations-dir', default=None, + help='Translations directory (default: auto-detect via PlatformIO libdeps)') + parser.add_argument('--validate', action='store_true', + help='Validate translation completeness (no build)') + args = parser.parse_args() + + if args.validate: + print(f"Validating translations for {args.locale}") + print("=" * 40) + ok = validate_translations(args.locale, args.translations_dir) + sys.exit(0 if ok else 1) + + print(f"WLED i18n Build — {args.locale}") + print("=" * 40) + + count = build_locale(args.locale, args.source_dir, args.output_dir, args.translations_dir) + if count == 0: + print("\nWarning: No translations applied!") + + +# PlatformIO pre-build integration +def pre_build(source, target, env): + """PlatformIO pre-build script entry point.""" + import re as _re + locale = None + for flag in env.get('BUILD_FLAGS', []): + m = _re.match(r'-D\s*WLED_LOCALE=(\S+)', flag) + if m: + locale = m.group(1).strip() + break + + if not locale: + print("[i18n] No WLED_LOCALE set, skipping translation") + return + + print(f"[i18n] Building with locale: {locale}") + build_dir = Path(env.subst('$BUILD_DIR')) / 'i18n' / locale + build_locale(locale, output_dir=build_dir) + + +try: + Import("env") + env.AddPreAction("buildprog", pre_build) +except NameError: + if __name__ == '__main__': + main() diff --git a/tools/i18n/check_upstream.py b/tools/i18n/check_upstream.py new file mode 100644 index 0000000000..7a9ca629c1 --- /dev/null +++ b/tools/i18n/check_upstream.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""WLED Upstream Update Checker + +Checks if WLED upstream has new commits since last extraction. + +Usage: + python3 check_upstream.py --wled-path ~/WLED --translations-path ~/WLED-translations +""" + +import json +import subprocess +import sys +import argparse +from pathlib import Path + + +def get_upstream_commit(wled_path): + """Get latest upstream commit hash and message.""" + # Try common remote/branch combinations + candidates = ['upstream/main', 'upstream/master', 'origin/main', 'origin/master'] + + # Also check which remotes actually exist + remotes_result = subprocess.run( + ['git', 'remote'], cwd=wled_path, + capture_output=True, text=True, timeout=10) + if remotes_result.returncode == 0: + remotes = remotes_result.stdout.strip().split('\n') + if 'upstream' not in remotes and 'origin' not in remotes: + print(f"Warning: No 'upstream' or 'origin' remote found. Available: {remotes}") + return None, None + + for ref in candidates: + result = subprocess.run( + ['git', 'rev-parse', ref], + cwd=wled_path, capture_output=True, text=True, timeout=10 + ) + if result.returncode == 0 and result.stdout.strip(): + full_hash = result.stdout.strip() + # Get commit message separately + msg_result = subprocess.run( + ['git', 'log', '--format=%s', '-1', ref], + cwd=wled_path, capture_output=True, text=True, timeout=10 + ) + msg = msg_result.stdout.strip() if msg_result.returncode == 0 else '' + return full_hash, msg + + print(f"Error: Could not find any upstream branch. Tried: {candidates}") + return None, None + + +def get_last_extracted_commit(translations_path): + """Get the commit hash from last extraction metadata.""" + metadata_file = translations_path / 'en_template' / 'metadata.json' + if not metadata_file.exists(): + return None + + try: + with open(metadata_file, encoding='utf-8') as f: + metadata = json.load(f) + return metadata.get('wled_commit') + except (json.JSONDecodeError, IOError): + return None + + +def main(): + parser = argparse.ArgumentParser( + description='Check WLED upstream for updates') + parser.add_argument('--wled-path', required=True, + help='Path to WLED repository') + parser.add_argument('--translations-path', required=True, + help='Path to WLED-translations repository') + parser.add_argument('--json', action='store_true', + help='Output as JSON') + args = parser.parse_args() + + wled_path = Path(args.wled_path) + translations_path = Path(args.translations_path) + + # Get current upstream commit + upstream_hash, upstream_msg = get_upstream_commit(wled_path) + if not upstream_hash: + print("Error: Could not get upstream commit") + sys.exit(1) + + # Get last extracted commit + last_hash = get_last_extracted_commit(translations_path) + + # Compare (handle both short and full hashes) + if last_hash is None: + has_updates = True + else: + has_updates = ( + not upstream_hash.startswith(last_hash) + and not last_hash.startswith(upstream_hash) + ) + + result = { + 'has_updates': has_updates, + 'upstream_commit': upstream_hash, + 'upstream_message': upstream_msg, + 'last_extracted_commit': last_hash, + 'translations_path': str(translations_path) + } + + if args.json: + print(json.dumps(result, indent=2, ensure_ascii=False)) + else: + if has_updates: + print(f"✓ New upstream commit: {upstream_hash}") + print(f" Message: {upstream_msg}") + if last_hash: + print(f" Last extracted: {last_hash}") + else: + print(f" No previous extraction found") + else: + print(f"No updates. Current: {upstream_hash}") + + +if __name__ == '__main__': + main() diff --git a/tools/i18n/common.py b/tools/i18n/common.py new file mode 100644 index 0000000000..e86dd73c9b --- /dev/null +++ b/tools/i18n/common.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Common utilities for WLED translation scripts. + +Shared functions to avoid code duplication across scripts. +""" + +import os +import json +import subprocess +import hashlib +from pathlib import Path +from typing import Any, Union + + +def load_json(path: Path) -> Any: + """Load JSON file with utf-8 encoding.""" + with open(path, encoding='utf-8') as f: + return json.load(f) + + +def save_json(path: Path, data: Any) -> None: + """Save JSON file with utf-8 encoding.""" + with open(path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def run_command(cmd: list[str], cwd: Union[str, os.PathLike, None] = None, + allow_nonzero_stdout: bool = False, + timeout: float = 300) -> tuple[bool, str]: + """Run a command and return (success, stdout). + + Args: + cmd: Command and arguments as list. + cwd: Working directory for the command. + allow_nonzero_stdout: If True, treat non-zero exit code with stdout as success + (used for diff.py which returns 1 when changes found). + timeout: Max seconds to wait (default 300). None for no limit. + + Returns: + (True, stdout) on success, (False, stderr_or_stdout) on failure. + """ + if not cmd: + return False, 'Empty command' + try: + result = subprocess.run(cmd, capture_output=True, text=True, + cwd=cwd, timeout=timeout) + except FileNotFoundError: + return False, f'Command not found: {cmd[0]}' + except OSError as e: + return False, str(e) + except subprocess.TimeoutExpired: + return False, f'Command timed out after {timeout}s: {cmd[0]}' + + if result.returncode != 0: + if allow_nonzero_stdout and result.stdout: + return True, result.stdout + return False, result.stderr or result.stdout + return True, result.stdout + + +def compute_hash(text: str) -> str: + """Compute short MD5 hash for text content (first 8 hex chars).""" + return hashlib.md5(text.encode('utf-8'), usedforsecurity=False).hexdigest()[:8] diff --git a/tools/i18n/convert_template.py b/tools/i18n/convert_template.py new file mode 100644 index 0000000000..03ac1d8344 --- /dev/null +++ b/tools/i18n/convert_template.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""WLED Template Format Converter + +Converts between different template formats: +- New format (extract.py): Nested by file, CSS selector keys +- Old format (v2 toolchain): Flat, file:hash keys + +Usage: + python3 convert_template.py --input new_template.json --output old_template.json --to-old + python3 convert_template.py --input old_template/ --output new_template.json --to-new +""" + +import sys +import argparse +from pathlib import Path +from typing import Any, Callable, Dict +from common import load_json, save_json, compute_hash + + +def extract_path_from_css_key(css_key: str) -> str: + """Extract meaningful path from CSS selector key. + + Handles formats like: + - "html:html > body > h1:text" -> "html > body > h1" + - "html:settings_2D.htm:06ce2a25" -> "settings_2D.htm" + - "js:edit.htm:77:ef93d8c8" -> "edit.htm:77" + - "html:button#power:title" -> "button#power" + - "html:div#fxlist > div > label > div > span:text" -> "div#fxlist > div > label > div > span" + """ + # Handle old format keys (js:file:line:hash or html:file:hash) + if css_key.startswith('js:') or css_key.startswith('html:'): + parts = css_key.split(':') + if len(parts) >= 3: + # js:edit.htm:77:ef93d8c8 -> edit.htm:77 + # html:settings_2D.htm:06ce2a25 -> settings_2D.htm + return ':'.join(parts[1:-1]) if len(parts) > 3 else parts[1] + return css_key + + # Handle CSS selector format + # Remove attribute suffixes (:text, :title, :placeholder, :aria-label, :nth-of-type) + path = css_key + for suffix in [':text', ':title', ':placeholder', ':aria-label']: + if path.endswith(suffix): + path = path[:-len(suffix)] + break + + # Handle :nth-of-type(N) - remove the whole suffix + if ':nth-of-type(' in path: + path = path[:path.rfind(':nth-of-type(')] + + return path + + +def new_to_old(new_template: Dict) -> Dict: + """Convert new format (nested by file) to old format (flat with file:hash keys).""" + old_template = {} + + for filename, file_entries in new_template.items(): + if not isinstance(file_entries, dict): + continue + + for css_key, entry in file_entries.items(): + if not isinstance(entry, dict): + continue + + en_text = entry.get('en', '') + if not en_text: + continue + + # Check if this is already in old format (e.g., js:edit.htm:77:ef93d8c8) + if css_key.startswith('js:') or css_key.startswith('html:'): + # Already in old format, use as-is + old_key = css_key + else: + # Create old format key: file:hash + # Include css_key in hash to avoid collision when same en_text appears in multiple selectors + content_hash = compute_hash(f"{css_key}:{en_text}") + old_key = f"{filename}:{content_hash}" + + # Extract path from CSS selector + path = extract_path_from_css_key(css_key) + + # Determine type + entry_type = entry.get('type', 'html_text') + if css_key.startswith('js:'): + entry_type = 'js' + + old_template[old_key] = { + 'en': en_text, + 'file': filename, + 'path': path, + 'type': entry_type + } + + # Copy translation if exists + if entry.get('translation'): + old_template[old_key]['translation'] = entry['translation'] + + return old_template + + +def old_to_new(old_template: Dict) -> Dict: + """Convert old format (flat) to new format (nested by file).""" + new_template = {} + + for key, entry in old_template.items(): + if not isinstance(entry, dict): + continue + + filename = entry.get('file', 'unknown') + en_text = entry.get('en', '') + path = entry.get('path', '') + entry_type = entry.get('type', 'html_text') + + if not en_text: + continue + + # Create CSS selector from path + if entry_type == 'js': + # JS entries keep their original key format + css_key = key + elif path: + # Reconstruct CSS selector + css_key = f"html:{path}:text" + else: + css_key = f"html:unknown:{key}" + + if filename not in new_template: + new_template[filename] = {} + + new_template[filename][css_key] = { + 'en': en_text, + 'translation': entry.get('translation', ''), + 'context': f"{filename}: ({entry_type})" + } + + return new_template + + +def _merge_extra(old_template: dict, merge_path: Path, + build_entry: Callable[[dict], dict], label: str) -> None: + """Merge extra entries (effects/palettes/index_js) into old_template.""" + if not merge_path.exists(): + return + data = load_json(merge_path) + for key, entry in data.items(): + if isinstance(entry, dict): + old_template[key] = build_entry(entry) + print(f"✓ Merged {len(data)} {label}") + + +def load_old_template_dir(template_dir: Path) -> Dict: + """Load old format template from directory (static.json, js.json, etc.).""" + merged = {} + + for json_file in sorted(template_dir.glob('*.json')): + if json_file.name == 'metadata.json': + continue + + data = load_json(json_file) + if isinstance(data, dict): + merged.update(data) + + return merged + + +def main(): + parser = argparse.ArgumentParser( + description='Convert WLED template formats') + parser.add_argument('--input', required=True, + help='Input file or directory') + parser.add_argument('--output', required=True, + help='Output file') + direction = parser.add_mutually_exclusive_group(required=True) + direction.add_argument('--to-old', action='store_true', + help='Convert new format to old format') + direction.add_argument('--to-new', action='store_true', + help='Convert old format to new format') + parser.add_argument('--merge-effects', + help='Merge effects.json into output') + parser.add_argument('--merge-palettes', + help='Merge palettes.json into output') + parser.add_argument('--merge-index-js', + help='Merge index_js.json into output') + args = parser.parse_args() + + input_path = Path(args.input) + output_path = Path(args.output) + + if not input_path.exists(): + print(f"Error: Input path does not exist: {input_path}") + return 1 + + if args.to_old: + # Load new format (single file or directory) + if input_path.is_dir(): + # Load all JSON files in directory + new_template = {} + for json_file in input_path.glob('*.json'): + if json_file.name in ['effects.json', 'palettes.json', 'index_js.json', 'metadata.json']: + continue + data = load_json(json_file) + if isinstance(data, dict): + new_template.update(data) + else: + new_template = load_json(input_path) + + old_template = new_to_old(new_template) + + # Merge extra data if provided + if args.merge_effects: + _merge_extra(old_template, Path(args.merge_effects), + lambda e: {'en': e.get('name', ''), 'type': 'effect', + 'file': 'effects.json', + 'metadata': e.get('metadata', ''), + 'full': e.get('full', '')}, + 'effects') + if args.merge_palettes: + _merge_extra(old_template, Path(args.merge_palettes), + lambda e: {'en': e.get('name', ''), 'type': 'palette', + 'file': 'palettes.json', + 'index': e.get('index', 0)}, + 'palettes') + if args.merge_index_js: + _merge_extra(old_template, Path(args.merge_index_js), + lambda e: {'en': e.get('en', ''), 'type': 'js', + 'file': 'index.js', + 'context': e.get('context', '')}, + 'index.js strings') + + save_json(output_path, old_template) + print(f"✓ Converted to old format: {len(old_template)} entries") + + elif args.to_new: + # Load old format + if input_path.is_dir(): + old_template = load_old_template_dir(input_path) + else: + old_template = load_json(input_path) + + new_template = old_to_new(old_template) + save_json(output_path, new_template) + print(f"✓ Converted to new format: {sum(len(v) for v in new_template.values())} entries in {len(new_template)} files") + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/i18n/diff.py b/tools/i18n/diff.py new file mode 100644 index 0000000000..f1bc05b3ce --- /dev/null +++ b/tools/i18n/diff.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +WLED i18n Translation Diff Tool +Compares two template versions and reports changes. + +Usage: + python3 diff.py --old v0.15.0.json --new v0.16.0.json + python3 diff.py --old en_template_old/ --new en_template/ --locale zh_CN + +Output: + JSON with added/removed/modified strings and stats. +""" + +import json +import sys +from pathlib import Path + + +def load_template(path): + """Load a single JSON template file.""" + with open(path, encoding='utf-8') as f: + return json.load(f) + + +def load_templates_from_dir(dir_path): + """Load all JSON files from a directory and merge into single dict.""" + result = {} + dir_path = Path(dir_path) + for json_file in sorted(dir_path.glob('*.json')): + if json_file.name == 'metadata.json': + continue + data = load_template(json_file) + result.update(data) + return result + + +def diff_templates(old, new): + """Compare two template dicts and return differences.""" + old_keys = set(old.keys()) + new_keys = set(new.keys()) + + added = new_keys - old_keys + removed = old_keys - new_keys + common = old_keys & new_keys + + modified = [] + for k in sorted(common): + old_en = old[k].get('en', '') + new_en = new[k].get('en', '') + if old_en != new_en: + modified.append({ + 'key': k, + 'old': old_en, + 'new': new_en, + }) + + return { + 'added': sorted(added), + 'removed': sorted(removed), + 'modified': modified, + 'stats': { + 'old_count': len(old_keys), + 'new_count': len(new_keys), + 'added': len(added), + 'removed': len(removed), + 'modified': len(modified), + } + } + + +def main(): + import argparse + parser = argparse.ArgumentParser( + description='Diff WLED i18n templates between versions' + ) + parser.add_argument( + '--old', required=True, + help='Old template JSON file or directory' + ) + parser.add_argument( + '--new', required=True, + help='New template JSON file or directory' + ) + parser.add_argument( + '--locale', + help='Locale to compare (e.g., zh_CN). Only used with directory mode.' + ) + args = parser.parse_args() + + old_path = Path(args.old) + new_path = Path(args.new) + + # Load old templates + if old_path.is_dir(): + if args.locale: + locale_dir = old_path / args.locale + if locale_dir.exists(): + old = load_templates_from_dir(locale_dir) + else: + old = load_templates_from_dir(old_path) + else: + old = load_templates_from_dir(old_path) + else: + old = load_template(old_path) + + # Load new templates + if new_path.is_dir(): + if args.locale: + locale_dir = new_path / args.locale + if locale_dir.exists(): + new = load_templates_from_dir(locale_dir) + else: + new = load_templates_from_dir(new_path) + else: + new = load_templates_from_dir(new_path) + else: + new = load_template(new_path) + + # Compute diff + result = diff_templates(old, new) + + # Output + print(json.dumps(result, indent=2, ensure_ascii=False)) + + # Exit code: 0 if no changes, 1 if changes found + stats = result['stats'] + if stats['added'] or stats['removed'] or stats['modified']: + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/i18n/extract.py b/tools/i18n/extract.py new file mode 100644 index 0000000000..3e656089ff --- /dev/null +++ b/tools/i18n/extract.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +""" +WLED i18n String Extractor +Extracts translatable strings from WLED Web UI HTML files. + +Usage: python3 extract.py [--locale zh_CN] + +Outputs: locales/.json (or locales/_template.json if no locale specified) + +Handles three layers: +1. Static HTML text (BeautifulSoup DOM parsing) +2. JS strings in