fix: read and write skills, benchmark data and run files as UTF-8

scripts/eval_only.py

@@ -48,6 +48,7 @@
 )
 from skillopt.model.common import default_model_for_backend, normalize_backend_name
 from skillopt.utils import compute_score
+from skillopt.utils.text_io import read_text
 
 _OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"}
 _ROLE_BACKEND_DEFAULTS = (None, "", "openai_chat")
@@ -608,8 +609,7 @@ def main() -> None:
 
     # Load skill
     skill_path = os.path.abspath(args.skill)
-    with open(skill_path) as f:
-        skill_content = f.read()
+    skill_content = read_text(skill_path)
     print(f"  [skill] {skill_path} ({len(skill_content)} chars)")
 
     # Configure models
@@ -766,7 +766,7 @@ def main() -> None:
         "hard": hard,
         "soft": soft,
     }
-    with open(os.path.join(out_root, "eval_summary.json"), "w") as f:
+    with open(os.path.join(out_root, "eval_summary.json"), "w", encoding="utf-8") as f:
         json.dump(summary, f, indent=2, ensure_ascii=False)
 
     print(f"  Saved to: {out_root}")

skillopt/envs/alfworld/adapter.py

@@ -18,6 +18,7 @@
     TASKS,
 )
 from skillopt.utils import compute_score
+from skillopt.utils.text_io import read_lines
 
 
 @dataclass(frozen=True)
@@ -329,12 +330,11 @@ def rollout(
         # Resume support
         if os.path.exists(results_path):
             existing: list[dict] = []
-            with open(results_path) as f:
-                for line in f:
-                    try:
-                        existing.append(json.loads(line))
-                    except Exception:
-                        pass
+            for line in read_lines(results_path):
+                try:
+                    existing.append(json.loads(line))
+                except Exception:
+                    pass
             if existing:
                 return existing
 
@@ -355,7 +355,7 @@ def rollout(
                 result_ids=getattr(env_manager, "_skillopt_result_ids", None),
             )
 
-        with open(results_path, "w") as f:
+        with open(results_path, "w", encoding="utf-8") as f:
             for r in results:
                 f.write(json.dumps(r, ensure_ascii=False) + "\n")
 

skillopt/envs/livemathematicianbench/dataloader.py

@@ -17,7 +17,7 @@
 
 
 def _load_json(path: str) -> Any:
-    with open(path) as f:
+    with open(path, encoding="utf-8") as f:
         return json.load(f)
 
 

skillopt/envs/livemathematicianbench/rollout.py

@@ -11,6 +11,7 @@
 from skillopt.model import chat_target, get_target_backend, is_target_exec_backend
 from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec
 from skillopt.prompts import load_prompt
+from skillopt.utils.text_io import read_lines
 
 
 def _build_system(skill_content: str) -> str:
@@ -202,7 +203,7 @@ def process_one(
                 f"Exact Match: {eval_result['em']}"
             )
             conversation.append({"role": "system", "content": eval_detail})
-            with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
+            with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
                 json.dump(conversation, f, ensure_ascii=False, indent=2)
             return result
 
@@ -280,7 +281,7 @@ def process_one(
         )
         conversation.append({"role": "system", "content": eval_detail})
 
-        with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
+        with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
             json.dump(conversation, f, ensure_ascii=False, indent=2)
 
     except Exception as e:  # noqa: BLE001
@@ -317,14 +318,13 @@ def run_batch(
     done_ids: set[str] = set()
     existing: list[dict] = []
     if os.path.exists(results_path):
-        with open(results_path) as f:
-            for line in f:
-                try:
-                    r = json.loads(line)
-                    done_ids.add(str(r["id"]))
-                    existing.append(r)
-                except Exception:
-                    pass
+        for line in read_lines(results_path):
+            try:
+                r = json.loads(line)
+                done_ids.add(str(r["id"]))
+                existing.append(r)
+            except Exception:
+                pass
 
     pending = [it for it in items if str(it["id"]) not in done_ids]
     if not pending:
@@ -380,7 +380,7 @@ def _error_result(it: dict, exc: Exception) -> dict:
         res["fail_reason"] = f"error: {type(exc).__name__}: {exc}"
         return res
 
-    with open(results_path, "a") as outf:
+    with open(results_path, "a", encoding="utf-8") as outf:
         ex = ThreadPoolExecutor(max_workers=workers)
         try:
             futs = {

skillopt/envs/searchqa/dataloader.py

@@ -10,7 +10,7 @@
 
 def _load_items(path: str) -> list[dict]:
     """Load items from JSON or JSONL file."""
-    with open(path) as f:
+    with open(path, encoding="utf-8") as f:
         content = f.read().strip()
     try:
         data = json.loads(content)

skillopt/envs/searchqa/rollout.py

@@ -21,6 +21,7 @@
 from skillopt.model import chat_target, is_target_exec_backend
 from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec
 from skillopt.prompts import load_prompt
+from skillopt.utils.text_io import read_lines
 
 # ── Prompt templates ─────────────────────────────────────────────────────────
 
@@ -233,11 +234,11 @@ def process_one(
             result["agent_ok"] = True
             result["n_turns"] = len(conversation)
 
-            with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w") as f:
+            with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f:
                 f.write(system)
-            with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w") as f:
+            with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f:
                 f.write(user)
-            with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
+            with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
                 json.dump(conversation, f, ensure_ascii=False, indent=2)
 
             eval_result = evaluate(response, gold_answers)
@@ -261,7 +262,7 @@ def process_one(
                 f"F1: {eval_result['f1']:.4f}"
             )
             conversation.append({"role": "system", "content": eval_detail})
-            with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
+            with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
                 json.dump(conversation, f, ensure_ascii=False, indent=2)
             return result
 
@@ -310,11 +311,11 @@ def process_one(
         result["n_turns"] = len(conversation)
 
         # Save conversation
-        with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w") as f:
+        with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f:
             f.write(system)
-        with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w") as f:
+        with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f:
             f.write(user)
-        with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
+        with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
             json.dump(conversation, f, ensure_ascii=False, indent=2)
 
         # Evaluate
@@ -346,7 +347,7 @@ def process_one(
             "content": eval_detail,
         })
         # Re-save enriched conversation
-        with open(os.path.join(pred_dir, "conversation.json"), "w") as f:
+        with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
             json.dump(conversation, f, ensure_ascii=False, indent=2)
 
     except Exception as e:  # noqa: BLE001
@@ -380,14 +381,13 @@ def run_batch(
     done_ids: set[str] = set()
     existing: list[dict] = []
     if os.path.exists(results_path):
-        with open(results_path) as f:
-            for line in f:
-                try:
-                    r = json.loads(line)
-                    done_ids.add(str(r["id"]))
-                    existing.append(r)
-                except Exception:
-                    pass
+        for line in read_lines(results_path):
+            try:
+                r = json.loads(line)
+                done_ids.add(str(r["id"]))
+                existing.append(r)
+            except Exception:
+                pass
 
     pending = [it for it in items if str(it["id"]) not in done_ids]
     if not pending:
@@ -441,7 +441,7 @@ def _run_one(item: dict) -> dict:
             max_completion_tokens,
         )
 
-    with open(results_path, "a") as outf:
+    with open(results_path, "a", encoding="utf-8") as outf:
         ex = ThreadPoolExecutor(max_workers=workers)
         try:
             futs = {ex.submit(_run_one, it): it for it in pending}

skillopt/envs/spreadsheetbench/adapter.py

@@ -17,6 +17,7 @@
     run_spreadsheet_batch_codegen,
 )
 from skillopt.model import get_target_backend, is_target_exec_backend
+from skillopt.utils.text_io import read_lines
 
 
 # Task types used for per-category breakdowns
@@ -109,12 +110,11 @@ def rollout(
         # Resume support
         if os.path.exists(results_path):
             existing: list[dict] = []
-            with open(results_path) as f:
-                for line in f:
-                    try:
-                        existing.append(json.loads(line))
-                    except (ValueError, RecursionError, TypeError):
-                        pass
+            for line in read_lines(results_path):
+                try:
+                    existing.append(json.loads(line))
+                except (ValueError, RecursionError, TypeError):
+                    pass
             if existing:
                 return existing
 
@@ -149,7 +149,7 @@ def rollout(
                 diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"),
             )
 
-        with open(results_path, "w") as f:
+        with open(results_path, "w", encoding="utf-8") as f:
             for r in results:
                 f.write(json.dumps(r, ensure_ascii=False) + "\n")
 

skillopt/envs/spreadsheetbench/react_agent.py

@@ -160,7 +160,7 @@ def _write_file(path: str, content: str, work_dir: str) -> str:
         parent = os.path.dirname(full_path)
         if parent:
             os.makedirs(parent, exist_ok=True)
-        with open(full_path, "w") as f:
+        with open(full_path, "w", encoding="utf-8") as f:
             f.write(content)
         return f"File written: {full_path} ({len(content)} chars)"
     except Exception as e:  # noqa: BLE001
@@ -176,7 +176,7 @@ def _auto_verify(work_dir: str) -> str:
     sol_path = os.path.join(work_dir, "solution.py")
     output_path = None
     if os.path.exists(sol_path):
-        with open(sol_path) as f:
+        with open(sol_path, encoding="utf-8") as f:
             for line in f:
                 stripped = line.strip()
                 if stripped.startswith("OUTPUT_PATH"):