From 69d34f07a86c3a09edd642d4a5c381d1db39f892 Mon Sep 17 00:00:00 2001 From: Tristen Pierson Date: Thu, 21 May 2026 13:53:29 -0400 Subject: [PATCH 1/3] =?UTF-8?q?fix(agent/run):=20silent=20no-response=20+?= =?UTF-8?q?=20provider=20visibility=20=E2=80=94=20closes=20#186-followup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root causes of specsmith run returning nothing: 1. DEFAULT_OLLAMA_MODEL=qwen2.5:7b not installed; Ollama 404 silently swallowed 2. run_chat returning None gave user zero feedback 3. EventEmitter always wrote JSONL even in interactive terminal mode Fixes: - chat_runner: _pick_ollama_model() queries /api/tags and selects the first installed model from a preference list (lighter models first); falls back to DEFAULT_OLLAMA_MODEL only when the API is unreachable or list is empty. SPECSMITH_OLLAMA_MODEL env var still wins unconditionally. - runner: _handle_command() prints an actionable hint when run_chat returns None — explains whether Ollama is running (with model name) or no provider is available at all. - events: PlainTextEmitter subclass — token() writes raw text to stdout, emit() is a no-op. Used by AgentRunner when json_events=False so LLM responses render as readable prose instead of JSONL blobs. - runner: check_providers() probes all four providers (Ollama, Anthropic, OpenAI, Gemini) and returns ProviderStatus(name, available, model, note). - runner: _print_banner() shows a provider status table in interactive mode so the user knows upfront which model will respond before typing. - cli: specsmith run --check validates providers and exits 0/1 without starting the REPL. - run_interactive: adds a trailing newline after each streamed response so the next prompt doesn't bleed onto the last output line. 19 new tests in test_agent_run_feedback.py (854 total, 0 failures). Co-Authored-By: Oz --- src/specsmith/agent/chat_runner.py | 63 ++++- src/specsmith/agent/events.py | 22 +- src/specsmith/agent/runner.py | 248 ++++++++++++++++++-- src/specsmith/cli.py | 39 ++++ tests/test_agent_run_feedback.py | 354 +++++++++++++++++++++++++++++ 5 files changed, 695 insertions(+), 31 deletions(-) create mode 100644 tests/test_agent_run_feedback.py diff --git a/src/specsmith/agent/chat_runner.py b/src/specsmith/agent/chat_runner.py index 95a4f07..fa74422 100644 --- a/src/specsmith/agent/chat_runner.py +++ b/src/specsmith/agent/chat_runner.py @@ -34,6 +34,24 @@ DEFAULT_OLLAMA_HOST = "http://127.0.0.1:11434" DEFAULT_OLLAMA_MODEL = os.environ.get("SPECSMITH_OLLAMA_MODEL", "qwen2.5:7b") + +# Ordered preference list used by _pick_ollama_model() when the default +# model is not installed. Lighter / faster models first so the REPL +# stays responsive on developer hardware. +_OLLAMA_MODEL_PREFERENCE = [ + "qwen2.5:7b", + "qwen2.5-coder:7b-instruct", + "qwen3:8b", + "mistral:7b-instruct-q4_0", + "llama3:8b-instruct-q4_K_M", + "qwen2.5:14b", + "qwen2.5-coder:14b", + "qwen3:14b", + "mistral-nemo:12b", + "phi4:14b-q4_K_M", + "deepseek-r1:14b", + "qwen3:30b-a3b", +] SYSTEM_PROMPT = ( "You are Nexus, the local-first agentic developer assistant inside " "Specsmith. Always end your response with the canonical contract:\n" @@ -239,6 +257,40 @@ def run_single_prompt(prompt: str, *, max_tokens: int = 500) -> str | None: # n # --------------------------------------------------------------------------- +def _ollama_alive(host: str) -> bool: + try: + with urlopen(f"{host}/api/tags", timeout=2): # noqa: S310 + return True + except (URLError, TimeoutError, OSError): + return False + + +def _pick_ollama_model(host: str) -> str: + """Return the best available Ollama model for this machine. + + Checks the installed model list from ``/api/tags`` and walks + ``_OLLAMA_MODEL_PREFERENCE`` to find the first match. Falls back + to ``DEFAULT_OLLAMA_MODEL`` when the API is unreachable or the list + is empty. An explicit ``SPECSMITH_OLLAMA_MODEL`` env var always wins. + """ + env_override = os.environ.get("SPECSMITH_OLLAMA_MODEL", "").strip() + if env_override: + return env_override + try: + with urlopen(f"{host}/api/tags", timeout=2) as resp: # noqa: S310 + data = json.loads(resp.read()) + installed = {m["name"] for m in data.get("models", []) if m.get("name")} + for candidate in _OLLAMA_MODEL_PREFERENCE: + if candidate in installed: + return candidate + # None of the preferred models found — use whatever is first + if installed: + return sorted(installed)[0] + except Exception: # noqa: BLE001 — best-effort; fall through to default + pass + return DEFAULT_OLLAMA_MODEL + + def _run_ollama( messages: list[dict[str, str]], emitter: EventEmitter, @@ -246,12 +298,13 @@ def _run_ollama( ) -> tuple[str | None, _UsageDelta]: """Stream from a local Ollama daemon using only stdlib.""" host = os.environ.get("OLLAMA_HOST", DEFAULT_OLLAMA_HOST).rstrip("/") - model = os.environ.get("SPECSMITH_OLLAMA_MODEL", DEFAULT_OLLAMA_MODEL) usage = _UsageDelta() if not _ollama_alive(host): return None, usage + model = _pick_ollama_model(host) + payload = json.dumps({"model": model, "messages": messages, "stream": True}).encode("utf-8") req = Request( # noqa: S310 - URL is a hardcoded localhost default f"{host}/api/chat", @@ -284,14 +337,6 @@ def _run_ollama( return ("".join(pieces) if pieces else None), usage -def _ollama_alive(host: str) -> bool: - try: - with urlopen(f"{host}/api/tags", timeout=2): # noqa: S310 - return True - except (URLError, TimeoutError, OSError): - return False - - def _run_anthropic( messages: list[dict[str, str]], emitter: EventEmitter, diff --git a/src/specsmith/agent/events.py b/src/specsmith/agent/events.py index e050187..8a8d78e 100644 --- a/src/specsmith/agent/events.py +++ b/src/specsmith/agent/events.py @@ -231,4 +231,24 @@ def task_complete( ) -__all__ = ["EventEmitter"] +class PlainTextEmitter(EventEmitter): + """Human-readable variant of EventEmitter for interactive terminal sessions. + + Writes LLM token chunks directly to the stream (no JSON wrapping) so + ``specsmith run`` without ``--json-events`` produces readable output. + All non-token protocol events are silently dropped — system messages + are handled separately by ``AgentRunner._default_emit_event``. + """ + + def token(self, block_id: str, text: str) -> None: # noqa: ARG002 + self.stream.write(text) + with contextlib.suppress(Exception): + self.stream.flush() + + def emit(self, event: dict[str, Any]) -> None: + # Drop JSONL protocol frames in plain-text mode; token() handles + # the only output that matters for interactive sessions. + pass + + +__all__ = ["EventEmitter", "PlainTextEmitter"] diff --git a/src/specsmith/agent/runner.py b/src/specsmith/agent/runner.py index 0acea0c..2dce4c8 100644 --- a/src/specsmith/agent/runner.py +++ b/src/specsmith/agent/runner.py @@ -27,13 +27,160 @@ from typing import Any from specsmith.agent.core import AgentState, ModelTier -from specsmith.agent.events import EventEmitter +from specsmith.agent.events import EventEmitter, PlainTextEmitter # These imports are kept lazy in the public API so that a busted optional # dependency (e.g. ``ag2``) doesn't keep the bridge from emitting ``ready``. # The import itself happens on the first call that actually needs the # orchestrator group chat. -__all__ = ["AgentRunner", "_capabilities"] +__all__ = ["AgentRunner", "ProviderStatus", "check_providers", "_capabilities"] + + +# --------------------------------------------------------------------------- +# Provider health check +# --------------------------------------------------------------------------- + + +from dataclasses import dataclass # noqa: E402 — after __all__ + + +@dataclass +class ProviderStatus: + """Health snapshot for a single LLM provider.""" + + name: str + available: bool + model: str = "" # resolved model name (empty when unavailable) + note: str = "" # human-readable reason (error or extra context) + model_count: int = 0 # number of installed models (Ollama only) + + @property + def icon(self) -> str: + return "\u2713" if self.available else "\u2717" + + +def check_providers() -> list[ProviderStatus]: + """Probe every supported LLM provider and return a status list. + + Safe to call at any time — never raises, never blocks for more than + a couple of seconds. Used by ``_print_banner`` and ``specsmith run + --check``. + """ + import importlib + import os + + from specsmith.agent.chat_runner import ( + _OLLAMA_MODEL_PREFERENCE, + DEFAULT_OLLAMA_HOST, + DEFAULT_OLLAMA_MODEL, + _ollama_alive, + ) + + results: list[ProviderStatus] = [] + + # ── Ollama ─────────────────────────────────────────────────────────── + host = os.environ.get("OLLAMA_HOST", DEFAULT_OLLAMA_HOST).rstrip("/") + if _ollama_alive(host): + try: + import json + from urllib.request import urlopen + + with urlopen(f"{host}/api/tags", timeout=2) as resp: # noqa: S310 + data = json.loads(resp.read()) + installed = [m["name"] for m in data.get("models", []) if m.get("name")] + installed_set = set(installed) + model = os.environ.get("SPECSMITH_OLLAMA_MODEL", "").strip() + source = "env" + if not model: + source = "auto" + for candidate in _OLLAMA_MODEL_PREFERENCE: + if candidate in installed_set: + model = candidate + break + if not model and installed: + model = sorted(installed_set)[0] + if not model: + model = DEFAULT_OLLAMA_MODEL + note = f"{source}, {len(installed)} installed" + results.append( + ProviderStatus( + name="ollama", + available=True, + model=model, + note=note, + model_count=len(installed), + ) + ) + except Exception as exc: # noqa: BLE001 + results.append( + ProviderStatus(name="ollama", available=False, note=f"error reading tags: {exc}") + ) + else: + results.append( + ProviderStatus( + name="ollama", + available=False, + note=f"not running at {host} — start with: ollama serve", + ) + ) + + # ── Anthropic ──────────────────────────────────────────────────────── + key = os.environ.get("ANTHROPIC_API_KEY", "") + if not key: + results.append( + ProviderStatus(name="anthropic", available=False, note="no ANTHROPIC_API_KEY") + ) + elif importlib.util.find_spec("anthropic") is None: + results.append( + ProviderStatus( + name="anthropic", + available=False, + note="key set but SDK missing — run: pipx inject specsmith anthropic", + ) + ) + else: + model = os.environ.get("ANTHROPIC_MODEL", "claude-haiku-4-5") + results.append( + ProviderStatus(name="anthropic", available=True, model=model, note="key configured") + ) + + # ── OpenAI ─────────────────────────────────────────────────────────── + key = os.environ.get("OPENAI_API_KEY", "") + if not key: + results.append(ProviderStatus(name="openai", available=False, note="no OPENAI_API_KEY")) + elif importlib.util.find_spec("openai") is None: + results.append( + ProviderStatus( + name="openai", + available=False, + note="key set but SDK missing — run: pipx inject specsmith openai", + ) + ) + else: + model = os.environ.get("OPENAI_MODEL", "gpt-4o-mini") + results.append( + ProviderStatus(name="openai", available=True, model=model, note="key configured") + ) + + # ── Gemini ─────────────────────────────────────────────────────────── + key = os.environ.get("GOOGLE_API_KEY", "") + if not key: + results.append(ProviderStatus(name="gemini", available=False, note="no GOOGLE_API_KEY")) + elif importlib.util.find_spec("google.genai") is None: + results.append( + ProviderStatus( + name="gemini", + available=False, + note="key set but SDK missing — run: pipx inject specsmith google-genai", + ) + ) + else: + model = os.environ.get("GEMINI_MODEL", "gemini-2.5-flash") + results.append( + ProviderStatus(name="gemini", available=True, model=model, note="key configured") + ) + + return results # --------------------------------------------------------------------------- @@ -156,7 +303,14 @@ def __init__( self.endpoint_id = (endpoint_id or "").strip() or None self.profile_id = (profile_id or "").strip() or None - self._emitter = emitter or EventEmitter(stream=sys.stdout) + # Use a plain-text emitter in interactive mode so LLM tokens render + # as readable prose instead of raw JSONL frames on the terminal. + if emitter is not None: + self._emitter = emitter + elif json_events: + self._emitter = EventEmitter(stream=sys.stdout) + else: + self._emitter = PlainTextEmitter(stream=sys.stdout) self._state = AgentState( provider_name=self.provider_name, model_name=self.model, @@ -187,37 +341,58 @@ def __init__( # ── Public lifecycle ─────────────────────────────────────────────── def _print_banner(self) -> None: - """Emit the ``ready`` handshake (or print a plain banner). + """Emit the ``ready`` handshake (or print a plain banner. Called exactly once at process start. The bridge waits up to 20 s - for this frame; when ``json_events`` is False we still emit a - terminal-friendly banner so interactive ``specsmith run`` users - see the same boot text they used to. + for this frame; when ``json_events`` is False we print a human- + readable provider status table so the user knows exactly what will + respond to their commands before they type anything. """ version = self._package_version() if self.json_events: + # When a model/provider was explicitly set (e.g. via CLI flag or test), + # use those. Only probe check_providers() when no model was pinned. + if self.model: + provider_for_ready = self.provider_name + model_for_ready = self.model + else: + statuses = check_providers() + active = next((s for s in statuses if s.available), None) + provider_for_ready = active.name if active else self.provider_name + model_for_ready = active.model if active else "" self._emitter.ready( agent="nexus", version=version, project_dir=self.project_dir, - provider=self.provider_name, - model=self.model, + provider=provider_for_ready, + model=model_for_ready, profile_id=self.profile_id or "", capabilities=_capabilities(), endpoint_id=self.endpoint_id or "", ) else: - print( - f"Nexus {version} — Local-first Agentic Development Environment " - f"(Specsmith-governed)\n" - f" project: {self.project_dir}\n" - f" provider: {self.provider_name}\n" - f" model: {self.model or '(default)'}\n" - f" profile: {self.profile_id or '(default)'}\n" - "Type plain English, or use slash commands " - "(/plan, /ask, /fix, /test, /commit, /pr, /why, /exit).", - flush=True, - ) + statuses = check_providers() + active_count = sum(1 for s in statuses if s.available) + lines = [ + f"Nexus {version} — specsmith run", + f" project : {self.project_dir}", + f" profile : {self.profile_id or '(default)'}", + "", + " LLM providers:", + ] + for s in statuses: + if s.available: + lines.append( + f" {s.icon} {s.name:<10} ✓ ready model: {s.model} ({s.note})" + ) + else: + lines.append(f" {s.icon} {s.name:<10} ✗ {s.note}") + lines.append("") + if active_count == 0: + lines.append(" ⚠ No provider available — commands will return no response.") + else: + lines.append(" Type plain English, or use /plan /ask /fix /test /commit /pr /exit") + print("\n".join(lines), flush=True) def run_interactive(self) -> None: """Read stdin lines and dispatch each to :meth:`_handle_command`.""" @@ -230,6 +405,10 @@ def run_interactive(self) -> None: if line.strip().lower() in {"/exit", "/quit"}: break self._handle_command(line) + # Ensure streamed tokens are followed by a newline so the + # next input prompt doesn't bleed onto the response line. + if not self.json_events: + print(flush=True) if self.json_events: self._emit_event(type="turn_done") if self._hard_stop: @@ -316,7 +495,12 @@ def _handle_command(self, text: str) -> Any: block_id = self._next_block_id() try: - from specsmith.agent.chat_runner import run_chat + from specsmith.agent.chat_runner import ( + DEFAULT_OLLAMA_HOST, + _ollama_alive, + _pick_ollama_model, + run_chat, + ) result = run_chat( full_prompt, @@ -336,6 +520,28 @@ def _handle_command(self, text: str) -> Any: ) return None + if result is None: + # All providers failed — give the user an actionable explanation + # rather than silent emptiness. + import os as _os + + host = _os.environ.get("OLLAMA_HOST", DEFAULT_OLLAMA_HOST).rstrip("/") + if _ollama_alive(host): + model = _pick_ollama_model(host) + hint = ( + f"Ollama is running but returned no response " + f"(model: {model}). " + "Try: ollama run " + model + ) + else: + hint = ( + "No LLM provider available. Options:\n" + " • Start Ollama: ollama serve\n" + " • Set ANTHROPIC_API_KEY / OPENAI_API_KEY / GOOGLE_API_KEY" + ) + self._emit_event(type="system", message=hint) + return None + # Aggregate metrics into the session state (C1). # ``run_chat`` now reports tokens_in / tokens_out / cost_usd off the # provider response (Ollama prompt_eval_count + eval_count, OpenAI diff --git a/src/specsmith/cli.py b/src/specsmith/cli.py index aad1822..ae5e5be 100644 --- a/src/specsmith/cli.py +++ b/src/specsmith/cli.py @@ -3661,6 +3661,13 @@ def abort_cmd(pid: int | None, abort_all_flag: bool, project_dir: str) -> None: @main.command(name="run") @click.option("--project-dir", type=click.Path(exists=True), default=".") +@click.option( + "--check", + "check_only", + is_flag=True, + default=False, + help="Check provider availability and exit (no REPL started).", +) @click.option( "--task", "task", @@ -3718,6 +3725,7 @@ def abort_cmd(pid: int | None, abort_all_flag: bool, project_dir: str) -> None: ) def run_cmd( project_dir: str, + check_only: bool, task: str, provider_name: str | None, model: str | None, @@ -3737,10 +3745,41 @@ def run_cmd( Ollama running → local LLMs (zero config) SPECSMITH_PROVIDER= → explicit override + Use --check to validate provider configuration without starting a session. + Install a provider: pipx inject specsmith anthropic # Claude pipx inject specsmith openai # GPT/O-series """ + from specsmith.agent.runner import check_providers + + if check_only: + statuses = check_providers() + any_ok = any(s.available for s in statuses) + console.print("[bold]specsmith run --check[/bold]\n") + for s in statuses: + if s.available: + console.print( + f" [green]\u2713[/green] {s.name:<10} " + f"model: [bold]{s.model}[/bold] ({s.note})" + ) + else: + console.print(f" [red]\u2717[/red] {s.name:<10} {s.note}") + console.print() + if any_ok: + active = next(s for s in statuses if s.available) + console.print( + f"[bold green]Ready.[/bold green] " + f"Primary provider: {active.name} / {active.model}" + ) + else: + console.print( + "[bold red]No provider available.[/bold red] " + "Start Ollama or set an API key." + ) + raise SystemExit(1) + return + from specsmith.agent.core import ModelTier from specsmith.agent.runner import AgentRunner diff --git a/tests/test_agent_run_feedback.py b/tests/test_agent_run_feedback.py new file mode 100644 index 0000000..ae66606 --- /dev/null +++ b/tests/test_agent_run_feedback.py @@ -0,0 +1,354 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 BitConcepts, LLC. All rights reserved. +"""Tests for specsmith run provider checks, model selection, and feedback. + +Regression coverage for the silent-no-response bug: +- Wrong default Ollama model (qwen2.5:7b not installed) → silent failure +- No feedback when run_chat returns None +- check_providers() accurately reflects real provider availability +- PlainTextEmitter writes tokens without JSON wrapping +- _pick_ollama_model prefers installed models over the hardcoded default +""" + +from __future__ import annotations + +import io +import json +from unittest.mock import MagicMock, patch + +# --------------------------------------------------------------------------- +# _pick_ollama_model — model selection logic +# --------------------------------------------------------------------------- + + +class TestPickOllamaModel: + def test_returns_env_override_without_querying(self, monkeypatch, tmp_path): + """SPECSMITH_OLLAMA_MODEL env var wins unconditionally.""" + monkeypatch.setenv("SPECSMITH_OLLAMA_MODEL", "phi4:14b-q4_K_M") + from specsmith.agent.chat_runner import _pick_ollama_model + + result = _pick_ollama_model("http://127.0.0.1:11434") + assert result == "phi4:14b-q4_K_M" + + def test_prefers_preferred_model_over_alphabetical_first(self, monkeypatch): + """Should pick the first preference-list match, not the alphabetically first.""" + monkeypatch.delenv("SPECSMITH_OLLAMA_MODEL", raising=False) + + installed = [ + {"name": "zz-model:latest"}, + {"name": "qwen2.5-coder:7b-instruct"}, + {"name": "phi4:14b-q4_K_M"}, + ] + fake_resp = MagicMock() + fake_resp.read.return_value = json.dumps({"models": installed}).encode() + fake_resp.__enter__ = lambda s: s + fake_resp.__exit__ = MagicMock(return_value=False) + + from specsmith.agent.chat_runner import _pick_ollama_model + + with patch("specsmith.agent.chat_runner.urlopen", return_value=fake_resp): + result = _pick_ollama_model("http://127.0.0.1:11434") + + # qwen2.5-coder:7b-instruct is higher in _OLLAMA_MODEL_PREFERENCE than phi4 + assert result == "qwen2.5-coder:7b-instruct" + + def test_falls_back_to_sorted_first_when_no_preference_match(self, monkeypatch): + """When none of the preference list is installed, pick sorted-first installed.""" + monkeypatch.delenv("SPECSMITH_OLLAMA_MODEL", raising=False) + + installed = [{"name": "custom-model-b:7b"}, {"name": "custom-model-a:7b"}] + fake_resp = MagicMock() + fake_resp.read.return_value = json.dumps({"models": installed}).encode() + fake_resp.__enter__ = lambda s: s + fake_resp.__exit__ = MagicMock(return_value=False) + + from specsmith.agent.chat_runner import _pick_ollama_model + + with patch("specsmith.agent.chat_runner.urlopen", return_value=fake_resp): + result = _pick_ollama_model("http://127.0.0.1:11434") + + assert result == "custom-model-a:7b" + + def test_falls_back_to_default_when_api_unreachable(self, monkeypatch): + """When /api/tags raises, return DEFAULT_OLLAMA_MODEL.""" + monkeypatch.delenv("SPECSMITH_OLLAMA_MODEL", raising=False) + from urllib.error import URLError + + from specsmith.agent.chat_runner import DEFAULT_OLLAMA_MODEL, _pick_ollama_model + + with patch("specsmith.agent.chat_runner.urlopen", side_effect=URLError("refused")): + result = _pick_ollama_model("http://127.0.0.1:11434") + + assert result == DEFAULT_OLLAMA_MODEL + + def test_uses_installed_model_not_missing_default(self, monkeypatch): + """Core regression: qwen2.5:7b not installed → picks qwen2.5:14b instead.""" + monkeypatch.delenv("SPECSMITH_OLLAMA_MODEL", raising=False) + + # Simulate the exact machine state that caused the bug + installed = [ + {"name": "phi4:14b-q4_K_M"}, + {"name": "qwen2.5:14b"}, + {"name": "qwen2.5-coder:7b-instruct"}, + {"name": "mistral:7b-instruct-q4_0"}, + ] + fake_resp = MagicMock() + fake_resp.read.return_value = json.dumps({"models": installed}).encode() + fake_resp.__enter__ = lambda s: s + fake_resp.__exit__ = MagicMock(return_value=False) + + from specsmith.agent.chat_runner import _pick_ollama_model + + with patch("specsmith.agent.chat_runner.urlopen", return_value=fake_resp): + result = _pick_ollama_model("http://127.0.0.1:11434") + + # qwen2.5:7b is not installed; qwen2.5-coder:7b-instruct is next preferred + assert result != "qwen2.5:7b" + assert result in {"qwen2.5-coder:7b-instruct", "mistral:7b-instruct-q4_0", "qwen2.5:14b"} + + +# --------------------------------------------------------------------------- +# PlainTextEmitter — interactive mode token output +# --------------------------------------------------------------------------- + + +class TestPlainTextEmitter: + def test_token_writes_text_not_json(self): + """PlainTextEmitter.token() must write raw text, not a JSON object.""" + from specsmith.agent.events import PlainTextEmitter + + buf = io.StringIO() + emitter = PlainTextEmitter(stream=buf) + emitter.token("blk_001", "Hello, ") + emitter.token("blk_001", "world!") + + output = buf.getvalue() + assert output == "Hello, world!" + # Must NOT be JSON + assert "{" not in output + + def test_emit_drops_protocol_frames(self): + """PlainTextEmitter.emit() discards JSONL frames silently.""" + from specsmith.agent.events import PlainTextEmitter + + buf = io.StringIO() + emitter = PlainTextEmitter(stream=buf) + emitter.emit({"type": "block_start", "block_id": "x", "kind": "message"}) + emitter.emit({"type": "turn_done"}) + + assert buf.getvalue() == "" + + def test_ready_does_not_write_in_plain_mode(self): + """PlainTextEmitter.ready() must be a no-op (inherits emit → no-op).""" + from specsmith.agent.events import PlainTextEmitter + + buf = io.StringIO() + emitter = PlainTextEmitter(stream=buf) + emitter.ready(agent="nexus", version="0.11.5") + assert buf.getvalue() == "" + + +# --------------------------------------------------------------------------- +# check_providers() — provider status table +# --------------------------------------------------------------------------- + + +class TestCheckProviders: + def test_returns_four_providers(self): + """check_providers() must always return exactly 4 entries.""" + from specsmith.agent.runner import check_providers + + # Patch Ollama alive check to avoid network + with patch("specsmith.agent.chat_runner._ollama_alive", return_value=False): + statuses = check_providers() + + names = [s.name for s in statuses] + assert names == ["ollama", "anthropic", "openai", "gemini"] + + def test_ollama_down_shows_not_available(self): + """When Ollama is not running, ollama status must be available=False.""" + from specsmith.agent.runner import check_providers + + with patch("specsmith.agent.chat_runner._ollama_alive", return_value=False): + statuses = check_providers() + + ollama = next(s for s in statuses if s.name == "ollama") + assert not ollama.available + assert "not running" in ollama.note or "ollama serve" in ollama.note + + def test_anthropic_without_key_is_unavailable(self, monkeypatch): + """Missing ANTHROPIC_API_KEY → anthropic unavailable.""" + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + from specsmith.agent.runner import check_providers + + with patch("specsmith.agent.chat_runner._ollama_alive", return_value=False): + statuses = check_providers() + + anthropic = next(s for s in statuses if s.name == "anthropic") + assert not anthropic.available + assert "ANTHROPIC_API_KEY" in anthropic.note + + def test_anthropic_with_key_and_sdk_is_available(self, monkeypatch): + """ANTHROPIC_API_KEY set + SDK present → anthropic available.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-key") + + from specsmith.agent.runner import check_providers + + with ( + patch("specsmith.agent.chat_runner._ollama_alive", return_value=False), + patch( + "importlib.util.find_spec", + lambda name: MagicMock() if name == "anthropic" else None, + ), + ): + statuses = check_providers() + + anthropic = next(s for s in statuses if s.name == "anthropic") + assert anthropic.available + assert "claude" in anthropic.model.lower() + + def test_ollama_available_with_model_shows_model_name(self, monkeypatch): + """When Ollama is alive, check_providers() reports the resolved model.""" + monkeypatch.delenv("SPECSMITH_OLLAMA_MODEL", raising=False) + + installed = [{"name": "qwen2.5-coder:7b-instruct"}] + fake_resp = MagicMock() + fake_resp.read.return_value = json.dumps({"models": installed}).encode() + fake_resp.__enter__ = lambda s: s + fake_resp.__exit__ = MagicMock(return_value=False) + + from specsmith.agent.runner import check_providers + + # check_providers() imports urlopen from urllib.request inside the function; + # patch at the source so both the alive-check and the tags call are controlled. + with ( + patch("specsmith.agent.chat_runner._ollama_alive", return_value=True), + patch("urllib.request.urlopen", return_value=fake_resp), + ): + statuses = check_providers() + + ollama = next(s for s in statuses if s.name == "ollama") + assert ollama.available + assert ollama.model == "qwen2.5-coder:7b-instruct" + assert ollama.model_count == 1 + + +# --------------------------------------------------------------------------- +# AgentRunner — feedback when no provider responds +# --------------------------------------------------------------------------- + + +class TestAgentRunnerFeedback: + def test_handle_command_prints_hint_when_no_provider(self, tmp_path, capsys): + """_handle_command must print a hint when run_chat returns None.""" + from specsmith.agent.runner import AgentRunner + + with ( + patch("specsmith.agent.chat_runner._ollama_alive", return_value=False), + patch("specsmith.agent.chat_runner.run_chat", return_value=None), + ): + runner = AgentRunner(project_dir=str(tmp_path), json_events=False) + result = runner._handle_command("hello specsmith") + + assert result is None + captured = capsys.readouterr() + # Must print something actionable — not stay silent + assert len(captured.out.strip()) > 0 + assert any( + keyword in captured.out.lower() + for keyword in ("provider", "ollama", "api_key", "anthropic", "options") + ) + + def test_handle_command_ollama_alive_hint_mentions_model(self, tmp_path, capsys): + """When Ollama is running but returns None, hint mentions the model name.""" + from specsmith.agent.runner import AgentRunner + + installed = [{"name": "mistral:7b-instruct-q4_0"}] + fake_resp = MagicMock() + fake_resp.read.return_value = json.dumps({"models": installed}).encode() + fake_resp.__enter__ = lambda s: s + fake_resp.__exit__ = MagicMock(return_value=False) + + with ( + patch("specsmith.agent.chat_runner._ollama_alive", return_value=True), + patch("specsmith.agent.chat_runner.urlopen", return_value=fake_resp), + patch("specsmith.agent.chat_runner.run_chat", return_value=None), + ): + runner = AgentRunner(project_dir=str(tmp_path), json_events=False) + runner._handle_command("test prompt") + + captured = capsys.readouterr() + assert "mistral" in captured.out or "ollama" in captured.out.lower() + + def test_plain_text_emitter_used_in_non_json_mode(self, tmp_path): + """AgentRunner must use PlainTextEmitter when json_events=False.""" + from specsmith.agent.events import PlainTextEmitter + from specsmith.agent.runner import AgentRunner + + with patch("specsmith.agent.chat_runner._ollama_alive", return_value=False): + runner = AgentRunner(project_dir=str(tmp_path), json_events=False) + + assert isinstance(runner._emitter, PlainTextEmitter) + + def test_jsonl_emitter_used_in_json_events_mode(self, tmp_path): + """AgentRunner must use EventEmitter (not PlainTextEmitter) when json_events=True.""" + from specsmith.agent.events import EventEmitter, PlainTextEmitter + from specsmith.agent.runner import AgentRunner + + with patch("specsmith.agent.chat_runner._ollama_alive", return_value=False): + runner = AgentRunner(project_dir=str(tmp_path), json_events=True) + + assert isinstance(runner._emitter, EventEmitter) + assert not isinstance(runner._emitter, PlainTextEmitter) + + +# --------------------------------------------------------------------------- +# specsmith run --check CLI integration +# --------------------------------------------------------------------------- + + +class TestRunCheckCLI: + def test_check_flag_exits_zero_when_provider_available(self, tmp_path): + """specsmith run --check exits 0 when at least one provider is available.""" + from click.testing import CliRunner + + from specsmith.agent.runner import ProviderStatus + from specsmith.cli import main + + fake_statuses = [ + ProviderStatus( + name="ollama", available=True, model="qwen3:8b", note="auto, 5 installed" + ), + ProviderStatus(name="anthropic", available=False, note="no ANTHROPIC_API_KEY"), + ProviderStatus(name="openai", available=False, note="no OPENAI_API_KEY"), + ProviderStatus(name="gemini", available=False, note="no GOOGLE_API_KEY"), + ] + + runner = CliRunner() + # check_providers is imported inside run_cmd via `from specsmith.agent.runner import` + with patch("specsmith.agent.runner.check_providers", return_value=fake_statuses): + result = runner.invoke(main, ["run", "--check", "--project-dir", str(tmp_path)]) + + assert result.exit_code == 0 + assert "qwen3:8b" in result.output + assert "Ready" in result.output + + def test_check_flag_exits_one_when_no_provider(self, tmp_path): + """specsmith run --check exits 1 when no provider is available.""" + from click.testing import CliRunner + from specsmith.cli import main + from specsmith.agent.runner import ProviderStatus + + fake_statuses = [ + ProviderStatus(name="ollama", available=False, note="not running"), + ProviderStatus(name="anthropic", available=False, note="no ANTHROPIC_API_KEY"), + ProviderStatus(name="openai", available=False, note="no OPENAI_API_KEY"), + ProviderStatus(name="gemini", available=False, note="no GOOGLE_API_KEY"), + ] + + runner = CliRunner() + with patch("specsmith.agent.runner.check_providers", return_value=fake_statuses): + result = runner.invoke(main, ["run", "--check", "--project-dir", str(tmp_path)]) + + assert result.exit_code == 1 + assert "No provider" in result.output From 36dbef27dbf039d0648d0c897ad0554bce68e44e Mon Sep 17 00:00:00 2001 From: Tristen Pierson Date: Thu, 21 May 2026 14:43:21 -0400 Subject: [PATCH 2/3] style: ruff format cli.py Co-Authored-By: Oz --- src/specsmith/cli.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/specsmith/cli.py b/src/specsmith/cli.py index ae5e5be..5ad6de3 100644 --- a/src/specsmith/cli.py +++ b/src/specsmith/cli.py @@ -3769,13 +3769,11 @@ def run_cmd( if any_ok: active = next(s for s in statuses if s.available) console.print( - f"[bold green]Ready.[/bold green] " - f"Primary provider: {active.name} / {active.model}" + f"[bold green]Ready.[/bold green] Primary provider: {active.name} / {active.model}" ) else: console.print( - "[bold red]No provider available.[/bold red] " - "Start Ollama or set an API key." + "[bold red]No provider available.[/bold red] Start Ollama or set an API key." ) raise SystemExit(1) return From 59392edf79bbd30bd5b9c23bafc4317306f5d0f7 Mon Sep 17 00:00:00 2001 From: Tristen Pierson Date: Thu, 21 May 2026 14:47:58 -0400 Subject: [PATCH 3/3] style: fix import sort in test_agent_run_feedback.py Co-Authored-By: Oz --- tests/test_agent_run_feedback.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_agent_run_feedback.py b/tests/test_agent_run_feedback.py index ae66606..5a22655 100644 --- a/tests/test_agent_run_feedback.py +++ b/tests/test_agent_run_feedback.py @@ -336,8 +336,9 @@ def test_check_flag_exits_zero_when_provider_available(self, tmp_path): def test_check_flag_exits_one_when_no_provider(self, tmp_path): """specsmith run --check exits 1 when no provider is available.""" from click.testing import CliRunner - from specsmith.cli import main + from specsmith.agent.runner import ProviderStatus + from specsmith.cli import main fake_statuses = [ ProviderStatus(name="ollama", available=False, note="not running"),