diff --git a/very-simplified-stack/cognito-backend/README.md b/very-simplified-stack/cognito-backend/README.md
index 9de67b8..0b7d186 100644
--- a/very-simplified-stack/cognito-backend/README.md
+++ b/very-simplified-stack/cognito-backend/README.md
@@ -62,6 +62,8 @@ Settings are loaded in the following order of priority:
- `app/core/agent_loop.py`: Turning text generation into a tool-executing agent loop.
- `app/core/session_manager.py`: Persistence and history management for AI sessions.
- `cli/cognito_cli.py`: Python CLI client for Cognito Agent.
+- `app/core/extensions/`: System for loading and managing extensions.
+- `app/services/escalation_routing.py`: Uncertainty-based subtask escalation mapping.
- `test-voice-api.ps1`: The main PowerShell profile script containing `cog` and `cogt`.
- `Install-CognitoProfile.ps1`: Installer for the PowerShell environment.
- `config.example.json`: Template for the user configuration file.
@@ -109,6 +111,21 @@ El backend incluye un cliente ligero en Python con tres modos de operación:
- **Project Trust**: Las herramientas de escritura y ejecución requieren que el directorio haya sido marcado como confiable.
- **AGENTS.md**: Si existe en el raíz del `cwd`, se inyecta automáticamente como contexto del sistema.
+### Sistema de Extensiones (Fase 4)
+El sistema permite extender el agente sin modificar el código fuente mediante ficheros Python cargados en runtime.
+
+- **Niveles de Carga**: Global (`~/.cognito/extensions/`), Configurado (`config.json`), y Local al Proyecto (`.cognito/extensions/`).
+- **Capacidades**: Registrar herramientas nuevas, backends, intents del orquestador, y suscribirse a eventos (hooks).
+- **Seguridad**: Las extensiones locales requieren que el proyecto sea marcado como confiable para extensiones (`set_extensions_trusted`).
+ - ⚠️ **ADVERTENCIA**: Marcar un repo con `extensions_trusted=True` concede a ese código el mismo nivel de acceso que el propio proceso del backend. No es un sandbox.
+
+### Escalado Adaptativo (Fase 5)
+El orquestador (`cognito-orchestrator`) ahora puede detectar si una subtarea se ha generado con alta incertidumbre y reintentarla automáticamente con un modelo de mayor capacidad.
+
+- **Umbral de Escalado**: configurable vía `COGNITO_ESCALATION_UNCERTAINTY_THRESHOLD` (default: 0.6).
+- **Mapeo de Escalado**: Definido en `app/services/escalation_routing.py`. Axel debe revisar este archivo para asegurar que los modelos de destino están disponibles en su entorno.
+- **Transparencia**: El escalado es automático y se registra en los logs del servidor. La respuesta final incluye metadatos sobre qué subtareas fueron escaladas.
+
## 🧪 Testing
To test the uncertainty features:
diff --git a/very-simplified-stack/cognito-backend/app/core/uncertainty.py b/very-simplified-stack/cognito-backend/app/core/uncertainty.py
index 0d357b4..1fd1bed 100644
--- a/very-simplified-stack/cognito-backend/app/core/uncertainty.py
+++ b/very-simplified-stack/cognito-backend/app/core/uncertainty.py
@@ -37,3 +37,11 @@ def compute_uncertainty(logprob_data: Any) -> Optional[float]:
except Exception as e:
logger.error("[Uncertainty] Error computing entropy: %s", e)
return None
+
+def aggregate_uncertainty(per_token_values: list[float]) -> Optional[float]:
+ """Media aritmética simple de las incertidumbres por token de una respuesta completa.
+ Devuelve None si la lista está vacía (el backend no devolvió logprobs).
+ Se usa la media y no el máximo para no disparar escalado por un único token ruidoso."""
+ if not per_token_values:
+ return None
+ return sum(per_token_values) / len(per_token_values)
diff --git a/very-simplified-stack/cognito-backend/app/services/backend_client.py b/very-simplified-stack/cognito-backend/app/services/backend_client.py
index de99536..f90d006 100644
--- a/very-simplified-stack/cognito-backend/app/services/backend_client.py
+++ b/very-simplified-stack/cognito-backend/app/services/backend_client.py
@@ -15,6 +15,7 @@
from typing import Any, AsyncGenerator, Dict, List, Optional, Set
from app.services.backend_registry import BackendConfig, BackendType
+from app.core.uncertainty import compute_uncertainty, aggregate_uncertainty
logger = logging.getLogger(__name__)
@@ -56,6 +57,35 @@ async def generate(
else:
return await self._call_openai(prompt, model_params)
+ async def generate_with_uncertainty(
+ self,
+ prompt: str,
+ model_params: Optional[Dict[str, Any]] = None,
+ ) -> tuple[str, Optional[float]]:
+ """
+ Executes generation and aggregates uncertainty from all tokens.
+ Returns (full_text, aggregated_uncertainty).
+ """
+ full_text = ""
+ uncertainties = []
+
+ if self.config.backend_type == BackendType.OLLAMA:
+ async for chunk in self._stream_ollama(prompt, model_params):
+ token = chunk.get("token", "")
+ full_text += token
+ u = compute_uncertainty(chunk.get("logprobs"))
+ if u is not None:
+ uncertainties.append(u)
+ else:
+ async for chunk in self._stream_openai(prompt, model_params):
+ token = chunk.get("token", "")
+ full_text += token
+ u = compute_uncertainty(chunk.get("logprobs"))
+ if u is not None:
+ uncertainties.append(u)
+
+ return full_text, aggregate_uncertainty(uncertainties)
+
async def generate_stream(
self,
prompt: str,
diff --git a/very-simplified-stack/cognito-backend/app/services/escalation_routing.py b/very-simplified-stack/cognito-backend/app/services/escalation_routing.py
new file mode 100644
index 0000000..035239b
--- /dev/null
+++ b/very-simplified-stack/cognito-backend/app/services/escalation_routing.py
@@ -0,0 +1,20 @@
+"""
+Mapeo de escalado: qué backend+modelo usar como reintento cuando la incertidumbre
+agregada de una subtarea supera el umbral. Es un punto de partida con los modelos
+que ya aparecen en MODEL_ROUTING (app/services/semantic_orchestrator.py) — no se
+inventan modelos nuevos que no estén confirmados como disponibles en el stack real.
+
+Axel: revisa y ajusta estas entradas según qué modelos tengas realmente disponibles.
+No todos los intents tienen un target de escalado por defecto — traducción, visión y
+edge son tareas especializadas donde "escalar" a un modelo de texto genérico rompería
+la tarea en vez de mejorarla, así que se dejan sin target (no se escalan).
+"""
+from typing import Dict
+
+ESCALATION_ROUTING: Dict[str, Dict[str, str]] = {
+ "fast": {"backend": "ollama-local", "model": "qwen3.5:9b"},
+ "general": {"backend": "ollama-local", "model": "phi4:latest"},
+ "coding": {"backend": "ollama-local", "model": "phi4:latest"},
+ "analysis": {"backend": "ollama-local", "model": "phi4:latest"},
+ # reasoning, translation, vision, edge: sin entrada -> no se escalan.
+}
diff --git a/very-simplified-stack/cognito-backend/app/services/semantic_orchestrator.py b/very-simplified-stack/cognito-backend/app/services/semantic_orchestrator.py
index 8b22459..dbfaf98 100644
--- a/very-simplified-stack/cognito-backend/app/services/semantic_orchestrator.py
+++ b/very-simplified-stack/cognito-backend/app/services/semantic_orchestrator.py
@@ -25,16 +25,25 @@
import asyncio
import logging
import re
+import os
import xml.etree.ElementTree as ET
from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Set
from app.services.backend_client import BackendClient
from app.services.backend_registry import BackendConfig, BackendType, BACKENDS_BY_PRIORITY
from app.models.ai import AIRequest, AIResponse
+from app.services.escalation_routing import ESCALATION_ROUTING
logger = logging.getLogger(__name__)
+# Escalation settings
+ESCALATION_ENABLED = os.environ.get("COGNITO_ESCALATION_ENABLED", "true").lower() == "true"
+ESCALATION_THRESHOLD = float(os.environ.get("COGNITO_ESCALATION_UNCERTAINTY_THRESHOLD", "0.6"))
+
+# In-memory tracking of backends/models that don't return logprobs
+_NO_LOGPROBS_WARNED: Set[tuple[str, str]] = set()
+
# ── Routing table: intent → (backend_name, model) ────────────────────────────
# Edit here to reassign tasks to different nodes.
@@ -209,6 +218,7 @@ def __init__(self, configs: List[BackendConfig], extra_routing: Optional[Dict[st
self._client_map = _build_client_map(configs)
self.routing = {**MODEL_ROUTING, **(extra_routing or {})}
self._orchestrator_client = self._build_orchestrator_client()
+ self._subtask_metadata: Dict[str, Dict[str, Any]] = {} # Track metadata like escalation
def add_intent_route(self, intent: str, backend_name: str, model: str) -> None:
"""Adds a new intent route at runtime."""
@@ -267,22 +277,33 @@ async def _execute_plan(self, plan: RoutingPlan, original_input: str) -> Dict[st
# Execute ready tasks concurrently
async def _run(task: SubTask) -> tuple[str, str]:
- client = _resolve_route(task.intent, self._client_map, self.routing)
+ # Resolve original route
+ route = self.routing.get(task.intent) or self.routing.get("general")
+ backend_name = route["backend"]
+ model_name = route["model"]
+
# Enrich the input slice with context from upstream tasks
- context = ""
+ context_str = ""
for dep_id in task.depends_on:
- context += f"\n{results[dep_id]}"
+ context_str += f"\n{results[dep_id]}"
full_prompt = task.input_slice or original_input
- if context:
- full_prompt = f"{context}\n\n{full_prompt}"
+ if context_str:
+ full_prompt = f"{context_str}\n\n{full_prompt}"
- logger.info(
- "[Orchestrator] Task %s → %s (%s) | intent=%s",
- task.id, client.config.name, client.config.model, task.intent,
+ # Execute with escalation logic
+ text, final_backend, final_model, escalated = await self._execute_subtask_with_escalation(
+ task, task.intent, backend_name, model_name, full_prompt
)
- res = await client.generate(prompt=full_prompt)
- return task.id, res.get("response", "")
+
+ # Store metadata for process() to use
+ self._subtask_metadata[task.id] = {
+ "backend": final_backend,
+ "model": final_model,
+ "escalated": escalated
+ }
+
+ return task.id, text
batch_results = await asyncio.gather(*[_run(t) for t in ready])
for task_id, response in batch_results:
@@ -292,6 +313,55 @@ async def _run(task: SubTask) -> tuple[str, str]:
return results
+ async def _execute_subtask_with_escalation(
+ self, task: SubTask, intent: str, backend_name: str, model_name: str, prompt: str, already_escalated: bool = False
+ ) -> tuple[str, str, str, bool]:
+ client = _resolve_route(intent if not already_escalated else "general", self._client_map, {**self.routing, "general": {"backend": backend_name, "model": model_name}})
+ # Actually, _resolve_route is simpler: it just needs intent and client_map.
+ # But we want to force a specific backend/model.
+
+ import copy
+ target_client = self._client_map.get(backend_name)
+ if target_client:
+ cfg = copy.copy(target_client.config)
+ cfg.model = model_name
+ client = BackendClient(cfg)
+ else:
+ client = next(iter(self._client_map.values()))
+
+ if not ESCALATION_ENABLED:
+ res = await client.generate(prompt=prompt)
+ return res.get("response", ""), backend_name, model_name, False
+
+ text, uncertainty = await client.generate_with_uncertainty(prompt)
+
+ if uncertainty is None:
+ cache_key = (backend_name, model_name)
+ if cache_key not in _NO_LOGPROBS_WARNED and intent in ESCALATION_ROUTING:
+ logger.warning(
+ "Backend %s / modelo %s no devuelve logprobs — el escalado no podrá activarse "
+ "para este backend/modelo aunque el intent lo tenga configurado",
+ backend_name, model_name
+ )
+ _NO_LOGPROBS_WARNED.add(cache_key)
+
+ if (
+ uncertainty is not None
+ and uncertainty > ESCALATION_THRESHOLD
+ and not already_escalated
+ and intent in ESCALATION_ROUTING
+ ):
+ target = ESCALATION_ROUTING[intent]
+ logger.info(
+ "Subtask %s (intent=%s) escalado de %s/%s a %s/%s: incertidumbre %.2f > umbral %.2f",
+ task.id, intent, backend_name, model_name, target["backend"], target["model"], uncertainty, ESCALATION_THRESHOLD,
+ )
+ return await self._execute_subtask_with_escalation(
+ task, intent, target["backend"], target["model"], prompt, already_escalated=True
+ )
+
+ return text, backend_name, model_name, already_escalated
+
# ── Public interface ───────────────────────────────────────────────────────
async def process(self, request: AIRequest) -> AIResponse:
@@ -328,16 +398,17 @@ async def process(self, request: AIRequest) -> AIResponse:
duration_ms = (time.perf_counter() - start) * 1000
# Build routing metadata for observability
- routing_info = [
- {
+ routing_info = []
+ for t in plan.subtasks:
+ meta = self._subtask_metadata.get(t.id, {})
+ routing_info.append({
"task_id": t.id,
"description": t.description,
"intent": t.intent,
- "backend": self.routing.get(t.intent, self.routing["general"])["backend"],
- "model": self.routing.get(t.intent, self.routing["general"])["model"],
- }
- for t in plan.subtasks
- ]
+ "backend": meta.get("backend", self.routing.get(t.intent, self.routing["general"])["backend"]),
+ "model": meta.get("model", self.routing.get(t.intent, self.routing["general"])["model"]),
+ "escalated": meta.get("escalated", False),
+ })
return AIResponse(
response=final_response,
diff --git a/very-simplified-stack/cognito-backend/tests/test_backend_client_uncertainty.py b/very-simplified-stack/cognito-backend/tests/test_backend_client_uncertainty.py
new file mode 100644
index 0000000..64e0ad9
--- /dev/null
+++ b/very-simplified-stack/cognito-backend/tests/test_backend_client_uncertainty.py
@@ -0,0 +1,76 @@
+import pytest
+import json
+import httpx
+from unittest.mock import AsyncMock, MagicMock
+from app.services.backend_client import BackendClient
+from app.services.backend_registry import BackendConfig, BackendType
+
+@pytest.mark.asyncio
+async def test_generate_with_uncertainty_ollama(respx_mock):
+ config = BackendConfig(
+ name="test-ollama",
+ base_url="http://ollama:11434",
+ backend_type=BackendType.OLLAMA,
+ model="test-model",
+ priority=1
+ )
+ client = BackendClient(config)
+
+ # Mock stream response for /api/generate (Ollama native)
+ # Logprobs must have multiple candidates to result in non-zero uncertainty
+ sse_lines = [
+ json.dumps({
+ "response": "Hello",
+ "done": False,
+ "logprobs": [{
+ "top_logprobs": [
+ {"token": "Hello", "logprob": -0.1},
+ {"token": "Hi", "logprob": -1.5}
+ ]
+ }]
+ }),
+ json.dumps({
+ "response": " world",
+ "done": True,
+ "logprobs": [{
+ "top_logprobs": [
+ {"token": " world", "logprob": -0.2},
+ {"token": " earth", "logprob": -1.2}
+ ]
+ }]
+ })
+ ]
+ respx_mock.post("http://ollama:11434/api/generate").return_value = httpx.Response(
+ 200, content="\n".join(sse_lines) + "\n"
+ )
+
+ text, uncertainty = await client.generate_with_uncertainty("test prompt")
+
+ assert text == "Hello world"
+ assert uncertainty is not None
+ assert uncertainty > 0
+
+@pytest.mark.asyncio
+async def test_generate_with_uncertainty_no_logprobs(respx_mock):
+ config = BackendConfig(
+ name="test-openai",
+ base_url="http://openai:8000",
+ backend_type=BackendType.OPENAI,
+ model="test-model",
+ priority=1
+ )
+ client = BackendClient(config)
+
+ # OpenAI style stream
+ sse_content = (
+ "data: {\"choices\": [{\"delta\": {\"content\": \"Hello\"}}]}\n\n"
+ "data: [DONE]\n\n"
+ )
+ respx_mock.post("http://openai:8000/v1/chat/completions").return_value = httpx.Response(
+ 200, content=sse_content
+ )
+
+ text, uncertainty = await client.generate_with_uncertainty("test prompt")
+
+ assert text == "Hello"
+ assert uncertainty is None
diff --git a/very-simplified-stack/cognito-backend/tests/test_escalation.py b/very-simplified-stack/cognito-backend/tests/test_escalation.py
new file mode 100644
index 0000000..8fd8093
--- /dev/null
+++ b/very-simplified-stack/cognito-backend/tests/test_escalation.py
@@ -0,0 +1,80 @@
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+from app.services.semantic_orchestrator import SemanticOrchestrator, SubTask, RoutingPlan
+from app.services.backend_registry import BackendConfig, BackendType
+from app.models.ai import AIRequest
+
+@pytest.fixture
+def mock_configs():
+ return [
+ BackendConfig(name="ollama-local", base_url="http://local", backend_type=BackendType.OLLAMA, model="m1", priority=1),
+ ]
+
+@pytest.mark.asyncio
+async def test_escalation_logic(mock_configs, monkeypatch):
+ orchestrator = SemanticOrchestrator(configs=mock_configs)
+
+ # Mock task and prompt
+ task = SubTask(id="1", description="desc", intent="general", input_slice="input", depends_on=[])
+ prompt = "input"
+
+ # Mock escalation routing
+ monkeypatch.setattr("app.services.semantic_orchestrator.ESCALATION_ROUTING", {
+ "general": {"backend": "ollama-local", "model": "phi4:latest"}
+ })
+ monkeypatch.setattr("app.services.semantic_orchestrator.ESCALATION_THRESHOLD", 0.5)
+ monkeypatch.setattr("app.services.semantic_orchestrator.ESCALATION_ENABLED", True)
+
+ # First attempt: high uncertainty (0.8)
+ # Second attempt: low uncertainty (0.2)
+ with patch("app.services.backend_client.BackendClient.generate_with_uncertainty") as mock_gen:
+ mock_gen.side_effect = [
+ ("Low quality response", 0.8),
+ ("High quality response", 0.2)
+ ]
+
+ text, final_backend, final_model, escalated = await orchestrator._execute_subtask_with_escalation(
+ task, "general", "ollama-local", "m1", prompt
+ )
+
+ assert escalated is True
+ assert text == "High quality response"
+ assert final_model == "phi4:latest"
+ assert mock_gen.call_count == 2
+
+@pytest.mark.asyncio
+async def test_no_escalation_when_disabled(mock_configs, monkeypatch):
+ orchestrator = SemanticOrchestrator(configs=mock_configs)
+ task = SubTask(id="1", description="desc", intent="general", input_slice="input", depends_on=[])
+
+ monkeypatch.setattr("app.services.semantic_orchestrator.ESCALATION_ENABLED", False)
+
+ with patch("app.services.backend_client.BackendClient.generate") as mock_gen:
+ mock_gen.return_value = {"response": "normal response"}
+
+ text, _, _, escalated = await orchestrator._execute_subtask_with_escalation(
+ task, "general", "ollama-local", "m1", "prompt"
+ )
+
+ assert escalated is False
+ assert text == "normal response"
+ mock_gen.assert_called_once()
+
+@pytest.mark.asyncio
+async def test_no_escalation_for_unmapped_intent(mock_configs, monkeypatch):
+ orchestrator = SemanticOrchestrator(configs=mock_configs)
+ task = SubTask(id="1", description="desc", intent="vision", input_slice="input", depends_on=[])
+
+ monkeypatch.setattr("app.services.semantic_orchestrator.ESCALATION_ROUTING", {})
+ monkeypatch.setattr("app.services.semantic_orchestrator.ESCALATION_THRESHOLD", 0.5)
+
+ with patch("app.services.backend_client.BackendClient.generate_with_uncertainty") as mock_gen:
+ mock_gen.return_value = ("uncertain but unmapped", 0.9)
+
+ text, _, _, escalated = await orchestrator._execute_subtask_with_escalation(
+ task, "vision", "ollama-local", "m1", "prompt"
+ )
+
+ assert escalated is False
+ assert text == "uncertain but unmapped"
+ assert mock_gen.call_count == 1
diff --git a/very-simplified-stack/cognito-backend/tests/test_uncertainty_aggregate.py b/very-simplified-stack/cognito-backend/tests/test_uncertainty_aggregate.py
new file mode 100644
index 0000000..ad97082
--- /dev/null
+++ b/very-simplified-stack/cognito-backend/tests/test_uncertainty_aggregate.py
@@ -0,0 +1,7 @@
+import pytest
+from app.core.uncertainty import aggregate_uncertainty
+
+def test_aggregate_uncertainty():
+ assert aggregate_uncertainty([]) is None
+ assert aggregate_uncertainty([0.1, 0.2, 0.3]) == pytest.approx(0.2)
+ assert aggregate_uncertainty([0.5]) == 0.5