Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,25 @@ PyPI version — not the changelog header.

---

## [2026-07-06]

### Fixed

- **prax log watchdog now covers branch `logs/` dirs — `.jsonl` runaway growth
caught.** Rotation was hardcoded to `.log` files, and several branches write
`.jsonl` logs via raw `open(path, "a")` appenders that bypass prax entirely —
`hooks/logs/engine.jsonl` had grown to 63 MB, `backup/logs/operations.jsonl`
to 31 MB, `trigger/logs/medic_suppressed.log` to 7 MB, all unrotated. The
log-watchdog safety net also only scanned `system_logs/*.log`. @prax extended
it: `scan_branch_log_files()` sweeps every `src/aipass/*/logs/` for `.log` +
`.jsonl` (WARN at 1 MB unrotated, CRITICAL at 10 MB),
`enforce_branch_log_limits()` truncates flagged files to the last 5000 lines,
and `drone @prax log-audit` now reports both system and branch scopes. 11 new
tests, full prax suite 947 green. The raw-appender writers themselves still
need per-owner caps — routed to @hooks, @backup, @trigger. (built by @prax)

---

## [2026-07-05]

### Fixed
Expand Down
151 changes: 145 additions & 6 deletions src/aipass/prax/apps/handlers/logging/log_watchdog.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,28 +9,33 @@
"""
System Log Size Watchdog

Scans the system_logs/ directory for oversized log files and enforces size limits.
Catches ALL log files regardless of how they were created — even those bypassing
PRAX's RotatingFileHandler (e.g., telegram bots using plain FileHandler).
Scans system_logs/ and all branch logs/ directories for oversized files and
enforces size limits. Catches ALL log files regardless of how they were created
— even those bypassing PRAX's RotatingFileHandler (e.g., raw open("a") appenders
writing .jsonl files, telegram bots using plain FileHandler).

This is the safety net: even if a branch misconfigures logging, the watchdog
prevents unbounded growth that caused the 2026-02-26 system crash (DPLAN-037).

Two scopes:
- system_logs/ (central) — scanned by scan_log_files()
- src/aipass/*/logs/ (branch-local) — scanned by scan_branch_log_files()

Two modes:
- audit: Report oversized files without changing anything
- enforce: Truncate oversized files to keep last max_lines
"""

import logging

logger = logging.getLogger(__name__)
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Tuple

from aipass.prax.apps.handlers.json import json_handler

logger = logging.getLogger(__name__)


# =============================================================================
# CONSTANTS
Expand All @@ -57,11 +62,16 @@ def _get_system_logs_dir() -> Path:
return _system_logs_dir_cache


# Thresholds
# Thresholds — system_logs (line-based)
WARN_THRESHOLD_LINES = 5000 # Fire warning at this line count
DEFAULT_MAX_LINES = 1000 # Truncate to this many lines (matches prax config)
CRITICAL_THRESHOLD_LINES = 10000 # Immediate action recommended

# Thresholds — branch logs (size-based, catches .jsonl and unrotated .log)
BRANCH_WARN_SIZE_MB = 1.0
BRANCH_CRITICAL_SIZE_MB = 10.0
BRANCH_DEFAULT_MAX_LINES = 5000


# =============================================================================
# SCANNING
Expand Down Expand Up @@ -277,6 +287,135 @@ def log_health_summary() -> Dict[str, Any]:
}


# =============================================================================
# BRANCH LOG SCANNING — covers .log and .jsonl in src/aipass/*/logs/
# =============================================================================


def _get_ecosystem_root() -> Path:
"""Find src/aipass/ directory."""
return _find_repo_root() / "src" / "aipass"


def _has_rotation_sibling(filepath: Path) -> bool:
"""Check if a file has a .1 rotation sibling."""
return (filepath.parent / f"{filepath.name}.1").exists()


def _classify_branch_log(log_file: Path, branch_name: str) -> Dict[str, Any]:
"""Build an audit record for a single branch log file."""
size_kb = _get_file_size_kb(log_file)
size_mb = size_kb / 1024.0
lines = _count_lines(log_file)
has_rotation = _has_rotation_sibling(log_file)

if size_mb >= BRANCH_CRITICAL_SIZE_MB:
status = "critical"
elif size_mb >= BRANCH_WARN_SIZE_MB and not has_rotation:
status = "warning"
else:
status = "ok"

return {
"path": str(log_file),
"name": log_file.name,
"branch": branch_name,
"lines": lines,
"size_kb": round(size_kb, 1),
"size_mb": round(size_mb, 1),
"has_rotation": has_rotation,
"status": status,
}


def scan_branch_log_files() -> List[Dict[str, Any]]:
"""
Scan all branch logs/ directories for files with unbounded growth.

Checks .log and .jsonl files. Flags unrotated files exceeding size
thresholds — the safety net for writers that bypass RotatingFileHandler.
"""
results: List[Dict[str, Any]] = []
eco_root = _get_ecosystem_root()

if not eco_root.exists():
return results

for branch_dir in sorted(eco_root.iterdir()):
logs_dir = branch_dir / "logs"
if not branch_dir.is_dir() or not logs_dir.is_dir():
continue
for log_file in sorted(logs_dir.glob("*.log")) + sorted(logs_dir.glob("*.jsonl")):
results.append(_classify_branch_log(log_file, branch_dir.name))

results.sort(key=lambda x: x["size_kb"], reverse=True)
json_handler.log_operation("branch_log_watchdog_check", {"files_scanned": len(results)})
return results


def get_oversized_branch_files() -> List[Dict[str, Any]]:
"""Get branch log files exceeding size thresholds."""
return [f for f in scan_branch_log_files() if f["status"] != "ok"]


def enforce_branch_log_limits(
max_lines: int = BRANCH_DEFAULT_MAX_LINES,
) -> List[Dict[str, Any]]:
"""
Truncate oversized branch log files (including .jsonl).

Only truncates files flagged as warning or critical by scan_branch_log_files().
"""
actions: List[Dict[str, Any]] = []

for file_info in get_oversized_branch_files():
filepath = Path(file_info["path"])
original, new = truncate_log_file(filepath, max_lines)

actions.append(
{
"name": file_info["name"],
"branch": file_info["branch"],
"original_lines": original,
"new_lines": new,
"size_mb": file_info["size_mb"],
"truncated": original != new,
}
)

return actions


def branch_log_health_summary() -> Dict[str, Any]:
"""Generate a health summary of branch logs."""
files = scan_branch_log_files()

if not files:
return {
"total_files": 0,
"oversized_count": 0,
"critical_count": 0,
"total_size_mb": 0.0,
"largest_file": None,
"healthy": True,
}

oversized = [f for f in files if f["status"] in ("warning", "critical")]
critical = [f for f in files if f["status"] == "critical"]
largest = files[0] if files else None
total_size = sum(f["size_mb"] for f in files)

return {
"total_files": len(files),
"oversized_count": len(oversized),
"critical_count": len(critical),
"total_size_mb": round(total_size, 1),
"largest_file": f"{largest['branch']}/{largest['name']}" if largest else None,
"largest_size_mb": largest["size_mb"] if largest else 0.0,
"healthy": len(oversized) == 0,
}


# =============================================================================
# CLI ENTRY POINT (for testing)
# =============================================================================
Expand Down
68 changes: 64 additions & 4 deletions src/aipass/prax/apps/modules/log_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,9 @@ def print_help():


def _display_audit(files: list, summary: dict) -> None:
"""Display audit results."""
"""Display system_logs/ audit results."""
console.print()
console.print("[bold cyan]System Log Audit[/bold cyan]")
console.print("[bold cyan]System Log Audit[/bold cyan] [dim](system_logs/)[/dim]")
console.print(f" Total files: {summary['total_files']}")
console.print(f" Total lines: {summary['total_lines']:,}")
if summary.get("largest_file"):
Expand All @@ -90,7 +90,6 @@ def _display_audit(files: list, summary: dict) -> None:
else:
error(f"Status: {summary['oversized_count']} oversized, {summary['critical_count']} critical")

# Show oversized files
oversized = [f for f in files if f["status"] != "ok"]
if oversized:
console.print()
Expand All @@ -101,8 +100,36 @@ def _display_audit(files: list, summary: dict) -> None:
f" [{status_color}]{f['status'].upper()}[/{status_color}] "
f"{f['name']}: {f['lines']:,} lines ({f['size_kb']} KB)"
)
console.print()


def _display_branch_audit(files: list, summary: dict) -> None:
"""Display branch logs/ audit results."""
console.print("[bold cyan]Branch Log Audit[/bold cyan] [dim](src/aipass/*/logs/)[/dim]")
console.print(f" Total files: {summary['total_files']}")
console.print(f" Total size: {summary['total_size_mb']} MB")
if summary.get("largest_file"):
console.print(f" Largest: {summary['largest_file']} ({summary.get('largest_size_mb', 0)} MB)")

if summary["healthy"]:
console.print("[green] Status: HEALTHY — no unbounded files[/green]")
else:
error(f"Status: {summary['oversized_count']} unbounded, {summary['critical_count']} critical")

oversized = [f for f in files if f["status"] != "ok"]
if oversized:
console.print()
console.print("[bold]Unbounded files (no rotation, exceeds size threshold):[/bold]")
for f in oversized:
status_color = "red" if f["status"] == "critical" else "yellow"
rotation = "[green]rotated[/green]" if f["has_rotation"] else "[red]unrotated[/red]"
console.print(
f" [{status_color}]{f['status'].upper()}[/{status_color}] "
f"{f['branch']}/{f['name']}: {f['size_mb']} MB, "
f"{f['lines']:,} lines, {rotation}"
)
console.print()
console.print("[dim]Run 'drone @prax log-audit enforce' to truncate oversized files[/dim]")
console.print("[dim]Run 'drone @prax log-audit enforce' to truncate[/dim]")
console.print()


Expand Down Expand Up @@ -140,10 +167,20 @@ def handle_command(command: str, args: List[str]) -> bool:
files = scan_log_files()
summary = log_health_summary()
_display_audit(files, summary)

from aipass.prax.apps.handlers.logging.log_watchdog import (
scan_branch_log_files,
branch_log_health_summary,
)

branch_files = scan_branch_log_files()
branch_summary = branch_log_health_summary()
_display_branch_audit(branch_files, branch_summary)
return True

if subcmd == "enforce":
_run_enforce()
_run_branch_enforce()
return True

error(f"Unknown log-audit subcommand: {subcmd}")
Expand Down Expand Up @@ -174,6 +211,29 @@ def _run_enforce():
logger.info("[log-audit] Enforced limits on %d files", len(actions))


def _run_branch_enforce():
"""Execute branch log enforcement and display results."""
from aipass.prax.apps.handlers.logging.log_watchdog import enforce_branch_log_limits

console.print("[bold cyan]Enforcing branch log limits...[/bold cyan]")
actions = enforce_branch_log_limits()

if not actions:
console.print("[green]All branch logs within limits — nothing to truncate[/green]\n")
return

for action in actions:
if action["truncated"]:
console.print(
f" [red]TRUNCATED[/red] {action['branch']}/{action['name']}: "
f"{action['size_mb']} MB, {action['original_lines']:,} → {action['new_lines']:,} lines"
)
else:
console.print(f" [green]OK[/green] {action['branch']}/{action['name']}: within limits")
console.print()
logger.info("[log-audit] Enforced branch log limits on %d files", len(actions))


if __name__ == "__main__":
if len(sys.argv) == 1:
print_introspection()
Expand Down
Loading
Loading