feat(cli): add progress reporting and summaries

Author: Scott Kostolni

docs/USAGE.md

@@ -77,7 +77,7 @@ mypy limitless_tools
 
 ## CLI commands
 
-All commands read `LIMITLESS_API_KEY` from the environment. Default data dir is `~/limitless_tools/data/lifelogs` (override with `--data-dir` or `LIMITLESS_DATA_DIR`). Default batch size is `50` (override with `--batch-size`). HTTP requests use a 30-second timeout by default; override with `LIMITLESS_HTTP_TIMEOUT` or the `http_timeout` config key. Use `-v/--verbose` to emit structured JSON debug logs to stderr for troubleshooting.
+All commands read `LIMITLESS_API_KEY` from the environment. Default data dir is `~/limitless_tools/data/lifelogs` (override with `--data-dir` or `LIMITLESS_DATA_DIR`). Default batch size is `50` (override with `--batch-size`). HTTP requests use a 30-second timeout by default; override with `LIMITLESS_HTTP_TIMEOUT` or the `http_timeout` config key. Use `-v/--verbose` to emit structured JSON debug logs to stderr for troubleshooting. Long-running `fetch`/`sync` runs print progress lines and a completion summary (new/updated/unchanged counts) to stderr so you always know the job status without polluting stdout/JSON output.
 
 - Fetch latest N lifelogs (saves JSON files): defaults include markdown and headings. Use `--json` to print a JSON array of saved item summaries to stdout.
 

limitless_tools/cli/main.py

@@ -4,14 +4,70 @@
 import logging
 import os
 import sys
+import time
+from collections.abc import Callable
 from pathlib import Path
 from zoneinfo import ZoneInfo
 
 from limitless_tools.config.config import default_config_path, get_profile, load_config
 from limitless_tools.config.env import load_env
 from limitless_tools.config.logging import setup_logging
 from limitless_tools.config.paths import default_data_dir, expand_path
-from limitless_tools.services.lifelog_service import LifelogService
+from limitless_tools.services.lifelog_service import LifelogService, SaveReport
+
+
+def _stderr_line(message: str) -> None:
+    sys.stderr.write(f"{message}\n")
+    sys.stderr.flush()
+
+
+class ProgressReporter:
+    def __init__(self, action: str):
+        self.action = action
+        self._start_ts: float | None = None
+        self._callback: Callable[[int, int], None] | None = None
+
+    def start(self) -> None:
+        if self._start_ts is None:
+            self._start_ts = time.perf_counter()
+            _stderr_line(f"{self.action.title()} started...")
+
+    def make_callback(self) -> Callable[[int, int], None]:
+        if self._callback is None:
+            def _cb(page: int, total: int) -> None:
+                _stderr_line(
+                    f"{self.action.title()} in progress: {total} lifelogs processed (page {page})"
+                )
+
+            self._callback = _cb
+        return self._callback
+
+    def finish(self, report: SaveReport | None) -> None:
+        if self._start_ts is None:
+            self.start()
+        duration = (time.perf_counter() - self._start_ts) if self._start_ts is not None else None
+        _stderr_line(_format_summary(self.action, report, duration))
+
+
+def _format_summary(action: str, report: SaveReport | None, duration: float | None) -> str:
+    title = action.title()
+    duration_text = ""
+    if duration is not None:
+        duration_text = f" in {duration:.1f}s"
+    if report is None:
+        return f"{title} complete{duration_text}."
+    if report.created or report.updated:
+        parts: list[str] = []
+        if report.created:
+            parts.append(f"{report.created} new")
+        if report.updated:
+            parts.append(f"{report.updated} updated")
+        if report.unchanged:
+            parts.append(f"{report.unchanged} unchanged")
+        return f"{title} complete{duration_text}: {', '.join(parts)}."
+    if report.unchanged:
+        return f"{title} complete{duration_text}: no changes (data already up to date)."
+    return f"{title} complete{duration_text}: no lifelogs returned."
 
 
 def _build_parser() -> argparse.ArgumentParser:
@@ -181,12 +237,15 @@ def _coerce_timeout_value(value: object) -> float | None:
             data_dir=args.data_dir,
             http_timeout=resolved_http_timeout,
         )
+        reporter = ProgressReporter("fetch")
+        reporter.start()
         saved = service.fetch(
             limit=args.limit,
             direction=args.direction,
             include_markdown=args.include_markdown,
             include_headings=args.include_headings,
             batch_size=max(1, int(args.batch_size)),
+            progress_callback=reporter.make_callback(),
         )
         if args.json:
             import json as _json
@@ -206,6 +265,7 @@ def _coerce_timeout_value(value: object) -> float | None:
                     log.debug("Skipping invalid saved lifelog %s: %s", p, exc)
                     continue
             print(_json.dumps(docs, ensure_ascii=False))
+        reporter.finish(getattr(service, "last_report", None))
         return 0
 
     if args.command == "sync":
@@ -224,13 +284,16 @@ def _coerce_timeout_value(value: object) -> float | None:
             data_dir=args.data_dir,
             http_timeout=resolved_http_timeout,
         )
+        reporter = ProgressReporter("sync")
+        reporter.start()
         saved = service.sync(
             date=args.date,
             start=args.start,
             end=args.end,
             timezone=args.timezone,
             is_starred=True if args.starred_only else None,
             batch_size=max(1, int(args.batch_size)),
+            progress_callback=reporter.make_callback(),
         )
         if args.json:
             import json as _json
@@ -268,6 +331,7 @@ def _coerce_timeout_value(value: object) -> float | None:
                 "items": items,
             }
             print(_json.dumps(result, ensure_ascii=False))
+        reporter.finish(getattr(service, "last_report", None))
         return 0
 
     if args.command == "list":

limitless_tools/http/client.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+from collections.abc import Callable
 from typing import Any
 
 try:
@@ -97,6 +98,7 @@ def get_lifelogs(
         is_starred: bool | None = None,
         batch_size: int = 50,
         cursor: str | None = None,
+        progress_callback: Callable[[int, int], None] | None = None,
     ) -> list[dict[str, Any]]:
         """
         Fetch lifelogs with automatic pagination. Returns a list of lifelog dicts.
@@ -112,7 +114,9 @@ def get_lifelogs(
         current_cursor: str | None = cursor
         self.last_next_cursor: str | None = None
 
+        page_number = 0
         while True:
+            page_number += 1
             params: dict[str, Any] = {
                 "limit": page_size,
                 "direction": direction,
@@ -186,6 +190,11 @@ def get_lifelogs(
             body = resp.json()
             page_items: list[dict[str, Any]] = body.get("data", {}).get("lifelogs", []) or []
             collected.extend(page_items)
+            if progress_callback is not None:
+                try:
+                    progress_callback(page_number, len(collected))
+                except Exception:
+                    log.debug("Progress callback failed", exc_info=True)
 
             if limit is not None and len(collected) >= limit:
                 return collected[:limit]

limitless_tools/services/lifelog_service.py

@@ -3,6 +3,7 @@
 import hashlib
 import json
 import logging
+from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -21,6 +22,24 @@ def _load_json(path: Path) -> object | None:
         log.debug("Failed to read JSON from %s: %s", path, exc)
         return None
 
+@dataclass
+class SaveReport:
+    created: int = 0
+    updated: int = 0
+    unchanged: int = 0
+
+    def record(self, status: str) -> None:
+        if status == "created":
+            self.created += 1
+        elif status == "updated":
+            self.updated += 1
+        else:
+            self.unchanged += 1
+
+    @property
+    def total(self) -> int:
+        return self.created + self.updated + self.unchanged
+
 
 @dataclass
 class LifelogService:
@@ -30,6 +49,7 @@ class LifelogService:
     client: LimitlessClient | None = None
     repo: JsonFileRepository | None = None
     http_timeout: float | None = None
+    last_report: SaveReport | None = None
 
     def fetch(
         self,
@@ -44,6 +64,7 @@ def fetch(
         timezone: str | None = None,
         is_starred: bool | None = None,
         batch_size: int = 50,
+        progress_callback: Callable[[int, int], None] | None = None,
     ) -> list[str]:
         """Fetch lifelogs from API and save them to JSON files. Returns saved file paths."""
 
@@ -65,12 +86,17 @@ def fetch(
             timezone=timezone,
             is_starred=is_starred,
             batch_size=batch_size,
+            progress_callback=progress_callback,
         )
 
+        report = SaveReport()
         saved_paths: list[str] = []
         for item in lifelogs:
-            saved_paths.append(repo.save_lifelog(item))
+            save_result = repo.save_lifelog(item)
+            saved_paths.append(save_result.path)
+            report.record(save_result.status)
 
+        self.last_report = report
         return saved_paths
 
     def sync(
@@ -82,6 +108,7 @@ def sync(
         timezone: str | None = None,
         is_starred: bool | None = None,
         batch_size: int = 50,
+        progress_callback: Callable[[int, int], None] | None = None,
     ) -> list[str]:
         client = self.client or LimitlessClient(
             api_key=self.api_key or "",
@@ -121,13 +148,16 @@ def sync(
             is_starred=is_starred,
             batch_size=batch_size,
             cursor=(sig_state.get("lastCursor") or st.get("lastCursor")) if not any([date, start, end]) else None,
+            progress_callback=progress_callback,
         )
 
+        report = SaveReport()
         saved_paths: list[str] = []
         index_rows: list[dict[str, str | bool | None]] = []
         for ll in lifelogs:
-            p = repo.save_lifelog(ll)
-            saved_paths.append(p)
+            save_result = repo.save_lifelog(ll)
+            saved_paths.append(save_result.path)
+            report.record(save_result.status)
             index_rows.append(
                 {
                     "id": ll.get("id"),
@@ -136,7 +166,7 @@ def sync(
                     "endTime": ll.get("endTime"),
                     "isStarred": ll.get("isStarred"),
                     "updatedAt": ll.get("updatedAt"),
-                    "path": p,
+                    "path": save_result.path,
                 }
             )
 
@@ -175,6 +205,7 @@ def sync(
             st["signatures"] = signatures
         state_repo.save(st)
 
+        self.last_report = report
         return saved_paths
 
     def list_local(

limitless_tools/storage/json_repo.py

@@ -1,8 +1,15 @@
 from __future__ import annotations
 
 import json
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal
+
+
+@dataclass
+class SaveResult:
+    path: str
+    status: Literal["created", "updated", "unchanged"]
 
 
 class JsonFileRepository:
@@ -16,8 +23,22 @@ def path_for_lifelog(self, lifelog: dict[str, Any]) -> str:
         file_path = dir_path / f"lifelog_{lifelog.get('id')}.json"
         return str(file_path)
 
-    def save_lifelog(self, lifelog: dict[str, Any]) -> str:
+    def save_lifelog(self, lifelog: dict[str, Any]) -> SaveResult:
         path = Path(self.path_for_lifelog(lifelog))
         path.parent.mkdir(parents=True, exist_ok=True)
-        path.write_text(json.dumps(lifelog, ensure_ascii=False, indent=2))
-        return str(path)
+        serialized = json.dumps(lifelog, ensure_ascii=False, indent=2)
+        status: Literal["created", "updated", "unchanged"]
+        if path.exists():
+            try:
+                existing = json.loads(path.read_text())
+            except (json.JSONDecodeError, OSError):
+                existing = None
+            if existing == lifelog:
+                status = "unchanged"
+            else:
+                status = "updated"
+        else:
+            status = "created"
+        if status != "unchanged":
+            path.write_text(serialized)
+        return SaveResult(str(path), status)

tests/unit/test_cli_progress_reporting.py

@@ -0,0 +1,67 @@
+"""
+CLI progress reporter tests ensure user-visible output occurs during long operations.
+Single assert per test using capsys.
+"""
+
+from pathlib import Path
+
+
+def test_fetch_command_emits_progress_and_summary(monkeypatch, tmp_path: Path, capsys):
+    from limitless_tools.cli import main as cli_main
+    from limitless_tools.services.lifelog_service import SaveReport
+
+    class FakeService:
+        def __init__(self, *_, **__):
+            self.last_report = SaveReport(created=1, updated=1, unchanged=0)
+
+        def fetch(self, **kwargs):
+            progress_callback = kwargs.get("progress_callback")
+            if callable(progress_callback):
+                progress_callback(1, 1)
+                progress_callback(2, 2)
+            return [str(tmp_path / "lifelog_a.json"), str(tmp_path / "lifelog_b.json")]
+
+    monkeypatch.setattr(cli_main, "LifelogService", FakeService)
+    monkeypatch.setattr(cli_main, "load_env", lambda: None)
+    monkeypatch.delenv("LIMITLESS_DATA_DIR", raising=False)
+
+    code = cli_main.main([
+        "fetch",
+        "--limit",
+        "2",
+        "--data-dir",
+        str(tmp_path),
+    ])
+    stderr = capsys.readouterr().err
+    assert code == 0 and "Fetch started" in stderr and "Fetch complete" in stderr
+
+
+def test_sync_reports_no_changes(monkeypatch, tmp_path: Path, capsys):
+    from limitless_tools.cli import main as cli_main
+    from limitless_tools.services.lifelog_service import SaveReport
+
+    class FakeService:
+        def __init__(self, *_, **__):
+            self.last_report = SaveReport(created=0, updated=0, unchanged=3)
+
+        def sync(self, **kwargs):
+            progress_callback = kwargs.get("progress_callback")
+            if callable(progress_callback):
+                progress_callback(1, 0)
+            return []
+
+    monkeypatch.setattr(cli_main, "LifelogService", FakeService)
+    monkeypatch.setattr(cli_main, "load_env", lambda: None)
+    monkeypatch.delenv("LIMITLESS_DATA_DIR", raising=False)
+
+    code = cli_main.main([
+        "sync",
+        "--start",
+        "2025-01-01",
+        "--end",
+        "2025-01-02",
+        "--data-dir",
+        str(tmp_path),
+    ])
+    stderr = capsys.readouterr().err
+    assert code == 0 and "no changes" in stderr