From 25c17f5e32a74f2793b8c26618f33bec26739f44 Mon Sep 17 00:00:00 2001 From: Kirill Korikov <11762090+yourconscience@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:05:37 +0200 Subject: [PATCH 1/6] add external data sources registry --- docs/sources.md | 233 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 docs/sources.md diff --git a/docs/sources.md b/docs/sources.md new file mode 100644 index 0000000..4eff9b4 --- /dev/null +++ b/docs/sources.md @@ -0,0 +1,233 @@ +# External Data Sources Registry + +Centralized reference for every external data source accessible to dotagents skills, MCPs, and CLIs. +Each source entry covers: what it is, how we access it, auth requirements, which skills use it, known limitations, and gaps. + +## Design goals + +1. One place to check "can I get data from X?" before writing a new skill or wiring a new MCP. +2. Each source has a canonical access method. Fallbacks are documented, not discovered ad-hoc. +3. Auth credentials live in known locations; never duplicated across tools or pasted into chat. +4. Read-only by default. Write/mutate actions require explicit opt-in per source. + +## Access method taxonomy + +| Method | Examples | Tradeoffs | +|---|---|---| +| **CLI tool** | `gh`, `x-cli`, `rdt-cli`, `tg`, `gws` | Best for agents; scriptable; output parseable. Preferred when available. | +| **MCP server** | linkedin-scraper-mcp, telegram-readonly, tavily | Native tool-use in agent context; higher setup cost; session lifecycle to manage. | +| **Public API** | HN Algolia, Greenhouse boards, HF Hub | No auth, no breakage risk from session rotation. Limited to public data. | +| **Web search** | Tavily MCP, native WebSearch, WebFetch | Fallback for anything; low precision; can't access authenticated content. | +| **Browser automation** | Playwright (LinkedIn MCP), x-cli login | Fragile; session/cookie rotation; ToS risk. Last resort. | + +--- + +## Source catalog + +### X.com / Twitter + +| | | +|---|---| +| Canonical access | `x-cli` CLI (Gladium-AI/x-cli) | +| Protocol | X internal GraphQL API via captured browser session | +| Auth | Browser cookies + bearer/CSRF in `~/.x-cli/credentials.json`. Login: `x-cli login` (requires Chrome). | +| Skills | x-cli, x-sim, tech-search, repo-eval | +| Read | timelines, tweets, users, search, followers/following | +| Write | None wired. x-sim is offline simulation only. | +| Fallback | `site:x.com ` via WebSearch | +| Limitations | GraphQL query ID drift breaks all commands until manual refresh. Rate-limited. No official API v2 wired (API key exists in research/API_KEYS.md but returns Unauthorized -- likely needs OAuth consumer flow). | +| Gaps | Official API v2 integration would eliminate endpoint drift risk. | + +### Hacker News + +| | | +|---|---| +| Canonical access | Algolia HN Search API (public, no auth) | +| Protocol | `GET https://hn.algolia.com/api/v1/search?query=&tags=story&hitsPerPage=N` | +| Auth | None | +| Skills | tech-search, repo-eval | +| Read | Story search by relevance. Thread bodies via WebFetch on `news.ycombinator.com/item?id=`. | +| Write | N/A | +| Fallback | `site:news.ycombinator.com` via WebSearch for older threads | +| Limitations | `search_by_date` endpoint returns zero-comment noise; disallowed in tech-search. | +| Gaps | None significant. Stable public API. | + +### Reddit + +| | | +|---|---| +| Canonical access | `rdt-cli` (Python, `uv tool install rdt-cli`) | +| Protocol | Reddit browser cookies; `rdt search`, `rdt read` | +| Auth | Browser cookies. VPS/headless = 403 without cookies. | +| Skills | tech-search | +| Read | Subreddit search, thread reading. Target subs: r/ExperiencedDevs, r/ClaudeAI, r/ClaudeCode, r/LocalLLaMA, r/MachineLearning, r/devops, r/commandline, r/neovim, r/Python, r/mcp, r/cybersecurity | +| Write | None | +| Fallback 1 | Pullpush API: `https://api.pullpush.io/reddit/search/submission/?q=&size=5&sort=desc&sort_type=score` (historical only, no auth) | +| Fallback 2 | `site:reddit.com ` via WebSearch | +| Limitations | Cookie-dependent; headless forbidden. Pullpush is historical, not real-time. | +| Gaps | No official Reddit API integration (requires app registration + OAuth2). Would solve headless access. | + +### Discord + +| | | +|---|---| +| Canonical access | `discord-cli` (Python, `uv tool install kabi-discord-cli`) | +| Protocol | Discord user-token API. Local SQLite sync for channels. | +| Auth | `DISCORD_TOKEN` env var (user token, not bot). CLI can scan local browser/Discord storage. | +| Skills | tech-search (opt-in only, ML/LLM/agents/evals topics) | +| Read | Channel search, message reading. Known guilds: NousResearch (`1053877538025386074`), Anthropic/Claude (`1456350064065904867`). | +| Write | None | +| Fallback | `rtk proxy curl` direct Discord API (same token) | +| Limitations | User-token usage may violate Discord ToS. Account risk noted. Opt-in only. Local search requires prior sync. | +| Gaps | Bot-token access would be ToS-safe but requires server admin approval. | + +### GitHub + +| | | +|---|---| +| Canonical access | `gh` CLI + `gh api graphql` | +| Protocol | GitHub REST + GraphQL via OAuth token | +| Auth | `gh auth login` (OAuth browser flow) | +| Skills | tech-search, repo-eval, pr-triage, jobs (career page detection) | +| Read | Repos, issues, PRs, releases, commits, check runs, languages, contributor stats. Clone to `~/Public/` for code analysis. | +| Write | PR comments via pr-triage (draft-then-review by default; direct replies to bot comments allowed). | +| Limitations | None significant. Always-available. | +| Gaps | None. | + +### LinkedIn + +| | | +|---|---| +| Canonical access | `linkedin-scraper-mcp` MCP server (Playwright browser automation) | +| Protocol | Headless Chrome via Playwright. No official LinkedIn API. | +| Auth | One-time browser login: `uvx linkedin-scraper-mcp --login`. Session managed by Playwright. | +| Skills | jobs | +| MCP tools | get_inbox, search_conversations, get_conversation, get_person_profile, get_company_profile, get_company_employees, get_job_details, search_people, search_jobs, search_companies | +| Read | DMs, profiles, job postings, company data | +| Write | connect_with_person, send_message exist but are NOT used autonomously. Jobs skill: "never send messages or connection requests unless user explicitly asks." | +| Config | dotagents.yaml, ~/.codex/config.toml, ~/.hermes/config.yaml (all enabled) | +| Limitations | Browser automation fragile. Session expires. MCP unavailability handled gracefully by jobs skill. | +| Gaps | No CLI wrapper. Official LinkedIn API requires company-level partnership. | + +### Telegram + +| | | +|---|---| +| Canonical access | `tg` CLI + `telegram-readonly` MCP server (both backed by same Telethon daemon) | +| Protocol | MTProto via Telethon. Singleton daemon on Unix socket `~/.local/share/dotagents/telegram-readonly/daemon.sock`. | +| Auth | `TELEGRAM_API_ID` + `TELEGRAM_API_HASH` in `~/.agents/mcp/telegram-readonly/.env`. Session file at `~/.local/share/dotagents/telegram-readonly/telegram.session`. Login: `uv run python login.py`. | +| Skills | tg | +| MCP tools | list_dialogs, get_recent_messages, search_messages, get_chat_info | +| CLI commands | `tg dialogs`, `tg read`, `tg search`, `tg info` | +| Read | Chat listing, message reading, search across chats | +| Write | None. Read-only enforced in both MCP and CLI. | +| Config | ~/.codex/config.toml, ~/.hermes/config.yaml | +| Limitations | Idle timeout 30m (daemon sleeps). Session can expire requiring re-login. | +| Gaps | No write capability by design. Hermes has `TELEGRAM_HOME_CHANNEL: 38369051` for its own gateway but that's separate from dotagents. | + +### Google Workspace (Gmail, Drive, Docs, Sheets, Calendar) + +| | | +|---|---| +| Canonical access | `gws` CLI | +| Protocol | Google REST APIs via OAuth | +| Auth | `gws auth login`. Credentials at `~/.config/gws/`. Also `~/.hermes/auth/google_oauth.json` for Hermes. | +| Skills | gws, jobs (Gmail recruiter signal queries) | +| Read | Full CRUD on Drive, Docs, Sheets; Gmail search/read; Calendar events | +| Write | Docs, Sheets, Drive file ops. Gmail: "never send mail unless explicitly asked." | +| Limitations | None significant. | +| Gaps | Claude Code MCP for Google Workspace exists (claude.ai Gmail/Drive/Calendar) but is separate from gws CLI. Parity not enforced. | + +### Web Search (general) + +| | | +|---|---| +| Canonical access | Tavily MCP server (primary) + native agent WebSearch (fallback) | +| Protocol | Tavily: `npx mcp-remote https://mcp.tavily.com/mcp`. Native: agent built-in. | +| Auth | Tavily: OAuth via mcp-remote (server-side, no local API key). Native: none. | +| Skills | tech-search, repo-eval, jobs, and general fallback for any source | +| MCP tools | tavily_search, tavily_extract, tavily_crawl, tavily_map, tavily_research | +| Read | Web search, page extraction, site mapping | +| Limitations | Tavily auth mechanism opaque (mcp-remote handles it). WebFetch blocked by many sites (RA, LinkedIn, etc.). | +| Gaps | Tavily auth needs verification if it stops working. | + +### Job ATS Portals (Greenhouse, Ashby, Lever) + +| | | +|---|---| +| Canonical access | `portals-scan` Go tool (skills/jobs/tools/portals-scan/) | +| Protocol | Direct HTTP to public board APIs. No auth. | +| Endpoints | Greenhouse: `boards-api.greenhouse.io/v1/boards/{slug}/jobs`; Ashby: `api.ashbyhq.com/posting-api/job-board/{slug}`; Lever: `api.lever.co/v0/postings/{slug}` | +| Skills | jobs (/jobs scan) | +| Read | Job listings with title, location, team, compensation (where available) | +| Write | None | +| Limitations | Only 3 ATS platforms. Companies with custom career portals need `enabled: false`. 10 concurrent scans. | +| Gaps | No Workday, iCIMS, SmartRecruiters, BambooHR. These cover a large chunk of enterprise hiring. | + +### Glassdoor / Levels.fyi + +| | | +|---|---| +| Canonical access | WebSearch only (no structured integration) | +| Skills | jobs (/jobs check -- comp research) | +| Read | Whatever WebSearch returns for `"company" "role" salary levels.fyi glassdoor` | +| Limitations | No structured scraping. "If no data, say so -- do not invent numbers." | +| Gaps | levels.fyi has an unofficial API. Glassdoor blocks scraping aggressively. Low priority unless comp research becomes frequent. | + +### Hugging Face + +| | | +|---|---| +| Canonical access | HF Hub REST API (public) | +| Protocol | `GET https://huggingface.co/api/models//` | +| Auth | None for public models | +| Skills | repo-eval (ML/model repos only) | +| Read | Model/dataset metadata: public/private state, downloads, likes, tags, files | +| Limitations | Read-only. Pickle formats flagged as trusted-code artifacts, never loaded. | +| Gaps | `huggingface-cli` exists but not wired. Would give authenticated access to gated models. | + +### Resident Advisor (RA) + +| | | +|---|---| +| Canonical access | **Not integrated.** Undocumented GraphQL endpoint, reverse-engineered. | +| Protocol | POST `https://ra.co/graphql` with operation `GET_EVENT_LISTINGS`. Area codes: Amsterdam=29. | +| Auth | None (public, but blocks standard user-agents and WebFetch/Tavily). Requires browser-like UA + Referer header. | +| Read | Event listings with title, venue, artists, genres, attendance, times | +| Limitations | No official API. GraphQL schema undocumented, reverse-engineered. May break. RA blocks all fetch tools (403). | +| Gaps | Needs a CLI wrapper or skill integration if event lookups become recurring. | + +--- + +## Not yet integrated (known wants) + +| Source | Why | Effort | Priority | +|---|---|---|---| +| **RA** | Event discovery in NL/EU cities | Low: wrap the known GraphQL endpoint in a CLI or skill step | Nice-to-have | +| **Glassdoor (structured)** | Comp research for job search | High: aggressive anti-scraping; unofficial APIs break | Low | +| **Workday/iCIMS ATS** | Enterprise career portals | Medium: each platform is different, no unified API | Low unless targeting large employers | +| **Reddit official API** | Headless-safe access without cookies | Medium: app registration + OAuth2 flow | Medium (fixes VPS access) | +| **X official API v2** | Eliminate endpoint drift from x-cli | Medium: need OAuth consumer flow, not just bearer token | Medium (stability) | +| **HF CLI** | Gated model access | Low: `pip install huggingface-cli`, wire into repo-eval | Low | + +## Auth credential locations + +| Source | Location | Rotation | +|---|---|---| +| X.com | `~/.x-cli/credentials.json` | Manual re-login on session expiry or GraphQL drift | +| Reddit | Browser cookies (managed by rdt-cli) | Manual; expires unpredictably | +| Discord | `DISCORD_TOKEN` env var | Manual; user-token, high risk | +| GitHub | `gh auth` keychain | Long-lived OAuth; rarely expires | +| LinkedIn | Playwright session (linkedin-scraper-mcp) | `uvx linkedin-scraper-mcp --login` on expiry | +| Telegram | `~/.local/share/dotagents/telegram-readonly/telegram.session` | Re-login via `uv run python login.py` | +| Google | `~/.config/gws/` + `~/.hermes/auth/google_oauth.json` | OAuth refresh; rarely expires | +| Tavily | mcp-remote managed (server-side) | Unknown; needs verification | + +## Principles + +1. **CLI-first.** If a CLI exists, prefer it over MCP. MCPs are for agent-native tool-use where CLI piping is awkward. +2. **Read-only default.** Write actions (sending messages, posting comments, connecting) require explicit user opt-in per invocation. +3. **Graceful degradation.** Every source with auth has a documented fallback (usually WebSearch). Skills must not block on MCP unavailability. +4. **No secrets in chat.** Credentials stay in their canonical locations. Agents never paste tokens, cookies, or API keys into conversation logs. +5. **ToS awareness.** Sources using unofficial access (x-cli, discord user-token, LinkedIn Playwright) are labeled with risk. Opt-in where risk is high. +6. **Agent-agnostic.** Source access methods work across Claude Code, Codex, Hermes, Droid. Agent-specific wiring details go in dotagents.yaml, not here. From 6cb5a52bb9462eb6ea122c0ad743cb4f1789f76e Mon Sep 17 00:00:00 2001 From: Kirill Korikov <11762090+yourconscience@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:29:33 +0200 Subject: [PATCH 2/6] add configurable sources system with dotagents sources command --- cmd/dotagents/config.go | 1 + cmd/dotagents/main.go | 4 + cmd/dotagents/sources.go | 271 ++++++++++++++++++++++++++++++++++ cmd/dotagents/sources_cli.go | 174 ++++++++++++++++++++++ cmd/dotagents/sources_test.go | 195 ++++++++++++++++++++++++ docs/sources.md | 14 +- dotagents.yaml | 27 ++++ skills/tech-search/SKILL.md | 28 ++-- 8 files changed, 703 insertions(+), 11 deletions(-) create mode 100644 cmd/dotagents/sources.go create mode 100644 cmd/dotagents/sources_cli.go create mode 100644 cmd/dotagents/sources_test.go diff --git a/cmd/dotagents/config.go b/cmd/dotagents/config.go index 18dbf4a..b8518cc 100644 --- a/cmd/dotagents/config.go +++ b/cmd/dotagents/config.go @@ -93,6 +93,7 @@ func mergeConfig(base *config, overlay config) { base.MCPServers = mergeByKey(base.MCPServers, overlay.MCPServers, func(s mcpServerConfig) string { return strings.TrimSpace(s.Name) }) base.Hooks = mergeByKey(base.Hooks, overlay.Hooks, func(h hookConfig) string { return strings.TrimSpace(h.Name) }) base.Plugins = mergeByKey(base.Plugins, overlay.Plugins, func(p pluginConfig) string { return strings.TrimSpace(p.Name) }) + base.Sources = mergeByKey(base.Sources, overlay.Sources, func(s sourceConfig) string { return strings.TrimSpace(s.Name) }) } func mergeByKey[T any](base []T, overlay []T, key func(T) string) []T { diff --git a/cmd/dotagents/main.go b/cmd/dotagents/main.go index 36bcc28..cfb4205 100644 --- a/cmd/dotagents/main.go +++ b/cmd/dotagents/main.go @@ -15,6 +15,7 @@ type config struct { ExternalSkills []externalSkillSource `yaml:"external_skills"` Plugins []pluginConfig `yaml:"plugins,omitempty"` Hooks []hookConfig `yaml:"hooks,omitempty"` + Sources []sourceConfig `yaml:"sources,omitempty"` } type pluginConfig struct { @@ -171,6 +172,8 @@ func run(args []string) error { return err } return runDoctor(opts) + case "sources": + return runSources(args[1:]) case "external": return runExternal(args[1:]) case "promote": @@ -245,6 +248,7 @@ func printUsage() { fmt.Println(" dotagents mcp add --command Add/update canonical managed MCP") fmt.Println(" dotagents mcp import Import native MCP into canonical config") fmt.Println(" dotagents mcp remove Remove canonical managed MCP") + fmt.Println(" dotagents sources [--json|--compact] [name] Show external data source availability") fmt.Println(" dotagents external list Show external skill sources and lock state") fmt.Println(" dotagents external update [name ...] Move external sources to latest and rewrite the lock") fmt.Println(" dotagents plugin add Install Claude Code plugin delivery for claude-code") diff --git a/cmd/dotagents/sources.go b/cmd/dotagents/sources.go new file mode 100644 index 0000000..118b298 --- /dev/null +++ b/cmd/dotagents/sources.go @@ -0,0 +1,271 @@ +package main + +import ( + "fmt" + "os/exec" + "path/filepath" + "strings" +) + +type sourceConfig struct { + Name string `yaml:"name"` + Enabled *bool `yaml:"enabled,omitempty"` + Preferred string `yaml:"preferred,omitempty"` +} + +type methodType string + +const ( + methodCLI methodType = "cli" + methodMCP methodType = "mcp" + methodAPI methodType = "api" + methodFallback methodType = "fallback" +) + +type sourceMethodDef struct { + Name string + Type methodType + Priority int + Detect string // binary name for LookPath + Check string // shell one-liner for deeper check (run via sh -c) + MCP string // MCP server name to look up in config + Auth string // credential location (display) + Setup string // one-line setup instruction +} + +type sourceDef struct { + Name string + Desc string + Methods []sourceMethodDef + DefaultOn bool + ToSRisk string // "", "high" +} + +type methodStatus struct { + Name string `json:"name"` + Type methodType `json:"type"` + Priority int `json:"priority"` + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` +} + +type sourceStatus struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Best string `json:"best"` + Methods []methodStatus `json:"methods"` +} + +var sourceRegistry = []sourceDef{ + { + Name: "x.com", Desc: "X.com / Twitter", DefaultOn: true, ToSRisk: "high", + Methods: []sourceMethodDef{ + {Name: "x-api-v2", Type: methodAPI, Priority: 0, Check: "test -s ~/.x-api/credentials.json", Auth: "~/.x-api/credentials.json", Setup: "register OAuth consumer app at developer.x.com"}, + {Name: "x-cli", Type: methodCLI, Priority: 1, Detect: "x-cli", Auth: "~/.x-cli/credentials.json", Setup: "x-cli auth login"}, + {Name: "websearch", Type: methodFallback, Priority: 99}, + }, + }, + { + Name: "reddit", Desc: "Reddit", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "rdt-cli", Type: methodCLI, Priority: 1, Detect: "rdt", Setup: "uv tool install rdt-cli"}, + {Name: "pullpush", Type: methodAPI, Priority: 2}, + {Name: "websearch", Type: methodFallback, Priority: 99}, + }, + }, + { + Name: "hacker-news", Desc: "Hacker News", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "algolia", Type: methodAPI, Priority: 1}, + }, + }, + { + Name: "discord", Desc: "Discord", DefaultOn: false, ToSRisk: "high", + Methods: []sourceMethodDef{ + {Name: "discord-cli", Type: methodCLI, Priority: 1, Detect: "discord", Check: "test -n \"$DISCORD_TOKEN\"", Auth: "$DISCORD_TOKEN env var", Setup: "uv tool install kabi-discord-cli"}, + {Name: "websearch", Type: methodFallback, Priority: 99}, + }, + }, + { + Name: "github", Desc: "GitHub", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "gh", Type: methodCLI, Priority: 1, Detect: "gh", Setup: "gh auth login"}, + }, + }, + { + Name: "linkedin", Desc: "LinkedIn", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "linkedin-mcp", Type: methodMCP, Priority: 1, MCP: "linkedin", Setup: "uvx linkedin-scraper-mcp --login"}, + {Name: "websearch", Type: methodFallback, Priority: 99}, + }, + }, + { + Name: "telegram", Desc: "Telegram", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "tg", Type: methodCLI, Priority: 1, Detect: "tg", Auth: "~/.local/share/dotagents/telegram-readonly/telegram.session", Setup: "cd ~/.agents/mcp/telegram-readonly && uv run python login.py"}, + {Name: "telegram-mcp", Type: methodMCP, Priority: 2, MCP: "telegram-readonly"}, + }, + }, + { + Name: "google", Desc: "Google Workspace", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "gws", Type: methodCLI, Priority: 1, Detect: "gws", Auth: "~/.config/gws/", Setup: "gws auth login"}, + }, + }, + { + Name: "web-search", Desc: "Web search (general)", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "tavily-mcp", Type: methodMCP, Priority: 1, MCP: "tavily"}, + {Name: "native", Type: methodFallback, Priority: 99}, + }, + }, + { + Name: "job-portals", Desc: "ATS portals (Greenhouse, Ashby, Lever)", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "portals-scan", Type: methodCLI, Priority: 1, Detect: "go", Check: "test -d ~/.agents/skills/jobs/tools/portals-scan"}, + }, + }, + { + Name: "glassdoor", Desc: "Glassdoor / Levels.fyi", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "websearch", Type: methodFallback, Priority: 99}, + }, + }, + { + Name: "hugging-face", Desc: "Hugging Face Hub", DefaultOn: true, + Methods: []sourceMethodDef{ + {Name: "hf-api", Type: methodAPI, Priority: 1}, + }, + }, + { + Name: "ra", Desc: "Resident Advisor", DefaultOn: false, + Methods: nil, + }, +} + +func checkMethodAvailability(m sourceMethodDef, cfg config, home string) (bool, string) { + switch m.Type { + case methodFallback: + return true, "" + case methodAPI: + if m.Check == "" { + return true, "" + } + return runShellCheck(m.Check, home) + case methodCLI: + if m.Detect != "" { + if _, err := exec.LookPath(m.Detect); err != nil { + return false, fmt.Sprintf("%s not on PATH", m.Detect) + } + } + if m.Check != "" { + return runShellCheck(m.Check, home) + } + return true, "" + case methodMCP: + if m.MCP == "" { + return false, "no MCP server name" + } + for _, srv := range cfg.MCPServers { + if srv.Name == m.MCP && srv.Enabled { + return true, "" + } + } + return false, fmt.Sprintf("MCP server %q not configured or disabled", m.MCP) + } + return false, "unknown method type" +} + +func runShellCheck(cmd string, home string) (bool, string) { + cmd = strings.ReplaceAll(cmd, "~", home) + out, err := exec.Command("sh", "-c", cmd).CombinedOutput() + if err != nil { + reason := strings.TrimSpace(string(out)) + if reason == "" { + if cmd != "" { + reason = "check failed" + } + } + return false, reason + } + return true, "" +} + +func resolveSourceStatus(cfg config, home string) []sourceStatus { + overrides := make(map[string]sourceConfig, len(cfg.Sources)) + for _, s := range cfg.Sources { + overrides[s.Name] = s + } + + var results []sourceStatus + for _, def := range sourceRegistry { + enabled := def.DefaultOn + preferred := "" + if ov, ok := overrides[def.Name]; ok { + if ov.Enabled != nil { + enabled = *ov.Enabled + } + preferred = ov.Preferred + } + + ss := sourceStatus{ + Name: def.Name, + Enabled: enabled, + } + + if !enabled || len(def.Methods) == 0 { + results = append(results, ss) + continue + } + + bestPriority := 999 + for _, m := range def.Methods { + avail, reason := checkMethodAvailability(m, cfg, home) + ms := methodStatus{ + Name: m.Name, + Type: m.Type, + Priority: m.Priority, + Available: avail, + Reason: reason, + } + ss.Methods = append(ss.Methods, ms) + + if avail && m.Name == preferred { + ss.Best = m.Name + bestPriority = -1 + } else if avail && m.Priority < bestPriority && ss.Best == "" { + ss.Best = m.Name + bestPriority = m.Priority + } + } + + // If preferred was set but not available, still pick best available + if ss.Best == "" { + for _, ms := range ss.Methods { + if ms.Available && ms.Priority < bestPriority { + ss.Best = ms.Name + bestPriority = ms.Priority + } + } + } + + results = append(results, ss) + } + return results +} + +func findSourceDef(name string) *sourceDef { + for i := range sourceRegistry { + if sourceRegistry[i].Name == name { + return &sourceRegistry[i] + } + } + return nil +} + +func expandAuthPath(auth string, home string) string { + if strings.HasPrefix(auth, "~/") { + return filepath.Join(home, auth[2:]) + } + return auth +} diff --git a/cmd/dotagents/sources_cli.go b/cmd/dotagents/sources_cli.go new file mode 100644 index 0000000..630806b --- /dev/null +++ b/cmd/dotagents/sources_cli.go @@ -0,0 +1,174 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "strings" +) + +func runSources(args []string) error { + fs := flag.NewFlagSet("sources", flag.ContinueOnError) + jsonFlag := fs.Bool("json", false, "output JSON") + compactFlag := fs.Bool("compact", false, "one-line output for skill consumption") + configPath := fs.String("config", "", "config file path") + if err := fs.Parse(args); err != nil { + return err + } + + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("resolve home: %w", err) + } + + repoRoot, _, err := findRoots() + if err != nil { + return fmt.Errorf("find roots: %w", err) + } + + cfg, err := loadConfig(repoRoot, home, *configPath) + if err != nil { + return fmt.Errorf("load config: %w", err) + } + + statuses := resolveSourceStatus(cfg, home) + + // Single source detail mode + if fs.NArg() > 0 { + name := fs.Arg(0) + for _, ss := range statuses { + if ss.Name == name { + if *jsonFlag { + return printJSON(ss) + } + return printSourceDetail(ss, home) + } + } + return fmt.Errorf("unknown source %q", name) + } + + if *jsonFlag { + return printJSON(statuses) + } + if *compactFlag { + return printCompact(statuses) + } + return printTable(statuses) +} + +func printJSON(v interface{}) error { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +func printCompact(statuses []sourceStatus) error { + var parts []string + for _, ss := range statuses { + if !ss.Enabled || ss.Best == "" { + continue + } + parts = append(parts, ss.Name+":"+ss.Best) + } + fmt.Println(strings.Join(parts, " ")) + return nil +} + +func printTable(statuses []sourceStatus) error { + for _, ss := range statuses { + name := padRight(ss.Name, 16) + if !ss.Enabled { + fmt.Printf("%s disabled\n", name) + continue + } + if len(ss.Methods) == 0 { + fmt.Printf("%s not integrated\n", name) + continue + } + + best := padRight(ss.Best, 16) + + var methodParts []string + var unavailable []string + for _, ms := range ss.Methods { + if ms.Available { + tag := "ok" + if ms.Type == methodFallback { + tag = "fallback" + } + methodParts = append(methodParts, fmt.Sprintf("%s [%s]", ms.Name, tag)) + } else { + reason := ms.Reason + if reason == "" { + reason = "unavailable" + } + unavailable = append(unavailable, fmt.Sprintf("%s: %s", ms.Name, reason)) + } + } + + methods := strings.Join(methodParts, " ") + line := fmt.Sprintf("%s %s %s", name, best, methods) + if len(unavailable) > 0 { + line += " (" + strings.Join(unavailable, "; ") + ")" + } + fmt.Println(line) + } + return nil +} + +func printSourceDetail(ss sourceStatus, home string) error { + def := findSourceDef(ss.Name) + if def == nil { + return fmt.Errorf("no definition for %q", ss.Name) + } + + fmt.Printf("%s (%s)\n", ss.Name, def.Desc) + fmt.Printf(" enabled: %v\n", ss.Enabled) + if def.ToSRisk != "" { + fmt.Printf(" tos-risk: %s\n", def.ToSRisk) + } + if ss.Best != "" { + fmt.Printf(" best: %s\n", ss.Best) + } + fmt.Println() + + if len(def.Methods) == 0 { + fmt.Println(" no methods (not integrated)") + return nil + } + + fmt.Println(" methods:") + for i, m := range def.Methods { + status := "NOT AVAILABLE" + reason := "" + if i < len(ss.Methods) { + if ss.Methods[i].Available { + status = "OK" + } else { + reason = ss.Methods[i].Reason + } + } + + marker := " " + if m.Name == ss.Best { + marker = "*" + } + fmt.Printf(" %s [%d] %-16s %-14s %s\n", marker, m.Priority, m.Name, status, reason) + + if m.Auth != "" { + fmt.Printf(" auth: %s\n", expandAuthPath(m.Auth, home)) + } + if m.Setup != "" { + fmt.Printf(" setup: %s\n", m.Setup) + } + } + return nil +} + +func padRight(s string, width int) string { + if len(s) >= width { + return s + } + return s + strings.Repeat(" ", width-len(s)) +} diff --git a/cmd/dotagents/sources_test.go b/cmd/dotagents/sources_test.go new file mode 100644 index 0000000..954e1e9 --- /dev/null +++ b/cmd/dotagents/sources_test.go @@ -0,0 +1,195 @@ +package main + +import ( + "testing" +) + +func TestResolveSourceStatus_defaults(t *testing.T) { + cfg := config{Version: 1} + statuses := resolveSourceStatus(cfg, "/tmp/fakehome") + + if len(statuses) != len(sourceRegistry) { + t.Fatalf("expected %d sources, got %d", len(sourceRegistry), len(statuses)) + } + + byName := make(map[string]sourceStatus) + for _, ss := range statuses { + byName[ss.Name] = ss + } + + // discord defaults to disabled + if byName["discord"].Enabled { + t.Error("discord should be disabled by default") + } + + // github defaults to enabled + if !byName["github"].Enabled { + t.Error("github should be enabled by default") + } + + // ra defaults to disabled + if byName["ra"].Enabled { + t.Error("ra should be disabled by default") + } + + // hacker-news algolia is always available (public API, no check) + hn := byName["hacker-news"] + if hn.Best != "algolia" { + t.Errorf("hacker-news best should be algolia, got %q", hn.Best) + } +} + +func TestResolveSourceStatus_enableOverride(t *testing.T) { + enabled := true + cfg := config{ + Version: 1, + Sources: []sourceConfig{ + {Name: "discord", Enabled: &enabled}, + }, + } + statuses := resolveSourceStatus(cfg, "/tmp/fakehome") + + for _, ss := range statuses { + if ss.Name == "discord" { + if !ss.Enabled { + t.Error("discord should be enabled via override") + } + return + } + } + t.Error("discord not found in statuses") +} + +func TestResolveSourceStatus_disableOverride(t *testing.T) { + disabled := false + cfg := config{ + Version: 1, + Sources: []sourceConfig{ + {Name: "github", Enabled: &disabled}, + }, + } + statuses := resolveSourceStatus(cfg, "/tmp/fakehome") + + for _, ss := range statuses { + if ss.Name == "github" { + if ss.Enabled { + t.Error("github should be disabled via override") + } + if ss.Best != "" { + t.Errorf("disabled source should have empty best, got %q", ss.Best) + } + return + } + } + t.Error("github not found in statuses") +} + +func TestResolveSourceStatus_mcpCheck(t *testing.T) { + cfg := config{ + Version: 1, + MCPServers: []mcpServerConfig{ + {Name: "tavily", Enabled: true, Command: "npx"}, + }, + } + statuses := resolveSourceStatus(cfg, "/tmp/fakehome") + + for _, ss := range statuses { + if ss.Name == "web-search" { + if ss.Best != "tavily-mcp" { + t.Errorf("web-search best should be tavily-mcp, got %q", ss.Best) + } + return + } + } + t.Error("web-search not found") +} + +func TestResolveSourceStatus_mcpMissing(t *testing.T) { + cfg := config{Version: 1} + statuses := resolveSourceStatus(cfg, "/tmp/fakehome") + + for _, ss := range statuses { + if ss.Name == "linkedin" { + // linkedin-mcp should be unavailable without MCP config + for _, ms := range ss.Methods { + if ms.Name == "linkedin-mcp" && ms.Available { + t.Error("linkedin-mcp should be unavailable without MCP server config") + } + } + // best should fall back to websearch + if ss.Best != "websearch" { + t.Errorf("linkedin best should be websearch without MCP, got %q", ss.Best) + } + return + } + } + t.Error("linkedin not found") +} + +func TestResolveSourceStatus_preferredOverride(t *testing.T) { + cfg := config{ + Version: 1, + Sources: []sourceConfig{ + {Name: "reddit", Preferred: "pullpush"}, + }, + } + statuses := resolveSourceStatus(cfg, "/tmp/fakehome") + + for _, ss := range statuses { + if ss.Name == "reddit" { + if ss.Best != "pullpush" { + t.Errorf("reddit best should be pullpush via preferred override, got %q", ss.Best) + } + return + } + } + t.Error("reddit not found") +} + +func TestFindSourceDef(t *testing.T) { + def := findSourceDef("x.com") + if def == nil { + t.Fatal("x.com not found in registry") + } + if def.Desc != "X.com / Twitter" { + t.Errorf("unexpected desc: %s", def.Desc) + } + + if findSourceDef("nonexistent") != nil { + t.Error("nonexistent source should return nil") + } +} + +func TestSourceConfigMerge(t *testing.T) { + enabled := true + base := config{ + Version: 1, + Agents: []agentConfig{{Name: "test", Enabled: true, SkillRoot: "/tmp"}}, + Sources: []sourceConfig{ + {Name: "discord"}, + {Name: "github"}, + }, + } + overlay := config{ + Sources: []sourceConfig{ + {Name: "discord", Enabled: &enabled, Preferred: "discord-cli"}, + }, + } + mergeConfig(&base, overlay) + + if len(base.Sources) != 2 { + t.Fatalf("expected 2 sources after merge, got %d", len(base.Sources)) + } + for _, s := range base.Sources { + if s.Name == "discord" { + if s.Enabled == nil || !*s.Enabled { + t.Error("discord enabled should be true after merge") + } + if s.Preferred != "discord-cli" { + t.Errorf("discord preferred should be discord-cli, got %q", s.Preferred) + } + return + } + } + t.Error("discord not found after merge") +} diff --git a/docs/sources.md b/docs/sources.md index 4eff9b4..42d69ea 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -3,12 +3,24 @@ Centralized reference for every external data source accessible to dotagents skills, MCPs, and CLIs. Each source entry covers: what it is, how we access it, auth requirements, which skills use it, known limitations, and gaps. +## Runtime status + +```bash +dotagents sources # table: what's available now +dotagents sources --compact # one-liner for skill prompts +dotagents sources --json # structured output +dotagents sources x.com # single source detail with auth/setup info +``` + +Configure in `dotagents.yaml` under `sources:`. Per-machine overrides in `dotagents.local.yaml`. + ## Design goals 1. One place to check "can I get data from X?" before writing a new skill or wiring a new MCP. -2. Each source has a canonical access method. Fallbacks are documented, not discovered ad-hoc. +2. Each source has a canonical access method with a priority. `dotagents sources` picks the best available. 3. Auth credentials live in known locations; never duplicated across tools or pasted into chat. 4. Read-only by default. Write/mutate actions require explicit opt-in per source. +5. Configurable per machine: set `preferred: x-api-v2` in `dotagents.local.yaml` to override method selection. ## Access method taxonomy diff --git a/dotagents.yaml b/dotagents.yaml index 83a518c..c4a391d 100644 --- a/dotagents.yaml +++ b/dotagents.yaml @@ -211,6 +211,33 @@ mcp_servers: - hermes - droid - pi +sources: + - name: x.com + enabled: true + - name: reddit + enabled: true + - name: hacker-news + enabled: true + - name: discord + enabled: false + - name: github + enabled: true + - name: linkedin + enabled: true + - name: telegram + enabled: true + - name: google + enabled: true + - name: web-search + enabled: true + - name: job-portals + enabled: true + - name: glassdoor + enabled: true + - name: hugging-face + enabled: true + - name: ra + enabled: false hooks: - name: memory-session-start enabled: true diff --git a/skills/tech-search/SKILL.md b/skills/tech-search/SKILL.md index 4428cde..823009f 100644 --- a/skills/tech-search/SKILL.md +++ b/skills/tech-search/SKILL.md @@ -31,7 +31,9 @@ The guiding rule: broad web finds the map; source-specific searches verify the t ## Sources -Search sources in parallel when practical, but do not force every source. Skipped sources are fine when they are irrelevant, unauthenticated, or low-signal. +Before searching, run `dotagents sources --compact` to discover available methods. Use the best available method for each source; fall back to the next if it fails at runtime. + +Search sources in parallel when practical, but do not force every source. Skipped sources are fine when they are irrelevant, unauthenticated, or low-signal. If a source shows as disabled in `dotagents sources`, do not attempt it. Reference: `references/reddit-discord-cli-eval.md` records the repo evaluation behind the `rdt-cli` and `discord-cli` recommendations. @@ -86,38 +88,44 @@ Read threads at `https://news.ycombinator.com/item?id=` and cite the H ### 4. Reddit -Preferred path when installed: +Use the method shown by `dotagents sources reddit`. + +**Target subreddits:** r/ExperiencedDevs, r/ClaudeAI, r/ClaudeCode, r/LocalLLaMA, r/MachineLearning, r/devops, r/commandline, r/neovim, r/Python, r/mcp, r/cybersecurity +Pick 2-3 relevant subreddits. Avoid broad Reddit for ambiguous terms; it will chase engagement from irrelevant communities. Add context keywords, e.g. `uv poetry Python packaging`, not `poetry`. + +**rdt-cli** (when best method is rdt-cli): ```bash rdt search "" -s relevance -t month -n 10 --compact --json rdt search "" -r -s top -t year -n 10 --compact --json rdt read -n 20 --json ``` -**Target subreddits:** r/ExperiencedDevs, r/ClaudeAI, r/ClaudeCode, r/LocalLLaMA, r/MachineLearning, r/devops, r/commandline, r/neovim, r/Python, r/mcp, r/cybersecurity +On VPS/headless hosts, `rdt search` may return Reddit `forbidden` without browser cookies; do not copy cookie secrets into chat. -Pick 2-3 relevant subreddits. Avoid broad Reddit for ambiguous terms; it will chase engagement from irrelevant communities. Add context keywords, e.g. `uv poetry Python packaging`, not `poetry`. +**pullpush** (fallback for historical posts or VPS): `https://api.pullpush.io/reddit/search/submission/?q=&size=5&sort=desc&sort_type=score` -Raw Reddit `.json` endpoints often 403 and should be treated as a fallback only. On VPS/headless hosts, `rdt search` may return Reddit `forbidden` without browser cookies; do not copy cookie secrets into chat. For historical posts or VPS fallback, use Pullpush API (`https://api.pullpush.io/reddit/search/submission/?q=&size=5&sort=desc&sort_type=score`). +**websearch** (fallback): `site:reddit.com ` via WebSearch. ### 5. X.com -Use `x-cli` for X.com searches. Check auth with `x-cli auth status`. +Use the method shown by `dotagents sources x.com`. Treat X as commentary unless the author is primary to the topic. -**Power users:** @karpathy, @fchollet, @hardmaru, @thorstenball, @thdxr, @steipete, @banteg +**Power users:** @karpathy, @fcholet, @hardmaru, @thorstenball, @thdxr, @steipete, @banteg +**x-cli** (when best method is x-cli): ```bash x-cli search "(from:karpathy OR from:fchollet) " --type latest --count 10 --json x-cli search " (recommended OR \"game changer\")" --type top --count 10 --json ``` -Fallback: `site:x.com ` via WebSearch if x-cli auth is broken. Treat X as commentary unless the author is primary to the topic. +**websearch** (fallback): `site:x.com ` via WebSearch. ### 6. Discord -Discord remains opt-in because it uses user-token auth and may carry account-risk. Search Discord only when the topic is relevant to known communities (ML, LLMs, Claude, agents, evals, fine-tuning, etc). Skip for generic/unrelated topics. +Discord is disabled by default (ToS risk). If `dotagents sources` shows it disabled, skip entirely. When enabled, search only for topics relevant to known communities (ML, LLMs, Claude, agents, evals, fine-tuning). Skip for generic/unrelated topics. -**Preferred CLI for repeated/community monitoring**: `discord-cli` (`uv tool install kabi-discord-cli`) can sync accessible Discord channels into local SQLite, then search/export them with structured YAML/JSON. Use it only for accounts the user controls. Do not ask the user to paste raw Discord tokens into chat logs. +**discord-cli** (when enabled and available): can sync accessible Discord channels into local SQLite, then search/export with structured YAML/JSON. Do not ask the user to paste raw Discord tokens into chat logs. ```bash discord status --yaml From 1b9cf62b9bf94ec5f4e6e8ce01cd223e5fd8d9ef Mon Sep 17 00:00:00 2001 From: Kirill Korikov <11762090+yourconscience@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:44:52 +0200 Subject: [PATCH 3/6] fix bot review: native Go checks, priority logic, x-api-v2 demotion, typo, render --- cmd/dotagents/sources.go | 51 +++++++++++++------ plugins/dotagents/skills/tech-search/SKILL.md | 26 ++++++---- skills/tech-search/SKILL.md | 2 +- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/cmd/dotagents/sources.go b/cmd/dotagents/sources.go index 118b298..60b67aa 100644 --- a/cmd/dotagents/sources.go +++ b/cmd/dotagents/sources.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "os/exec" "path/filepath" "strings" @@ -60,8 +61,8 @@ var sourceRegistry = []sourceDef{ { Name: "x.com", Desc: "X.com / Twitter", DefaultOn: true, ToSRisk: "high", Methods: []sourceMethodDef{ - {Name: "x-api-v2", Type: methodAPI, Priority: 0, Check: "test -s ~/.x-api/credentials.json", Auth: "~/.x-api/credentials.json", Setup: "register OAuth consumer app at developer.x.com"}, {Name: "x-cli", Type: methodCLI, Priority: 1, Detect: "x-cli", Auth: "~/.x-cli/credentials.json", Setup: "x-cli auth login"}, + {Name: "x-api-v2", Type: methodAPI, Priority: 2, Check: "test -s ~/.x-api/credentials.json", Auth: "~/.x-api/credentials.json", Setup: "register OAuth consumer app at developer.x.com"}, {Name: "websearch", Type: methodFallback, Priority: 99}, }, }, @@ -178,13 +179,43 @@ func checkMethodAvailability(m sourceMethodDef, cfg config, home string) (bool, func runShellCheck(cmd string, home string) (bool, string) { cmd = strings.ReplaceAll(cmd, "~", home) + parts := strings.Fields(cmd) + if len(parts) >= 3 && parts[0] == "test" { + op := parts[1] + arg := strings.Trim(parts[2], "\"'") + switch op { + case "-s": + info, err := os.Stat(arg) + if err != nil { + return false, err.Error() + } + if info.Size() == 0 { + return false, "file is empty" + } + return true, "" + case "-d": + info, err := os.Stat(arg) + if err != nil { + return false, err.Error() + } + if !info.IsDir() { + return false, "not a directory" + } + return true, "" + case "-n": + if strings.HasPrefix(arg, "$") { + if os.Getenv(strings.TrimPrefix(arg, "$")) == "" { + return false, "environment variable not set" + } + return true, "" + } + } + } out, err := exec.Command("sh", "-c", cmd).CombinedOutput() if err != nil { reason := strings.TrimSpace(string(out)) if reason == "" { - if cmd != "" { - reason = "check failed" - } + reason = "check failed" } return false, reason } @@ -233,22 +264,12 @@ func resolveSourceStatus(cfg config, home string) []sourceStatus { if avail && m.Name == preferred { ss.Best = m.Name bestPriority = -1 - } else if avail && m.Priority < bestPriority && ss.Best == "" { + } else if avail && m.Priority < bestPriority { ss.Best = m.Name bestPriority = m.Priority } } - // If preferred was set but not available, still pick best available - if ss.Best == "" { - for _, ms := range ss.Methods { - if ms.Available && ms.Priority < bestPriority { - ss.Best = ms.Name - bestPriority = ms.Priority - } - } - } - results = append(results, ss) } return results diff --git a/plugins/dotagents/skills/tech-search/SKILL.md b/plugins/dotagents/skills/tech-search/SKILL.md index 4428cde..6024bd6 100644 --- a/plugins/dotagents/skills/tech-search/SKILL.md +++ b/plugins/dotagents/skills/tech-search/SKILL.md @@ -31,7 +31,9 @@ The guiding rule: broad web finds the map; source-specific searches verify the t ## Sources -Search sources in parallel when practical, but do not force every source. Skipped sources are fine when they are irrelevant, unauthenticated, or low-signal. +Before searching, run `dotagents sources --compact` to discover available methods. Use the best available method for each source; fall back to the next if it fails at runtime. + +Search sources in parallel when practical, but do not force every source. Skipped sources are fine when they are irrelevant, unauthenticated, or low-signal. If a source shows as disabled in `dotagents sources`, do not attempt it. Reference: `references/reddit-discord-cli-eval.md` records the repo evaluation behind the `rdt-cli` and `discord-cli` recommendations. @@ -86,38 +88,44 @@ Read threads at `https://news.ycombinator.com/item?id=` and cite the H ### 4. Reddit -Preferred path when installed: +Use the method shown by `dotagents sources reddit`. + +**Target subreddits:** r/ExperiencedDevs, r/ClaudeAI, r/ClaudeCode, r/LocalLLaMA, r/MachineLearning, r/devops, r/commandline, r/neovim, r/Python, r/mcp, r/cybersecurity +Pick 2-3 relevant subreddits. Avoid broad Reddit for ambiguous terms; it will chase engagement from irrelevant communities. Add context keywords, e.g. `uv poetry Python packaging`, not `poetry`. + +**rdt-cli** (when best method is rdt-cli): ```bash rdt search "" -s relevance -t month -n 10 --compact --json rdt search "" -r -s top -t year -n 10 --compact --json rdt read -n 20 --json ``` -**Target subreddits:** r/ExperiencedDevs, r/ClaudeAI, r/ClaudeCode, r/LocalLLaMA, r/MachineLearning, r/devops, r/commandline, r/neovim, r/Python, r/mcp, r/cybersecurity +On VPS/headless hosts, `rdt search` may return Reddit `forbidden` without browser cookies; do not copy cookie secrets into chat. -Pick 2-3 relevant subreddits. Avoid broad Reddit for ambiguous terms; it will chase engagement from irrelevant communities. Add context keywords, e.g. `uv poetry Python packaging`, not `poetry`. +**pullpush** (fallback for historical posts or VPS): `https://api.pullpush.io/reddit/search/submission/?q=&size=5&sort=desc&sort_type=score` -Raw Reddit `.json` endpoints often 403 and should be treated as a fallback only. On VPS/headless hosts, `rdt search` may return Reddit `forbidden` without browser cookies; do not copy cookie secrets into chat. For historical posts or VPS fallback, use Pullpush API (`https://api.pullpush.io/reddit/search/submission/?q=&size=5&sort=desc&sort_type=score`). +**websearch** (fallback): `site:reddit.com ` via WebSearch. ### 5. X.com -Use `x-cli` for X.com searches. Check auth with `x-cli auth status`. +Use the method shown by `dotagents sources x.com`. Treat X as commentary unless the author is primary to the topic. **Power users:** @karpathy, @fchollet, @hardmaru, @thorstenball, @thdxr, @steipete, @banteg +**x-cli** (when best method is x-cli): ```bash x-cli search "(from:karpathy OR from:fchollet) " --type latest --count 10 --json x-cli search " (recommended OR \"game changer\")" --type top --count 10 --json ``` -Fallback: `site:x.com ` via WebSearch if x-cli auth is broken. Treat X as commentary unless the author is primary to the topic. +**websearch** (fallback): `site:x.com ` via WebSearch. ### 6. Discord -Discord remains opt-in because it uses user-token auth and may carry account-risk. Search Discord only when the topic is relevant to known communities (ML, LLMs, Claude, agents, evals, fine-tuning, etc). Skip for generic/unrelated topics. +Discord is disabled by default (ToS risk). If `dotagents sources` shows it disabled, skip entirely. When enabled, search only for topics relevant to known communities (ML, LLMs, Claude, agents, evals, fine-tuning). Skip for generic/unrelated topics. -**Preferred CLI for repeated/community monitoring**: `discord-cli` (`uv tool install kabi-discord-cli`) can sync accessible Discord channels into local SQLite, then search/export them with structured YAML/JSON. Use it only for accounts the user controls. Do not ask the user to paste raw Discord tokens into chat logs. +**discord-cli** (when enabled and available): can sync accessible Discord channels into local SQLite, then search/export with structured YAML/JSON. Do not ask the user to paste raw Discord tokens into chat logs. ```bash discord status --yaml diff --git a/skills/tech-search/SKILL.md b/skills/tech-search/SKILL.md index 823009f..6024bd6 100644 --- a/skills/tech-search/SKILL.md +++ b/skills/tech-search/SKILL.md @@ -111,7 +111,7 @@ On VPS/headless hosts, `rdt search` may return Reddit `forbidden` without browse Use the method shown by `dotagents sources x.com`. Treat X as commentary unless the author is primary to the topic. -**Power users:** @karpathy, @fcholet, @hardmaru, @thorstenball, @thdxr, @steipete, @banteg +**Power users:** @karpathy, @fchollet, @hardmaru, @thorstenball, @thdxr, @steipete, @banteg **x-cli** (when best method is x-cli): ```bash From cf40d83679061f54c74cb0f28f4cd326bf46fb0e Mon Sep 17 00:00:00 2001 From: Kirill Korikov <11762090+yourconscience@users.noreply.github.com> Date: Mon, 15 Jun 2026 14:58:19 +0200 Subject: [PATCH 4/6] simplify README, code, and landscape table for public release --- README.md | 286 +++++------------------- cmd/dotagents/agents.go | 7 +- cmd/dotagents/hooks.go | 89 ++------ cmd/dotagents/inspect.go | 4 +- cmd/dotagents/mcp.go | 9 +- docs/site/index.html | 25 +-- plugins/dotagents/skills/spawn/SKILL.md | 3 + skills/cmux/SKILL.md | 2 - skills/spawn/SKILL.md | 24 -- skills/tmux/SKILL.md | 4 +- 10 files changed, 88 insertions(+), 365 deletions(-) diff --git a/README.md b/README.md index 27e5957..3a05681 100644 --- a/README.md +++ b/README.md @@ -1,277 +1,93 @@ # dotagents -Cross-agent sync CLI for managing shared skills, agent roles, and MCP servers across the primary coding-agent stack: Claude Code, Codex, Factory Droid, Hermes, and Pi/OMP. Amp, OpenClaw, OpenCode, and similar non-primary harnesses are compatibility-only unless explicitly configured. - -This repo is the canonical `~/.agents` layer. It detects installed agent platforms, syncs shared skills and MCP entries to each platform's native format, validates drift, and self-tests. +Go CLI that keeps skills, MCP servers, hooks, and agent roles in one `~/.agents` repo and syncs them into Claude Code, Codex, Droid, Hermes, and Pi. ![dotagents harness map](./docs/harness-map.png) -[Open the full harness map.](./docs/harness-map.html) - -## Agent instructions - -[`AGENTS.md`](./AGENTS.md) is the canonical instruction file. [`CLAUDE.md`](./CLAUDE.md) is only a compatibility shim for agents that look for Claude-style project memory. - ## Install -Two ways to consume this repo - pick exactly one per machine and harness: - -| You want | Do this | -|---|---| -| Just the skills in Claude Code | `/plugin marketplace add yourconscience/dotagents` then `/plugin install dotagents@yourconscience` | -| Just the skills in Codex | `codex plugin marketplace add https://github.com/yourconscience/dotagents` then `codex plugin add dotagents@yourconscience` | -| The full managed setup (skills, roles, MCP, hooks) on any supported harness - Claude Code, Codex, Hermes, Factory Droid, Pi/OMP | Install the `dotagents` CLI (below), clone the repo, run `dotagents setup` | - -Plugins snapshot the repo at install time and update through each harness's plugin update flow. `dotagents sync` keeps live symlinks instead. Do not combine both on the same harness or skills appear twice; see "Installing this repo as a plugin" for details and how `delivery:` arbitrates this for Claude Code. +**Plugin only** (Claude Code or Codex, no CLI needed): -### CLI install - -Prebuilt binaries for macOS and Linux (amd64/arm64) are attached to [GitHub Releases](https://github.com/yourconscience/dotagents/releases). With Go installed: - -```bash -go install github.com/yourconscience/dotagents/cmd/dotagents@latest ``` - -Or from a clone: - -```bash -go install ./cmd/dotagents +/plugin marketplace add yourconscience/dotagents +/plugin install dotagents@yourconscience ``` -Ensure the Go install directory is on `PATH`. If `go env GOBIN` is non-empty, add that directory; otherwise add `$(go env GOPATH)/bin`. - -After that, run first-time machine setup (creates `~/.agents`, patches detected agent configs, syncs), or inspect state directly: +**Full setup** (all harnesses): ```bash +go install github.com/yourconscience/dotagents/cmd/dotagents@latest +git clone https://github.com/yourconscience/dotagents ~/.agents dotagents setup -dotagents status -dotagents deps check ``` -Releases are cut by pushing a `v*` tag; CI runs GoReleaser, which builds the archives and publishes the GitHub Release. - -## Alternatives - -How dotagents compares to other cross-agent config sync tools: - -| | dotagents | [skillshare](https://github.com/runkids/skillshare) | [vsync](https://github.com/nicepkg/vsync) | [agents-cli](https://github.com/amtiYo/agents) | -|---|---|---|---|---| -| Skills sync | yes (symlinks + config-driven dirs) | yes | yes | yes | -| MCP sync | yes | no | yes | yes | -| Hooks sync | yes (Claude Code, Codex, Hermes, Droid) | no | no | no | -| Native subagent roles | yes (Claude Code, Codex, Droid) | agents as files | yes | no | -| Plugin catalog | yes (first-party `dotagents.yaml` entries) | no | no | no | -| External skill pinning | yes (`dotagents.lock`) | version tracking | no | no | -| Skill security audit | yes (`dotagents doctor`) | yes | no | no | -| Local private overlay | yes (`dotagents.local.yaml`) | no | no | no | -| Target agents | Claude Code, Codex, Amp, Hermes, Factory Droid, Pi/OpenClaw | Claude Code, Codex, Cursor, Gemini, 60+ | Claude Code, Cursor, OpenCode, Codex | Codex, Claude Code, Gemini CLI, Cursor, Copilot, others | -| Language | Go | Go | TypeScript | TypeScript | - -dotagents focuses on the post-IDE agent stack (Hermes, Amp, Droid, OpenClaw/Pi alongside Claude Code and Codex) and on syncing the full surface - skills, MCP, hooks, roles, plugins, root instructions - from one canonical `~/.agents` layer. - -## Agents - -Reusable agent role definitions for agent-native subagents. Canonical roles live in `agents/*.yaml`; `dotagents sync` renders them to each configured native format: +Prebuilt binaries for macOS and Linux (amd64/arm64) on [Releases](https://github.com/yourconscience/dotagents/releases). -- Claude Code: `~/.claude/agents/.md` -- Codex: `~/.codex/agents/.toml` -- Factory Droid: `~/.factory/droids/.md` +## What it does -- `architect` - designs system architecture, telemetry schemas, and technical plans. Sonnet, read + write. -- `builder` - implements code changes following specs or architect designs. Sonnet, read + write. -- `researcher` - investigates codebases, APIs, repos, and web sources. Sonnet, read + write + web. -- `reviewer` - reviews code against specs, finds bugs and security issues. Sonnet, read-only. - -Reference these from TeamCreate teammates, Claude Code subagent types, or Codex native subagent roles. See `skills/spawn/SKILL.md` for usage patterns. +| Harness | Skills | Roles | MCP | Hooks | Status | +|---|---|---|---|---|---| +| Claude Code | yes | yes | yes | yes | managed | +| Codex | yes | yes | yes | yes | managed | +| Factory Droid | yes | yes | yes | yes | managed | +| Hermes | yes | -- | yes | yes | managed | +| Pi/OMP | yes | -- | yes | -- | managed | +| Amp | -- | -- | -- | -- | compat | +| OpenCode | -- | -- | -- | -- | compat | +| OpenClaw | -- | -- | -- | -- | compat | ## Skills -Grouped by category, with the execution surface each skill drives. See `docs/reports/skill-categorization.md` for the full analysis. - -Orchestration and delegation: - -- `spawn` - decide how to delegate work to subagents or teams across Droid, Claude Code, Hermes, Codex, and multiplexer surfaces (tmux/cmux/Ghostex). -- `cmux` - control cmux workspaces, panes, terminal/browser surfaces, markdown viewers, and visible agent workspaces. Surface: `cmux` CLI. Under replacement trial by Ghostex; kept until the trial concludes. -- `tmux` - generic tmux reference for sessions, windows, panes, screen capture, and input. Surface: `tmux` CLI; fallback when neither cmux nor Ghostex manages the terminal. -- `remote-access` - search local Droid/Codex sessions and send scoped continuation instructions through the Mac bridge from mobile. Surface: ssh/Tailscale + takopi. - -Ghostex ships its own bundled skills (`ghostex-agent-orchestration`, `ghostex-browser-use`, etc.); dotagents does not duplicate them - skills detect the surface and defer to the bundled reference when running inside Ghostex. - -Research: - -- `repo-eval` - find, triage, and deep-evaluate GitHub repos for a given need. -- `tech-search` - gather high-signal opinions from tech communities and blogs on a topic. -- `x-sim` - offline X audience simulation for draft tweets and handle positioning. +16 skills ship with this repo: -Writing and process: +`spawn` `cmux` `tmux` `remote-access` `repo-eval` `tech-search` `x-sim` `grill-me` `humanizer` `spec` `jobs` `pr-triage` `gws` `tg` `x-cli` `dotagents` -- `grill-me` - pressure-test a plan one question at a time until scope and decisions are concrete. -- `humanizer` - final-pass rewriting for concise writing that keeps the user's voice. -- `spec` - produce a small `SPEC.md` for complex or ambiguous work before implementation. -- `jobs` - track job search pipeline, analyze fit for postings, generate interview quizzes, grade answers. -- `pr-triage` - inspect PR failures and unresolved review threads, then drive a single fix-commit-push loop. +A skill is a `SKILL.md` in a directory under `skills/`. Add one, run `dotagents sync`, it shows up everywhere. -Integrations (CLI wrappers): +## Agent roles -- `gws` - Google Workspace workflows. On Hermes, prefer the bundled native `google-workspace` skill; this repo's `skills/gws` remains the shared source for Claude Code/Codex and CLI helpers. -- `tg` - read Telegram chats, search messages, and list dialogs through the read-only `tg` CLI. -- `x-cli` - unofficial CLI for `x` tooling. +Four roles defined in `agents/*.yaml`, rendered to each harness's native format: -Infrastructure: - -- `dotagents` - inspect and sync the repo-owned skill links across supported coding agents. - -## Installing these skills without dotagents - -The repo doubles as a [Claude Code plugin marketplace](https://code.claude.com/docs/en/discover-plugins): `.claude-plugin/marketplace.json` exposes the portable skills (`tech-search`, `grill-me`, `humanizer`, `repo-eval`, `spec`, `pr-triage`, `tmux`) as single-skill plugins. - -```text -/plugin marketplace add yourconscience/dotagents -/plugin install tech-search@yourconscience -``` +`architect` `builder` `researcher` `reviewer` -For any agent managed by dotagents, consume the same skills as an external source with a `skills` allowlist: +## Key commands -```yaml -external_skills: - - url: https://github.com/yourconscience/dotagents - skill_dir: skills - branch: main - skills: [tech-search, grill-me, humanizer, repo-eval, spec, pr-triage, tmux] +```bash +dotagents status # what's synced where +dotagents sync # push changes to all harnesses +dotagents doctor # validate config, check for drift +dotagents render # regenerate plugin copies and agent files +dotagents plugin add # switch Claude Code to plugin delivery +dotagents deps check # verify external tool dependencies ``` -Other sync tools that install skills from a git repo (e.g. skillshare) can point at the `skills/` directory directly. - -## External Skills +## External skills -Skills from external git repos can be synced alongside local skills. Declare sources in `dotagents.yaml` when needed: +Pull skills from other repos: ```yaml +# dotagents.yaml external_skills: - url: https://github.com/example/shared-skills skill_dir: skill branch: main - skills: [alpha, beta] # optional allowlist; omit to take every skill + skills: [alpha, beta] ``` -`dotagents sync` clones or updates each repo into `~/.agents/external//` and symlinks discovered skills into agent skill roots. `dotagents status` shows external sources with their commit hash. `dotagents doctor` validates that clones exist and contain valid skills. - -External sources are pinned in `dotagents.lock` (commit this file): the first sync records each source's commit, and later syncs keep the source at the pinned commit instead of silently tracking the branch. `dotagents external list` shows pin state; `dotagents external update [name ...]` moves sources to the latest branch head and rewrites the lock. `dotagents doctor` warns when a source is unpinned or its cache drifts from the lock, and runs a content audit over external skills that flags risky patterns (pipe-to-shell installs, base64-decode-to-shell, prompt-injection phrasing, credential paths) for human review. +Sources are pinned in `dotagents.lock`. `dotagents doctor` audits external skills for risky patterns. -## Local overlay - -`dotagents.local.yaml` next to `dotagents.yaml` (gitignored) holds personal additions that should stay out of public git: extra agents, external skill sources, MCP servers, hooks, or plugin entries. Entries merge by name (external sources by repo name); a matching name replaces the public entry wholesale, everything else is appended. - -## Plugins - -Dotagents treats third-party plugins as first-party catalog entries in `dotagents.yaml`, not as committed `.codex-plugin`, `.amp/`, or `.hermes/` runtime directories. (The repo's own self-publication manifests - `.claude-plugin/`, `.agents/plugins/marketplace.json`, and `plugins/dotagents/` - are the exception; see "Installing this repo as a plugin" below.) A plugin entry records its source format, runtime surfaces, target agents, and review notes: - -```yaml -plugins: - - name: feature-dev - enabled: false - source: claude:claude-plugins-official/feature-dev - format: claude-plugin - surfaces: [skills, agents, commands, native-plugin] - agents: [claude-code, codex, hermes, droid, pi] -``` +Private additions go in `dotagents.local.yaml` (gitignored). -Enabled plugin `skills/` surfaces are discovered from portable plugin source IDs. `codex:/` resolves under `DOTAGENTS_CODEX_PLUGIN_ROOT`; `claude:/` resolves under `DOTAGENTS_CLAUDE_PLUGIN_ROOT`. For Codex, Factory Droid, and Pi/OMP, `dotagents sync` manages those plugin skills as symlinks in the native skill roots. Claude Code uses either symlink sync or this repo's native Claude plugin based on `agents[].delivery`. For Hermes, `dotagents setup` adds the plugin `skills/` directories to `skills.external_dirs`. Amp remains compatibility-only and must be targeted explicitly in a local config if needed. +## Landscape -`dotagents status` prints each plugin's compatibility across known harness descriptors; non-primary harnesses show as `not targeted` unless explicitly configured. `dotagents doctor` validates the catalog and warns when an enabled plugin targets an agent that has no supported surface for it. - -Compatibility model: - -- `skills` work through managed symlinks for Claude Code/Codex/Factory Droid/Pi and `skills.external_dirs` for Hermes. -- `mcp` works through managed MCP entries. -- `agents` currently renders to Claude Code, Codex, and Droid. -- `hooks` are supported only where dotagents has verified hook config support. -- `native-plugin` is host-specific: `.codex-plugin` stays Codex-native and `.claude-plugin` stays Claude-native. -- `commands` are currently Claude-native unless re-modeled as skills, hooks, MCP, or a repo-owned CLI. - -### Installing this repo as a plugin - -The repo self-publishes as a plugin for the two harnesses that have plugin systems: - -- Claude Code, via `.claude-plugin/{plugin,marketplace}.json` at the repo root. -- Codex, via `.agents/plugins/marketplace.json` and the `plugins/dotagents/` plugin directory. - -Hermes, Factory Droid, and Pi/OMP have no plugin system; they consume skills through `dotagents setup` / `dotagents sync`. - -For Claude Code: - -``` -/plugin marketplace add yourconscience/dotagents -/plugin install dotagents@yourconscience -``` - -The repo is private, so `marketplace add` requires working GitHub git auth (ssh or `gh auth`) on the machine. - -The plugin ships every skill under `skills/` (namespaced as `/dotagents:`) plus the four subagent roles. The roles are rendered from `agents/*.yaml` into `agents/*.md` (beside the sources) by `dotagents render`; Claude Code auto-discovers them from the top-level `agents/` directory. `dotagents doctor` and CI tests fail (`plugin agents` check) when the rendered copies drift from the YAML. - -Plugin skills are byte-identical to the symlink-synced ones - same directories, same repo. A machine should use exactly one Claude Code delivery channel: - -```yaml -agents: - - name: claude-code - delivery: sync # default: dotagents sync manages ~/.claude/skills and ~/.claude/agents -``` - -Use the CLI wrapper to switch Claude Code to plugin delivery: - -```bash -dotagents plugin add -``` - -That command runs the Claude plugin install flow, sets `delivery: plugin` for `claude-code`, and prunes dotagents-managed symlinks and generated Claude agent files so skills do not appear twice (`/tg` and `/dotagents:tg`). `dotagents doctor` includes a `claude delivery` check that fails when `delivery: plugin` is set but `dotagents@yourconscience` is not installed, or when plugin delivery still has managed sync artifacts. - -Use this to return Claude Code to symlink sync: - -```bash -dotagents plugin remove -``` - -That uninstalls the Claude plugin, removes the marketplace entry, sets `delivery: sync`, and runs `dotagents sync --agents=claude-code`. Plugin installs snapshot the repo at install time; consumers pick up new skills with `/plugin update`, unlike the always-live symlinks. The plugin manifest intentionally omits a fixed `version` so Claude Code uses the git commit SHA and every new commit can be updated. - -### Installing this repo as a Codex plugin - -For Codex: - -```bash -codex plugin marketplace add https://github.com/yourconscience/dotagents -codex plugin add dotagents@yourconscience -``` - -The repo is private, so `marketplace add` requires working GitHub git auth on the machine. - -Codex plugin installs require a *real, copied* `skills/` inside the plugin directory - symlinks are silently dropped and a plugin at the marketplace root is rejected (verified against `codex 0.136.0`). So `plugins/dotagents/skills/` is a rendered copy of the canonical `skills/` tree (tracked files only, a few hundred KB), regenerated by `dotagents render` alongside the Claude agent renders. `dotagents doctor` (`codex plugin` check) and CI fail when the copy drifts, which keeps the single-source-of-truth guarantee. - -There is no `delivery:` switch for Codex: a machine managed by dotagents keeps the live symlink sync and should not install the Codex plugin on top (skills would appear twice). The plugin is the install path for machines that do not run dotagents. Like the Claude plugin, the manifest intentionally omits a fixed `version` so updates never hide behind a stale version pin; to pick up new skills, run `codex plugin marketplace upgrade`, then `codex plugin remove dotagents@yourconscience` and `codex plugin add dotagents@yourconscience` (re-adding refreshes the cached copy - verified against codex 0.136.0). - -Plugins ship full skills, tools included. Skill tool commands resolve relative to the skill's own directory (the base directory the harness reports when a skill loads), never to a fixed checkout path, so bundled CLIs like `pr-triage` inspect and `x-sim` run from any install root - checkout, symlink, or plugin cache. The exceptions are the machine-management skills (`dotagents`, `cmux` hooks, memory sync) which inherently operate on a dotagents-managed machine and say so. - -`SKILL.md` directories remain the genuinely portable cross-tool convention; harnesses without a plugin system (Hermes, Droid, Pi, Amp) consume them through `dotagents sync` or by pointing at `skills/` directly. - -## Agent Integration Status - -Dotagents keeps `~/.agents` as the source of truth and adapts each agent through symlinks, targeted config patches, or generated native files. Do not commit agent-specific project runtime directories such as `.amp/` or `.hermes/` to this repo. - -| Agent | Shared skills | Native subagents | MCP sync | Hook sync | Root instructions | Integration notes | -|---|---|---|---|---|---|---| -| Claude Code | `delivery: sync` symlink mirror to `~/.claude/skills`; `delivery: plugin` via `dotagents@yourconscience` | `delivery: sync` generated to `~/.claude/agents`; `delivery: plugin` from plugin `agents/*.md` | `~/.claude/settings.json` | `~/.claude/settings.json` | `CLAUDE.md` shim points to `AGENTS.md` | Use exactly one skill/role delivery channel; MCP and supported hooks remain dotagents-managed. | -| Codex | Symlink mirror to `~/.codex/skills` | Generated to `~/.codex/agents` | `~/.codex/config.toml` | `~/.codex/hooks.json` plus `[features].hooks = true` | Reads `AGENTS.md` | Full managed mirror for skills, roles, MCP, and supported hooks. | -| Hermes | Config path to `~/.agents/skills` | Not managed | `~/.hermes/config.yaml` | `~/.hermes/config.yaml` for known lifecycle hooks | Reads configured Hermes context | Uses `skills.external_dirs`; do not mirror into `~/.hermes/skills` because bundled categories can collide. | -| Factory Droid | Symlink mirror to `~/.factory/skills` | Generated to `~/.factory/droids` | `~/.factory/mcp.json` | `~/.factory/settings.json` | `~/.factory/AGENTS.md` symlink | Full managed mirror for skills, roles, MCP, and supported hooks. | -| Pi/OMP | Symlink mirror to `~/.omp/agent/skills` | Not managed | `~/.omp/agent/mcp.json` | Not managed | Reads configured OMP context | Primary OMP target for shared skills, MCP entries, and portable plugin skill surfaces. | -| Amp | Compatibility-only via explicit local config | Not managed | Supported by CLI when locally targeted | Not managed | Not managed | Intentionally absent from canonical `dotagents.yaml`; use a gitignored local overlay for one-off migration or preservation work. | -| OpenCode | Not managed | Not managed | Not managed | Not managed | Not managed | Compatibility research only; no verified native skill/MCP surface is managed by dotagents yet. | -| OpenClaw | Not managed | Not managed | Not managed | Not managed | Not managed | Compatibility research only; needs an owner and verified config surface before dotagents treats it as managed. | - -Compatibility-only harness support may remain in the CLI for migration, hook cleanup, trailer stripping, or one-off local configs. Those harnesses are intentionally absent from the canonical `dotagents.yaml` managed target list. - - -Managed hook declarations live in `dotagents.yaml`. `dotagents sync` may patch supported hook config, but it never approves hook execution. Host-specific hook approval remains manual and lifecycle-sensitive. - -## Experimental - -`experimental/` holds evaluation notes, landscape comparisons, and alternatives explored. Not necessarily implemented or supported - more about what was tried, what the options are, and why certain choices were made. Reference material for future decisions. +| | dotagents | [gstack](https://github.com/garrytan/gstack) | [SuperClaude](https://github.com/SuperClaude-Org/SuperClaude_Framework) | [skillshare](https://github.com/runkids/skillshare) | +|---|---|---|---|---| +| Skills sync | yes | -- | -- | yes | +| MCP sync | yes | no | no | no | +| Hooks sync | yes | no | yes | no | +| Agent roles | yes | 23 built-in | 20 built-in | no | +| Security audit | yes | no | no | yes | +| Multi-harness | 5 managed + 3 compat | Claude Code | Claude Code | 60+ | +| Install | plugin or `go install` | CC plugin | pipx | `go install` | + +gstack and SuperClaude are content packs for Claude Code. skillshare syncs skills broadly but not MCP, hooks, or roles. dotagents is the full config layer across harnesses. diff --git a/cmd/dotagents/agents.go b/cmd/dotagents/agents.go index 63f739a..5160683 100644 --- a/cmd/dotagents/agents.go +++ b/cmd/dotagents/agents.go @@ -346,20 +346,17 @@ func writeTOMLString(b *strings.Builder, key string, value string) { } b.WriteString(key) b.WriteString(" = ") - b.WriteString(tomlQuote(value)) + b.WriteString(strconv.Quote(value)) b.WriteString("\n") } func writeTOMLMultiline(b *strings.Builder, key string, value string) { b.WriteString(key) b.WriteString(" = ") - b.WriteString(tomlQuote(value)) + b.WriteString(strconv.Quote(value)) b.WriteString("\n") } -func tomlQuote(value string) string { - return strconv.Quote(value) -} func codexModelFor(model string) string { switch strings.ToLower(strings.TrimSpace(model)) { diff --git a/cmd/dotagents/hooks.go b/cmd/dotagents/hooks.go index 2538826..778859d 100644 --- a/cmd/dotagents/hooks.go +++ b/cmd/dotagents/hooks.go @@ -271,83 +271,14 @@ func claudeHooksConfigPath(home string) string { } func inspectClaudeHookMap(raw map[string]interface{}, hook hookConfig) string { - hooksRoot, ok := raw["hooks"].(map[string]interface{}) - if !ok { - return stateMissing - } - groups, ok := hooksRoot[hook.Event].([]interface{}) - if !ok { - return stateMissing - } - found := false - for _, groupRaw := range groups { - group, ok := groupRaw.(map[string]interface{}) - if !ok { - continue - } - items, ok := group["hooks"].([]interface{}) - if !ok { - continue - } - for _, itemRaw := range items { - item, ok := itemRaw.(map[string]interface{}) - if !ok || !hookCommandMatches(item["command"], hook.Command) { - continue - } - found = true - if hookTimeoutMatches(item, hook.Timeout) { - return stateSynced - } - } - } - if found { - return stateDrifted - } - return stateMissing + return inspectGroupedHookMap(raw, hook, false) } -func upsertClaudeHookMap(raw map[string]interface{}, hook hookConfig) { - hooksRoot, _ := raw["hooks"].(map[string]interface{}) - if hooksRoot == nil { - hooksRoot = map[string]interface{}{} - raw["hooks"] = hooksRoot - } - groups, _ := hooksRoot[hook.Event].([]interface{}) - if len(groups) == 0 { - groups = []interface{}{map[string]interface{}{"hooks": []interface{}{}}} - } - targetIndex := 0 - for i, groupRaw := range groups { - group, ok := groupRaw.(map[string]interface{}) - if !ok { - continue - } - items, _ := group["hooks"].([]interface{}) - if containsHookCommand(items, hook.Command) { - targetIndex = i - break - } - } - for i, groupRaw := range groups { - group, _ := groupRaw.(map[string]interface{}) - if group == nil { - if i != targetIndex { - continue - } - group = map[string]interface{}{} - groups[i] = group - } - items, _ := group["hooks"].([]interface{}) - items = removeHookCommand(items, hook.Command) - if i == targetIndex { - items = append(items, renderHookEntry(hook)) - } - group["hooks"] = items - } - hooksRoot[hook.Event] = groups +func inspectNestedJSONHookMap(raw map[string]interface{}, hook hookConfig) string { + return inspectGroupedHookMap(raw, hook, true) } -func inspectNestedJSONHookMap(raw map[string]interface{}, hook hookConfig) string { +func inspectGroupedHookMap(raw map[string]interface{}, hook hookConfig, requireType bool) string { hooksRoot, ok := raw["hooks"].(map[string]interface{}) if !ok { return stateMissing @@ -372,7 +303,7 @@ func inspectNestedJSONHookMap(raw map[string]interface{}, hook hookConfig) strin continue } found = true - if item["type"] == "command" && hookTimeoutMatches(item, hook.Timeout) { + if (!requireType || item["type"] == "command") && hookTimeoutMatches(item, hook.Timeout) { return stateSynced } } @@ -383,7 +314,15 @@ func inspectNestedJSONHookMap(raw map[string]interface{}, hook hookConfig) strin return stateMissing } +func upsertClaudeHookMap(raw map[string]interface{}, hook hookConfig) { + upsertGroupedHookMap(raw, hook, renderHookEntry) +} + func upsertNestedJSONHookMap(raw map[string]interface{}, hook hookConfig) { + upsertGroupedHookMap(raw, hook, renderNestedHookEntry) +} + +func upsertGroupedHookMap(raw map[string]interface{}, hook hookConfig, render func(hookConfig) map[string]interface{}) { hooksRoot, _ := raw["hooks"].(map[string]interface{}) if hooksRoot == nil { hooksRoot = map[string]interface{}{} @@ -417,7 +356,7 @@ func upsertNestedJSONHookMap(raw map[string]interface{}, hook hookConfig) { items, _ := group["hooks"].([]interface{}) items = removeHookCommand(items, hook.Command) if i == targetIndex { - items = append(items, renderNestedHookEntry(hook)) + items = append(items, render(hook)) } group["hooks"] = items } diff --git a/cmd/dotagents/inspect.go b/cmd/dotagents/inspect.go index b3a3efd..fefd3ac 100644 --- a/cmd/dotagents/inspect.go +++ b/cmd/dotagents/inspect.go @@ -335,7 +335,7 @@ func inspectAmpAgent(agent agentConfig, expected map[string]string, cfg config, } sortReportLists(&report) - report.Synced = len(report.Missing) == 0 && len(report.Drifted) == 0 && len(report.Conflicts) == 0 && len(report.StaleManaged) == 0 && len(report.MissingMCP) == 0 && len(report.DriftedMCP) == 0 && len(report.MissingHook) == 0 && len(report.DriftedHook) == 0 + report.Synced = isReportSynced(report) return report, nil } @@ -408,7 +408,7 @@ func inspectHermesAgent(agent agentConfig, expected map[string]string, agentsSki } sortReportLists(&report) - report.Synced = len(report.Missing) == 0 && len(report.Drifted) == 0 && len(report.Conflicts) == 0 && len(report.StaleManaged) == 0 && len(report.MissingMCP) == 0 && len(report.DriftedMCP) == 0 && len(report.MissingHook) == 0 && len(report.DriftedHook) == 0 + report.Synced = isReportSynced(report) return report, nil } diff --git a/cmd/dotagents/mcp.go b/cmd/dotagents/mcp.go index 5c180df..d53c46b 100644 --- a/cmd/dotagents/mcp.go +++ b/cmd/dotagents/mcp.go @@ -103,7 +103,7 @@ func applyAgentMCPSync(reports []agentReport, cfg config, home string) error { } func inspectMCPServer(agentName string, server mcpServerConfig, home string) (string, error) { - target, err := mcpTargetForAgent(agentName) + target, err := mcpTargetForHarness(agentName) if err != nil { return stateMissing, err } @@ -111,7 +111,7 @@ func inspectMCPServer(agentName string, server mcpServerConfig, home string) (st } func patchMCPServer(agentName string, server mcpServerConfig, home string) error { - target, err := mcpTargetForAgent(agentName) + target, err := mcpTargetForHarness(agentName) if err != nil { return err } @@ -119,16 +119,13 @@ func patchMCPServer(agentName string, server mcpServerConfig, home string) error } func readNativeMCPServer(agentName string, name string, home string) (mcpServerConfig, error) { - target, err := mcpTargetForAgent(agentName) + target, err := mcpTargetForHarness(agentName) if err != nil { return mcpServerConfig{}, err } return target.read(target, name, home) } -func mcpTargetForAgent(agentName string) (mcpTarget, error) { - return mcpTargetForHarness(agentName) -} func inspectJSONMCPServer(target mcpTarget, server mcpServerConfig, home string) (string, error) { configPath := target.configPath(home) diff --git a/docs/site/index.html b/docs/site/index.html index 5a7c041..fc20c22 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -395,25 +395,24 @@

Sync surface by harness

- +
-

Alternatives

-

Other tools solve parts of this. dotagents covers the full surface across the post-IDE agent stack.

+

Landscape

+

Content packs target one agent. Skills syncing covers breadth but not depth. dotagents syncs the full config surface across harnesses.

-

Comparison

+

How it compares

- + - - - - - - - - + + + + + + +
dotagentsskillsharevsyncagents-cli
dotagentsgstackSuperClaudeskillshare
Skills syncyesyesyesyes
MCP syncyesnoyesyes
Hooks syncyesnonono
Agent rolesyesnoyesno
Plugin catalogyesnonono
Skill pinningyesyesnono
Security audityesyesnono
Private overlayyesnonono
Skills syncyes----yes
MCP syncyesnonono
Hooks syncyesnoyesno
Agent rolesyes23 built-in20 built-inno
Security audityesnonoyes
Multi-harness5 managed + 3 compatClaude CodeClaude Code60+
Installplugin or CLICC pluginpipxgo install
diff --git a/plugins/dotagents/skills/spawn/SKILL.md b/plugins/dotagents/skills/spawn/SKILL.md index 7ed9d28..9ec6a84 100644 --- a/plugins/dotagents/skills/spawn/SKILL.md +++ b/plugins/dotagents/skills/spawn/SKILL.md @@ -218,3 +218,6 @@ Enable `multi_agent = true` in `~/.codex/config.toml`. Codex spawns child agents - No shared task list across sessions - No bi-directional messaging between independent sessions - Coordination is single-session only + +--- + diff --git a/skills/cmux/SKILL.md b/skills/cmux/SKILL.md index 86a755a..464587e 100644 --- a/skills/cmux/SKILL.md +++ b/skills/cmux/SKILL.md @@ -7,8 +7,6 @@ description: Control cmux workspaces, panes, terminal and browser surfaces, mark Use this skill when the current terminal is managed by cmux, or when the task needs cmux browser surfaces, workspace layout, markdown viewing, or cmux-specific agent launchers. -Status: under replacement trial by Ghostex (since 2026-06-12). If the terminal is managed by Ghostex instead (`GHOSTEX_*` env vars set, no `CMUX_*` env), use Ghostex's bundled `ghostex-agent-orchestration` and `ghostex-browser-use` skills. This skill stays until the trial concludes. - ## Detection ```bash diff --git a/skills/spawn/SKILL.md b/skills/spawn/SKILL.md index 2a711bf..9ec6a84 100644 --- a/skills/spawn/SKILL.md +++ b/skills/spawn/SKILL.md @@ -221,27 +221,3 @@ Enable `multi_agent = true` in `~/.codex/config.toml`. Codex spawns child agents --- -## Ghostex execution - -Use when the terminal is managed by Ghostex (detect: `GHOSTEX_*` env vars set, e.g. `GHOSTEX_ZMX_BIN`; `gx state` succeeding alone only proves the server is running somewhere). The bundled `ghostex-agent-orchestration` skill in the app is the authoritative command reference; this section only covers the spawn loop. - -```bash -gx sessions --json # inspect projects/sessions -gx create-session "task title" --project-id # registers the session -gx send-message "self-contained prompt" # send prompt + Enter -gx read-text --lines 80 --json # read output -gx kill --json # clean up -``` - -Known gaps (gxserver 0.1.0): - -- CLI-created sessions lazy-start: the PTY does not exist until a client attaches. Headless workaround: pre-start the daemon with the bundled zmx (not on PATH) before sending input: - - ```bash - zmx="/Applications/Ghostex.app/Contents/Resources/Web/bin/zmx" - "$zmx" run "" -d --initial-command /bin/zsh -lic 'exec /bin/zsh -li' - ``` - -- `gx send-key ctrl-c`, `gx wait-for`, `gx focus` are broken headlessly; interrupt via `printf '\x03' | "$zmx" send ` and wait by polling `read-text`. -- Agent buttons (`create-agent`/`run-agent`) do not auto-dispatch their startup command without the UI; send it manually after the zmx pre-start. -- No cmux-style markdown viewer: show reports by opening the .md in the embedded Code editor. `gx edit ` is the intended path but currently fails (missing `openPaths` gxserver endpoint); until it lands, ask the user to open the file in the Ghostex editor, or print the path. Upstream CLI TDZ-crash fix: maddada/Ghostex#21. diff --git a/skills/tmux/SKILL.md b/skills/tmux/SKILL.md index a241ff0..61f1ade 100644 --- a/skills/tmux/SKILL.md +++ b/skills/tmux/SKILL.md @@ -5,20 +5,18 @@ description: Generic tmux reference for sessions, windows, panes, screen capture # tmux -Use this skill for terminal multiplexing when the current terminal is not managed by cmux or Ghostex. +Use this skill for terminal multiplexing when the current terminal is not managed by cmux. ## Detection ```bash env | grep '^CMUX_' # if present, switch to the cmux skill -env | grep '^GHOSTEX_' # if present, the terminal is Ghostex-managed: use its bundled ghostex-agent-orchestration skill test -n "$TMUX" && echo tmux command -v tmux ``` Routing: - If `CMUX_WORKSPACE_ID` and `CMUX_SURFACE_ID` are set, use the `cmux` skill. -- Else if `GHOSTEX_*` env vars are set (e.g. `GHOSTEX_ZMX_BIN`), the terminal is Ghostex-managed: use the bundled `ghostex-agent-orchestration` skill (zmx underneath). A merely-running Ghostex app elsewhere does not count - `gx state` succeeding is server state, not terminal ownership. - Else if `TMUX` is set, target the current tmux server/session. - Else if `tmux` exists, create or attach a tmux session before relying on pane operations. - Else, no supported queryable pane manager is active. From 0ee0c5f03512ae51d4d570d99db2bac207bdfc31 Mon Sep 17 00:00:00 2001 From: Kirill Korikov <11762090+yourconscience@users.noreply.github.com> Date: Mon, 15 Jun 2026 15:00:24 +0200 Subject: [PATCH 5/6] fix gofmt formatting --- cmd/dotagents/agents.go | 1 - cmd/dotagents/mcp.go | 1 - 2 files changed, 2 deletions(-) diff --git a/cmd/dotagents/agents.go b/cmd/dotagents/agents.go index 5160683..f3392a5 100644 --- a/cmd/dotagents/agents.go +++ b/cmd/dotagents/agents.go @@ -357,7 +357,6 @@ func writeTOMLMultiline(b *strings.Builder, key string, value string) { b.WriteString("\n") } - func codexModelFor(model string) string { switch strings.ToLower(strings.TrimSpace(model)) { case "haiku": diff --git a/cmd/dotagents/mcp.go b/cmd/dotagents/mcp.go index d53c46b..8eede48 100644 --- a/cmd/dotagents/mcp.go +++ b/cmd/dotagents/mcp.go @@ -126,7 +126,6 @@ func readNativeMCPServer(agentName string, name string, home string) (mcpServerC return target.read(target, name, home) } - func inspectJSONMCPServer(target mcpTarget, server mcpServerConfig, home string) (string, error) { configPath := target.configPath(home) data, err := os.ReadFile(configPath) From 0a1ad21bd21f01a4b65fa8bec58eda8c02b783fc Mon Sep 17 00:00:00 2001 From: Kirill Korikov <11762090+yourconscience@users.noreply.github.com> Date: Mon, 15 Jun 2026 15:01:40 +0200 Subject: [PATCH 6/6] fix runShellCheck breaking on home paths with spaces --- cmd/dotagents/sources.go | 64 +++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/cmd/dotagents/sources.go b/cmd/dotagents/sources.go index 60b67aa..96aa160 100644 --- a/cmd/dotagents/sources.go +++ b/cmd/dotagents/sources.go @@ -178,40 +178,44 @@ func checkMethodAvailability(m sourceMethodDef, cfg config, home string) (bool, } func runShellCheck(cmd string, home string) (bool, string) { - cmd = strings.ReplaceAll(cmd, "~", home) - parts := strings.Fields(cmd) - if len(parts) >= 3 && parts[0] == "test" { - op := parts[1] - arg := strings.Trim(parts[2], "\"'") - switch op { - case "-s": - info, err := os.Stat(arg) - if err != nil { - return false, err.Error() - } - if info.Size() == 0 { - return false, "file is empty" - } - return true, "" - case "-d": - info, err := os.Stat(arg) - if err != nil { - return false, err.Error() - } - if !info.IsDir() { - return false, "not a directory" + if strings.HasPrefix(cmd, "test -s ") { + path := strings.TrimSpace(strings.TrimPrefix(cmd, "test -s ")) + path = strings.Trim(path, "\"'") + path = strings.ReplaceAll(path, "~", home) + info, err := os.Stat(path) + if err != nil { + return false, err.Error() + } + if info.Size() == 0 { + return false, "file is empty" + } + return true, "" + } + if strings.HasPrefix(cmd, "test -d ") { + path := strings.TrimSpace(strings.TrimPrefix(cmd, "test -d ")) + path = strings.Trim(path, "\"'") + path = strings.ReplaceAll(path, "~", home) + info, err := os.Stat(path) + if err != nil { + return false, err.Error() + } + if !info.IsDir() { + return false, "not a directory" + } + return true, "" + } + if strings.HasPrefix(cmd, "test -n ") { + val := strings.TrimSpace(strings.TrimPrefix(cmd, "test -n ")) + val = strings.Trim(val, "\"'") + if strings.HasPrefix(val, "$") { + if os.Getenv(strings.TrimPrefix(val, "$")) == "" { + return false, "environment variable not set" } return true, "" - case "-n": - if strings.HasPrefix(arg, "$") { - if os.Getenv(strings.TrimPrefix(arg, "$")) == "" { - return false, "environment variable not set" - } - return true, "" - } } } - out, err := exec.Command("sh", "-c", cmd).CombinedOutput() + cmdExpanded := strings.ReplaceAll(cmd, "~", home) + out, err := exec.Command("sh", "-c", cmdExpanded).CombinedOutput() if err != nil { reason := strings.TrimSpace(string(out)) if reason == "" {