diff --git a/vajra-cli/src/main.rs b/vajra-cli/src/main.rs index 1d06249..d3629c9 100644 --- a/vajra-cli/src/main.rs +++ b/vajra-cli/src/main.rs @@ -19,8 +19,9 @@ use vajra_fingerprint::{ cluster_documents, FingerprintAnalyzer, FingerprintResult, StreamingFingerprintAccumulator, }; use vajra_stats::{ - extract_json_path, linear_regression, shannon_entropy_from_counts, StatsAnalyzer, StatsResult, - StreamingStatsAccumulator, + commit_records_from_json, detect_core_team, extract_json_path, governance_analysis, + linear_regression, render_core_team_text, render_governance_markdown, render_governance_text, + shannon_entropy_from_counts, StatsAnalyzer, StatsResult, StreamingStatsAccumulator, }; use vajra_types::scoring::{compute_health_score, HealthMetrics, HealthScore, HealthWeights}; use vajra_types::traits::{ConcernProfile, OutputFormat}; @@ -220,6 +221,39 @@ enum Command { }, /// List all available profiles (built-in and custom) Profiles, + /// Governance metrics: bus factor, merge equity, contributor churn + Governance { + /// Path to JSON file, or `-` for stdin + input: String, + /// JSONPath to the author field (e.g. '$.author') + #[arg(long, default_value = "$.author")] + author_field: String, + /// JSONPath to the timestamp field (e.g. '$.date') + #[arg(long, default_value = "$.date")] + time_field: String, + }, + /// Ingest GitHub repository data (PRs, issues, commits, releases) via gh CLI + IngestGithub { + /// Owner/repo identifier (e.g. 'facebook/react') + repo: String, + /// Output directory for ingested JSON files + #[arg(long, default_value = ".")] + output: PathBuf, + /// Maximum number of pull requests to fetch + #[arg(long, default_value = "500")] + pr_limit: usize, + /// Maximum number of issues to fetch + #[arg(long, default_value = "500")] + issue_limit: usize, + /// Maximum number of commits to fetch + #[arg(long, default_value = "800")] + commit_limit: usize, + }, + /// Detect core team members from commit patterns + CoreTeam { + /// Path to JSON file containing commit data, or `-` for stdin + input: String, + }, } // --------------------------------------------------------------------------- @@ -289,6 +323,19 @@ fn main() { &cli, ), Command::Profiles => cmd_profiles(&cli), + Command::Governance { + input, + author_field, + time_field, + } => cmd_governance(input, author_field, time_field, &cli), + Command::IngestGithub { + repo, + output, + pr_limit, + issue_limit, + commit_limit, + } => cmd_ingest_github(repo, output, *pr_limit, *issue_limit, *commit_limit, &cli), + Command::CoreTeam { input } => cmd_core_team(input, &cli), }; if let Err(e) = result { @@ -2390,3 +2437,139 @@ fn cascade_md(r: &vajra_cascade::CascadeResult) -> String { } o } + +// --------------------------------------------------------------------------- +// governance command +// --------------------------------------------------------------------------- + +fn cmd_governance(input: &str, author_field: &str, time_field: &str, cli: &Cli) -> Result<()> { + let doc = load_document(input, cli)?; + let records = match doc.value().as_array() { + Some(arr) => arr.clone(), + None => { + anyhow::bail!( + "governance command expects a JSON array of records (e.g. commit or PR data)" + ); + } + }; + + if records.is_empty() { + anyhow::bail!("governance command received an empty array — no records to analyze"); + } + + let report = governance_analysis(&records, author_field, time_field) + .map_err(|e| anyhow::anyhow!("{e}"))?; + + match cli.format { + Format::Json => { + let out = serde_json::to_string_pretty(&report).context("JSON serialization failed")?; + let out = maybe_redact(&out, cli); + println!("{out}"); + } + Format::Markdown => { + let md = render_governance_markdown(&report); + let md = maybe_redact(&md, cli); + print!("{md}"); + } + Format::Text | Format::CompactAi => { + let txt = render_governance_text(&report); + let txt = maybe_redact(&txt, cli); + print!("{txt}"); + } + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// ingest-github command +// --------------------------------------------------------------------------- + +fn cmd_ingest_github( + repo: &str, + output: &Path, + pr_limit: usize, + issue_limit: usize, + commit_limit: usize, + cli: &Cli, +) -> Result<()> { + let config = vajra_core::GitHubIngestConfig { + owner_repo: repo.to_string(), + output_dir: output.to_path_buf(), + pr_limit, + issue_limit, + commit_limit, + }; + + if !cli.quiet { + eprintln!("vajra: ingesting {} into {}", repo, output.display()); + } + + let result = vajra_core::ingest_github(&config).map_err(|e| anyhow::anyhow!("{e}"))?; + + match cli.format { + Format::Json | Format::CompactAi => { + let summary = serde_json::json!({ + "repo": repo, + "output_dir": result.output_dir.display().to_string(), + "commits": result.commits, + "pull_requests": result.pull_requests, + "issues": result.issues, + "releases": result.releases, + }); + let out = if matches!(cli.format, Format::Json) { + serde_json::to_string_pretty(&summary).context("JSON serialization failed")? + } else { + serde_json::to_string(&summary).context("JSON serialization failed")? + }; + println!("{out}"); + } + Format::Text | Format::Markdown => { + println!("=== GitHub Ingestion Summary ==="); + println!(" Repository: {repo}"); + println!(" Output dir: {}", result.output_dir.display()); + println!(" Commits: {}", result.commits); + println!(" Pull requests: {}", result.pull_requests); + println!(" Issues: {}", result.issues); + println!(" Releases: {}", result.releases); + } + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// core-team command +// --------------------------------------------------------------------------- + +fn cmd_core_team(input: &str, cli: &Cli) -> Result<()> { + let doc = load_document(input, cli)?; + let records = commit_records_from_json(doc.value()); + + if records.is_empty() { + anyhow::bail!( + "core-team command received no valid commit records — expected JSON array with author_name, author_email, date fields" + ); + } + + let result = detect_core_team(&records).map_err(|e| anyhow::anyhow!("{e}"))?; + + match cli.format { + Format::Json | Format::CompactAi => { + let out = if matches!(cli.format, Format::Json) { + serde_json::to_string_pretty(&result).context("JSON serialization failed")? + } else { + serde_json::to_string(&result).context("JSON serialization failed")? + }; + let out = maybe_redact(&out, cli); + println!("{out}"); + } + Format::Text | Format::Markdown => { + let txt = render_core_team_text(&result); + let txt = maybe_redact(&txt, cli); + print!("{txt}"); + } + } + + Ok(()) +} diff --git a/vajra-core/src/github_ingest.rs b/vajra-core/src/github_ingest.rs new file mode 100644 index 0000000..fd8e2ea --- /dev/null +++ b/vajra-core/src/github_ingest.rs @@ -0,0 +1,642 @@ +//! Native GitHub API ingestion adapter. +//! +//! Shells out to the `gh` CLI (GitHub CLI) to fetch repository metadata +//! (pull requests, issues, releases) and to `git` for commit history. +//! Produces flattened JSON files suitable for Vajra analysis. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use vajra_types::VajraError; + +/// Configuration for GitHub repository ingestion. +#[derive(Debug, Clone)] +pub struct GitHubIngestConfig { + /// Owner/repo identifier, e.g. "facebook/react". + pub owner_repo: String, + /// Directory to write output files into. + pub output_dir: PathBuf, + /// Maximum number of pull requests to fetch. + pub pr_limit: usize, + /// Maximum number of issues to fetch. + pub issue_limit: usize, + /// Maximum number of commits to fetch. + pub commit_limit: usize, +} + +impl Default for GitHubIngestConfig { + fn default() -> Self { + Self { + owner_repo: String::new(), + output_dir: PathBuf::from("."), + pr_limit: 500, + issue_limit: 500, + commit_limit: 800, + } + } +} + +/// Summary of what was ingested. +#[derive(Debug)] +pub struct IngestResult { + /// Number of commits written. + pub commits: usize, + /// Number of pull requests written. + pub pull_requests: usize, + /// Number of issues written. + pub issues: usize, + /// Number of releases written. + pub releases: usize, + /// Output directory path. + pub output_dir: PathBuf, +} + +/// Run the full GitHub ingestion pipeline. +/// +/// Fetches commits (via shallow clone + git log), pull requests, issues, +/// and releases for the given repository, writing each as a JSON array +/// file in `config.output_dir`. +/// +/// # Errors +/// +/// Returns [`VajraError::Config`] if `gh` is not installed, authentication +/// is missing, or the repository is not found. +/// Returns [`VajraError::Io`] if the output directory cannot be created. +pub fn ingest_github(config: &GitHubIngestConfig) -> Result { + validate_gh_cli()?; + ensure_output_dir(&config.output_dir)?; + + let commits = ingest_commits(config)?; + let prs = ingest_prs(config)?; + let issues = ingest_issues(config)?; + let releases = ingest_releases(config)?; + + Ok(IngestResult { + commits, + pull_requests: prs, + issues, + releases, + output_dir: config.output_dir.clone(), + }) +} + +// --------------------------------------------------------------------------- +// Validation helpers +// --------------------------------------------------------------------------- + +/// Verify that the `gh` CLI is available and authenticated. +fn validate_gh_cli() -> Result<(), VajraError> { + let output = Command::new("gh") + .arg("--version") + .output() + .map_err(|_| VajraError::Config { + message: "GitHub CLI (gh) not found. Install from https://cli.github.com".to_string(), + })?; + + if !output.status.success() { + return Err(VajraError::Config { + message: "GitHub CLI (gh) not found. Install from https://cli.github.com".to_string(), + }); + } + + // Check authentication status + let auth_output = Command::new("gh") + .args(["auth", "status"]) + .output() + .map_err(|_| VajraError::Config { + message: "gh auth login required. Run: gh auth login".to_string(), + })?; + + if !auth_output.status.success() { + return Err(VajraError::Config { + message: "gh auth login required. Run: gh auth login".to_string(), + }); + } + + Ok(()) +} + +/// Create the output directory if it does not exist. +fn ensure_output_dir(dir: &Path) -> Result<(), VajraError> { + fs::create_dir_all(dir).map_err(|e| VajraError::Io { + path: dir.to_path_buf(), + source: e, + }) +} + +// --------------------------------------------------------------------------- +// Commit ingestion (via shallow clone + git log) +// --------------------------------------------------------------------------- + +fn ingest_commits(config: &GitHubIngestConfig) -> Result { + let clone_url = format!("https://github.com/{}.git", config.owner_repo); + let clone_dir = config.output_dir.join("_repo_clone"); + + // Shallow clone + let clone_output = Command::new("git") + .args([ + "clone", + "--depth", + &config.commit_limit.to_string(), + "--single-branch", + &clone_url, + ]) + .arg(&clone_dir) + .output() + .map_err(|e| VajraError::Io { + path: PathBuf::from("git"), + source: std::io::Error::new(e.kind(), format!("failed to run git clone: {e}")), + })?; + + if !clone_output.status.success() { + let stderr = String::from_utf8_lossy(&clone_output.stderr); + // Clean up partial clone + let _ = fs::remove_dir_all(&clone_dir); + return Err(classify_git_error(&stderr, &config.owner_repo)); + } + + // Run git log + let log_config = crate::git::GitLogConfig { + limit: config.commit_limit, + branch: None, + }; + let doc = crate::git::load_git_log(&clone_dir, &log_config)?; + + // The document contains a JSON array; count elements and write it out + let value = doc.value(); + let count = value.as_array().map_or(0, Vec::len); + + let out_path = config.output_dir.join("commits.json"); + let pretty = serde_json::to_string_pretty(value).unwrap_or_else(|_| "[]".to_string()); + fs::write(&out_path, pretty).map_err(|e| VajraError::Io { + path: out_path, + source: e, + })?; + + // Clean up clone directory + let _ = fs::remove_dir_all(&clone_dir); + + Ok(count) +} + +fn classify_git_error(stderr: &str, owner_repo: &str) -> VajraError { + let lower = stderr.to_lowercase(); + if lower.contains("not found") || lower.contains("does not exist") { + VajraError::Config { + message: format!("repository not found: {owner_repo}"), + } + } else { + VajraError::Config { + message: format!("git clone failed for {owner_repo}: {}", stderr.trim()), + } + } +} + +// --------------------------------------------------------------------------- +// Pull request ingestion +// --------------------------------------------------------------------------- + +fn ingest_prs(config: &GitHubIngestConfig) -> Result { + let json_fields = + "number,author,createdAt,title,state,mergedAt,additions,deletions,labels,closedAt"; + let raw = run_gh_command(&[ + "pr", + "list", + "--repo", + &config.owner_repo, + "--state", + "all", + "--limit", + &config.pr_limit.to_string(), + "--json", + json_fields, + ])?; + + let mut items: Vec = + serde_json::from_str(&raw).unwrap_or_else(|_| Vec::new()); + + for item in &mut items { + flatten_author(item); + flatten_labels(item); + } + + let count = items.len(); + let out_path = config.output_dir.join("prs.json"); + let pretty = serde_json::to_string_pretty(&items).unwrap_or_else(|_| "[]".to_string()); + fs::write(&out_path, pretty).map_err(|e| VajraError::Io { + path: out_path, + source: e, + })?; + + Ok(count) +} + +// --------------------------------------------------------------------------- +// Issue ingestion +// --------------------------------------------------------------------------- + +fn ingest_issues(config: &GitHubIngestConfig) -> Result { + let json_fields = "number,author,createdAt,title,state,closedAt,labels,comments"; + let raw = run_gh_command(&[ + "issue", + "list", + "--repo", + &config.owner_repo, + "--state", + "all", + "--limit", + &config.issue_limit.to_string(), + "--json", + json_fields, + ])?; + + let mut items: Vec = + serde_json::from_str(&raw).unwrap_or_else(|_| Vec::new()); + + for item in &mut items { + flatten_author(item); + flatten_labels(item); + flatten_comments(item); + } + + let count = items.len(); + let out_path = config.output_dir.join("issues.json"); + let pretty = serde_json::to_string_pretty(&items).unwrap_or_else(|_| "[]".to_string()); + fs::write(&out_path, pretty).map_err(|e| VajraError::Io { + path: out_path, + source: e, + })?; + + Ok(count) +} + +// --------------------------------------------------------------------------- +// Release ingestion +// --------------------------------------------------------------------------- + +fn ingest_releases(config: &GitHubIngestConfig) -> Result { + let json_fields = "tagName,createdAt,name,isPrerelease"; + let raw = run_gh_command(&[ + "release", + "list", + "--repo", + &config.owner_repo, + "--limit", + "50", + "--json", + json_fields, + ])?; + + let items: Vec = serde_json::from_str(&raw).unwrap_or_else(|_| Vec::new()); + + let count = items.len(); + let out_path = config.output_dir.join("releases.json"); + let pretty = serde_json::to_string_pretty(&items).unwrap_or_else(|_| "[]".to_string()); + fs::write(&out_path, pretty).map_err(|e| VajraError::Io { + path: out_path, + source: e, + })?; + + Ok(count) +} + +// --------------------------------------------------------------------------- +// gh CLI runner +// --------------------------------------------------------------------------- + +fn run_gh_command(args: &[&str]) -> Result { + let output = Command::new("gh") + .args(args) + .output() + .map_err(|_| VajraError::Config { + message: "GitHub CLI (gh) not found. Install from https://cli.github.com".to_string(), + })?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(classify_gh_error(&stderr, args)); + } + + String::from_utf8(output.stdout).map_err(|e| VajraError::Parse { + byte_offset: 0, + message: format!("gh output is not valid UTF-8: {e}"), + source_path: None, + }) +} + +fn classify_gh_error(stderr: &str, args: &[&str]) -> VajraError { + let lower = stderr.to_lowercase(); + // Extract the repo from args if present + let repo = args + .windows(2) + .find(|w| w[0] == "--repo") + .map(|w| w[1]) + .unwrap_or("unknown"); + + if lower.contains("could not resolve") || lower.contains("not found") { + VajraError::Config { + message: format!("repository not found: {repo}"), + } + } else if lower.contains("auth") || lower.contains("login") { + VajraError::Config { + message: "gh auth login required. Run: gh auth login".to_string(), + } + } else { + VajraError::Config { + message: format!("gh command failed: {}", stderr.trim()), + } + } +} + +// --------------------------------------------------------------------------- +// JSON flattening helpers +// --------------------------------------------------------------------------- + +/// Flatten `{"author": {"login": "user"}}` to `{"author": "user"}`. +pub fn flatten_author(value: &mut serde_json::Value) { + if let Some(obj) = value.as_object_mut() { + if let Some(author) = obj.get("author").cloned() { + if let Some(login) = author.get("login").and_then(|v| v.as_str()) { + obj.insert( + "author".to_string(), + serde_json::Value::String(login.to_string()), + ); + } + } + } +} + +/// Flatten `{"labels": [{"name": "bug"}, {"name": "high"}]}` to `{"labels": "bug,high"}`. +pub fn flatten_labels(value: &mut serde_json::Value) { + if let Some(obj) = value.as_object_mut() { + if let Some(labels) = obj.get("labels").cloned() { + if let Some(arr) = labels.as_array() { + let names: Vec<&str> = arr + .iter() + .filter_map(|v| v.get("name").and_then(|n| n.as_str())) + .collect(); + obj.insert( + "labels".to_string(), + serde_json::Value::String(names.join(",")), + ); + } + } + } +} + +/// Replace `{"comments": [...]}` with `{"comment_count": N}`. +pub fn flatten_comments(value: &mut serde_json::Value) { + if let Some(obj) = value.as_object_mut() { + if let Some(comments) = obj.get("comments").cloned() { + if let Some(arr) = comments.as_array() { + let count = arr.len(); + obj.remove("comments"); + obj.insert( + "comment_count".to_string(), + serde_json::Value::Number(serde_json::Number::from(count)), + ); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ----------------------------------------------------------------------- + // Flattening tests + // ----------------------------------------------------------------------- + + #[test] + fn flatten_author_extracts_login() { + let mut v = serde_json::json!({ + "number": 1, + "author": {"login": "tim-smart"}, + "title": "Fix bug" + }); + flatten_author(&mut v); + assert_eq!(v["author"], "tim-smart"); + } + + #[test] + fn flatten_author_noop_when_already_string() { + let mut v = serde_json::json!({"author": "alice"}); + flatten_author(&mut v); + assert_eq!(v["author"], "alice"); + } + + #[test] + fn flatten_author_noop_when_missing() { + let mut v = serde_json::json!({"title": "no author"}); + flatten_author(&mut v); + assert!(v.get("author").is_none()); + } + + #[test] + fn flatten_author_null_login() { + let mut v = serde_json::json!({"author": {"login": null}}); + flatten_author(&mut v); + // login is null, not a string, so author remains as the object + assert!(v["author"].is_object()); + } + + #[test] + fn flatten_labels_joins_names() { + let mut v = serde_json::json!({ + "labels": [{"name": "bug"}, {"name": "high"}] + }); + flatten_labels(&mut v); + assert_eq!(v["labels"], "bug,high"); + } + + #[test] + fn flatten_labels_empty_array() { + let mut v = serde_json::json!({"labels": []}); + flatten_labels(&mut v); + assert_eq!(v["labels"], ""); + } + + #[test] + fn flatten_labels_single() { + let mut v = serde_json::json!({"labels": [{"name": "enhancement"}]}); + flatten_labels(&mut v); + assert_eq!(v["labels"], "enhancement"); + } + + #[test] + fn flatten_labels_noop_when_missing() { + let mut v = serde_json::json!({"title": "no labels"}); + flatten_labels(&mut v); + assert!(v.get("labels").is_none()); + } + + #[test] + fn flatten_comments_to_count() { + let mut v = serde_json::json!({ + "comments": [ + {"body": "looks good"}, + {"body": "needs work"}, + {"body": "approved"} + ] + }); + flatten_comments(&mut v); + assert!(v.get("comments").is_none()); + assert_eq!(v["comment_count"], 3); + } + + #[test] + fn flatten_comments_empty() { + let mut v = serde_json::json!({"comments": []}); + flatten_comments(&mut v); + assert_eq!(v["comment_count"], 0); + } + + #[test] + fn flatten_comments_noop_when_missing() { + let mut v = serde_json::json!({"title": "no comments"}); + flatten_comments(&mut v); + assert!(v.get("comment_count").is_none()); + } + + #[test] + fn flatten_all_fields_combined() { + let mut v = serde_json::json!({ + "number": 42, + "author": {"login": "dependabot[bot]"}, + "labels": [{"name": "dependencies"}, {"name": "javascript"}], + "comments": [{"body": "auto-merge"}, {"body": "merged"}], + "title": "Bump lodash" + }); + flatten_author(&mut v); + flatten_labels(&mut v); + flatten_comments(&mut v); + + assert_eq!(v["author"], "dependabot[bot]"); + assert_eq!(v["labels"], "dependencies,javascript"); + assert_eq!(v["comment_count"], 2); + assert_eq!(v["number"], 42); + assert_eq!(v["title"], "Bump lodash"); + } + + // ----------------------------------------------------------------------- + // Error classification tests + // ----------------------------------------------------------------------- + + #[test] + fn classify_gh_error_not_found() { + let err = classify_gh_error( + "GraphQL: Could not resolve to a Repository", + &["pr", "list", "--repo", "noexist/norepo"], + ); + match err { + VajraError::Config { message } => { + assert!( + message.contains("repository not found"), + "unexpected message: {message}" + ); + } + other => panic!("expected Config error, got: {other:?}"), + } + } + + #[test] + fn classify_gh_error_auth() { + let err = classify_gh_error( + "error: gh auth login required", + &["pr", "list", "--repo", "owner/repo"], + ); + match err { + VajraError::Config { message } => { + assert!( + message.contains("gh auth login required"), + "unexpected message: {message}" + ); + } + other => panic!("expected Config error, got: {other:?}"), + } + } + + #[test] + fn classify_gh_error_generic() { + let err = classify_gh_error( + "some unexpected error occurred", + &["pr", "list", "--repo", "owner/repo"], + ); + match err { + VajraError::Config { message } => { + assert!( + message.contains("gh command failed"), + "unexpected message: {message}" + ); + } + other => panic!("expected Config error, got: {other:?}"), + } + } + + #[test] + fn classify_git_error_not_found() { + let err = classify_git_error( + "fatal: repository 'https://github.com/noexist/norepo.git/' not found", + "noexist/norepo", + ); + match err { + VajraError::Config { message } => { + assert!( + message.contains("repository not found"), + "unexpected message: {message}" + ); + } + other => panic!("expected Config error, got: {other:?}"), + } + } + + #[test] + fn classify_git_error_generic() { + let err = classify_git_error("fatal: something else went wrong", "owner/repo"); + match err { + VajraError::Config { message } => { + assert!( + message.contains("git clone failed"), + "unexpected message: {message}" + ); + } + other => panic!("expected Config error, got: {other:?}"), + } + } + + // ----------------------------------------------------------------------- + // Directory creation test + // ----------------------------------------------------------------------- + + #[test] + fn ensure_output_dir_creates_nested() { + let tmp = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir failed: {e}")); + let nested = tmp.path().join("a").join("b").join("c"); + assert!(!nested.exists()); + ensure_output_dir(&nested).unwrap_or_else(|e| panic!("ensure_output_dir failed: {e}")); + assert!(nested.is_dir()); + } + + #[test] + fn ensure_output_dir_existing_is_ok() { + let tmp = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir failed: {e}")); + ensure_output_dir(tmp.path()) + .unwrap_or_else(|e| panic!("ensure_output_dir on existing dir failed: {e}")); + assert!(tmp.path().is_dir()); + } + + // ----------------------------------------------------------------------- + // Config defaults + // ----------------------------------------------------------------------- + + #[test] + fn default_config_values() { + let cfg = GitHubIngestConfig::default(); + assert_eq!(cfg.pr_limit, 500); + assert_eq!(cfg.issue_limit, 500); + assert_eq!(cfg.commit_limit, 800); + assert!(cfg.owner_repo.is_empty()); + } +} diff --git a/vajra-core/src/lib.rs b/vajra-core/src/lib.rs index b7fc527..9259194 100644 --- a/vajra-core/src/lib.rs +++ b/vajra-core/src/lib.rs @@ -12,6 +12,7 @@ pub mod event; pub mod fetch; pub mod formats; pub mod git; +pub mod github_ingest; pub mod input; pub mod markdown; pub mod parse; @@ -29,6 +30,7 @@ pub use formats::{ detect_format, parse_auto, parse_csv, parse_file_auto, parse_ndjson, parse_yaml, InputFormat, }; pub use git::{is_git_repo, load_git_log, GitLogConfig}; +pub use github_ingest::{ingest_github, GitHubIngestConfig, IngestResult}; pub use input::{ load_documents, load_documents_aggregated, load_input, resolve_input, InputSource, LoadedInput, }; diff --git a/vajra-stats/src/core_team.rs b/vajra-stats/src/core_team.rs new file mode 100644 index 0000000..ae16618 --- /dev/null +++ b/vajra-stats/src/core_team.rs @@ -0,0 +1,1186 @@ +//! Core team auto-detection from commit patterns. +//! +//! Classifies repository contributors into three roles—**Core**, **Bot**, and +//! **Community**—using deterministic heuristics applied to commit metadata. +//! +//! # Detection algorithm (O(n log n)) +//! +//! 1. **Bot detection** — name/email pattern matching (highest confidence). +//! 2. **Email domain clustering** — dominant employer domain detection. +//! 3. **Pareto detection** — authors contributing to top 80% of commits. +//! 4. **Temporal consistency** — activity span across months refines confidence. +//! 5. **Signal combination** — merges the above into a final classification. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use vajra_types::InferenceConfidence; + +// --------------------------------------------------------------------------- +// Serde bridge for InferenceConfidence (defined in vajra-types without Serde) +// --------------------------------------------------------------------------- + +mod confidence_serde { + use super::*; + + pub fn serialize( + val: &InferenceConfidence, + serializer: S, + ) -> Result { + let s = match val { + InferenceConfidence::Heuristic => "Heuristic", + InferenceConfidence::Dominant => "Dominant", + InferenceConfidence::Definite => "Definite", + }; + serializer.serialize_str(s) + } + + pub fn deserialize<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> Result { + let s = String::deserialize(deserializer)?; + match s.as_str() { + "Heuristic" => Ok(InferenceConfidence::Heuristic), + "Dominant" => Ok(InferenceConfidence::Dominant), + "Definite" => Ok(InferenceConfidence::Definite), + other => Err(serde::de::Error::custom(format!( + "unknown confidence level: {other}" + ))), + } + } +} + +// --------------------------------------------------------------------------- +// Public types +// --------------------------------------------------------------------------- + +/// Overall result of the core-team detection algorithm. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CoreTeamResult { + /// Authors classified as core team members. + pub core: Vec, + /// Authors classified as bots or automated accounts. + pub bots: Vec, + /// Authors classified as community / occasional contributors. + pub community: Vec, + /// Human-readable description of the detection method(s) used. + pub detection_method: String, +} + +/// A single author with their classification and supporting evidence. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AuthorClassification { + /// Author display name. + pub name: String, + /// Author email address (if available). + pub email: Option, + /// Total number of commits. + pub commits: usize, + /// Assigned role. + pub classification: AuthorRole, + /// Confidence in the assignment. + #[serde(with = "confidence_serde")] + pub confidence: InferenceConfidence, + /// Machine-readable reason string (e.g. `"pareto_80pct"`, `"bot_suffix"`). + pub reason: String, +} + +/// Role assigned to a repository contributor. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum AuthorRole { + /// Core / maintainer. + Core, + /// Automated bot account. + Bot, + /// Community / occasional contributor. + Community, +} + +impl std::fmt::Display for AuthorRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Core => write!(f, "Core"), + Self::Bot => write!(f, "Bot"), + Self::Community => write!(f, "Community"), + } + } +} + +// --------------------------------------------------------------------------- +// Input representation +// --------------------------------------------------------------------------- + +/// A single commit record consumed by the detection engine. +#[derive(Debug, Clone)] +pub struct CommitRecord { + /// Author display name. + pub author_name: String, + /// Author email address. + pub author_email: String, + /// ISO-8601 date string (only the YYYY-MM prefix is used). + pub date: String, +} + +// --------------------------------------------------------------------------- +// Internal bookkeeping +// --------------------------------------------------------------------------- + +/// Aggregated per-author statistics built during the first pass. +#[derive(Debug, Clone)] +struct AuthorStats { + name: String, + email: String, + commits: usize, + /// Set of distinct YYYY-MM months the author committed in. + active_months: BTreeMap, +} + +/// Intermediate signal produced by each detector pass. +#[derive(Debug, Clone)] +struct Signal { + role: AuthorRole, + confidence: InferenceConfidence, + reason: String, +} + +// --------------------------------------------------------------------------- +// Public entry point +// --------------------------------------------------------------------------- + +/// Detect core team members from a sequence of commit records. +/// +/// The algorithm is fully deterministic: given the same input in the same +/// order it will always produce the same output. All internal collections +/// are [`BTreeMap`]-based to guarantee stable iteration order. +/// +/// # Errors +/// +/// Returns `Err` if the input is empty (nothing to classify). +pub fn detect_core_team(commits: &[CommitRecord]) -> Result { + if commits.is_empty() { + return Err("no commits provided for core team detection".to_string()); + } + + // -- Step 0: aggregate per-author stats -------------------------------- + let (author_stats, total_months) = aggregate_authors(commits); + + // -- Step 1-4: compute signals ----------------------------------------- + let mut signals: BTreeMap> = BTreeMap::new(); + + for stats in author_stats.values() { + let key = author_key(&stats.name, &stats.email); + let mut author_signals = Vec::new(); + + // 1. Bot detection + if let Some(sig) = detect_bot(&stats.name, &stats.email) { + author_signals.push(sig); + } + + signals.insert(key, author_signals); + } + + // 2. Email domain clustering + apply_email_domain_clustering(&author_stats, &mut signals); + + // 3. Pareto detection + apply_pareto_detection(&author_stats, &mut signals); + + // 4. Temporal consistency (refinement) + apply_temporal_refinement(&author_stats, total_months, &mut signals); + + // -- Step 5: combine signals ------------------------------------------- + let mut core = Vec::new(); + let mut bots = Vec::new(); + let mut community = Vec::new(); + + let mut methods_used: BTreeMap = BTreeMap::new(); + + for stats in author_stats.values() { + let key = author_key(&stats.name, &stats.email); + let author_signals = signals.get(&key).cloned().unwrap_or_default(); + let (role, confidence, reason) = combine_signals(&author_signals); + + // Track which methods contributed + for sig in &author_signals { + let method = sig.reason.split(':').next().unwrap_or(&sig.reason); + methods_used.insert(method.to_string(), ()); + } + + let classification = AuthorClassification { + name: stats.name.clone(), + email: if stats.email.is_empty() { + None + } else { + Some(stats.email.clone()) + }, + commits: stats.commits, + classification: role, + confidence, + reason, + }; + + match role { + AuthorRole::Bot => bots.push(classification), + AuthorRole::Core => core.push(classification), + AuthorRole::Community => community.push(classification), + } + } + + // Sort each bucket by commits descending, then name ascending for stability. + let sort_fn = |a: &AuthorClassification, b: &AuthorClassification| { + b.commits.cmp(&a.commits).then_with(|| a.name.cmp(&b.name)) + }; + core.sort_by(sort_fn); + bots.sort_by(sort_fn); + community.sort_by(sort_fn); + + let detection_method = if methods_used.is_empty() { + "none".to_string() + } else { + methods_used.keys().cloned().collect::>().join("+") + }; + + Ok(CoreTeamResult { + core, + bots, + community, + detection_method, + }) +} + +// --------------------------------------------------------------------------- +// Step 0: aggregation +// --------------------------------------------------------------------------- + +/// Aggregate raw commits into per-author stats. +/// +/// Returns the map of author stats and the total number of distinct months +/// across the entire commit history. +fn aggregate_authors(commits: &[CommitRecord]) -> (BTreeMap, usize) { + let mut authors: BTreeMap = BTreeMap::new(); + let mut all_months: BTreeMap = BTreeMap::new(); + + for c in commits { + let key = author_key(&c.author_name, &c.author_email); + let month = extract_month(&c.date); + + if let Some(ref m) = month { + all_months.insert(m.clone(), ()); + } + + let entry = authors.entry(key).or_insert_with(|| AuthorStats { + name: c.author_name.clone(), + email: c.author_email.clone(), + commits: 0, + active_months: BTreeMap::new(), + }); + entry.commits += 1; + if let Some(m) = month { + entry.active_months.insert(m, ()); + } + } + + (authors, all_months.len()) +} + +// --------------------------------------------------------------------------- +// Step 1: bot detection +// --------------------------------------------------------------------------- + +fn detect_bot(name: &str, email: &str) -> Option { + // Name ends with [bot] + if name.ends_with("[bot]") { + return Some(Signal { + role: AuthorRole::Bot, + confidence: InferenceConfidence::Definite, + reason: "bot_suffix".to_string(), + }); + } + + // Name starts with app/ + if name.starts_with("app/") { + return Some(Signal { + role: AuthorRole::Bot, + confidence: InferenceConfidence::Definite, + reason: "bot_app_prefix".to_string(), + }); + } + + // Email contains noreply or bot + let email_lower = email.to_lowercase(); + if email_lower.contains("noreply") || email_lower.contains("bot") { + return Some(Signal { + role: AuthorRole::Bot, + confidence: InferenceConfidence::Dominant, + reason: "bot_email_pattern".to_string(), + }); + } + + None +} + +// --------------------------------------------------------------------------- +// Step 2: email domain clustering +// --------------------------------------------------------------------------- + +fn apply_email_domain_clustering( + authors: &BTreeMap, + signals: &mut BTreeMap>, +) { + // Count non-bot commits per email domain. + let mut domain_commits: BTreeMap = BTreeMap::new(); + let mut total_non_bot: usize = 0; + + for stats in authors.values() { + // Skip authors already flagged as bots. + let key = author_key(&stats.name, &stats.email); + let is_bot = signals + .get(&key) + .map(|sigs| sigs.iter().any(|s| s.role == AuthorRole::Bot)) + .unwrap_or(false); + if is_bot { + continue; + } + + if let Some(domain) = extract_domain(&stats.email) { + // Ignore generic domains that don't indicate an employer. + if !is_generic_domain(&domain) { + *domain_commits.entry(domain).or_insert(0) += stats.commits; + } + total_non_bot += stats.commits; + } else { + total_non_bot += stats.commits; + } + } + + if total_non_bot == 0 { + return; + } + + // Find the domain with the most commits. + let top_domain = domain_commits + .iter() + .max_by_key(|(_, &count)| count) + .map(|(domain, &count)| (domain.clone(), count)); + + let (top_domain_name, top_domain_commits) = match top_domain { + Some(t) => t, + None => return, + }; + + // Check the >50% threshold using integer arithmetic to avoid float. + // top_domain_commits / total_non_bot > 0.5 + // ⟺ top_domain_commits * 2 > total_non_bot + if top_domain_commits * 2 <= total_non_bot { + return; + } + + // Tag every non-bot author with the top domain as Core. + for stats in authors.values() { + let key = author_key(&stats.name, &stats.email); + if let Some(domain) = extract_domain(&stats.email) { + if domain == top_domain_name { + let sigs = signals.entry(key).or_default(); + sigs.push(Signal { + role: AuthorRole::Core, + confidence: InferenceConfidence::Dominant, + reason: format!("email_domain:{top_domain_name}"), + }); + } + } + } +} + +// --------------------------------------------------------------------------- +// Step 3: Pareto detection +// --------------------------------------------------------------------------- + +fn apply_pareto_detection( + authors: &BTreeMap, + signals: &mut BTreeMap>, +) { + // Collect non-bot authors sorted by commits descending. + let mut non_bot: Vec<(&String, &AuthorStats)> = authors + .iter() + .filter(|(_, stats)| { + let key = author_key(&stats.name, &stats.email); + !signals + .get(&key) + .map(|sigs| sigs.iter().any(|s| s.role == AuthorRole::Bot)) + .unwrap_or(false) + }) + .collect(); + + // Sort descending by commits, then by key for determinism. + non_bot.sort_by(|a, b| b.1.commits.cmp(&a.1.commits).then_with(|| a.0.cmp(b.0))); + + let total_non_bot_commits: usize = non_bot.iter().map(|(_, s)| s.commits).sum(); + if total_non_bot_commits == 0 { + return; + } + + // 80% threshold using integer arithmetic: + // cumulative / total >= 0.8 ⟺ cumulative * 5 >= total * 4 + let threshold = total_non_bot_commits * 4; + let mut cumulative: usize = 0; + + for (_key, stats) in &non_bot { + cumulative += stats.commits; + let author_key_val = author_key(&stats.name, &stats.email); + let sigs = signals.entry(author_key_val).or_default(); + sigs.push(Signal { + role: AuthorRole::Core, + confidence: InferenceConfidence::Heuristic, + reason: "pareto_80pct".to_string(), + }); + // Once cumulative reaches 80%, stop adding more. + if cumulative * 5 >= threshold { + break; + } + } +} + +// --------------------------------------------------------------------------- +// Step 4: temporal consistency +// --------------------------------------------------------------------------- + +fn apply_temporal_refinement( + authors: &BTreeMap, + total_months: usize, + signals: &mut BTreeMap>, +) { + if total_months == 0 { + return; + } + + for stats in authors.values() { + let key = author_key(&stats.name, &stats.email); + let active = stats.active_months.len(); + + // > 50%: active * 2 > total_months + // < 20%: active * 5 < total_months + let sigs = signals.entry(key).or_default(); + + if active * 2 > total_months { + sigs.push(Signal { + role: AuthorRole::Core, + confidence: InferenceConfidence::Heuristic, + reason: "temporal_consistent".to_string(), + }); + } else if active * 5 < total_months { + sigs.push(Signal { + role: AuthorRole::Community, + confidence: InferenceConfidence::Heuristic, + reason: "temporal_sporadic".to_string(), + }); + } + } +} + +// --------------------------------------------------------------------------- +// Step 5: signal combination +// --------------------------------------------------------------------------- + +fn combine_signals(signals: &[Signal]) -> (AuthorRole, InferenceConfidence, String) { + if signals.is_empty() { + return ( + AuthorRole::Community, + InferenceConfidence::Heuristic, + "no_signal".to_string(), + ); + } + + // Rule: any bot signal -> Bot + let has_bot = signals.iter().any(|s| s.role == AuthorRole::Bot); + if has_bot { + let best_bot = signals + .iter() + .filter(|s| s.role == AuthorRole::Bot) + .max_by_key(|s| s.confidence); + return match best_bot { + Some(s) => (AuthorRole::Bot, s.confidence, s.reason.clone()), + None => ( + AuthorRole::Bot, + InferenceConfidence::Heuristic, + "bot_signal".to_string(), + ), + }; + } + + let has_email_domain = signals + .iter() + .any(|s| s.role == AuthorRole::Core && s.reason.starts_with("email_domain:")); + let has_pareto = signals + .iter() + .any(|s| s.role == AuthorRole::Core && s.reason == "pareto_80pct"); + let has_temporal_boost = signals + .iter() + .any(|s| s.role == AuthorRole::Core && s.reason == "temporal_consistent"); + let has_temporal_demote = signals + .iter() + .any(|s| s.role == AuthorRole::Community && s.reason == "temporal_sporadic"); + + // Email domain + Pareto -> Core (Dominant) + if has_email_domain && has_pareto { + let domain_sig = signals + .iter() + .find(|s| s.reason.starts_with("email_domain:")); + let reason = domain_sig + .map(|s| format!("{}+pareto_80pct", s.reason)) + .unwrap_or_else(|| "email_domain+pareto_80pct".to_string()); + return (AuthorRole::Core, InferenceConfidence::Dominant, reason); + } + + // Email domain only -> Core (Dominant) + if has_email_domain { + let domain_sig = signals + .iter() + .find(|s| s.reason.starts_with("email_domain:")); + let reason = domain_sig + .map(|s| s.reason.clone()) + .unwrap_or_else(|| "email_domain".to_string()); + return (AuthorRole::Core, InferenceConfidence::Dominant, reason); + } + + // Pareto with temporal demote -> Community (Heuristic) + if has_pareto && has_temporal_demote && !has_temporal_boost { + return ( + AuthorRole::Community, + InferenceConfidence::Heuristic, + "pareto_80pct+temporal_sporadic".to_string(), + ); + } + + // Pareto only -> Core (Heuristic), boosted to Dominant if temporal_consistent + if has_pareto { + let confidence = if has_temporal_boost { + InferenceConfidence::Dominant + } else { + InferenceConfidence::Heuristic + }; + let reason = if has_temporal_boost { + "pareto_80pct+temporal_consistent".to_string() + } else { + "pareto_80pct".to_string() + }; + return (AuthorRole::Core, confidence, reason); + } + + // Everything else -> Community + ( + AuthorRole::Community, + InferenceConfidence::Heuristic, + "community_default".to_string(), + ) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Build a deterministic key from name + email. +fn author_key(name: &str, email: &str) -> String { + format!("{name} <{email}>") +} + +/// Extract the YYYY-MM prefix from an ISO-8601-ish date string. +fn extract_month(date: &str) -> Option { + if date.len() >= 7 && date.as_bytes().get(4).copied() == Some(b'-') { + Some(date[..7].to_string()) + } else { + None + } +} + +/// Extract the domain part of an email address (lowercased). +fn extract_domain(email: &str) -> Option { + email + .rsplit_once('@') + .map(|(_, domain)| domain.to_lowercase()) +} + +/// Returns `true` for generic email providers that do not indicate an employer. +fn is_generic_domain(domain: &str) -> bool { + matches!( + domain, + "gmail.com" + | "yahoo.com" + | "hotmail.com" + | "outlook.com" + | "live.com" + | "icloud.com" + | "me.com" + | "mac.com" + | "aol.com" + | "protonmail.com" + | "proton.me" + | "users.noreply.github.com" + | "mail.com" + | "gmx.com" + | "gmx.net" + | "yandex.com" + | "fastmail.com" + | "hey.com" + | "pm.me" + | "qq.com" + | "163.com" + | "126.com" + | "foxmail.com" + ) +} + +// --------------------------------------------------------------------------- +// Convenience: build CommitRecords from serde_json values +// --------------------------------------------------------------------------- + +/// Extract [`CommitRecord`]s from a JSON array of commit objects. +/// +/// Each object is expected to have `author_name`, `author_email`, and `date` +/// string fields (matching the schema produced by `vajra-core`'s git adapter). +/// Objects that are missing both name and email are silently skipped. +pub fn commit_records_from_json(value: &serde_json::Value) -> Vec { + let arr = match value.as_array() { + Some(a) => a, + None => return Vec::new(), + }; + + let mut records = Vec::with_capacity(arr.len()); + for obj in arr { + let name = obj + .get("author_name") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + let email = obj + .get("author_email") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + let date = obj.get("date").and_then(|v| v.as_str()).unwrap_or_default(); + + if name.is_empty() && email.is_empty() { + continue; + } + + records.push(CommitRecord { + author_name: name.to_string(), + author_email: email.to_string(), + date: date.to_string(), + }); + } + + records +} + +// --------------------------------------------------------------------------- +// Text / JSON rendering +// --------------------------------------------------------------------------- + +/// Render a [`CoreTeamResult`] as human-readable text. +pub fn render_core_team_text(result: &CoreTeamResult) -> String { + use std::fmt::Write; + let mut o = String::new(); + + let _ = writeln!(o, "=== Core Team Detection ==="); + let _ = writeln!(o, " Method: {}", result.detection_method); + let _ = writeln!(o); + + if !result.core.is_empty() { + let _ = writeln!(o, " CORE ({}):", result.core.len()); + for a in &result.core { + let email_str = a.email.as_deref().unwrap_or("(no email)"); + let _ = writeln!( + o, + " {} <{}> — {} commits — {:?} — {}", + a.name, email_str, a.commits, a.confidence, a.reason, + ); + } + let _ = writeln!(o); + } + + if !result.bots.is_empty() { + let _ = writeln!(o, " BOTS ({}):", result.bots.len()); + for a in &result.bots { + let email_str = a.email.as_deref().unwrap_or("(no email)"); + let _ = writeln!( + o, + " {} <{}> — {} commits — {:?} — {}", + a.name, email_str, a.commits, a.confidence, a.reason, + ); + } + let _ = writeln!(o); + } + + if !result.community.is_empty() { + let _ = writeln!(o, " COMMUNITY ({}):", result.community.len()); + for a in &result.community { + let email_str = a.email.as_deref().unwrap_or("(no email)"); + let _ = writeln!( + o, + " {} <{}> — {} commits — {:?} — {}", + a.name, email_str, a.commits, a.confidence, a.reason, + ); + } + } + + o +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // -- Helper: build a batch of commits from (name, email, date) tuples -- + + fn make_commits(data: &[(&str, &str, &str)]) -> Vec { + data.iter() + .map(|(name, email, date)| CommitRecord { + author_name: name.to_string(), + author_email: email.to_string(), + date: date.to_string(), + }) + .collect() + } + + /// Extract Ok value, asserting no error. Uses assert! (allowed in tests) not panic!. + fn unwrap_result(r: Result) -> CoreTeamResult { + assert!(r.is_ok(), "unexpected error: {:?}", r.as_ref().err()); + // SAFETY: we just asserted is_ok above; if_let avoids unwrap/expect lint. + match r { + Ok(v) => v, + Err(_) => CoreTeamResult { + core: vec![], + bots: vec![], + community: vec![], + detection_method: String::new(), + }, + } + } + + // -- Bot detection ----------------------------------------------------- + + #[test] + fn bot_suffix_detected() { + let commits = make_commits(&[ + ("dependabot[bot]", "dependabot@github.com", "2024-01-01"), + ("Alice", "alice@example.com", "2024-01-01"), + ]); + let r = unwrap_result(detect_core_team(&commits)); + assert_eq!(r.bots.len(), 1); + assert_eq!(r.bots[0].name, "dependabot[bot]"); + assert_eq!(r.bots[0].confidence, InferenceConfidence::Definite); + assert_eq!(r.bots[0].reason, "bot_suffix"); + } + + #[test] + fn bot_app_prefix_detected() { + let commits = make_commits(&[ + ("app/dependabot", "noreply@github.com", "2024-01-01"), + ("Alice", "alice@example.com", "2024-01-01"), + ]); + let r = unwrap_result(detect_core_team(&commits)); + assert_eq!(r.bots.len(), 1); + assert_eq!(r.bots[0].name, "app/dependabot"); + assert_eq!(r.bots[0].confidence, InferenceConfidence::Definite); + } + + #[test] + fn bot_email_noreply() { + let commits = make_commits(&[( + "github-actions", + "41898282+github-actions[bot]@users.noreply.github.com", + "2024-01-01", + )]); + let r = unwrap_result(detect_core_team(&commits)); + assert_eq!(r.bots.len(), 1); + assert_eq!(r.bots[0].name, "github-actions"); + } + + #[test] + fn bot_email_bot_keyword() { + let commits = make_commits(&[ + ("renovate", "renovate-bot@renovateapp.com", "2024-01-01"), + ("Alice", "alice@example.com", "2024-01-01"), + ]); + let r = unwrap_result(detect_core_team(&commits)); + assert_eq!(r.bots.len(), 1); + assert_eq!(r.bots[0].name, "renovate"); + assert_eq!(r.bots[0].confidence, InferenceConfidence::Dominant); + } + + // -- Email domain clustering ------------------------------------------- + + #[test] + fn email_domain_majority() { + let mut commits = Vec::new(); + let meta_authors = [ + "Alice", "Bob", "Charlie", "Diana", "Eve", "Frank", "Grace", "Heidi", + ]; + for (i, author) in meta_authors.iter().enumerate() { + for m in 1..=5 { + commits.push(CommitRecord { + author_name: author.to_string(), + author_email: format!("{author}@meta.com").to_lowercase(), + date: format!("2024-{m:02}-{:02}", (i % 28) + 1), + }); + } + } + // 2 community authors with 2 commits each + for m in 1..=2 { + commits.push(CommitRecord { + author_name: "Outsider1".to_string(), + author_email: "outsider1@gmail.com".to_string(), + date: format!("2024-{m:02}-15"), + }); + commits.push(CommitRecord { + author_name: "Outsider2".to_string(), + author_email: "outsider2@yahoo.com".to_string(), + date: format!("2024-{m:02}-20"), + }); + } + + let r = unwrap_result(detect_core_team(&commits)); + + // All 8 Meta authors should be Core. + assert_eq!(r.core.len(), 8); + for c in &r.core { + assert_eq!(c.classification, AuthorRole::Core); + assert!( + c.reason.contains("email_domain:meta.com"), + "expected email_domain reason, got: {}", + c.reason + ); + } + // Community authors + assert_eq!(r.community.len(), 2); + } + + // -- Pareto detection -------------------------------------------------- + + #[test] + fn pareto_top_authors() { + let mut commits = Vec::new(); + for i in 0..80 { + let m = (i % 12) + 1; + commits.push(CommitRecord { + author_name: "TimSmart".to_string(), + author_email: "tim@example.com".to_string(), + date: format!("2024-{m:02}-01"), + }); + } + for i in 0..15 { + let m = (i % 6) + 1; + commits.push(CommitRecord { + author_name: "Contributor2".to_string(), + author_email: "c2@example.com".to_string(), + date: format!("2024-{m:02}-01"), + }); + } + for i in 0..5 { + let m = (i % 2) + 1; + commits.push(CommitRecord { + author_name: "Contributor3".to_string(), + author_email: "c3@example.com".to_string(), + date: format!("2024-{m:02}-01"), + }); + } + + let r = unwrap_result(detect_core_team(&commits)); + + let tim = r.core.iter().find(|c| c.name == "TimSmart"); + assert!(tim.is_some(), "TimSmart should be Core"); + assert!( + tim.map(|t| t.reason.contains("pareto_80pct")) + .unwrap_or(false), + "expected pareto reason" + ); + } + + // -- Temporal consistency ---------------------------------------------- + + #[test] + fn temporal_sporadic_demotes_pareto() { + let mut commits = Vec::new(); + + // 12 months of history from a steady contributor (2 commits/month = 24) + for m in 1..=12 { + for _ in 0..2 { + commits.push(CommitRecord { + author_name: "Steady".to_string(), + author_email: "steady@gmail.com".to_string(), + date: format!("2024-{m:02}-10"), + }); + } + } + + // Burst contributor: 30 commits all in January + for _ in 0..30 { + commits.push(CommitRecord { + author_name: "Burst".to_string(), + author_email: "burst@yahoo.com".to_string(), + date: "2024-01-15".to_string(), + }); + } + + let r = unwrap_result(detect_core_team(&commits)); + + // Steady should be Core. + let steady = r.core.iter().find(|c| c.name == "Steady"); + assert!(steady.is_some(), "Steady should be Core"); + + // Burst should be demoted to Community by temporal_sporadic. + let burst = r.community.iter().find(|c| c.name == "Burst"); + assert!( + burst.is_some(), + "Burst should be Community (temporal demote). core={:?}, community={:?}", + r.core.iter().map(|c| &c.name).collect::>(), + r.community.iter().map(|c| &c.name).collect::>(), + ); + } + + // -- Edge cases -------------------------------------------------------- + + #[test] + fn single_author() { + let commits = make_commits(&[ + ("Solo", "solo@example.com", "2024-01-01"), + ("Solo", "solo@example.com", "2024-02-01"), + ]); + let r = unwrap_result(detect_core_team(&commits)); + assert_eq!(r.core.len(), 1); + assert_eq!(r.core[0].name, "Solo"); + } + + #[test] + fn all_equal_commits() { + let commits = make_commits(&[ + ("A", "a@example.com", "2024-01-01"), + ("B", "b@example.com", "2024-01-01"), + ("C", "c@example.com", "2024-01-01"), + ("D", "d@example.com", "2024-01-01"), + ]); + let r = unwrap_result(detect_core_team(&commits)); + // With all equal (1 commit each), Pareto grabs until 80%: + // Each has 1/4 = 25%. Need cumulative >= 80%. + // Integer: threshold=4*4=16, cumulative*5: 5,10,15,20. 20>=16 at 4th. + assert_eq!(r.core.len(), 4); + } + + #[test] + fn no_emails() { + let commits = make_commits(&[ + ("A", "", "2024-01-01"), + ("A", "", "2024-02-01"), + ("B", "", "2024-01-01"), + ]); + let r = unwrap_result(detect_core_team(&commits)); + assert!(!r.core.is_empty()); + // A's email field should be None + let a = r.core.iter().find(|c| c.name == "A"); + assert!(a.is_some()); + assert!(a.map(|x| x.email.is_none()).unwrap_or(false)); + } + + #[test] + fn empty_commits_returns_error() { + let result = detect_core_team(&[]); + assert!(result.is_err()); + } + + // -- JSON parsing ------------------------------------------------------ + + #[test] + fn commit_records_from_json_basic() { + let json = serde_json::json!([ + { + "hash": "abc123", + "author_name": "Alice", + "author_email": "alice@meta.com", + "date": "2024-01-15T10:00:00+00:00", + "subject": "Initial commit" + }, + { + "hash": "def456", + "author_name": "dependabot[bot]", + "author_email": "dependabot@github.com", + "date": "2024-01-16T11:00:00+00:00", + "subject": "Bump deps" + } + ]); + let records = commit_records_from_json(&json); + assert_eq!(records.len(), 2); + assert_eq!(records[0].author_name, "Alice"); + assert_eq!(records[0].author_email, "alice@meta.com"); + assert_eq!(records[1].author_name, "dependabot[bot]"); + } + + #[test] + fn commit_records_from_json_not_array() { + let json = serde_json::json!({"key": "value"}); + let records = commit_records_from_json(&json); + assert!(records.is_empty()); + } + + #[test] + fn commit_records_from_json_missing_fields() { + let json = serde_json::json!([ + {"hash": "abc123", "subject": "No author fields"}, + {"author_name": "Alice", "author_email": "alice@example.com", "date": "2024-01-01"} + ]); + let records = commit_records_from_json(&json); + assert_eq!(records.len(), 1); + assert_eq!(records[0].author_name, "Alice"); + } + + // -- Helper tests ------------------------------------------------------ + + #[test] + fn extract_month_iso_date() { + assert_eq!(extract_month("2024-01-15"), Some("2024-01".to_string())); + } + + #[test] + fn extract_month_iso_datetime() { + assert_eq!( + extract_month("2024-03-20T10:30:00+00:00"), + Some("2024-03".to_string()) + ); + } + + #[test] + fn extract_month_short_string() { + assert_eq!(extract_month("short"), None); + } + + #[test] + fn extract_domain_works() { + assert_eq!( + extract_domain("alice@Meta.Com"), + Some("meta.com".to_string()) + ); + } + + #[test] + fn extract_domain_no_at() { + assert_eq!(extract_domain("noatsign"), None); + } + + #[test] + fn generic_domains_detected() { + assert!(is_generic_domain("gmail.com")); + assert!(is_generic_domain("users.noreply.github.com")); + assert!(!is_generic_domain("meta.com")); + } + + // -- Combined detection scenario: React-like repo ---------------------- + + #[test] + fn react_like_scenario() { + let mut commits = Vec::new(); + + // 5 Meta employees, each with many commits across many months. + let meta_devs = ["Dan", "Andrew", "Sophie", "Rick", "Luna"]; + for dev in &meta_devs { + for m in 1..=10 { + for _ in 0..8 { + commits.push(CommitRecord { + author_name: dev.to_string(), + author_email: format!("{dev}@meta.com").to_lowercase(), + date: format!("2024-{m:02}-01"), + }); + } + } + } + + // 1 bot + for m in 1..=10 { + commits.push(CommitRecord { + author_name: "dependabot[bot]".to_string(), + author_email: "dependabot@github.com".to_string(), + date: format!("2024-{m:02}-15"), + }); + } + + // 10 community contributors, 1-3 commits each in 1-2 months. + for i in 0..10 { + let count = (i % 3) + 1; + for c in 0..count { + let m = (c % 2) + 1; + commits.push(CommitRecord { + author_name: format!("Community{i}"), + author_email: format!("community{i}@gmail.com"), + date: format!("2024-{m:02}-20"), + }); + } + } + + let r = unwrap_result(detect_core_team(&commits)); + + assert_eq!(r.core.len(), 5, "expected 5 core, got {:?}", r.core); + for c in &r.core { + assert!( + meta_devs.iter().any(|d| *d == c.name), + "unexpected core member: {}", + c.name + ); + assert!( + c.reason.contains("email_domain:meta.com"), + "expected email_domain reason for {}, got: {}", + c.name, + c.reason + ); + } + + assert_eq!(r.bots.len(), 1); + assert_eq!(r.bots[0].name, "dependabot[bot]"); + assert_eq!(r.community.len(), 10); + + assert!( + r.detection_method.contains("email_domain"), + "detection_method should mention email_domain: {}", + r.detection_method + ); + } + + // -- Rendering --------------------------------------------------------- + + #[test] + fn render_text_non_empty() { + let r = unwrap_result(detect_core_team(&make_commits(&[ + ("Alice", "alice@example.com", "2024-01-01"), + ("Alice", "alice@example.com", "2024-02-01"), + ("Bob", "bob@example.com", "2024-01-01"), + ]))); + let text = render_core_team_text(&r); + assert!(text.contains("Core Team Detection")); + assert!(text.contains("Alice")); + } + + // -- Determinism: same input -> same output ---------------------------- + + #[test] + fn deterministic_output() { + let commits = make_commits(&[ + ("Zara", "zara@company.com", "2024-01-01"), + ("Zara", "zara@company.com", "2024-02-01"), + ("Zara", "zara@company.com", "2024-03-01"), + ("Adam", "adam@company.com", "2024-01-01"), + ("Adam", "adam@company.com", "2024-02-01"), + ("Mike", "mike@gmail.com", "2024-01-15"), + ]); + + let r1 = unwrap_result(detect_core_team(&commits)); + let r2 = unwrap_result(detect_core_team(&commits)); + + // Same names in same order. + let names1: Vec<&str> = r1.core.iter().map(|c| c.name.as_str()).collect(); + let names2: Vec<&str> = r2.core.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names1, names2); + + let comm1: Vec<&str> = r1.community.iter().map(|c| c.name.as_str()).collect(); + let comm2: Vec<&str> = r2.community.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(comm1, comm2); + } +} diff --git a/vajra-stats/src/governance.rs b/vajra-stats/src/governance.rs new file mode 100644 index 0000000..46e1146 --- /dev/null +++ b/vajra-stats/src/governance.rs @@ -0,0 +1,834 @@ +//! Governance metrics: bus factor, merge equity, contributor churn. +//! +//! All algorithms are O(n log n) or better. Uses `BTreeMap` for determinism. +//! No `unwrap`/`expect`/`panic`. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::temporal::{parse_iso8601, truncate_to_window, value_to_epoch, WindowGranularity}; + +// --------------------------------------------------------------------------- +// Public types +// --------------------------------------------------------------------------- + +/// Governance metrics computed from author/item distributions. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GovernanceMetrics { + /// Minimum authors covering 80% of items. + pub bus_factor_80: usize, + /// Minimum authors covering 50% of items. + pub bus_factor_50: usize, + /// Shannon entropy of the author distribution (bits). + pub author_entropy: f64, + /// Gini coefficient: 0 = equal, 1 = one person does everything. + pub gini_coefficient: f64, + /// Fraction of items by the top author. + pub top1_share: f64, + /// Fraction of items by the top 3 authors. + pub top3_share: f64, + /// Fraction of items by the top 5 authors. + pub top5_share: f64, + /// Fraction of authors with exactly 1 item. + pub one_commit_rate: f64, + /// Number of distinct authors. + pub unique_authors: usize, + /// Total number of items. + pub total_items: usize, +} + +/// Contributor churn analysis over monthly windows. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChurnMetrics { + /// Per-month new/returning/churned/active breakdown. + pub monthly: Vec, + /// Mean net contributor growth per month. + pub mean_net_growth: f64, + /// Median author tenure (first to last item), in days. + pub contributor_half_life_days: Option, +} + +/// A single month's contributor churn breakdown. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MonthChurn { + /// Month label (YYYY-MM). + pub month: String, + /// Authors seen for the first time this month. + pub new: usize, + /// Authors who were seen before and are active this month. + pub returning: usize, + /// Authors active last month but not this month. + pub churned: usize, + /// Total unique active authors this month. + pub active: usize, + /// Net = new - churned. + pub net: i64, +} + +/// Combined governance + churn report. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GovernanceReport { + #[serde(flatten)] + pub governance: GovernanceMetrics, + pub churn: ChurnMetrics, +} + +/// Errors from governance computation. +#[derive(Debug, thiserror::Error)] +pub enum GovernanceError { + #[error("no items provided")] + EmptyInput, + #[error("author field not found in record")] + AuthorFieldMissing, + #[error("time field not found in record")] + TimeFieldMissing, +} + +// --------------------------------------------------------------------------- +// Input extraction +// --------------------------------------------------------------------------- + +/// A single authored, timestamped item extracted from a JSON record. +struct AuthoredItem { + author: String, + epoch: i64, +} + +/// Extract `(author, epoch)` pairs from a JSON array using JSONPath-style field selectors. +fn extract_items( + records: &[serde_json::Value], + author_field: &str, + time_field: &str, +) -> Result, GovernanceError> { + let mut items = Vec::with_capacity(records.len()); + for record in records { + let author_val = crate::temporal::extract_json_path(record, author_field) + .ok_or(GovernanceError::AuthorFieldMissing)?; + let author = match author_val.as_str() { + Some(s) => s.to_owned(), + None => author_val.to_string(), + }; + + let time_val = crate::temporal::extract_json_path(record, time_field) + .ok_or(GovernanceError::TimeFieldMissing)?; + let epoch = value_to_epoch(time_val).or_else(|| { + // Fallback: try string representation + time_val.as_str().and_then(parse_iso8601) + }); + + // Skip records where the time cannot be parsed rather than failing hard. + if let Some(epoch) = epoch { + items.push(AuthoredItem { author, epoch }); + } + } + Ok(items) +} + +// --------------------------------------------------------------------------- +// Bus factor +// --------------------------------------------------------------------------- + +/// Compute the bus factor at a given threshold: minimum number of top authors +/// whose items cover at least `threshold` fraction of total items. +/// +/// `counts` must be sorted descending. Returns at least 1 if there is any author. +fn bus_factor(counts_desc: &[usize], total: usize, threshold: f64) -> usize { + if counts_desc.is_empty() || total == 0 { + return 0; + } + let target = (total as f64 * threshold).ceil() as usize; + let mut cumulative: usize = 0; + for (i, &c) in counts_desc.iter().enumerate() { + cumulative = cumulative.saturating_add(c); + if cumulative >= target { + return i + 1; + } + } + counts_desc.len() +} + +// --------------------------------------------------------------------------- +// Gini coefficient +// --------------------------------------------------------------------------- + +/// Gini coefficient from a slice of non-negative values. +/// +/// Formula (sorted ascending): +/// G = (2 * sum(i * x_i)) / (n * sum(x_i)) - (n + 1) / n +/// +/// O(n log n) for sort. +fn gini(values: &[usize]) -> f64 { + if values.is_empty() { + return 0.0; + } + let mut sorted: Vec = values.to_vec(); + sorted.sort_unstable(); + + let n = sorted.len(); + #[allow(clippy::cast_precision_loss)] + let n_f = n as f64; + let total: usize = sorted.iter().sum(); + if total == 0 { + return 0.0; + } + #[allow(clippy::cast_precision_loss)] + let total_f = total as f64; + + let mut weighted_sum: f64 = 0.0; + for (i, &x) in sorted.iter().enumerate() { + #[allow(clippy::cast_precision_loss)] + let idx = (i + 1) as f64; // 1-based + #[allow(clippy::cast_precision_loss)] + let x_f = x as f64; + weighted_sum += idx * x_f; + } + + let g = (2.0 * weighted_sum) / (n_f * total_f) - (n_f + 1.0) / n_f; + g.clamp(0.0, 1.0) +} + +// --------------------------------------------------------------------------- +// Shannon entropy of author distribution +// --------------------------------------------------------------------------- + +fn author_entropy(counts: &[usize], total: usize) -> f64 { + if total == 0 { + return 0.0; + } + #[allow(clippy::cast_precision_loss)] + let total_f = total as f64; + let mut h = 0.0_f64; + for &c in counts { + if c == 0 { + continue; + } + #[allow(clippy::cast_precision_loss)] + let p = c as f64 / total_f; + h -= p * p.log2(); + } + if h < 0.0 { + 0.0 + } else { + h + } +} + +// --------------------------------------------------------------------------- +// Top-K share +// --------------------------------------------------------------------------- + +fn top_k_share(counts_desc: &[usize], total: usize, k: usize) -> f64 { + if total == 0 { + return 0.0; + } + let top_sum: usize = counts_desc.iter().take(k).sum(); + #[allow(clippy::cast_precision_loss)] + let share = top_sum as f64 / total as f64; + share.clamp(0.0, 1.0) +} + +// --------------------------------------------------------------------------- +// Governance metrics (from author->count map) +// --------------------------------------------------------------------------- + +fn compute_governance(author_counts: &BTreeMap, total: usize) -> GovernanceMetrics { + let unique_authors = author_counts.len(); + let counts: Vec = author_counts.values().copied().collect(); + + // Descending-sorted counts for bus factor / top-K + let mut counts_desc = counts.clone(); + counts_desc.sort_unstable_by(|a, b| b.cmp(a)); + + let one_commit_count = counts.iter().filter(|&&c| c == 1).count(); + #[allow(clippy::cast_precision_loss)] + let one_commit_rate = if unique_authors == 0 { + 0.0 + } else { + one_commit_count as f64 / unique_authors as f64 + }; + + GovernanceMetrics { + bus_factor_80: bus_factor(&counts_desc, total, 0.80), + bus_factor_50: bus_factor(&counts_desc, total, 0.50), + author_entropy: author_entropy(&counts, total), + gini_coefficient: gini(&counts), + top1_share: top_k_share(&counts_desc, total, 1), + top3_share: top_k_share(&counts_desc, total, 3), + top5_share: top_k_share(&counts_desc, total, 5), + one_commit_rate, + unique_authors, + total_items: total, + } +} + +// --------------------------------------------------------------------------- +// Churn metrics +// --------------------------------------------------------------------------- + +fn compute_churn(items: &[AuthoredItem]) -> ChurnMetrics { + if items.is_empty() { + return ChurnMetrics { + monthly: Vec::new(), + mean_net_growth: 0.0, + contributor_half_life_days: None, + }; + } + + // Group authors by month + let mut month_authors: BTreeMap> = BTreeMap::new(); + for item in items { + if let Some(label) = truncate_to_window(item.epoch, WindowGranularity::Month) { + month_authors + .entry(label) + .or_default() + .insert(item.author.clone()); + } + } + + let months: Vec = month_authors.keys().cloned().collect(); + let mut ever_seen: BTreeSet = BTreeSet::new(); + let mut prev_active: BTreeSet = BTreeSet::new(); + let mut monthly: Vec = Vec::with_capacity(months.len()); + + for month in &months { + let active = match month_authors.get(month) { + Some(set) => set.clone(), + None => BTreeSet::new(), + }; + + let new_authors: usize = active.iter().filter(|a| !ever_seen.contains(*a)).count(); + let returning: usize = active.iter().filter(|a| ever_seen.contains(*a)).count(); + let churned: usize = prev_active.iter().filter(|a| !active.contains(*a)).count(); + + #[allow(clippy::cast_possible_wrap)] + let net = new_authors as i64 - churned as i64; + + monthly.push(MonthChurn { + month: month.clone(), + new: new_authors, + returning, + churned, + active: active.len(), + net, + }); + + for a in &active { + ever_seen.insert(a.clone()); + } + prev_active = active; + } + + // Mean net growth + #[allow(clippy::cast_precision_loss)] + let mean_net_growth = if monthly.is_empty() { + 0.0 + } else { + let sum: i64 = monthly.iter().map(|m| m.net).sum(); + sum as f64 / monthly.len() as f64 + }; + + // Contributor half-life: median tenure in days + let contributor_half_life_days = compute_half_life(items); + + ChurnMetrics { + monthly, + mean_net_growth, + contributor_half_life_days, + } +} + +/// Compute median author tenure (first item to last item) in days. +fn compute_half_life(items: &[AuthoredItem]) -> Option { + if items.is_empty() { + return None; + } + + // For each author, find min/max epoch + let mut author_range: BTreeMap<&str, (i64, i64)> = BTreeMap::new(); + for item in items { + let entry = author_range + .entry(item.author.as_str()) + .or_insert((item.epoch, item.epoch)); + if item.epoch < entry.0 { + entry.0 = item.epoch; + } + if item.epoch > entry.1 { + entry.1 = item.epoch; + } + } + + let mut tenures: Vec = author_range + .values() + .map(|(first, last)| (last - first) as f64 / 86400.0) + .collect(); + + if tenures.is_empty() { + return None; + } + + tenures.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let mid = tenures.len() / 2; + if tenures.len().is_multiple_of(2) && tenures.len() >= 2 { + Some((tenures[mid - 1] + tenures[mid]) / 2.0) + } else { + Some(tenures[mid]) + } +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/// Compute governance and churn metrics from a JSON array of records. +/// +/// `author_field` and `time_field` are JSONPath-style selectors (e.g. `$.author`). +pub fn governance_analysis( + records: &[serde_json::Value], + author_field: &str, + time_field: &str, +) -> Result { + if records.is_empty() { + return Err(GovernanceError::EmptyInput); + } + + let items = extract_items(records, author_field, time_field)?; + if items.is_empty() { + return Err(GovernanceError::EmptyInput); + } + + // Build author -> count map (deterministic via BTreeMap) + let mut author_counts: BTreeMap = BTreeMap::new(); + for item in &items { + *author_counts.entry(item.author.clone()).or_insert(0) += 1; + } + + let total = items.len(); + let governance = compute_governance(&author_counts, total); + let churn = compute_churn(&items); + + Ok(GovernanceReport { governance, churn }) +} + +// --------------------------------------------------------------------------- +// Output rendering +// --------------------------------------------------------------------------- + +/// Render the governance report as a human-readable text string. +pub fn render_text(report: &GovernanceReport) -> String { + use std::fmt::Write; + let mut o = String::new(); + + let g = &report.governance; + let _ = writeln!(o, "=== Governance Metrics ==="); + let _ = writeln!(o, " Bus factor (80%): {}", g.bus_factor_80); + let _ = writeln!(o, " Bus factor (50%): {}", g.bus_factor_50); + let _ = writeln!(o, " Author entropy: {:.2} bits", g.author_entropy); + let _ = writeln!(o, " Gini coefficient: {:.4}", g.gini_coefficient); + let _ = writeln!(o, " Top-1 share: {:.3}", g.top1_share); + let _ = writeln!(o, " Top-3 share: {:.3}", g.top3_share); + let _ = writeln!(o, " Top-5 share: {:.3}", g.top5_share); + let _ = writeln!(o, " One-commit rate: {:.3}", g.one_commit_rate); + let _ = writeln!(o, " Unique authors: {}", g.unique_authors); + let _ = writeln!(o, " Total items: {}", g.total_items); + let _ = writeln!(o); + + let c = &report.churn; + let _ = writeln!(o, "=== Contributor Churn ==="); + let _ = writeln!(o, " Mean net growth: {:.2}", c.mean_net_growth); + match c.contributor_half_life_days { + Some(hl) => { + let _ = writeln!(o, " Half-life (days): {:.1}", hl); + } + None => { + let _ = writeln!(o, " Half-life (days): N/A"); + } + } + let _ = writeln!(o); + + if !c.monthly.is_empty() { + let _ = writeln!(o, " MONTH NEW RETURN CHURNED ACTIVE NET"); + for m in &c.monthly { + let _ = writeln!( + o, + " {:<10} {:>4} {:>6} {:>7} {:>6} {:>4}", + m.month, m.new, m.returning, m.churned, m.active, m.net + ); + } + } + + o +} + +/// Render the governance report as a Markdown string. +pub fn render_markdown(report: &GovernanceReport) -> String { + use std::fmt::Write; + let mut o = String::new(); + + let g = &report.governance; + let _ = writeln!(o, "# Governance Metrics\n"); + let _ = writeln!(o, "| Metric | Value |"); + let _ = writeln!(o, "|--------|-------|"); + let _ = writeln!(o, "| Bus factor (80%) | {} |", g.bus_factor_80); + let _ = writeln!(o, "| Bus factor (50%) | {} |", g.bus_factor_50); + let _ = writeln!(o, "| Author entropy | {:.2} bits |", g.author_entropy); + let _ = writeln!(o, "| Gini coefficient | {:.4} |", g.gini_coefficient); + let _ = writeln!(o, "| Top-1 share | {:.3} |", g.top1_share); + let _ = writeln!(o, "| Top-3 share | {:.3} |", g.top3_share); + let _ = writeln!(o, "| Top-5 share | {:.3} |", g.top5_share); + let _ = writeln!(o, "| One-commit rate | {:.3} |", g.one_commit_rate); + let _ = writeln!(o, "| Unique authors | {} |", g.unique_authors); + let _ = writeln!(o, "| Total items | {} |", g.total_items); + let _ = writeln!(o); + + let c = &report.churn; + let _ = writeln!(o, "## Contributor Churn\n"); + let _ = writeln!(o, "| Metric | Value |"); + let _ = writeln!(o, "|--------|-------|"); + let _ = writeln!(o, "| Mean net growth | {:.2} |", c.mean_net_growth); + match c.contributor_half_life_days { + Some(hl) => { + let _ = writeln!(o, "| Half-life (days) | {:.1} |", hl); + } + None => { + let _ = writeln!(o, "| Half-life (days) | N/A |"); + } + } + let _ = writeln!(o); + + if !c.monthly.is_empty() { + let _ = writeln!(o, "### Monthly Breakdown\n"); + let _ = writeln!(o, "| Month | New | Returning | Churned | Active | Net |"); + let _ = writeln!(o, "|-------|-----|-----------|---------|--------|-----|"); + for m in &c.monthly { + let _ = writeln!( + o, + "| {} | {} | {} | {} | {} | {} |", + m.month, m.new, m.returning, m.churned, m.active, m.net + ); + } + } + + o +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + const EPS: f64 = 1e-6; + + // ---- bus factor ---- + + #[test] + fn bus_factor_three_authors_30_30_40() { + // 3 authors: 40%, 30%, 30% + // Cumulative desc = [40, 70, 100] + // 40 < 80, 70 < 80, 100 >= 80 -> bus_factor_80 = 3 + // 40 < 50, 70 >= 50 -> bus_factor_50 = 2 + let counts_desc = vec![40, 30, 30]; + assert_eq!(bus_factor(&counts_desc, 100, 0.80), 3); + assert_eq!(bus_factor(&counts_desc, 100, 0.50), 2); + } + + #[test] + fn bus_factor_single_dominant_author() { + // One author does 90% + let counts_desc = vec![90, 5, 3, 2]; + assert_eq!(bus_factor(&counts_desc, 100, 0.80), 1); + assert_eq!(bus_factor(&counts_desc, 100, 0.50), 1); + } + + #[test] + fn bus_factor_equal_authors() { + // 10 authors, 10 items each -> bus_factor_80 = 8 + let counts_desc = vec![10; 10]; + assert_eq!(bus_factor(&counts_desc, 100, 0.80), 8); + assert_eq!(bus_factor(&counts_desc, 100, 0.50), 5); + } + + #[test] + fn bus_factor_empty() { + let counts_desc: Vec = vec![]; + assert_eq!(bus_factor(&counts_desc, 0, 0.80), 0); + } + + // ---- gini coefficient ---- + + #[test] + fn gini_equal_distribution() { + // All equal -> Gini = 0 + let values = vec![10, 10, 10, 10]; + let g = gini(&values); + assert!(g.abs() < EPS, "expected ~0, got {g}"); + } + + #[test] + fn gini_single_author_does_everything() { + // One author: 100, rest: 0 -> Gini close to 1 + let values = vec![100, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let g = gini(&values); + assert!(g > 0.7, "expected high gini, got {g}"); + } + + #[test] + fn gini_two_equal() { + let values = vec![50, 50]; + let g = gini(&values); + assert!(g.abs() < EPS, "expected ~0, got {g}"); + } + + #[test] + fn gini_in_range() { + let values = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 100]; + let g = gini(&values); + assert!((0.0..=1.0).contains(&g), "gini out of range: {g}"); + } + + // ---- one-commit rate ---- + + #[test] + fn one_commit_rate_known() { + let mut counts = BTreeMap::new(); + counts.insert("alice".to_owned(), 10); + counts.insert("bob".to_owned(), 1); + counts.insert("carol".to_owned(), 1); + counts.insert("dave".to_owned(), 5); + // 2 out of 4 have 1 item -> 0.5 + let gov = compute_governance(&counts, 17); + assert!((gov.one_commit_rate - 0.5).abs() < EPS); + } + + // ---- churn: 3-month dataset ---- + + #[test] + fn churn_three_months() { + // Month 1: alice, bob + // Month 2: bob, carol (alice churned, carol new) + // Month 3: carol, dave (bob churned, dave new) + let items = vec![ + AuthoredItem { + author: "alice".to_owned(), + epoch: 1672531200, // 2023-01-01 + }, + AuthoredItem { + author: "bob".to_owned(), + epoch: 1672617600, // 2023-01-02 + }, + AuthoredItem { + author: "bob".to_owned(), + epoch: 1675209600, // 2023-02-01 + }, + AuthoredItem { + author: "carol".to_owned(), + epoch: 1675296000, // 2023-02-02 + }, + AuthoredItem { + author: "carol".to_owned(), + epoch: 1677628800, // 2023-03-01 + }, + AuthoredItem { + author: "dave".to_owned(), + epoch: 1677715200, // 2023-03-02 + }, + ]; + + let churn = compute_churn(&items); + assert_eq!(churn.monthly.len(), 3); + + // Month 1: new=2 (alice,bob), returning=0, churned=0, active=2, net=2 + assert_eq!(churn.monthly[0].new, 2); + assert_eq!(churn.monthly[0].returning, 0); + assert_eq!(churn.monthly[0].churned, 0); + assert_eq!(churn.monthly[0].active, 2); + assert_eq!(churn.monthly[0].net, 2); + + // Month 2: new=1 (carol), returning=1 (bob), churned=1 (alice), active=2, net=0 + assert_eq!(churn.monthly[1].new, 1); + assert_eq!(churn.monthly[1].returning, 1); + assert_eq!(churn.monthly[1].churned, 1); + assert_eq!(churn.monthly[1].active, 2); + assert_eq!(churn.monthly[1].net, 0); + + // Month 3: new=1 (dave), returning=1 (carol), churned=1 (bob), active=2, net=0 + assert_eq!(churn.monthly[2].new, 1); + assert_eq!(churn.monthly[2].returning, 1); + assert_eq!(churn.monthly[2].churned, 1); + assert_eq!(churn.monthly[2].active, 2); + assert_eq!(churn.monthly[2].net, 0); + } + + // ---- half-life ---- + + #[test] + fn half_life_known_tenure() { + // alice: day 0 to day 10 -> tenure 10 days + // bob: day 0 to day 0 -> tenure 0 days + // carol: day 0 to day 20 -> tenure 20 days + // Sorted tenures: [0, 10, 20] -> median = 10 + let items = vec![ + AuthoredItem { + author: "alice".to_owned(), + epoch: 0, + }, + AuthoredItem { + author: "alice".to_owned(), + epoch: 10 * 86400, + }, + AuthoredItem { + author: "bob".to_owned(), + epoch: 0, + }, + AuthoredItem { + author: "carol".to_owned(), + epoch: 0, + }, + AuthoredItem { + author: "carol".to_owned(), + epoch: 20 * 86400, + }, + ]; + + let hl = compute_half_life(&items); + assert!((hl.unwrap_or(0.0) - 10.0).abs() < EPS); + } + + // ---- property tests via known constraints ---- + + #[test] + fn property_bus_factor_at_least_one() { + let counts_desc = vec![50, 30, 20]; + assert!(bus_factor(&counts_desc, 100, 0.80) >= 1); + assert!(bus_factor(&counts_desc, 100, 0.50) >= 1); + } + + #[test] + fn property_gini_in_zero_one() { + for distribution in [ + vec![1], + vec![1, 1], + vec![1, 100], + vec![1, 1, 1, 1000], + vec![10, 10, 10, 10, 10], + ] { + let g = gini(&distribution); + assert!( + (0.0..=1.0).contains(&g), + "gini out of range for {distribution:?}: {g}" + ); + } + } + + #[test] + fn property_rates_in_zero_one() { + let mut counts = BTreeMap::new(); + counts.insert("a".to_owned(), 50); + counts.insert("b".to_owned(), 30); + counts.insert("c".to_owned(), 20); + let gov = compute_governance(&counts, 100); + assert!((0.0..=1.0).contains(&gov.top1_share)); + assert!((0.0..=1.0).contains(&gov.top3_share)); + assert!((0.0..=1.0).contains(&gov.top5_share)); + assert!((0.0..=1.0).contains(&gov.one_commit_rate)); + assert!((0.0..=1.0).contains(&gov.gini_coefficient)); + } + + // ---- end-to-end via JSON records ---- + + #[test] + fn end_to_end_governance_analysis() { + let records: Vec = vec![ + serde_json::json!({"author": "alice", "date": "2023-01-15T00:00:00Z"}), + serde_json::json!({"author": "alice", "date": "2023-01-20T00:00:00Z"}), + serde_json::json!({"author": "alice", "date": "2023-02-01T00:00:00Z"}), + serde_json::json!({"author": "bob", "date": "2023-01-25T00:00:00Z"}), + serde_json::json!({"author": "bob", "date": "2023-02-10T00:00:00Z"}), + serde_json::json!({"author": "carol", "date": "2023-03-01T00:00:00Z"}), + ]; + + let report = governance_analysis(&records, "$.author", "$.date"); + assert!(report.is_ok()); + let report = match report { + Ok(r) => r, + Err(_) => return, + }; + + assert_eq!(report.governance.unique_authors, 3); + assert_eq!(report.governance.total_items, 6); + assert!(report.governance.bus_factor_80 >= 1); + assert!((0.0..=1.0).contains(&report.governance.gini_coefficient)); + assert!(!report.churn.monthly.is_empty()); + } + + #[test] + fn empty_records_returns_error() { + let records: Vec = vec![]; + let result = governance_analysis(&records, "$.author", "$.date"); + assert!(result.is_err()); + } + + // ---- rendering smoke tests ---- + + #[test] + fn render_text_non_empty() { + let report = GovernanceReport { + governance: GovernanceMetrics { + bus_factor_80: 4, + bus_factor_50: 2, + author_entropy: 3.71, + gini_coefficient: 0.82, + top1_share: 0.234, + top3_share: 0.587, + top5_share: 0.712, + one_commit_rate: 0.591, + unique_authors: 66, + total_items: 800, + }, + churn: ChurnMetrics { + monthly: vec![MonthChurn { + month: "2023-01".to_owned(), + new: 5, + returning: 0, + churned: 0, + active: 5, + net: 5, + }], + mean_net_growth: -1.08, + contributor_half_life_days: Some(14.0), + }, + }; + let text = render_text(&report); + assert!(text.contains("Bus factor (80%):")); + assert!(text.contains("Gini coefficient:")); + } + + #[test] + fn render_markdown_non_empty() { + let report = GovernanceReport { + governance: GovernanceMetrics { + bus_factor_80: 2, + bus_factor_50: 1, + author_entropy: 1.0, + gini_coefficient: 0.5, + top1_share: 0.5, + top3_share: 1.0, + top5_share: 1.0, + one_commit_rate: 0.0, + unique_authors: 2, + total_items: 10, + }, + churn: ChurnMetrics { + monthly: vec![], + mean_net_growth: 0.0, + contributor_half_life_days: None, + }, + }; + let md = render_markdown(&report); + assert!(md.contains("# Governance Metrics")); + assert!(md.contains("| Bus factor (80%)")); + } +} diff --git a/vajra-stats/src/lib.rs b/vajra-stats/src/lib.rs index 53805ba..e8b924f 100644 --- a/vajra-stats/src/lib.rs +++ b/vajra-stats/src/lib.rs @@ -13,9 +13,11 @@ pub mod analyzer; pub mod benford; pub mod cms; +pub mod core_team; pub mod ddsketch; pub mod entropy; pub mod frequency; +pub mod governance; pub mod mad; pub mod numeric; pub mod relationships; @@ -29,9 +31,18 @@ pub use benford::{ leading_digit_distribution, BenfordResult, }; pub use cms::CountMinSketch; +pub use core_team::{ + commit_records_from_json, detect_core_team, render_core_team_text, AuthorClassification, + AuthorRole, CommitRecord, CoreTeamResult, +}; pub use ddsketch::DDSketch; pub use entropy::{normalized_entropy, shannon_entropy_from_counts}; pub use frequency::FrequencyCounter; +pub use governance::{ + governance_analysis, render_markdown as render_governance_markdown, + render_text as render_governance_text, ChurnMetrics, GovernanceError, GovernanceMetrics, + GovernanceReport, MonthChurn, +}; pub use mad::{mad, median, modified_z_score}; pub use numeric::{compute_numeric_stats, percentile, NumericStats}; pub use relationships::{conditional_entropy, discover_relationships, pmi, FieldRelationship};