Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 47 additions & 46 deletions src/aggregate.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,12 @@
import tempfile
from pathlib import Path

from aggregation import (
load_config,
save_config,
DocsFetcher,
transform_directory_structure,
copy_targeted_docs,
process_all_markdown,
)
from aggregation.releases import generate_release_docs
from aggregation.release_notes import generate_release_notes_docs
from aggregation import (DocsFetcher, copy_targeted_docs, load_config,
process_all_markdown, save_config,
transform_directory_structure)
from aggregation.flavor_matrix import generate_flavor_matrix_docs
from aggregation.release_notes import generate_release_notes_docs
from aggregation.releases import generate_release_docs


def transform_repo_docs(
Expand All @@ -34,14 +29,20 @@ def transform_repo_docs(
print(f"\n{'='*60}")
print(f"Transforming docs for: {repo_name}")
print(f"{'='*60}")

source_dir = temp_dir / repo_name
target_dir = docs_dir / repo.target_path

# Step 1: Copy files with 'github_target_path:' frontmatter
print(f"\nStep 1: Processing targeted files...")
copy_targeted_docs(str(source_dir), str(docs_dir), repo_name, repo.media_directories, repo.root_files)

copy_targeted_docs(
str(source_dir),
str(docs_dir),
repo_name,
repo.media_directories,
repo.root_files,
)

# Step 2: Transform project structure
print(f"\nStep 2: Transforming project structure...")
transform_directory_structure(
Expand All @@ -51,11 +52,11 @@ def transform_repo_docs(
repo.special_files,
repo.media_directories,
)

# Step 3: Process markdown files
print(f"\nStep 3: Processing markdown files...")
process_all_markdown(str(target_dir), repo_name)

print(f"\n✓ Transformation complete for {repo_name}")
return True

Expand All @@ -68,32 +69,32 @@ def aggregate_repo(
) -> tuple:
"""
Aggregate documentation for a single repository.

Returns:
Tuple of (success, resolved_commit_hash)
"""
print(f"\n{'='*60}")
print(f"Aggregating: {repo.name}")
print(f"{'='*60}")

# Create output directory for this repo
repo_output_dir = temp_dir / repo.name
repo_output_dir.mkdir(parents=True, exist_ok=True)

# Fetch the repository
result = fetcher.fetch(repo, repo_output_dir)

if not result.success:
print(f"✗ Failed to fetch {repo.name}")
return False, result.resolved_commit

# Transform the fetched docs
transform_success = transform_repo_docs(repo, docs_dir, temp_dir)

if not transform_success:
print(f"✗ Failed to transform {repo.name}")
return False, result.resolved_commit

return True, result.resolved_commit


Expand All @@ -117,7 +118,7 @@ def main() -> int:
%(prog)s --update-locks
""",
)

parser.add_argument(
"--config",
default="repos-config.json",
Expand All @@ -137,25 +138,25 @@ def main() -> int:
action="store_true",
help="Update commit locks: fetch and update config with resolved commit hashes",
)

args = parser.parse_args()

# Determine script directory
script_dir = Path(__file__).parent.resolve()
project_root = script_dir.parent

# Resolve paths
# Config files are in project root, not in src/
if not Path(args.config).is_absolute():
config_path = project_root / args.config
else:
config_path = Path(args.config)

if not Path(args.docs_dir).is_absolute():
docs_dir = project_root / args.docs_dir
else:
docs_dir = Path(args.docs_dir)

# Load configuration
print(f"{'='*60}")
print("Garden Linux Documentation Aggregation")
Expand All @@ -167,56 +168,56 @@ def main() -> int:
if args.update_locks:
print("Update commit locks: ENABLED")
print()

repos = load_config(str(config_path))

# Create temporary directory for fetched docs
with tempfile.TemporaryDirectory() as temp_dir_str:
temp_dir = Path(temp_dir_str)
print(f"Temporary directory: {temp_dir}\n")

# Initialize fetcher
fetcher = DocsFetcher(project_root, update_locks=args.update_locks)

# Track resolved commits for locking
resolved_commits = {}
success_count = 0
fail_count = 0

# Aggregate each repository
for repo in repos:
# Filter by repo if specified
if args.repo and repo.name != args.repo:
continue

success, resolved_commit = aggregate_repo(
repo,
docs_dir,
temp_dir,
fetcher,
)

if success:
success_count += 1
if resolved_commit:
resolved_commits[repo.name] = resolved_commit
else:
fail_count += 1

# Update config with resolved commits if locking
if args.update_locks and resolved_commits:
print(f"\n{'='*60}")
print("Updating config with resolved commits...")
print(f"{'='*60}\n")

for repo in repos:
if repo.name in resolved_commits:
repo.commit = resolved_commits[repo.name]
print(f" {repo.name}: {resolved_commits[repo.name]}")

save_config(str(config_path), repos)
print(f"\n✓ Config updated: {config_path}")

# Generate flavor matrix documentation after all repos are aggregated
# Use docs/projects/gardenlinux path since temp_dir is cleaned up
gardenlinux_docs_path = docs_dir / "projects" / "gardenlinux"
Expand All @@ -225,33 +226,33 @@ def main() -> int:
print("Generating flavor matrix documentation...")
print(f"{'='*60}\n")
generate_flavor_matrix_docs(docs_dir, gardenlinux_docs_path)

# Generate release documentation from GLRD
print(f"\n{'='*60}")
print("Generating release documentation...")
print(f"{'='*60}\n")
generate_release_docs(docs_dir)

# Generate release notes from GitHub
print(f"\n{'='*60}")
print("Fetching release notes from GitHub...")
print(f"{'='*60}\n")
generate_release_notes_docs(docs_dir)

# Summary
print(f"\n{'='*60}")
print("Documentation aggregation complete!")
print(f"{'='*60}\n")
print(f"Successful: {success_count}")
print(f"Failed: {fail_count}")

print("\nNext steps:")
print(" 1. Review the changes in docs/projects/")
print(" 2. Run 'make dev' or 'pnpm run docs:dev' to preview")
print(" 3. Commit the changes if satisfied")

return 0 if fail_count == 0 else 1


if __name__ == "__main__":
sys.exit(main())
sys.exit(main())
22 changes: 7 additions & 15 deletions src/aggregation/__init__.py
Original file line number Diff line number Diff line change
@@ -1,23 +1,15 @@
"""Aggregation package for docs-ng documentation aggregation."""

# Re-export commonly used functions for backward compatibility with tests
from .transformer import (
rewrite_links,
ensure_frontmatter,
quote_yaml_value,
parse_frontmatter,
)

from .models import RepoConfig, AggregateResult
from .config import load_config, save_config
from .fetcher import DocsFetcher
from .structure import (
transform_directory_structure,
copy_targeted_docs,
process_all_markdown,
)
from .releases import generate_release_docs
from .flavor_matrix import generate_flavor_matrix_docs
from .models import AggregateResult, RepoConfig
from .releases import generate_release_docs
from .structure import (copy_targeted_docs, process_all_markdown,
transform_directory_structure)
from .transformer import (ensure_frontmatter, parse_frontmatter,
quote_yaml_value, rewrite_links)

__all__ = [
# Models
Expand All @@ -41,4 +33,4 @@
"generate_release_docs",
# Flavor Matrix
"generate_flavor_matrix_docs",
]
]
24 changes: 14 additions & 10 deletions src/aggregation/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,34 +2,34 @@

import json
import sys
from typing import Dict, List
from typing import List

from .models import RepoConfig


def load_config(config_path: str) -> List[RepoConfig]:
"""
Load and validate repository configuration.

Args:
config_path: Path to JSON configuration file

Returns:
List of validated RepoConfig objects
"""
try:
with open(config_path, "r", encoding="utf-8") as f:
config = json.load(f)

if "repos" not in config:
raise ValueError("Configuration must have 'repos' array")

repos = []
for repo_dict in config["repos"]:
repo = RepoConfig.from_dict(repo_dict)
repo.validate()
repos.append(repo)

return repos
except json.JSONDecodeError as e:
print(f"Error: Invalid JSON in config file: {e}", file=sys.stderr)
Expand All @@ -42,7 +42,7 @@ def load_config(config_path: str) -> List[RepoConfig]:
def save_config(config_path: str, repos: List[RepoConfig]) -> None:
"""
Save repository configuration to JSON file.

Args:
config_path: Path to JSON configuration file
repos: List of RepoConfig objects to save
Expand All @@ -60,12 +60,16 @@ def save_config(config_path: str, repos: List[RepoConfig]) -> None:
**({"root_files": repo.root_files} if repo.root_files else {}),
**({"structure": repo.structure} if repo.structure != "flat" else {}),
**({"special_files": repo.special_files} if repo.special_files else {}),
**({"media_directories": repo.media_directories} if repo.media_directories else {}),
**(
{"media_directories": repo.media_directories}
if repo.media_directories
else {}
),
}
for repo in repos
]
}

with open(config_path, "w", encoding="utf-8") as f:
json.dump(config, f, indent=2)
f.write("\n")
f.write("\n")
Loading
Loading