diff --git a/.copier/main-test.yaml b/.copier/main-test.yaml new file mode 100644 index 0000000..a7b8ec3 --- /dev/null +++ b/.copier/main-test.yaml @@ -0,0 +1,66 @@ +# Main Configuration File for the Dev Docs Copier App +# This is the central config file that references individual workflow configs +# Specified in app's env.yaml as MAIN_CONFIG_FILE + +# ============================================================================ +# GLOBAL DEFAULTS +# ============================================================================ +# These defaults apply to all workflows across all workflow config files +# unless overridden at the workflow config level or individual workflow level + +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + deprecation_check: + enabled: true + file: "deprecated_examples.json" + +# ============================================================================ +# WORKFLOW CONFIG REFERENCES +# ============================================================================ +# App will auto-discover installation ID for source repo, then fetch the workflow config + +workflow_configs: + + # -------------------------------------------------------------------------- + # SAMPLE APPS + # -------------------------------------------------------------------------- + - source: "repo" + repo: "mongodb/docs-sample-apps" + branch: "main" # optional, defaults to main + path: ".copier/config.yaml" + enabled: true + + # -------------------------------------------------------------------------- + # MONOREPO + # -------------------------------------------------------------------------- + - source: "repo" + repo: "10gen/docs-mongodb-internal" + branch: "main" + path: ".copier/config.yaml" + enabled: true + + # -------------------------------------------------------------------------- + # DOCS CODE EXAMPLES (DISABLED) + # -------------------------------------------------------------------------- + - source: "repo" + repo: "mongodb/docs-code-examples" + branch: "main" + path: ".copier/config.yaml" + enabled: false + + # -------------------------------------------------------------------------- + # ** TESTING ** + # -------------------------------------------------------------------------- + - source: "repo" + repo: "cbullinger/aggregation-tasks" + branch: "main" + path: "copier-config.yaml" + enabled: true + + - source: "repo" + repo: "cbullinger/copier-app-source-test" + branch: "main" + path: ".copier/test-main.yaml" + enabled: true diff --git a/.cursorignore b/.cursorignore new file mode 100644 index 0000000..2ebfa8c --- /dev/null +++ b/.cursorignore @@ -0,0 +1,50 @@ +# Git +.git/ + +# Binaries +github-copier +code-copier +copier +config-validator +test-webhook +test-pem +*.exe +*.dll +*.so +*.dylib +*.test + +# Dependencies +vendor/ +go.sum + +# Build/Coverage output +*.out + +# Environment files (secrets) +.env +.env.* +!.env.test + +# Private keys +*.pem +*.key + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db + +# Logs and temp +*.log +tmp/ +temp/ + +# Large test fixtures (JSON payloads) +testdata/*.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6489772..567cdec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ name: CI on: push: branches: [main] + tags: ['v*'] pull_request: branches: [main] @@ -14,15 +15,13 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.24' + go-version: '1.26' - name: Download dependencies run: go mod download - name: Run tests - # Note: -race disabled due to pre-existing race conditions in tests that spawn - # background goroutines. These should be fixed by adding proper synchronization. - run: go test -v ./... + run: go test -race -v ./... lint: runs-on: ubuntu-latest @@ -31,12 +30,12 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.24' + go-version: '1.26' - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v7 with: - version: latest + version: v2.9.0 security: runs-on: ubuntu-latest @@ -45,16 +44,15 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.24' + go-version: '1.26' + + - name: Install gosec + run: go install github.com/securego/gosec/v2/cmd/gosec@latest - name: Run gosec - uses: securego/gosec@master - with: - # Exclude G101 (hardcoded credentials - false positive on env var names) - # Exclude G115 (integer overflow - false positive for PR numbers) - # Exclude G304 (file inclusion - intentional for CLI tools) - # Exclude G306 (file permissions - config files don't need 0600) - args: -exclude=G101,G115,G304,G306 ./... + # All false positives are suppressed with inline #nosec comments. + # No global exclusions โ€” every suppression is documented at the call site. + run: gosec ./... build: runs-on: ubuntu-latest @@ -64,16 +62,34 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.24' + go-version: '1.26' - name: Build run: go build -v ./... + scan: + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v4 + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + scan-ref: '.' + severity: 'CRITICAL,HIGH' + exit-code: '1' + deploy: runs-on: ubuntu-latest - needs: [build, security] - # Only deploy on push to main (not on PRs) - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: [build, security, scan] + # Only deploy on version tag pushes (e.g. v1.0.0) + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + + environment: + name: production + url: ${{ steps.show-url.outputs.url }} permissions: contents: read @@ -87,6 +103,10 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Extract version from tag + id: version + run: echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + - name: Authenticate to Google Cloud uses: google-github-actions/auth@v2 with: @@ -104,6 +124,9 @@ jobs: --project $PROJECT_ID \ --allow-unauthenticated \ --env-vars-file=env-cloudrun.yaml \ + --set-env-vars="GITHUB_APP_ID=${{ secrets.GITHUB_APP_ID }},INSTALLATION_ID=${{ secrets.INSTALLATION_ID }}" \ + --build-arg="VERSION=${{ steps.version.outputs.tag }}" \ + --tag="${{ steps.version.outputs.tag }}" \ --max-instances=10 \ --cpu=1 \ --memory=512Mi \ @@ -113,10 +136,11 @@ jobs: --platform=managed - name: Show deployment URL + id: show-url run: | URL=$(gcloud run services describe $SERVICE_NAME \ --region $REGION \ --project $PROJECT_ID \ --format='value(status.url)') - echo "๐Ÿš€ Deployed to: $URL" - + echo "url=$URL" >> $GITHUB_OUTPUT + echo "Deployed ${{ steps.version.outputs.tag }} to: $URL" diff --git a/.gitignore b/.gitignore index 37a0615..d5c14d9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ github-copier code-copier copier +config-validator +test-webhook +test-pem *.exe *.exe~ *.dll @@ -60,4 +63,3 @@ Thumbs.db # Temporary files tmp/ temp/ -RECOMMENDATIONS.md diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 0000000..87417f0 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,5 @@ +# Example placeholder string in .env.local.example (not a real key) +configs/.env.local.example:private-key:77 + +# Purpose-generated test-only PEM key in .env.test (never associated with a real GitHub App) +.env.test:private-key:30 diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..e8b1f95 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,56 @@ +# golangci-lint v2 configuration +# Matches CI (golangci-lint v2.9.0) and local pre-commit. +# Docs: https://golangci-lint.run/usage/configuration/ +version: "2" + +linters: + default: none + enable: + # Bug detection (default set) + - errcheck # unchecked errors + - govet # suspicious constructs + - ineffassign # unused assignments + - staticcheck # advanced static analysis (includes gosimple) + - unused # unused code + + # Style & quality + - misspell # common typos in comments/strings + - revive # extensible linter (replaces golint) + + settings: + errcheck: + # Ignore intentionally discarded errors in defer cleanup. + # All such cases use the _ = expr pattern for explicitness. + exclude-functions: + - (io.Closer).Close + + revive: + rules: + - name: exported + disabled: true # too noisy for internal-only code + + staticcheck: + checks: + - "all" + - "-SA1029" # context.WithValue key type โ€” acceptable for request-scoped data + - "-ST1000" # package comments โ€” not enforced for this project + - "-ST1003" # naming conventions โ€” existing codebase uses mixed styles + + # In golangci-lint v2, exclusion rules live under linters.exclusions (not issues). + exclusions: + rules: + # Test files: allow dot-imports and unused parameters. + - path: _test\.go + linters: + - revive + text: "dot-imports|unused-parameter" + +formatters: + enable: + - gofmt + - goimports + +issues: + # Don't limit the number of reported issues per linter. + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8c909a..fe7f979 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,11 +5,16 @@ repos: hooks: - id: gitleaks - # Go linting - - repo: https://github.com/golangci/golangci-lint - rev: v1.62.2 + # Go linting - requires golangci-lint v2 installed locally: + # go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.9.0 + - repo: local hooks: - id: golangci-lint + name: golangci-lint + entry: golangci-lint run --fix + language: system + pass_filenames: false + types: [go] # Local Go hooks - repo: local diff --git a/AGENT.md b/AGENT.md index ab2277c..f5c9680 100644 --- a/AGENT.md +++ b/AGENT.md @@ -5,23 +5,47 @@ Webhook service: PR merged โ†’ match files โ†’ transform paths โ†’ copy to targe ## File Map ``` -app.go # entrypoint, HTTP server +app.go # Entrypoint, HTTP server, graceful shutdown services/ - webhook_handler_new.go # HandleWebhookWithContainer() - workflow_processor.go # ProcessWorkflow() - core logic + webhook_handler_new.go # HandleWebhookWithContainer() orchestrator + workflow_processor.go # ProcessWorkflow() - core file matching logic pattern_matcher.go # MatchFile(pattern, path) bool - github_auth.go # ConfigurePermissions() error - github_read.go # GetFilesChangedInPr(), RetrieveFileContents() - github_write_to_target.go # AddFilesToTargetRepoBranch() - github_write_to_source.go # UpdateDeprecationFile() - file_state_service.go # tracks upload/deprecate queues + token_manager.go # TokenManager (thread-safe token state, sync.RWMutex) + github_auth.go # ConfigurePermissions(), JWT generation + github_read.go # GetFilesChangedInPr() (GraphQL), RetrieveFileContents() + github_write_to_target.go # AddFilesToTargetRepos(), addFilesViaPR() + github_write_to_source.go # UpdateDeprecationFile(filesToDeprecate) + rate_limit.go # RateLimitTransport (auto-retry on 403/429) + delivery_tracker.go # DeliveryTracker (webhook idempotency via X-GitHub-Delivery) + errors.go # Sentinel errors (ErrRateLimited, ErrNotFound, etc.) + logger.go # slog JSON handler, LogCritical, LogAndReturnError + file_state_service.go # Tracks upload/deprecate queues (thread-safe) main_config_loader.go # LoadConfig() with $ref support + config_loader.go # Config loading & validation + config_cache.go # CachedConfigLoader (TTL-based config caching) service_container.go # DI container + health_metrics.go # /health (liveness), /ready (readiness), /metrics + audit_logger.go # MongoDB audit logging + slack_notifier.go # Slack notifications + pr_template_fetcher.go # PR template resolution from target repos types/ config.go # Workflow, Transformation, SourcePattern structs types.go # ChangedFile, UploadKey, UploadFileContent configs/environment.go # Config struct, LoadEnvironment() -tests/utils.go # test helpers, httpmock setup +tests/utils.go # Test helpers, httpmock setup +cmd/ + config-validator/ # CLI: validate configs, test patterns, init templates + test-webhook/ # CLI: send test webhook payloads (with delivery ID) + test-pem/ # CLI: verify PEM key + App ID against GitHub API +scripts/ + ci-local.sh # Run full CI pipeline locally (build, test, lint, vet) + run-local.sh # Run app locally with dev settings + deploy-cloudrun.sh # Deploy to Google Cloud Run + integration-test.sh # End-to-end integration test + release.sh # Create versioned release (tag, changelog, GitHub Release) + test-slack.sh # Test Slack notification integration + diagnose-github-auth.sh # Debug GitHub App authentication issues + check-installation-repos.sh # List repos accessible to GitHub App installation ``` ## Key Types @@ -32,11 +56,13 @@ type PatternType string // "prefix" | "glob" | "regex" type TransformationType string // "move" | "copy" | "glob" | "regex" type Workflow struct { - Name string - Source SourceConfig // Repo, Branch, Patterns []SourcePattern - Destination DestinationConfig // Repo, Branch - Transformations []Transformation // Type, From, To, Pattern, Replacement - Commit CommitConfig // Strategy, Message, PRTitle, AutoMerge + Name string + Source Source // Repo, Branch, InstallationID + Destination Destination // Repo, Branch + Transformations []Transformation // Type, From, To, Pattern, Replacement + Exclude []string + CommitStrategy *CommitStrategyConfig // Type (direct|pull_request), PRTitle, PRBody, AutoMerge + DeprecationCheck *DeprecationConfig } // types/types.go @@ -44,15 +70,24 @@ type ChangedFile struct { Path, Status string } // Status: "ADDED"|"MODIFIED"|" type UploadKey struct { RepoName, BranchPath string } ``` -## Global State (โš ๏ธ mutable) +## State Management -```go -// services/github_write_to_target.go -var FilesToUpload map[UploadKey]UploadFileContent -// services/github_auth.go -var InstallationAccessToken string -var OrgTokens map[string]string -``` +All mutable state is encapsulated in `TokenManager` (thread-safe via `sync.RWMutex`): +- Installation access token +- Per-org installation tokens with expiry +- Cached JWT +- HTTP client + +Per-request file state is managed via `FileStateService` in the `ServiceContainer`. + +Webhook idempotency is handled by `DeliveryTracker` (TTL-based, in-memory). + +## Target Repo Batching + +Multiple workflows targeting the **same destination repo** are batched into a single commit/PR. +The last workflow's commit strategy, PR title/body, and auto-merge setting wins. +To get separate PRs per workflow, use different destination repos or branches. +See `docs/ARCHITECTURE.md` ยง "Target Repo Batching" for full details. ## Config Example @@ -62,16 +97,57 @@ workflows: source: { repo: "org/src", branch: "main", patterns: [{type: glob, pattern: "docs/**"}] } destination: { repo: "org/dest", branch: "main" } transformations: [{ type: move, from: "docs/", to: "public/" }] - commit: { strategy: pr, message: "Sync" } # strategy: direct|pr + commit_strategy: { type: pull_request, pr_title: "Sync docs" } # type: direct|pull_request ``` -## Test Commands +## Quick Reference ```bash -go test ./... # all -go test ./services/... -run TestWorkflow -v # specific +# Build & Run +make build # build binary +make run # run with .env +./github-copier -env .env.test # run with specific env file + +# Testing +go test -race ./... # all tests with race detector +go test ./services/... -run TestWorkflow -v # specific test +make test # run all tests via Makefile + +# Linting +golangci-lint run ./... # lint (config: .golangci.yml) +make lint # lint via Makefile + +# CI (local) +./scripts/ci-local.sh # full CI: build, test, lint, vet + +# Release +./scripts/release.sh v1.2.3 # create release (see below) +./scripts/release.sh v1.2.3 --dry-run # preview without changes ``` +## Release Process + +Releases use semantic versioning (`vMAJOR.MINOR.PATCH`) and are automated via `scripts/release.sh`. + +**Prerequisites:** +- Clean working tree on `main` branch +- `gh` CLI authenticated +- `CHANGELOG.md` has content in `[Unreleased]` section + +**What the script does:** +1. Validates version format and prerequisites +2. Renames `[Unreleased]` โ†’ `[vX.Y.Z] - YYYY-MM-DD` in `CHANGELOG.md` +3. Adds fresh `[Unreleased]` section +4. Commits: `Release vX.Y.Z` +5. Creates annotated git tag +6. Pushes to origin (triggers CI deploy via `v*` tag) +7. Creates GitHub Release with changelog excerpt + +**Changelog convention:** +- Follow [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) format +- Sections: `Added`, `Changed`, `Fixed`, `Security`, `Deprecated`, `Removed` +- Add entries to `[Unreleased]` as you work; the release script promotes them + ## Edit Patterns | Task | Files to modify | @@ -80,11 +156,28 @@ go test ./services/... -run TestWorkflow -v # specific | New pattern type | `types/config.go` (PatternType) โ†’ `pattern_matcher.go` | | New config field | `types/config.go` (struct) โ†’ consumers in `workflow_processor.go` | | Webhook logic | `webhook_handler_new.go` | +| Rate limit behavior | `rate_limit.go` | +| Auth flow | `github_auth.go` + `token_manager.go` | +| CLI tool | `cmd//main.go` + `cmd//README.md` | ## Conventions - Return `error`, never `log.Fatal` - Wrap errors: `fmt.Errorf("context: %w", err)` +- Use sentinel errors from `errors.go` where appropriate - Nil-check GitHub API responses before dereference +- Use `log/slog` for all logging (never `log` or `fmt.Print` for operational output) - Tests use `httpmock`; see `tests/utils.go` +- Always run tests with `-race` flag - **Changelog**: Update `CHANGELOG.md` for all notable changes (follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)) + +## Key Documentation + +| Doc | Purpose | +|-----|---------| +| `docs/ARCHITECTURE.md` | System design, data flow, batching behavior | +| `docs/CONFIG-REFERENCE.md` | Full config schema and field reference | +| `docs/DEPLOYMENT.md` | Cloud Run deployment, secrets setup | +| `docs/TROUBLESHOOTING.md` | Common issues and debugging | +| `docs/LOCAL-TESTING.md` | Running and testing locally | +| `testdata/README.md` | Test fixtures and webhook payload examples | diff --git a/CHANGELOG.md b/CHANGELOG.md index 96daa5d..7653814 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,58 @@ # Changelog -All notable changes to this project will be documented in this file. +All notable changes to the github-copier application are documented in this file. -## 17 Dec 2025 +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [Unreleased] + +### Added + +- **`.golangci.yml` config** โ€” Pinned linter and formatter configuration (v2 format) for consistent CI and local behavior. Enabled linters: `errcheck`, `govet`, `ineffassign`, `staticcheck`, `unused`, `misspell`, `revive`. +- **Structured error alerting** โ€” `ErrorEvent` now includes `DeliveryID` and `Attempts` fields. Slack failure notifications include the GitHub delivery ID and attempt count for full traceability. +- **Integration test for target repo batching** โ€” `TestIntegration_TargetRepoBatching_MixedStrategies` verifies that workflows with different commit strategies produce separate operations, while same-strategy workflows batch correctly. +- **End-to-end integration tests** โ€” `TestIntegration_MergedPR_DirectCommit` covers the full webhook-to-commit pipeline; additional tests cover no-matching-workflows, config-load errors, and webhook signature verification. +- **Config reference doc** โ€” `docs/CONFIG-REFERENCE.md` provides a single-page reference for all environment variables and workflow YAML schema. +- **Webhook routing guide** โ€” Added a "Webhook Routing" section to `docs/LOCAL-TESTING.md` documenting how to avoid dual-delivery (local + Cloud Run processing the same webhook). +- **Webhook processing timeout** โ€” Background goroutine now applies `context.WithTimeout` (configurable via `WEBHOOK_PROCESSING_TIMEOUT_SECONDS`, default 300s). +- **Retry with exponential backoff** โ€” `processWebhookWithRetry` retries failed webhook processing with configurable max retries and initial delay. Panics are recovered and retried. Slack alert sent after exhaustion. +- **Graceful partial failure** โ€” `processFilesWithWorkflows` processes each workflow independently and returns per-workflow errors. One workflow failure no longer blocks others. +- **Config caching** โ€” `CachedConfigLoader` caches resolved workflow configs with a configurable TTL (default 5 min, via `CONFIG_CACHE_TTL_SECONDS`). +- **Parallel file fetching** โ€” `ProcessWorkflow` now fetches file contents concurrently via `errgroup` (concurrency limit of 5). +- **PR deduplication** โ€” `addFilesViaPR` checks for existing `copier/*` PRs before creating new ones; pushes to existing branch and updates metadata instead. +- **Empty commit prevention** โ€” `createCommitTree` returns base tree SHA; commits are skipped when the new tree is identical to HEAD. +- **Mixed commit strategy fix** โ€” `UploadKey` now includes `CommitStrategy`, separating write operations for `direct` vs `pull_request` workflows targeting the same repo. Config-time warning for conflicting strategies. +- **PR metadata overwrite logging** โ€” Logs when a subsequent workflow overwrites a batched commit message or PR title. +- **Health check probes** โ€” Liveness (`/health`) and readiness (`/ready`) endpoints. +- **Webhook idempotency** โ€” In-memory `DeliveryTracker` prevents duplicate processing of the same `X-GitHub-Delivery` header within a single instance. +- **Rate limiting** โ€” GitHub API retry logic with exponential backoff. +- **CLI tools** โ€” `config-validator`, `test-webhook`, and `test-pem` utilities under `cmd/`. +- **`/config` diagnostic endpoint** โ€” Read-only endpoint showing resolved runtime config (secrets redacted) and workflow summary. +- **Transient vs permanent error classification** โ€” `IsPermanentError()` detects non-retryable failures (404, 403, config validation, etc.); retry loop skips retries immediately for permanent errors. +- **Version stamping** โ€” Binary version set via `-ldflags` at build time; exposed on `/health`, `/config`, startup banner, and `-version` flag. +- **Release script** โ€” `scripts/release.sh` automates CHANGELOG update, git tagging, push, and GitHub Release creation. + +### Changed + +- **Go version** โ€” Upgraded to Go 1.26.0. +- **golangci-lint** โ€” Upgraded to v2.9.0 (action v7 in CI). +- **go-github** โ€” Upgraded to v82; replaced deprecated `github.String`/`Int`/`Bool` with `github.Ptr`. +- **Logging** โ€” Migrated to `log/slog` with structured JSON output. +- **Pre-commit hooks** โ€” `golangci-lint` hook uses `language: system` with `--fix`; requires local v2.9.0 install. +- **App banner** โ€” Now displays version and `EffectiveConfigFile()` instead of the legacy `ConfigFile` default. +- **CI deploy trigger** โ€” Deployment now triggers on version tag pushes (`v*`) instead of every push to `main`. Tags stamp the version into the Cloud Run revision. +- **Legacy config deprecation** โ€” `DefaultConfigLoader` (single-file config) is marked deprecated with runtime warnings; dead code (`ConfigValidator`, unused struct fields) removed. + +### Fixed + +- **CI lint/security failures** โ€” Resolved `golangci-lint` Go version incompatibility, `gosec` taint analysis false positives (G703โ€“G706), and all `errcheck`/`staticcheck`/`unused` issues. +- **gitleaks false positive** โ€” Added `.gitleaksignore` entries for example and test-only PEM keys. +- **Tightened gosec exclusions** โ€” Removed all global `gosec` exclusions from CI; sole remaining false positive suppressed with inline `#nosec G115`. + +### Security +- **Go toolchain directive** โ€” Added `toolchain go1.26.0` to `go.mod` for deterministic builds. + +## [0.1.0] - 2025-12-17 ### Added - CI/CD pipeline with GitHub Actions (`.github/workflows/ci.yml`) @@ -38,7 +88,7 @@ All notable changes to this project will be documented in this file. - Added gitleaks pre-commit hook for secrets detection - Added gosec security scanning in CI pipeline -## Initial Release (Migration from mongodb/code-example-tooling) +## [0.0.1] - Initial Release (Migration from mongodb/code-example-tooling) ### Features - Webhook service for automated file copying on PR merge @@ -50,5 +100,4 @@ All notable changes to this project will be documented in this file. - Slack notifications for operational visibility - MongoDB audit logging (optional) - Google Cloud Logging integration -- Dry-run mode for testing - +- Dry-run mode for testing \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2bedd4f..6292220 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,8 @@ # Build stage -FROM golang:1.24.0-alpine AS builder +FROM golang:1.26.0-alpine AS builder + +# Version is set at build time (e.g. docker build --build-arg VERSION=v1.0.0) +ARG VERSION=dev # Install build dependencies RUN apk add --no-cache git ca-certificates @@ -15,24 +18,33 @@ COPY . . # Build the binary # CGO_ENABLED=0 for static binary (no C dependencies) -# -ldflags="-w -s" strips debug info to reduce size -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o examples-copier . +# -ldflags: -w -s strips debug info; -X stamps version into the binary +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s -X main.version=${VERSION}" -o github-copier . -# Runtime stage -FROM alpine:latest +# Runtime stage - pin to specific version for reproducible builds +FROM alpine:3.21 # Install ca-certificates for HTTPS requests RUN apk --no-cache add ca-certificates -WORKDIR /root/ +# Create non-root user +RUN addgroup -S appgroup && adduser -S appuser -G appgroup + +WORKDIR /home/appuser # Copy binary from builder -COPY --from=builder /app/examples-copier . +COPY --from=builder /app/github-copier . + +# Switch to non-root user +USER appuser # Cloud Run sets PORT environment variable # Our app reads it from config.Port (defaults to 8080) EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 + # Run the binary -CMD ["./examples-copier"] +CMD ["./github-copier"] diff --git a/Makefile b/Makefile index b586e4c..015a953 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,10 @@ # Default target help: - @echo "Examples Copier - Makefile" + @echo "GitHub Copier - Makefile" @echo "" - @echo "Available targets:" + @echo "Build & Run:" @echo " make build - Build all binaries" - @echo " make test - Run all tests" - @echo " make test-unit - Run unit tests only" - @echo " make test-webhook - Build webhook test tool" @echo " make run - Run application" @echo " make run-dry - Run in dry-run mode" @echo " make run-local - Run in local dev mode (recommended)" @@ -16,19 +13,32 @@ help: @echo " make install - Install all tools to \$$GOPATH/bin" @echo " make clean - Remove built binaries" @echo "" - @echo "Testing with webhooks:" - @echo " make test-webhook-example - Test with example payload" + @echo "Testing:" + @echo " make test - Run all tests" + @echo " make test-unit - Run unit tests only" + @echo " make test-coverage - Run tests with coverage report" + @echo " make test-integration - Run integration tests" + @echo " make test-integration-quick - Quick integration smoke test" + @echo "" + @echo "Webhook Testing:" + @echo " make test-webhook - Build webhook test tool" + @echo " make test-webhook-example - Test with example payload (no signature)" + @echo " make test-webhook-gcp - Test with GCP webhook secret" @echo " make test-webhook-pr PR=123 OWNER=org REPO=repo - Test with real PR" @echo "" + @echo "CI Pipelines:" + @echo " make ci-local - Run local CI checks (build, test, lint, vet)" + @echo " make ci-full - Full CI with integration tests" + @echo "" @echo "Quick start for local testing:" - @echo " make run-local-quick # Start app (Terminal 1)" + @echo " make run-local-quick # Start app (Terminal 1)" @echo " make test-webhook-example # Send test webhook (Terminal 2)" @echo "" # Build all binaries build: - @echo "Building examples-copier..." - @go build -o examples-copier . + @echo "Building github-copier..." + @go build -o github-copier . @echo "Building config-validator..." @go build -o config-validator ./cmd/config-validator @echo "Building test-webhook..." @@ -42,7 +52,7 @@ test: test-unit # Run unit tests test-unit: @echo "Running unit tests..." - @go test ./services -v + @go test -race ./services -v # Run unit tests with coverage test-coverage: @@ -58,17 +68,23 @@ test-webhook: @echo "โœ“ test-webhook built" # Test with example payload +# Uses WEBHOOK_SECRET env var if set, otherwise defaults to "test-secret" (matches .env.test) +# To use GCP secret: make test-webhook-gcp test-webhook-example: test-webhook @echo "Testing with example payload..." - @if [ -z "$$WEBHOOK_SECRET" ]; then \ - echo "Fetching webhook secret from Secret Manager..."; \ - export WEBHOOK_SECRET=$$(gcloud secrets versions access latest --secret=webhook-secret 2>/dev/null); \ - fi; \ + @SECRET=$${WEBHOOK_SECRET:-test-secret}; \ + echo "Using webhook secret: $$SECRET"; \ + ./test-webhook -payload testdata/example-pr-merged.json -secret "$$SECRET" + +# Test with example payload using GCP secret (for testing against Cloud Run) +test-webhook-gcp: test-webhook + @echo "Testing with example payload (using GCP secret)..." + @WEBHOOK_SECRET=$$(gcloud secrets versions access latest --secret=webhook-secret 2>/dev/null); \ if [ -n "$$WEBHOOK_SECRET" ]; then \ - ./test-webhook -payload test-payloads/example-pr-merged.json -secret "$$WEBHOOK_SECRET"; \ + ./test-webhook -payload testdata/example-pr-merged.json -secret "$$WEBHOOK_SECRET"; \ else \ - echo "Warning: WEBHOOK_SECRET not set, sending without signature"; \ - ./test-webhook -payload test-payloads/example-pr-merged.json; \ + echo "Error: Could not fetch webhook secret from GCP"; \ + exit 1; \ fi # Test with real PR (requires PR, OWNER, REPO variables) @@ -78,14 +94,10 @@ test-webhook-pr: test-webhook echo "Usage: make test-webhook-pr PR=123 OWNER=myorg REPO=myrepo"; \ exit 1; \ fi - @if [ -z "$$WEBHOOK_SECRET" ]; then \ - echo "Fetching webhook secret from Secret Manager..."; \ - export WEBHOOK_SECRET=$$(gcloud secrets versions access latest --secret=webhook-secret 2>/dev/null); \ - fi; \ - if [ -n "$$WEBHOOK_SECRET" ]; then \ + @if [ -n "$$WEBHOOK_SECRET" ]; then \ ./test-webhook -pr $(PR) -owner $(OWNER) -repo $(REPO) -secret "$$WEBHOOK_SECRET"; \ else \ - echo "Warning: WEBHOOK_SECRET not set, sending without signature"; \ + echo "No WEBHOOK_SECRET set, sending without signature"; \ ./test-webhook -pr $(PR) -owner $(OWNER) -repo $(REPO); \ fi @@ -100,28 +112,28 @@ test-pr: # Run application run: build - @echo "Starting examples-copier..." - @./examples-copier + @echo "Starting github-copier..." + @./github-copier # Run in dry-run mode run-dry: build - @echo "Starting examples-copier in dry-run mode..." - @DRY_RUN=true ./examples-copier + @echo "Starting github-copier in dry-run mode..." + @DRY_RUN=true ./github-copier # Run in local development mode (recommended) run-local: build - @echo "Starting examples-copier in local development mode..." + @echo "Starting github-copier in local development mode..." @./scripts/run-local.sh # Run with cloud logging disabled (quick local testing) run-local-quick: build - @echo "Starting examples-copier (local, no cloud logging)..." - @COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./examples-copier + @echo "Starting github-copier (local, no cloud logging)..." + @COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./github-copier # Validate configuration validate: build @echo "Validating configuration..." - @./examples-copier -validate + @./github-copier -validate # Install binaries to $GOPATH/bin install: @@ -129,12 +141,13 @@ install: @go install . @go install ./cmd/config-validator @go install ./cmd/test-webhook + @go install ./cmd/test-pem @echo "โœ“ Binaries installed to \$$GOPATH/bin" # Clean built binaries clean: @echo "Cleaning built binaries..." - @rm -f examples-copier config-validator test-webhook + @rm -f github-copier config-validator test-webhook test-pem @rm -f coverage.out coverage.html @echo "โœ“ Clean complete" @@ -169,6 +182,8 @@ version: dev-setup: deps build @echo "Setting up development environment..." @chmod +x scripts/test-with-pr.sh + @chmod +x scripts/integration-test.sh + @chmod +x scripts/ci-local.sh @echo "โœ“ Development environment ready" # Quick test cycle @@ -179,3 +194,21 @@ quick-test: build test-unit full-test: build test-unit test-webhook-example @echo "โœ“ Full test cycle complete" +# Integration tests +test-integration: build test-webhook + @echo "Running integration tests..." + @./scripts/integration-test.sh + +# Quick integration tests +test-integration-quick: build test-webhook + @echo "Running quick integration tests..." + @./scripts/integration-test.sh --quick + +# Local CI pipeline +ci-local: + @./scripts/ci-local.sh + +# Full CI with integration tests +ci-full: + @./scripts/ci-local.sh --full + diff --git a/QUICK-REFERENCE.md b/QUICK-REFERENCE.md deleted file mode 100644 index 28fe49a..0000000 --- a/QUICK-REFERENCE.md +++ /dev/null @@ -1,544 +0,0 @@ -# Quick Reference Guide - -## Command Line - -### Application - -```bash -# Run with default settings -./github-copier - -# Run with custom environment -./github-copier -env ./configs/.env.production - -# Dry-run mode (no actual commits) -./github-copier -dry-run - -# Validate configuration only -./github-copier -validate - -# Show help -./github-copier -help -``` - -### CLI Validator - -```bash -# Validate config -./config-validator validate -config copier-config.yaml -v - -# Test pattern -./config-validator test-pattern -type regex -pattern "^examples/(?P[^/]+)/.*$" -file "examples/go/main.go" - -# Test transformation -./config-validator test-transform -source "examples/go/main.go" -template "docs/${lang}/${file}" -vars "lang=go,file=main.go" - -# Initialize new config -./config-validator init -output copier-config.yaml - -# Convert formats -./config-validator convert -input config.json -output copier-config.yaml -``` - -## Configuration Patterns - -### Move Transformation -```yaml -transformations: - - move: - from: "examples/go" - to: "code/go" -``` - -### Glob Transformation -```yaml -transformations: - - glob: - pattern: "examples/*/main.go" - transform: "code/${relative_path}" -``` - -### Regex Transformation -```yaml -transformations: - - regex: - pattern: "^examples/(?P[^/]+)/(?P.+)$" - transform: "code/${lang}/${file}" -``` - -### Workflow with Exclusions -```yaml -workflows: - - name: "Copy examples" - transformations: - - move: - from: "examples" - to: "code" - exclude: - - "**/.gitignore" - - "**/node_modules/**" - - "**/.env" - - "**/dist/**" -``` - -## Path Transformations - -Path transformations are used with **`glob`** and **`regex`** transformation types using the `transform` parameter. - -### Built-in Variables -- `${path}` - Full source path -- `${filename}` - File name only -- `${dir}` - Directory path -- `${ext}` - File extension -- `${relative_path}` - Path relative to glob pattern prefix (glob only) - -### Glob Transformation Examples -```yaml -# Keep same path -transformations: - - glob: - pattern: "examples/**/*.go" - transform: "${path}" - -# Change directory -transformations: - - glob: - pattern: "examples/**/*.go" - transform: "docs/${relative_path}" - -# Reorganize structure (using custom variables from regex) -transformations: - - regex: - pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" - transform: "docs/${lang}/${category}/${file}" - -# Change extension -transformations: - - glob: - pattern: "examples/**/*.txt" - transform: "${dir}/${filename}.md" -``` - -## Commit Strategies - -### Direct Commit -```yaml -commit_strategy: - type: "direct" - commit_message: "Update examples" -``` - -### Pull Request -```yaml -commit_strategy: - type: "pull_request" - commit_message: "Update examples" - pr_title: "Update code examples" - pr_body: "Automated update" - use_pr_template: true # Fetch PR template from target repo - auto_merge: true -``` - -## Advanced Features - -### Exclude Patterns -Exclude unwanted files from being copied at the workflow level: - -```yaml -workflows: - - name: "Copy examples" - exclude: - - "**/.gitignore" # Exclude .gitignore - - "**/node_modules/**" # Exclude dependencies - - "**/.env" # Exclude .env files - - "**/.env.*" # Exclude .env.local, .env.production, etc. - - "**/dist/**" # Exclude build output - - "**/build/**" # Exclude build artifacts - - "**/*.test.js" # Exclude test files -``` - -### PR Template Integration -Fetch and merge PR templates from target repos: - -```yaml -commit_strategy: - type: "pull_request" - pr_body: | - ๐Ÿค– Automated update - Files: ${file_count} - use_pr_template: true # Fetches .github/pull_request_template.md -``` - -**Result:** PR description shows: -1. Target repo's PR template (checklists, guidelines) -2. Separator (`---`) -3. Your configured content (automation info) - -## Message Templates - -### Available Variables -- `${rule_name}` - Copy rule name (e.g., "java-aggregation-examples") -- `${source_repo}` - Source repository (e.g., "mongodb/aggregation-tasks") -- `${target_repo}` - Target repository (e.g., "mongodb/vector-search") -- `${source_branch}` - Source branch (e.g., "main") -- `${target_branch}` - Target branch (e.g., "main") -- `${file_count}` - Number of files (e.g., "3") -- `${pr_number}` - Source PR number (e.g., "42") -- `${commit_sha}` - Source commit SHA (e.g., "abc123") -- Custom variables from regex patterns (e.g., `${lang}`, `${file}`) - -### Examples -```yaml -commit_message: "Update ${category} examples from ${lang}" -pr_title: "Update ${lang} examples" -pr_body: | - Files updated: ${file_count} using ${rule_name} match pattern - - Source: ${source_repo} - PR: #${pr_number} -``` - -## API Endpoints - -### Health Check -```bash -curl http://localhost:8080/health -``` - -### Metrics -```bash -curl http://localhost:8080/metrics -``` - -### Webhook -```bash -curl -X POST http://localhost:8080/webhook \ - -H "Content-Type: application/json" \ - -H "X-Hub-Signature-256: sha256=..." \ - -d @webhook-payload.json -``` - -## Environment Variables - -### Required -```bash -REPO_OWNER=your-org -REPO_NAME=your-repo -GITHUB_APP_ID=123456 -GITHUB_INSTALLATION_ID=789012 -GCP_PROJECT_ID=your-project -PEM_KEY_NAME=projects/123/secrets/KEY/versions/latest -``` - -### Optional -```bash -# Application -PORT=8080 -CONFIG_FILE=copier-config.yaml -DEPRECATION_FILE=deprecated_examples.json -DRY_RUN=false - -# Logging -LOG_LEVEL=info -COPIER_DEBUG=false -COPIER_DISABLE_CLOUD_LOGGING=false - -# Audit -AUDIT_ENABLED=true -MONGO_URI=mongodb+srv://... -AUDIT_DATABASE=code_copier -AUDIT_COLLECTION=audit_events - -# Metrics -METRICS_ENABLED=true - -# Webhook -WEBHOOK_SECRET=your-secret -``` - -## MongoDB Queries - -### Recent Events -```javascript -db.audit_events.find().sort({timestamp: -1}).limit(10) -``` - -### Failed Operations -```javascript -db.audit_events.find({success: false}).sort({timestamp: -1}) -``` - -### Events by Rule -```javascript -db.audit_events.find({rule_name: "Copy Go examples"}) -``` - -### Statistics -```javascript -db.audit_events.aggregate([ - {$match: {event_type: "copy"}}, - {$group: { - _id: "$rule_name", - count: {$sum: 1}, - avg_duration: {$avg: "$duration_ms"} - }} -]) -``` - -### Success Rate -```javascript -db.audit_events.aggregate([ - {$group: { - _id: "$success", - count: {$sum: 1} - }} -]) -``` - -## Testing - -### Run Unit Tests -```bash -# All tests -go test ./services -v - -# Specific test -go test ./services -v -run TestPatternMatcher - -# With coverage -go test ./services -cover -``` - -### Test with Webhooks - -#### Option 1: Use Example Payload -```bash -# Build test tool -go build -o test-webhook ./cmd/test-webhook - -# Send example payload -./test-webhook -payload testdata/example-pr-merged.json - -# Dry-run (see payload without sending) -./test-webhook -payload testdata/example-pr-merged.json -dry-run -``` - -#### Option 2: Use Real PR Data -```bash -# Set GitHub token -export GITHUB_TOKEN=ghp_your_token_here - -# Fetch and send real PR data -./test-webhook -pr 123 -owner myorg -repo myrepo - -# Test against production -./test-webhook -pr 123 -owner myorg -repo myrepo \ - -url https://myapp.appspot.com/webhook \ - -secret "my-webhook-secret" -``` - -#### Option 3: Use Helper Script (Interactive) -```bash -# Make executable -chmod +x scripts/test-with-pr.sh - -# Run interactive test -./scripts/test-with-pr.sh 123 myorg myrepo -``` - -### Test in Dry-Run Mode -```bash -# Start app in dry-run mode -DRY_RUN=true ./github-copier & - -# Send test webhook -./test-webhook -pr 123 -owner myorg -repo myrepo - -# Check logs (no actual commits made) -``` - -### Build -```bash -# Main application -go build -o github-copier . - -# CLI validator -go build -o config-validator ./cmd/config-validator - -# Test webhook tool -go build -o test-webhook ./cmd/test-webhook - -# All tools -go build -o github-copier . && \ -go build -o config-validator ./cmd/config-validator && \ -go build -o test-webhook ./cmd/test-webhook -``` - -## Common Patterns - -### Copy All Go Files -```yaml -workflows: - - name: "Copy Go files" - source: - repo: "org/source" - branch: "main" - destination: - repo: "org/docs" - branch: "main" - transformations: - - regex: - pattern: "^examples/.*\\.go$" - transform: "code/${path}" -``` - -### Organize by Language -```yaml -workflows: - - name: "Organize by language" - transformations: - - regex: - pattern: "^examples/(?P[^/]+)/(?P.+)$" - transform: "languages/${lang}/${rest}" -``` - -### Multiple Workflows for Different Destinations -```yaml -workflows: - - name: "Copy to docs-v1" - destination: - repo: "org/docs-v1" - branch: "main" - transformations: - - move: - from: "examples" - to: "examples" - - - name: "Copy to docs-v2" - destination: - repo: "org/docs-v2" - branch: "main" - transformations: - - move: - from: "examples" - to: "code-samples" -``` - -### Conditional Copying (by file type) -```yaml -workflows: - - name: "Copy by file type" - transformations: - - regex: - pattern: "^examples/.*\\.(?Pgo|py|js)$" - transform: "code/${ext}/${filename}" -``` - -## Troubleshooting - -### Check Logs -```bash -# Application logs -gcloud app logs tail -s default - -# Local logs -LOG_LEVEL=debug ./github-copier -``` - -### Validate Config -```bash -./config-validator validate -config copier-config.yaml -v -``` - -### Test Pattern Matching -```bash -./config-validator test-pattern \ - -type regex \ - -pattern "your-pattern" \ - -file "test/file.go" -``` - -### Dry Run -```bash -DRY_RUN=true ./github-copier -``` - -### Check Health -```bash -curl http://localhost:8080/health -``` - -### Check Metrics -```bash -curl http://localhost:8080/metrics | jq -``` - -## Deployment - -### Google Cloud Quick Commands - -```bash -# Deploy (env.yaml is included via 'includes' directive in app.yaml) -gcloud app deploy app.yaml - -# View logs -gcloud app logs tail -s default - -# Check health -curl https://github-copy-code-examples.appspot.com/health - -# List secrets -gcloud secrets list - -# Grant access -./grant-secret-access.sh -``` - - - -## File Locations - -``` -github-copier/ -โ”œโ”€โ”€ README.md # Main documentation -โ”œโ”€โ”€ QUICK-REFERENCE.md # This file -โ”œโ”€โ”€ docs/ -โ”‚ โ”œโ”€โ”€ ARCHITECTURE.md # Architecture overview -โ”‚ โ”œโ”€โ”€ CONFIGURATION-GUIDE.md # Complete config reference -โ”‚ โ”œโ”€โ”€ DEPLOYMENT.md # Deployment guide -โ”‚ โ”œโ”€โ”€ DEPLOYMENT-CHECKLIST.md # Deployment checklist -โ”‚ โ”œโ”€โ”€ FAQ.md # Frequently asked questions -โ”‚ โ”œโ”€โ”€ LOCAL-TESTING.md # Local testing guide -โ”‚ โ”œโ”€โ”€ PATTERN-MATCHING-GUIDE.md # Pattern matching guide -โ”‚ โ”œโ”€โ”€ PATTERN-MATCHING-CHEATSHEET.md # Quick pattern reference -โ”‚ โ”œโ”€โ”€ TROUBLESHOOTING.md # Troubleshooting guide -โ”‚ โ””โ”€โ”€ WEBHOOK-TESTING.md # Webhook testing guide -โ”œโ”€โ”€ configs/ -โ”‚ โ”œโ”€โ”€ .env # Environment config -โ”‚ โ”œโ”€โ”€ env.yaml.example # Environment template -โ”‚ โ””โ”€โ”€ copier-config.example.yaml # Config template -โ””โ”€โ”€ cmd/ - โ”œโ”€โ”€ config-validator/ # CLI validation tool - โ””โ”€โ”€ test-webhook/ # Webhook testing tool -``` - -## Quick Start Checklist - -- [ ] Clone repository -- [ ] Copy `configs/.env.local.example` to `configs/.env` -- [ ] Set required environment variables -- [ ] Create `copier-config.yaml` in source repo -- [ ] Validate config: `./config-validator validate -config copier-config.yaml` -- [ ] Test in dry-run: `DRY_RUN=true ./github-copier` -- [ ] Deploy: `./github-copier` -- [ ] Configure GitHub webhook -- [ ] Monitor: `curl http://localhost:8080/health` - -## Support - -- **Documentation**: [README.md](README.md) -- **Configuration**: [Configuration Guide](./docs/CONFIGURATION-GUIDE.md) -- **Deployment**: [Deployment Guide](./docs/DEPLOYMENT.md) -- **Troubleshooting**: [Troubleshooting Guide](./docs/TROUBLESHOOTING.md) -- **FAQ**: [Frequently Asked Questions](./docs/FAQ.md) - diff --git a/README.md b/README.md index db0a9aa..2a74c22 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,10 @@ A GitHub app that automatically copies code examples and files from source repos - **PR Template Integration** - Fetch and merge PR templates from target repos - **File Exclusion** - Exclude patterns to filter out unwanted files - **Audit Logging** - MongoDB-based event tracking for all operations -- **Health & Metrics** - `/health` and `/metrics` endpoints for monitoring +- **Health & Metrics** - `/health`, `/ready`, and `/metrics` endpoints for monitoring +- **Rate Limit Handling** - Automatic GitHub API rate limit detection, backoff, and retry +- **Webhook Idempotency** - Deduplication via `X-GitHub-Delivery` header tracking +- **Structured Logging** - JSON structured logging via `log/slog` (Cloud Logging compatible) - **Development Tools** - Dry-run mode, CLI validation, enhanced logging - **Thread-Safe** - Concurrent webhook processing with proper state management @@ -30,7 +33,7 @@ A GitHub app that automatically copies code examples and files from source repos ### Prerequisites -- Go 1.23.4+ +- Go 1.26+ - GitHub App credentials - Google Cloud project (for Secret Manager and logging) - MongoDB Atlas (optional, for audit logging) @@ -358,29 +361,20 @@ Validate and test configurations before deployment: ## Monitoring -### Health Endpoint +### Health Endpoint (Liveness) -Check application health: +Basic liveness check: ```bash curl http://localhost:8080/health ``` -Response: -```json -{ - "status": "healthy", - "started": true, - "github": { - "status": "healthy", - "authenticated": true - }, - "queues": { - "upload_count": 0, - "deprecation_count": 0 - }, - "uptime": "1h23m45s" -} +### Readiness Endpoint + +Deep readiness probe (checks GitHub auth, rate limits, MongoDB): + +```bash +curl http://localhost:8080/ready ``` ### Metrics Endpoint @@ -391,31 +385,6 @@ Get performance metrics: curl http://localhost:8080/metrics ``` -Response: -```json -{ - "webhooks": { - "received": 42, - "processed": 40, - "failed": 2, - "success_rate": 95.24, - "processing_time": { - "avg_ms": 234.5, - "p50_ms": 200, - "p95_ms": 450, - "p99_ms": 890 - } - }, - "files": { - "matched": 150, - "uploaded": 145, - "upload_failed": 5, - "deprecated": 3, - "upload_success_rate": 96.67 - } -} -``` - ## Audit Logging When enabled, all operations are logged to MongoDB: @@ -445,18 +414,18 @@ db.audit_events.aggregate([ ## Testing -### Run Unit Tests +### Run Tests ```bash -# Run all tests -go test ./services -v +# Run all tests with race detector +go test -race ./... # Run specific test suite go test ./services -v -run TestPatternMatcher # Run with coverage -go test ./services -cover -go test ./services -coverprofile=coverage.out +go test -race ./services -cover +go test -race ./services -coverprofile=coverage.out go tool cover -html=coverage.out ``` @@ -467,18 +436,18 @@ go tool cover -html=coverage.out Test without making actual changes: ```bash -DRY_RUN=true ./github-copier +./github-copier -env .env.test -dry-run ``` In dry-run mode: -- Webhooks are processed -- Files are matched and transformed -- Audit events are logged -- **NO actual commits or PRs are created** +- Webhooks are received and processed through the full pipeline +- Files are matched and path transformations are applied +- GitHub auth failures are tolerated (logged as warnings) +- **No commits, PRs, or file uploads are created** -### Enhanced Logging +### Structured Logging -Enable detailed logging: +The app uses `log/slog` with JSON output. Enable debug logging: ```bash LOG_LEVEL=debug ./github-copier @@ -493,35 +462,50 @@ COPIER_DEBUG=true ./github-copier ``` github-copier/ โ”œโ”€โ”€ app.go # Main application entry point +โ”œโ”€โ”€ github-app-manifest.yml # GitHub App permissions documentation โ”œโ”€โ”€ cmd/ โ”‚ โ”œโ”€โ”€ config-validator/ # CLI validation tool +โ”‚ โ”œโ”€โ”€ test-pem/ # PEM key validation tool โ”‚ โ””โ”€โ”€ test-webhook/ # Webhook testing tool โ”œโ”€โ”€ configs/ โ”‚ โ”œโ”€โ”€ environment.go # Environment configuration โ”‚ โ”œโ”€โ”€ .env.local.example # Local environment template -โ”‚ โ”œโ”€โ”€ env.yaml.example # YAML environment template โ”‚ โ””โ”€โ”€ copier-config.example.yaml # Config template +โ”œโ”€โ”€ scripts/ +โ”‚ โ”œโ”€โ”€ release.sh # Create versioned releases +โ”‚ โ”œโ”€โ”€ deploy-cloudrun.sh # Cloud Run deployment +โ”‚ โ”œโ”€โ”€ ci-local.sh # Run CI checks locally +โ”‚ โ”œโ”€โ”€ integration-test.sh # End-to-end integration tests +โ”‚ โ””โ”€โ”€ ... # Additional helper scripts โ”œโ”€โ”€ services/ -โ”‚ โ”œโ”€โ”€ pattern_matcher.go # Pattern matching engine -โ”‚ โ”œโ”€โ”€ config_loader.go # Config loading & validation -โ”‚ โ”œโ”€โ”€ audit_logger.go # MongoDB audit logging -โ”‚ โ”œโ”€โ”€ health_metrics.go # Health & metrics endpoints -โ”‚ โ”œโ”€โ”€ file_state_service.go # Thread-safe state management -โ”‚ โ”œโ”€โ”€ service_container.go # Dependency injection -โ”‚ โ”œโ”€โ”€ webhook_handler_new.go # Webhook handler -โ”‚ โ”œโ”€โ”€ github_auth.go # GitHub authentication -โ”‚ โ”œโ”€โ”€ github_read.go # GitHub read operations +โ”‚ โ”œโ”€โ”€ webhook_handler_new.go # Webhook handler (orchestrator) +โ”‚ โ”œโ”€โ”€ workflow_processor.go # ProcessWorkflow() - core logic +โ”‚ โ”œโ”€โ”€ pattern_matcher.go # Pattern matching engine +โ”‚ โ”œโ”€โ”€ config_loader.go # Config loading & validation +โ”‚ โ”œโ”€โ”€ main_config_loader.go # Main config with $ref support +โ”‚ โ”œโ”€โ”€ github_auth.go # GitHub App authentication +โ”‚ โ”œโ”€โ”€ github_read.go # GitHub read operations (REST + GraphQL) โ”‚ โ”œโ”€โ”€ github_write_to_target.go # GitHub write operations -โ”‚ โ””โ”€โ”€ slack_notifier.go # Slack notifications +โ”‚ โ”œโ”€โ”€ github_write_to_source.go # Deprecation file updates +โ”‚ โ”œโ”€โ”€ token_manager.go # Thread-safe token state management +โ”‚ โ”œโ”€โ”€ rate_limit.go # GitHub API rate limit handling +โ”‚ โ”œโ”€โ”€ delivery_tracker.go # Webhook idempotency (deduplication) +โ”‚ โ”œโ”€โ”€ errors.go # Sentinel errors & classification +โ”‚ โ”œโ”€โ”€ logger.go # Structured logging (slog) +โ”‚ โ”œโ”€โ”€ service_container.go # Dependency injection container +โ”‚ โ”œโ”€โ”€ file_state_service.go # Thread-safe upload/deprecation queues +โ”‚ โ”œโ”€โ”€ health_metrics.go # Health, readiness, metrics & config endpoints +โ”‚ โ”œโ”€โ”€ slack_notifier.go # Slack notifications +โ”‚ โ””โ”€โ”€ pr_template_fetcher.go # PR template resolution โ”œโ”€โ”€ types/ -โ”‚ โ”œโ”€โ”€ config.go # Configuration types -โ”‚ โ””โ”€โ”€ types.go # Core types +โ”‚ โ”œโ”€โ”€ config.go # Configuration types +โ”‚ โ””โ”€โ”€ types.go # Core types โ””โ”€โ”€ docs/ - โ”œโ”€โ”€ ARCHITECTURE.md # Architecture overview - โ”œโ”€โ”€ CONFIGURATION-GUIDE.md # Complete config reference - โ”œโ”€โ”€ DEPLOYMENT.md # Deployment guide - โ”œโ”€โ”€ FAQ.md # Frequently asked questions - โ””โ”€โ”€ ... # Additional documentation + โ”œโ”€โ”€ DEPLOYMENT.md # Deployment & rollback guide + โ”œโ”€โ”€ CONFIG-REFERENCE.md # Environment variables & YAML schema + โ”œโ”€โ”€ WEBHOOK-TESTING.md # Webhook testing guide + โ”œโ”€โ”€ SLACK-NOTIFICATIONS.md # Slack integration guide + โ””โ”€โ”€ ... # Additional documentation ``` ### Service Container @@ -533,53 +517,85 @@ container := NewServiceContainer(config) // All services initialized and wired together ``` -## Deployment +## Releasing + +The project uses semantic versioning (`vMAJOR.MINOR.PATCH`) with GitHub Releases. Pushing a version tag triggers CI to build, test, and deploy to Cloud Run. -See [DEPLOYMENT.md](./docs/DEPLOYMENT.md) for complete deployment guide. +### Release Workflow -### Google Cloud Run +1. Merge your changes to `main`. +2. Run the release script: ```bash -cd github-copier -./scripts/deploy-cloudrun.sh +# Preview what will happen (no changes made) +./scripts/release.sh v1.2.0 --dry-run + +# Create the release +./scripts/release.sh v1.2.0 ``` -### Docker +The script: +1. Validates the version format and that the working tree is clean on `main` +2. Renames the `[Unreleased]` section in `CHANGELOG.md` to `[v1.2.0] - YYYY-MM-DD` +3. Commits the changelog update and creates an annotated git tag +4. Pushes the tag to origin โ€” this triggers the CI `deploy` job +5. Creates a GitHub Release with the changelog excerpt + +### CI Deploy Pipeline + +The `deploy` job in `.github/workflows/ci.yml` runs only on version tag pushes: + +- Authenticates to Google Cloud via Workload Identity Federation +- Deploys to Cloud Run with the version stamped as a build arg (`VERSION`) +- Tags the Cloud Run revision with the version for easy rollback + +### Version Stamping + +The version tag is injected at build time via `-ldflags`: ```bash -docker build -t github-copier . -docker run -p 8080:8080 --env-file env.yaml github-copier +go build -ldflags "-X main.Version=v1.2.0" -o github-copier . ``` +The version appears in the startup banner and the `/health` endpoint response. + +## Deployment + +See [DEPLOYMENT.md](./docs/DEPLOYMENT.md) for the complete deployment and rollback guide. + ## Security - **Webhook Signature Verification** - HMAC-SHA256 validation +- **Webhook Idempotency** - Duplicate delivery detection via `X-GitHub-Delivery` - **Secret Management** - Google Cloud Secret Manager -- **Least Privilege** - Minimal GitHub App permissions -- **Audit Trail** - Complete operation logging +- **Least Privilege** - Minimal GitHub App permissions (see `github-app-manifest.yml`) ## Documentation ### Getting Started -- **[Main Config README](configs/copier-config-examples/MAIN-CONFIG-README.md)** - Complete main config documentation -- **[Quick Start Guide](configs/copier-config-examples/QUICK-START-MAIN-CONFIG.md)** - Get started in 5 minutes +- **[Main Config README](configs/copier-config-examples/MAIN-CONFIG-README.md)** - Main config architecture +- **[Source Repo README](configs/copier-config-examples/SOURCE-REPO-README.md)** - Workflow config guide for source repos - **[Pattern Matching Guide](docs/PATTERN-MATCHING-GUIDE.md)** - Pattern matching with examples - **[Local Testing](docs/LOCAL-TESTING.md)** - Test locally before deploying -- **[Deployment Guide](docs/DEPLOYMENT.md)** - Deploy to production +- **[Deployment Guide](docs/DEPLOYMENT.md)** - Deploy to Cloud Run ### Reference +- **[Config Reference](docs/CONFIG-REFERENCE.md)** - Environment variables and YAML schema - **[Architecture](docs/ARCHITECTURE.md)** - System design and components - **[Troubleshooting](docs/TROUBLESHOOTING.md)** - Common issues and solutions - **[FAQ](docs/FAQ.md)** - Frequently asked questions -- **[Deprecation Tracking](docs/DEPRECATION-TRACKING-EXPLAINED.md)** - How deprecation tracking works +- **[Changelog](CHANGELOG.md)** - Release history ### Features - **[Slack Notifications](docs/SLACK-NOTIFICATIONS.md)** - Slack integration guide -- **[Webhook Testing](docs/WEBHOOK-TESTING.md)** - Test with real PR data +- **[Webhook Testing](docs/WEBHOOK-TESTING.md)** - Webhook testing guide +- **[GitHub App Manifest](github-app-manifest.yml)** - Required permissions and events ### Tools -- **[Scripts](scripts/README.md)** - Helper scripts for deployment and testing +- **[Config Validator](cmd/config-validator/README.md)** - CLI tool for validating configs +- **[Test Webhook](cmd/test-webhook/README.md)** - CLI tool for testing webhooks +- **[Scripts](scripts/README.md)** - Helper scripts for deployment, testing, and releases diff --git a/app.go b/app.go index f104059..00c50b4 100644 --- a/app.go +++ b/app.go @@ -4,7 +4,6 @@ import ( "context" "flag" "fmt" - "log" "net/http" "os" "os/signal" @@ -15,6 +14,13 @@ import ( "github.com/grove-platform/github-copier/services" ) +// version is set at build time via -ldflags: +// +// go build -ldflags "-X main.version=v1.0.0" +// +// When not set (local dev builds), it defaults to "dev". +var version = "dev" + func main() { // Parse command line flags var envFile string @@ -24,9 +30,15 @@ func main() { flag.StringVar(&envFile, "env", "./configs/.env", "Path to environment file") flag.BoolVar(&dryRun, "dry-run", false, "Enable dry-run mode (no actual changes)") flag.BoolVar(&validateOnly, "validate", false, "Validate configuration and exit") + showVersion := flag.Bool("version", false, "Print version and exit") help := flag.Bool("help", false, "Show help") flag.Parse() + if *showVersion { + fmt.Println(version) + return + } + if *help { printHelp() return @@ -40,12 +52,13 @@ func main() { } // Load secrets from Secret Manager if not directly provided - if err := services.LoadWebhookSecret(config); err != nil { + ctx := context.Background() + if err := services.LoadWebhookSecret(ctx, config); err != nil { fmt.Printf("โŒ Error loading webhook secret: %v\n", err) os.Exit(1) } - if err := services.LoadMongoURI(config); err != nil { + if err := services.LoadMongoURI(ctx, config); err != nil { fmt.Printf("โŒ Error loading MongoDB URI: %v\n", err) os.Exit(1) } @@ -61,7 +74,7 @@ func main() { fmt.Printf("โŒ Failed to initialize services: %v\n", err) os.Exit(1) } - defer container.Close(context.Background()) + defer func() { _ = container.Close(context.Background()) }() // If validate-only mode, validate config and exit if validateOnly { @@ -74,13 +87,18 @@ func main() { } // Initialize Google Cloud logging - services.InitializeGoogleLogger() + services.InitializeLogger(config) defer services.CloseGoogleLogger() // Configure GitHub permissions - if err := services.ConfigurePermissions(); err != nil { - fmt.Printf("โŒ Failed to configure GitHub permissions: %v\n", err) - os.Exit(1) + if err := services.ConfigurePermissions(ctx, config); err != nil { + if config.DryRun { + services.LogWarning("GitHub authentication failed (non-fatal in dry-run mode)", "error", err) + fmt.Printf("โš ๏ธ GitHub auth skipped (dry-run): %v\n", err) + } else { + fmt.Printf("โŒ Failed to configure GitHub permissions: %v\n", err) + os.Exit(1) + } } // Print startup banner @@ -115,12 +133,14 @@ func printBanner(config *configs.Config, container *services.ServiceContainer) { fmt.Println("โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—") fmt.Println("โ•‘ GitHub Code Example Copier โ•‘") fmt.Println("โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ") + fmt.Printf("โ•‘ Version: %-48sโ•‘\n", version) fmt.Printf("โ•‘ Port: %-48sโ•‘\n", config.Port) fmt.Printf("โ•‘ Webhook Path: %-48sโ•‘\n", config.WebserverPath) - fmt.Printf("โ•‘ Config File: %-48sโ•‘\n", config.ConfigFile) + fmt.Printf("โ•‘ Config File: %-48sโ•‘\n", config.EffectiveConfigFile()) fmt.Printf("โ•‘ Dry Run: %-48vโ•‘\n", config.DryRun) fmt.Printf("โ•‘ Audit Log: %-48vโ•‘\n", config.AuditEnabled) fmt.Printf("โ•‘ Metrics: %-48vโ•‘\n", config.MetricsEnabled) + fmt.Printf("โ•‘ Slack: %-48vโ•‘\n", config.SlackEnabled) fmt.Println("โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•") fmt.Println() } @@ -140,14 +160,20 @@ func startWebServer(config *configs.Config, container *services.ServiceContainer handleWebhook(w, r, config, container) }) - // Health endpoint - mux.HandleFunc("/health", services.HealthHandler(container.FileStateService, container.StartTime)) + // Liveness probe โ€” lightweight, always 200 if process is running + mux.HandleFunc("/health", services.HealthHandler(container.StartTime, version)) + + // Readiness probe โ€” checks GitHub auth, MongoDB connectivity + mux.HandleFunc("/ready", services.ReadinessHandler(container)) // Metrics endpoint (if enabled) if config.MetricsEnabled { mux.HandleFunc("/metrics", services.MetricsHandler(container.MetricsCollector, container.FileStateService)) } + // Config diagnostic endpoint โ€” shows resolved config with secrets redacted + mux.HandleFunc("/config", services.ConfigDiagnosticHandler(container, version)) + // Info endpoint mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { @@ -155,11 +181,13 @@ func startWebServer(config *configs.Config, container *services.ServiceContainer return } w.Header().Set("Content-Type", "text/plain") - fmt.Fprintf(w, "GitHub Code Example Copier\n") - fmt.Fprintf(w, "Webhook endpoint: %s\n", config.WebserverPath) - fmt.Fprintf(w, "Health check: /health\n") + _, _ = fmt.Fprintf(w, "GitHub Code Example Copier %s\n", version) + _, _ = fmt.Fprintf(w, "Webhook endpoint: %s\n", config.WebserverPath) + _, _ = fmt.Fprintf(w, "Health check: /health\n") + _, _ = fmt.Fprintf(w, "Readiness check: /ready\n") + _, _ = fmt.Fprintf(w, "Config diagnostic: /config\n") if config.MetricsEnabled { - fmt.Fprintf(w, "Metrics: /metrics\n") + _, _ = fmt.Fprintf(w, "Metrics: /metrics\n") } }) @@ -178,7 +206,7 @@ func startWebServer(config *configs.Config, container *services.ServiceContainer // Start server in goroutine go func() { - services.LogInfo(fmt.Sprintf("Starting web server on port %s", port)) + services.LogInfo("Starting web server", "port", port) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { serverErr <- fmt.Errorf("server error: %w", err) } @@ -196,27 +224,27 @@ func startWebServer(config *configs.Config, container *services.ServiceContainer return err } case sig := <-sigChan: - log.Printf("Received signal %v, initiating graceful shutdown...", sig) + services.LogInfo("Received signal, initiating graceful shutdown", "signal", sig) } // Graceful shutdown with timeout shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - log.Println("Waiting for in-flight requests to complete...") + services.LogInfo("Waiting for in-flight requests to complete") if err := server.Shutdown(shutdownCtx); err != nil { - log.Printf("Server shutdown error: %v", err) + services.LogError("Server shutdown error", "error", err) } else { - log.Println("Server stopped accepting new connections") + services.LogInfo("Server stopped accepting new connections") } // Cleanup resources (flush audit logs, close connections) - log.Println("Cleaning up resources...") + services.LogInfo("Cleaning up resources") if err := container.Close(shutdownCtx); err != nil { - log.Printf("Cleanup error: %v", err) + services.LogError("Cleanup error", "error", err) } - log.Println("Shutdown complete") + services.LogInfo("Shutdown complete") return nil } diff --git a/app.yaml b/app.yaml deleted file mode 100644 index 6c78988..0000000 --- a/app.yaml +++ /dev/null @@ -1,44 +0,0 @@ -runtime: go -runtime_config: - operating_system: "ubuntu22" - runtime_version: "1.23" -env: flex - -includes: - - env.yaml - -# Automatic scaling configuration -# Keeps at least 1 instance running to avoid cold starts -automatic_scaling: - min_num_instances: 1 - max_num_instances: 10 - cool_down_period_sec: 120 - cpu_utilization: - target_utilization: 0.6 - -# Network configuration -network: - session_affinity: true - -# Health check configuration -# These ensure the app is ready before receiving traffic -liveness_check: - path: "/health" - check_interval_sec: 30 - timeout_sec: 4 - failure_threshold: 2 - success_threshold: 2 - -readiness_check: - path: "/health" - check_interval_sec: 5 - timeout_sec: 4 - failure_threshold: 2 - success_threshold: 2 - app_start_timeout_sec: 300 - -# Resources configuration -resources: - cpu: 1 - memory_gb: 2 - disk_size_gb: 10 diff --git a/cmd/config-validator/README.md b/cmd/config-validator/README.md index cfd0c52..664f391 100644 --- a/cmd/config-validator/README.md +++ b/cmd/config-validator/README.md @@ -42,9 +42,6 @@ Validate a configuration file. # Validate with verbose output ./config-validator validate -config .copier/workflows/config.yaml -v - -# Validate legacy JSON config -./config-validator validate -config config.json ``` **Output:** @@ -185,7 +182,7 @@ When files aren't matching your pattern: 1. **Get actual file paths from logs:** ```bash - grep "sample file path" logs/app.log + # Check stdout for "sample file path" entries ``` 2. **Test your pattern:** @@ -235,19 +232,6 @@ Before deploying a new configuration: -vars "lang=go,file=main.go" ``` -### Migrating from JSON to YAML - -```bash -# Validate -./config-validator validate -config workflow-config.yaml -v - -# Test patterns -./config-validator test-pattern \ - -type prefix \ - -pattern "examples/" \ - -file "examples/go/main.go" -``` - ## Exit Codes - `0` - Success diff --git a/cmd/config-validator/main.go b/cmd/config-validator/main.go index 9c3eee7..f0afccd 100644 --- a/cmd/config-validator/main.go +++ b/cmd/config-validator/main.go @@ -93,7 +93,7 @@ func printUsage() { } func validateConfig(configFile string, verbose bool) { - content, err := os.ReadFile(configFile) + content, err := os.ReadFile(configFile) // #nosec G304 -- CLI tool, path from user arg if err != nil { fmt.Printf("โŒ Error reading config file: %v\n", err) os.Exit(1) @@ -141,13 +141,18 @@ func testPattern(patternType, pattern, filePath string) { os.Exit(1) } - validator := services.NewConfigValidator() - result, err := validator.TestPattern(pt, pattern, filePath) - if err != nil { + sp := types.SourcePattern{ + Type: pt, + Pattern: pattern, + } + if err := sp.Validate(); err != nil { fmt.Printf("โŒ Pattern validation error: %v\n", err) os.Exit(1) } + matcher := services.NewPatternMatcher() + result := matcher.Match(filePath, sp) + if result.Matched { fmt.Println("โœ… Pattern matched!") if len(result.Variables) > 0 { @@ -174,8 +179,8 @@ func testTransform(source, template, varsStr string) { } } - validator := services.NewConfigValidator() - result, err := validator.TestTransform(source, template, variables) + transformer := services.NewPathTransformer() + result, err := transformer.Transform(source, template, variables) if err != nil { fmt.Printf("โŒ Transform error: %v\n", err) os.Exit(1) @@ -187,35 +192,73 @@ func testTransform(source, template, varsStr string) { } func initConfig(templateName, output string) { - // Simple workflow config template - template := `# Workflow Configuration -# This file defines workflows for copying code examples between repositories + templates := map[string]string{ + "basic": `# Workflow Configuration โ€” Basic (move transformation) +# This file defines workflows for copying code examples between repositories. workflows: - name: "example-workflow" - source: - repo: "mongodb/source-repo" - branch: "main" - path: "examples" + # source.repo and source.branch are inherited from the workflow config reference destination: - repo: "mongodb/dest-repo" + repo: "your-org/dest-repo" branch: "main" transformations: - move: from: "examples" to: "code-examples" commit_strategy: - type: "pr" + type: "pull_request" pr_title: "Update code examples" pr_body: "Automated update from source repository" -` +`, + "glob": `# Workflow Configuration โ€” Glob transformation +# Uses glob patterns for flexible file matching. + +workflows: + - name: "copy-go-examples" + destination: + repo: "your-org/dest-repo" + branch: "main" + transformations: + - glob: + pattern: "examples/**/*.go" + transform: "code/${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update Go examples" + auto_merge: false +`, + "regex": `# Workflow Configuration โ€” Regex transformation +# Uses regex with named capture groups for precise path control. + +workflows: + - name: "organize-by-language" + destination: + repo: "your-org/dest-repo" + branch: "main" + transformations: + - regex: + pattern: "^examples/(?P[^/]+)/(?P.+)$" + transform: "code/${lang}/${file}" + commit_strategy: + type: "pull_request" + pr_title: "Update ${lang} examples" + auto_merge: false +`, + } + + tmpl, ok := templates[templateName] + if !ok { + fmt.Printf("โŒ Unknown template: %s (must be basic, glob, or regex)\n", templateName) + os.Exit(1) + } - err := os.WriteFile(output, []byte(template), 0644) + err := os.WriteFile(output, []byte(tmpl), 0644) // #nosec G306 -- config template file, 0644 is intentional if err != nil { fmt.Printf("โŒ Error writing config file: %v\n", err) os.Exit(1) } - fmt.Printf("โœ… Created workflow config file: %s\n", output) + fmt.Printf("โœ… Created workflow config file: %s (template: %s)\n", output, templateName) fmt.Println("Edit this file to configure your workflows") } diff --git a/cmd/test-pem/README.md b/cmd/test-pem/README.md new file mode 100644 index 0000000..2fd02ce --- /dev/null +++ b/cmd/test-pem/README.md @@ -0,0 +1,61 @@ +# test-pem + +Verify a GitHub App PEM private key by generating a JWT and calling the GitHub `/app` endpoint. + +## Purpose + +Quickly confirm that: + +- The PEM file is valid and correctly formatted (PKCS#1 or PKCS#8) +- The App ID matches the key +- GitHub accepts the resulting JWT + +## Build + +```bash +go build -o test-pem ./cmd/test-pem +``` + +## Usage + +```bash +./test-pem +``` + +**Arguments:** + +| Argument | Description | +|-----------|------------------------------------| +| pem-file | Path to the `.pem` private key | +| app-id | GitHub App ID (numeric) | + +## Example + +```bash +$ ./test-pem github-app.pem 123456 +โœ“ Read PEM file: github-app.pem (1674 bytes) +โœ“ Parsed RSA private key (size: 2048 bits) +โœ“ Generated JWT for App ID 123456 + +Contacting GitHub API... +Status: 200 +โœ… Authentication successful! + +App info: +{"id":123456,"slug":"my-app","name":"My App",...} +``` + +## Troubleshooting + +| Error | Cause | Fix | +|-------|-------|-----| +| `Failed to read PEM file` | File not found or unreadable | Check path and permissions | +| `Failed to parse RSA private key` | Not a valid PEM key | Re-download from GitHub App settings | +| `HTTP 401` | App ID doesn't match key, or key is revoked | Verify App ID; regenerate key if needed | + +## Exit Codes + +| Code | Meaning | +|------|---------| +| 0 | Authentication succeeded | +| 1 | Any error (file, parse, network, auth) | diff --git a/cmd/test-pem/main.go b/cmd/test-pem/main.go new file mode 100644 index 0000000..91e157a --- /dev/null +++ b/cmd/test-pem/main.go @@ -0,0 +1,97 @@ +// test-pem verifies a GitHub App PEM private key by generating a JWT +// and calling the GitHub API's /app endpoint. This confirms the key +// is valid, correctly formatted, and matches the App ID. +package main + +import ( + "crypto/rsa" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +func main() { + if len(os.Args) < 3 { + fmt.Fprintf(os.Stderr, `test-pem โ€” verify a GitHub App PEM key + +Usage: test-pem + +Arguments: + pem-file Path to the .pem private key file + app-id GitHub App ID (numeric) + +Example: + test-pem github-app.pem 123456 +`) + os.Exit(1) + } + + pemPath := os.Args[1] + appID := os.Args[2] + + pemData, err := os.ReadFile(pemPath) // #nosec G304 G703 -- CLI tool, path from user arg + if err != nil { + fmt.Fprintf(os.Stderr, "โŒ Failed to read PEM file %q: %v\n", pemPath, err) // #nosec G705 -- CLI stderr, not web output + os.Exit(1) + } + fmt.Printf("โœ“ Read PEM file: %s (%d bytes)\n", pemPath, len(pemData)) + + privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(pemData) + if err != nil { + fmt.Fprintf(os.Stderr, "โŒ Failed to parse RSA private key: %v\n", err) + fmt.Fprintf(os.Stderr, " Ensure the file is a valid PKCS#1 or PKCS#8 PEM-encoded RSA key.\n") + os.Exit(1) + } + fmt.Printf("โœ“ Parsed RSA private key (size: %d bits)\n", privateKey.N.BitLen()) + + token, err := generateJWT(appID, privateKey) + if err != nil { + fmt.Fprintf(os.Stderr, "โŒ Failed to generate JWT: %v\n", err) + os.Exit(1) + } + fmt.Printf("โœ“ Generated JWT for App ID %s\n", appID) + + req, err := http.NewRequest("GET", "https://api.github.com/app", nil) + if err != nil { + fmt.Fprintf(os.Stderr, "โŒ Failed to create request: %v\n", err) + os.Exit(1) + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Accept", "application/vnd.github+json") + + fmt.Println("\nContacting GitHub API...") + resp, err := http.DefaultClient.Do(req) // #nosec G704 -- URL is hardcoded to api.github.com/app + if err != nil { + fmt.Fprintf(os.Stderr, "โŒ API request failed: %v\n", err) + os.Exit(1) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "โŒ Failed to read response: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Status: %d\n", resp.StatusCode) + if resp.StatusCode == http.StatusOK { + fmt.Printf("โœ… Authentication successful!\n\nApp info:\n%s\n", body) + } else { + fmt.Fprintf(os.Stderr, "โŒ Authentication failed (HTTP %d)\n%s\n", resp.StatusCode, body) // #nosec G705 -- CLI stderr, not web output + os.Exit(1) + } +} + +func generateJWT(appID string, pk *rsa.PrivateKey) (string, error) { + now := time.Now() + claims := jwt.MapClaims{ + "iat": now.Unix(), + "exp": now.Add(10 * time.Minute).Unix(), + "iss": appID, + } + return jwt.NewWithClaims(jwt.SigningMethodRS256, claims).SignedString(pk) +} diff --git a/cmd/test-webhook/README.md b/cmd/test-webhook/README.md index 10070bf..6903385 100644 --- a/cmd/test-webhook/README.md +++ b/cmd/test-webhook/README.md @@ -31,7 +31,7 @@ Send a pre-made example payload to the webhook endpoint. **Options:** - `-payload` - Path to JSON payload file (required) -- `-url` - Webhook URL (default: `http://localhost:8080/webhook`) +- `-url` - Webhook URL (default: `http://localhost:8080/events`) **Example:** @@ -41,7 +41,7 @@ Send a pre-made example payload to the webhook endpoint. # Use custom URL ./test-webhook -payload testdata/example-pr-merged.json \ - -url http://localhost:8080/webhook + -url http://localhost:8080/events ``` **Output:** @@ -68,7 +68,7 @@ Fetch real PR data from GitHub and send it to the webhook. - `-pr` - Pull request number (required) - `-owner` - Repository owner (required) - `-repo` - Repository name (required) -- `-url` - Webhook URL (default: `http://localhost:8080/webhook`) +- `-url` - Webhook URL (default: `http://localhost:8080/events`) **Environment Variables:** - `GITHUB_TOKEN` - GitHub personal access token (required for real PR data) @@ -84,7 +84,7 @@ export GITHUB_TOKEN=ghp_your_token_here # Test with custom URL ./test-webhook -pr 42 -owner mongodb -repo docs-code-examples \ - -url http://localhost:8080/webhook + -url http://localhost:8080/events ``` **Output:** @@ -108,13 +108,12 @@ Test your configuration locally before deploying: ```bash # 1. Start app in dry-run mode -DRY_RUN=true make run-local-quick +./scripts/run-local.sh # 2. In another terminal, send test webhook ./test-webhook -payload testdata/example-pr-merged.json -# 3. Check logs -tail -f logs/app.log +# 3. Check logs (JSON format on stdout via slog) ``` ### Testing Pattern Matching @@ -123,7 +122,7 @@ Test if your patterns match real PR files: ```bash # 1. Start app -make run-local-quick +./scripts/run-local.sh # 2. Send webhook with real PR data export GITHUB_TOKEN=ghp_... @@ -145,7 +144,7 @@ DRY_RUN=true ./github-copier & ./test-webhook -payload testdata/example-pr-merged.json # 3. Check logs for transformed paths -grep "transformed path" logs/app.log +# Check stdout for "transformed path" entries ``` ### Testing Slack Notifications @@ -176,7 +175,7 @@ export LOG_LEVEL=debug ./test-webhook -payload testdata/example-pr-merged.json # 3. Review detailed logs -grep "DEBUG" logs/app.log +# Run with LOG_LEVEL=debug for verbose output ``` ## Example Payloads @@ -247,7 +246,7 @@ export GITHUB_TOKEN=ghp_... ./test-webhook -pr 42 -owner myorg -repo myrepo # 5. Review logs -grep "matched" logs/app.log +# Check stdout for "matched" entries ``` ## Troubleshooting @@ -286,8 +285,8 @@ Response: 404 Not Found **Solution:** Check the webhook URL: ```bash -# Default is /webhook -./test-webhook -payload test.json -url http://localhost:8080/webhook +# Default is /events +./test-webhook -payload test.json -url http://localhost:8080/events ``` ### GitHub API Rate Limit @@ -364,37 +363,6 @@ chmod +x run-tests.sh ./run-tests.sh ``` -### Integration with CI/CD - -```yaml -# .github/workflows/test.yml -name: Test Examples Copier - -on: [push, pull_request] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.23.4 - - - name: Build - run: | - go build -o github-copier . - go build -o test-webhook ./cmd/test-webhook - - - name: Test - run: | - DRY_RUN=true ./github-copier & - sleep 2 - ./test-webhook -payload testdata/example-pr-merged.json -``` - ## Exit Codes - `0` - Success @@ -405,5 +373,4 @@ jobs: - [Webhook Testing Guide](../../docs/WEBHOOK-TESTING.md) - Comprehensive testing guide - [Local Testing](../../docs/LOCAL-TESTING.md) - Local development - [Test Payloads](../../testdata/README.md) - Example payloads -- [Quick Reference](../../QUICK-REFERENCE.md) - All commands diff --git a/cmd/test-webhook/main.go b/cmd/test-webhook/main.go index a84c3b8..3c6343a 100644 --- a/cmd/test-webhook/main.go +++ b/cmd/test-webhook/main.go @@ -12,7 +12,8 @@ import ( "net/http" "os" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" + "github.com/google/uuid" ) func main() { @@ -38,7 +39,7 @@ func main() { // Option 1: Use custom payload file if *payloadFile != "" { - payload, err = os.ReadFile(*payloadFile) + payload, err = os.ReadFile(*payloadFile) // #nosec G304 -- CLI tool, path from user flag if err != nil { fmt.Printf("Error reading payload file: %v\n", err) os.Exit(1) @@ -117,7 +118,7 @@ Examples: # Send to production with secret test-webhook -pr 123 -owner myorg -repo myrepo \ - -url https://myapp.appspot.com/events \ + -url https://your-service.run.app/events \ -secret "my-webhook-secret" Environment Variables: @@ -143,11 +144,11 @@ func fetchPRPayload(owner, repo string, prNumber int) ([]byte, error) { req.Header.Set("Accept", "application/vnd.github.v3+json") client := &http.Client{} - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- URL is hardcoded to api.github.com if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != 200 { body, _ := io.ReadAll(resp.Body) @@ -169,11 +170,11 @@ func fetchPRPayload(owner, repo string, prNumber int) ([]byte, error) { filesReq.Header.Set("Authorization", "Bearer "+token) filesReq.Header.Set("Accept", "application/vnd.github.v3+json") - filesResp, err := client.Do(filesReq) + filesResp, err := client.Do(filesReq) // #nosec G704 -- URL is hardcoded to api.github.com if err != nil { return nil, err } - defer filesResp.Body.Close() + defer func() { _ = filesResp.Body.Close() }() var files []map[string]interface{} if err := json.NewDecoder(filesResp.Body).Decode(&files); err != nil { @@ -185,9 +186,9 @@ func fetchPRPayload(owner, repo string, prNumber int) ([]byte, error) { "action": "closed", "number": prNumber, "pull_request": map[string]interface{}{ - "number": pr.GetNumber(), - "state": pr.GetState(), - "merged": pr.GetMerged(), + "number": pr.GetNumber(), + "state": pr.GetState(), + "merged": pr.GetMerged(), "merge_commit_sha": pr.GetMergeCommitSHA(), "head": map[string]interface{}{ "ref": pr.GetHead().GetRef(), @@ -223,9 +224,9 @@ func createExamplePayload() []byte { "action": "closed", "number": 42, "pull_request": map[string]interface{}{ - "number": 42, - "state": "closed", - "merged": true, + "number": 42, + "state": "closed", + "merged": true, "merge_commit_sha": "abc123def456", "head": map[string]interface{}{ "ref": "feature-branch", @@ -265,6 +266,11 @@ func sendWebhook(url string, payload []byte, secret string) error { req.Header.Set("Content-Type", "application/json") req.Header.Set("X-GitHub-Event", "pull_request") + // Add unique delivery ID for idempotency tracking + deliveryID := uuid.New().String() + req.Header.Set("X-GitHub-Delivery", deliveryID) + fmt.Printf("โœ“ Delivery ID: %s\n", deliveryID) + // Add signature if secret provided if secret != "" { signature := generateSignature(payload, secret) @@ -273,11 +279,11 @@ func sendWebhook(url string, payload []byte, secret string) error { } client := &http.Client{} - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- URL is user-provided target for local webhook testing if err != nil { return err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() body, _ := io.ReadAll(resp.Body) @@ -298,4 +304,3 @@ func generateSignature(payload []byte, secret string) string { mac.Write(payload) return "sha256=" + hex.EncodeToString(mac.Sum(nil)) } - diff --git a/configs/.env.local.example b/configs/.env.local.example index 01656ae..bc5ba7b 100644 --- a/configs/.env.local.example +++ b/configs/.env.local.example @@ -3,13 +3,13 @@ # To use this file, copy it to .env and edit with your values: # cp configs/.env.local configs/.env # source configs/.env -# ./examples-copier +# ./github-copier # Or use with make: # make run-dry # Or run directly: -# COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./examples-copier +# COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./github-copier # ============================================================================ # REQUIRED FOR LOCAL TESTING @@ -54,27 +54,36 @@ METRICS_ENABLED=true PORT=8080 # ============================================================================ -# GITHUB CREDENTIALS (REQUIRED FOR REAL PR TESTING) +# GITHUB APP CREDENTIALS (REQUIRED โ€” the app authenticates on startup) # ============================================================================ -# GitHub App ID (get from GitHub App settings) -# GITHUB_APP_ID=123456 +# GitHub App ID and Installation ID (get from GitHub App settings) +GITHUB_APP_ID= +INSTALLATION_ID= -# GitHub Installation ID (get from GitHub App installation) -# GITHUB_INSTALLATION_ID=789012 +# --- PEM Key: choose ONE of the options below --- -# For local testing, you can use a Personal Access Token instead -# Get from: https://github.com/settings/tokens -# Required scopes: repo (for reading PRs and files) -GITHUB_TOKEN= +# Option A: Fetch PEM from GCP Secret Manager (requires: gcloud auth application-default login) +# Just leave SKIP_SECRET_MANAGER unset. The app reads the secret named in PEM_NAME +# (default: CODE_COPIER_PEM) from your GOOGLE_CLOUD_PROJECT_ID. +# PEM_NAME=CODE_COPIER_PEM +# GOOGLE_CLOUD_PROJECT_ID=github-copy-code-examples + +# Option B: Provide the PEM key directly (no GCP access needed) +# Set SKIP_SECRET_MANAGER=true and provide the key via one of: +# SKIP_SECRET_MANAGER=true +# GITHUB_APP_PRIVATE_KEY_B64= +# Or inline (replace newlines with \n): +# GITHUB_APP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" # ============================================================================ -# OPTIONAL: GOOGLE CLOUD (ONLY IF TESTING CLOUD FEATURES) +# OPTIONAL: GITHUB TOKEN (for test-webhook CLI / test-with-pr.sh only) # ============================================================================ -# Only set these if you want to test with actual GCP -# GCP_PROJECT_ID=your-project-id -# PEM_KEY_NAME=projects/123/secrets/CODE_COPIER_PEM/versions/latest +# A PAT is NOT used by the app itself โ€” only by the test-webhook CLI tool +# to fetch real PR data from the GitHub API. +# Get from: https://github.com/settings/tokens (scope: repo) +GITHUB_TOKEN= # ============================================================================ # OPTIONAL: MONGODB AUDIT LOGGING (FOR LOCAL TESTING) @@ -108,7 +117,7 @@ SLACK_WEBHOOK_URL= SLACK_CHANNEL=#code-examples # Slack bot username (default: Examples Copier) -SLACK_USERNAME=Examples Copier +SLACK_USERNAME=GitHub Copier # Slack bot icon emoji (default: :robot_face:) SLACK_ICON_EMOJI=:robot_face: @@ -168,7 +177,7 @@ SLACK_ENABLED=false # # 6. TESTING: # - Use CONFIG_FILE="copier-config.example.yaml" for testing -# - Set WEBSERVER_PATH="/events" to match GitHub webhook +# - WEBSERVER_PATH defaults to "/events" to match GitHub webhook # - Use PORT="3000" or any available port # ============================================================================= diff --git a/configs/README.md b/configs/README.md index 5a4ab16..953e0fe 100644 --- a/configs/README.md +++ b/configs/README.md @@ -101,35 +101,13 @@ env_variables: --- -## Deployment Targets +## Deployment Target -This service supports **two Google Cloud deployment options**: +This service deploys to **Google Cloud Run**. -### App Engine (Flexible Environment) +### Cloud Run -**Config file:** `env.yaml` (with `env_variables:` wrapper) - -**Format:** -```yaml -env_variables: - GITHUB_APP_ID: "123456" - REPO_OWNER: "mongodb" -``` - -**Deploy:** -```bash -cp configs/env.yaml.production env.yaml -# Edit env.yaml with your values -gcloud app deploy app.yaml # Includes env.yaml automatically -``` - -**Best for:** Long-running services, always-on applications - ---- - -### Cloud Run (Serverless Containers) - -**Config file:** `env-cloudrun.yaml` (plain YAML, no wrapper) +**Config file:** `env-cloudrun.yaml` (plain YAML key-value pairs) **Format:** ```yaml @@ -140,13 +118,10 @@ REPO_OWNER: "mongodb" **Deploy:** ```bash cp configs/env.yaml.production env-cloudrun.yaml -# Remove the 'env_variables:' wrapper # Edit env-cloudrun.yaml with your values -gcloud run deploy github-copier --source . --env-vars-file=env-cloudrun.yaml +./scripts/deploy-cloudrun.sh ``` -**Best for:** Cost-effective, scales to zero, serverless - --- ## Usage Scenarios @@ -157,10 +132,10 @@ gcloud run deploy github-copier --source . --env-vars-file=env-cloudrun.yaml ```bash # Quick start -cp configs/env.yaml.production env.yaml -nano env.yaml # Update PROJECT_NUMBER and values +cp configs/env.yaml.production env-cloudrun.yaml +nano env-cloudrun.yaml # Update PROJECT_NUMBER and values ./scripts/grant-secret-access.sh -gcloud app deploy app.yaml # env.yaml is included via 'includes' directive +./scripts/deploy-cloudrun.sh ``` **Why:** Pre-configured with production best practices, minimal setup required. @@ -216,7 +191,7 @@ nano env.yaml # - Set custom defaults # Deploy -gcloud app deploy app.yaml # env.yaml is included via 'includes' directive +./scripts/deploy-cloudrun.sh ``` **Why:** Need advanced features not in production template. @@ -240,22 +215,9 @@ Or manually convert: GITHUB_APP_ID=123456 REPO_OWNER=mongodb -# env.yaml format: -env_variables: - GITHUB_APP_ID: "123456" - REPO_OWNER: "mongodb" -``` - -### Between App Engine and Cloud Run formats - -Use the format conversion script: - -```bash -# Convert App Engine โ†’ Cloud Run -./scripts/convert-env-format.sh to-cloudrun env.yaml env-cloudrun.yaml - -# Convert Cloud Run โ†’ App Engine -./scripts/convert-env-format.sh to-appengine env-cloudrun.yaml env.yaml +# Cloud Run YAML format: +GITHUB_APP_ID: "123456" +REPO_OWNER: "mongodb" ``` **Key difference:** @@ -337,7 +299,6 @@ github-copier/ ## See Also -- [CONFIGURATION-GUIDE.md](../docs/CONFIGURATION-GUIDE.md) - Variable validation and reference - [DEPLOYMENT.md](../docs/DEPLOYMENT.md) - Complete deployment guide - [LOCAL-TESTING.md](../docs/LOCAL-TESTING.md) - Local development guide diff --git a/configs/copier-config-examples/SOURCE-REPO-README.md b/configs/copier-config-examples/SOURCE-REPO-README.md index b48b54c..c1f5240 100644 --- a/configs/copier-config-examples/SOURCE-REPO-README.md +++ b/configs/copier-config-examples/SOURCE-REPO-README.md @@ -24,7 +24,7 @@ This directory contains workflow configurations for automatically copying code e **Where are the logs?** ```bash -gcloud app logs read --limit=100 | grep "your-repo-name" +gcloud run services logs read github-copier --limit=100 ``` ## Quick Start @@ -455,14 +455,8 @@ workflows: ### How do I view the logs? ```bash -# View recent logs -gcloud app logs read --limit=100 - -# Search for your PR -gcloud app logs read --limit=200 | grep "PR #123" - -# Search for your repo -gcloud app logs read --limit=200 | grep "your-repo-name" +# View recent logs (Cloud Run) +gcloud run services logs read github-copier --limit=100 ``` ### How do I test my configuration? diff --git a/configs/environment.go b/configs/environment.go index bf9ff1f..f9f5b01 100644 --- a/configs/environment.go +++ b/configs/environment.go @@ -44,11 +44,13 @@ type Config struct { MetricsEnabled bool // Slack notifications - SlackWebhookURL string - SlackChannel string - SlackUsername string - SlackIconEmoji string - SlackEnabled bool + SlackWebhookURL string + SlackChannel string + SlackUsername string + SlackIconEmoji string + SlackEnabled bool + SlackPlainText bool // Use plain text only (required for Workflow Builder webhooks) + SlackMessageVariable string // Variable name for Workflow Builder webhooks (default: "text") // GitHub API retry configuration GitHubAPIMaxRetries int @@ -57,71 +59,91 @@ type Config struct { // PR merge polling configuration PRMergePollMaxAttempts int PRMergePollInterval int // in milliseconds + + // Config cache TTL in seconds (0 = disabled) + ConfigCacheTTLSeconds int + + // Webhook background processing timeout in seconds (0 = no timeout) + WebhookProcessingTimeoutSeconds int + + // Webhook retry configuration + WebhookMaxRetries int // max retry attempts for failed webhook processing + WebhookRetryInitialDelay int // initial delay between retries in seconds (doubles each attempt) } const ( - EnvFile = "ENV" - Port = "PORT" - ConfigRepoName = "CONFIG_REPO_NAME" - ConfigRepoOwner = "CONFIG_REPO_OWNER" - AppId = "GITHUB_APP_ID" - AppClientId = "GITHUB_APP_CLIENT_ID" - InstallationId = "INSTALLATION_ID" - CommitterName = "COMMITTER_NAME" - CommitterEmail = "COMMITTER_EMAIL" - ConfigFile = "CONFIG_FILE" - MainConfigFile = "MAIN_CONFIG_FILE" - UseMainConfig = "USE_MAIN_CONFIG" - DeprecationFile = "DEPRECATION_FILE" - WebserverPath = "WEBSERVER_PATH" - ConfigRepoBranch = "CONFIG_REPO_BRANCH" - PEMKeyName = "PEM_NAME" - WebhookSecretName = "WEBHOOK_SECRET_NAME" - WebhookSecret = "WEBHOOK_SECRET" - CopierLogName = "COPIER_LOG_NAME" - GoogleCloudProjectId = "GOOGLE_CLOUD_PROJECT_ID" - DefaultRecursiveCopy = "DEFAULT_RECURSIVE_COPY" - DefaultPRMerge = "DEFAULT_PR_MERGE" - DefaultCommitMessage = "DEFAULT_COMMIT_MESSAGE" - DryRun = "DRY_RUN" - AuditEnabled = "AUDIT_ENABLED" - MongoURI = "MONGO_URI" - MongoURISecretName = "MONGO_URI_SECRET_NAME" - AuditDatabase = "AUDIT_DATABASE" - AuditCollection = "AUDIT_COLLECTION" - MetricsEnabled = "METRICS_ENABLED" - SlackWebhookURL = "SLACK_WEBHOOK_URL" - SlackChannel = "SLACK_CHANNEL" - SlackUsername = "SLACK_USERNAME" - SlackIconEmoji = "SLACK_ICON_EMOJI" - SlackEnabled = "SLACK_ENABLED" - GitHubAPIMaxRetries = "GITHUB_API_MAX_RETRIES" - GitHubAPIInitialRetryDelay = "GITHUB_API_INITIAL_RETRY_DELAY" - PRMergePollMaxAttempts = "PR_MERGE_POLL_MAX_ATTEMPTS" - PRMergePollInterval = "PR_MERGE_POLL_INTERVAL" + EnvFile = "ENV" + Port = "PORT" + ConfigRepoName = "CONFIG_REPO_NAME" + ConfigRepoOwner = "CONFIG_REPO_OWNER" + AppId = "GITHUB_APP_ID" + AppClientId = "GITHUB_APP_CLIENT_ID" + InstallationId = "INSTALLATION_ID" + CommitterName = "COMMITTER_NAME" + CommitterEmail = "COMMITTER_EMAIL" + ConfigFile = "CONFIG_FILE" + MainConfigFile = "MAIN_CONFIG_FILE" + UseMainConfig = "USE_MAIN_CONFIG" + DeprecationFile = "DEPRECATION_FILE" + WebserverPath = "WEBSERVER_PATH" + ConfigRepoBranch = "CONFIG_REPO_BRANCH" + PEMKeyName = "PEM_NAME" // #nosec G101 -- env var name, not a credential + WebhookSecretName = "WEBHOOK_SECRET_NAME" // #nosec G101 -- env var name, not a credential + WebhookSecret = "WEBHOOK_SECRET" // #nosec G101 -- env var name, not a credential + CopierLogName = "COPIER_LOG_NAME" + GoogleCloudProjectId = "GOOGLE_CLOUD_PROJECT_ID" + DefaultRecursiveCopy = "DEFAULT_RECURSIVE_COPY" + DefaultPRMerge = "DEFAULT_PR_MERGE" + DefaultCommitMessage = "DEFAULT_COMMIT_MESSAGE" + DryRun = "DRY_RUN" + AuditEnabled = "AUDIT_ENABLED" + MongoURI = "MONGO_URI" + MongoURISecretName = "MONGO_URI_SECRET_NAME" // #nosec G101 -- env var name, not a credential + AuditDatabase = "AUDIT_DATABASE" + AuditCollection = "AUDIT_COLLECTION" + MetricsEnabled = "METRICS_ENABLED" + SlackWebhookURL = "SLACK_WEBHOOK_URL" // #nosec G101 -- env var name, not a credential + SlackChannel = "SLACK_CHANNEL" + SlackUsername = "SLACK_USERNAME" + SlackIconEmoji = "SLACK_ICON_EMOJI" + SlackEnabled = "SLACK_ENABLED" + SlackPlainText = "SLACK_PLAIN_TEXT" // Use for Workflow Builder webhooks + SlackMessageVariable = "SLACK_MESSAGE_VARIABLE" // Variable name for Workflow Builder (default: "text") + GitHubAPIMaxRetries = "GITHUB_API_MAX_RETRIES" + GitHubAPIInitialRetryDelay = "GITHUB_API_INITIAL_RETRY_DELAY" + PRMergePollMaxAttempts = "PR_MERGE_POLL_MAX_ATTEMPTS" + PRMergePollInterval = "PR_MERGE_POLL_INTERVAL" + ConfigCacheTTLSeconds = "CONFIG_CACHE_TTL_SECONDS" + WebhookProcessingTimeoutSeconds = "WEBHOOK_PROCESSING_TIMEOUT_SECONDS" + WebhookMaxRetries = "WEBHOOK_MAX_RETRIES" + WebhookRetryInitialDelay = "WEBHOOK_RETRY_INITIAL_DELAY" //nolint:gosec // env var name, not a credential ) // NewConfig returns a new Config instance with default values func NewConfig() *Config { return &Config{ - Port: "8080", - CommitterName: "Copier Bot", - CommitterEmail: "bot@example.com", - ConfigFile: "copier-config.yaml", - DeprecationFile: "deprecated_examples.json", - WebserverPath: "/webhook", - ConfigRepoBranch: "main", // Default branch to fetch config file from - PEMKeyName: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest", // default secret name for GCP Secret Manager - WebhookSecretName: "projects/1054147886816/secrets/webhook-secret/versions/latest", // default webhook secret name for GCP Secret Manager - CopierLogName: "copy-copier-log", // default log name for logging to GCP - GoogleCloudProjectId: "github-copy-code-examples", // default project ID for logging to GCP - DefaultRecursiveCopy: true, // system-wide default for recursive copying that individual config entries can override. - DefaultPRMerge: false, // system-wide default for PR merge without review that individual config entries can override. - DefaultCommitMessage: "Automated PR with updated examples", // default commit message used when per-config commit_message is absent. - GitHubAPIMaxRetries: 3, // default number of retry attempts for GitHub API calls - GitHubAPIInitialRetryDelay: 500, // default initial retry delay in milliseconds (exponential backoff) - PRMergePollMaxAttempts: 20, // default max attempts to poll PR for mergeability (~10 seconds with 500ms interval) - PRMergePollInterval: 500, // default polling interval in milliseconds + Port: "8080", + CommitterName: "Copier Bot", + CommitterEmail: "bot@example.com", + ConfigFile: "copier-config.yaml", + DeprecationFile: "deprecated_examples.json", + WebserverPath: "/events", + ConfigRepoBranch: "main", // Default branch to fetch config file from + PEMKeyName: "CODE_COPIER_PEM", // short secret name; resolved to full path at runtime via SecretPath() + WebhookSecretName: "webhook-secret", // short secret name; resolved to full path at runtime via SecretPath() + CopierLogName: "copy-copier-log", // default log name for logging to GCP + GoogleCloudProjectId: "github-copy-code-examples", // default project ID for logging to GCP + DefaultRecursiveCopy: true, // system-wide default for recursive copying that individual config entries can override. + DefaultPRMerge: false, // system-wide default for PR merge without review that individual config entries can override. + DefaultCommitMessage: "Automated PR with updated examples", // default commit message used when per-config commit_message is absent. + GitHubAPIMaxRetries: 3, // default number of retry attempts for GitHub API calls + GitHubAPIInitialRetryDelay: 500, // default initial retry delay in milliseconds (exponential backoff) + PRMergePollMaxAttempts: 20, // default max attempts to poll PR for mergeability (~10 seconds with 500ms interval) + PRMergePollInterval: 500, // default polling interval in milliseconds + ConfigCacheTTLSeconds: 300, // default 5 minutes; set to 0 to disable caching + WebhookProcessingTimeoutSeconds: 300, // default 5 minutes; 0 = no timeout + WebhookMaxRetries: 2, // default 2 retries (3 total attempts) + WebhookRetryInitialDelay: 5, // default 5 seconds initial delay (doubles each retry) } } @@ -194,6 +216,8 @@ func LoadEnvironment(envFile string) (*Config, error) { config.SlackUsername = getEnvWithDefault(SlackUsername, "Examples Copier") config.SlackIconEmoji = getEnvWithDefault(SlackIconEmoji, ":robot_face:") config.SlackEnabled = getBoolEnvWithDefault(SlackEnabled, config.SlackWebhookURL != "") + config.SlackPlainText = getBoolEnvWithDefault(SlackPlainText, false) + config.SlackMessageVariable = getEnvWithDefault(SlackMessageVariable, "text") // GitHub API retry configuration config.GitHubAPIMaxRetries = getIntEnvWithDefault(GitHubAPIMaxRetries, config.GitHubAPIMaxRetries) @@ -203,27 +227,13 @@ func LoadEnvironment(envFile string) (*Config, error) { config.PRMergePollMaxAttempts = getIntEnvWithDefault(PRMergePollMaxAttempts, config.PRMergePollMaxAttempts) config.PRMergePollInterval = getIntEnvWithDefault(PRMergePollInterval, config.PRMergePollInterval) - // Export resolved values back into environment so downstream os.Getenv sees defaults - _ = os.Setenv(Port, config.Port) - _ = os.Setenv(ConfigRepoName, config.ConfigRepoName) - _ = os.Setenv(ConfigRepoOwner, config.ConfigRepoOwner) - _ = os.Setenv(AppId, config.AppId) - _ = os.Setenv(AppClientId, config.AppClientId) - _ = os.Setenv(InstallationId, config.InstallationId) - _ = os.Setenv(CommitterName, config.CommitterName) - _ = os.Setenv(CommitterEmail, config.CommitterEmail) - _ = os.Setenv(ConfigFile, config.ConfigFile) - _ = os.Setenv(MainConfigFile, config.MainConfigFile) - _ = os.Setenv(UseMainConfig, fmt.Sprintf("%t", config.UseMainConfig)) - _ = os.Setenv(DeprecationFile, config.DeprecationFile) - _ = os.Setenv(WebserverPath, config.WebserverPath) - _ = os.Setenv(ConfigRepoBranch, config.ConfigRepoBranch) - _ = os.Setenv(PEMKeyName, config.PEMKeyName) - _ = os.Setenv(CopierLogName, config.CopierLogName) - _ = os.Setenv(GoogleCloudProjectId, config.GoogleCloudProjectId) - _ = os.Setenv(DefaultRecursiveCopy, fmt.Sprintf("%t", config.DefaultRecursiveCopy)) - _ = os.Setenv(DefaultPRMerge, fmt.Sprintf("%t", config.DefaultPRMerge)) - _ = os.Setenv(DefaultCommitMessage, config.DefaultCommitMessage) + // Config cache + config.ConfigCacheTTLSeconds = getIntEnvWithDefault(ConfigCacheTTLSeconds, config.ConfigCacheTTLSeconds) + + // Webhook processing + config.WebhookProcessingTimeoutSeconds = getIntEnvWithDefault(WebhookProcessingTimeoutSeconds, config.WebhookProcessingTimeoutSeconds) + config.WebhookMaxRetries = getIntEnvWithDefault(WebhookMaxRetries, config.WebhookMaxRetries) + config.WebhookRetryInitialDelay = getIntEnvWithDefault(WebhookRetryInitialDelay, config.WebhookRetryInitialDelay) if err := validateConfig(config); err != nil { return nil, err @@ -232,6 +242,25 @@ func LoadEnvironment(envFile string) (*Config, error) { return config, nil } +// EffectiveConfigFile returns the config file path that the app will actually use. +// If MainConfigFile is set (USE_MAIN_CONFIG=true), it takes precedence over ConfigFile. +func (c *Config) EffectiveConfigFile() string { + if c.UseMainConfig && c.MainConfigFile != "" { + return c.MainConfigFile + } + return c.ConfigFile +} + +// SecretPath resolves a secret name to a fully-qualified GCP Secret Manager resource path. +// If the name already contains "projects/", it is returned as-is (for backward compatibility). +// Otherwise, it builds the full path using the configured GoogleCloudProjectId. +func (c *Config) SecretPath(secretName string) string { + if strings.HasPrefix(secretName, "projects/") { + return secretName + } + return fmt.Sprintf("projects/%s/secrets/%s/versions/latest", c.GoogleCloudProjectId, secretName) +} + // getEnvWithDefault returns the environment variable value or default if not set func getEnvWithDefault(key, defaultValue string) string { value := os.Getenv(key) @@ -284,5 +313,15 @@ func validateConfig(config *Config) error { return fmt.Errorf("missing required environment variables: %s", strings.Join(missingVars, ", ")) } + // Warn if webhook secret is not configured. + // In production, webhook signature verification should always be enabled + // to prevent unauthorized requests from being processed. + env := getEnvWithDefault(EnvFile, "") + if config.WebhookSecret == "" && config.WebhookSecretName == "" { + if env == "production" || env == "prod" { + return fmt.Errorf("WEBHOOK_SECRET or WEBHOOK_SECRET_NAME is required in production to enable webhook signature verification") + } + } + return nil } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 32c29cb..a9fe4fe 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -481,11 +481,80 @@ if file.Status == "DELETED" { - Deprecation queue: `{repo}:{targetPath}` - Thread-safe operations with mutex locks -#### 5. Batch Operations -- All files for same target are batched together -- Single commit per target repository -- Single PR per target (if using PR strategy) -- Deprecation file updated once with all entries +#### 5. Target Repo Batching + +When a webhook is processed, matched files from multiple workflows are grouped by a composite key of **(destination repo, branch, commit strategy)**. Workflows that share all three values are batched into a single write operation; workflows that differ on any dimension produce separate operations. + +**How it works:** + +1. All workflows are processed in order; each matched file is queued by `(repo, branch, strategy)` +2. After all workflows finish, files sharing the same key are combined +3. One commit or PR is created per unique key (not per workflow) + +**Implications:** + +| Scenario | Behavior | +|----------|----------| +| 2 workflows โ†’ same repo, both `direct` | One direct commit with all files | +| 2 workflows โ†’ same repo, both `pull_request` | One PR with all files | +| 1 `direct` + 1 `pull_request` โ†’ same repo | **Two separate operations** โ€” one direct commit and one PR | +| 2 workflows โ†’ different repos | Separate commit/PR per repo (independent) | + +**What gets merged when workflows share a key:** + +- **Files**: All matched files from all workflows with the same key are combined into one commit tree +- **Commit message**: Uses the last workflow's `commit_message` +- **PR title/body**: Uses the last workflow's `pr_title` and `pr_body` +- **Auto-merge**: Uses the last workflow's `auto_merge` setting +- **PR template**: Fetched once if any workflow sets `use_pr_template: true` + +> **Note:** At config load time the app logs a warning when workflows target the same `(repo, branch)` with different commit strategies, so operators are aware that multiple operations will be created. + +**Example โ€” same strategy (batched):** + +```yaml +# These two workflows share repo, branch, AND strategy โ†’ batched into one PR: +workflows: + - name: "go-examples" + destination: { repo: "org/docs", branch: "main" } + commit_strategy: + type: "pull_request" + pr_title: "Update Go examples" + + - name: "python-examples" + destination: { repo: "org/docs", branch: "main" } + commit_strategy: + type: "pull_request" + pr_title: "Update Python examples" +``` + +Result: **One PR** with files from both workflows. The PR title will be "Update Python examples" (last workflow wins for metadata). + +**Example โ€” different strategies (separate operations):** + +```yaml +# These two workflows share repo and branch but differ on strategy โ†’ two operations: +workflows: + - name: "go-examples" + destination: { repo: "org/docs", branch: "main" } + commit_strategy: + type: "direct" + + - name: "python-examples" + destination: { repo: "org/docs", branch: "main" } + commit_strategy: + type: "pull_request" + pr_title: "Update Python examples" +``` + +Result: **One direct commit** (Go files) and **one PR** (Python files). + +**Design rationale:** + +- Avoids multiple PRs or commits to the same branch from a single source event *when strategies match* +- Respects each workflow's intended commit strategy when they differ +- Reduces noise in target repos while remaining predictable +- A config-time warning alerts operators to mixed-strategy destinations ## Configuration Examples @@ -581,19 +650,37 @@ auto_merge: false The application is designed for concurrent operations: +- **TokenManager**: Thread-safe token state via `sync.RWMutex` - **FileStateService**: Thread-safe with `sync.RWMutex` +- **DeliveryTracker**: Thread-safe webhook deduplication via `sync.Mutex` - **MetricsCollector**: Thread-safe counters - **AuditLogger**: Thread-safe MongoDB operations - **ServiceContainer**: Immutable after initialization ## Error Handling +- Sentinel errors in `services/errors.go` (`ErrRateLimited`, `ErrNotFound`, etc.) +- All errors wrapped with `%w` for `errors.Is()`/`errors.As()` compatibility - Context-aware cancellation support - Graceful degradation (audit logging optional) -- Detailed error logging with full context +- Structured error logging with full context via `log/slog` - Metrics tracking for failed operations - No-op implementations for optional features +## Rate Limit Handling + +The `RateLimitTransport` (`services/rate_limit.go`) wraps the HTTP transport to automatically: +- Detect 403/429 responses with `X-RateLimit-Remaining: 0` or `Retry-After` headers +- Wait and retry with appropriate backoff +- Log rate limit events with structured context + +## Webhook Idempotency + +The `DeliveryTracker` (`services/delivery_tracker.go`) prevents duplicate processing: +- Tracks `X-GitHub-Delivery` header from each webhook +- TTL-based cleanup to prevent unbounded memory growth +- Returns 200 OK for already-processed deliveries + ## Performance Considerations - **Batch Operations**: Multiple files committed in single operation @@ -639,8 +726,9 @@ METRICS_ENABLED: "true" **Health Monitoring:** - `/health` endpoint for liveness checks +- `/ready` endpoint for readiness probes (checks GitHub auth, rate limits, MongoDB) - `/metrics` endpoint for monitoring -- Structured logs for analysis +- Structured JSON logs via `log/slog` for analysis ## Future Enhancements @@ -649,6 +737,4 @@ Potential improvements: 1. **Automatic Cleanup PRs** - Create PRs to remove deprecated files from targets 2. **Expiration Dates** - Auto-remove deprecation entries after X days 3. **Config Validation CLI** - Enhanced validation tool -4. **Retry Logic** - Automatic retry for failed GitHub API calls -5. **Rate Limiting** - Respect GitHub API rate limits diff --git a/docs/CONFIG-REFERENCE.md b/docs/CONFIG-REFERENCE.md new file mode 100644 index 0000000..41895fb --- /dev/null +++ b/docs/CONFIG-REFERENCE.md @@ -0,0 +1,387 @@ +# Configuration Reference + +Complete reference for all github-copier configuration options: environment variables (app config) and YAML schema (workflow config). + +## Table of Contents + +- [Environment Variables](#environment-variables) + - [Required](#required) + - [Config Repository](#config-repository) + - [GitHub App Authentication](#github-app-authentication) + - [Webhook](#webhook) + - [Commit Defaults](#commit-defaults) + - [Dry Run / Debug](#dry-run--debug) + - [Slack Notifications](#slack-notifications) + - [Audit Logging](#audit-logging) + - [GitHub API Tuning](#github-api-tuning) + - [Webhook Processing](#webhook-processing) + - [Google Cloud](#google-cloud) +- [Workflow YAML Schema](#workflow-yaml-schema) + - [Main Config](#main-config) + - [Workflow Config](#workflow-config) + - [Workflow](#workflow) + - [Source / Destination](#source--destination) + - [Transformations](#transformations) + - [Commit Strategy](#commit-strategy) + - [Defaults](#defaults) + - [$ref Support](#ref-support) + +--- + +## Environment Variables + +Set via `.env` files, `env-cloudrun.yaml`, or process environment. + +### Required + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `CONFIG_REPO_NAME` | string | โ€” | Repository containing the workflow config files. | +| `CONFIG_REPO_OWNER` | string | โ€” | Owner (org or user) of the config repository. | +| `GITHUB_APP_ID` | string | โ€” | GitHub App ID for authentication. | +| `INSTALLATION_ID` | string | โ€” | GitHub App installation ID. | + +### Config Repository + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `CONFIG_REPO_BRANCH` | string | `main` | Branch to fetch config files from. | +| `CONFIG_FILE` | string | `copier-config.yaml` | Legacy single-file config path. Ignored when `USE_MAIN_CONFIG=true`. | +| `MAIN_CONFIG_FILE` | string | โ€” | Path to the main config file (e.g. `.copier/main.yaml`). Enables multi-file config. | +| `USE_MAIN_CONFIG` | bool | `true` if `MAIN_CONFIG_FILE` is set | Whether to use the main config format. | +| `DEPRECATION_FILE` | string | `deprecated_examples.json` | Path to the deprecation tracking file. | +| `CONFIG_CACHE_TTL_SECONDS` | int | `300` | How long (seconds) to cache resolved workflow configs. `0` disables caching. | + +### GitHub App Authentication + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `GITHUB_APP_CLIENT_ID` | string | โ€” | GitHub App client ID (optional). | +| `PEM_NAME` | string | `CODE_COPIER_PEM` | GCP Secret Manager secret name for the PEM key. Resolved to full path via `SecretPath()`. | +| `SKIP_SECRET_MANAGER` | bool | `false` | If `true`, skip GCP Secret Manager and use `GITHUB_APP_PRIVATE_KEY_B64` instead. | +| `GITHUB_APP_PRIVATE_KEY_B64` | string | โ€” | Base64-encoded PEM private key. Used when `SKIP_SECRET_MANAGER=true`. | + +### Webhook + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `PORT` | string | `8080` | HTTP server listen port. | +| `WEBSERVER_PATH` | string | `/events` | Path for the webhook endpoint. | +| `WEBHOOK_SECRET` | string | โ€” | Webhook HMAC secret for signature verification. Leave empty to disable (not recommended in production). | +| `WEBHOOK_SECRET_NAME` | string | `webhook-secret` | GCP Secret Manager name for the webhook secret. | + +### Commit Defaults + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `COMMITTER_NAME` | string | `Copier Bot` | Git committer name for generated commits. | +| `COMMITTER_EMAIL` | string | `bot@example.com` | Git committer email. | +| `DEFAULT_COMMIT_MESSAGE` | string | `Automated PR with updated examples` | Fallback commit message when per-workflow message is absent. | +| `DEFAULT_RECURSIVE_COPY` | bool | `true` | System-wide default for recursive copying (individual configs can override). | +| `DEFAULT_PR_MERGE` | bool | `false` | System-wide default for PR auto-merge without review. | + +### Dry Run / Debug + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `DRY_RUN` | bool | `false` | Process webhooks but don't make actual commits/PRs. | +| `LOG_LEVEL` | string | โ€” | Log level (`debug`, `info`, `warn`, `error`). | +| `COPIER_DEBUG` | bool | `false` | Enable extra debug info in logs. | +| `COPIER_DISABLE_CLOUD_LOGGING` | bool | `false` | Use stdout instead of GCP Cloud Logging. | +| `METRICS_ENABLED` | bool | `true` | Enable the `/metrics` endpoint. | + +### Slack Notifications + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `SLACK_WEBHOOK_URL` | string | โ€” | Slack incoming webhook URL. Leave empty to disable notifications. | +| `SLACK_CHANNEL` | string | `#code-examples` | Slack channel for notifications. | +| `SLACK_USERNAME` | string | `Examples Copier` | Bot username shown in Slack. | +| `SLACK_ICON_EMOJI` | string | `:robot_face:` | Bot icon emoji. | +| `SLACK_ENABLED` | bool | `true` if URL is set | Explicitly enable/disable Slack. | + +### Audit Logging + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `AUDIT_ENABLED` | bool | `false` | Enable MongoDB audit logging. | +| `MONGO_URI` | string | โ€” | MongoDB connection URI. | +| `MONGO_URI_SECRET_NAME` | string | โ€” | GCP Secret Manager name for the MongoDB URI. | +| `AUDIT_DATABASE` | string | `copier_audit` | MongoDB database name. | +| `AUDIT_COLLECTION` | string | `events` | MongoDB collection name. | + +### GitHub API Tuning + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `GITHUB_API_MAX_RETRIES` | int | `3` | Number of retry attempts for GitHub API calls (rate limits, transient errors). | +| `GITHUB_API_INITIAL_RETRY_DELAY` | int | `500` | Initial retry delay in **milliseconds** (doubles each attempt). | +| `PR_MERGE_POLL_MAX_ATTEMPTS` | int | `20` | Max attempts to poll PR mergeability after creation. | +| `PR_MERGE_POLL_INTERVAL` | int | `500` | Polling interval in **milliseconds**. | + +### Webhook Processing + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `WEBHOOK_PROCESSING_TIMEOUT_SECONDS` | int | `300` | Timeout for the background webhook processing goroutine. `0` disables the timeout. | +| `WEBHOOK_MAX_RETRIES` | int | `2` | Max retry attempts for failed webhook processing (total attempts = retries + 1). | +| `WEBHOOK_RETRY_INITIAL_DELAY` | int | `5` | Initial delay between retries in **seconds** (doubles each attempt). | + +### Google Cloud + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `GOOGLE_CLOUD_PROJECT_ID` | string | `github-copy-code-examples` | GCP project ID for Cloud Logging and Secret Manager. | +| `COPIER_LOG_NAME` | string | `copy-copier-log` | Log name in GCP Cloud Logging. | + +--- + +## Workflow YAML Schema + +### Main Config + +The main config file (e.g. `.copier/main.yaml`) references one or more workflow config files. + +```yaml +# Optional global defaults applied to all workflows +defaults: + commit_strategy: + type: pull_request + commit_message: "chore: sync from source" + exclude: + - "*.test.go" + +# List of workflow config sources +workflow_configs: + # Load from a file in the same repo + - source: local + path: .copier/workflows/docs.yaml + + # Load from a different repo + - source: repo + repo: org/other-repo + path: copier-config.yaml + branch: main # default: main + + # Inline workflows directly + - source: inline + workflows: + - name: inline-example + source: + repo: org/source + destination: + repo: org/target + transformations: + - copy: + from: README.md + to: README.md + + # Disable a config without removing it + - source: local + path: .copier/workflows/experimental.yaml + enabled: false +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `defaults` | [Defaults](#defaults) | No | Global defaults inherited by all workflows. | +| `workflow_configs` | array | Yes | List of workflow config references. | +| `workflow_configs[].source` | string | Yes | `local`, `repo`, or `inline`. | +| `workflow_configs[].path` | string | Yes (local/repo) | Path to the config file. | +| `workflow_configs[].repo` | string | Yes (repo) | `owner/name` of the repository. | +| `workflow_configs[].branch` | string | No | Branch to fetch from. Default: `main`. | +| `workflow_configs[].enabled` | bool | No | Set `false` to skip this config. Default: `true`. | +| `workflow_configs[].workflows` | array | Yes (inline) | Inline workflow definitions. | + +### Workflow Config + +A workflow config file (referenced by the main config) contains one or more workflows and optional local defaults. + +```yaml +defaults: + commit_strategy: + type: direct + +workflows: + - name: sync-examples + source: + repo: org/source-repo + branch: main + destination: + repo: org/target-repo + branch: main + transformations: + - copy: + from: examples/hello.go + to: examples/hello.go +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `defaults` | [Defaults](#defaults) | No | Local defaults for workflows in this file. | +| `workflows` | array | Yes | One or more workflow definitions. | + +### Workflow + +```yaml +- name: my-workflow # Required: unique name + source: + repo: org/source-repo # Required: owner/name + branch: main # Optional, default: main + installation_id: "12345" # Optional: override default installation + destination: + repo: org/target-repo # Required: owner/name + branch: main # Optional, default: main + installation_id: "67890" # Optional: override + transformations: # Required: at least one + - copy: + from: docs/guide.md + to: docs/guide.md + exclude: # Optional: regex patterns + - ".*_test\\.go$" + commit_strategy: # Optional (inherits from defaults) + type: pull_request + pr_title: "Copier: sync docs" + deprecation_check: # Optional + enabled: true + file: deprecated_examples.json +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique workflow name (used in logs and metrics). | +| `source` | [Source](#source--destination) | Yes | Where to read files from. | +| `destination` | [Source](#source--destination) | Yes | Where to write files to. | +| `transformations` | array | Yes | List of file transformations. At least one required. | +| `exclude` | string[] | No | Regex patterns for files to skip. | +| `commit_strategy` | [CommitStrategy](#commit-strategy) | No | How to deliver changes. Inherits from defaults. | +| `deprecation_check` | object | No | Track file deletions. | + +### Source / Destination + +```yaml +source: + repo: org/repo-name # Required + branch: main # Optional, default: main + installation_id: "12345" # Optional: GitHub App installation override +``` + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `repo` | string | Yes | โ€” | `owner/name` format. | +| `branch` | string | No | `main` | Branch to read from / write to. | +| `installation_id` | string | No | โ€” | Override the default GitHub App installation ID. | + +### Transformations + +Each transformation matches source files and maps them to destination paths. Exactly one type per entry. + +#### `copy` โ€” Single file copy + +```yaml +- copy: + from: path/to/source.go # Exact source path + to: path/to/dest.go # Exact destination path +``` + +#### `move` โ€” Directory move (recursive) + +```yaml +- move: + from: examples/go/ # Source directory prefix + to: sdk/go/ # Destination directory prefix +``` + +Files under `from` are placed under `to` preserving relative structure. + +#### `glob` โ€” Glob pattern with template + +```yaml +- glob: + pattern: "examples/**/*.go" # Glob pattern + transform: "sdk/${relative_path}" # Template with variables +``` + +**Built-in variables:** `${path}`, `${filename}`, `${dir}`, `${ext}`, `${relative_path}`. + +#### `regex` โ€” Regex with named capture groups + +```yaml +- regex: + pattern: "^examples/(?P[^/]+)/(?P.+)$" + transform: "sdk/${lang}/${file}" +``` + +Named groups (`?P`) become template variables. Built-in variables are also available. + +### Commit Strategy + +Controls how changes are delivered to the destination repository. + +```yaml +commit_strategy: + type: pull_request # "direct" or "pull_request" + commit_message: "chore: sync from source" + pr_title: "Copier: update examples" + pr_body: "Automated sync from source repo." + use_pr_template: false # Fetch PR template from target repo + auto_merge: false # Auto-merge the PR (requires branch protection) +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `type` | string | `pull_request` | `direct` (commit to branch) or `pull_request` (open PR). | +| `commit_message` | string | env `DEFAULT_COMMIT_MESSAGE` | Git commit message. | +| `pr_title` | string | same as `commit_message` | PR title (only for `pull_request` type). | +| `pr_body` | string | โ€” | PR body text. | +| `use_pr_template` | bool | `false` | If true, fetch `PULL_REQUEST_TEMPLATE.md` from target repo. | +| `auto_merge` | bool | `false` | If true, enable auto-merge on the created PR. | + +**Important:** If multiple workflows target the same `(repo, branch)` with the same strategy, their files are **batched into a single commit/PR**. The last workflow's metadata (title, body, message) wins. A warning is logged at config load time if workflows use different strategies for the same target. + +### Defaults + +Defaults can be set at three levels (highest to lowest priority): +1. **Workflow-level** โ€” set directly on the workflow. +2. **Config-file-level** โ€” set in the workflow config file's `defaults` block. +3. **Global-level** โ€” set in the main config file's `defaults` block. + +```yaml +defaults: + commit_strategy: + type: direct + commit_message: "chore: automated sync" + deprecation_check: + enabled: true + file: deprecated_examples.json + exclude: + - ".*_test\\.go$" + - "\\.github/.*" +``` + +| Field | Type | Description | +|-------|------|-------------| +| `commit_strategy` | [CommitStrategy](#commit-strategy) | Default commit strategy for workflows that don't specify one. | +| `deprecation_check` | object | Default deprecation settings. | +| `exclude` | string[] | Default exclude patterns (regex). | + +### $ref Support + +Workflow fields (`transformations`, `commit_strategy`, `exclude`) support `$ref` to reference shared definitions in other files: + +```yaml +workflows: + - name: example + source: + repo: org/source + destination: + repo: org/target + transformations: + $ref: shared/transforms.yaml + commit_strategy: + $ref: shared/pr-strategy.yaml + exclude: + $ref: shared/excludes.yaml +``` + +The referenced file is loaded relative to the config repository root. diff --git a/docs/DEBUG-LOGGING.md b/docs/DEBUG-LOGGING.md deleted file mode 100644 index 05f3682..0000000 --- a/docs/DEBUG-LOGGING.md +++ /dev/null @@ -1,376 +0,0 @@ -# Debug Logging Guide - -This guide explains how to enable and use debug logging in the Examples Copier application. - -## Overview - -The Examples Copier supports configurable logging levels to help with development, troubleshooting, and debugging. By default, the application logs at the INFO level, but you can enable DEBUG logging for more verbose output. - -## Environment Variables - -### LOG_LEVEL - -**Purpose:** Set the logging level for the application - -**Values:** -- `info` (default) - Standard operational logs -- `debug` - Verbose debug logs with detailed operation information - -**Example:** -```bash -LOG_LEVEL="debug" -``` - -### COPIER_DEBUG - -**Purpose:** Alternative way to enable debug mode - -**Values:** -- `true` - Enable debug logging -- `false` (default) - Standard logging - -**Example:** -```bash -COPIER_DEBUG="true" -``` - -**Note:** Either `LOG_LEVEL="debug"` OR `COPIER_DEBUG="true"` will enable debug logging. You only need to set one. - -### COPIER_DISABLE_CLOUD_LOGGING - -**Purpose:** Disable Google Cloud Logging (useful for local development) - -**Values:** -- `true` - Disable GCP logging, only log to stdout -- `false` (default) - Enable GCP logging if configured - -**Example:** -```bash -COPIER_DISABLE_CLOUD_LOGGING="true" -``` - -**Use case:** When developing locally, you may not want logs sent to Google Cloud. This flag keeps all logs local. - ---- - -## How It Works - -### Code Implementation - -The logging system is implemented in `services/logger.go`: - -```go -// LogDebug writes debug logs only when LOG_LEVEL=debug or COPIER_DEBUG=true. -func LogDebug(message string) { - if !isDebugEnabled() { - return - } - // Mirror to GCP as info if available, plus prefix to stdout - if googleInfoLogger != nil && gcpLoggingEnabled { - googleInfoLogger.Println("[DEBUG] " + message) - } - log.Println("[DEBUG] " + message) -} - -func isDebugEnabled() bool { - if strings.EqualFold(os.Getenv("LOG_LEVEL"), "debug") { - return true - } - return strings.EqualFold(os.Getenv("COPIER_DEBUG"), "true") -} - -func isCloudLoggingDisabled() bool { - return strings.EqualFold(os.Getenv("COPIER_DISABLE_CLOUD_LOGGING"), "true") -} -``` - -### Log Levels - -The application supports the following log levels: - -| Level | Function | When to Use | Example | -|-------|----------|-------------|---------| -| **DEBUG** | `LogDebug()` | Detailed operation logs, file matching, API calls | `[DEBUG] Matched file: src/example.js` | -| **INFO** | `LogInfo()` | Standard operational logs | `[INFO] Processing webhook event` | -| **WARN** | `LogWarning()` | Warning conditions | `[WARN] File not found, skipping` | -| **ERROR** | `LogError()` | Error conditions | `[ERROR] Failed to create PR` | -| **CRITICAL** | `LogCritical()` | Critical failures | `[CRITICAL] Database connection failed` | - ---- - -## Usage Examples - -### Local Development with Debug Logging - -**Using .env file:** -```bash -# configs/.env -LOG_LEVEL="debug" -COPIER_DISABLE_CLOUD_LOGGING="true" -DRY_RUN="true" -``` - -**Using environment variables:** -```bash -export LOG_LEVEL=debug -export COPIER_DISABLE_CLOUD_LOGGING=true -export DRY_RUN=true -go run app.go -``` - -### Production with Debug Logging (Temporary) - -**env.yaml:** -```yaml -env_variables: - LOG_LEVEL: "debug" - # ... other variables -``` - -**Deploy:** -```bash -gcloud app deploy app.yaml # env.yaml is included via 'includes' directive -``` - -**Important:** Remember to disable debug logging after troubleshooting to reduce log volume and costs. - -### Local Development without Cloud Logging - -```bash -# configs/.env -COPIER_DISABLE_CLOUD_LOGGING="true" -``` - -This keeps all logs local (stdout only), which is faster and doesn't require GCP credentials. - ---- - -## What Gets Logged at DEBUG Level? - -When debug logging is enabled, you'll see additional information about: - -### 1. **File Matching Operations** -``` -[DEBUG] Checking pattern: src/**/*.js -[DEBUG] Matched file: src/examples/example1.js -[DEBUG] Excluded file: src/tests/test.js (matches exclude pattern) -``` - -### 2. **GitHub API Calls** -``` -[DEBUG] Fetching file from GitHub: src/example.js -[DEBUG] Creating PR for target repo: mongodb/docs-code-examples -[DEBUG] GitHub API response: 200 OK -``` - -### 3. **Configuration Loading** -``` -[DEBUG] Loading config file: copier-config.yaml -[DEBUG] Found 5 copy rules -[DEBUG] Rule 1: Copy src/**/*.js to examples/ -``` - -### 4. **Webhook Processing** -``` -[DEBUG] Received webhook event: pull_request -[DEBUG] PR action: closed -[DEBUG] PR merged: true -[DEBUG] Processing 3 changed files -``` - -### 5. **Pattern Matching** -``` -[DEBUG] Testing pattern: src/**/*.{js,ts} -[DEBUG] File matches: true -[DEBUG] Applying transformations: 2 -``` - ---- - -## Best Practices - -### โœ… DO - -- **Enable debug logging when troubleshooting issues** - ```bash - LOG_LEVEL="debug" - ``` - -- **Disable cloud logging for local development** - ```bash - COPIER_DISABLE_CLOUD_LOGGING="true" - ``` - -- **Use debug logging with dry run mode for testing** - ```bash - LOG_LEVEL="debug" - DRY_RUN="true" - ``` - -- **Disable debug logging in production after troubleshooting** - - High log volume can increase costs - - May expose sensitive information - -### โŒ DON'T - -- **Don't leave debug logging enabled in production long-term** - - Increases log volume and storage costs - - May impact performance - - Can expose internal implementation details - -- **Don't rely on debug logs for critical monitoring** - - Use INFO/WARN/ERROR levels for operational monitoring - - Debug logs may be disabled in production - -- **Don't log sensitive data even in debug mode** - - The code already avoids logging secrets - - Be careful when adding new debug logs - ---- - -## Troubleshooting - -### Debug Logs Not Appearing - -**Problem:** Set `LOG_LEVEL="debug"` but not seeing debug logs - -**Solutions:** - -1. **Check the variable is set correctly:** - ```bash - echo $LOG_LEVEL - # Should output: debug - ``` - -2. **Try the alternative flag:** - ```bash - COPIER_DEBUG="true" - ``` - -3. **Check case sensitivity:** - ```bash - # Both work (case-insensitive): - LOG_LEVEL="debug" - LOG_LEVEL="DEBUG" - ``` - -4. **Verify the code is calling LogDebug():** - - Not all operations have debug logs - - Check `services/logger.go` for `LogDebug()` calls - -### Logs Not Going to Google Cloud - -**Problem:** Logs appear in stdout but not in Google Cloud Logging - -**Solutions:** - -1. **Check if cloud logging is disabled:** - ```bash - # Remove or set to false: - # COPIER_DISABLE_CLOUD_LOGGING="true" - ``` - -2. **Verify GCP credentials:** - ```bash - gcloud auth application-default login - ``` - -3. **Check project ID is set:** - ```bash - GOOGLE_CLOUD_PROJECT_ID="your-project-id" - ``` - -4. **Check log name is set:** - ```bash - COPIER_LOG_NAME="code-copier-log" - ``` - -### Too Many Logs - -**Problem:** Debug logging produces too much output - -**Solutions:** - -1. **Disable debug logging:** - ```bash - # Remove or comment out: - # LOG_LEVEL="debug" - # COPIER_DEBUG="true" - ``` - -2. **Use grep to filter:** - ```bash - # Show only errors: - go run app.go 2>&1 | grep ERROR - - # Show only specific operations: - go run app.go 2>&1 | grep "pattern matching" - ``` - -3. **Redirect to file:** - ```bash - go run app.go > debug.log 2>&1 - ``` - ---- - -## Configuration Examples - -### Example 1: Local Development (Recommended) - -```bash -# configs/.env -LOG_LEVEL="debug" -COPIER_DISABLE_CLOUD_LOGGING="true" -DRY_RUN="true" -AUDIT_ENABLED="false" -METRICS_ENABLED="true" -``` - -**Why:** -- Debug logs help understand what's happening -- No cloud logging keeps it fast and local -- Dry run prevents accidental changes -- No audit logging (simpler setup) - -### Example 2: Production Troubleshooting - -```yaml -# env.yaml -env_variables: - LOG_LEVEL: "debug" - GOOGLE_CLOUD_PROJECT_ID: "your-project-id" - COPIER_LOG_NAME: "code-copier-log" - # ... other variables -``` - -**Why:** -- Temporarily enable debug for troubleshooting -- Logs go to Cloud Logging for analysis -- Remember to disable after fixing issue - -### Example 3: Local with Cloud Logging - -```bash -# configs/.env -LOG_LEVEL="debug" -GOOGLE_CLOUD_PROJECT_ID="your-project-id" -COPIER_LOG_NAME="code-copier-log-dev" -# COPIER_DISABLE_CLOUD_LOGGING not set (defaults to false) -``` - -**Why:** -- Test cloud logging integration locally -- Separate log name for dev environment -- Useful for testing logging infrastructure - ---- - -## See Also - -- [LOCAL-TESTING.md](LOCAL-TESTING.md) - Local development guide -- [TROUBLESHOOTING.md](TROUBLESHOOTING.md) - General troubleshooting -- [CONFIGURATION-GUIDE.md](CONFIGURATION-GUIDE.md) - Complete configuration reference -- [../configs/env.yaml.example](../configs/env.yaml.example) - All environment variables -- [../configs/.env.example](../configs/.env.example) - Local development template - diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index b7d312d..8235616 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -18,7 +18,7 @@ Complete guide for deploying the GitHub Code Example Copier to Google Cloud Run ### Required Tools -- **Go 1.23+** - For local development and testing +- **Go 1.26+** - For local development and testing - **Google Cloud SDK** - For deployment - **GitHub App** - With appropriate permissions - **MongoDB Atlas** (optional) - For audit logging @@ -432,7 +432,7 @@ curl ${SERVICE_URL}/health - Go to: `https://github.com/YOUR_ORG/YOUR_REPO/settings/hooks` 2. **Add or edit webhook** - - **Payload URL:** `https://examples-copier-XXXXXXXXXX-uc.a.run.app/events` (use your Cloud Run URL) + - **Payload URL:** `https://YOUR_SERVICE_URL/events` (use your Cloud Run URL) - **Content type:** `application/json` - **Secret:** (the webhook secret from Secret Manager) - **Events:** Select "Pull requests" @@ -451,7 +451,7 @@ curl ${SERVICE_URL}/health ```bash # Create and merge a test PR # Watch logs for webhook receipt -gcloud app logs tail -s default | grep webhook +gcloud run services logs read github-copier --limit=50 ``` **Option B: Redeliver from GitHub** @@ -465,7 +465,7 @@ gcloud app logs tail -s default | grep webhook ```bash # Check logs for successful processing -gcloud app logs read --limit=50 +gcloud run services logs read github-copier --limit=50 # Look for: # โœ… "Starting web server on port :8080" @@ -485,24 +485,18 @@ gcloud app logs read --limit=50 ### View Logs ```bash -# Real-time logs -gcloud app logs tail -s default - # Recent logs -gcloud app logs read --limit=100 - -# Filter for errors -gcloud app logs read --limit=100 | grep ERROR +gcloud run services logs read github-copier --limit=100 -# Filter for webhooks -gcloud app logs read --limit=100 | grep webhook +# Real-time log streaming +gcloud beta run services logs tail github-copier ``` ### Check Metrics ```bash # Metrics endpoint -curl https://YOUR_APP.appspot.com/metrics +curl https://YOUR_SERVICE_URL/metrics # Response includes: # - webhooks_received @@ -616,7 +610,8 @@ cd github-copier ```bash PROJECT_NUMBER=$(gcloud projects describe $(gcloud config get-value project) --format="value(projectNumber)") -SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" +# Use the Cloud Run service account (default compute, or a custom one) +SERVICE_ACCOUNT="${PROJECT_NUMBER}-compute@developer.gserviceaccount.com" gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ --member="serviceAccount:${SERVICE_ACCOUNT}" \ @@ -633,9 +628,9 @@ gcloud secrets add-iam-policy-binding mongo-uri \ **Verify:** ```bash -gcloud secrets get-iam-policy CODE_COPIER_PEM | grep @appspot -gcloud secrets get-iam-policy webhook-secret | grep @appspot -gcloud secrets get-iam-policy mongo-uri | grep @appspot +gcloud secrets get-iam-policy CODE_COPIER_PEM | grep compute +gcloud secrets get-iam-policy webhook-secret | grep compute +gcloud secrets get-iam-policy mongo-uri | grep compute ``` ### โ˜ 5. Create env.yaml @@ -673,27 +668,13 @@ grep "env.yaml" .gitignore echo "env.yaml" >> .gitignore ``` -### โ˜ 7. Verify app.yaml Configuration - -```bash -cat app.yaml -``` - -**Should contain:** -```yaml -runtime: go -runtime_config: - operating_system: "ubuntu22" - runtime_version: "1.23" -env: flex -``` +### โ˜ 7. Verify Dockerfile -**Should NOT contain:** -- โŒ `env_variables:` section (those go in env.yaml) +Ensure the `Dockerfile` exists and is correctly configured. The app uses a multi-stage build with a non-root user and `HEALTHCHECK`. --- -## ๐Ÿš€ Deployment +## Deployment ### โ˜ 8. Deploy to Cloud Run @@ -704,29 +685,18 @@ cd github-copier ./scripts/deploy-cloudrun.sh ``` -**Expected output:** -``` -Updating service [default]...done. -Setting traffic split for service [default]...done. -Deployed service [default] to [https://YOUR_APP.appspot.com] -``` - ### โ˜ 9. Verify Deployment ```bash -# Check versions -gcloud app versions list - -# Get app URL -APP_URL=$(gcloud app describe --format="value(defaultHostname)") -echo "App URL: https://${APP_URL}" +# Get service URL +gcloud run services describe github-copier --format="value(status.url)" ``` ### โ˜ 10. Check Logs ```bash -# View real-time logs -gcloud app logs tail -s default +# View recent logs +gcloud run services logs read github-copier --limit=50 ``` **Look for:** @@ -743,11 +713,14 @@ gcloud app logs tail -s default ### โ˜ 11. Test Health Endpoint ```bash -# Get app URL -APP_URL=$(gcloud app describe --format="value(defaultHostname)") +# Get service URL +SERVICE_URL=$(gcloud run services describe github-copier --format="value(status.url)") -# Test health -curl https://${APP_URL}/health +# Test health (liveness) +curl ${SERVICE_URL}/health + +# Test readiness +curl ${SERVICE_URL}/ready ``` **Expected response:** @@ -786,7 +759,7 @@ gcloud secrets versions access latest --secret=webhook-secret - URL: `https://github.com/YOUR_ORG/YOUR_REPO/settings/hooks` 2. **Add or edit webhook** - - **Payload URL:** `https://YOUR_APP.appspot.com/events` + - **Payload URL:** `https://YOUR_SERVICE_URL/events` - **Content type:** `application/json` - **Secret:** (paste the value from step 12) - **SSL verification:** Enable SSL verification @@ -810,18 +783,18 @@ gcloud secrets versions access latest --secret=webhook-secret ```bash # Watch logs -gcloud app logs tail -s default | grep webhook +gcloud run services logs read github-copier --limit=50 ``` --- -## โœ… Post-Deployment Verification +## Post-Deployment Verification ### โ˜ 15. Verify Secrets Loaded ```bash # Check logs for secret loading -gcloud app logs read --limit=100 | grep -i "secret" +gcloud run services logs read github-copier --limit=100 ``` **Should NOT see:** @@ -832,23 +805,23 @@ gcloud app logs read --limit=100 | grep -i "secret" ```bash # Watch logs during webhook delivery -gcloud app logs tail -s default +gcloud run services logs read github-copier --limit=50 ``` **Look for:** -- โœ… "webhook received" -- โœ… "signature verified" -- โœ… "processing webhook" +- "webhook received" +- "signature verified" +- "processing webhook" **Should NOT see:** -- โŒ "webhook signature verification failed" -- โŒ "invalid signature" +- "webhook signature verification failed" +- "invalid signature" ### โ˜ 17. Verify File Copying ```bash # Watch logs during PR merge -gcloud app logs tail -s default +gcloud run services logs read github-copier --limit=50 ``` **Look for:** @@ -870,7 +843,7 @@ db.audit_events.find().sort({timestamp: -1}).limit(5) ```bash # Check metrics endpoint -curl https://YOUR_APP.appspot.com/metrics +curl https://YOUR_SERVICE_URL/metrics ``` **Expected response:** @@ -901,9 +874,9 @@ git status | grep env.yaml # Should show: nothing to commit (or untracked) # Verify IAM permissions -gcloud secrets get-iam-policy CODE_COPIER_PEM | grep @appspot -gcloud secrets get-iam-policy webhook-secret | grep @appspot -# Should see the service account +gcloud secrets get-iam-policy CODE_COPIER_PEM | grep compute +gcloud secrets get-iam-policy webhook-secret | grep compute +# Should see the Cloud Run service account ``` --- @@ -939,13 +912,13 @@ gcloud secrets versions access latest --secret=webhook-secret **Fix:** ```bash # Option 1: Disable audit logging -# In env.yaml: AUDIT_ENABLED: "false" +# In env-cloudrun.yaml: AUDIT_ENABLED: "false" # Option 2: Ensure MONGO_URI_SECRET_NAME is set -# In env.yaml: MONGO_URI_SECRET_NAME: "projects/.../secrets/mongo-uri/versions/latest" +# In env-cloudrun.yaml: MONGO_URI_SECRET_NAME: "projects/.../secrets/mongo-uri/versions/latest" # Redeploy -gcloud app deploy app.yaml +./scripts/deploy-cloudrun.sh ``` ### Error: "Config file not found" @@ -996,20 +969,23 @@ Your application is deployed with: --- -## ๐Ÿ“š Quick Reference +## Quick Reference ```bash # Deploy -gcloud app deploy app.yaml +./scripts/deploy-cloudrun.sh # View logs -gcloud app logs tail -s default +gcloud run services logs read github-copier --limit=100 # Check health -curl https://YOUR_APP.appspot.com/health +curl https://YOUR_SERVICE_URL/health + +# Check readiness +curl https://YOUR_SERVICE_URL/ready # Check metrics -curl https://YOUR_APP.appspot.com/metrics +curl https://YOUR_SERVICE_URL/metrics # List secrets gcloud secrets list @@ -1019,10 +995,6 @@ gcloud secrets versions access latest --secret=SECRET_NAME # Grant access ./scripts/grant-secret-access.sh - -# Rollback -gcloud app versions list -gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 ``` --- @@ -1047,10 +1019,10 @@ gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 gcloud secrets versions access latest --secret=webhook-secret # Disable audit logging -# In env.yaml: AUDIT_ENABLED: "false" +# In env-cloudrun.yaml: AUDIT_ENABLED: "false" # Redeploy -gcloud app deploy app.yaml +./scripts/deploy-cloudrun.sh ``` ## Next Steps diff --git a/docs/DEPRECATION-TRACKING-EXPLAINED.md b/docs/DEPRECATION-TRACKING-EXPLAINED.md index a5947f8..3a550a8 100644 --- a/docs/DEPRECATION-TRACKING-EXPLAINED.md +++ b/docs/DEPRECATION-TRACKING-EXPLAINED.md @@ -320,7 +320,6 @@ Potential improvements: --- **See Also:** -- [Configuration Guide](CONFIGURATION-GUIDE.md) - Deprecation configuration - [Architecture](ARCHITECTURE.md) - System design - [Troubleshooting](TROUBLESHOOTING.md) - Common issues diff --git a/docs/FAQ.md b/docs/FAQ.md index 2b8a479..ed1ee9b 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -22,7 +22,7 @@ The GitHub copier is a GitHub app that automatically copies code examples and fi - Path transformations with variable substitution - Multiple target repositories - Flexible commit strategies (direct or PR) -- **Batch PRs** - Combine multiple rules into one PR per target repo +- **Target Repo Batching** - All workflows targeting the same repo are combined into one commit/PR - **PR Template Integration** - Fetch and merge PR templates from target repos - **File Exclusion** - Exclude patterns to filter out unwanted files - Deprecation tracking @@ -68,6 +68,30 @@ workflows: Yes. A file can match multiple workflows and be copied to multiple targets. This is useful for copying the same file to different repositories or branches. +### What happens when multiple workflows target the same repo? + +Files from all workflows that share the same destination repo are **batched into a single commit or PR**. The app does not create separate commits/PRs per workflow. + +**Key behaviors:** +- All matched files are combined into one commit tree +- The **last workflow's commit strategy wins** (PR title, body, commit message, auto-merge setting) +- If one workflow uses `direct` and another uses `pull_request` for the same target, the last strategy wins โ€” they are not separated + +**Example:** If workflow A (direct commit) and workflow B (PR with auto-merge) both target `org/docs`, the result is a single operation using workflow B's strategy โ€” because it runs last. + +**To get separate PRs per workflow**, use different destination repos or branches: + +```yaml +# These create separate PRs because the branches differ: +- name: "go-examples" + destination: { repo: "org/docs", branch: "copier/go" } + +- name: "python-examples" + destination: { repo: "org/docs", branch: "copier/python" } +``` + +See [Architecture > Target Repo Batching](ARCHITECTURE.md#5-target-repo-batching) for details. + ### Where should I store the config file? **Main config:** Store in a central config repository and set `MAIN_CONFIG_FILE` in env.yaml. @@ -168,7 +192,7 @@ path_transform: "all-examples/${filename}" ### What are the prerequisites? -- Go 1.23.4+ +- Go 1.26+ - GitHub App credentials - Google Cloud project (for Secret Manager and logging) - MongoDB Atlas (optional, for audit logging) @@ -179,7 +203,7 @@ Yes! See [Local Testing](LOCAL-TESTING.md) for instructions. ### How do I deploy to Google Cloud? -See [Deployment Guide](DEPLOYMENT.md) for complete guide and [Deployment Checklist](DEPLOYMENT-CHECKLIST.md) for step-by-step instructions. +See [Deployment Guide](DEPLOYMENT.md) for the complete guide including the deployment checklist. ### Do I need MongoDB? diff --git a/docs/LOCAL-TESTING.md b/docs/LOCAL-TESTING.md index 0c4ee64..3c74dca 100644 --- a/docs/LOCAL-TESTING.md +++ b/docs/LOCAL-TESTING.md @@ -40,28 +40,58 @@ cp configs/.env.local.example configs/.env nano configs/.env ``` -### 2. Minimal Configuration +### 2. GitHub App Credentials (Required) -For basic local testing, you only need: +The app authenticates with GitHub on startup, even in dry-run mode. You need your App ID, Installation ID, and PEM key. + +**Option A โ€” PEM from GCP Secret Manager** (if you have `gcloud` access): ```bash +# Run once to authenticate locally: +gcloud auth application-default login + # configs/.env +GITHUB_APP_ID=123456 +INSTALLATION_ID=789012 +GOOGLE_CLOUD_PROJECT_ID=github-copy-code-examples +PEM_NAME=CODE_COPIER_PEM +``` + +**Option B โ€” PEM key provided directly** (no GCP needed): + +```bash +# configs/.env +GITHUB_APP_ID=123456 +INSTALLATION_ID=789012 +SKIP_SECRET_MANAGER=true +GITHUB_APP_PRIVATE_KEY_B64=$(base64 -i /path/to/your-key.pem) +``` + +You can verify your PEM key independently with: + +```bash +go build -o test-pem ./cmd/test-pem +./test-pem /path/to/your-key.pem 123456 +``` + +### 3. Additional Settings (Recommended) + +```bash +# configs/.env (add below the credentials) COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true MAIN_CONFIG_FILE=.copier/workflows/main.yaml USE_MAIN_CONFIG=true ``` -### 3. For Testing with Real PRs +### 4. For Testing with Real PRs -Add a GitHub token: +The `test-webhook` CLI and `test-with-pr.sh` script use a GitHub PAT (not the App credentials) to fetch PR data from the API: ```bash # Get token from: https://github.com/settings/tokens # Required scope: repo (read access) - -# Add to configs/.env -GITHUB_TOKEN=ghp_your_token_here +export GITHUB_TOKEN=ghp_your_token_here ``` ## Running Locally @@ -149,7 +179,7 @@ export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-s nano .copier/workflows/main.yaml # 2. Validate it -./config-validator validate -config config.json -v +./config-validator validate -config copier-config.yaml -v # 3. Start app make run-local @@ -207,13 +237,11 @@ Logs go to stdout when cloud logging is disabled: ```bash # You'll see logs like: -[INFO] Webhook received: pull_request event -[INFO] PR #42 merged: "Add Go database examples" -[INFO] Processing 5 files from PR -[DEBUG] Testing pattern: ^examples/(?P[^/]+)/(?P[^/]+)/.*$ -[INFO] Pattern matched: examples/go/database/connect.go -[INFO] โ†’ Transformed to: docs/go/database/connect.go -[INFO] โ†’ Variables: lang=go, category=database +{"level":"INFO","msg":"Webhook received","event":"pull_request"} +{"level":"INFO","msg":"PR merged","pr":42,"title":"Add Go database examples"} +{"level":"INFO","msg":"Processing files from PR","count":5} +{"level":"DEBUG","msg":"Testing pattern","pattern":"^examples/(?P[^/]+)/(?P[^/]+)/.*$"} +{"level":"INFO","msg":"Pattern matched","file":"examples/go/database/connect.go","target":"docs/go/database/connect.go"} [DRY-RUN] Would create commit with 2 files [DRY-RUN] Would create PR: "Update database examples" ``` @@ -266,9 +294,22 @@ curl http://localhost:8080/health | jq ## Environment Variables for Local Testing -### Required (Minimal) +### Required ```bash +# GitHub App credentials (app authenticates on startup) +GITHUB_APP_ID=123456 +INSTALLATION_ID=789012 + +# PEM key โ€” Option A: via Secret Manager (requires gcloud auth) +GOOGLE_CLOUD_PROJECT_ID=github-copy-code-examples +PEM_NAME=CODE_COPIER_PEM + +# PEM key โ€” Option B: direct (no GCP needed) +SKIP_SECRET_MANAGER=true +GITHUB_APP_PRIVATE_KEY_B64= + +# Local dev overrides COPIER_DISABLE_CLOUD_LOGGING=true # Use stdout instead of GCP DRY_RUN=true # Don't make actual commits ``` @@ -283,10 +324,10 @@ MAIN_CONFIG_FILE=.copier/workflows/main.yaml # Your main config file USE_MAIN_CONFIG=true # Enable main config system ``` -### Optional (for Real PR Testing) +### Optional (for test-webhook CLI / test-with-pr.sh) ```bash -GITHUB_TOKEN=ghp_... # For fetching real PRs +GITHUB_TOKEN=ghp_... # PAT for fetching real PR data REPO_OWNER=mongodb # Default repo owner REPO_NAME=docs-realm # Default repo name ``` @@ -304,9 +345,26 @@ AUDIT_COLLECTION=audit_events ## Troubleshooting +### Error: "A JSON web token could not be decoded" / "Failed to configure GitHub permissions" + +**Problem:** The app needs GitHub App credentials (App ID + PEM key) to authenticate on startup, even in dry-run mode. + +**Solution:** +```bash +# Add to configs/.env: +GITHUB_APP_ID=123456 +INSTALLATION_ID=789012 + +# Then provide the PEM key โ€” either via Secret Manager: +gcloud auth application-default login +# Or directly: +SKIP_SECRET_MANAGER=true +GITHUB_APP_PRIVATE_KEY_B64=$(base64 -i /path/to/your-key.pem) +``` + ### Error: "projects/GOOGLE_CLOUD_PROJECT_ID is not a valid resource name" -**Problem:** Cloud logging is enabled but GCP_PROJECT_ID is not set +**Problem:** Cloud logging is enabled but GCP_PROJECT_ID is not set. **Solution:** ```bash @@ -360,7 +418,7 @@ export GITHUB_TOKEN=ghp_your_token_here -file "examples/go/main.go" # Check config file -./config-validator validate -config config.json -v +./config-validator validate -config copier-config.yaml -v ``` ## Complete Testing Workflow @@ -372,7 +430,7 @@ export GITHUB_TOKEN=ghp_your_token_here make build # 2. Validate configuration -./config-validator validate -config config.json -v +./config-validator validate -config copier-config.yaml -v # 3. Test pattern matching ./config-validator test-pattern \ @@ -398,6 +456,92 @@ export GITHUB_TOKEN=ghp_... # 9. Stop app (Ctrl+C in Terminal 1) ``` +## Webhook Routing: Avoiding Dual Delivery + +When testing locally with a **smee.io** proxy while a **Cloud Run** instance is also running, the same GitHub webhook can be processed by both instances simultaneously. This causes duplicate commits, duplicate PRs, or empty commits in target repositories. + +### Why It Happens + +The GitHub App's webhook URL is a global setting. When set to the Cloud Run URL (`https://...run.app/events`), only Cloud Run receives webhooks. When set to a smee.io channel, your local app receives them โ€” but if you forget to switch back, Cloud Run stops receiving them. If you use smee as a *forwarding proxy* while Cloud Run is also pointed at the same webhook URL, both receive the event. + +The in-memory `DeliveryTracker` prevents duplicate processing within a single instance, but it cannot deduplicate across separate processes. + +### Recommended Strategies + +#### Strategy 1: Swap the webhook URL (simplest) + +Point the GitHub App webhook URL at your smee channel during local testing, then switch it back to Cloud Run when done. + +``` +# Local testing: +GitHub App โ†’ Webhook URL: https://smee.io/your-channel + +# Production: +GitHub App โ†’ Webhook URL: https://your-service.run.app/events +``` + +**Pros:** Zero risk of dual delivery. +**Cons:** Requires manual toggling in GitHub App settings; Cloud Run receives nothing while you test. + +#### Strategy 2: Local dry-run + Cloud Run live (safest) + +Keep the webhook URL pointed at Cloud Run. Run your local app in **dry-run mode** with a smee proxy. The local app processes the webhook but makes no commits or PRs, so duplicate delivery is harmless. + +```bash +# configs/.env +DRY_RUN=true +``` + +``` +GitHub App โ†’ Webhook URL: https://your-service.run.app/events +smee.io โ†’ forwards a copy to localhost:8080/events +``` + +**Pros:** Cloud Run continues operating normally; local testing is safe. +**Cons:** You can't test actual commit/PR creation locally. + +#### Strategy 3: Pause Cloud Run during local testing + +Set Cloud Run to 0 instances while testing locally, then restore it. + +```bash +# Pause Cloud Run +gcloud run services update examples-copier \ + --max-instances=0 --region=us-central1 + +# Resume after testing +gcloud run services update examples-copier \ + --max-instances=10 --region=us-central1 +``` + +**Pros:** Full live testing locally without dual delivery. +**Cons:** Webhooks received by Cloud Run during the pause window are lost (GitHub retries a few times, but may give up). + +#### Strategy 4: Use a test-only source repository + +Create a separate test source repo (e.g. `copier-app-source-test`) that is **not** in the production main config. Point your local `.env` at a test config that includes it: + +```bash +# configs/.env +CONFIG_REPO_OWNER=cbullinger +CONFIG_REPO_NAME=copier-app-source-test +MAIN_CONFIG_FILE=.copier/test-main.yaml +``` + +Webhooks from this test repo will only match workflows in your test config. The production Cloud Run instance uses a different config that doesn't include this repo, so even if it receives the webhook, no workflows match and no work is done. + +**Pros:** Full isolation; no risk to production workflows. +**Cons:** Requires maintaining a separate test repo and config. + +### Quick Decision Guide + +| Scenario | Recommended Strategy | +|----------|---------------------| +| Quick config validation | Strategy 2 (dry-run) | +| Testing actual commits/PRs | Strategy 1 (swap URL) or Strategy 4 (test repo) | +| Extended local development session | Strategy 3 (pause Cloud Run) | +| CI / automated testing | Strategy 4 (test repo) | + ## Tips for Effective Local Testing 1. **Always start with dry-run mode** - Never test with real commits locally diff --git a/docs/PATTERN-MATCHING-GUIDE.md b/docs/PATTERN-MATCHING-GUIDE.md index 59bf661..e324e71 100644 --- a/docs/PATTERN-MATCHING-GUIDE.md +++ b/docs/PATTERN-MATCHING-GUIDE.md @@ -846,6 +846,6 @@ ${name} # Name without ext: main ## See Also - [Local Testing](LOCAL-TESTING.md) - How to test locally -- [Quick Reference](QUICK-REFERENCE.md) - Quick command reference +- [FAQ](FAQ.md) - Frequently asked questions - [Webhook Testing](WEBHOOK-TESTING.md) - Testing with webhooks diff --git a/docs/RECOMMENDATIONS.md b/docs/RECOMMENDATIONS.md new file mode 100644 index 0000000..f7fb217 --- /dev/null +++ b/docs/RECOMMENDATIONS.md @@ -0,0 +1,311 @@ +# Recommendations & Backlog + +Improvement recommendations for the github-copier application, organized by category and priority. + +Last updated: 2026-02-15 + +## Progress Summary + +| Category | Resolved | Remaining | +|---------------|----------|-----------| +| Bugs | 3 | 0 | +| Reliability | 6 | 1 | +| Security | 3 | 0 | +| Performance | 3 | 0 | +| Code Quality | 2 | 1 | +| Testing | 3 | 1 | +| Operational | 2 | 2 | +| Documentation | 3 | 0 | +| **Total** | **25** | **4** | + +## Bugs / Correctness + +### 1. ~~Mixed commit strategies for same target repo~~ (RESOLVED) + +**Priority:** High โ€” **Status: Fixed** + +~~When workflows with `direct` and `pull_request` strategies target the same destination repo, files are batched together and the last workflow's strategy silently wins.~~ + +**Resolution:** Implemented options (a) and (b): +- `UploadKey` now includes `CommitStrategy`, so files are naturally separated by strategy and produce independent write operations (direct commit vs. PR). +- A config-load-time warning alerts operators when workflows target the same `(repo, branch)` with different strategies. +- Write-phase log messages now include `strategy_source` and `file_count`. + +**Related:** [Architecture > Target Repo Batching](ARCHITECTURE.md#5-target-repo-batching) + +### 2. ~~Empty commits on duplicate processing~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~When two instances process the same webhook (e.g., local + Cloud Run), the second instance creates a commit with 0 file changes because the tree is already at HEAD.~~ + +**Resolution:** `createCommitTree` now returns the base commit's tree SHA alongside the new tree SHA. Both `addFilesToBranch` (direct path) and `commitFilesToBranch` (PR path) compare the two and skip the commit entirely when they match, logging `"Skipping empty commit โ€” new tree is identical to HEAD tree"`. + +**Files:** `services/github_write_to_target.go` + +### 3. ~~PR title/body "last wins" is opaque~~ (RESOLVED) + +**Priority:** Low โ€” **Status: Fixed** + +~~When multiple workflows batch into one PR, the commit message, PR title, and body come from whichever workflow ran last. There's no logging indicating which workflow's metadata was used.~~ + +**Resolution:** `workflow_processor.go` now logs when a subsequent workflow overwrites a batched commit message or PR title, including the previous and new values and the workflow name responsible. + +## Reliability + +### 4. Shared delivery tracking for multi-instance + +**Priority:** Medium + +The in-memory `DeliveryTracker` prevents duplicate processing within a single instance, but not across instances (e.g., local + Cloud Run, or multiple Cloud Run revisions during a deployment). Since MongoDB is already wired up for audit logging, add a `processed_deliveries` collection as an optional shared backend. + +**Files:** `services/delivery_tracker.go` + +### 5. ~~PR deduplication in target repos~~ (RESOLVED) + +**Priority:** Low โ€” **Status: Fixed** + +~~If the app processes two source PRs in quick succession, it can create duplicate open PRs in the target repo.~~ + +**Resolution:** `addFilesViaPR` now calls `findExistingCopierPR` before creating a new branch. If an open PR from a `copier/*` branch targeting the same base branch exists, the app pushes new commits to that branch and updates the PR title/body instead of creating a duplicate. + +### 6. ~~Graceful partial failure~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~If 3 workflows match but the 2nd fails mid-processing, the 3rd never runs.~~ + +**Resolution:** `processFilesWithWorkflows` now processes each workflow independently and returns a `map[string]error` of per-workflow failures. A failed workflow no longer blocks others. `handleMergedPRWithContainer` returns an aggregate error when any workflows fail, enabling the retry mechanism to re-attempt the full batch. + +**Files:** `services/webhook_handler_new.go` + +### 7. ~~Retry failed background processing~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~The webhook handler sends 200 OK immediately and processes in a background goroutine. If the goroutine fails, the webhook is lost.~~ + +**Resolution:** Implemented option (a) โ€” `processWebhookWithRetry` wraps `handleMergedPRWithContainer` with exponential-backoff retries (configurable via `WEBHOOK_MAX_RETRIES`, default 2, and `WEBHOOK_RETRY_INITIAL_DELAY`, default 5s). Panics are converted to errors via `runWithRecovery`. After all retries are exhausted, a Slack alert is sent with the delivery context. Retries are skipped if the context deadline is exceeded. + +**Files:** `services/webhook_handler_new.go`, `configs/environment.go` + +### 8. ~~Distinguish transient vs permanent errors~~ (RESOLVED) + +**Priority:** Low โ€” **Status: Fixed** + +~~API rate limits and network timeouts are retryable; 404 (repo not found) and 403 (no permission) are not. The retry logic in `rate_limit.go` handles rate limits, but other transient failures in the write path aren't retried.~~ + +**Resolution:** Added `IsPermanentError()` classifier in `errors.go` that detects non-retryable failures by checking: (a) sentinel errors (`ErrConfigLoad`, `ErrConfigValidation`, `ErrInstallationNotFound`, `ErrMergeConflict`) via `errors.Is()`, and (b) GitHub API `ErrorResponse` status codes (403, 404, 409, 410, 422) via `errors.As()`. The `processWebhookWithRetry` loop now calls `IsPermanentError()` after each failed attempt and breaks immediately for permanent errors, avoiding wasted retries. Slack alerts distinguish `webhook_processing_permanent_error` from `webhook_processing_exhausted`. Comprehensive tests cover all sentinel types, HTTP status codes, wrapped errors, and edge cases. + +**Files:** `services/errors.go`, `services/errors_test.go`, `services/webhook_handler_new.go` + +### 9. ~~Background processing timeout~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~The background goroutine that processes webhooks has no timeout. A stuck GitHub API call could leave it running indefinitely.~~ + +**Resolution:** The background goroutine now applies `context.WithTimeout` using the configurable `WEBHOOK_PROCESSING_TIMEOUT_SECONDS` env var (default 300s / 5 minutes). When the timeout fires, the context is cancelled, in-flight API calls abort, and the retry loop stops with a log and Slack alert. Set to 0 to disable the timeout. + +**Files:** `services/webhook_handler_new.go`, `configs/environment.go` + +## Security + +### 10. ~~Rotate the test PEM key in `.env.test`~~ (RESOLVED) + +**Priority:** High โ€” **Status: Fixed** + +~~`.env.test` contains a real (expired) base64-encoded private key.~~ + +**Resolution:** Replaced the old key with a purpose-generated 2048-bit RSA test-only key that was never associated with any GitHub App. Added the fingerprint to `.gitleaksignore` and a comment clarifying its test-only nature. + +**Files:** `.env.test`, `.gitleaksignore` + +### 11. ~~Tighten gosec exclusions~~ (RESOLVED) + +**Priority:** Low โ€” **Status: Fixed** + +~~CI globally excludes `G703-G706` (taint analysis rules). These are broad.~~ + +**Resolution:** Removed all global `gosec` exclusions (`G115`, `G703`-`G706`) from the CI workflow. G703-G706 no longer fire (code changes and gosec updates eliminated them). The single remaining G115 hit (safe `int -> int32` for PR numbers) is suppressed with an inline `#nosec G115` comment. All 16 existing suppressions are now inline with explanations. + +**Files:** `.github/workflows/ci.yml`, `services/github_read.go` + +### 12. ~~Add `toolchain` directive to `go.mod`~~ (RESOLVED) + +**Priority:** Low โ€” **Status: Fixed** + +~~The `go.mod` says `go 1.26.0` but nothing prevents contributors from building with an older toolchain.~~ + +**Resolution:** Added `toolchain go1.26.0` directive to `go.mod`. Contributors using Go 1.21+ will now automatically download and use the correct toolchain version, ensuring deterministic builds. + +**Files:** `go.mod` + +## Performance + +### 13. ~~Cache resolved workflow configs~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~The app fetches all workflow configs from GitHub on every webhook (4+ API calls to resolve the main config and remote refs).~~ + +**Resolution:** Added `CachedConfigLoader` (decorator around `ConfigLoader`) with a configurable TTL (default 5 minutes, via `CONFIG_CACHE_TTL_SECONDS` env var). Repeated webhooks within the TTL window reuse the cached config without any GitHub API calls. Set to 0 to disable. + +**Files:** `services/config_cache.go`, `services/service_container.go`, `configs/environment.go` + +### 14. ~~Fetch file contents in parallel~~ (RESOLVED) + +**Priority:** Low โ€” **Status: Fixed** + +~~`RetrieveFileContents()` is called sequentially for each matched file.~~ + +**Resolution:** Refactored `ProcessWorkflow` into three phases: match (sequential, fast), fetch (parallel via `errgroup` with a concurrency limit of 5), and queue (sequential, mutates shared state). File content fetches now run concurrently within each workflow, significantly reducing latency for PRs with many matched files. + +**Files:** `services/workflow_processor.go` + +### 15. ~~Handle GitHub API pagination for large PRs~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Already implemented** + +~~Large PRs with 100+ changed files may require pagination in `GetFilesChangedInPr()`.~~ + +**Resolution:** `GetFilesChangedInPr()` already implements cursor-based pagination with `hasNextPage` and `first: 100` per page. No changes needed. + +**Files:** `services/github_read.go` + +## Code Quality + +### 16. ~~Add a `.golangci.yml` config file~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~No linter config file exists; all settings are implicit defaults.~~ + +**Resolution:** Created `.golangci.yml` (v2 format) pinning enabled linters (`errcheck`, `govet`, `ineffassign`, `staticcheck`, `unused`, `misspell`, `revive`), formatters (`gofmt`, `goimports`), and documenting suppressed staticcheck rules (`ST1000`, `ST1003`, `SA1029`). CI and local pre-commit now share the same config. + +**Files:** `.golangci.yml` + +### 17. Split the `services/` package + +**Priority:** Low + +Everything lives in one package: auth, webhook handling, file processing, Slack, audit logging, config loading. Consider sub-packages like `services/github`, `services/config`, `services/notify` to reduce coupling and improve testability. + +### 18. ~~Remove dead code paths~~ (RESOLVED) + +**Priority:** Low โ€” **Status: Fixed** + +~~The legacy `ConfigLoader` (non-main-config path) and `CONFIG_FILE` env var are still present but unused in any real deployment. Deprecation-log them now, remove in a future release.~~ + +**Resolution:** Removed truly dead code from `config_loader.go`: `ValidateConfig`, `ConfigValidator`, `NewConfigValidator`, `ValidatePattern`, `TestPattern`, `TestTransform` (none were called anywhere). Removed unused `configLoader` field from `DefaultMainConfigLoader` in `main_config_loader.go`. Added deprecation warnings in `service_container.go` when the legacy single-file config path is used (`USE_MAIN_CONFIG=false`). Updated `cmd/config-validator/main.go` to call `PatternMatcher` and `PathTransformer` directly instead of through the removed `ConfigValidator` wrapper. The legacy `DefaultConfigLoader` and `CONFIG_FILE` env var are retained for backward compatibility but clearly marked as deprecated. + +**Files:** `services/config_loader.go`, `services/main_config_loader.go`, `services/service_container.go`, `cmd/config-validator/main.go` + +## Testing + +### 19. ~~Integration test for target repo batching~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~Add a test that verifies correct behavior when multiple workflows target the same repo.~~ + +**Resolution:** Added `TestIntegration_TargetRepoBatching_MixedStrategies` which sends a webhook that triggers 3 workflows (2 direct + 1 PR) targeting the same repo. Verifies that the 2 direct workflows batch into 1 commit and the PR workflow produces a separate PR. Also updated the existing direct-commit integration test with the `MockGetCommit` mock for empty-commit detection. + +**Files:** `services/integration_test.go` + +### 20. ~~End-to-end webhook-to-commit test~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~There's no integration test that sends a webhook payload and verifies the resulting GitHub API calls (create tree, create commit, create PR).~~ + +**Resolution:** Multiple integration tests now cover this path: +- `TestIntegration_MergedPR_DirectCommit` โ€” full webhook-to-direct-commit pipeline (config load, GraphQL file list, source fetch, tree/commit/ref update). +- `TestIntegration_TargetRepoBatching_MixedStrategies` โ€” webhook-to-commit *and* webhook-to-PR, verifying batching and separate strategy handling. +- `TestIntegration_MergedPR_NoMatchingWorkflows` and `TestIntegration_MergedPR_ConfigLoadError` โ€” error/edge case flows. + +**Files:** `services/integration_test.go` + +### 21. Contract tests for GitHub API responses + +**Priority:** Low + +Tests use `httpmock` with hardcoded response bodies. If GitHub changes their API response shape, tests still pass but production breaks. Consider recording real API responses as fixtures and replaying them. + +### 22. Benchmarks for pattern matching + +**Priority:** Low + +If a config has many workflows with complex regex patterns, `ProcessWorkflow()` could become a bottleneck. Add `Benchmark` tests for the hot path to catch regressions. + +**Files:** `services/workflow_processor_test.go` + +## Operational + +### 23. ~~Structured error alerting~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~Slack notifications exist but aren't connected to processing failures.~~ + +**Resolution:** `ErrorEvent` now includes `DeliveryID` and `Attempts` fields. `NotifyError` renders them as Slack message fields. `processWebhookWithRetry` threads the GitHub delivery ID through to the final Slack alert, providing full traceability from webhook receipt to failure notification. + +**Files:** `services/slack_notifier.go`, `services/webhook_handler_new.go` + +### 24. ~~Add a `/config` diagnostic endpoint~~ (RESOLVED) + +**Priority:** Low โ€” **Status: Fixed** + +~~A read-only endpoint that shows the resolved effective config (all workflow configs resolved, defaults merged) without secrets. Useful for debugging "why didn't my workflow match?" without reading logs.~~ + +**Resolution:** Added a `GET /config` endpoint that returns a JSON diagnostic view containing: (a) a sanitised environment summary with all secret fields redacted to `[SET]`/`[NOT SET]`, and (b) the resolved workflow list (name, source/dest repos and branches, commit strategy, transform count, excludes). On config load failure the response includes a `load_error` field while still returning the environment section. Tests cover both the error path and a mock-injected workflow summary. + +**Files:** `services/health_metrics.go`, `services/health_metrics_test.go`, `app.go` + +### 25. Cloud Run `min-instances` + +**Priority:** Low + +Cold starts on Cloud Run mean the first webhook after idle takes extra time (Go binary startup + config resolution + GitHub auth). Setting `min-instances: 1` in the deploy config eliminates this at a small cost. + +**Files:** `.github/workflows/ci.yml` (deploy step), `scripts/deploy-cloudrun.sh` + +### 26. Per-workflow logging in write phase + +**Priority:** Low + +The processing phase logs which workflow matched each file, but the write phase (commit/PR creation) only logs the target repo. Add the workflow name(s) that contributed files to each commit/PR. + +**Files:** `services/github_write_to_target.go` + +## Documentation + +### 27. ~~Add a CHANGELOG~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~There's no record of what changed between deployments.~~ + +**Resolution:** Created `CHANGELOG.md` in the repository root following the [Keep a Changelog](https://keepachangelog.com/) format. Documents all Added, Changed, Fixed, and Security items from the current development cycle with cross-references to recommendation numbers. + +**Files:** `CHANGELOG.md` + +### 28. ~~Config reference doc~~ (RESOLVED) + +**Priority:** Medium โ€” **Status: Fixed** + +~~There's no single-page reference of every config option with types, defaults, and examples.~~ + +**Resolution:** Created `docs/CONFIG-REFERENCE.md` with two major sections: (1) all environment variables organized by category with types, defaults, and descriptions; (2) complete workflow YAML schema covering main config, workflow config, transformations, commit strategy, defaults, and `$ref` support. + +**Files:** `docs/CONFIG-REFERENCE.md` + +### 29. ~~Local testing: webhook routing~~ (RESOLVED) + +**Priority:** Low โ€” **Status: Fixed** + +~~Document how to avoid the dual-delivery problem (local + Cloud Run both processing the same webhook).~~ + +**Resolution:** Added a "Webhook Routing: Avoiding Dual Delivery" section to `docs/LOCAL-TESTING.md` explaining why dual delivery happens and documenting four strategies: (1) swap the webhook URL, (2) local dry-run + Cloud Run live, (3) pause Cloud Run, (4) use a test-only source repository. Includes a quick decision guide table. + +**Files:** `docs/LOCAL-TESTING.md` diff --git a/docs/SLACK-NOTIFICATIONS.md b/docs/SLACK-NOTIFICATIONS.md index 50fef96..887f79b 100644 --- a/docs/SLACK-NOTIFICATIONS.md +++ b/docs/SLACK-NOTIFICATIONS.md @@ -158,13 +158,54 @@ File Count: 2 ### Environment Variables -| Variable | Description | Default | Required | -|---------------------|------------------------------|---------------------------|----------| -| `SLACK_WEBHOOK_URL` | Slack incoming webhook URL | - | Yes | -| `SLACK_CHANNEL` | Channel to post to | `#code-examples` | No | -| `SLACK_USERNAME` | Bot username | `Examples Copier` | No | -| `SLACK_ICON_EMOJI` | Bot icon emoji | `:robot_face:` | No | -| `SLACK_ENABLED` | Enable/disable notifications | `true` if webhook URL set | No | +| Variable | Description | Default | Required | +|-------------------------|------------------------------|---------------------------|----------| +| `SLACK_WEBHOOK_URL` | Slack incoming webhook URL | - | Yes | +| `SLACK_CHANNEL` | Channel to post to | `#code-examples` | No | +| `SLACK_USERNAME` | Bot username | `Examples Copier` | No | +| `SLACK_ICON_EMOJI` | Bot icon emoji | `:robot_face:` | No | +| `SLACK_ENABLED` | Enable/disable notifications | `true` if webhook URL set | No | +| `SLACK_PLAIN_TEXT` | Use plain text only (for Workflow Builder) | `false` | No | +| `SLACK_MESSAGE_VARIABLE`| Variable name for Workflow Builder | `text` | No | + +### Webhook Types + +There are two types of Slack webhooks, and they have different capabilities: + +#### Slack App Incoming Webhook (Recommended) + +Created via a Slack App, supports full rich formatting: +- Rich message attachments with colors +- Custom username and icon +- Channel override +- Formatted fields + +**Setup:** +1. Go to https://api.slack.com/apps +2. Click "Create New App" โ†’ "From scratch" +3. Add the "Incoming Webhooks" feature +4. Activate and create a webhook for your channel +5. Copy the webhook URL + +#### Workflow Builder Webhook + +Created via Slack Workflow Builder, only supports plain text: +- No attachments or blocks +- No custom username/icon +- No channel override + +Workflow Builder webhooks use URLs like `https://hooks.slack.com/triggers/...` (note: `/triggers/` instead of `/services/`). + +**The notifier auto-detects Workflow Builder webhooks** by checking for `/triggers/` in the URL and automatically uses plain text mode. + +**Important:** You must set `SLACK_MESSAGE_VARIABLE` to match the variable name you configured in your Slack Workflow. When creating a workflow with "Starts with a webhook" trigger, Slack prompts you to define input variables. Set this to whatever name you used (e.g., `text`, `data`, `message`): + +```bash +SLACK_WEBHOOK_URL="https://hooks.slack.com/triggers/..." +SLACK_MESSAGE_VARIABLE=data # Must match your workflow's input variable name +``` + +The notifier will send a payload like `{"data": "message content"}` which your workflow can then use in a "Send a message" step. ### Disabling Notifications diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index f2cc55d..8563f74 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -88,8 +88,8 @@ source configs/.env 1. **Check actual file paths:** ```bash - # Look for "sample file path" in logs - grep "sample file path" logs/app.log + # Look for "sample file path" in logs (stdout or Cloud Logging) + # Locally: check terminal output or redirect stdout to a file ``` 2. **Test your pattern:** @@ -206,11 +206,11 @@ pattern: "^examples/(?P[^/]+)/(?P.+)$" 1. **Check application logs:** ```bash - # Local - tail -f logs/app.log + # Local: logs go to stdout (JSON format) + LOG_LEVEL=debug ./github-copier - # GCP - gcloud app logs tail -s default + # Cloud Run + gcloud run services logs read github-copier --limit=50 ``` 2. **Check for common errors:** @@ -333,10 +333,7 @@ export COPIER_DISABLE_CLOUD_LOGGING=true # Should be: true ``` -4. **Check application logs:** - ```bash - grep "slack" logs/app.log - ``` +4. **Check application logs** (search stdout for "slack" in local output) ### Slack Notifications in Wrong Channel @@ -469,10 +466,7 @@ db.audit_events.find().sort({timestamp: -1}).limit(10).pretty() ### Trace a Specific Request -```bash -# Look for request ID in logs -grep "request_id=abc123" logs/app.log -``` +Logs are structured JSON (via `log/slog`) and written to stdout. In production on Cloud Run, use Cloud Logging to search by structured fields. ## Getting Help diff --git a/docs/WEBHOOK-TESTING.md b/docs/WEBHOOK-TESTING.md index 5042e90..01f1d41 100644 --- a/docs/WEBHOOK-TESTING.md +++ b/docs/WEBHOOK-TESTING.md @@ -1,453 +1,232 @@ # Webhook Testing Guide -This guide explains how to test the github-copier application with webhooks using real PR data or example payloads. +Test the github-copier app locally using the `test-webhook` CLI tool and/or [smee.io](https://smee.io) webhook forwarding. + +## Prerequisites + +- The app built locally: `go build -o github-copier .` +- A `.env.test` file configured (see `configs/.env.local.example`) +- (Optional) A GitHub personal access token for fetching real PR data ## Quick Start ### 1. Build the Test Tool ```bash -# Using Make make test-webhook -# Or manually +# Or manually: go build -o test-webhook ./cmd/test-webhook ``` -### 2. Test with Example Payload +### 2. Start the App in Dry-Run Mode ```bash -# Send example payload to local server -./test-webhook -payload testdata/example-pr-merged.json - -# See payload without sending -./test-webhook -payload testdata/example-pr-merged.json -dry-run +./github-copier -env .env.test -dry-run ``` -### 3. Test with Real PR Data +The `-dry-run` flag skips actual writes to target repos and tolerates auth failures with a test-only PEM key. + +### 3. Send a Test Webhook ```bash -# Set GitHub token -export GITHUB_TOKEN=ghp_your_token_here +# Send the example payload to the local server +./test-webhook -payload testdata/example-pr-merged.json -# Fetch and send real PR data -./test-webhook -pr 123 -owner myorg -repo myrepo +# Preview the payload without sending +./test-webhook -payload testdata/example-pr-merged.json -dry-run -# Interactive testing with helper script -./scripts/test-with-pr.sh 123 myorg myrepo +# Send with no arguments to use a built-in example payload +./test-webhook ``` -## Testing Scenarios +## Test Scenarios -### Scenario 1: Local Development Testing +### Local Dry-Run -Test your configuration changes locally before deploying: +Test configuration, pattern matching, and transformations without touching GitHub: ```bash -# Terminal 1: Start app in dry-run mode -DRY_RUN=true ./github-copier +# Terminal 1: start the app +./github-copier -env .env.test -dry-run -# Terminal 2: Send test webhook +# Terminal 2: send a test webhook ./test-webhook -payload testdata/example-pr-merged.json - -# Check Terminal 1 for processing logs ``` -**What to verify:** -- Files are matched by patterns -- Path transformations are correct -- Message templates render properly -- No errors in processing +Check Terminal 1 logs for: +- `PR event received` โ€” webhook parsed successfully +- `found matching workflows` โ€” config loaded and workflows matched +- `File matched transformation` โ€” pattern matching + path transforms working +- `[DRY-RUN] Would upload files` โ€” files would be written (skipped in dry-run) -### Scenario 2: Test with Real PR +### Test with a Real PR -Test with actual PR data from your repository: +Fetch actual PR metadata and file lists from GitHub: ```bash -# Set environment export GITHUB_TOKEN=ghp_your_token_here -export REPO_OWNER=myorg -export REPO_NAME=myrepo - -# Use helper script (interactive) -./scripts/test-with-pr.sh 456 -# Or use test-webhook directly -./test-webhook -pr 456 -owner myorg -repo myrepo +./test-webhook -pr 123 -owner myorg -repo myrepo ``` -**What to verify:** -- Real file paths match your patterns -- Actual PR metadata is used correctly -- All files from PR are processed - -### Scenario 3: Test Against Staging - -Test against your staging environment: +Or use the interactive helper script: ```bash -# Set staging URL -export WEBHOOK_URL=https://staging-myapp.appspot.com/webhook -export WEBHOOK_SECRET=your-staging-secret - -# Test with real PR -./test-webhook -pr 123 -owner myorg -repo myrepo \ - -url $WEBHOOK_URL \ - -secret $WEBHOOK_SECRET +./scripts/test-with-pr.sh 123 myorg myrepo ``` -**What to verify:** -- Webhook signature verification works -- Staging environment processes correctly -- Audit logs are created (if enabled) -- Metrics are updated - -### Scenario 4: Test Pattern Matching - -Create custom payloads to test specific patterns: +### Test Against a Remote Environment ```bash -# Create test payload -cat > test-go-only.json < test-deprecation.json <[^/]+)/(?P[^/]+)/(?P.+)$` -Should extract: `lang=go`, `category=database`, `file=connect.go` - -**Test Multiple Languages:** -```json -{ - "files": [ - {"filename": "examples/go/main.go", "status": "added"}, - {"filename": "examples/python/main.py", "status": "added"}, - {"filename": "examples/javascript/main.js", "status": "added"} - ] -} -``` +### Diagnostic Endpoints -**Test Path Transformations:** -```json -{ - "files": [ - {"filename": "examples/go/database/connect.go", "status": "added"} - ] -} -``` -With transform: `docs/${lang}/${category}/${file}` -Should produce: `docs/go/database/connect.go` - -## Validation Checklist - -After sending a test webhook, verify: - -### Application Logs ```bash -# Local -tail -f logs/app.log +# Liveness probe +curl http://localhost:8080/health | jq -# GCP -gcloud app logs tail -s default -``` +# Readiness probe (checks GitHub auth) +curl http://localhost:8080/ready | jq -**Look for:** -- โœ… Webhook received -- โœ… Files matched by pattern -- โœ… Path transformations applied -- โœ… Variables extracted correctly -- โœ… No errors in processing +# Resolved config with secrets redacted +curl http://localhost:8080/config | jq -### Metrics Endpoint -```bash +# Metrics (if enabled) curl http://localhost:8080/metrics | jq ``` -**Verify:** -- โœ… `webhooks.received` incremented -- โœ… `webhooks.processed` incremented -- โœ… `files.matched` shows correct count -- โœ… `files.uploaded` updated (if not dry-run) - -### Health Endpoint -```bash -curl http://localhost:8080/health | jq -``` - -**Verify:** -- โœ… Status is "healthy" -- โœ… GitHub authentication working -- โœ… Queue counts are correct - -### Audit Logs (if enabled) -```javascript -// MongoDB query -db.audit_events.find().sort({timestamp: -1}).limit(10) -``` - -**Verify:** -- โœ… Event created for webhook -- โœ… Correct event type (copy/deprecation) -- โœ… File paths are correct -- โœ… Rule name matches config - ## Troubleshooting ### Webhook Returns 401 Unauthorized -**Problem:** Signature verification failed - -**Solution:** -```bash -# Make sure secret matches -./test-webhook -payload test.json -secret "correct-secret" +The webhook secret doesn't match. Ensure the `-secret` flag (or `WEBHOOK_SECRET` env var) matches the app's `WEBHOOK_SECRET` config value. If testing locally without signature verification, leave `WEBHOOK_SECRET` empty in `.env.test`. -# Or disable signature check for testing -# (remove signature verification in code temporarily) -``` +### Duplicate Delivery Skipped -### Files Not Matched +The app deduplicates webhooks by the `X-GitHub-Delivery` header. If you redeliver the same webhook, restart the app to clear the in-memory tracker (TTL is 1 hour). -**Problem:** Pattern doesn't match files +### Config Cache Returns Stale Data -**Solution:** -```bash -# Test pattern with config-validator -./config-validator test-pattern \ - -type regex \ - -pattern "^examples/(?P[^/]+)/.*$" \ - -file "examples/go/main.go" - -# Check config file pattern syntax -./config-validator validate -config copier-config.yaml -v -``` +Workflow configs are cached for 5 minutes (configurable via `CONFIG_CACHE_TTL_SECONDS`). Restart the app to force a fresh load after changing a remote workflow config. -### Path Transformation Wrong +### Files Not Matched -**Problem:** Transformed path is incorrect +Use the `config-validator` CLI to test patterns: -**Solution:** ```bash -# Test transformation -./config-validator test-transform \ - -template "docs/${lang}/${file}" \ - -file "examples/go/main.go" \ - -pattern "^examples/(?P[^/]+)/(?P.+)$" +go build -o config-validator ./cmd/config-validator -# Check variable names match -# Pattern: (?P...) -> Template: ${lang} -``` +# Test a glob pattern +./config-validator test-pattern -type glob -pattern 'examples/**/*.go' -file 'examples/go/main.go' -### No Response from Webhook +# Test a regex pattern +./config-validator test-pattern -type regex -pattern '^examples/(?P[^/]+)/.*$' -file 'examples/go/main.go' +``` -**Problem:** Webhook doesn't respond +### Path Transformation Wrong -**Solution:** ```bash -# Check if app is running -curl http://localhost:8080/health - -# Check webhook URL -./test-webhook -payload test.json -url http://localhost:8080/webhook - -# Check application logs for errors +./config-validator test-transform \ + -source 'examples/go/main.go' \ + -template 'code/${filename}' ``` -### Real PR Fetch Fails +### Auth Fails in Dry-Run Mode -**Problem:** Can't fetch PR data from GitHub - -**Solution:** -```bash -# Verify token is set -echo $GITHUB_TOKEN +Expected when using a test-only PEM key. The app will log a warning and continue: -# Test token manually -curl -H "Authorization: Bearer $GITHUB_TOKEN" \ - https://api.github.com/repos/owner/repo/pulls/123 - -# Check token permissions (needs repo read access) ``` - -## Best Practices - -1. **Always test locally first** with dry-run mode -2. **Use real PR data** for realistic testing -3. **Create custom payloads** for edge cases -4. **Verify all metrics** after testing -5. **Check audit logs** to ensure tracking works -6. **Test with multiple file types** to verify patterns -7. **Test deprecation** by including removed files -8. **Use helper script** for interactive testing -9. **Keep test payloads** in version control -10. **Document test scenarios** for your specific use case - -## Integration with CI/CD - -Add webhook testing to your CI pipeline: - -```yaml -# .github/workflows/test.yml -- name: Test webhook processing - run: | - # Start app in background - DRY_RUN=true ./github-copier & - APP_PID=$! - - # Wait for app to start - sleep 5 - - # Run webhook tests - ./test-webhook -payload testdata/example-pr-merged.json - - # Stop app - kill $APP_PID +โš ๏ธ GitHub auth skipped (dry-run): ... ``` -## Next Steps +The webhook handler will also fail auth and classify it as a permanent error (no retries). This confirms the routing logic works โ€” use smee.io with a real PEM for full end-to-end testing. -After successful webhook testing: +### No Matching Workflows -1. Deploy to staging environment -2. Configure GitHub webhook in repository settings -3. Test with real PR merge -4. Monitor metrics and logs -5. Deploy to production -6. Set up alerts for failures +Check that: +1. The workflow config in the source repo uses the `workflows:` schema (not `workflow_configs:`) +2. The `source.repo` and `source.branch` in the workflow match the webhook's repository and base branch +3. The workflow config reference is `enabled: true` in the main config -See [DEPLOYMENT.md](DEPLOYMENT.md) for deployment instructions. +## Makefile Targets +```bash +make test-webhook # Build the test-webhook tool +make test-webhook-example # Build + send example payload +make test-webhook-pr PR=123 OWNER=org REPO=repo # Build + send real PR data +``` diff --git a/env-cloudrun.yaml b/env-cloudrun.yaml index a255f63..48fe1fd 100644 --- a/env-cloudrun.yaml +++ b/env-cloudrun.yaml @@ -2,19 +2,19 @@ # Format: KEY: "VALUE" (no env_variables wrapper needed) # Note: PORT is automatically set by Cloud Run, don't override it -# GitHub Configuration -GITHUB_APP_ID: "1166559" -INSTALLATION_ID: "95537167" # grove-platform +# Org-specific values (GITHUB_APP_ID, INSTALLATION_ID) are passed via +# --set-env-vars in the deploy step using GitHub Actions secrets. +# Do NOT commit org-specific values here. # Config Repository (where main config file is stored) CONFIG_REPO_OWNER: "grove-platform" CONFIG_REPO_NAME: "github-copier" CONFIG_REPO_BRANCH: "main" -# Secret Manager References -GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest" -WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" -MONGO_URI_SECRET_NAME: "projects/1054147886816/secrets/mongo-uri/versions/latest" +# Secret Manager References (short names โ€” resolved at runtime via SecretPath()) +PEM_NAME: "CODE_COPIER_PEM" +WEBHOOK_SECRET_NAME: "webhook-secret" +MONGO_URI_SECRET_NAME: "mongo-uri" # Application Settings WEBSERVER_PATH: "/events" @@ -33,4 +33,3 @@ COPIER_LOG_NAME: "code-copier-log" # Feature Flags AUDIT_ENABLED: "false" METRICS_ENABLED: "true" - diff --git a/github-app-manifest.yml b/github-app-manifest.yml new file mode 100644 index 0000000..3e00db1 --- /dev/null +++ b/github-app-manifest.yml @@ -0,0 +1,46 @@ +# GitHub App Manifest +# Documents the minimum permissions and events required by this GitHub App. +# Reference: https://docs.github.com/en/apps/sharing-github-apps/registering-a-github-app-from-a-manifest +# +# This file serves as living documentation. It can also be used with GitHub's +# "Register a GitHub App from a manifest" flow to create a correctly configured app. + +name: GitHub Copier +description: Copies code examples between repositories when pull requests are merged. +url: https://github.com/grove-platform/github-copier + +# --- Permissions --- +# The minimum set of permissions needed for the app to function. + +default_permissions: + # Read & Write: read source files, create branches/trees/commits, push to target repos, + # read config repos, update deprecation files. + contents: write + + # Read & Write: read changed files via GraphQL, create PRs in target repos, + # check mergeability, auto-merge PRs. + pull_requests: write + + # Read: implicit for all GitHub Apps; required for basic repository access. + metadata: read + +# --- Webhook Events --- +# The app only needs the pull_request event. It processes merged PRs +# (action == "closed" && merged == true) and ignores all other events. + +default_events: + - pull_request + +# --- Installation --- +# The app must be installed in every organization whose repositories it accesses: +# - Source repos (where PRs trigger the webhook) +# - Target repos (where files are copied to) +# - Config repo (where workflow YAML files live) +# +# The app discovers per-org installation IDs at runtime via GET /app/installations +# and creates scoped installation access tokens for each org. + +# --- Webhook Configuration --- +# The app listens on /events for webhook deliveries. +# Configure the webhook URL to point to your Cloud Run service URL + /events. +# A webhook secret (WEBHOOK_SECRET) should always be configured in production. diff --git a/go.mod b/go.mod index 3a6d555..e36cd9c 100644 --- a/go.mod +++ b/go.mod @@ -1,63 +1,62 @@ module github.com/grove-platform/github-copier -go 1.24.0 +go 1.26.0 require ( - cloud.google.com/go/logging v1.13.0 - cloud.google.com/go/secretmanager v1.14.6 - github.com/bmatcuk/doublestar/v4 v4.9.1 - github.com/golang-jwt/jwt/v5 v5.2.2 - github.com/google/go-github/v48 v48.2.0 - github.com/jarcoal/httpmock v1.4.0 + cloud.google.com/go/logging v1.13.2 + cloud.google.com/go/secretmanager v1.16.0 + github.com/bmatcuk/doublestar/v4 v4.10.0 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/google/go-github/v82 v82.0.0 + github.com/google/uuid v1.6.0 + github.com/jarcoal/httpmock v1.4.1 github.com/joho/godotenv v1.5.1 - github.com/pkg/errors v0.9.1 - github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 - github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 - github.com/stretchr/testify v1.10.0 - go.mongodb.org/mongo-driver v1.17.1 - golang.org/x/oauth2 v0.28.0 + github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed + github.com/shurcooL/graphql v0.0.0-20240915155400-7ee5256398cf + github.com/stretchr/testify v1.11.1 + go.mongodb.org/mongo-driver/v2 v2.5.0 + golang.org/x/oauth2 v0.35.0 + golang.org/x/sync v0.19.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - cloud.google.com/go v0.118.3 // indirect - cloud.google.com/go/auth v0.15.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect - cloud.google.com/go/compute/metadata v0.6.0 // indirect - cloud.google.com/go/iam v1.4.1 // indirect - cloud.google.com/go/longrunning v0.6.4 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.18.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/longrunning v0.8.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-querystring v1.1.0 // indirect + github.com/google/go-querystring v1.2.0 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.5 // indirect - github.com/googleapis/gax-go/v2 v2.14.1 // indirect - github.com/klauspost/compress v1.13.6 // indirect - github.com/montanaflynn/stats v0.7.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect + github.com/googleapis/gax-go/v2 v2.16.0 // indirect + github.com/klauspost/compress v1.17.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect - go.opentelemetry.io/otel v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.34.0 // indirect - go.opentelemetry.io/otel/trace v1.34.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect - golang.org/x/time v0.10.0 // indirect - google.golang.org/api v0.224.0 // indirect - google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e // indirect - google.golang.org/grpc v1.71.0 // indirect - google.golang.org/protobuf v1.36.5 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/api v0.259.0 // indirect + google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 2e3e5ab..ca3e2ee 100644 --- a/go.sum +++ b/go.sum @@ -1,158 +1,164 @@ -cloud.google.com/go v0.118.3 h1:jsypSnrE/w4mJysioGdMBg4MiW/hHx/sArFpaBWHdME= -cloud.google.com/go v0.118.3/go.mod h1:Lhs3YLnBlwJ4KA6nuObNMZ/fCbOQBPuWKPoE0Wa/9Vc= -cloud.google.com/go/auth v0.15.0 h1:Ly0u4aA5vG/fsSsxu98qCQBemXtAtJf+95z9HK+cxps= -cloud.google.com/go/auth v0.15.0/go.mod h1:WJDGqZ1o9E9wKIL+IwStfyn/+s59zl4Bi+1KQNVXLZ8= -cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= -cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= -cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= -cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= -cloud.google.com/go/iam v1.4.1 h1:cFC25Nv+u5BkTR/BT1tXdoF2daiVbZ1RLx2eqfQ9RMM= -cloud.google.com/go/iam v1.4.1/go.mod h1:2vUEJpUG3Q9p2UdsyksaKpDzlwOrnMzS30isdReIcLM= -cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= -cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= -cloud.google.com/go/longrunning v0.6.4 h1:3tyw9rO3E2XVXzSApn1gyEEnH2K9SynNQjMlBi3uHLg= -cloud.google.com/go/longrunning v0.6.4/go.mod h1:ttZpLCe6e7EXvn9OxpBRx7kZEB0efv8yBO6YnVMfhJs= -cloud.google.com/go/secretmanager v1.14.6 h1:/ooktIMSORaWk9gm3vf8+Mg+zSrUplJFKBztP993oL0= -cloud.google.com/go/secretmanager v1.14.6/go.mod h1:0OWeM3qpJ2n71MGgNfKsgjC/9LfVTcUqXFUlGxo5PzY= -github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= -github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0= +cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= +cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/secretmanager v1.16.0 h1:19QT7ZsLJ8FSP1k+4esQvuCD7npMJml6hYzilxVyT+k= +cloud.google.com/go/secretmanager v1.16.0/go.mod h1://C/e4I8D26SDTz1f3TQcddhcmiC3rMEl0S1Cakvs3Q= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= +github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= +github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v48 v48.2.0 h1:68puzySE6WqUY9KWmpOsDEQfDZsso98rT6pZcz9HqcE= -github.com/google/go-github/v48 v48.2.0/go.mod h1:dDlehKBDo850ZPvCTK0sEqTCVWcrGl2LcDiajkYi89Y= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/go-github/v82 v82.0.0 h1:OH09ESON2QwKCUVMYmMcVu1IFKFoaZHwqYaUtr/MVfk= +github.com/google/go-github/v82 v82.0.0/go.mod h1:hQ6Xo0VKfL8RZ7z1hSfB4fvISg0QqHOqe9BP0qo+WvM= +github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= +github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.5 h1:VgzTY2jogw3xt39CusEnFJWm7rlsq5yL5q9XdLOuP5g= -github.com/googleapis/enterprise-certificate-proxy v0.3.5/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= -github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= -github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= -github.com/jarcoal/httpmock v1.4.0 h1:BvhqnH0JAYbNudL2GMJKgOHe2CtKlzJ/5rWKyp+hc2k= -github.com/jarcoal/httpmock v1.4.0/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= +github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= +github.com/jarcoal/httpmock v1.4.1 h1:0Ju+VCFuARfFlhVXFc2HxlcQkfB+Xq12/EotHko+x2A= +github.com/jarcoal/httpmock v1.4.1/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX48g9wI= github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= -github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= -github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 h1:cYCy18SHPKRkvclm+pWm1Lk4YrREb4IOIb/YdFO0p2M= -github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= -github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= -github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed h1:KT7hI8vYXgU0s2qaMkrfq9tCA1w/iEPgfredVP+4Tzw= +github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= +github.com/shurcooL/graphql v0.0.0-20240915155400-7ee5256398cf h1:o1uxfymjZ7jZ4MsgCErcwWGtVKSiNAXtS59Lhs6uI/g= +github.com/shurcooL/graphql v0.0.0-20240915155400-7ee5256398cf/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= +github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= -go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= -golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= -golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.224.0 h1:Ir4UPtDsNiwIOHdExr3fAj4xZ42QjK7uQte3lORLJwU= -google.golang.org/api v0.224.0/go.mod h1:3V39my2xAGkodXy0vEqcEtkqgw2GtrFL5WuBZlCTCOQ= -google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= -google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e h1:YA5lmSs3zc/5w+xsRcHqpETkaYyK63ivEPzNTcUUlSA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= -google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ= +google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4= +google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934= +google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b h1:uA40e2M6fYRBf0+8uN5mLlqUtV192iiksiICIBkYJ1E= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:Xa7le7qx2vmqB/SzWUBa7KdMjpdpAHlh5QCSnjessQk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/scripts/README.md b/scripts/README.md index c6a3c01..4286520 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,233 +1,156 @@ # Helper Scripts -Collection of helper scripts for testing and running the github-copier application. +Collection of helper scripts for developing, testing, and deploying the github-copier application. ## Scripts -### run-local.sh +### ci-local.sh -Start the github-copier application locally with proper environment configuration. +Run the full CI pipeline locally before pushing. Mirrors what `.github/workflows/ci.yml` does. **Usage:** ```bash -./scripts/run-local.sh +./scripts/ci-local.sh ``` **What it does:** -- Loads environment variables from `configs/.env.local` -- Disables Google Cloud logging (uses stdout instead) -- Sets up local configuration -- Starts the application +1. `go build ./...` +2. `go test -race ./...` +3. `golangci-lint run ./...` +4. `go vet ./...` + +### run-local.sh + +Start the github-copier application locally with development settings. -**Example:** +**Usage:** ```bash -# Start app locally ./scripts/run-local.sh - -# In another terminal, test it -./test-webhook -payload testdata/example-pr-merged.json ``` -**Environment:** -- Uses `configs/.env.local` for configuration -- Sets `COPIER_DISABLE_CLOUD_LOGGING=true` -- Sets `CONFIG_FILE=copier-config.yaml` +**What it does:** +- Builds the `github-copier` binary if needed +- Disables Google Cloud logging (uses stdout) +- Enables dry-run mode, debug logging, and metrics +- Loads env from `configs/.env` if present +- Starts the application on port 8080 -### test-and-check.sh +### deploy-cloudrun.sh -Send a test webhook and check the metrics. +Deploy to Google Cloud Run. **Usage:** ```bash -./scripts/test-and-check.sh +./scripts/deploy-cloudrun.sh [region] ``` **What it does:** -1. Sends test webhook with example payload -2. Waits for processing -3. Fetches and displays metrics -4. Shows recent application logs +- Validates `env-cloudrun.yaml` exists +- Confirms deployment with user +- Deploys via `gcloud run deploy` with Dockerfile +- Prints service URL and next steps -**Example:** -```bash -# Start app first -./scripts/run-local.sh +### grant-secret-access.sh -# In another terminal, test and check -./scripts/test-and-check.sh -``` +Grant the Cloud Run service account access to all required secrets in Secret Manager. -**Output:** -``` -Testing webhook with example payload... - -โœ“ Loaded payload from testdata/example-pr-merged.json -โœ“ Response: 200 OK -โœ“ Webhook sent successfully - -Webhook sent! Waiting 2 seconds for processing... - -=== Metrics === -{ - "webhooks": { - "received": 1, - "processed": 1, - "failed": 0 - }, - "files": { - "matched": 20, - "uploaded": 0 - } -} - -=== Recent Logs === -[INFO] loaded config from local file -[INFO] retrieved changed files | {"count":21} -[INFO] processing files with pattern matching -[INFO] file matched pattern | {"file":"..."} +**Usage:** +```bash +./scripts/grant-secret-access.sh ``` -### test-slack.sh +**Secrets configured:** `CODE_COPIER_PEM`, `webhook-secret`, `mongo-uri` -Test Slack notifications by sending example messages. +### test-and-check.sh + +Send a test webhook and check health/metrics. **Usage:** ```bash -./scripts/test-slack.sh [webhook-url] +./scripts/test-and-check.sh ``` -**Arguments:** -- `webhook-url` - Slack webhook URL (optional, uses `$SLACK_WEBHOOK_URL` if not provided) - **What it does:** -1. Sends simple test message -2. Sends PR processed notification -3. Sends error notification -4. Sends files copied notification -5. Sends deprecation notification +1. Sends test webhook with example payload +2. Waits for processing +3. Fetches and displays `/metrics` and `/health` -**Example:** -```bash -# Using environment variable -export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/..." -./scripts/test-slack.sh +### test-with-pr.sh -# Or pass URL directly -./scripts/test-slack.sh "https://hooks.slack.com/services/..." -``` +Fetch real PR data from GitHub and send it to the webhook. -**Output:** +**Usage:** +```bash +./scripts/test-with-pr.sh [owner] [repo] ``` -Testing Slack Notifications - -Webhook URL: https://hooks.slack.com/services/... - -Test 1: Sending simple test message... -โœ“ Simple message sent - -Test 2: Sending PR processed notification... -โœ“ PR processed notification sent - -Test 3: Sending error notification... -โœ“ Error notification sent - -Test 4: Sending files copied notification... -โœ“ Files copied notification sent - -Test 5: Sending deprecation notification... -โœ“ Deprecation notification sent - -=== All Tests Complete === -Check your Slack channel for 5 test notifications -``` +**Environment:** +- `GITHUB_TOKEN` โ€” GitHub personal access token (required) +- `WEBHOOK_URL` โ€” Webhook endpoint (default: `http://localhost:8080/events`) +- `WEBHOOK_SECRET` โ€” Webhook secret for HMAC signature -### convert-env-format.sh +### integration-test.sh -Convert between App Engine and Cloud Run environment file formats. +End-to-end integration test: sends a webhook payload and optionally verifies destination repos. **Usage:** ```bash -./scripts/convert-env-format.sh to-cloudrun -./scripts/convert-env-format.sh to-appengine +./scripts/integration-test.sh webhook # Send test webhook +./scripts/integration-test.sh verify # Check dest repos +./scripts/integration-test.sh full # Both + wait ``` -**What it does:** -- Converts between App Engine format (with `env_variables:` wrapper) and Cloud Run format (plain YAML) -- Handles indentation automatically -- Validates input file exists -- Prompts before overwriting existing files - -**Example:** -```bash -# Convert App Engine โ†’ Cloud Run -./scripts/convert-env-format.sh to-cloudrun env.yaml env-cloudrun.yaml +**Environment:** +- `APP_URL` โ€” App URL (default: `http://localhost:8080`) +- `WEBHOOK_SECRET` โ€” Webhook secret (default: reads from `.env.test`) +- `DEST_REPO_1`, `DEST_PATH_1` โ€” First destination repo/path to verify +- `DEST_REPO_2`, `DEST_PATH_2` โ€” Second destination repo/path to verify -# Convert Cloud Run โ†’ App Engine -./scripts/convert-env-format.sh to-appengine env-cloudrun.yaml env.yaml -``` +### test-slack.sh -**Format differences:** -```yaml -# App Engine format (env.yaml) -env_variables: - GITHUB_APP_ID: "123456" - REPO_OWNER: "mongodb" +Test Slack notifications by sending example messages (success, error, deprecation). -# Cloud Run format (env-cloudrun.yaml) -GITHUB_APP_ID: "123456" -REPO_OWNER: "mongodb" +**Usage:** +```bash +./scripts/test-slack.sh [webhook-url] +# Or: export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/..." ``` -### test-with-pr.sh +### diagnose-github-auth.sh -Fetch real PR data from GitHub and send it to the webhook. +Diagnostic script for GitHub App authentication issues. Checks Secret Manager, private key format, env config, and Cloud Run service health/readiness. **Usage:** ```bash -./scripts/test-with-pr.sh [webhook-url] +./scripts/diagnose-github-auth.sh ``` -**Arguments:** -- `pr-number` - Pull request number (required) -- `owner` - Repository owner (required) -- `repo` - Repository name (required) -- `webhook-url` - Webhook URL (optional, default: `http://localhost:8080/webhook`) +### test-github-access.sh -**Environment Variables:** -- `GITHUB_TOKEN` - GitHub personal access token (required) +Test if the deployed Cloud Run service can access the configured GitHub repository. Checks the `/ready` endpoint and recent logs for 401 errors. -**Example:** +**Usage:** ```bash -# Set GitHub token -export GITHUB_TOKEN=ghp_your_token_here - -# Test with real PR -./scripts/test-with-pr.sh 42 mongodb docs-code-examples - -# Test with custom webhook URL -./scripts/test-with-pr.sh 42 mongodb docs-code-examples http://localhost:8080/webhook +./scripts/test-github-access.sh ``` -**Output:** -``` -Fetching PR #42 from mongodb/docs-code-examples... +### check-installation-repos.sh -โœ“ PR fetched successfully -โœ“ PR Title: Add Go database examples -โœ“ Files changed: 21 -โœ“ Sending to webhook... -โœ“ Response: 200 OK +List all repositories accessible to a GitHub App installation. Generates a JWT, exchanges it for an installation token, and queries the GitHub API. -Check application logs for processing details. +**Usage:** +```bash +./scripts/check-installation-repos.sh ``` +**Requires:** `jq`, `ruby` with `jwt` gem, `gcloud` + ## Common Workflows ### Local Development ```bash -# 1. Start app locally +# 1. Start app locally (dry-run + debug) ./scripts/run-local.sh # 2. In another terminal, test it @@ -237,6 +160,12 @@ Check application logs for processing details. curl http://localhost:8080/metrics | jq ``` +### Pre-Push Validation + +```bash +./scripts/ci-local.sh +``` + ### Testing with Real Data ```bash @@ -248,183 +177,30 @@ export GITHUB_TOKEN=ghp_... # 3. Test with real PR ./scripts/test-with-pr.sh 42 myorg myrepo - -# 4. Check results -./scripts/test-and-check.sh ``` ### Testing Slack Integration ```bash -# 1. Test Slack webhook export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/..." ./scripts/test-slack.sh - -# 2. Start app with Slack enabled -./scripts/run-local.sh - -# 3. Send test webhook -./scripts/test-and-check.sh - -# 4. Check Slack channel for notification -``` - -### Dry-Run Testing - -```bash -# 1. Start app in dry-run mode -DRY_RUN=true ./scripts/run-local.sh - -# 2. Test processing -./scripts/test-and-check.sh - -# 3. Verify no commits were made (check logs) -``` - -## Script Details - -### Environment Variables - -All scripts respect these environment variables: - -**Application:** -- `CONFIG_FILE` - Configuration file path -- `DRY_RUN` - Enable dry-run mode -- `LOG_LEVEL` - Logging level (debug, info, warn, error) -- `COPIER_DISABLE_CLOUD_LOGGING` - Disable Google Cloud logging - -**GitHub:** -- `GITHUB_TOKEN` - GitHub personal access token -- `GITHUB_APP_ID` - GitHub App ID -- `GITHUB_INSTALLATION_ID` - GitHub Installation ID - -**Slack:** -- `SLACK_WEBHOOK_URL` - Slack webhook URL -- `SLACK_CHANNEL` - Slack channel -- `SLACK_ENABLED` - Enable/disable Slack notifications - -**MongoDB:** -- `MONGO_URI` - MongoDB connection string -- `AUDIT_ENABLED` - Enable/disable audit logging - -### Exit Codes - -All scripts use standard exit codes: -- `0` - Success -- `1` - Error occurred - -### Error Handling - -Scripts include error handling and will: -- Display clear error messages -- Exit with non-zero code on failure -- Provide troubleshooting hints - -## Creating Custom Scripts - -### Template - -```bash -#!/bin/bash - -# Script description -# Usage: ./my-script.sh [args] - -set -e # Exit on error - -# Colors for output -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -# Check prerequisites -if [ -z "$REQUIRED_VAR" ]; then - echo -e "${RED}Error: REQUIRED_VAR not set${NC}" - exit 1 -fi - -# Main logic -echo -e "${GREEN}Starting...${NC}" - -# Do work -# ... - -echo -e "${GREEN}Complete!${NC}" ``` -### Best Practices - -1. **Use `set -e`** to exit on errors -2. **Check prerequisites** before running -3. **Provide clear output** with colors -4. **Include usage instructions** in comments -5. **Handle errors gracefully** -6. **Make scripts executable**: `chmod +x script.sh` +### Diagnosing Auth Issues -## Troubleshooting - -### Script Not Executable - -**Error:** -``` -Permission denied: ./scripts/run-local.sh -``` - -**Solution:** ```bash -chmod +x scripts/*.sh -``` +# Full diagnostic +./scripts/diagnose-github-auth.sh -### Environment Variables Not Set - -**Error:** -``` -Error: GITHUB_TOKEN not set -``` +# Check repo access +./scripts/test-github-access.sh -**Solution:** -```bash -export GITHUB_TOKEN=ghp_your_token_here -# Or add to configs/.env.local -``` - -### App Not Running - -**Error:** -``` -Connection refused -``` - -**Solution:** -```bash -# Start the app first -./scripts/run-local.sh - -# Then run tests in another terminal -``` - -### Slack Webhook Fails - -**Error:** -``` -Error: invalid_payload -``` - -**Solution:** -```bash -# Verify webhook URL -echo $SLACK_WEBHOOK_URL - -# Test directly -curl -X POST -H 'Content-type: application/json' \ - --data '{"text":"Test"}' \ - "$SLACK_WEBHOOK_URL" +# List accessible repos +./scripts/check-installation-repos.sh ``` ## See Also - [Local Testing Guide](../docs/LOCAL-TESTING.md) - Local development - [Webhook Testing Guide](../docs/WEBHOOK-TESTING.md) - Testing webhooks -- [Quick Reference](../docs/QUICK-REFERENCE.md) - All commands - [test-webhook Tool](../cmd/test-webhook/README.md) - Test webhook tool - diff --git a/scripts/check-installation-repos.sh b/scripts/check-installation-repos.sh index a504c8d..ff4c68b 100755 --- a/scripts/check-installation-repos.sh +++ b/scripts/check-installation-repos.sh @@ -16,11 +16,20 @@ if ! command -v jq &> /dev/null; then exit 1 fi -# Get the installation ID from env.yaml -INSTALLATION_ID=$(grep "INSTALLATION_ID:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') +# Get the installation ID from env-cloudrun.yaml (or env.yaml fallback) +ENV_FILE="env-cloudrun.yaml" +if [ ! -f "$ENV_FILE" ]; then + ENV_FILE="env.yaml" +fi +if [ ! -f "$ENV_FILE" ]; then + echo "โŒ Neither env-cloudrun.yaml nor env.yaml found" + exit 1 +fi + +INSTALLATION_ID=$(grep "INSTALLATION_ID:" "$ENV_FILE" | grep -v "#" | awk '{print $2}' | tr -d '"') if [ -z "$INSTALLATION_ID" ]; then - echo "โŒ INSTALLATION_ID not found in env.yaml" + echo "โŒ INSTALLATION_ID not found in $ENV_FILE" exit 1 fi @@ -39,11 +48,11 @@ fi echo "โœ… Private key retrieved" echo "" -# Get the GitHub App ID from env.yaml -APP_ID=$(grep "GITHUB_APP_ID:" env.yaml | awk '{print $2}' | tr -d '"') +# Get the GitHub App ID +APP_ID=$(grep "GITHUB_APP_ID:" "$ENV_FILE" | awk '{print $2}' | tr -d '"') if [ -z "$APP_ID" ]; then - echo "โŒ GITHUB_APP_ID not found in env.yaml" + echo "โŒ GITHUB_APP_ID not found in $ENV_FILE" exit 1 fi diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh new file mode 100755 index 0000000..56ac0f3 --- /dev/null +++ b/scripts/ci-local.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +# Run the full CI pipeline locally before pushing +# Mirrors what .github/workflows/ci.yml does +# +# Usage: +# ./scripts/ci-local.sh # Run standard CI checks +# ./scripts/ci-local.sh --full # Include integration tests +# ./scripts/ci-local.sh --quick # Fast checks only (no race detector) + +set -e + +# Colors +GREEN='\033[0;32m' +RED='\033[0;31m' +BLUE='\033[0;34m' +YELLOW='\033[0;33m' +NC='\033[0m' # No Color + +PASS="${GREEN}PASS${NC}" +FAIL="${RED}FAIL${NC}" +SKIP="${YELLOW}SKIP${NC}" + +# Options +RUN_INTEGRATION=false +QUICK_MODE=false + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --full|-f) + RUN_INTEGRATION=true + shift + ;; + --quick|-q) + QUICK_MODE=true + shift + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --full, -f Include integration tests" + echo " --quick, -q Fast checks only (no race detector)" + echo " --help, -h Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +echo -e "${BLUE}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" +echo -e "${BLUE}โ•‘ GitHub Copier - Local CI Pipeline โ•‘${NC}" +echo -e "${BLUE}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" +echo "" + +FAILED=0 + +# 1. Build +echo -n "Build... " +if go build ./... 2>&1; then + echo -e "$PASS" +else + echo -e "$FAIL" + ((FAILED++)) +fi + +# 2. Test with race detector (skip in quick mode) +if [[ "$QUICK_MODE" == "true" ]]; then + echo -n "Test (no race)... " + if go test ./... 2>&1; then + echo -e "$PASS" + else + echo -e "$FAIL" + ((FAILED++)) + fi +else + echo -n "Test (race)... " + if go test -race ./... 2>&1; then + echo -e "$PASS" + else + echo -e "$FAIL" + ((FAILED++)) + fi +fi + +# 3. Lint +echo -n "Lint... " +if command -v golangci-lint &> /dev/null; then + if golangci-lint run ./... 2>&1; then + echo -e "$PASS" + else + echo -e "$FAIL" + ((FAILED++)) + fi +else + echo -e "$SKIP (golangci-lint not installed)" + echo " Install: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest" +fi + +# 4. Vet +echo -n "Vet... " +if go vet ./... 2>&1; then + echo -e "$PASS" +else + echo -e "$FAIL" + ((FAILED++)) +fi + +# 5. Test coverage (optional, quick summary) +if [[ "$QUICK_MODE" != "true" ]]; then + echo -n "Coverage... " + COVERAGE=$(go test ./services -cover 2>&1 | grep -o 'coverage: [0-9.]*%' | grep -o '[0-9.]*' || echo "0") + if [[ -n "$COVERAGE" ]]; then + echo -e "${GREEN}${COVERAGE}%${NC}" + else + echo -e "$SKIP" + fi +fi + +# 6. Integration tests (if --full flag) +if [[ "$RUN_INTEGRATION" == "true" ]]; then + echo "" + echo -e "${BLUE}Running integration tests...${NC}" + if [[ -x "./scripts/integration-test.sh" ]]; then + if ./scripts/integration-test.sh --quick 2>&1; then + echo -e "Integration tests... $PASS" + else + echo -e "Integration tests... $FAIL" + ((FAILED++)) + fi + else + echo -e "Integration tests... $SKIP (script not found)" + fi +fi + +# Summary +echo "" +echo "โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•" +if [[ $FAILED -eq 0 ]]; then + echo -e "${GREEN}All checks passed!${NC}" + exit 0 +else + echo -e "${RED}$FAILED check(s) failed${NC}" + exit 1 +fi diff --git a/scripts/convert-env-format.sh b/scripts/convert-env-format.sh deleted file mode 100755 index 44bf7b9..0000000 --- a/scripts/convert-env-format.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -# Convert between App Engine (env.yaml) and Cloud Run (env-cloudrun.yaml) formats -# -# Usage: -# ./convert-env-format.sh to-cloudrun env.yaml env-cloudrun.yaml -# ./convert-env-format.sh to-appengine env-cloudrun.yaml env.yaml - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Print usage -usage() { - echo "Convert between App Engine and Cloud Run environment file formats" - echo "" - echo "Usage:" - echo " $0 to-cloudrun " - echo " $0 to-appengine " - echo "" - echo "Examples:" - echo " # Convert App Engine format to Cloud Run format" - echo " $0 to-cloudrun env.yaml env-cloudrun.yaml" - echo "" - echo " # Convert Cloud Run format to App Engine format" - echo " $0 to-appengine env-cloudrun.yaml env.yaml" - echo "" - echo "Formats:" - echo " App Engine: env_variables: wrapper with indented keys" - echo " Cloud Run: Plain YAML without wrapper" - exit 1 -} - -# Check arguments -if [ $# -ne 3 ]; then - usage -fi - -COMMAND=$1 -INPUT=$2 -OUTPUT=$3 - -# Validate command -if [ "$COMMAND" != "to-cloudrun" ] && [ "$COMMAND" != "to-appengine" ]; then - echo -e "${RED}Error: Invalid command '$COMMAND'${NC}" - echo "Must be 'to-cloudrun' or 'to-appengine'" - usage -fi - -# Check input file exists -if [ ! -f "$INPUT" ]; then - echo -e "${RED}Error: Input file '$INPUT' not found${NC}" - exit 1 -fi - -# Check if output file exists -if [ -f "$OUTPUT" ]; then - echo -e "${YELLOW}Warning: Output file '$OUTPUT' already exists${NC}" - read -p "Overwrite? (y/N) " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Aborted" - exit 1 - fi -fi - -# Convert to Cloud Run format (remove env_variables wrapper and unindent) -if [ "$COMMAND" = "to-cloudrun" ]; then - echo -e "${BLUE}Converting App Engine format to Cloud Run format...${NC}" - - # Remove 'env_variables:' line and unindent by 2 spaces - sed '/^env_variables:/d' "$INPUT" | sed 's/^ //' > "$OUTPUT" - - echo -e "${GREEN}โœ“ Converted to Cloud Run format: $OUTPUT${NC}" - echo "" - echo "Deploy with:" - echo " gcloud run deploy examples-copier --source . --env-vars-file=$OUTPUT" -fi - -# Convert to App Engine format (add env_variables wrapper and indent) -if [ "$COMMAND" = "to-appengine" ]; then - echo -e "${BLUE}Converting Cloud Run format to App Engine format...${NC}" - - # Add 'env_variables:' header and indent all lines by 2 spaces - echo "env_variables:" > "$OUTPUT" - sed 's/^/ /' "$INPUT" >> "$OUTPUT" - - echo -e "${GREEN}โœ“ Converted to App Engine format: $OUTPUT${NC}" - echo "" - echo "Deploy with:" - echo " gcloud app deploy app.yaml # Includes $OUTPUT automatically" -fi - -echo "" -echo -e "${YELLOW}Note: Review the output file before deploying!${NC}" - diff --git a/scripts/convert-env-to-yaml.sh b/scripts/convert-env-to-yaml.sh deleted file mode 100755 index 3a41ace..0000000 --- a/scripts/convert-env-to-yaml.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# Convert .env file to env.yaml format for Google Cloud App Engine - -set -e - -# Default input/output files -INPUT_FILE="${1:-.env}" -OUTPUT_FILE="${2:-env.yaml}" - -if [ ! -f "$INPUT_FILE" ]; then - echo "Error: Input file '$INPUT_FILE' not found" - echo "Usage: $0 [input-file] [output-file]" - echo "Example: $0 .env.production env.yaml" - exit 1 -fi - -echo "Converting $INPUT_FILE to $OUTPUT_FILE..." - -# Start the YAML file -echo "env_variables:" > "$OUTPUT_FILE" - -# Read the .env file and convert to YAML -while IFS= read -r line || [ -n "$line" ]; do - # Skip empty lines and comments - if [[ -z "$line" ]] || [[ "$line" =~ ^[[:space:]]*# ]]; then - continue - fi - - # Extract key and value - if [[ "$line" =~ ^([^=]+)=(.*)$ ]]; then - key="${BASH_REMATCH[1]}" - value="${BASH_REMATCH[2]}" - - # Remove leading/trailing whitespace from key - key=$(echo "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - # Remove quotes from value if present - value=$(echo "$value" | sed 's/^["'\'']\(.*\)["'\'']$/\1/') - - # Write to YAML file with proper indentation - echo " $key: \"$value\"" >> "$OUTPUT_FILE" - fi -done < "$INPUT_FILE" - -echo "โœ… Conversion complete: $OUTPUT_FILE" -echo "" -echo "โš ๏ธ IMPORTANT: Review $OUTPUT_FILE before deploying!" -echo " - Verify all values are correct" -echo " - Check for sensitive data" -echo " - Ensure $OUTPUT_FILE is in .gitignore" - diff --git a/scripts/deploy-cloudrun.sh b/scripts/deploy-cloudrun.sh index 7052b96..4f885f3 100755 --- a/scripts/deploy-cloudrun.sh +++ b/scripts/deploy-cloudrun.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Deploy examples-copier to Google Cloud Run +# Deploy github-copier to Google Cloud Run # Usage: ./scripts/deploy-cloudrun.sh [region] set -e diff --git a/scripts/diagnose-github-auth.sh b/scripts/diagnose-github-auth.sh index e6c89f9..dc38a28 100755 --- a/scripts/diagnose-github-auth.sh +++ b/scripts/diagnose-github-auth.sh @@ -5,7 +5,7 @@ set -e -echo "๐Ÿ” GitHub App Authentication Diagnostics" +echo "GitHub App Authentication Diagnostics" echo "==========================================" echo "" @@ -17,58 +17,59 @@ NC='\033[0m' # No Color # Check if gcloud is installed if ! command -v gcloud &> /dev/null; then - echo -e "${RED}โŒ gcloud CLI not found${NC}" + echo -e "${RED}gcloud CLI not found${NC}" echo "Please install: https://cloud.google.com/sdk/docs/install" exit 1 fi -echo -e "${GREEN}โœ… gcloud CLI found${NC}" +echo -e "${GREEN}gcloud CLI found${NC}" # Get project info PROJECT_ID=$(gcloud config get-value project 2>/dev/null) if [ -z "$PROJECT_ID" ]; then - echo -e "${RED}โŒ No GCP project set${NC}" + echo -e "${RED}No GCP project set${NC}" echo "Run: gcloud config set project YOUR_PROJECT_ID" exit 1 fi -echo -e "${GREEN}โœ… GCP Project: $PROJECT_ID${NC}" +echo -e "${GREEN}GCP Project: $PROJECT_ID${NC}" PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectNumber)") -echo -e "${GREEN}โœ… Project Number: $PROJECT_NUMBER${NC}" +echo -e "${GREEN}Project Number: $PROJECT_NUMBER${NC}" -SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" +# Cloud Run uses the default Compute Engine SA unless a custom SA is configured +SERVICE_ACCOUNT="${PROJECT_NUMBER}-compute@developer.gserviceaccount.com" echo -e " Service Account: $SERVICE_ACCOUNT" echo "" # Check Secret Manager API -echo "๐Ÿ“ฆ Checking Secret Manager..." +echo "Checking Secret Manager..." if gcloud services list --enabled --filter="name:secretmanager.googleapis.com" --format="value(name)" | grep -q secretmanager; then - echo -e "${GREEN}โœ… Secret Manager API enabled${NC}" + echo -e "${GREEN}Secret Manager API enabled${NC}" else - echo -e "${RED}โŒ Secret Manager API not enabled${NC}" + echo -e "${RED}Secret Manager API not enabled${NC}" echo "Run: gcloud services enable secretmanager.googleapis.com" exit 1 fi # Check if secrets exist echo "" -echo "๐Ÿ” Checking Secrets..." +echo "Checking Secrets..." check_secret() { local secret_name=$1 if gcloud secrets describe "$secret_name" &>/dev/null; then - echo -e "${GREEN}โœ… Secret exists: $secret_name${NC}" + echo -e "${GREEN}Secret exists: $secret_name${NC}" # Check IAM permissions if gcloud secrets get-iam-policy "$secret_name" --format="value(bindings.members)" | grep -q "$SERVICE_ACCOUNT"; then - echo -e "${GREEN} โœ… Service account has access${NC}" + echo -e "${GREEN} Service account has access${NC}" else - echo -e "${RED} โŒ Service account does NOT have access${NC}" + echo -e "${RED} Service account does NOT have access${NC}" echo -e "${YELLOW} Fix: gcloud secrets add-iam-policy-binding $secret_name --member=\"serviceAccount:${SERVICE_ACCOUNT}\" --role=\"roles/secretmanager.secretAccessor\"${NC}" fi else - echo -e "${RED}โŒ Secret NOT found: $secret_name${NC}" + echo -e "${RED}Secret NOT found: $secret_name${NC}" fi } @@ -77,98 +78,110 @@ check_secret "webhook-secret" # Check if we can access the PEM key echo "" -echo "๐Ÿ”‘ Checking GitHub App Private Key..." +echo "Checking GitHub App Private Key..." if gcloud secrets versions access latest --secret=CODE_COPIER_PEM &>/dev/null; then PEM_FIRST_LINE=$(gcloud secrets versions access latest --secret=CODE_COPIER_PEM | head -n 1) if [[ "$PEM_FIRST_LINE" == "-----BEGIN RSA PRIVATE KEY-----" ]] || [[ "$PEM_FIRST_LINE" == "-----BEGIN PRIVATE KEY-----" ]]; then - echo -e "${GREEN}โœ… Private key format looks correct${NC}" + echo -e "${GREEN}Private key format looks correct${NC}" else - echo -e "${RED}โŒ Private key format looks incorrect${NC}" + echo -e "${RED}Private key format looks incorrect${NC}" echo " First line: $PEM_FIRST_LINE" fi else - echo -e "${RED}โŒ Cannot access private key${NC}" + echo -e "${RED}Cannot access private key${NC}" fi -# Check env.yaml +# Check env-cloudrun.yaml echo "" -echo "โš™๏ธ Checking env.yaml configuration..." -if [ -f "env.yaml" ]; then - echo -e "${GREEN}โœ… env.yaml found${NC}" +echo "Checking env-cloudrun.yaml configuration..." +ENV_FILE="env-cloudrun.yaml" +if [ -f "$ENV_FILE" ]; then + echo -e "${GREEN}$ENV_FILE found${NC}" - # Extract values - GITHUB_APP_ID=$(grep "GITHUB_APP_ID:" env.yaml | awk '{print $2}' | tr -d '"') - INSTALLATION_ID=$(grep "INSTALLATION_ID:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') - REPO_OWNER=$(grep "REPO_OWNER:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') - REPO_NAME=$(grep "REPO_NAME:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') + # Extract values (plain YAML, no env_variables wrapper) + GITHUB_APP_ID=$(grep "GITHUB_APP_ID:" "$ENV_FILE" | awk '{print $2}' | tr -d '"') + INSTALLATION_ID=$(grep "INSTALLATION_ID:" "$ENV_FILE" | grep -v "#" | awk '{print $2}' | tr -d '"') + REPO_OWNER=$(grep "REPO_OWNER:" "$ENV_FILE" | grep -v "#" | awk '{print $2}' | tr -d '"') + REPO_NAME=$(grep "REPO_NAME:" "$ENV_FILE" | grep -v "#" | awk '{print $2}' | tr -d '"') echo " GitHub App ID: $GITHUB_APP_ID" echo " Installation ID: $INSTALLATION_ID" echo " Repository: $REPO_OWNER/$REPO_NAME" - if [ -z "$GITHUB_APP_ID" ] || [ -z "$INSTALLATION_ID" ] || [ -z "$REPO_OWNER" ] || [ -z "$REPO_NAME" ]; then - echo -e "${RED}โŒ Missing required configuration${NC}" + if [ -z "$GITHUB_APP_ID" ] || [ -z "$REPO_OWNER" ] || [ -z "$REPO_NAME" ]; then + echo -e "${RED}Missing required configuration${NC}" else - echo -e "${GREEN}โœ… Configuration looks complete${NC}" + echo -e "${GREEN}Configuration looks complete${NC}" fi else - echo -e "${RED}โŒ env.yaml not found${NC}" + echo -e "${RED}$ENV_FILE not found${NC}" + echo "Create it: cp configs/env.yaml.production env-cloudrun.yaml" fi -# Check App Engine deployment +# Check Cloud Run deployment echo "" -echo "๐Ÿš€ Checking App Engine deployment..." -if gcloud app describe &>/dev/null; then - APP_URL=$(gcloud app describe --format="value(defaultHostname)") - echo -e "${GREEN}โœ… App Engine app exists${NC}" - echo " URL: https://$APP_URL" +echo "Checking Cloud Run deployment..." +SERVICE_URL=$(gcloud run services describe github-copier --format="value(status.url)" 2>/dev/null) +if [ -n "$SERVICE_URL" ]; then + echo -e "${GREEN}Cloud Run service exists${NC}" + echo " URL: $SERVICE_URL" # Try to hit health endpoint echo "" - echo "๐Ÿฅ Checking health endpoint..." - if curl -s -f "https://$APP_URL/health" &>/dev/null; then - echo -e "${GREEN}โœ… Health endpoint responding${NC}" - curl -s "https://$APP_URL/health" | python3 -m json.tool 2>/dev/null || echo "" + echo "Checking health endpoint..." + if curl -s -f "$SERVICE_URL/health" &>/dev/null; then + echo -e "${GREEN}Health endpoint responding${NC}" + curl -s "$SERVICE_URL/health" | python3 -m json.tool 2>/dev/null || echo "" else - echo -e "${RED}โŒ Health endpoint not responding${NC}" + echo -e "${RED}Health endpoint not responding${NC}" + fi + + echo "" + echo "Checking readiness endpoint..." + if curl -s -f "$SERVICE_URL/ready" &>/dev/null; then + echo -e "${GREEN}Readiness endpoint responding${NC}" + curl -s "$SERVICE_URL/ready" | python3 -m json.tool 2>/dev/null || echo "" + else + echo -e "${YELLOW}Readiness endpoint returned non-200 (may indicate auth or connectivity issue)${NC}" + curl -s "$SERVICE_URL/ready" | python3 -m json.tool 2>/dev/null || echo "" fi else - echo -e "${YELLOW}โš ๏ธ No App Engine app deployed yet${NC}" + echo -e "${YELLOW}No Cloud Run service 'github-copier' found${NC}" fi # Summary echo "" -echo "๐Ÿ“‹ Summary & Next Steps" +echo "Summary & Next Steps" echo "=======================" echo "" # Check for common issues ISSUES_FOUND=0 -if ! gcloud secrets get-iam-policy CODE_COPIER_PEM --format="value(bindings.members)" | grep -q "$SERVICE_ACCOUNT"; then - echo -e "${RED}โŒ Issue: Service account doesn't have access to CODE_COPIER_PEM${NC}" +if ! gcloud secrets get-iam-policy CODE_COPIER_PEM --format="value(bindings.members)" 2>/dev/null | grep -q "$SERVICE_ACCOUNT"; then + echo -e "${RED}Issue: Service account doesn't have access to CODE_COPIER_PEM${NC}" echo " Fix: Run ./scripts/grant-secret-access.sh" ISSUES_FOUND=$((ISSUES_FOUND + 1)) fi -if [ ! -f "env.yaml" ]; then - echo -e "${RED}โŒ Issue: env.yaml not found${NC}" - echo " Fix: cp configs/env.yaml.example env.yaml && nano env.yaml" +if [ ! -f "$ENV_FILE" ]; then + echo -e "${RED}Issue: $ENV_FILE not found${NC}" + echo " Fix: cp configs/env.yaml.production env-cloudrun.yaml && nano env-cloudrun.yaml" ISSUES_FOUND=$((ISSUES_FOUND + 1)) fi if [ $ISSUES_FOUND -eq 0 ]; then - echo -e "${GREEN}โœ… No obvious issues found${NC}" + echo -e "${GREEN}No obvious issues found${NC}" echo "" echo "If you're still seeing 401 errors, check:" echo "1. GitHub App is installed on the repository: https://github.com/settings/installations" echo "2. Installation ID matches the repository" echo "3. Private key in Secret Manager matches the GitHub App" - echo "4. GitHub App has 'Contents' read permission" + echo "4. GitHub App has 'Contents' write and 'Pull requests' write permissions" + echo " (see github-app-manifest.yml for required permissions)" echo "" - echo "View logs: gcloud app logs tail -s default" + echo "View logs: gcloud run services logs read github-copier --limit=50" else echo "" echo -e "${YELLOW}Found $ISSUES_FOUND issue(s) - please fix them and try again${NC}" fi - diff --git a/scripts/grant-secret-access.sh b/scripts/grant-secret-access.sh index 023832c..d08cc8a 100755 --- a/scripts/grant-secret-access.sh +++ b/scripts/grant-secret-access.sh @@ -1,13 +1,20 @@ #!/bin/bash -# Grant App Engine access to all secrets +# Grant Cloud Run service account access to all secrets +# +# NOTE: If you use a custom service account for Cloud Run (recommended), +# update SERVICE_ACCOUNT below to match. The default Compute Engine SA +# is used here as a baseline. set -e PROJECT_ID="github-copy-code-examples" PROJECT_NUMBER="1054147886816" -SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" -echo "Granting App Engine service account access to secrets..." +# Cloud Run uses the default Compute Engine service account unless a custom SA is configured. +# Previously this script incorrectly targeted the App Engine SA (${PROJECT_NUMBER}@appspot.gserviceaccount.com). +SERVICE_ACCOUNT="${PROJECT_NUMBER}-compute@developer.gserviceaccount.com" + +echo "Granting Cloud Run service account access to secrets..." echo "Service Account: ${SERVICE_ACCOUNT}" echo "" @@ -27,7 +34,7 @@ for SECRET in "${SECRETS[@]}"; do echo "" done -echo "โœ… Done! Verifying permissions..." +echo "Done! Verifying permissions..." echo "" for SECRET in "${SECRETS[@]}"; do @@ -38,5 +45,4 @@ for SECRET in "${SECRETS[@]}"; do echo "" done -echo "โœ… All secrets are now accessible by App Engine!" - +echo "All secrets are now accessible by Cloud Run!" diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index 1914487..7f3d12e 100755 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -1,142 +1,322 @@ #!/bin/bash + # Integration test script for github-copier -# Sends test webhook payload to locally running app +# Runs automated end-to-end tests locally without external dependencies # # Usage: -# ./scripts/integration-test.sh webhook # Send test webhook -# ./scripts/integration-test.sh verify # Check dest repos -# ./scripts/integration-test.sh full # Both + wait -# -# Environment: -# APP_URL - App URL (default: http://localhost:8080) -# WEBHOOK_SECRET - Webhook secret (default: reads from .env.test) -# PAYLOAD_FILE - Payload file (default: testdata/test-pr-merged.json) +# ./scripts/integration-test.sh # Run all tests +# ./scripts/integration-test.sh --quick # Run quick smoke tests only +# ./scripts/integration-test.sh --verbose # Show detailed output set -e -# Load webhook secret from .env.test if it exists and WEBHOOK_SECRET not set -if [[ -z "$WEBHOOK_SECRET" && -f ".env.test" ]]; then - WEBHOOK_SECRET=$(grep -E "^WEBHOOK_SECRET=" .env.test | cut -d'=' -f2- | tr -d '"' | tr -d "'") -fi - -# Configuration -APP_URL="${APP_URL:-http://localhost:8080}" -WEBHOOK_SECRET="${WEBHOOK_SECRET:-test-secret}" -PAYLOAD_FILE="${PAYLOAD_FILE:-testdata/test-pr-merged.json}" - # Colors -RED='\033[0;31m' GREEN='\033[0;32m' -YELLOW='\033[1;33m' +RED='\033[0;31m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' NC='\033[0m' # No Color -log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +PASS="${GREEN}PASS${NC}" +FAIL="${RED}FAIL${NC}" +SKIP="${YELLOW}SKIP${NC}" + +# Configuration +APP_PORT="${PORT:-8080}" +APP_URL="http://localhost:${APP_PORT}" +WEBHOOK_ENDPOINT="${APP_URL}/events" +HEALTH_ENDPOINT="${APP_URL}/health" +METRICS_ENDPOINT="${APP_URL}/metrics" +TESTDATA_DIR="testdata" +APP_PID="" +VERBOSE=false +QUICK_MODE=false + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --verbose|-v) + VERBOSE=true + shift + ;; + --quick|-q) + QUICK_MODE=true + shift + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --verbose, -v Show detailed output" + echo " --quick, -q Run quick smoke tests only" + echo " --help, -h Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +log() { + if [[ "$VERBOSE" == "true" ]]; then + echo -e "$1" + fi +} + +log_always() { + echo -e "$1" +} + +cleanup() { + if [[ -n "$APP_PID" ]] && kill -0 "$APP_PID" 2>/dev/null; then + log "Stopping application (PID: $APP_PID)..." + kill "$APP_PID" 2>/dev/null || true + wait "$APP_PID" 2>/dev/null || true + fi +} + +trap cleanup EXIT + +# Build the application and test tools +build_app() { + log_always -n "Building application... " + if go build -o github-copier . 2>&1; then + log_always -e "$PASS" + else + log_always -e "$FAIL" + exit 1 + fi + + log_always -n "Building test-webhook... " + if go build -o test-webhook ./cmd/test-webhook 2>&1; then + log_always -e "$PASS" + else + log_always -e "$FAIL" + exit 1 + fi +} -# Check if app is running -check_app() { - log_info "Checking if app is running at $APP_URL..." - if curl -s "$APP_URL/health" > /dev/null 2>&1; then - log_info "App is running" +# Start the application in dry-run mode +start_app() { + log_always -n "Starting app (dry-run)... " + + # Check if something is already listening on the port + if lsof -i ":${APP_PORT}" >/dev/null 2>&1; then + log_always -e "${YELLOW}Port ${APP_PORT} in use${NC}" + log_always "Stop existing process or set PORT env var" + exit 1 + fi + + # Start with minimal config for testing + COPIER_DISABLE_CLOUD_LOGGING=true \ + DRY_RUN=true \ + AUDIT_ENABLED=false \ + LOG_LEVEL=error \ + PORT="${APP_PORT}" \ + ./github-copier > /tmp/integration-test-app.log 2>&1 & + + APP_PID=$! + + # Wait for app to be ready (max 30 seconds) + local max_wait=30 + local waited=0 + while [[ $waited -lt $max_wait ]]; do + if curl -s "${HEALTH_ENDPOINT}" >/dev/null 2>&1; then + log_always -e "$PASS" + return 0 + fi + + # Check if process died + if ! kill -0 "$APP_PID" 2>/dev/null; then + log_always -e "$FAIL" + log_always "Application failed to start. Logs:" + cat /tmp/integration-test-app.log + exit 1 + fi + + sleep 1 + ((waited++)) + done + + log_always -e "$FAIL (timeout)" + exit 1 +} + +# Test health endpoint +test_health() { + log_always -n "Health check... " + local response + response=$(curl -s -w "\n%{http_code}" "${HEALTH_ENDPOINT}") + local http_code + http_code=$(echo "$response" | tail -n1) + + if [[ "$http_code" == "200" ]]; then + log_always -e "$PASS" return 0 else - log_error "App is not running at $APP_URL" - log_info "Start the app with: go run app.go" + log_always -e "$FAIL (HTTP $http_code)" return 1 fi } -# Generate HMAC signature for webhook -generate_signature() { - local payload="$1" - echo -n "$payload" | openssl dgst -sha256 -hmac "$WEBHOOK_SECRET" | sed 's/^.* //' +# Test metrics endpoint +test_metrics() { + log_always -n "Metrics endpoint... " + local response + response=$(curl -s -w "\n%{http_code}" "${METRICS_ENDPOINT}") + local http_code + http_code=$(echo "$response" | tail -n1) + + if [[ "$http_code" == "200" ]]; then + log_always -e "$PASS" + return 0 + else + log_always -e "$FAIL (HTTP $http_code)" + return 1 + fi } -# Send webhook payload +# Send a test webhook payload send_webhook() { local payload_file="$1" + local expected_status="${2:-202}" + local description="$3" if [[ ! -f "$payload_file" ]]; then - log_error "Payload file not found: $payload_file" + log_always -e " $description: ${SKIP} (file not found)" + return 0 + fi + + log_always -n " $description... " + + local response + response=$(./test-webhook -payload "$payload_file" -url "${WEBHOOK_ENDPOINT}" 2>&1) + local exit_code=$? + + if [[ $exit_code -eq 0 ]]; then + log_always -e "$PASS" + log "$response" + return 0 + else + log_always -e "$FAIL" + log "$response" return 1 fi +} + +# Test webhook payloads +test_webhooks() { + log_always "" + log_always -e "${BLUE}Testing webhook payloads:${NC}" - local payload=$(cat "$payload_file") - local signature="sha256=$(generate_signature "$payload")" + local failed=0 - log_info "Sending webhook payload from $payload_file..." - log_info "Signature: $signature" + # Test merged PR (should be processed) + send_webhook "${TESTDATA_DIR}/example-pr-merged.json" 202 "Merged PR (example)" || ((failed++)) + send_webhook "${TESTDATA_DIR}/test-pr-merged.json" 202 "Merged PR (test)" || ((failed++)) - response=$(curl -s -w "\n%{http_code}" \ - -X POST "$APP_URL/webhook" \ - -H "Content-Type: application/json" \ - -H "X-GitHub-Event: pull_request" \ - -H "X-Hub-Signature-256: $signature" \ - -d "$payload") + if [[ "$QUICK_MODE" == "true" ]]; then + log_always -e " ${YELLOW}Quick mode: skipping additional payloads${NC}" + return $failed + fi - http_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') + # Test various PR scenarios + send_webhook "${TESTDATA_DIR}/pr-closed-not-merged.json" 202 "Closed (not merged)" || ((failed++)) + send_webhook "${TESTDATA_DIR}/pr-opened.json" 202 "PR opened" || ((failed++)) + send_webhook "${TESTDATA_DIR}/pr-synchronize.json" 202 "PR synchronized" || ((failed++)) + send_webhook "${TESTDATA_DIR}/pr-multiple-workflows.json" 202 "Multiple workflows" || ((failed++)) + send_webhook "${TESTDATA_DIR}/pr-no-matching-files.json" 202 "No matching files" || ((failed++)) + send_webhook "${TESTDATA_DIR}/pr-large-changeset.json" 202 "Large changeset" || ((failed++)) + send_webhook "${TESTDATA_DIR}/pr-renamed-files.json" 202 "Renamed files" || ((failed++)) + send_webhook "${TESTDATA_DIR}/pr-with-deprecations.json" 202 "With deprecations" || ((failed++)) - if [[ "$http_code" == "200" ]]; then - log_info "Webhook accepted (HTTP $http_code)" - echo "$body" + # Test push event (if supported) + send_webhook "${TESTDATA_DIR}/push-to-main.json" 202 "Push to main" || ((failed++)) + + return $failed +} + +# Verify metrics after tests +verify_metrics() { + log_always "" + log_always -n "Verifying metrics... " + + local metrics + metrics=$(curl -s "${METRICS_ENDPOINT}") + + # Check that webhooks were received + local received + received=$(echo "$metrics" | grep -o '"received":[0-9]*' | grep -o '[0-9]*' || echo "0") + + if [[ "$received" -gt 0 ]]; then + log_always -e "$PASS (received: $received)" return 0 else - log_error "Webhook rejected (HTTP $http_code)" - echo "$body" - return 1 + log_always -e "${YELLOW}WARN${NC} (no webhooks recorded)" + return 0 # Not a failure, might be expected in some configs fi } -# Verify files in destination repo (requires gh CLI) -verify_dest_repo() { - local repo="$1" - local path="$2" +# Check logs for errors +check_logs() { + log_always -n "Checking for errors... " - log_info "Checking $repo for $path..." - if gh api "repos/$repo/contents/$path" > /dev/null 2>&1; then - log_info "โœ“ Found $path in $repo" - return 0 + if grep -q '"level":"ERROR"' /tmp/integration-test-app.log 2>/dev/null; then + local error_count + error_count=$(grep -c '"level":"ERROR"' /tmp/integration-test-app.log) + log_always -e "${YELLOW}WARN${NC} ($error_count errors in log)" + if [[ "$VERBOSE" == "true" ]]; then + grep '"level":"ERROR"' /tmp/integration-test-app.log | head -5 + fi else - log_warn "โœ— Not found: $path in $repo" - return 1 + log_always -e "$PASS" fi } -# Main +# Main execution main() { - echo "==========================================" - echo "GitHub Copier Integration Test" - echo "==========================================" - - case "${1:-webhook}" in - webhook) - check_app || exit 1 - send_webhook "$PAYLOAD_FILE" - ;; - verify) - log_info "Verifying destination repos..." - verify_dest_repo "cbullinger/copier-app-dest-1" "go-examples" - verify_dest_repo "cbullinger/copier-app-dest-2" "python-examples" - ;; - full) - check_app || exit 1 - send_webhook "$PAYLOAD_FILE" - log_info "Waiting 10s for processing..." - sleep 10 - verify_dest_repo "cbullinger/copier-app-dest-1" "go-examples" - verify_dest_repo "cbullinger/copier-app-dest-2" "python-examples" - ;; - *) - echo "Usage: $0 [webhook|verify|full]" - echo " webhook - Send test webhook to app (default)" - echo " verify - Check destination repos for expected files" - echo " full - Send webhook and verify results" - exit 1 - ;; - esac + log_always "" + log_always -e "${BLUE}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" + log_always -e "${BLUE}โ•‘ GitHub Copier - Integration Tests โ•‘${NC}" + log_always -e "${BLUE}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" + log_always "" + + local total_failed=0 + + # Build + build_app + + # Start app + start_app + + log_always "" + log_always -e "${BLUE}Running endpoint tests:${NC}" + + # Test endpoints + test_health || ((total_failed++)) + test_metrics || ((total_failed++)) + + # Test webhooks + test_webhooks || ((total_failed++)) + + # Verify results + verify_metrics + check_logs + + log_always "" + if [[ $total_failed -eq 0 ]]; then + log_always -e "${GREEN}โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" + log_always -e "${GREEN} All integration tests passed!${NC}" + log_always -e "${GREEN}โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" + exit 0 + else + log_always -e "${RED}โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" + log_always -e "${RED} $total_failed test(s) failed${NC}" + log_always -e "${RED}โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" + exit 1 + fi } -main "$@" - +main diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..f3c5ba0 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,207 @@ +#!/bin/bash +# Create a versioned release: update CHANGELOG, commit, tag, push, and create GitHub Release. +# +# Usage: +# ./scripts/release.sh +# ./scripts/release.sh v1.0.0 +# ./scripts/release.sh v0.5.1 --dry-run # show what would happen without making changes +# +# Prerequisites: +# - git (with a clean working tree on main branch) +# - gh (GitHub CLI, authenticated) +# - CHANGELOG.md with an [Unreleased] section (Keep a Changelog format) +# +# What it does: +# 1. Validates the version tag format (vMAJOR.MINOR.PATCH) +# 2. Checks that the working tree is clean and on the main branch +# 3. Renames [Unreleased] โ†’ [vX.Y.Z] - YYYY-MM-DD in CHANGELOG.md +# 4. Adds a fresh [Unreleased] section +# 5. Commits the CHANGELOG update +# 6. Creates an annotated git tag +# 7. Pushes the tag (triggers CI deploy) +# 8. Creates a GitHub Release with the changelog excerpt + +set -euo pipefail + +# โ”€โ”€โ”€ Colors โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# โ”€โ”€โ”€ Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +info() { echo -e "${BLUE}โ„น ${NC}$*"; } +ok() { echo -e "${GREEN}โœ“ ${NC}$*"; } +warn() { echo -e "${YELLOW}โš  ${NC}$*"; } +err() { echo -e "${RED}โœ— ${NC}$*" >&2; } +die() { err "$@"; exit 1; } + +# โ”€โ”€โ”€ Parse arguments โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +DRY_RUN=false +VERSION="" + +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + v*) VERSION="$arg" ;; + *) die "Unknown argument: $arg" ;; + esac +done + +if [ -z "$VERSION" ]; then + echo "Usage: $0 [--dry-run]" + echo "" + echo "Examples:" + echo " $0 v1.0.0" + echo " $0 v0.5.1 --dry-run" + exit 1 +fi + +# โ”€โ”€โ”€ Validate version format โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +if ! echo "$VERSION" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+$'; then + die "Invalid version format: $VERSION (expected vMAJOR.MINOR.PATCH, e.g. v1.0.0)" +fi + +# โ”€โ”€โ”€ Validate prerequisites โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +command -v git >/dev/null 2>&1 || die "git is required" +command -v gh >/dev/null 2>&1 || die "gh (GitHub CLI) is required" + +# Move to repo root +REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || die "Not inside a git repository" +cd "$REPO_ROOT" + +# Check clean working tree +if ! git diff --quiet || ! git diff --cached --quiet; then + die "Working tree is not clean. Commit or stash changes first." +fi + +# Check we're on main +BRANCH="$(git rev-parse --abbrev-ref HEAD)" +if [ "$BRANCH" != "main" ]; then + die "Must be on main branch (currently on: $BRANCH)" +fi + +# Check tag doesn't already exist +if git rev-parse "$VERSION" >/dev/null 2>&1; then + die "Tag $VERSION already exists" +fi + +# Check CHANGELOG.md exists and has [Unreleased] +CHANGELOG="CHANGELOG.md" +if [ ! -f "$CHANGELOG" ]; then + die "$CHANGELOG not found" +fi + +if ! grep -q '^\## \[Unreleased\]' "$CHANGELOG"; then + die "$CHANGELOG does not contain an [Unreleased] section" +fi + +# Check that [Unreleased] has content (not just the heading) +UNRELEASED_CONTENT=$(sed -n '/^## \[Unreleased\]/,/^## \[/{/^## \[/d;p;}' "$CHANGELOG" | grep -v '^$' | head -1) +if [ -z "$UNRELEASED_CONTENT" ]; then + die "[Unreleased] section in $CHANGELOG is empty. Nothing to release." +fi + +# โ”€โ”€โ”€ Show plan โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +TODAY=$(date +%Y-%m-%d) + +echo "" +echo -e "${BLUE}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" +echo -e "${BLUE}โ•‘ Release: ${VERSION}$(printf '%*s' $((42 - ${#VERSION})) '') โ•‘${NC}" +echo -e "${BLUE}โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ${NC}" +echo -e "${BLUE}โ•‘${NC} Date: ${TODAY} ${BLUE}โ•‘${NC}" +echo -e "${BLUE}โ•‘${NC} Branch: ${BRANCH} ${BLUE}โ•‘${NC}" +echo -e "${BLUE}โ•‘${NC} Dry run: ${DRY_RUN} ${BLUE}โ•‘${NC}" +echo -e "${BLUE}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" +echo "" + +if [ "$DRY_RUN" = true ]; then + warn "Dry run mode โ€” no changes will be made" + echo "" +fi + +# โ”€โ”€โ”€ Step 1: Update CHANGELOG โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +info "Updating $CHANGELOG..." + +# Replace [Unreleased] with versioned heading and add a new [Unreleased] above it +REPLACEMENT="## [Unreleased]\n\n## [${VERSION}] - ${TODAY}" + +if [ "$DRY_RUN" = true ]; then + info "Would replace '## [Unreleased]' with:" + echo -e " $REPLACEMENT" +else + # Use perl for reliable in-place replacement (works on macOS and Linux) + perl -i -pe "s/^## \\[Unreleased\\]/## [Unreleased]\n\n## [${VERSION}] - ${TODAY}/" "$CHANGELOG" + ok "Updated $CHANGELOG" +fi + +# โ”€โ”€โ”€ Step 2: Extract release notes โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +info "Extracting release notes for $VERSION..." + +# Extract the section between this version and the next ## heading +RELEASE_NOTES=$(sed -n "/^## \[${VERSION}\]/,/^## \[/{/^## \[${VERSION}\]/d;/^## \[/d;p;}" "$CHANGELOG" 2>/dev/null || true) + +# If dry run, extract from the [Unreleased] section instead (CHANGELOG not yet modified) +if [ "$DRY_RUN" = true ]; then + RELEASE_NOTES=$(sed -n '/^## \[Unreleased\]/,/^## \[/{/^## \[/d;p;}' "$CHANGELOG") +fi + +# Trim leading/trailing whitespace +RELEASE_NOTES=$(echo "$RELEASE_NOTES" | sed -e '/./,$!d' -e :a -e '/^\n*$/{$d;N;ba}') + +if [ -z "$RELEASE_NOTES" ]; then + warn "No release notes extracted โ€” the GitHub Release will have minimal content" +fi + +# โ”€โ”€โ”€ Step 3: Commit, tag, push โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +if [ "$DRY_RUN" = true ]; then + info "Would commit: 'Release $VERSION'" + info "Would create annotated tag: $VERSION" + info "Would push tag to origin" + info "Would create GitHub Release: $VERSION" + echo "" + info "Release notes preview:" + echo "---" + echo "$RELEASE_NOTES" + echo "---" + echo "" + ok "Dry run complete. Run without --dry-run to execute." + exit 0 +fi + +info "Committing CHANGELOG update..." +git add "$CHANGELOG" +git commit -m "$(cat </dev/null) +if [ -z "$SERVICE_URL" ]; then + echo -e "${RED}Cloud Run service 'github-copier' not found${NC}" + exit 1 +fi + +echo "Service URL: $SERVICE_URL" +echo "" + # Check health endpoint -echo "๐Ÿ“Š Checking application health..." -HEALTH=$(curl -s https://github-copy-code-examples.ue.r.appspot.com/health) -AUTH_STATUS=$(echo "$HEALTH" | python3 -c "import sys, json; print(json.load(sys.stdin)['github']['authenticated'])") +echo "Checking application health..." +HEALTH=$(curl -s "$SERVICE_URL/health") +echo "$HEALTH" | python3 -m json.tool 2>/dev/null || echo "$HEALTH" +echo "" + +# Check readiness (includes GitHub auth check) +echo "Checking readiness (includes GitHub auth)..." +READY_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$SERVICE_URL/ready") +READY_BODY=$(curl -s "$SERVICE_URL/ready") -if [ "$AUTH_STATUS" == "True" ]; then - echo "โœ… GitHub authentication is working" +if [ "$READY_CODE" == "200" ]; then + echo -e "${GREEN}Readiness check passed (HTTP $READY_CODE)${NC}" + echo "$READY_BODY" | python3 -m json.tool 2>/dev/null || echo "$READY_BODY" else - echo "โŒ GitHub authentication is NOT working" - exit 1 + echo -e "${RED}Readiness check failed (HTTP $READY_CODE)${NC}" + echo "$READY_BODY" | python3 -m json.tool 2>/dev/null || echo "$READY_BODY" fi echo "" -# Check recent logs for 401 errors -echo "๐Ÿ” Checking recent logs for 401 errors..." -RECENT_ERRORS=$(gcloud logging read "resource.type=gae_app AND severity>=ERROR AND textPayload=~'401 Bad credentials'" --limit=5 --format="value(timestamp,textPayload)" --freshness=30m 2>/dev/null) +# Check recent logs for errors +echo "Checking recent logs for errors..." +RECENT_ERRORS=$(gcloud run services logs read github-copier --limit=20 2>/dev/null | grep -i "401\|bad credentials\|unauthorized" || true) if [ -z "$RECENT_ERRORS" ]; then - echo "โœ… No recent 401 errors found!" - echo "" - echo "๐ŸŽ‰ GitHub App can successfully access the repository!" + echo -e "${GREEN}No recent 401 errors found${NC}" else - echo "โŒ Found recent 401 errors:" - echo "" + echo -e "${RED}Found recent authentication errors:${NC}" echo "$RECENT_ERRORS" echo "" - echo "This means the GitHub App cannot access one or more repositories." - echo "" echo "Possible causes:" echo "1. GitHub App is not installed on the repository" echo "2. Installation ID doesn't match the repository" - echo "3. GitHub App doesn't have 'Contents' read permission" + echo "3. GitHub App private key is incorrect or expired" + echo "4. GitHub App doesn't have required permissions (see github-app-manifest.yml)" echo "" echo "To fix:" echo "1. Go to: https://github.com/settings/installations" echo "2. Find your GitHub App installation" echo "3. Make sure $REPO_OWNER/$REPO_NAME is in the list of accessible repositories" - echo "4. If not, click 'Configure' and add it" fi echo "" -echo "๐Ÿ“‹ Summary" +echo "Summary" echo "==========" echo "Repository: $REPO_OWNER/$REPO_NAME" echo "Installation ID: $INSTALLATION_ID" -echo "Authentication: $AUTH_STATUS" +echo "Readiness: HTTP $READY_CODE" -if [ -z "$RECENT_ERRORS" ]; then - echo "Status: โœ… WORKING" +if [ "$READY_CODE" == "200" ] && [ -z "$RECENT_ERRORS" ]; then + echo -e "Status: ${GREEN}WORKING${NC}" exit 0 else - echo "Status: โŒ NEEDS ATTENTION" + echo -e "Status: ${RED}NEEDS ATTENTION${NC}" exit 1 fi - diff --git a/scripts/test-slack.sh b/scripts/test-slack.sh index 1cf5b0f..1e7a25e 100755 --- a/scripts/test-slack.sh +++ b/scripts/test-slack.sh @@ -38,7 +38,7 @@ echo "" # Test 1: Simple message echo -e "${BLUE}Test 1: Sending simple test message...${NC}" curl -X POST -H 'Content-type: application/json' \ - --data '{"text":"๐Ÿงช Test message from examples-copier"}' \ + --data '{"text":"๐Ÿงช Test message from github-copier"}' \ "$WEBHOOK_URL" echo "" echo -e "${GREEN}โœ“ Simple message sent${NC}" diff --git a/scripts/test-with-pr.sh b/scripts/test-with-pr.sh index 01805e3..38d9a19 100755 --- a/scripts/test-with-pr.sh +++ b/scripts/test-with-pr.sh @@ -16,7 +16,7 @@ NC='\033[0m' # No Color PR_NUMBER=${1:-} OWNER=${2:-${REPO_OWNER:-}} REPO=${3:-${REPO_NAME:-}} -WEBHOOK_URL=${WEBHOOK_URL:-http://localhost:8080/webhook} +WEBHOOK_URL=${WEBHOOK_URL:-http://localhost:8080/events} WEBHOOK_SECRET=${WEBHOOK_SECRET:-} # Help message @@ -32,7 +32,7 @@ if [ "$1" == "-h" ] || [ "$1" == "--help" ] || [ -z "$PR_NUMBER" ]; then echo "" echo "Environment Variables:" echo " GITHUB_TOKEN GitHub token for API access (required)" - echo " WEBHOOK_URL Webhook endpoint (default: http://localhost:8080/webhook)" + echo " WEBHOOK_URL Webhook endpoint (default: http://localhost:8080/events)" echo " WEBHOOK_SECRET Webhook secret for signature" echo " REPO_OWNER Default repository owner" echo " REPO_NAME Default repository name" @@ -45,7 +45,7 @@ if [ "$1" == "-h" ] || [ "$1" == "--help" ] || [ -z "$PR_NUMBER" ]; then echo " $0 123 myorg myrepo" echo "" echo " # Test against production" - echo " WEBHOOK_URL=https://myapp.appspot.com/webhook $0 123" + echo " WEBHOOK_URL=https://your-service.run.app/events $0 123" echo "" exit 0 fi @@ -81,7 +81,7 @@ if [[ "$WEBHOOK_URL" == http://localhost* ]]; then echo -e "${BLUE}Checking if application is running...${NC}" if ! curl -s -f "$WEBHOOK_URL" > /dev/null 2>&1; then echo -e "${YELLOW}Warning: Application doesn't seem to be running at $WEBHOOK_URL${NC}" - echo -e "${YELLOW}Start it with: DRY_RUN=true ./examples-copier${NC}" + echo -e "${YELLOW}Start it with: DRY_RUN=true ./github-copier${NC}" read -p "Continue anyway? (y/N) " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then @@ -144,6 +144,6 @@ echo "" echo -e "${GREEN}โœ“ Test complete!${NC}" echo "" echo "Check application logs for processing details:" -echo " Local: Check terminal output" -echo " GCP: gcloud app logs tail -s default" +echo " Local: Check terminal output (JSON via slog)" +echo " Cloud Run: gcloud run services logs read github-copier --limit=50" diff --git a/scripts/validate-config-detailed.py b/scripts/validate-config-detailed.py deleted file mode 100755 index be7d415..0000000 --- a/scripts/validate-config-detailed.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -""" -Detailed validation script for copier-config.yaml files -""" - -import sys -import yaml -import re - -def validate_config(file_path): - """Validate a copier-config.yaml file and report all issues""" - - issues = [] - warnings = [] - - # Try to load the YAML - try: - with open(file_path, 'r') as f: - config = yaml.safe_load(f) - except yaml.YAMLError as e: - print(f"โŒ YAML Parsing Error:") - print(f" {e}") - return False - except Exception as e: - print(f"โŒ Error reading file: {e}") - return False - - print("โœ… YAML syntax is valid") - print() - - # Validate structure - if not isinstance(config, dict): - issues.append("Config must be a dictionary") - return False - - # Check required fields - if 'source_repo' not in config: - issues.append("Missing required field: source_repo") - - if 'copy_rules' not in config: - issues.append("Missing required field: copy_rules") - - if issues: - print("โŒ Structural Issues:") - for issue in issues: - print(f" - {issue}") - return False - - print(f"๐Ÿ“‹ Config Summary:") - print(f" Source: {config.get('source_repo')}") - print(f" Branch: {config.get('source_branch', 'main')}") - print(f" Rules: {len(config.get('copy_rules', []))}") - print() - - # Validate each rule - rules = config.get('copy_rules', []) - for i, rule in enumerate(rules, 1): - rule_name = rule.get('name', f'Rule {i}') - print(f"๐Ÿ” Validating Rule {i}: {rule_name}") - - # Check rule structure - if 'source_pattern' not in rule: - issues.append(f"Rule '{rule_name}': Missing source_pattern") - continue - - if 'targets' not in rule: - issues.append(f"Rule '{rule_name}': Missing targets") - continue - - # Validate source_pattern - pattern = rule['source_pattern'] - if not isinstance(pattern, dict): - issues.append(f"Rule '{rule_name}': source_pattern must be a dictionary") - continue - - pattern_type = pattern.get('type') - pattern_str = pattern.get('pattern') - - if not pattern_type: - issues.append(f"Rule '{rule_name}': Missing pattern type") - elif pattern_type not in ['prefix', 'glob', 'regex']: - issues.append(f"Rule '{rule_name}': Invalid pattern type '{pattern_type}' (must be prefix, glob, or regex)") - - if not pattern_str: - issues.append(f"Rule '{rule_name}': Missing pattern string") - else: - # Check for type/pattern mismatch - has_regex_syntax = bool(re.search(r'\(\?P<\w+>', pattern_str)) - - if pattern_type == 'prefix' and has_regex_syntax: - issues.append(f"Rule '{rule_name}': Pattern type is 'prefix' but pattern contains regex syntax '(?P<...>)'") - warnings.append(f"Rule '{rule_name}': Should use type: 'regex' instead of 'prefix'") - - # Validate regex patterns - if pattern_type == 'regex': - try: - re.compile(pattern_str) - except re.error as e: - issues.append(f"Rule '{rule_name}': Invalid regex pattern: {e}") - - # Validate targets - targets = rule.get('targets', []) - if not isinstance(targets, list): - issues.append(f"Rule '{rule_name}': targets must be a list") - continue - - if len(targets) == 0: - warnings.append(f"Rule '{rule_name}': No targets defined") - - for j, target in enumerate(targets, 1): - if not isinstance(target, dict): - issues.append(f"Rule '{rule_name}', Target {j}: Must be a dictionary") - continue - - # Check required target fields - if 'repo' not in target: - issues.append(f"Rule '{rule_name}', Target {j}: Missing 'repo' field") - - if 'branch' not in target: - warnings.append(f"Rule '{rule_name}', Target {j}: Missing 'branch' field (will use default)") - - if 'path_transform' not in target: - warnings.append(f"Rule '{rule_name}', Target {j}: Missing 'path_transform' field") - - # Validate commit_strategy - if 'commit_strategy' in target: - strategy = target['commit_strategy'] - if not isinstance(strategy, dict): - issues.append(f"Rule '{rule_name}', Target {j}: commit_strategy must be a dictionary") - else: - strategy_type = strategy.get('type') - if strategy_type and strategy_type not in ['direct', 'pull_request']: - issues.append(f"Rule '{rule_name}', Target {j}: Invalid commit_strategy type '{strategy_type}'") - - print(f" โœ“ Rule validated") - - print() - - # Print summary - if issues: - print("โŒ VALIDATION FAILED") - print() - print("Issues found:") - for issue in issues: - print(f" โŒ {issue}") - print() - - if warnings: - print("โš ๏ธ Warnings:") - for warning in warnings: - print(f" โš ๏ธ {warning}") - print() - - if not issues and not warnings: - print("โœ… Configuration is valid with no issues!") - return True - elif not issues: - print("โœ… Configuration is valid (with warnings)") - return True - else: - return False - -if __name__ == '__main__': - if len(sys.argv) != 2: - print("Usage: validate-config-detailed.py ") - sys.exit(1) - - file_path = sys.argv[1] - success = validate_config(file_path) - sys.exit(0 if success else 1) - diff --git a/services/audit_logger.go b/services/audit_logger.go index b40a62c..7fa6fca 100644 --- a/services/audit_logger.go +++ b/services/audit_logger.go @@ -5,9 +5,9 @@ import ( "fmt" "time" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" ) // AuditEventType represents the type of audit event @@ -21,21 +21,21 @@ const ( // AuditEvent represents an audit log entry type AuditEvent struct { - ID string `bson:"_id,omitempty"` - Timestamp time.Time `bson:"timestamp"` - EventType AuditEventType `bson:"event_type"` - RuleName string `bson:"rule_name,omitempty"` - SourceRepo string `bson:"source_repo"` - SourcePath string `bson:"source_path"` - TargetRepo string `bson:"target_repo,omitempty"` - TargetPath string `bson:"target_path,omitempty"` - CommitSHA string `bson:"commit_sha,omitempty"` - PRNumber int `bson:"pr_number,omitempty"` - Success bool `bson:"success"` - ErrorMessage string `bson:"error_message,omitempty"` - DurationMs int64 `bson:"duration_ms,omitempty"` - FileSize int64 `bson:"file_size,omitempty"` - AdditionalData map[string]any `bson:"additional_data,omitempty"` + ID string `bson:"_id,omitempty"` + Timestamp time.Time `bson:"timestamp"` + EventType AuditEventType `bson:"event_type"` + RuleName string `bson:"rule_name,omitempty"` + SourceRepo string `bson:"source_repo"` + SourcePath string `bson:"source_path"` + TargetRepo string `bson:"target_repo,omitempty"` + TargetPath string `bson:"target_path,omitempty"` + CommitSHA string `bson:"commit_sha,omitempty"` + PRNumber int `bson:"pr_number,omitempty"` + Success bool `bson:"success"` + ErrorMessage string `bson:"error_message,omitempty"` + DurationMs int64 `bson:"duration_ms,omitempty"` + FileSize int64 `bson:"file_size,omitempty"` + AdditionalData map[string]any `bson:"additional_data,omitempty"` } // AuditLogger handles audit logging to MongoDB @@ -48,24 +48,25 @@ type AuditLogger interface { GetEventsByRule(ctx context.Context, ruleName string, limit int) ([]AuditEvent, error) GetStatsByRule(ctx context.Context) (map[string]RuleStats, error) GetDailyVolume(ctx context.Context, days int) ([]DailyStats, error) + Ping(ctx context.Context) error Close(ctx context.Context) error } // RuleStats represents statistics for a specific rule type RuleStats struct { - RuleName string `bson:"_id"` - TotalCopies int `bson:"total_copies"` - SuccessCount int `bson:"success_count"` - FailureCount int `bson:"failure_count"` + RuleName string `bson:"_id"` + TotalCopies int `bson:"total_copies"` + SuccessCount int `bson:"success_count"` + FailureCount int `bson:"failure_count"` AvgDuration float64 `bson:"avg_duration"` } // DailyStats represents daily copy volume statistics type DailyStats struct { - Date string `bson:"_id"` - TotalCopies int `bson:"total_copies"` - SuccessCount int `bson:"success_count"` - FailureCount int `bson:"failure_count"` + Date string `bson:"_id"` + TotalCopies int `bson:"total_copies"` + SuccessCount int `bson:"success_count"` + FailureCount int `bson:"failure_count"` } // MongoAuditLogger implements AuditLogger using MongoDB @@ -85,8 +86,14 @@ func NewMongoAuditLogger(ctx context.Context, mongoURI, database, collection str return nil, fmt.Errorf("MONGO_URI is required when audit logging is enabled") } - clientOptions := options.Client().ApplyURI(mongoURI) - client, err := mongo.Connect(ctx, clientOptions) + clientOptions := options.Client(). + ApplyURI(mongoURI). + SetServerSelectionTimeout(5 * time.Second). + SetConnectTimeout(5 * time.Second). + SetTimeout(10 * time.Second). + SetMaxPoolSize(10). + SetRetryWrites(true) + client, err := mongo.Connect(clientOptions) if err != nil { return nil, fmt.Errorf("failed to connect to MongoDB: %w", err) } @@ -168,7 +175,7 @@ func (mal *MongoAuditLogger) GetRecentEvents(ctx context.Context, limit int) ([] if err != nil { return nil, err } - defer cursor.Close(ctx) + defer func() { _ = cursor.Close(ctx) }() var events []AuditEvent if err := cursor.All(ctx, &events); err != nil { @@ -185,7 +192,7 @@ func (mal *MongoAuditLogger) GetFailedEvents(ctx context.Context, limit int) ([] if err != nil { return nil, err } - defer cursor.Close(ctx) + defer func() { _ = cursor.Close(ctx) }() var events []AuditEvent if err := cursor.All(ctx, &events); err != nil { @@ -202,7 +209,7 @@ func (mal *MongoAuditLogger) GetEventsByRule(ctx context.Context, ruleName strin if err != nil { return nil, err } - defer cursor.Close(ctx) + defer func() { _ = cursor.Close(ctx) }() var events []AuditEvent if err := cursor.All(ctx, &events); err != nil { @@ -228,7 +235,7 @@ func (mal *MongoAuditLogger) GetStatsByRule(ctx context.Context) (map[string]Rul if err != nil { return nil, err } - defer cursor.Close(ctx) + defer func() { _ = cursor.Close(ctx) }() var stats []RuleStats if err := cursor.All(ctx, &stats); err != nil { @@ -245,7 +252,7 @@ func (mal *MongoAuditLogger) GetStatsByRule(ctx context.Context) (map[string]Rul // GetDailyVolume retrieves daily copy volume statistics func (mal *MongoAuditLogger) GetDailyVolume(ctx context.Context, days int) ([]DailyStats, error) { startDate := time.Now().AddDate(0, 0, -days) - + pipeline := mongo.Pipeline{ {{Key: "$match", Value: bson.M{ "event_type": AuditEventCopy, @@ -269,7 +276,7 @@ func (mal *MongoAuditLogger) GetDailyVolume(ctx context.Context, days int) ([]Da if err != nil { return nil, err } - defer cursor.Close(ctx) + defer func() { _ = cursor.Close(ctx) }() var stats []DailyStats if err := cursor.All(ctx, &stats); err != nil { @@ -278,7 +285,12 @@ func (mal *MongoAuditLogger) GetDailyVolume(ctx context.Context, days int) ([]Da return stats, nil } -// Close closes the MongoDB connection +// Ping checks MongoDB connectivity. +func (mal *MongoAuditLogger) Ping(ctx context.Context) error { + return mal.client.Ping(ctx, nil) +} + +// Close closes the MongoDB connection. func (mal *MongoAuditLogger) Close(ctx context.Context) error { return mal.client.Disconnect(ctx) } @@ -286,9 +298,11 @@ func (mal *MongoAuditLogger) Close(ctx context.Context) error { // NoOpAuditLogger is a no-op implementation when audit logging is disabled type NoOpAuditLogger struct{} -func (nal *NoOpAuditLogger) LogCopyEvent(ctx context.Context, event *AuditEvent) error { return nil } -func (nal *NoOpAuditLogger) LogDeprecationEvent(ctx context.Context, event *AuditEvent) error { return nil } -func (nal *NoOpAuditLogger) LogErrorEvent(ctx context.Context, event *AuditEvent) error { return nil } +func (nal *NoOpAuditLogger) LogCopyEvent(ctx context.Context, event *AuditEvent) error { return nil } +func (nal *NoOpAuditLogger) LogDeprecationEvent(ctx context.Context, event *AuditEvent) error { + return nil +} +func (nal *NoOpAuditLogger) LogErrorEvent(ctx context.Context, event *AuditEvent) error { return nil } func (nal *NoOpAuditLogger) GetRecentEvents(ctx context.Context, limit int) ([]AuditEvent, error) { return []AuditEvent{}, nil } @@ -304,5 +318,5 @@ func (nal *NoOpAuditLogger) GetStatsByRule(ctx context.Context) (map[string]Rule func (nal *NoOpAuditLogger) GetDailyVolume(ctx context.Context, days int) ([]DailyStats, error) { return []DailyStats{}, nil } +func (nal *NoOpAuditLogger) Ping(ctx context.Context) error { return nil } func (nal *NoOpAuditLogger) Close(ctx context.Context) error { return nil } - diff --git a/services/audit_logger_test.go b/services/audit_logger_test.go index 2aed1e8..1f89afa 100644 --- a/services/audit_logger_test.go +++ b/services/audit_logger_test.go @@ -8,7 +8,7 @@ import ( func TestNewMongoAuditLogger_Disabled(t *testing.T) { ctx := context.Background() - + // When enabled=false, should return NoOpAuditLogger logger, err := NewMongoAuditLogger(ctx, "", "testdb", "testcoll", false) if err != nil { @@ -28,7 +28,7 @@ func TestNewMongoAuditLogger_Disabled(t *testing.T) { func TestNewMongoAuditLogger_EnabledWithoutURI(t *testing.T) { ctx := context.Background() - + // When enabled=true but no URI, should return error _, err := NewMongoAuditLogger(ctx, "", "testdb", "testcoll", true) if err == nil { @@ -46,17 +46,17 @@ func TestNoOpAuditLogger_LogCopyEvent(t *testing.T) { ctx := context.Background() event := &AuditEvent{ - EventType: AuditEventCopy, - RuleName: "test-rule", - SourceRepo: "test/source", - SourcePath: "test.go", - TargetRepo: "test/target", - TargetPath: "copied/test.go", - CommitSHA: "abc123", - PRNumber: 123, - Success: true, - DurationMs: 100, - FileSize: 1024, + EventType: AuditEventCopy, + RuleName: "test-rule", + SourceRepo: "test/source", + SourcePath: "test.go", + TargetRepo: "test/target", + TargetPath: "copied/test.go", + CommitSHA: "abc123", + PRNumber: 123, + Success: true, + DurationMs: 100, + FileSize: 1024, } err := logger.LogCopyEvent(ctx, event) @@ -301,4 +301,3 @@ func TestDailyStats_Structure(t *testing.T) { t.Errorf("TotalCopies = %d, want 50", stats.TotalCopies) } } - diff --git a/services/config_cache.go b/services/config_cache.go new file mode 100644 index 0000000..49d2083 --- /dev/null +++ b/services/config_cache.go @@ -0,0 +1,91 @@ +package services + +import ( + "context" + "sync" + "time" + + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/types" +) + +// CachedConfigLoader wraps a ConfigLoader and caches the resolved *YAMLConfig +// for a configurable TTL. Subsequent calls within the TTL window return the +// cached config without making any GitHub API calls. +// +// The cache is safe for concurrent access. It is keyed on the effective config +// file path, so different configs don't collide. +type CachedConfigLoader struct { + inner ConfigLoader + ttl time.Duration + + mu sync.RWMutex + entries map[string]*cacheEntry +} + +type cacheEntry struct { + config *types.YAMLConfig + fetchedAt time.Time +} + +// NewCachedConfigLoader wraps inner with a TTL cache. +// A TTL of 0 disables caching (every call delegates directly). +func NewCachedConfigLoader(inner ConfigLoader, ttl time.Duration) ConfigLoader { + if ttl <= 0 { + return inner + } + return &CachedConfigLoader{ + inner: inner, + ttl: ttl, + entries: make(map[string]*cacheEntry), + } +} + +// LoadConfig returns a cached config if fresh, otherwise delegates to the inner loader. +func (c *CachedConfigLoader) LoadConfig(ctx context.Context, config *configs.Config) (*types.YAMLConfig, error) { + key := config.EffectiveConfigFile() + + // Fast path: check under read lock + c.mu.RLock() + if entry, ok := c.entries[key]; ok && time.Since(entry.fetchedAt) < c.ttl { + c.mu.RUnlock() + LogInfoCtx(ctx, "using cached workflow config", map[string]interface{}{ + "config_file": key, + "age_seconds": int(time.Since(entry.fetchedAt).Seconds()), + "ttl_seconds": int(c.ttl.Seconds()), + }) + return entry.config, nil + } + c.mu.RUnlock() + + // Slow path: fetch and cache + result, err := c.inner.LoadConfig(ctx, config) + if err != nil { + return nil, err + } + + c.mu.Lock() + c.entries[key] = &cacheEntry{config: result, fetchedAt: time.Now()} + c.mu.Unlock() + + LogInfoCtx(ctx, "cached workflow config", map[string]interface{}{ + "config_file": key, + "ttl_seconds": int(c.ttl.Seconds()), + }) + + return result, nil +} + +// LoadConfigFromContent delegates directly โ€” content-based loads are not cached +// since the caller already has the content in hand. +func (c *CachedConfigLoader) LoadConfigFromContent(content string, filename string) (*types.YAMLConfig, error) { + return c.inner.LoadConfigFromContent(content, filename) +} + +// InvalidateCache clears all cached entries. Useful after a config-repo push event. +func (c *CachedConfigLoader) InvalidateCache() { + c.mu.Lock() + c.entries = make(map[string]*cacheEntry) + c.mu.Unlock() + LogInfo("workflow config cache invalidated") +} diff --git a/services/config_loader.go b/services/config_loader.go index f3567dd..ad657db 100644 --- a/services/config_loader.go +++ b/services/config_loader.go @@ -4,8 +4,9 @@ import ( "context" "fmt" "os" + "strings" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "gopkg.in/yaml.v3" "github.com/grove-platform/github-copier/configs" @@ -18,10 +19,13 @@ type ConfigLoader interface { LoadConfigFromContent(content string, filename string) (*types.YAMLConfig, error) } -// DefaultConfigLoader implements the ConfigLoader interface +// DefaultConfigLoader implements the ConfigLoader interface for the legacy +// single-file config format. Deprecated: migrate to the main config format +// (USE_MAIN_CONFIG=true) and use DefaultMainConfigLoader instead. type DefaultConfigLoader struct{} -// NewConfigLoader creates a new config loader +// NewConfigLoader creates a new legacy config loader. +// Deprecated: use NewMainConfigLoader instead. func NewConfigLoader() ConfigLoader { return &DefaultConfigLoader{} } @@ -51,14 +55,14 @@ func (cl *DefaultConfigLoader) LoadConfig(ctx context.Context, config *configs.C // LoadConfigFromContent loads configuration from a string func (cl *DefaultConfigLoader) LoadConfigFromContent(content string, filename string) (*types.YAMLConfig, error) { if content == "" { - return nil, fmt.Errorf("config file is empty") + return nil, fmt.Errorf("%w: config file is empty", ErrConfigLoad) } // Parse as YAML (supports both YAML and JSON since YAML is a superset of JSON) var yamlConfig types.YAMLConfig err := yaml.Unmarshal([]byte(content), &yamlConfig) if err != nil { - return nil, fmt.Errorf("failed to parse config file: %w", err) + return nil, fmt.Errorf("%w: failed to parse config file: %v", ErrConfigLoad, err) } // Set defaults @@ -66,7 +70,7 @@ func (cl *DefaultConfigLoader) LoadConfigFromContent(content string, filename st // Validate if err := yamlConfig.Validate(); err != nil { - return nil, fmt.Errorf("config validation failed: %w", err) + return nil, fmt.Errorf("%w: %v", ErrConfigValidation, err) } return &yamlConfig, nil @@ -75,7 +79,7 @@ func (cl *DefaultConfigLoader) LoadConfigFromContent(content string, filename st // retrieveConfigFileContent fetches the config file content from the repository func retrieveConfigFileContent(ctx context.Context, filePath string, config *configs.Config) (string, error) { // Get GitHub client for the config repo's org (auto-discovers installation ID) - client, err := GetRestClientForOrg(config.ConfigRepoOwner) + client, err := GetRestClientForOrg(ctx, config, config.ConfigRepoOwner) if err != nil { return "", fmt.Errorf("failed to get GitHub client for org %s: %w", config.ConfigRepoOwner, err) } @@ -91,10 +95,15 @@ func retrieveConfigFileContent(ctx context.Context, filePath string, config *con }, ) if err != nil { + // Check if this is an authentication error + errStr := err.Error() + if strings.Contains(errStr, "401") || strings.Contains(errStr, "Bad credentials") { + return "", fmt.Errorf("%w: unable to fetch config file. The GitHub App private key (PEM) may be invalid or expired. Please check the CODE_COPIER_PEM secret in GCP Secret Manager. Original error: %v", ErrAuthentication, err) + } return "", fmt.Errorf("failed to get config file: %w", err) } if fileContent == nil { - return "", fmt.Errorf("config file content is nil for path: %s", filePath) + return "", fmt.Errorf("%w: config file at path: %s", ErrContentNil, filePath) } // Decode content @@ -106,54 +115,11 @@ func retrieveConfigFileContent(ctx context.Context, filePath string, config *con return content, nil } -// ValidateConfig validates a YAML configuration -func ValidateConfig(config *types.YAMLConfig) error { - return config.Validate() -} - -// ConfigValidator provides validation utilities -type ConfigValidator struct{} - -// NewConfigValidator creates a new config validator -func NewConfigValidator() *ConfigValidator { - return &ConfigValidator{} -} - -// ValidatePattern validates a pattern and returns any errors -func (cv *ConfigValidator) ValidatePattern(patternType types.PatternType, pattern string) error { - sp := types.SourcePattern{ - Type: patternType, - Pattern: pattern, - } - return sp.Validate() -} - -// TestPattern tests a pattern against a file path -func (cv *ConfigValidator) TestPattern(patternType types.PatternType, pattern string, filePath string) (types.MatchResult, error) { - sp := types.SourcePattern{ - Type: patternType, - Pattern: pattern, - } - - if err := sp.Validate(); err != nil { - return types.NewMatchResult(false, nil), err - } - - matcher := NewPatternMatcher() - return matcher.Match(filePath, sp), nil -} - -// TestTransform tests a path transformation -func (cv *ConfigValidator) TestTransform(sourcePath string, template string, variables map[string]string) (string, error) { - transformer := NewPathTransformer() - return transformer.Transform(sourcePath, template, variables) -} - // loadLocalConfigFile attempts to load config from a local file // This is useful for local testing and development func loadLocalConfigFile(filename string) (string, error) { // Try to read from current directory - data, err := os.ReadFile(filename) + data, err := os.ReadFile(filename) // #nosec G304 -- local dev config path from caller if err != nil { return "", err } diff --git a/services/delivery_tracker.go b/services/delivery_tracker.go new file mode 100644 index 0000000..fe36de0 --- /dev/null +++ b/services/delivery_tracker.go @@ -0,0 +1,90 @@ +package services + +import ( + "sync" + "time" +) + +// DeliveryTracker tracks processed GitHub webhook delivery IDs to prevent +// duplicate processing. GitHub retries deliveries on timeout or error, and +// the X-GitHub-Delivery header uniquely identifies each delivery. +// +// This is an in-memory implementation suitable for single-instance deployments. +// For multi-instance deployments, replace with a shared store (e.g. MongoDB or Redis). +type DeliveryTracker struct { + mu sync.Mutex + entries map[string]time.Time + ttl time.Duration + + // stopCleanup signals the background goroutine to stop + stopCleanup chan struct{} +} + +// NewDeliveryTracker creates a tracker that expires entries after the given TTL. +// A background goroutine periodically purges expired entries. +func NewDeliveryTracker(ttl time.Duration) *DeliveryTracker { + dt := &DeliveryTracker{ + entries: make(map[string]time.Time), + ttl: ttl, + stopCleanup: make(chan struct{}), + } + go dt.cleanupLoop() + return dt +} + +// TryRecord attempts to record a delivery ID. Returns true if the ID is new +// (not a duplicate), false if it was already seen within the TTL window. +func (dt *DeliveryTracker) TryRecord(deliveryID string) bool { + dt.mu.Lock() + defer dt.mu.Unlock() + + if seenAt, exists := dt.entries[deliveryID]; exists { + if time.Since(seenAt) < dt.ttl { + return false // duplicate within TTL + } + // Expired entry โ€” allow reprocessing + } + + dt.entries[deliveryID] = time.Now() + return true +} + +// Len returns the current number of tracked delivery IDs (for diagnostics). +func (dt *DeliveryTracker) Len() int { + dt.mu.Lock() + defer dt.mu.Unlock() + return len(dt.entries) +} + +// Stop halts the background cleanup goroutine. +func (dt *DeliveryTracker) Stop() { + close(dt.stopCleanup) +} + +// cleanupLoop periodically removes expired entries to bound memory usage. +func (dt *DeliveryTracker) cleanupLoop() { + ticker := time.NewTicker(dt.ttl / 2) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + dt.purgeExpired() + case <-dt.stopCleanup: + return + } + } +} + +// purgeExpired removes all entries older than TTL. +func (dt *DeliveryTracker) purgeExpired() { + dt.mu.Lock() + defer dt.mu.Unlock() + + now := time.Now() + for id, seenAt := range dt.entries { + if now.Sub(seenAt) >= dt.ttl { + delete(dt.entries, id) + } + } +} diff --git a/services/delivery_tracker_test.go b/services/delivery_tracker_test.go new file mode 100644 index 0000000..0ebb725 --- /dev/null +++ b/services/delivery_tracker_test.go @@ -0,0 +1,137 @@ +package services + +import ( + "fmt" + "sync" + "testing" + "time" +) + +func TestDeliveryTracker_TryRecord(t *testing.T) { + dt := NewDeliveryTracker(1 * time.Hour) + defer dt.Stop() + + // First call should succeed + if !dt.TryRecord("delivery-1") { + t.Error("expected first TryRecord to return true") + } + + // Duplicate should be rejected + if dt.TryRecord("delivery-1") { + t.Error("expected duplicate TryRecord to return false") + } + + // Different ID should succeed + if !dt.TryRecord("delivery-2") { + t.Error("expected TryRecord for new ID to return true") + } +} + +func TestDeliveryTracker_TTLExpiry(t *testing.T) { + dt := NewDeliveryTracker(50 * time.Millisecond) + defer dt.Stop() + + if !dt.TryRecord("delivery-1") { + t.Error("expected first TryRecord to return true") + } + + // Wait for TTL to expire + time.Sleep(60 * time.Millisecond) + + // Should allow reprocessing after expiry + if !dt.TryRecord("delivery-1") { + t.Error("expected TryRecord to return true after TTL expiry") + } +} + +func TestDeliveryTracker_Len(t *testing.T) { + dt := NewDeliveryTracker(1 * time.Hour) + defer dt.Stop() + + if dt.Len() != 0 { + t.Errorf("expected Len()=0, got %d", dt.Len()) + } + + dt.TryRecord("a") + dt.TryRecord("b") + dt.TryRecord("c") + dt.TryRecord("a") // duplicate, should not increase count + + if dt.Len() != 3 { + t.Errorf("expected Len()=3, got %d", dt.Len()) + } +} + +func TestDeliveryTracker_PurgeExpired(t *testing.T) { + dt := NewDeliveryTracker(50 * time.Millisecond) + defer dt.Stop() + + dt.TryRecord("a") + dt.TryRecord("b") + + if dt.Len() != 2 { + t.Errorf("expected Len()=2, got %d", dt.Len()) + } + + // Wait for expiry + cleanup cycle (ttl/2 = 25ms, so total ~75ms should trigger) + time.Sleep(100 * time.Millisecond) + + if dt.Len() != 0 { + t.Errorf("expected Len()=0 after purge, got %d", dt.Len()) + } +} + +func TestDeliveryTracker_ConcurrentAccess(t *testing.T) { + dt := NewDeliveryTracker(1 * time.Hour) + defer dt.Stop() + + const goroutines = 50 + var wg sync.WaitGroup + results := make([]bool, goroutines) + + // All goroutines try to record the same delivery ID + wg.Add(goroutines) + for i := 0; i < goroutines; i++ { + go func(idx int) { + defer wg.Done() + results[idx] = dt.TryRecord("same-delivery") + }(i) + } + wg.Wait() + + // Exactly one goroutine should succeed + successCount := 0 + for _, ok := range results { + if ok { + successCount++ + } + } + if successCount != 1 { + t.Errorf("expected exactly 1 success, got %d", successCount) + } +} + +func TestDeliveryTracker_ConcurrentDifferentIDs(t *testing.T) { + dt := NewDeliveryTracker(1 * time.Hour) + defer dt.Stop() + + const goroutines = 100 + var wg sync.WaitGroup + + // Each goroutine records a unique ID + wg.Add(goroutines) + for i := 0; i < goroutines; i++ { + go func(idx int) { + defer wg.Done() + id := fmt.Sprintf("delivery-%d", idx) + if !dt.TryRecord(id) { + t.Errorf("expected TryRecord(%s) to return true", id) + } + }(i) + } + wg.Wait() + + if dt.Len() != goroutines { + t.Errorf("expected Len()=%d, got %d", goroutines, dt.Len()) + } +} diff --git a/services/errors.go b/services/errors.go new file mode 100644 index 0000000..e0ac1b9 --- /dev/null +++ b/services/errors.go @@ -0,0 +1,91 @@ +package services + +import ( + "errors" + "net/http" + + "github.com/google/go-github/v82/github" +) + +// Sentinel errors for common failure modes. Wrap these with fmt.Errorf("%w", ...) +// so callers can use errors.Is() to detect specific failure categories. +var ( + // ErrAuthentication indicates a GitHub App authentication failure + // (invalid PEM, expired key, bad JWT, etc.). + ErrAuthentication = errors.New("github app authentication failed") + + // ErrSecretAccess indicates a failure to retrieve a secret from + // GCP Secret Manager or from the local environment fallback. + ErrSecretAccess = errors.New("secret access failed") + + // ErrConfigLoad indicates a failure to load or parse the YAML configuration file. + ErrConfigLoad = errors.New("config load failed") + + // ErrConfigValidation indicates the configuration was loaded but failed validation. + ErrConfigValidation = errors.New("config validation failed") + + // ErrContentNil indicates that the GitHub API returned a nil content body + // for a file that was expected to exist. + ErrContentNil = errors.New("file content is nil") + + // ErrInstallationNotFound indicates that no GitHub App installation was found + // for the given organization. + ErrInstallationNotFound = errors.New("no installation found for organization") + + // ErrMergeConflict indicates a PR or ref update could not be completed + // due to merge conflicts or non-fast-forward push. + ErrMergeConflict = errors.New("merge conflict") +) + +// permanentSentinels lists sentinel errors that indicate a permanent failure. +// These errors will not resolve by retrying the same operation. +var permanentSentinels = []error{ + ErrAuthentication, + ErrConfigLoad, + ErrConfigValidation, + ErrInstallationNotFound, + ErrMergeConflict, +} + +// permanentHTTPStatuses lists HTTP status codes from the GitHub API that +// indicate a permanent (non-retryable) failure. +var permanentHTTPStatuses = map[int]bool{ + http.StatusNotFound: true, // 404 โ€” repo, ref, or resource does not exist + http.StatusForbidden: true, // 403 โ€” no permission (rate limits handled separately) + http.StatusUnprocessableEntity: true, // 422 โ€” validation error, merge conflict + http.StatusConflict: true, // 409 โ€” ref update conflict + http.StatusGone: true, // 410 โ€” resource permanently removed +} + +// IsPermanentError returns true if err represents a failure that will not +// resolve by retrying the same operation. The retry loop should break +// immediately when this returns true. +// +// Permanent errors include: +// - Known sentinel errors (config validation, installation not found, merge conflict) +// - GitHub API responses with 403, 404, 409, 410, or 422 status codes +// +// Note: HTTP 403 from rate limiting is handled separately by rateLimitTransport +// before it reaches this layer. A 403 that reaches here is a true permission denial. +func IsPermanentError(err error) bool { + if err == nil { + return false + } + + // Check sentinel errors + for _, sentinel := range permanentSentinels { + if errors.Is(err, sentinel) { + return true + } + } + + // Check GitHub API error responses + var ghErr *github.ErrorResponse + if errors.As(err, &ghErr) { + if ghErr.Response != nil { + return permanentHTTPStatuses[ghErr.Response.StatusCode] + } + } + + return false +} diff --git a/services/errors_test.go b/services/errors_test.go new file mode 100644 index 0000000..2d2463f --- /dev/null +++ b/services/errors_test.go @@ -0,0 +1,127 @@ +package services_test + +import ( + "fmt" + "net/http" + "testing" + + "github.com/google/go-github/v82/github" + "github.com/grove-platform/github-copier/services" + "github.com/stretchr/testify/assert" +) + +func TestIsPermanentError_NilError(t *testing.T) { + assert.False(t, services.IsPermanentError(nil)) +} + +func TestIsPermanentError_TransientErrors(t *testing.T) { + tests := []struct { + name string + err error + }{ + {"generic error", fmt.Errorf("something went wrong")}, + {"network timeout", fmt.Errorf("dial tcp: i/o timeout")}, + {"wrapped generic", fmt.Errorf("operation failed: %w", fmt.Errorf("transient"))}, + {"github 500", &github.ErrorResponse{ + Response: &http.Response{StatusCode: http.StatusInternalServerError}, + Message: "Internal Server Error", + }}, + {"github 502", &github.ErrorResponse{ + Response: &http.Response{StatusCode: http.StatusBadGateway}, + Message: "Bad Gateway", + }}, + {"github 503", &github.ErrorResponse{ + Response: &http.Response{StatusCode: http.StatusServiceUnavailable}, + Message: "Service Unavailable", + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.False(t, services.IsPermanentError(tt.err), "expected transient, got permanent") + }) + } +} + +func TestIsPermanentError_PermanentSentinels(t *testing.T) { + tests := []struct { + name string + err error + sentinel error + }{ + {"authentication", services.ErrAuthentication, services.ErrAuthentication}, + {"config load", services.ErrConfigLoad, services.ErrConfigLoad}, + {"config validation", services.ErrConfigValidation, services.ErrConfigValidation}, + {"installation not found", services.ErrInstallationNotFound, services.ErrInstallationNotFound}, + {"merge conflict", services.ErrMergeConflict, services.ErrMergeConflict}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Direct sentinel + assert.True(t, services.IsPermanentError(tt.err)) + + // Wrapped sentinel + wrapped := fmt.Errorf("webhook failed: %w", tt.err) + assert.True(t, services.IsPermanentError(wrapped)) + + // Double-wrapped sentinel + doubleWrapped := fmt.Errorf("outer: %w", wrapped) + assert.True(t, services.IsPermanentError(doubleWrapped)) + }) + } +} + +func TestIsPermanentError_NonPermanentSentinels(t *testing.T) { + // These sentinel errors are NOT classified as permanent โ€” they may + // resolve on retry (e.g. secrets re-fetched, content retried). + tests := []struct { + name string + err error + }{ + {"secret access", services.ErrSecretAccess}, + {"content nil", services.ErrContentNil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.False(t, services.IsPermanentError(tt.err)) + }) + } +} + +func TestIsPermanentError_GitHubAPIStatuses(t *testing.T) { + permanentCodes := []struct { + code int + name string + }{ + {http.StatusNotFound, "404 Not Found"}, + {http.StatusForbidden, "403 Forbidden"}, + {http.StatusUnprocessableEntity, "422 Unprocessable Entity"}, + {http.StatusConflict, "409 Conflict"}, + {http.StatusGone, "410 Gone"}, + } + + for _, tt := range permanentCodes { + t.Run(tt.name, func(t *testing.T) { + err := &github.ErrorResponse{ + Response: &http.Response{StatusCode: tt.code}, + Message: tt.name, + } + assert.True(t, services.IsPermanentError(err), "expected permanent for %s", tt.name) + + // Also works when wrapped + wrapped := fmt.Errorf("github: %w", err) + assert.True(t, services.IsPermanentError(wrapped), "expected permanent for wrapped %s", tt.name) + }) + } +} + +func TestIsPermanentError_GitHubAPINilResponse(t *testing.T) { + // ErrorResponse with nil Response should not panic and should be treated as transient + err := &github.ErrorResponse{ + Response: nil, + Message: "unknown error", + } + assert.False(t, services.IsPermanentError(err)) +} diff --git a/services/file_state_service_test.go b/services/file_state_service_test.go index 39bb2d9..92cd8bf 100644 --- a/services/file_state_service_test.go +++ b/services/file_state_service_test.go @@ -4,7 +4,7 @@ import ( "sync" "testing" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/grove-platform/github-copier/services" "github.com/grove-platform/github-copier/types" "github.com/stretchr/testify/assert" @@ -26,7 +26,7 @@ func TestFileStateService_AddAndGetFilesToUpload(t *testing.T) { CommitStrategy: types.CommitStrategyDirect, CommitMessage: "Test commit", Content: []github.RepositoryContent{ - {Path: github.String("test.go")}, + {Path: github.Ptr("test.go")}, }, } @@ -193,7 +193,7 @@ func TestFileStateService_UpdateExistingFile(t *testing.T) { content1 := types.UploadFileContent{ TargetBranch: "main", Content: []github.RepositoryContent{ - {Path: github.String("file1.go")}, + {Path: github.Ptr("file1.go")}, }, } service.AddFileToUpload(key, content1) @@ -202,8 +202,8 @@ func TestFileStateService_UpdateExistingFile(t *testing.T) { content2 := types.UploadFileContent{ TargetBranch: "main", Content: []github.RepositoryContent{ - {Path: github.String("file1.go")}, - {Path: github.String("file2.go")}, + {Path: github.Ptr("file1.go")}, + {Path: github.Ptr("file2.go")}, }, } service.AddFileToUpload(key, content2) @@ -271,14 +271,14 @@ func TestFileStateService_MultipleRepos(t *testing.T) { content1 := types.UploadFileContent{ TargetBranch: "main", Content: []github.RepositoryContent{ - {Path: github.String("file1.go")}, + {Path: github.Ptr("file1.go")}, }, } content2 := types.UploadFileContent{ TargetBranch: "develop", Content: []github.RepositoryContent{ - {Path: github.String("file2.go")}, + {Path: github.Ptr("file2.go")}, }, } @@ -305,7 +305,7 @@ func TestFileStateService_IsolatedCopies(t *testing.T) { content := types.UploadFileContent{ TargetBranch: "main", Content: []github.RepositoryContent{ - {Path: github.String("file1.go")}, + {Path: github.Ptr("file1.go")}, }, } diff --git a/services/github_auth.go b/services/github_auth.go index 683463f..8317d0a 100644 --- a/services/github_auth.go +++ b/services/github_auth.go @@ -14,7 +14,7 @@ import ( secretmanager "cloud.google.com/go/secretmanager/apiv1" "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" "github.com/golang-jwt/jwt/v5" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/grove-platform/github-copier/configs" "github.com/shurcooL/graphql" "golang.org/x/oauth2" @@ -25,28 +25,11 @@ type transport struct { token string } -var InstallationAccessToken string -var HTTPClient = http.DefaultClient - -// installationTokenCache caches installation access tokens by organization name -var installationTokenCache = make(map[string]string) - -// jwtToken caches the GitHub App JWT token -var jwtToken string -var jwtExpiry time.Time - // ConfigurePermissions sets up the necessary permissions to interact with the GitHub API. // It retrieves the GitHub App's private key from Google Secret Manager, generates a JWT, -// and exchanges it for an installation access token. -func ConfigurePermissions() error { - envFilePath := os.Getenv("ENV_FILE") - - _, err := configs.LoadEnvironment(envFilePath) - if err != nil { - return fmt.Errorf("failed to load environment: %w", err) - } - - pemKey, err := getPrivateKeyFromSecret() +// and exchanges it for an installation access token stored in the TokenManager. +func ConfigurePermissions(ctx context.Context, config *configs.Config) error { + pemKey, err := getPrivateKeyFromSecret(ctx, config) if err != nil { return fmt.Errorf("failed to get private key: %w", err) } @@ -57,30 +40,29 @@ func ConfigurePermissions() error { } // Generate JWT โ€” use the numeric GitHub App ID (GITHUB_APP_ID) as "iss" - token, err := generateGitHubJWT(os.Getenv(configs.AppId), privateKey) + token, err := generateGitHubJWT(config.AppId, privateKey) if err != nil { return fmt.Errorf("error generating JWT: %w", err) } - installationToken, err := getInstallationAccessToken("", token, HTTPClient) + hc := defaultTokenManager.GetHTTPClient() + installationToken, _, err := getInstallationAccessToken(config.InstallationId, token, hc) if err != nil { return fmt.Errorf("error getting installation access token: %w", err) } - InstallationAccessToken = installationToken + defaultTokenManager.SetInstallationAccessToken(installationToken) return nil } // generateGitHubJWT creates a JWT for GitHub App authentication. func generateGitHubJWT(appID string, privateKey *rsa.PrivateKey) (string, error) { - // Create a new JWT token now := time.Now() claims := jwt.MapClaims{ - "iat": now.Unix(), // Issued at - "exp": now.Add(time.Minute * 10).Unix(), // Expiration time, 10 minutes from issue - "iss": appID, // GitHub App ID + "iat": now.Unix(), + "exp": now.Add(time.Minute * 10).Unix(), + "iss": appID, } token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) - // Sign the JWT with the private key signedToken, err := token.SignedString(privateKey) if err != nil { return "", fmt.Errorf("unable to sign JWT: %v", err) @@ -90,8 +72,8 @@ func generateGitHubJWT(appID string, privateKey *rsa.PrivateKey) (string, error) // getPrivateKeyFromSecret retrieves the GitHub App's private key from Google Secret Manager. // It supports local testing by allowing the key to be provided via environment variables. -func getPrivateKeyFromSecret() ([]byte, error) { - if os.Getenv("SKIP_SECRET_MANAGER") == "true" { // for tests and local runs +func getPrivateKeyFromSecret(ctx context.Context, config *configs.Config) ([]byte, error) { + if os.Getenv("SKIP_SECRET_MANAGER") == "true" { if pem := os.Getenv("GITHUB_APP_PRIVATE_KEY"); pem != "" { return []byte(pem), nil } @@ -102,67 +84,56 @@ func getPrivateKeyFromSecret() ([]byte, error) { } return dec, nil } - return nil, fmt.Errorf("SKIP_SECRET_MANAGER=true but no GITHUB_APP_PRIVATE_KEY or GITHUB_APP_PRIVATE_KEY_B64 set") + return nil, fmt.Errorf("%w: SKIP_SECRET_MANAGER=true but no GITHUB_APP_PRIVATE_KEY or GITHUB_APP_PRIVATE_KEY_B64 set", ErrSecretAccess) } - ctx := context.Background() client, err := secretmanager.NewClient(ctx) - if err != nil { - return nil, fmt.Errorf("failed to create Secret Manager client: %w", err) - } - defer client.Close() - - secretName := os.Getenv(configs.PEMKeyName) - if secretName == "" { - secretName = configs.NewConfig().PEMKeyName + return nil, fmt.Errorf("%w: failed to create Secret Manager client: %v", ErrSecretAccess, err) } + defer func() { _ = client.Close() }() req := &secretmanagerpb.AccessSecretVersionRequest{ - Name: secretName, + Name: config.SecretPath(config.PEMKeyName), } result, err := client.AccessSecretVersion(ctx, req) if err != nil { - return nil, fmt.Errorf("failed to access secret version: %w", err) + return nil, fmt.Errorf("%w: %v", ErrSecretAccess, err) } return result.Payload.Data, nil } // getWebhookSecretFromSecretManager retrieves the webhook secret from Google Cloud Secret Manager -func getWebhookSecretFromSecretManager(secretName string) (string, error) { +func getWebhookSecretFromSecretManager(ctx context.Context, secretName string) (string, error) { if os.Getenv("SKIP_SECRET_MANAGER") == "true" { - // For tests and local runs, use direct env var if secret := os.Getenv(configs.WebhookSecret); secret != "" { return secret, nil } - return "", fmt.Errorf("SKIP_SECRET_MANAGER=true but no WEBHOOK_SECRET set") + return "", fmt.Errorf("%w: SKIP_SECRET_MANAGER=true but no WEBHOOK_SECRET set", ErrSecretAccess) } - ctx := context.Background() client, err := secretmanager.NewClient(ctx) if err != nil { - return "", fmt.Errorf("failed to create Secret Manager client: %w", err) + return "", fmt.Errorf("%w: failed to create Secret Manager client: %v", ErrSecretAccess, err) } - defer client.Close() + defer func() { _ = client.Close() }() req := &secretmanagerpb.AccessSecretVersionRequest{ Name: secretName, } result, err := client.AccessSecretVersion(ctx, req) if err != nil { - return "", fmt.Errorf("failed to access secret version: %w", err) + return "", fmt.Errorf("%w: %v", ErrSecretAccess, err) } return string(result.Payload.Data), nil } // LoadWebhookSecret loads the webhook secret from Secret Manager or environment variable -func LoadWebhookSecret(config *configs.Config) error { - // If webhook secret is already set directly, use it +func LoadWebhookSecret(ctx context.Context, config *configs.Config) error { if config.WebhookSecret != "" { return nil } - - // Otherwise, load from Secret Manager - secret, err := getWebhookSecretFromSecretManager(config.WebhookSecretName) + resolvedName := config.SecretPath(config.WebhookSecretName) + secret, err := getWebhookSecretFromSecretManager(ctx, resolvedName) if err != nil { return fmt.Errorf("failed to load webhook secret: %w", err) } @@ -171,19 +142,15 @@ func LoadWebhookSecret(config *configs.Config) error { } // LoadMongoURI loads the MongoDB URI from Secret Manager or environment variable -func LoadMongoURI(config *configs.Config) error { - // If MongoDB URI is already set directly, use it +func LoadMongoURI(ctx context.Context, config *configs.Config) error { if config.MongoURI != "" { return nil } - - // If no secret name is configured, skip (audit logging is optional) if config.MongoURISecretName == "" { return nil } - - // Load from Secret Manager - uri, err := getSecretFromSecretManager(config.MongoURISecretName, "MONGO_URI") + resolvedName := config.SecretPath(config.MongoURISecretName) + uri, err := getSecretFromSecretManager(ctx, resolvedName, "MONGO_URI") if err != nil { return fmt.Errorf("failed to load MongoDB URI: %w", err) } @@ -192,110 +159,136 @@ func LoadMongoURI(config *configs.Config) error { } // getSecretFromSecretManager is a generic function to retrieve any secret from Secret Manager -func getSecretFromSecretManager(secretName, envVarName string) (string, error) { +func getSecretFromSecretManager(ctx context.Context, secretName, envVarName string) (string, error) { if os.Getenv("SKIP_SECRET_MANAGER") == "true" { - // For tests and local runs, use direct env var if secret := os.Getenv(envVarName); secret != "" { return secret, nil } - return "", fmt.Errorf("SKIP_SECRET_MANAGER=true but no %s set", envVarName) + return "", fmt.Errorf("%w: SKIP_SECRET_MANAGER=true but no %s set", ErrSecretAccess, envVarName) } - ctx := context.Background() client, err := secretmanager.NewClient(ctx) if err != nil { - return "", fmt.Errorf("failed to create Secret Manager client: %w", err) + return "", fmt.Errorf("%w: failed to create Secret Manager client: %v", ErrSecretAccess, err) } - defer client.Close() + defer func() { _ = client.Close() }() req := &secretmanagerpb.AccessSecretVersionRequest{ Name: secretName, } result, err := client.AccessSecretVersion(ctx, req) if err != nil { - return "", fmt.Errorf("failed to access secret version: %w", err) + return "", fmt.Errorf("%w: %v", ErrSecretAccess, err) } return string(result.Payload.Data), nil } // getInstallationAccessToken exchanges a JWT for a GitHub App installation access token. -func getInstallationAccessToken(installationId, jwtToken string, hc *http.Client) (string, error) { - if installationId == "" || installationId == configs.InstallationId { - installationId = os.Getenv(configs.InstallationId) - } +// Returns the token, its expiry time, and any error. +func getInstallationAccessToken(installationId, jwtTokenStr string, hc *http.Client) (string, time.Time, error) { if installationId == "" { - return "", fmt.Errorf("missing installation ID") + return "", time.Time{}, fmt.Errorf("missing installation ID") } url := fmt.Sprintf("https://api.github.com/app/installations/%s/access_tokens", installationId) req, err := http.NewRequest("POST", url, nil) if err != nil { - return "", fmt.Errorf("create request: %w", err) + return "", time.Time{}, fmt.Errorf("create request: %w", err) } - req.Header.Set("Authorization", "Bearer "+jwtToken) + req.Header.Set("Authorization", "Bearer "+jwtTokenStr) req.Header.Set("Accept", "application/vnd.github+json") if hc == nil { hc = http.DefaultClient } - resp, err := hc.Do(req) + resp, err := hc.Do(req) // #nosec G704 -- URL is hardcoded to api.github.com; installationId is from trusted config if err != nil { - return "", fmt.Errorf("execute request: %w", err) + return "", time.Time{}, fmt.Errorf("execute request: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusCreated { - b, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("status %d: %s", resp.StatusCode, string(b)) + b, readErr := io.ReadAll(resp.Body) + if readErr != nil { + b = []byte(fmt.Sprintf("", readErr)) + } + if resp.StatusCode == http.StatusUnauthorized { + return "", time.Time{}, fmt.Errorf("%w: failed to get installation access token (401). The GitHub App private key (PEM) may be invalid or expired. Please check the CODE_COPIER_PEM secret in GCP Secret Manager. Response: %s", ErrAuthentication, string(b)) + } + return "", time.Time{}, fmt.Errorf("%w: status %d: %s", ErrAuthentication, resp.StatusCode, string(b)) } var out struct { - Token string `json:"token"` + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` } if err = json.NewDecoder(resp.Body).Decode(&out); err != nil { - return "", fmt.Errorf("decode: %w", err) + return "", time.Time{}, fmt.Errorf("decode: %w", err) } - return out.Token, nil + return out.Token, out.ExpiresAt, nil } -// GetRestClient returns a GitHub REST API client authenticated with the installation access token. +// GetRestClient returns a GitHub REST API client authenticated with the default installation access token. func GetRestClient() *github.Client { - src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: InstallationAccessToken}) + tm := defaultTokenManager + token := tm.GetInstallationAccessToken() + hc := tm.GetHTTPClient() + return newGitHubRESTClient(token, hc) +} - base := http.DefaultTransport - if HTTPClient != nil && HTTPClient.Transport != nil { - base = HTTPClient.Transport +// GetGraphQLClient returns a GitHub GraphQL API client authenticated with the default installation access token. +func GetGraphQLClient(ctx context.Context, config *configs.Config) (*graphql.Client, error) { + if defaultTokenManager.GetInstallationAccessToken() == "" { + if err := ConfigurePermissions(ctx, config); err != nil { + return nil, fmt.Errorf("failed to configure permissions: %w", err) + } } + client := graphql.NewClient("https://api.github.com/graphql", &http.Client{ + Transport: newRateLimitTransport(&transport{token: defaultTokenManager.GetInstallationAccessToken()}, nil), + }) + return client, nil +} - httpClient := &http.Client{ - Transport: &oauth2.Transport{ - Source: src, - Base: base, - }, +// GetGraphQLClientForOrg returns a GitHub GraphQL API client authenticated for a specific organization. +// Uses the TokenManager for thread-safe token caching with expiry tracking. +func GetGraphQLClientForOrg(ctx context.Context, config *configs.Config, org string) (*graphql.Client, error) { + if token, ok := defaultTokenManager.GetTokenForOrg(org); ok { + client := graphql.NewClient("https://api.github.com/graphql", &http.Client{ + Transport: newRateLimitTransport(&transport{token: token}, nil), + }) + return client, nil } - return github.NewClient(httpClient) -} -func GetGraphQLClient() (*graphql.Client, error) { - if InstallationAccessToken == "" { - if err := ConfigurePermissions(); err != nil { - return nil, fmt.Errorf("failed to configure permissions: %w", err) - } + installationID, err := getInstallationIDForOrg(ctx, config, org) + if err != nil { + return nil, fmt.Errorf("failed to get installation ID for org %s: %w", org, err) + } + + token, err := getOrRefreshJWT(ctx, config) + if err != nil { + return nil, fmt.Errorf("failed to get JWT: %w", err) } + + hc := defaultTokenManager.GetHTTPClient() + installationToken, expiresAt, err := getInstallationAccessToken(installationID, token, hc) + if err != nil { + return nil, fmt.Errorf("failed to get installation token for org %s: %w", org, err) + } + + defaultTokenManager.SetTokenForOrg(org, installationToken, expiresAt) + client := graphql.NewClient("https://api.github.com/graphql", &http.Client{ - Transport: &transport{token: InstallationAccessToken}, + Transport: newRateLimitTransport(&transport{token: installationToken}, nil), }) return client, nil } -// getOrRefreshJWT returns a valid JWT token, generating a new one if expired -func getOrRefreshJWT() (string, error) { - // Check if we have a valid cached JWT - if jwtToken != "" && time.Now().Before(jwtExpiry) { - return jwtToken, nil +// getOrRefreshJWT returns a valid JWT token, generating a new one if expired. +func getOrRefreshJWT(ctx context.Context, config *configs.Config) (string, error) { + if cachedToken, ok := defaultTokenManager.GetCachedJWT(); ok { + return cachedToken, nil } - // Generate new JWT - pemKey, err := getPrivateKeyFromSecret() + pemKey, err := getPrivateKeyFromSecret(ctx, config) if err != nil { return "", fmt.Errorf("failed to get private key: %w", err) } @@ -305,21 +298,18 @@ func getOrRefreshJWT() (string, error) { return "", fmt.Errorf("unable to parse RSA private key: %w", err) } - token, err := generateGitHubJWT(os.Getenv(configs.AppId), privateKey) + token, err := generateGitHubJWT(config.AppId, privateKey) if err != nil { return "", fmt.Errorf("error generating JWT: %w", err) } - // Cache the JWT (expires in 10 minutes, cache for 9 to be safe) - jwtToken = token - jwtExpiry = time.Now().Add(9 * time.Minute) - + defaultTokenManager.SetCachedJWT(token, time.Now().Add(9*time.Minute)) return token, nil } // getInstallationIDForOrg retrieves the installation ID for a specific organization -func getInstallationIDForOrg(org string) (string, error) { - token, err := getOrRefreshJWT() +func getInstallationIDForOrg(ctx context.Context, config *configs.Config, org string) (string, error) { + token, err := getOrRefreshJWT(ctx, config) if err != nil { return "", fmt.Errorf("failed to get JWT: %w", err) } @@ -332,20 +322,22 @@ func getInstallationIDForOrg(org string) (string, error) { req.Header.Set("Authorization", "Bearer "+token) req.Header.Set("Accept", "application/vnd.github+json") - hc := HTTPClient - if hc == nil { - hc = http.DefaultClient - } - - resp, err := hc.Do(req) + hc := defaultTokenManager.GetHTTPClient() + resp, err := hc.Do(req) // #nosec G704 -- URL is hardcoded to api.github.com/app/installations if err != nil { return "", fmt.Errorf("request failed: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("GET %s: %d %s %s", url, resp.StatusCode, resp.Status, body) + body, readErr := io.ReadAll(resp.Body) + if readErr != nil { + body = []byte(fmt.Sprintf("", readErr)) + } + if resp.StatusCode == http.StatusUnauthorized { + return "", fmt.Errorf("%w: the GitHub App private key (PEM) may be invalid or expired. Please check the CODE_COPIER_PEM secret in GCP Secret Manager. Response: %s", ErrAuthentication, string(body)) + } + return "", fmt.Errorf("%w: GET %s: %d %s %s", ErrAuthentication, url, resp.StatusCode, resp.Status, body) } var installations []struct { @@ -360,74 +352,65 @@ func getInstallationIDForOrg(org string) (string, error) { return "", fmt.Errorf("decode response: %w", err) } - // Find the installation for the specified organization for _, inst := range installations { if inst.Account.Login == org { return fmt.Sprintf("%d", inst.ID), nil } } - return "", fmt.Errorf("no installation found for organization: %s", org) + return "", fmt.Errorf("%w: %s", ErrInstallationNotFound, org) } // SetInstallationTokenForOrg sets a cached installation token for an organization. // This is primarily used for testing to bypass the GitHub App authentication flow. func SetInstallationTokenForOrg(org, token string) { - installationTokenCache[org] = token + defaultTokenManager.SetTokenForOrgNoExpiry(org, token) } -// GetRestClientForOrg returns a GitHub REST API client authenticated for a specific organization -func GetRestClientForOrg(org string) (*github.Client, error) { - // Check if we have a cached token for this org - if token, ok := installationTokenCache[org]; ok && token != "" { - src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) - base := http.DefaultTransport - if HTTPClient != nil && HTTPClient.Transport != nil { - base = HTTPClient.Transport - } - httpClient := &http.Client{ - Transport: &oauth2.Transport{ - Source: src, - Base: base, - }, - } - return github.NewClient(httpClient), nil +// GetRestClientForOrg returns a GitHub REST API client authenticated for a specific organization. +func GetRestClientForOrg(ctx context.Context, config *configs.Config, org string) (*github.Client, error) { + tm := defaultTokenManager + hc := tm.GetHTTPClient() + + if token, ok := tm.GetTokenForOrg(org); ok { + return newGitHubRESTClient(token, hc), nil } - // Get installation ID for the organization - installationID, err := getInstallationIDForOrg(org) + installationID, err := getInstallationIDForOrg(ctx, config, org) if err != nil { return nil, fmt.Errorf("failed to get installation ID for org %s: %w", org, err) } - // Get JWT token - token, err := getOrRefreshJWT() + token, err := getOrRefreshJWT(ctx, config) if err != nil { return nil, fmt.Errorf("failed to get JWT: %w", err) } - // Get installation access token - installationToken, err := getInstallationAccessToken(installationID, token, HTTPClient) + installationToken, expiresAt, err := getInstallationAccessToken(installationID, token, hc) if err != nil { return nil, fmt.Errorf("failed to get installation token for org %s: %w", org, err) } - // Cache the token - installationTokenCache[org] = installationToken + tm.SetTokenForOrg(org, installationToken, expiresAt) + return newGitHubRESTClient(installationToken, hc), nil +} - // Create and return client - src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: installationToken}) +// newGitHubRESTClient creates a GitHub REST client with the given token and base HTTP client. +// The transport chain is: rateLimitTransport โ†’ oauth2.Transport โ†’ base. +func newGitHubRESTClient(token string, hc *http.Client) *github.Client { + src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) base := http.DefaultTransport - if HTTPClient != nil && HTTPClient.Transport != nil { - base = HTTPClient.Transport + if hc != nil && hc.Transport != nil { + base = hc.Transport + } + oauthTransport := &oauth2.Transport{ + Source: src, + Base: base, } httpClient := &http.Client{ - Transport: &oauth2.Transport{ - Source: src, - Base: base, - }, + Transport: newRateLimitTransport(oauthTransport, nil), } - return github.NewClient(httpClient), nil + return github.NewClient(httpClient) } // RoundTrip adds the Authorization header to each request. diff --git a/services/github_auth_test.go b/services/github_auth_test.go index cbb6f6c..ed40f31 100644 --- a/services/github_auth_test.go +++ b/services/github_auth_test.go @@ -1,116 +1,178 @@ package services import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "net/http" "os" "testing" "time" + "github.com/golang-jwt/jwt/v5" "github.com/grove-platform/github-copier/configs" + "github.com/jarcoal/httpmock" ) -func TestGenerateGitHubJWT_EmptyAppID(t *testing.T) { - // Note: generateGitHubJWT requires appID string and *rsa.PrivateKey - // Testing this requires creating a valid RSA private key, which is complex - // This test documents the expected behavior - t.Skip("Skipping test that requires valid RSA private key generation") +func TestGenerateGitHubJWT(t *testing.T) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + + tests := []struct { + name string + appID string + wantErr bool + }{ + {name: "valid app ID", appID: "123456", wantErr: false}, + {name: "empty app ID still produces JWT", appID: "", wantErr: false}, + } - // Expected behavior: - // - Should return error with empty app ID - // - Should return error with nil private key - // - Should generate valid JWT with valid inputs + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + token, err := generateGitHubJWT(tt.appID, key) + if (err != nil) != tt.wantErr { + t.Fatalf("generateGitHubJWT() error = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr { + if token == "" { + t.Error("expected non-empty JWT token") + } + // Verify the token can be parsed and has correct claims + parsed, err := jwt.Parse(token, func(t *jwt.Token) (interface{}, error) { + return &key.PublicKey, nil + }) + if err != nil { + t.Fatalf("jwt.Parse: %v", err) + } + claims, ok := parsed.Claims.(jwt.MapClaims) + if !ok { + t.Fatal("expected MapClaims") + } + if iss, _ := claims["iss"].(string); iss != tt.appID { + t.Errorf("iss = %q, want %q", iss, tt.appID) + } + if _, ok := claims["iat"]; !ok { + t.Error("missing 'iat' claim") + } + if _, ok := claims["exp"]; !ok { + t.Error("missing 'exp' claim") + } + } + }) + } } -func TestJWTCaching(t *testing.T) { - // Test JWT caching behavior - originalToken := jwtToken - originalExpiry := jwtExpiry +func TestGenerateGitHubJWT_NilKey(t *testing.T) { + // The JWT library panics on nil key, so verify we get a panic. defer func() { - jwtToken = originalToken - jwtExpiry = originalExpiry + if r := recover(); r == nil { + t.Error("expected panic with nil private key") + } }() + _, _ = generateGitHubJWT("123456", nil) +} + +func TestJWTCaching(t *testing.T) { + tm := NewTokenManager() // Set a cached token that hasn't expired - jwtToken = "cached-token" - jwtExpiry = time.Now().Add(5 * time.Minute) + tm.SetCachedJWT("cached-token", time.Now().Add(5*time.Minute)) + + token, ok := tm.GetCachedJWT() + if !ok { + t.Error("Expected cached JWT to be valid") + } + if token != "cached-token" { + t.Errorf("Cached JWT = %s, want cached-token", token) + } + + // Set an expired token + tm.SetCachedJWT("expired-token", time.Now().Add(-1*time.Minute)) - // Note: getOrRefreshJWT is not exported, so we can't test it directly - // This test documents the expected caching behavior: - // - If jwtToken is set and jwtExpiry is in the future, return cached token - // - If jwtToken is empty or jwtExpiry is in the past, generate new token - // - Cache the new token and set expiry to 9 minutes from now + _, ok = tm.GetCachedJWT() + if ok { + t.Error("Expected expired JWT to not be returned") + } } func TestInstallationTokenCache_Structure(t *testing.T) { - // Test that we can manipulate the installation token cache - originalCache := installationTokenCache - defer func() { - installationTokenCache = originalCache - }() + tm := NewTokenManager() - // Initialize cache (it's a map[string]string) - installationTokenCache = make(map[string]string) - - // Add a token testToken := "test-token-value" - installationTokenCache["test-org"] = testToken + tm.SetTokenForOrgNoExpiry("test-org", testToken) - // Verify it was added - cached, exists := installationTokenCache["test-org"] - if !exists { + cached, ok := tm.GetTokenForOrg("test-org") + if !ok { t.Error("Token not found in cache") } - if cached != testToken { t.Errorf("Cached token = %s, want %s", cached, testToken) } } -func TestLoadWebhookSecret_FromEnv(t *testing.T) { - // Test loading webhook secret from environment variable - testSecret := "test-webhook-secret" - os.Setenv("WEBHOOK_SECRET", testSecret) - defer os.Unsetenv("WEBHOOK_SECRET") +func TestInstallationTokenCache_ExpiryTracking(t *testing.T) { + tm := NewTokenManager() - // LoadWebhookSecret requires a config parameter - config := &configs.Config{ - WebhookSecret: "", + // Token with future expiry should be valid + tm.SetTokenForOrg("future-org", "valid-token", time.Now().Add(1*time.Hour)) + token, ok := tm.GetTokenForOrg("future-org") + if !ok { + t.Error("Expected valid token to be returned") } + if token != "valid-token" { + t.Errorf("Token = %s, want valid-token", token) + } + + // Token within the 5-minute buffer should be treated as expired + tm.SetTokenForOrg("expiring-org", "expiring-token", time.Now().Add(3*time.Minute)) + _, ok = tm.GetTokenForOrg("expiring-org") + if ok { + t.Error("Expected token within 5-minute buffer to not be returned") + } + + // Already-expired token should not be returned + tm.SetTokenForOrg("expired-org", "expired-token", time.Now().Add(-10*time.Minute)) + _, ok = tm.GetTokenForOrg("expired-org") + if ok { + t.Error("Expected expired token to not be returned") + } +} - // Note: LoadWebhookSecret tries Secret Manager first, which will fail in test environment - // This is expected behavior - the function should handle the error gracefully - _ = LoadWebhookSecret(config) +func TestLoadWebhookSecret_FromEnv(t *testing.T) { + testSecret := "test-webhook-secret" + _ = os.Setenv("WEBHOOK_SECRET", testSecret) + defer func() { _ = os.Unsetenv("WEBHOOK_SECRET") }() + + config := &configs.Config{WebhookSecret: ""} + _ = LoadWebhookSecret(context.Background(), config) - // Verify the environment variable is set (even if Secret Manager fails) envSecret := os.Getenv("WEBHOOK_SECRET") if envSecret != testSecret { t.Errorf("WEBHOOK_SECRET env var = %s, want %s", envSecret, testSecret) } - - // Note: In production, LoadWebhookSecret would populate config.WebhookSecret - // from Secret Manager or fall back to the environment variable } func TestLoadMongoURI_FromEnv(t *testing.T) { - // Test loading MongoDB URI from environment variable testURI := "mongodb://localhost:27017/test" - os.Setenv("MONGO_URI", testURI) - defer os.Unsetenv("MONGO_URI") + _ = os.Setenv("MONGO_URI", testURI) + defer func() { _ = os.Unsetenv("MONGO_URI") }() - // Verify the environment variable is set envURI := os.Getenv("MONGO_URI") if envURI != testURI { t.Errorf("MONGO_URI env var = %s, want %s", envURI, testURI) } - - // Note: LoadMongoURI function signature needs to be checked - // This test documents that MONGO_URI can be set via environment } func TestGitHubAppID_FromEnv(t *testing.T) { - // Test that GITHUB_APP_ID can be read from environment testAppID := "123456" - os.Setenv("GITHUB_APP_ID", testAppID) - defer os.Unsetenv("GITHUB_APP_ID") + _ = os.Setenv("GITHUB_APP_ID", testAppID) + defer func() { _ = os.Unsetenv("GITHUB_APP_ID") }() appID := os.Getenv("GITHUB_APP_ID") if appID != testAppID { @@ -119,10 +181,9 @@ func TestGitHubAppID_FromEnv(t *testing.T) { } func TestGitHubInstallationID_FromEnv(t *testing.T) { - // Test that GITHUB_INSTALLATION_ID can be read from environment testInstallID := "789012" - os.Setenv("GITHUB_INSTALLATION_ID", testInstallID) - defer os.Unsetenv("GITHUB_INSTALLATION_ID") + _ = os.Setenv("GITHUB_INSTALLATION_ID", testInstallID) + defer func() { _ = os.Unsetenv("GITHUB_INSTALLATION_ID") }() installID := os.Getenv("GITHUB_INSTALLATION_ID") if installID != testInstallID { @@ -131,10 +192,9 @@ func TestGitHubInstallationID_FromEnv(t *testing.T) { } func TestGitHubPrivateKeyPath_FromEnv(t *testing.T) { - // Test that GITHUB_PRIVATE_KEY_PATH can be read from environment testPath := "/path/to/private-key.pem" - os.Setenv("GITHUB_PRIVATE_KEY_PATH", testPath) - defer os.Unsetenv("GITHUB_PRIVATE_KEY_PATH") + _ = os.Setenv("GITHUB_PRIVATE_KEY_PATH", testPath) + defer func() { _ = os.Unsetenv("GITHUB_PRIVATE_KEY_PATH") }() keyPath := os.Getenv("GITHUB_PRIVATE_KEY_PATH") if keyPath != testPath { @@ -142,83 +202,320 @@ func TestGitHubPrivateKeyPath_FromEnv(t *testing.T) { } } -func TestInstallationAccessToken_GlobalVariable(t *testing.T) { - // Test that we can manipulate the global InstallationAccessToken - originalToken := InstallationAccessToken - defer func() { - InstallationAccessToken = originalToken - }() +func TestTokenManager_InstallationAccessToken(t *testing.T) { + tm := NewTokenManager() - testToken := "ghs_test_token_123" - InstallationAccessToken = testToken + if got := tm.GetInstallationAccessToken(); got != "" { + t.Errorf("Expected empty token, got %q", got) + } - if InstallationAccessToken != testToken { - t.Errorf("InstallationAccessToken = %s, want %s", InstallationAccessToken, testToken) + tm.SetInstallationAccessToken("ghs_test_token_123") + if got := tm.GetInstallationAccessToken(); got != "ghs_test_token_123" { + t.Errorf("InstallationAccessToken = %s, want ghs_test_token_123", got) } } -func TestHTTPClient_GlobalVariable(t *testing.T) { - // Test that HTTPClient is initialized - if HTTPClient == nil { +func TestTokenManager_HTTPClient(t *testing.T) { + tm := NewTokenManager() + + // Default client should not be nil + if tm.GetHTTPClient() == nil { t.Error("HTTPClient should not be nil") } - // Note: HTTPClient is initialized to http.DefaultClient which has Timeout = 0 (no timeout) - // This is the default behavior in Go's http package - // The test just verifies the client exists + // Should be able to swap clients + custom := &http.Client{} + tm.SetHTTPClient(custom) + if tm.GetHTTPClient() != custom { + t.Error("Expected custom HTTP client after SetHTTPClient") + } } -func TestJWTExpiry_GlobalVariable(t *testing.T) { - // Test that we can manipulate the JWT expiry time - originalExpiry := jwtExpiry - defer func() { - jwtExpiry = originalExpiry - }() +func TestTokenManager_JWTExpiry(t *testing.T) { + tm := NewTokenManager() - // Set a future expiry futureExpiry := time.Now().Add(1 * time.Hour) - jwtExpiry = futureExpiry - - if time.Now().After(jwtExpiry) { + tm.SetCachedJWT("future-jwt", futureExpiry) + _, ok := tm.GetCachedJWT() + if !ok { t.Error("JWT should not be expired") } - // Set a past expiry pastExpiry := time.Now().Add(-1 * time.Hour) - jwtExpiry = pastExpiry - - if !time.Now().After(jwtExpiry) { + tm.SetCachedJWT("past-jwt", pastExpiry) + _, ok = tm.GetCachedJWT() + if ok { t.Error("JWT should be expired") } } -// TODO https://jira.mongodb.org/browse/DOCSP-54727 -// Note: Comprehensive testing of github_auth.go would require: -// 1. Mocking the Secret Manager client -// 2. Mocking the GitHub API client -// 3. Testing the full authentication flow: -// - JWT generation with valid PEM key -// - Installation token retrieval -// - Token caching and refresh logic -// - Organization-specific client creation -// - Error handling for API failures -// -// Example test scenarios that would require mocking: -// - TestConfigurePermissions_Success -// - TestConfigurePermissions_MissingAppID -// - TestConfigurePermissions_InvalidPEM -// - TestGetInstallationAccessToken_Success -// - TestGetInstallationAccessToken_Cached -// - TestGetInstallationAccessToken_Expired -// - TestGetRestClientForOrg_Success -// - TestGetRestClientForOrg_Cached -// - TestGetPrivateKeyFromSecret_SecretManager -// - TestGetPrivateKeyFromSecret_LocalFile -// - TestGetPrivateKeyFromSecret_EnvVar -// -// Refactoring suggestions for better testability: -// 1. Accept Secret Manager client as parameter instead of creating it internally -// 2. Accept GitHub client factory as parameter -// 3. Return errors instead of calling log.Fatal -// 4. Use dependency injection for HTTP client -// 5. Make JWT generation and caching logic more modular +func TestTokenManager_ThreadSafety(t *testing.T) { + tm := NewTokenManager() + + done := make(chan bool, 10) + + for i := range 5 { + go func(n int) { + defer func() { done <- true }() + org := "org-" + string(rune('A'+n)) + tm.SetTokenForOrg(org, "token-"+org, time.Now().Add(1*time.Hour)) + tm.SetCachedJWT("jwt-"+org, time.Now().Add(9*time.Minute)) + tm.SetInstallationAccessToken("iat-" + org) + }(i) + } + + for i := range 5 { + go func(n int) { + defer func() { done <- true }() + org := "org-" + string(rune('A'+n)) + _, _ = tm.GetTokenForOrg(org) + _, _ = tm.GetCachedJWT() + _ = tm.GetInstallationAccessToken() + _ = tm.GetHTTPClient() + }(i) + } + + for i := 0; i < 10; i++ { + <-done + } +} + +func TestGetInstallationAccessToken_Success(t *testing.T) { + c := &http.Client{} + httpmock.ActivateNonDefault(c) + t.Cleanup(httpmock.DeactivateAndReset) + + httpmock.RegisterResponder("POST", + "https://api.github.com/app/installations/12345/access_tokens", + httpmock.NewJsonResponderOrPanic(201, map[string]any{ + "token": "ghs_test123", + "expires_at": "2030-01-01T00:00:00Z", + }), + ) + + token, expiresAt, err := getInstallationAccessToken("12345", "fake-jwt", c) + if err != nil { + t.Fatalf("getInstallationAccessToken: %v", err) + } + if token != "ghs_test123" { + t.Errorf("token = %q, want ghs_test123", token) + } + if expiresAt.IsZero() { + t.Error("expected non-zero expiresAt") + } +} + +func TestGetInstallationAccessToken_MissingInstallationID(t *testing.T) { + _, _, err := getInstallationAccessToken("", "fake-jwt", nil) + if err == nil { + t.Error("expected error with empty installation ID") + } +} + +func TestGetInstallationAccessToken_Unauthorized(t *testing.T) { + c := &http.Client{} + httpmock.ActivateNonDefault(c) + t.Cleanup(httpmock.DeactivateAndReset) + + httpmock.RegisterResponder("POST", + "https://api.github.com/app/installations/12345/access_tokens", + httpmock.NewJsonResponderOrPanic(401, map[string]any{ + "message": "Bad credentials", + }), + ) + + _, _, err := getInstallationAccessToken("12345", "bad-jwt", c) + if err == nil { + t.Fatal("expected error on 401") + } + if !errors.Is(err, ErrAuthentication) { + t.Errorf("expected ErrAuthentication, got: %v", err) + } +} + +func TestConfigurePermissions_FullFlow(t *testing.T) { + // Generate a real RSA key for JWT signing + key, _ := rsa.GenerateKey(rand.Reader, 2048) + der := x509.MarshalPKCS1PrivateKey(key) + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der}) + + t.Setenv("SKIP_SECRET_MANAGER", "true") + t.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) + + // Use a fresh TokenManager to avoid polluting other tests + tm := NewTokenManager() + prev := defaultTokenManager + defaultTokenManager = tm + t.Cleanup(func() { defaultTokenManager = prev }) + + c := &http.Client{} + httpmock.ActivateNonDefault(c) + t.Cleanup(httpmock.DeactivateAndReset) + tm.SetHTTPClient(c) + + httpmock.RegisterResponder("POST", + "https://api.github.com/app/installations/99999/access_tokens", + httpmock.NewJsonResponderOrPanic(201, map[string]any{ + "token": "ghs_configured_token", + "expires_at": "2030-01-01T00:00:00Z", + }), + ) + + config := &configs.Config{ + AppId: "123456", + InstallationId: "99999", + } + + err := ConfigurePermissions(context.Background(), config) + if err != nil { + t.Fatalf("ConfigurePermissions: %v", err) + } + + if got := tm.GetInstallationAccessToken(); got != "ghs_configured_token" { + t.Errorf("installation token = %q, want ghs_configured_token", got) + } + + // Verify the token endpoint was called exactly once + info := httpmock.GetCallCountInfo() + if info["POST https://api.github.com/app/installations/99999/access_tokens"] != 1 { + t.Errorf("expected exactly 1 call to token endpoint, got %d", + info["POST https://api.github.com/app/installations/99999/access_tokens"]) + } +} + +func TestGetPrivateKeyFromSecret_EnvVar(t *testing.T) { + key, _ := rsa.GenerateKey(rand.Reader, 2048) + der := x509.MarshalPKCS1PrivateKey(key) + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der}) + + t.Setenv("SKIP_SECRET_MANAGER", "true") + t.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) + + got, err := getPrivateKeyFromSecret(context.Background(), &configs.Config{}) + if err != nil { + t.Fatalf("getPrivateKeyFromSecret: %v", err) + } + if string(got) != string(pemBytes) { + t.Error("returned key does not match expected PEM bytes") + } +} + +func TestGetPrivateKeyFromSecret_Base64(t *testing.T) { + key, _ := rsa.GenerateKey(rand.Reader, 2048) + der := x509.MarshalPKCS1PrivateKey(key) + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der}) + + t.Setenv("SKIP_SECRET_MANAGER", "true") + // Clear direct env var to force base64 path + t.Setenv("GITHUB_APP_PRIVATE_KEY", "") + t.Setenv("GITHUB_APP_PRIVATE_KEY_B64", base64.StdEncoding.EncodeToString(pemBytes)) + + got, err := getPrivateKeyFromSecret(context.Background(), &configs.Config{}) + if err != nil { + t.Fatalf("getPrivateKeyFromSecret: %v", err) + } + if string(got) != string(pemBytes) { + t.Error("returned key does not match expected PEM bytes") + } +} + +func TestGetPrivateKeyFromSecret_MissingAllEnvVars(t *testing.T) { + t.Setenv("SKIP_SECRET_MANAGER", "true") + t.Setenv("GITHUB_APP_PRIVATE_KEY", "") + t.Setenv("GITHUB_APP_PRIVATE_KEY_B64", "") + + _, err := getPrivateKeyFromSecret(context.Background(), &configs.Config{}) + if err == nil { + t.Error("expected error when no key env vars set") + } + if !errors.Is(err, ErrSecretAccess) { + t.Errorf("expected ErrSecretAccess, got: %v", err) + } +} + +func TestGetInstallationIDForOrg_Success(t *testing.T) { + // Set up a fresh TokenManager with a cached JWT to avoid private key lookup + tm := NewTokenManager() + tm.SetCachedJWT("fake-jwt", time.Now().Add(5*time.Minute)) + prev := defaultTokenManager + defaultTokenManager = tm + t.Cleanup(func() { defaultTokenManager = prev }) + + c := &http.Client{} + httpmock.ActivateNonDefault(c) + t.Cleanup(httpmock.DeactivateAndReset) + tm.SetHTTPClient(c) + + httpmock.RegisterResponder("GET", "https://api.github.com/app/installations", + httpmock.NewJsonResponderOrPanic(200, []map[string]any{ + {"id": 111, "account": map[string]any{"login": "org-a", "type": "Organization"}}, + {"id": 222, "account": map[string]any{"login": "org-b", "type": "Organization"}}, + }), + ) + + config := &configs.Config{AppId: "123"} + + id, err := getInstallationIDForOrg(context.Background(), config, "org-b") + if err != nil { + t.Fatalf("getInstallationIDForOrg: %v", err) + } + if id != "222" { + t.Errorf("installationID = %q, want 222", id) + } +} + +func TestGetInstallationIDForOrg_NotFound(t *testing.T) { + tm := NewTokenManager() + tm.SetCachedJWT("fake-jwt", time.Now().Add(5*time.Minute)) + prev := defaultTokenManager + defaultTokenManager = tm + t.Cleanup(func() { defaultTokenManager = prev }) + + c := &http.Client{} + httpmock.ActivateNonDefault(c) + t.Cleanup(httpmock.DeactivateAndReset) + tm.SetHTTPClient(c) + + httpmock.RegisterResponder("GET", "https://api.github.com/app/installations", + httpmock.NewJsonResponderOrPanic(200, []map[string]any{ + {"id": 111, "account": map[string]any{"login": "org-a", "type": "Organization"}}, + }), + ) + + config := &configs.Config{AppId: "123"} + + _, err := getInstallationIDForOrg(context.Background(), config, "no-such-org") + if err == nil { + t.Fatal("expected error for unknown org") + } + if !errors.Is(err, ErrInstallationNotFound) { + t.Errorf("expected ErrInstallationNotFound, got: %v", err) + } +} + +func TestGetInstallationIDForOrg_Unauthorized(t *testing.T) { + tm := NewTokenManager() + tm.SetCachedJWT("bad-jwt", time.Now().Add(5*time.Minute)) + prev := defaultTokenManager + defaultTokenManager = tm + t.Cleanup(func() { defaultTokenManager = prev }) + + c := &http.Client{} + httpmock.ActivateNonDefault(c) + t.Cleanup(httpmock.DeactivateAndReset) + tm.SetHTTPClient(c) + + httpmock.RegisterResponder("GET", "https://api.github.com/app/installations", + httpmock.NewJsonResponderOrPanic(401, map[string]any{"message": "Bad credentials"}), + ) + + config := &configs.Config{AppId: "123"} + + _, err := getInstallationIDForOrg(context.Background(), config, "org-a") + if err == nil { + t.Fatal("expected error on 401") + } + if !errors.Is(err, ErrAuthentication) { + t.Errorf("expected ErrAuthentication, got: %v", err) + } +} diff --git a/services/github_read.go b/services/github_read.go index e213712..3605871 100644 --- a/services/github_read.go +++ b/services/github_read.go @@ -3,12 +3,10 @@ package services import ( "context" "fmt" - "log" - "os" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/grove-platform/github-copier/configs" - . "github.com/grove-platform/github-copier/types" + "github.com/grove-platform/github-copier/types" "github.com/shurcooL/githubv4" ) @@ -18,46 +16,46 @@ import ( // - owner: The repository owner (e.g., "mongodb") // - repo: The repository name (e.g., "docs-sample-apps") // - pr_number: The pull request number -func GetFilesChangedInPr(owner string, repo string, pr_number int) ([]ChangedFile, error) { - if InstallationAccessToken == "" { - log.Println("No installation token provided") - if err := ConfigurePermissions(); err != nil { +func GetFilesChangedInPr(ctx context.Context, config *configs.Config, owner string, repo string, pr_number int) ([]types.ChangedFile, error) { + if defaultTokenManager.GetInstallationAccessToken() == "" { + LogWarning("No installation token provided, configuring permissions") + if err := ConfigurePermissions(ctx, config); err != nil { return nil, fmt.Errorf("failed to configure permissions: %w", err) } } - client, err := GetGraphQLClient() + // Use org-specific client to ensure we have the right installation token + client, err := GetGraphQLClientForOrg(ctx, config, owner) if err != nil { - return nil, fmt.Errorf("failed to get GraphQL client: %w", err) + return nil, fmt.Errorf("failed to get GraphQL client for org %s: %w", owner, err) } - ctx := context.Background() - var changedFiles []ChangedFile + var changedFiles []types.ChangedFile var cursor *githubv4.String = nil hasNextPage := true // Paginate through all files for hasNextPage { - var prQuery PullRequestQuery + var prQuery types.PullRequestQuery variables := map[string]interface{}{ "owner": githubv4.String(owner), "name": githubv4.String(repo), - "number": githubv4.Int(pr_number), + "number": githubv4.Int(pr_number), // #nosec G115 -- PR numbers won't overflow int32 "cursor": cursor, } err := client.Query(ctx, &prQuery, variables) if err != nil { - LogCritical(fmt.Sprintf("Failed to execute query GetFilesChanged: %v", err)) + LogCritical("Failed to execute query GetFilesChanged", "error", err) return nil, err } // Append files from this page for _, edge := range prQuery.Repository.PullRequest.Files.Edges { - changedFiles = append(changedFiles, ChangedFile{ + changedFiles = append(changedFiles, types.ChangedFile{ Path: string(edge.Node.Path), - Additions: int(edge.Node.Additions), - Deletions: int(edge.Node.Deletions), + Additions: int(edge.Node.Additions), // #nosec G115 -- PR additions/deletions fit in int + Deletions: int(edge.Node.Deletions), // #nosec G115 -- PR additions/deletions fit in int Status: string(edge.Node.ChangeType), }) } @@ -69,52 +67,31 @@ func GetFilesChangedInPr(owner string, repo string, pr_number int) ([]ChangedFil } } - LogInfo(fmt.Sprintf("PR has %d changed files.", len(changedFiles))) - - // Log all files for debugging (especially to see if server files are included) - LogInfo("=== ALL FILES FROM GRAPHQL API ===") - for i, file := range changedFiles { - LogInfo(fmt.Sprintf(" [%d] %s (status: %s)", i, file.Path, file.Status)) - } - LogInfo("=== END FILE LIST ===") - - // Count files by directory for debugging - clientCount := 0 - serverCount := 0 - otherCount := 0 - for _, file := range changedFiles { - if len(file.Path) >= 13 && file.Path[:13] == "mflix/client/" { - clientCount++ - } else if len(file.Path) >= 13 && file.Path[:13] == "mflix/server/" { - serverCount++ - } else { - otherCount++ - } - } - LogInfo(fmt.Sprintf("File breakdown: client=%d, server=%d, other=%d", clientCount, serverCount, otherCount)) + LogInfoCtx(ctx, "Retrieved changed files from PR", map[string]interface{}{ + "file_count": len(changedFiles), + }) return changedFiles, nil } // RetrieveFileContents fetches the contents of a file from the config repository at the specified path. // It returns a github.RepositoryContent object containing the file details. -func RetrieveFileContents(filePath string) (github.RepositoryContent, error) { - owner := os.Getenv(configs.ConfigRepoOwner) - repo := os.Getenv(configs.ConfigRepoName) +func RetrieveFileContents(ctx context.Context, config *configs.Config, filePath string) (github.RepositoryContent, error) { + owner := config.ConfigRepoOwner + repo := config.ConfigRepoName client := GetRestClient() - ctx := context.Background() fileContent, _, _, err := client.Repositories.GetContents(ctx, owner, repo, filePath, &github.RepositoryContentGetOptions{ - Ref: os.Getenv(configs.ConfigRepoBranch), + Ref: config.ConfigRepoBranch, }) if err != nil { return github.RepositoryContent{}, fmt.Errorf("failed to get file content for %s: %w", filePath, err) } if fileContent == nil { - return github.RepositoryContent{}, fmt.Errorf("file content is nil for path: %s", filePath) + return github.RepositoryContent{}, fmt.Errorf("%w: %s", ErrContentNil, filePath) } return *fileContent, nil } diff --git a/services/github_read_test.go b/services/github_read_test.go index 3f79a59..6a3fd18 100644 --- a/services/github_read_test.go +++ b/services/github_read_test.go @@ -1,10 +1,11 @@ package services_test import ( + "context" "encoding/base64" "testing" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/grove-platform/github-copier/services" "github.com/stretchr/testify/require" @@ -99,7 +100,8 @@ func TestRetrieveFileContents_Success(t *testing.T) { payload := "hello" stubContentsForBothOwners(path, b64(payload), owner, repo) - rc, err := services.RetrieveFileContents(path) + cfg := test.TestConfig() + rc, err := services.RetrieveFileContents(context.Background(), cfg, path) require.NoError(t, err, "expected RetrieveFileContents to succeed") require.IsType(t, github.RepositoryContent{}, rc) require.Equal(t, path, rc.GetPath()) diff --git a/services/github_write_to_source.go b/services/github_write_to_source.go index c1941cd..894119c 100644 --- a/services/github_write_to_source.go +++ b/services/github_write_to_source.go @@ -4,106 +4,163 @@ import ( "context" "encoding/json" "fmt" - "os" "time" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/grove-platform/github-copier/configs" - . "github.com/grove-platform/github-copier/types" + "github.com/grove-platform/github-copier/types" ) -func UpdateDeprecationFile() { +// UpdateDeprecationFile updates the deprecation file in the source repo with the provided data map. +// The deprecation file tracks which files have been deleted from the source repo. +func UpdateDeprecationFile(ctx context.Context, config *configs.Config, filesToDeprecate map[string]types.Configs, sourceRepoOwner, sourceRepoName, sourceBranch string) { // Early return if there are no files to deprecate - prevents blank commits - if len(FilesToDeprecate) == 0 { + if len(filesToDeprecate) == 0 { LogInfo("No deprecated files to record; skipping deprecation file update") return } - // Fetch the deprecation file from the repository + sourceRepo := fmt.Sprintf("%s/%s", sourceRepoOwner, sourceRepoName) + + if config.DryRun { + LogInfo("[DRY-RUN] Would update deprecation file", + "file", config.DeprecationFile, + "source_repo", sourceRepo, + "source_branch", sourceBranch, + "deprecated_count", len(filesToDeprecate), + ) + for path := range filesToDeprecate { + LogInfo("[DRY-RUN] Would mark as deprecated", "path", path) + } + return + } + + // Fetch the deprecation file from the source repository client := GetRestClient() - ctx := context.Background() fileContent, _, _, err := client.Repositories.GetContents( ctx, - os.Getenv(configs.ConfigRepoOwner), - os.Getenv(configs.ConfigRepoName), - os.Getenv(configs.DeprecationFile), + sourceRepoOwner, + sourceRepoName, + config.DeprecationFile, &github.RepositoryContentGetOptions{ - Ref: os.Getenv(configs.ConfigRepoBranch), + Ref: sourceBranch, }, ) + + var deprecationFile types.DeprecationFile + if err != nil { - LogError(fmt.Sprintf("Error getting deprecation file: %v", err)) - return - } - if fileContent == nil { + // If file doesn't exist, start with empty array + LogInfo("Deprecation file not found, will create new one", + "file", config.DeprecationFile, + "source_repo", sourceRepo, + ) + deprecationFile = types.DeprecationFile{} + } else if fileContent == nil { LogError("Deprecation file content is nil") return - } + } else { + content, err := fileContent.GetContent() + if err != nil { + LogError("Error decoding deprecation file", "error", err) + return + } - content, err := fileContent.GetContent() - if err != nil { - LogError(fmt.Sprintf("Error decoding deprecation file: %v", err)) - return + err = json.Unmarshal([]byte(content), &deprecationFile) + if err != nil { + LogError("Failed to unmarshal deprecation file", "file", config.DeprecationFile, "error", err) + return + } } - var deprecationFile DeprecationFile - err = json.Unmarshal([]byte(content), &deprecationFile) - if err != nil { - LogError(fmt.Sprintf("Failed to unmarshal %s: %v", configs.DeprecationFile, err)) - return + // Build a set of existing entries for duplicate detection + // Key format: "filename|repo|branch" to identify unique entries + existingEntries := make(map[string]bool) + for _, entry := range deprecationFile { + key := entry.FileName + "|" + entry.Repo + "|" + entry.Branch + existingEntries[key] = true } - for key, value := range FilesToDeprecate { - newDeprecatedFileEntry := DeprecatedFileEntry{ + entriesAdded := 0 + for key, value := range filesToDeprecate { + // Check for duplicates before appending (prevents issues with webhook redelivery) + entryKey := key + "|" + value.TargetRepo + "|" + value.TargetBranch + if existingEntries[entryKey] { + LogInfo("Skipping duplicate deprecation entry", + "filename", key, + "repo", value.TargetRepo, + "branch", value.TargetBranch, + ) + continue + } + + newDeprecatedFileEntry := types.DeprecatedFileEntry{ FileName: key, Repo: value.TargetRepo, Branch: value.TargetBranch, DeletedOn: time.Now().Format(time.RFC3339), } deprecationFile = append(deprecationFile, newDeprecatedFileEntry) + existingEntries[entryKey] = true // Mark as added to prevent duplicates within current batch + entriesAdded++ + } + + // Early return if all entries were duplicates + if entriesAdded == 0 { + LogInfo("All deprecation entries already exist; skipping update") + return } updatedJSON, err := json.MarshalIndent(deprecationFile, "", " ") if err != nil { - LogError(fmt.Sprintf("Error marshaling JSON: %v", err)) + LogError("Error marshaling JSON", "error", err) + return } - message := fmt.Sprintf("Updating %s.", os.Getenv(configs.DeprecationFile)) - uploadDeprecationFileChanges(message, string(updatedJSON)) + message := fmt.Sprintf("Updating %s.", config.DeprecationFile) + uploadDeprecationFileChanges(ctx, config, message, string(updatedJSON), sourceRepoOwner, sourceRepoName, sourceBranch, fileContent) - LogInfo(fmt.Sprintf("Successfully updated %s with %d entries", os.Getenv(configs.DeprecationFile), len(FilesToDeprecate))) + LogInfo("Successfully updated deprecation file", + "file", config.DeprecationFile, + "source_repo", sourceRepo, + "entries", len(filesToDeprecate), + ) } -func uploadDeprecationFileChanges(message string, newDeprecationFileContents string) { +func uploadDeprecationFileChanges(ctx context.Context, config *configs.Config, message string, newDeprecationFileContents string, sourceRepoOwner, sourceRepoName, sourceBranch string, existingContent *github.RepositoryContent) { client := GetRestClient() - ctx := context.Background() - - targetFileContent, _, _, err := client.Repositories.GetContents(ctx, os.Getenv(configs.ConfigRepoOwner), os.Getenv(configs.ConfigRepoName), - os.Getenv(configs.DeprecationFile), &github.RepositoryContentGetOptions{Ref: os.Getenv(configs.ConfigRepoBranch)}) - - if err != nil { - LogError(fmt.Sprintf("Error getting deprecation file contents: %v", err)) - return - } - if targetFileContent == nil { - LogError("Target deprecation file content is nil") - return - } options := &github.RepositoryContentFileOptions{ - Message: github.String(message), + Message: github.Ptr(message), Content: []byte(newDeprecationFileContents), - Branch: github.String(os.Getenv(configs.ConfigRepoBranch)), - Committer: &github.CommitAuthor{Name: github.String(os.Getenv(configs.CommitterName)), - Email: github.String(os.Getenv(configs.CommitterEmail))}, + Branch: github.Ptr(sourceBranch), + Committer: &github.CommitAuthor{ + Name: github.Ptr(config.CommitterName), + Email: github.Ptr(config.CommitterEmail), + }, + } + + var err error + if existingContent != nil && existingContent.SHA != nil { + // Update existing file + options.SHA = existingContent.SHA + _, _, err = client.Repositories.UpdateFile(ctx, sourceRepoOwner, sourceRepoName, config.DeprecationFile, options) + } else { + // Create new file + _, _, err = client.Repositories.CreateFile(ctx, sourceRepoOwner, sourceRepoName, config.DeprecationFile, options) } - options.SHA = targetFileContent.SHA - _, _, err = client.Repositories.UpdateFile(ctx, os.Getenv(configs.ConfigRepoOwner), os.Getenv(configs.ConfigRepoName), os.Getenv(configs.DeprecationFile), options) if err != nil { - LogError(fmt.Sprintf("Cannot update deprecation file: %v", err)) + LogError("Cannot update deprecation file", + "error", err, + "source_repo", fmt.Sprintf("%s/%s", sourceRepoOwner, sourceRepoName), + ) + return } - LogInfo("Deprecation file updated.") + LogInfo("Deprecation file updated.", + "source_repo", fmt.Sprintf("%s/%s", sourceRepoOwner, sourceRepoName), + "branch", sourceBranch, + ) } diff --git a/services/github_write_to_source_test.go b/services/github_write_to_source_test.go index c9af808..0d36c4a 100644 --- a/services/github_write_to_source_test.go +++ b/services/github_write_to_source_test.go @@ -1,92 +1,28 @@ package services import ( + "context" "testing" - . "github.com/grove-platform/github-copier/types" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/types" ) -func TestUpdateDeprecationFile_EmptyList(t *testing.T) { - // When FilesToDeprecate is empty, UpdateDeprecationFile should return early - // FilesToDeprecate is a map[string]Configs - originalFiles := FilesToDeprecate - defer func() { - FilesToDeprecate = originalFiles - }() - - FilesToDeprecate = make(map[string]Configs) - - // This should not panic or error - it should return early - // Note: This test doesn't verify the actual GitHub API call since that would - // require mocking the GitHub client, which is a global variable - UpdateDeprecationFile() - - // If we get here without panic, the test passes +func TestUpdateDeprecationFile_EmptyMap(t *testing.T) { + // When the map is empty, UpdateDeprecationFile should return early without panic. + UpdateDeprecationFile(context.Background(), configs.NewConfig(), map[string]types.Configs{}, "owner", "repo", "main") } func TestUpdateDeprecationFile_WithFiles(t *testing.T) { - // Set up files to deprecate - originalFiles := FilesToDeprecate - defer func() { - FilesToDeprecate = originalFiles - }() - - FilesToDeprecate = map[string]Configs{ - "examples/old-example.go": { - TargetRepo: "test/target", - TargetBranch: "main", - }, - "examples/deprecated.go": { - TargetRepo: "test/target", - TargetBranch: "main", - }, - } - - // Note: This test will fail if it actually tries to call GitHub API + // Note: This test will fail if it actually tries to call GitHub API. // In a real test environment, we would need to: // 1. Mock the GetRestClient() function // 2. Mock the GitHub API responses // 3. Verify the correct API calls were made - // - // For now, this test documents the expected behavior - // The actual implementation would require refactoring to inject dependencies - - // Since we can't easily test this without mocking, we'll skip the actual call t.Skip("Skipping test that requires GitHub API mocking") } -func TestFilesToDeprecate_GlobalVariable(t *testing.T) { - // Test that we can manipulate the global FilesToDeprecate variable - originalFiles := FilesToDeprecate - defer func() { - FilesToDeprecate = originalFiles - }() - - // Set test files (FilesToDeprecate is a map[string]Configs) - testFiles := map[string]Configs{ - "file1.go": {TargetRepo: "test/repo1", TargetBranch: "main"}, - "file2.go": {TargetRepo: "test/repo2", TargetBranch: "develop"}, - "file3.go": {TargetRepo: "test/repo3", TargetBranch: "main"}, - } - FilesToDeprecate = testFiles - - if len(FilesToDeprecate) != 3 { - t.Errorf("FilesToDeprecate length = %d, want 3", len(FilesToDeprecate)) - } - - for file, config := range testFiles { - if deprecatedConfig, exists := FilesToDeprecate[file]; !exists { - t.Errorf("FilesToDeprecate missing file %s", file) - } else if deprecatedConfig.TargetRepo != config.TargetRepo { - t.Errorf("FilesToDeprecate[%s].TargetRepo = %s, want %s", file, deprecatedConfig.TargetRepo, config.TargetRepo) - } - } -} - func TestDeprecationFileEnvironmentVariables(t *testing.T) { - // Test that deprecation file configuration can be set via environment variables - // The UpdateDeprecationFile function uses os.Getenv to read these values - tests := []struct { name string deprecationFile string @@ -107,8 +43,6 @@ func TestDeprecationFileEnvironmentVariables(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // The deprecation file path is typically configured via environment variables - // This test documents the expected configuration approach if tt.deprecationFile == "" { t.Error("Deprecation file path should not be empty") } diff --git a/services/github_write_to_target.go b/services/github_write_to_target.go index e77066e..2159c51 100644 --- a/services/github_write_to_target.go +++ b/services/github_write_to_target.go @@ -4,152 +4,179 @@ import ( "context" "fmt" "net/http" - "os" "strings" "time" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/grove-platform/github-copier/configs" - . "github.com/grove-platform/github-copier/types" - "github.com/pkg/errors" + "github.com/grove-platform/github-copier/types" ) -// FilesToUpload is a map where the key is the repo name -// and the value is of type [UploadFileContent], which -// contains the target branch name and the collection of files -// to be uploaded. -var FilesToUpload map[UploadKey]UploadFileContent -var FilesToDeprecate map[string]Configs - -// repoOwner returns the config repository owner from environment variables. -func repoOwner() string { return os.Getenv(configs.ConfigRepoOwner) } - // parseRepoPath parses a repository path in the format "owner/repo" and returns owner and repo separately. -// If the path doesn't contain a slash, it returns the source repo owner from env and the path as repo name. -func parseRepoPath(repoPath string) (owner, repo string) { +// If the path doesn't contain a slash, it returns defaultOwner and the path as repo name. +func parseRepoPath(repoPath string, defaultOwner string) (owner, repo string) { parts := strings.Split(repoPath, "/") if len(parts) == 2 { return parts[0], parts[1] } - // Fallback to source repo owner if no slash found (backward compatibility) - return repoOwner(), repoPath + // Fallback to default owner if no slash found (backward compatibility) + return defaultOwner, repoPath } // normalizeRepoName ensures a repository name includes the owner prefix. // If the repo name already has an owner (contains "/"), returns it as-is. -// Otherwise, prepends the default repo owner from environment. -func normalizeRepoName(repoName string) string { +// Otherwise, prepends the defaultOwner. +func normalizeRepoName(repoName string, defaultOwner string) string { if strings.Contains(repoName, "/") { return repoName } - return repoOwner() + "/" + repoName + return defaultOwner + "/" + repoName } -// AddFilesToTargetRepoBranch uploads files to the target repository branch -// using the specified commit strategy (direct or via pull request). -func AddFilesToTargetRepoBranch() { - AddFilesToTargetRepoBranchWithFetcher(nil, nil) -} +// normalizeRefPath ensures a ref path is in the correct format for different GitHub API calls. +// For GetRef: expects "heads/main" (no "refs/" prefix) +// For UpdateRef: expects "refs/heads/main" (full ref path) +func normalizeRefPath(branchPath string, fullPath bool) string { + // Strip "refs/" prefix if present + refPath := strings.TrimPrefix(branchPath, "refs/") -// AddFilesToTargetRepoBranchWithFetcher uploads files to the target repository branch -// using the specified commit strategy (direct or via pull request). -// If prTemplateFetcher is provided, it will be used to fetch PR templates when use_pr_template is true. -// If metricsCollector is provided, it will be used to record upload failures. -func AddFilesToTargetRepoBranchWithFetcher(prTemplateFetcher PRTemplateFetcher, metricsCollector *MetricsCollector) { - ctx := context.Background() + // Ensure "heads/" prefix exists (unless it's a tag) + if !strings.HasPrefix(refPath, "heads/") && !strings.HasPrefix(refPath, "tags/") { + refPath = "heads/" + refPath + } - for key, value := range FilesToUpload { - // Parse the repository to get the organization - owner, _ := parseRepoPath(key.RepoName) + // Add "refs/" prefix back if full path is needed + if fullPath { + return "refs/" + refPath + } + return refPath +} - // Get a client authenticated for this organization - client, err := GetRestClientForOrg(owner) - if err != nil { - LogCritical(fmt.Sprintf("Failed to get GitHub client for org %s: %v", owner, err)) - // Record failure for each file in this batch - if metricsCollector != nil { - for range value.Content { - metricsCollector.RecordFileUploadFailed() - } +// AddFilesToTargetRepos uploads files to target repository branches. +// It accepts the upload map as a parameter for concurrency safety. +func AddFilesToTargetRepos(ctx context.Context, config *configs.Config, filesToUpload map[types.UploadKey]types.UploadFileContent, prTemplateFetcher PRTemplateFetcher, metricsCollector *MetricsCollector) { + if config.DryRun { + for key, value := range filesToUpload { + LogInfo("[DRY-RUN] Would upload files to target repo", + "repo", key.RepoName, + "branch", key.BranchPath, + "file_count", len(value.Content), + "strategy", value.CommitStrategy, + ) + for path := range value.Content { + LogInfo("[DRY-RUN] Would write file", "repo", key.RepoName, "path", path) } - continue } + return + } - // Determine commit strategy from value (set by pattern-matching system) - strategy := string(value.CommitStrategy) - if strategy == "" { - strategy = "direct" // default + for key, value := range filesToUpload { + if err := uploadToTarget(ctx, config, key, value, prTemplateFetcher); err != nil { + LogCritical("Failed to upload files", "repo", key.RepoName, "error", err) + recordBatchFailure(metricsCollector, len(value.Content)) } + } +} - // Get commit message from value or use default - commitMsg := value.CommitMessage - if strings.TrimSpace(commitMsg) == "" { - commitMsg = os.Getenv(configs.DefaultCommitMessage) - if strings.TrimSpace(commitMsg) == "" { - commitMsg = configs.NewConfig().DefaultCommitMessage - } - } +// uploadToTarget handles a single upload-key: authenticates for the target org, +// resolves commit parameters, and dispatches to the appropriate strategy. +func uploadToTarget(ctx context.Context, config *configs.Config, key types.UploadKey, value types.UploadFileContent, prTemplateFetcher PRTemplateFetcher) error { + owner, _ := parseRepoPath(key.RepoName, config.ConfigRepoOwner) - // Get PR title from value or use commit message - prTitle := value.PRTitle - if strings.TrimSpace(prTitle) == "" { - prTitle = commitMsg - } + client, err := GetRestClientForOrg(ctx, config, owner) + if err != nil { + return fmt.Errorf("get GitHub client for org %s: %w", owner, err) + } + + params := resolveCommitParams(config, key, value, prTemplateFetcher, client, ctx) + + switch params.strategy { + case "direct": + LogInfo("Using direct commit strategy", + "repo", key.RepoName, + "branch", key.BranchPath, + "strategy_source", key.CommitStrategy, + "file_count", len(value.Content), + ) + return addFilesToBranch(ctx, config, client, key, value.Content, params.commitMsg) + default: // "pr" or "pull_request" + LogInfo("Using PR commit strategy", + "repo", key.RepoName, + "branch", key.BranchPath, + "strategy_source", key.CommitStrategy, + "file_count", len(value.Content), + "auto_merge", params.mergeWithoutReview, + ) + return addFilesViaPR(ctx, config, client, key, value.Content, params.commitMsg, params.prTitle, params.prBody, params.mergeWithoutReview) + } +} - // Get PR body from value - prBody := value.PRBody - - // Fetch and merge PR template if requested - if value.UsePRTemplate && prTemplateFetcher != nil && strategy != "direct" { - targetBranch := strings.TrimPrefix(key.BranchPath, "refs/heads/") - template, err := prTemplateFetcher.FetchPRTemplate(ctx, client, key.RepoName, targetBranch) - if err != nil { - LogWarning(fmt.Sprintf("Failed to fetch PR template for %s: %v", key.RepoName, err)) - } else if template != "" { - // Merge configured body with template - prBody = MergePRBodyWithTemplate(prBody, template) - LogInfo(fmt.Sprintf("Merged PR template for %s", key.RepoName)) - } - } +// commitParams groups the resolved parameters for a single upload operation. +type commitParams struct { + strategy string + commitMsg string + prTitle string + prBody string + mergeWithoutReview bool +} - // Get auto-merge setting from value - mergeWithoutReview := value.AutoMergePR - - switch strategy { - case "direct": // commits directly to the target branch - LogInfo(fmt.Sprintf("Using direct commit strategy for %s on branch %s", key.RepoName, key.BranchPath)) - if err := addFilesToBranch(ctx, client, key, value.Content, commitMsg); err != nil { - LogCritical(fmt.Sprintf("Failed to add files to target branch: %v\n", err)) - // Record failure for each file in this batch - if metricsCollector != nil { - for range value.Content { - metricsCollector.RecordFileUploadFailed() - } - } - } - default: // "pr" or "pull_request" strategy - LogInfo(fmt.Sprintf("Using PR commit strategy for %s on branch %s (auto_merge=%v)", key.RepoName, key.BranchPath, mergeWithoutReview)) - if err := addFilesViaPR(ctx, client, key, value.Content, commitMsg, prTitle, prBody, mergeWithoutReview); err != nil { - LogCritical(fmt.Sprintf("Failed via PR path: %v\n", err)) - // Record failure for each file in this batch - if metricsCollector != nil { - for range value.Content { - metricsCollector.RecordFileUploadFailed() - } - } - } +// resolveCommitParams derives commit strategy, message, PR title/body, and template +// from the upload value and config defaults. +func resolveCommitParams(config *configs.Config, key types.UploadKey, value types.UploadFileContent, prTemplateFetcher PRTemplateFetcher, client *github.Client, ctx context.Context) commitParams { + strategy := string(value.CommitStrategy) + if strategy == "" { + strategy = "direct" + } + + commitMsg := value.CommitMessage + if strings.TrimSpace(commitMsg) == "" { + commitMsg = config.DefaultCommitMessage + } + + prTitle := value.PRTitle + if strings.TrimSpace(prTitle) == "" { + prTitle = commitMsg + } + + prBody := value.PRBody + if value.UsePRTemplate && prTemplateFetcher != nil && strategy != "direct" { + targetBranch := strings.TrimPrefix(key.BranchPath, "refs/heads/") + template, err := prTemplateFetcher.FetchPRTemplate(ctx, client, key.RepoName, targetBranch) + if err != nil { + LogWarning("Failed to fetch PR template", "repo", key.RepoName, "error", err) + } else if template != "" { + prBody = MergePRBodyWithTemplate(prBody, template) + LogInfo("Merged PR template", "repo", key.RepoName) } } + + return commitParams{ + strategy: strategy, + commitMsg: commitMsg, + prTitle: prTitle, + prBody: prBody, + mergeWithoutReview: value.AutoMergePR, + } +} + +// recordBatchFailure records n file upload failures on the metrics collector. +func recordBatchFailure(mc *MetricsCollector, n int) { + if mc == nil { + return + } + for i := 0; i < n; i++ { + mc.RecordFileUploadFailed() + } } // createPullRequest opens a pull request from head to base in the specified repository. -func createPullRequest(ctx context.Context, client *github.Client, repo, head, base, title, body string) (*github.PullRequest, error) { - owner, repoName := parseRepoPath(repo) +func createPullRequest(ctx context.Context, client *github.Client, defaultOwner, repo, head, base, title, body string) (*github.PullRequest, error) { + owner, repoName := parseRepoPath(repo, defaultOwner) pr := &github.NewPullRequest{ - Title: github.String(title), - Head: github.String(head), // for same-repo branches, just "branch"; for forks, use "owner:branch" - Base: github.String(base), // e.g. "main" - Body: github.String(body), + Title: github.Ptr(title), + Head: github.Ptr(head), // for same-repo branches, just "branch"; for forks, use "owner:branch" + Base: github.Ptr(base), // e.g. "main" + Body: github.Ptr(body), } created, _, err := client.PullRequests.Create(ctx, owner, repoName, pr) if err != nil { @@ -158,123 +185,213 @@ func createPullRequest(ctx context.Context, client *github.Client, repo, head, b return created, nil } +// findExistingCopierPR searches for an open PR whose head branch starts with "copier/" +// targeting the given base branch. Returns nil if none found. +func findExistingCopierPR(ctx context.Context, client *github.Client, owner, repoName, baseBranch string) *github.PullRequest { + prs, _, err := client.PullRequests.List(ctx, owner, repoName, &github.PullRequestListOptions{ + State: "open", + Base: baseBranch, + ListOptions: github.ListOptions{ + PerPage: 50, + }, + }) + if err != nil { + LogWarning("Failed to list PRs for dedup check; will create new PR", "repo", owner+"/"+repoName, "error", err) + return nil + } + for _, pr := range prs { + if strings.HasPrefix(pr.GetHead().GetRef(), "copier/") { + return pr + } + } + return nil +} + // addFilesViaPR creates a temporary branch, commits files to it using the provided commitMessage, // opens a pull request with prTitle and prBody, and optionally merges it automatically. -func addFilesViaPR(ctx context.Context, client *github.Client, key UploadKey, +// If an existing open PR from a copier/* branch is found, the files are pushed to that +// branch and the PR is updated instead of creating a duplicate. +func addFilesViaPR(ctx context.Context, config *configs.Config, client *github.Client, key types.UploadKey, files []github.RepositoryContent, commitMessage string, prTitle string, prBody string, mergeWithoutReview bool, ) error { + defaultOwner := config.ConfigRepoOwner + baseBranch := strings.TrimPrefix(key.BranchPath, "refs/heads/") + owner, repoName := parseRepoPath(key.RepoName, defaultOwner) + + // 0. Check for an existing open copier PR targeting this base branch. + existingPR := findExistingCopierPR(ctx, client, owner, repoName, baseBranch) + if existingPR != nil { + existingBranch := existingPR.GetHead().GetRef() + LogInfo("Found existing open copier PR; updating instead of creating new", + "pr_number", existingPR.GetNumber(), + "branch", existingBranch, + "repo", key.RepoName, + ) + + // Push new files to the existing branch + if err := commitFilesToBranch(ctx, config, client, key, files, existingBranch, commitMessage); err != nil { + return fmt.Errorf("commit to existing copier branch %s: %w", existingBranch, err) + } + + // Update the PR title/body to reflect the latest content + _, _, err := client.PullRequests.Edit(ctx, owner, repoName, existingPR.GetNumber(), &github.PullRequest{ + Title: github.Ptr(prTitle), + Body: github.Ptr(prBody), + }) + if err != nil { + LogWarning("Failed to update existing PR title/body", "pr_number", existingPR.GetNumber(), "error", err) + } + + if mergeWithoutReview { + return autoMergePR(ctx, config, client, key.RepoName, defaultOwner, existingPR.GetNumber(), existingBranch) + } + LogInfo("Existing PR updated and awaiting review", "pr_number", existingPR.GetNumber()) + return nil + } + + // No existing PR โ€” create a new temp branch and PR. tempBranch := "copier/" + time.Now().UTC().Format("20060102-150405") - // 1) Create branch off the target branch specified in key.BranchPath or default to "main" - baseBranch := strings.TrimPrefix(key.BranchPath, "refs/heads/") - newRef, err := createBranch(ctx, client, key.RepoName, tempBranch, baseBranch) - if err != nil { + // 1. Create branch off the target + if _, err := createBranch(ctx, client, defaultOwner, key.RepoName, tempBranch, baseBranch); err != nil { return fmt.Errorf("create branch: %w", err) } - _ = newRef // we just need it created; ref is not reused directly - // 2) Commit files to temp branch + // 2. Commit files to temp branch + if err := commitFilesToBranch(ctx, config, client, key, files, tempBranch, commitMessage); err != nil { + return err + } + + // 3. Open PR from temp branch โ†’ base branch + pr, err := createPullRequest(ctx, client, defaultOwner, key.RepoName, tempBranch, baseBranch, prTitle, prBody) + if err != nil { + return fmt.Errorf("create PR: %w", err) + } + + LogInfo("PR created", "pr_number", pr.GetNumber(), "from_branch", tempBranch, "base_branch", baseBranch) + LogInfo("PR URL", "url", pr.GetHTMLURL()) + + // 4. Optionally auto-merge and clean up + if mergeWithoutReview { + return autoMergePR(ctx, config, client, key.RepoName, defaultOwner, pr.GetNumber(), tempBranch) + } + LogInfo("PR created and awaiting review", "pr_number", pr.GetNumber()) + return nil +} + +// commitFilesToBranch decodes file contents and creates a tree + commit on the temp branch. +// If the resulting tree is identical to the branch's current tree, the commit is skipped. +func commitFilesToBranch(ctx context.Context, config *configs.Config, client *github.Client, key types.UploadKey, + files []github.RepositoryContent, tempBranch string, commitMessage string, +) error { entries := make(map[string]string, len(files)) for _, f := range files { - content, _ := f.GetContent() + content, err := f.GetContent() + if err != nil { + return fmt.Errorf("decode content for %s: %w", f.GetName(), err) + } entries[f.GetName()] = content } - tempKey := UploadKey{RepoName: key.RepoName, BranchPath: "refs/heads/" + tempBranch} - treeSHA, baseSHA, err := createCommitTree(ctx, client, tempKey, entries) + tempKey := types.UploadKey{RepoName: key.RepoName, BranchPath: "refs/heads/" + tempBranch} + tr, err := createCommitTree(ctx, config, client, tempKey, entries) if err != nil { return fmt.Errorf("create tree on temp branch: %w", err) } - if err = createCommit(ctx, client, tempKey, baseSHA, treeSHA, commitMessage); err != nil { + + if tr.TreeSHA == tr.BaseTreeSHA { + LogInfo("Skipping empty commit on temp branch โ€” tree unchanged", + "repo", key.RepoName, + "branch", tempBranch, + "tree_sha", tr.TreeSHA, + ) + return nil + } + + if err = createCommit(ctx, client, config.ConfigRepoOwner, tempKey, tr.BaseSHA, tr.TreeSHA, commitMessage); err != nil { return fmt.Errorf("create commit on temp branch: %w", err) } + return nil +} - // 3) Create PR from temp branch to base branch - base := strings.TrimPrefix(key.BranchPath, "refs/heads/") - pr, err := createPullRequest(ctx, client, key.RepoName, tempBranch, base, prTitle, prBody) - if err != nil { - return fmt.Errorf("create PR: %w", err) +// autoMergePR polls the PR for mergeability, merges it, and deletes the temp branch. +func autoMergePR(ctx context.Context, config *configs.Config, client *github.Client, repo string, defaultOwner string, prNumber int, tempBranch string) error { + owner, repoName := parseRepoPath(repo, defaultOwner) + + mergeable, state := pollMergeability(ctx, client, owner, repoName, prNumber, config.PRMergePollMaxAttempts, config.PRMergePollInterval) + if mergeable != nil && !*mergeable || strings.EqualFold(state, "dirty") { + LogWarning("PR is not mergeable; leaving open for manual resolution", "pr_number", prNumber, "state", state) + return fmt.Errorf("%w: pull request #%d has conflicts (state=%s)", ErrMergeConflict, prNumber, state) } - // 4) Optionally merge the PR without review if MergeWithoutReview is true - LogInfo(fmt.Sprintf("PR created: #%d from %s to %s", pr.GetNumber(), tempBranch, base)) - LogInfo(fmt.Sprintf("PR URL: %s", pr.GetHTMLURL())) - if mergeWithoutReview { - // Poll PR for mergeability; GitHub may take a moment to compute it - // Get polling configuration from environment or use defaults - cfg := configs.NewConfig() - maxAttempts := cfg.PRMergePollMaxAttempts - if envAttempts := os.Getenv(configs.PRMergePollMaxAttempts); envAttempts != "" { - if parsed, err := parseIntWithDefault(envAttempts, maxAttempts); err == nil { - maxAttempts = parsed - } - } + if err := mergePR(ctx, client, defaultOwner, repo, prNumber); err != nil { + return fmt.Errorf("merge PR: %w", err) + } - pollInterval := cfg.PRMergePollInterval - if envInterval := os.Getenv(configs.PRMergePollInterval); envInterval != "" { - if parsed, err := parseIntWithDefault(envInterval, pollInterval); err == nil { - pollInterval = parsed - } - } + if err := deleteBranchIfExists(ctx, client, defaultOwner, repo, &github.Reference{Ref: github.Ptr("refs/heads/" + tempBranch)}); err != nil { + LogWarning("Failed to delete temp branch after merge", "error", err) + } + return nil +} - var mergeable *bool - var mergeableState string - owner, repoName := parseRepoPath(key.RepoName) - for i := 0; i < maxAttempts; i++ { - current, _, gerr := client.PullRequests.Get(ctx, owner, repoName, pr.GetNumber()) - if gerr == nil && current != nil { - mergeable = current.Mergeable - mergeableState = current.GetMergeableState() - if mergeable != nil { // computed - break - } +// pollMergeability polls the GitHub API until the PR's mergeability is computed or attempts are exhausted. +func pollMergeability(ctx context.Context, client *github.Client, owner string, repo string, prNumber int, maxAttempts int, pollIntervalMs int) (mergeable *bool, state string) { + for i := 0; i < maxAttempts; i++ { + current, _, err := client.PullRequests.Get(ctx, owner, repo, prNumber) + if err == nil && current != nil { + mergeable = current.Mergeable + state = current.GetMergeableState() + if mergeable != nil { + return } - time.Sleep(time.Duration(pollInterval) * time.Millisecond) - } - if mergeable != nil && !*mergeable || strings.EqualFold(mergeableState, "dirty") { - LogWarning(fmt.Sprintf("PR #%d is not mergeable (state=%s). Likely merge conflicts. Leaving PR open for manual resolution.", pr.GetNumber(), mergeableState)) - return fmt.Errorf("pull request #%d has merge conflicts (state=%s)", pr.GetNumber(), mergeableState) } - if err = mergePR(ctx, client, key.RepoName, pr.GetNumber()); err != nil { - return fmt.Errorf("merge PR: %w", err) - } - if err = deleteBranchIfExists(ctx, client, key.RepoName, &github.Reference{Ref: github.String("refs/heads/" + tempBranch)}); err != nil { - // Log but don't fail - branch cleanup is not critical - LogWarning(fmt.Sprintf("Failed to delete temp branch after merge: %v", err)) - } - } else { - LogInfo(fmt.Sprintf("PR created and awaiting review: #%d", pr.GetNumber())) + time.Sleep(time.Duration(pollIntervalMs) * time.Millisecond) } - return nil + return } -// addFilesToBranch builds a tree, creates a commit, and updates the ref (direct to target branch) -func addFilesToBranch(ctx context.Context, client *github.Client, key UploadKey, +// addFilesToBranch builds a tree, creates a commit, and updates the ref (direct to target branch). +// If the resulting tree is identical to the current HEAD tree, the commit is skipped to avoid +// empty commits (e.g., when a duplicate webhook creates changes already at HEAD). +func addFilesToBranch(ctx context.Context, config *configs.Config, client *github.Client, key types.UploadKey, files []github.RepositoryContent, message string) error { entries := make(map[string]string, len(files)) for _, f := range files { - content, _ := f.GetContent() + content, err := f.GetContent() + if err != nil { + return fmt.Errorf("decode content for %s: %w", f.GetName(), err) + } entries[f.GetName()] = content } - treeSHA, baseSHA, err := createCommitTree(ctx, client, key, entries) + tr, err := createCommitTree(ctx, config, client, key, entries) if err != nil { - LogCritical(fmt.Sprintf("Error creating commit tree: %v\n", err)) + LogCritical("Error creating commit tree", "error", err) return err } - if err := createCommit(ctx, client, key, baseSHA, treeSHA, message); err != nil { - LogCritical(fmt.Sprintf("Error creating commit: %v\n", err)) + + if tr.TreeSHA == tr.BaseTreeSHA { + LogInfo("Skipping empty commit โ€” new tree is identical to HEAD tree", + "repo", key.RepoName, + "branch", key.BranchPath, + "tree_sha", tr.TreeSHA, + ) + return nil + } + + if err := createCommit(ctx, client, config.ConfigRepoOwner, key, tr.BaseSHA, tr.TreeSHA, message); err != nil { + LogCritical("Error creating commit", "error", err) return err } return nil } // createBranch creates a new branch from the specified base branch (defaults to 'main') and deletes it first if it already exists. -func createBranch(ctx context.Context, client *github.Client, repo, newBranch string, baseBranch ...string) (*github.Reference, error) { +func createBranch(ctx context.Context, client *github.Client, defaultOwner, repo, newBranch string, baseBranch ...string) (*github.Reference, error) { // Normalize repo name for consistent logging and operations - normalizedRepo := normalizeRepoName(repo) - owner, repoName := parseRepoPath(normalizedRepo) + normalizedRepo := normalizeRepoName(repo, defaultOwner) + owner, repoName := parseRepoPath(normalizedRepo, defaultOwner) // Use provided base branch or default to "main" base := "main" @@ -284,75 +401,68 @@ func createBranch(ctx context.Context, client *github.Client, repo, newBranch st baseRef, _, err := client.Git.GetRef(ctx, owner, repoName, "refs/heads/"+base) if err != nil { - LogCritical(fmt.Sprintf("Failed to get '%s' baseRef: %s", base, err)) + LogCritical("Failed to get baseRef", "base", base, "error", err) return nil, err } - // *** Check if branch (newBranchRef) already exists and delete it *** - newBranchRef, _, _ := client.Git.GetRef(ctx, owner, repoName, fmt.Sprintf("%s%s", "refs/heads/", newBranch)) - if err := deleteBranchIfExists(ctx, client, normalizedRepo, newBranchRef); err != nil { + // Check if branch already exists and delete it (404 is expected when it doesn't exist) + newBranchRef, _, _ := client.Git.GetRef(ctx, owner, repoName, fmt.Sprintf("%s%s", "refs/heads/", newBranch)) //nolint:errcheck // 404 expected + if err := deleteBranchIfExists(ctx, client, defaultOwner, normalizedRepo, newBranchRef); err != nil { return nil, fmt.Errorf("failed to delete existing branch %s: %w", newBranch, err) } - newRef := &github.Reference{ - Ref: github.String(fmt.Sprintf("%s%s", "refs/heads/", newBranch)), - Object: &github.GitObject{ - SHA: baseRef.Object.SHA, - }, + createRef := github.CreateRef{ + Ref: fmt.Sprintf("refs/heads/%s", newBranch), + SHA: baseRef.Object.GetSHA(), } - newBranchRef, _, err = client.Git.CreateRef(ctx, owner, repoName, newRef) + newBranchRef, _, err = client.Git.CreateRef(ctx, owner, repoName, createRef) if err != nil { - LogCritical(fmt.Sprintf("Failed to create newBranchRef %s: %s", newRef, err)) + LogCritical("Failed to create newBranchRef", "ref", createRef.Ref, "error", err) return nil, err } - LogInfo(fmt.Sprintf("Branch created successfully: %s on %s (from %s)", newRef, normalizedRepo, base)) + LogInfo("Branch created successfully", "ref", createRef.Ref, "repo", normalizedRepo, "base", base) return newBranchRef, nil } +// treeResult holds the output of createCommitTree so callers can detect no-op trees. +type treeResult struct { + TreeSHA string // SHA of the newly created tree + BaseSHA string // SHA of the base commit (parent for the new commit) + BaseTreeSHA string // SHA of the base commit's tree โ€” if equal to TreeSHA, nothing changed +} + // createCommitTree looks up the branch ref once, then builds a tree on top of that base commit. -func createCommitTree(ctx context.Context, client *github.Client, targetBranch UploadKey, - files map[string]string) (treeSHA string, baseSHA string, err error) { +func createCommitTree(ctx context.Context, config *configs.Config, client *github.Client, targetBranch types.UploadKey, + files map[string]string) (treeResult, error) { + defaultOwner := config.ConfigRepoOwner // Normalize repo name for consistent logging - normalizedRepo := normalizeRepoName(targetBranch.RepoName) - owner, repoName := parseRepoPath(normalizedRepo) - LogInfo(fmt.Sprintf("DEBUG createCommitTree: targetBranch.RepoName=%q, normalized=%q, parsed owner=%q, repoName=%q", - targetBranch.RepoName, normalizedRepo, owner, repoName)) + normalizedRepo := normalizeRepoName(targetBranch.RepoName, defaultOwner) + owner, repoName := parseRepoPath(normalizedRepo, defaultOwner) + LogInfo("DEBUG createCommitTree", "target_repo_name", targetBranch.RepoName, "normalized", normalizedRepo, "owner", owner, "repo_name", repoName) // 1) Get current ref with retry logic to handle GitHub API eventual consistency // When a branch is just created, it may take a moment to be visible var ref *github.Reference + var err error - // Get retry configuration from environment or use defaults - cfg := configs.NewConfig() - maxRetries := cfg.GitHubAPIMaxRetries - if envRetries := os.Getenv(configs.GitHubAPIMaxRetries); envRetries != "" { - if parsed, err := parseIntWithDefault(envRetries, maxRetries); err == nil { - maxRetries = parsed - } - } - - initialRetryDelay := cfg.GitHubAPIInitialRetryDelay - if envDelay := os.Getenv(configs.GitHubAPIInitialRetryDelay); envDelay != "" { - if parsed, err := parseIntWithDefault(envDelay, initialRetryDelay); err == nil { - initialRetryDelay = parsed - } - } + maxRetries := config.GitHubAPIMaxRetries + retryDelay := time.Duration(config.GitHubAPIInitialRetryDelay) * time.Millisecond - retryDelay := time.Duration(initialRetryDelay) * time.Millisecond + // GetRef expects "heads/main" format (no "refs/" prefix) + refPath := normalizeRefPath(targetBranch.BranchPath, false) for attempt := 1; attempt <= maxRetries; attempt++ { - ref, _, err = client.Git.GetRef(ctx, owner, repoName, targetBranch.BranchPath) + ref, _, err = client.Git.GetRef(ctx, owner, repoName, refPath) if err == nil && ref != nil { break // Success } if attempt < maxRetries { - LogWarning(fmt.Sprintf("Failed to get ref for %s (attempt %d/%d): %v. Retrying in %v...", - normalizedRepo, attempt, maxRetries, err, retryDelay)) + LogWarning("Failed to get ref; retrying", "repo", normalizedRepo, "attempt", attempt, "max_retries", maxRetries, "error", err, "retry_delay", retryDelay) time.Sleep(retryDelay) retryDelay *= 2 // Exponential backoff } @@ -360,60 +470,74 @@ func createCommitTree(ctx context.Context, client *github.Client, targetBranch U if err != nil || ref == nil { if err == nil { - err = errors.Errorf("targetRef is nil after %d attempts", maxRetries) + err = fmt.Errorf("targetRef is nil after %d attempts", maxRetries) } - LogCritical(fmt.Sprintf("Failed to get ref for %s after %d attempts: %v\n", normalizedRepo, maxRetries, err)) - return "", "", err + LogCritical("Failed to get ref after max attempts", "repo", normalizedRepo, "attempts", maxRetries, "error", err) + return treeResult{}, err } - baseSHA = ref.GetObject().GetSHA() + baseSHA := ref.GetObject().GetSHA() + + // 1b) Fetch the base commit to get its tree SHA (for no-op detection) + baseCommit, _, err := client.Git.GetCommit(ctx, owner, repoName, baseSHA) + if err != nil { + return treeResult{}, fmt.Errorf("failed to get base commit %s: %w", baseSHA, err) + } + baseTreeSHA := baseCommit.GetTree().GetSHA() // 2) Build tree entries var treeEntries []*github.TreeEntry for path, content := range files { treeEntries = append(treeEntries, &github.TreeEntry{ - Path: github.String(path), - Type: github.String("blob"), - Mode: github.String("100644"), - Content: github.String(content), + Path: github.Ptr(path), + Type: github.Ptr("blob"), + Mode: github.Ptr("100644"), + Content: github.Ptr(content), }) } // 3) Create tree on top of baseSHA tree, _, err := client.Git.CreateTree(ctx, owner, repoName, baseSHA, treeEntries) if err != nil { - return "", "", fmt.Errorf("failed to create tree: %w", err) + return treeResult{}, fmt.Errorf("failed to create tree: %w", err) } - return tree.GetSHA(), baseSHA, nil + return treeResult{ + TreeSHA: tree.GetSHA(), + BaseSHA: baseSHA, + BaseTreeSHA: baseTreeSHA, + }, nil } // createCommit makes the commit using the provided baseSHA, and updates the branch ref to the new commit. -func createCommit(ctx context.Context, client *github.Client, targetBranch UploadKey, +func createCommit(ctx context.Context, client *github.Client, defaultOwner string, targetBranch types.UploadKey, baseSHA string, treeSHA string, message string) error { - owner, repoName := parseRepoPath(targetBranch.RepoName) + owner, repoName := parseRepoPath(targetBranch.RepoName, defaultOwner) - parent := &github.Commit{SHA: github.String(baseSHA)} - commit := &github.Commit{ - Message: github.String(message), - Tree: &github.Tree{SHA: github.String(treeSHA)}, + parent := &github.Commit{SHA: github.Ptr(baseSHA)} + commit := github.Commit{ + Message: github.Ptr(message), + Tree: &github.Tree{SHA: github.Ptr(treeSHA)}, Parents: []*github.Commit{parent}, } - newCommit, _, err := client.Git.CreateCommit(ctx, owner, repoName, commit) + newCommit, _, err := client.Git.CreateCommit(ctx, owner, repoName, commit, nil) if err != nil { return fmt.Errorf("could not create commit: %w", err) } // Update branch ref directly (no second GET) - ref := &github.Reference{ - Ref: github.String(targetBranch.BranchPath), // e.g., "refs/heads/main" - Object: &github.GitObject{SHA: github.String(newCommit.GetSHA())}, - } - if _, _, err := client.Git.UpdateRef(ctx, owner, repoName, ref, false); err != nil { + // UpdateRef expects ref path like "heads/main" (without "refs/" prefix) + fullRefPath := normalizeRefPath(targetBranch.BranchPath, true) + refPath := strings.TrimPrefix(fullRefPath, "refs/") + updateRef := github.UpdateRef{ + SHA: newCommit.GetSHA(), + Force: github.Ptr(false), + } + if _, _, err := client.Git.UpdateRef(ctx, owner, repoName, refPath, updateRef); err != nil { // Detect non-fast-forward / conflict scenarios and provide a clearer error if eresp, ok := err.(*github.ErrorResponse); ok { if eresp.Response != nil && eresp.Response.StatusCode == http.StatusUnprocessableEntity { - return fmt.Errorf("failed to update ref: non-fast-forward (possible conflict). Consider using PR strategy: %w", err) + return fmt.Errorf("%w: failed to update ref: non-fast-forward. Consider using PR strategy: %v", ErrMergeConflict, err) } } return fmt.Errorf("failed to update ref to new commit: %w", err) @@ -422,50 +546,50 @@ func createCommit(ctx context.Context, client *github.Client, targetBranch Uploa } // mergePR merges the specified pull request in the given repository. -func mergePR(ctx context.Context, client *github.Client, repo string, pr_number int) error { - owner, repoName := parseRepoPath(repo) +func mergePR(ctx context.Context, client *github.Client, defaultOwner, repo string, pr_number int) error { + owner, repoName := parseRepoPath(repo, defaultOwner) options := &github.PullRequestOptions{ MergeMethod: "merge", // Other options: "squash" or "rebase" } result, _, err := client.PullRequests.Merge(ctx, owner, repoName, pr_number, "Merging the pull request", options) if err != nil { - LogCritical(fmt.Sprintf("Failed to merge PR: %v\n", err)) + LogCritical("Failed to merge PR", "error", err) return err } if result.GetMerged() { - LogInfo(fmt.Sprintf("Successfully merged PR #%d\n", pr_number)) + LogInfo("Successfully merged PR", "pr_number", pr_number) return nil } else { - LogError(fmt.Sprintf("Failed to merge PR #%d: %s", pr_number, result.GetMessage())) + LogError("Failed to merge PR", "pr_number", pr_number, "message", result.GetMessage()) return fmt.Errorf("failed to merge PR #%d: %s", pr_number, result.GetMessage()) } } // deleteBranchIfExists deletes the specified branch if it exists, except for 'main'. // Returns an error if attempting to delete the main branch or if deletion fails. -func deleteBranchIfExists(backgroundContext context.Context, client *github.Client, repo string, ref *github.Reference) error { +func deleteBranchIfExists(backgroundContext context.Context, client *github.Client, defaultOwner, repo string, ref *github.Reference) error { // Early return if ref is nil (branch doesn't exist) if ref == nil { return nil } // Normalize repo name for consistent logging - normalizedRepo := normalizeRepoName(repo) - owner, repoName := parseRepoPath(normalizedRepo) + normalizedRepo := normalizeRepoName(repo, defaultOwner) + owner, repoName := parseRepoPath(normalizedRepo, defaultOwner) if ref.GetRef() == "refs/heads/main" { LogError("I refuse to delete branch 'main'.") return fmt.Errorf("refusing to delete protected branch 'main'") } - LogInfo(fmt.Sprintf("Deleting branch %s on %s", ref.GetRef(), normalizedRepo)) + LogInfo("Deleting branch", "ref", ref.GetRef(), "repo", normalizedRepo) _, _, err := client.Git.GetRef(backgroundContext, owner, repoName, ref.GetRef()) if err == nil { // Branch exists (there was no error fetching it) _, err = client.Git.DeleteRef(backgroundContext, owner, repoName, ref.GetRef()) if err != nil { - LogCritical(fmt.Sprintf("Error deleting branch: %v\n", err)) + LogCritical("Error deleting branch", "error", err) return fmt.Errorf("failed to delete branch %s: %w", ref.GetRef(), err) } } @@ -473,15 +597,6 @@ func deleteBranchIfExists(backgroundContext context.Context, client *github.Clie } // DeleteBranchIfExistsExported is an exported wrapper for testing deleteBranchIfExists -func DeleteBranchIfExistsExported(ctx context.Context, client *github.Client, repo string, ref *github.Reference) error { - return deleteBranchIfExists(ctx, client, repo, ref) -} - -// parseIntWithDefault parses a string to int, returning defaultValue on error -func parseIntWithDefault(s string, defaultValue int) (int, error) { - var result int - if _, err := fmt.Sscanf(s, "%d", &result); err != nil { - return defaultValue, err - } - return result, nil +func DeleteBranchIfExistsExported(ctx context.Context, client *github.Client, defaultOwner, repo string, ref *github.Reference) error { + return deleteBranchIfExists(ctx, client, defaultOwner, repo, ref) } diff --git a/services/github_write_to_target_test.go b/services/github_write_to_target_test.go index a645836..05e1383 100644 --- a/services/github_write_to_target_test.go +++ b/services/github_write_to_target_test.go @@ -14,248 +14,80 @@ import ( "strings" "testing" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/grove-platform/github-copier/configs" "github.com/grove-platform/github-copier/services" "github.com/grove-platform/github-copier/types" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/require" - // test helpers (utils.go) test "github.com/grove-platform/github-copier/tests" ) func TestMain(m *testing.M) { - // Minimal env so init() and any env readers are happy. - os.Setenv(configs.ConfigRepoOwner, "my-org") - os.Setenv(configs.ConfigRepoName, "config-repo") - os.Setenv(configs.InstallationId, "12345") - os.Setenv(configs.AppId, "1166559") - os.Setenv(configs.AppClientId, "IvTestClientId") - os.Setenv("SKIP_SECRET_MANAGER", "true") - os.Setenv(configs.ConfigRepoBranch, "main") - - // Provide an RSA private key (both raw and b64) so ConfigurePermissions can parse. + _ = os.Setenv(configs.ConfigRepoOwner, "my-org") + _ = os.Setenv(configs.ConfigRepoName, "config-repo") + _ = os.Setenv(configs.InstallationId, "12345") + _ = os.Setenv(configs.AppId, "1166559") + _ = os.Setenv(configs.AppClientId, "IvTestClientId") + _ = os.Setenv("SKIP_SECRET_MANAGER", "true") + _ = os.Setenv(configs.ConfigRepoBranch, "main") + key, _ := rsa.GenerateKey(rand.Reader, 1024) der := x509.MarshalPKCS1PrivateKey(key) pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der}) - os.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) - os.Setenv("GITHUB_APP_PRIVATE_KEY_B64", base64.StdEncoding.EncodeToString(pemBytes)) + _ = os.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) + _ = os.Setenv("GITHUB_APP_PRIVATE_KEY_B64", base64.StdEncoding.EncodeToString(pemBytes)) code := m.Run() - // Cleanup - os.Unsetenv(configs.ConfigRepoOwner) - os.Unsetenv(configs.ConfigRepoName) - os.Unsetenv(configs.InstallationId) - os.Unsetenv(configs.AppId) - os.Unsetenv(configs.AppClientId) - os.Unsetenv("SKIP_SECRET_MANAGER") - os.Unsetenv("SRC_BRANCH") - os.Unsetenv("GITHUB_APP_PRIVATE_KEY") - os.Unsetenv("GITHUB_APP_PRIVATE_KEY_B64") + _ = os.Unsetenv(configs.ConfigRepoOwner) + _ = os.Unsetenv(configs.ConfigRepoName) + _ = os.Unsetenv(configs.InstallationId) + _ = os.Unsetenv(configs.AppId) + _ = os.Unsetenv(configs.AppClientId) + _ = os.Unsetenv("SKIP_SECRET_MANAGER") + _ = os.Unsetenv("SRC_BRANCH") + _ = os.Unsetenv("GITHUB_APP_PRIVATE_KEY") + _ = os.Unsetenv("GITHUB_APP_PRIVATE_KEY_B64") os.Exit(code) } -// LEGACY TESTS - These tests are for legacy code that was removed in commit a64726c -// The AddToRepoAndFilesMap and IterateFilesForCopy functions were removed as part of the -// migration to the new pattern-matching system. These tests are commented out but kept for reference. -// -// The new system uses pattern matching rules defined in YAML config files. -// See pattern_matcher_test.go for tests of the new system. - -/* -func TestAddToRepoAndFilesMap_NewEntry(t *testing.T) { - services.FilesToUpload = nil - - name := "example.txt" - dummyFile := github.RepositoryContent{Name: &name} - - services.AddToRepoAndFilesMap("TargetRepo1", "main", dummyFile) - - require.NotNil(t, services.FilesToUpload, "FilesToUpload map should be initialized") - key := types.UploadKey{RepoName: "TargetRepo1", BranchPath: "refs/heads/main", RuleName: "", CommitStrategy: ""} - entry, exists := services.FilesToUpload[key] - require.True(t, exists, "Entry for TargetRepo1/main should exist") - require.Equal(t, "main", entry.TargetBranch) - require.Len(t, entry.Content, 1) - require.Equal(t, "example.txt", *entry.Content[0].Name) -} - -func TestAddToRepoAndFilesMap_AppendEntry(t *testing.T) { - services.FilesToUpload = make(map[types.UploadKey]types.UploadFileContent) - key := types.UploadKey{RepoName: "TargetRepo1", BranchPath: "refs/heads/main", RuleName: "", CommitStrategy: ""} - - initialName := "first.txt" - services.FilesToUpload[key] = types.UploadFileContent{ - TargetBranch: "main", - Content: []github.RepositoryContent{{Name: &initialName}}, - } - - newName := "second.txt" - newFile := github.RepositoryContent{Name: &newName} - services.AddToRepoAndFilesMap("TargetRepo1", "main", newFile) - - entry := services.FilesToUpload[key] - require.Len(t, entry.Content, 2) - require.ElementsMatch(t, []string{"first.txt", "second.txt"}, - []string{*entry.Content[0].Name, *entry.Content[1].Name}) -} - -func TestAddToRepoAndFilesMap_NestedFiles(t *testing.T) { - services.FilesToUpload = make(map[types.UploadKey]types.UploadFileContent) - key := types.UploadKey{RepoName: "TargetRepo1", BranchPath: "refs/heads/main", RuleName: "", CommitStrategy: ""} - - initialName := "level1/first.txt" - services.FilesToUpload[key] = types.UploadFileContent{ - TargetBranch: "main", - Content: []github.RepositoryContent{{Name: &initialName}}, - } - - newName := "level1/level2/level3/nested-second.txt" - newFile := github.RepositoryContent{Name: &newName} - services.AddToRepoAndFilesMap("TargetRepo1", "main", newFile) - - entry := services.FilesToUpload[key] - require.Len(t, entry.Content, 2) - require.ElementsMatch(t, []string{"level1/first.txt", "level1/level2/level3/nested-second.txt"}, - []string{*entry.Content[0].Name, *entry.Content[1].Name}) -} - -func TestIterateFilesForCopy_Deletes(t *testing.T) { - cfg := types.Configs{ - SourceDirectory: "src/examples", - TargetRepo: "TargetRepo1", - TargetBranch: "main", - TargetDirectory: "dest/examples", - RecursiveCopy: false, - } - configFile := types.ConfigFileType{cfg} - changed := []types.ChangedFile{{ - Path: "src/examples/sample.txt", - Status: "DELETED", - }} - - services.FilesToUpload = nil - services.FilesToDeprecate = nil - - err := services.IterateFilesForCopy(changed, configFile) - require.NoError(t, err) - - targetPath := "dest/examples/sample.txt" - require.Contains(t, services.FilesToDeprecate, targetPath) - require.Equal(t, cfg, services.FilesToDeprecate[targetPath]) - require.Nil(t, services.FilesToUpload) -} - -func TestIterateFilesForCopy_RecursiveVsNonRecursive(t *testing.T) { - t.Setenv("SRC_BRANCH", "main") - _ = test.WithHTTPMock(t) - - owner, repo := test.EnvOwnerRepo(t) - - // Simulate changes under the source directory - changed := []types.ChangedFile{ - test.MakeChanged("ADDED", "examples/a.txt"), - test.MakeChanged("MODIFIED", "examples/sub/b.txt"), - test.MakeChanged("ADDED", "examples/sub/deeper/c.txt"), - } - - // Helper to base64-encode small content blobs - b64 := func(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) } - - // Register responders for owner/repo - for _, or := range [][2]string{{owner, repo}, {"REPO_OWNER", "REPO_NAME"}} { - test.MockContentsEndpoint(or[0], or[1], "examples/a.txt", b64("A")) - test.MockContentsEndpoint(or[0], or[1], "examples/sub/b.txt", b64("B")) - test.MockContentsEndpoint(or[0], or[1], "examples/sub/deeper/c.txt", b64("C")) - } - - // Same source; two configs exercising recursive vs non-recursive and different targets - cases := []struct { - name string - cfg types.Configs - expect []string // expected TARGET paths - }{ - { - name: "recursive=true copies all depths", - cfg: types.Configs{ - SourceDirectory: "examples", - TargetRepo: "TargetRepoR", - TargetBranch: "main", - TargetDirectory: "dest", - RecursiveCopy: true, - }, - expect: []string{ - "dest/a.txt", - "dest/sub/b.txt", - "dest/sub/deeper/c.txt", - }, - }, - { - name: "recursive=false copies only root files", - cfg: types.Configs{ - SourceDirectory: "examples", - TargetRepo: "TargetRepoNR", - TargetBranch: "main", - TargetDirectory: "dest", - RecursiveCopy: false, - }, - expect: []string{ - "dest/a.txt", - }, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - test.ResetGlobals() - err := services.IterateFilesForCopy(changed, types.ConfigFileType{tc.cfg}) - require.NoError(t, err) - // Compares staged entries cfg.SourceDirectory -> cfg.TargetDirectory. - test.AssertUploadedPathsFromConfig(t, tc.cfg, tc.expect) - }) - } -} -*/ - -func TestAddFilesToTargetRepoBranch_Succeeds(t *testing.T) { +func TestAddFilesToTargetRepos_Direct_Succeeds(t *testing.T) { _ = test.WithHTTPMock(t) owner, repo := test.EnvOwnerRepo(t) branch := "main" - // Set up cached token for the org to bypass GitHub App auth test.SetupOrgToken(owner, "test-token") baseRefURL, commitsURL, updateRefURL := test.MockGitHubWriteEndpoints(owner, repo, branch) files := []github.RepositoryContent{ { - Name: github.String("dir/example1.txt"), - Path: github.String("dir/example1.txt"), - Content: github.String(base64.StdEncoding.EncodeToString([]byte("hello 1"))), + Name: github.Ptr("dir/example1.txt"), + Path: github.Ptr("dir/example1.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("hello 1"))), }, { - Name: github.String("dir/example2.txt"), - Path: github.String("dir/example2.txt"), - Content: github.String(base64.StdEncoding.EncodeToString([]byte("hello 2"))), + Name: github.Ptr("dir/example2.txt"), + Path: github.Ptr("dir/example2.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("hello 2"))), }, } - services.FilesToUpload = map[types.UploadKey]types.UploadFileContent{ + filesToUpload := map[types.UploadKey]types.UploadFileContent{ {RepoName: repo, BranchPath: "refs/heads/" + branch}: { TargetBranch: branch, Content: files, }, } - services.AddFilesToTargetRepoBranch() + services.AddFilesToTargetRepos(context.Background(), test.TestConfig(), filesToUpload, nil, nil) info := httpmock.GetCallCountInfo() require.Equal(t, 1, info["GET "+baseRefURL]) - // POST /git/trees is registered via regex; sum by prefix treeCalls := 0 for k, v := range info { if strings.HasPrefix(k, "POST https://api.github.com/repos/"+owner+"/"+repo+"/git/trees") { @@ -263,29 +95,30 @@ func TestAddFilesToTargetRepoBranch_Succeeds(t *testing.T) { } } require.Equal(t, 1, treeCalls) - require.Equal(t, 1, info["POST "+commitsURL]) require.Equal(t, 1, info["PATCH "+updateRefURL]) - - services.FilesToUpload = nil } -func TestAddFilesToTargetRepoBranch_ViaPR_Succeeds(t *testing.T) { +func TestAddFilesToTargetRepos_ViaPR_Succeeds(t *testing.T) { _ = test.WithHTTPMock(t) t.Setenv("COPIER_COMMIT_STRATEGY", "pr") owner, repo := test.EnvOwnerRepo(t) baseBranch := "main" - // Force fresh token; stub token endpoint then configure permissions. - services.InstallationAccessToken = "" - test.MockGitHubAppTokenEndpoint(os.Getenv(configs.InstallationId)) - err := services.ConfigurePermissions() + // Force fresh token + services.DefaultTokenManager().SetInstallationAccessToken("") + cfg := test.TestConfig() + test.MockGitHubAppTokenEndpoint(cfg.InstallationId) + err := services.ConfigurePermissions(context.Background(), cfg) require.NoError(t, err, "ConfigurePermissions should succeed") - // Set up cached token for the org to bypass GitHub App auth test.SetupOrgToken(owner, "test-token") + // No existing open PRs + test.MockListOpenPRs(owner, repo, nil) + test.MockGetCommit(owner, repo, "baseSha", "oldTreeSha") + // Base ref used to create temp branch httpmock.RegisterRegexpResponder("GET", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/ref/(?:refs/)?heads/`+baseBranch+`$`), @@ -294,10 +127,8 @@ func TestAddFilesToTargetRepoBranch_ViaPR_Succeeds(t *testing.T) { }), ) - // Create temp branch createRefURL := test.MockCreateRef(owner, repo) - // Temp branch: GET ref, POST tree, POST commit, PATCH ref tempHead := `copier/\d{8}-\d{6}` httpmock.RegisterRegexpResponder("GET", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/ref/(?:refs/)?heads/`+tempHead+`$`), @@ -318,24 +149,22 @@ func TestAddFilesToTargetRepoBranch_ViaPR_Succeeds(t *testing.T) { httpmock.NewStringResponder(200, "{}"), ) - // PR create + merge; delete temp branch test.MockPullsAndMerge(owner, repo, 42) test.MockDeleteTempRef(owner, repo) - // Stage files to baseBranch; service will write via temp branch โ†’ PR merge files := []github.RepositoryContent{ { - Name: github.String("dir/example1.txt"), - Path: github.String("dir/example1.txt"), - Content: github.String(base64.StdEncoding.EncodeToString([]byte("hello 1"))), + Name: github.Ptr("dir/example1.txt"), + Path: github.Ptr("dir/example1.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("hello 1"))), }, { - Name: github.String("dir/example2.txt"), - Path: github.String("dir/example2.txt"), - Content: github.String(base64.StdEncoding.EncodeToString([]byte("hello 2"))), + Name: github.Ptr("dir/example2.txt"), + Path: github.Ptr("dir/example2.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("hello 2"))), }, } - services.FilesToUpload = map[types.UploadKey]types.UploadFileContent{ + filesToUpload := map[types.UploadKey]types.UploadFileContent{ {RepoName: repo, BranchPath: "refs/heads/" + baseBranch}: { TargetBranch: baseBranch, Content: files, @@ -344,11 +173,10 @@ func TestAddFilesToTargetRepoBranch_ViaPR_Succeeds(t *testing.T) { }, } - services.AddFilesToTargetRepoBranch() + services.AddFilesToTargetRepos(context.Background(), cfg, filesToUpload, nil, nil) - // Assertions require.Equal(t, 1, test.CountByMethodAndURLRegexp("POST", - regexp.MustCompile(`/app/installations/`+regexp.QuoteMeta(os.Getenv(configs.InstallationId))+`/access_tokens$`), + regexp.MustCompile(`/app/installations/`+regexp.QuoteMeta(cfg.InstallationId)+`/access_tokens$`), )) info := httpmock.GetCallCountInfo() require.Equal(t, 1, info["POST "+createRefURL]) @@ -386,11 +214,46 @@ func TestAddFilesToTargetRepoBranch_ViaPR_Succeeds(t *testing.T) { regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/git/refs/heads/copier/\d{8}-\d{6}$`)), 1, ) - - services.FilesToUpload = nil } -// --- Added critical tests for merge conflicts and configuration/default priorities --- +// TestAddFilesToTargetRepos_Direct_SkipsEmptyCommit verifies that when the new +// tree SHA equals the base commit's tree SHA (i.e. all files already at HEAD), +// no commit or ref update is created. +func TestAddFilesToTargetRepos_Direct_SkipsEmptyCommit(t *testing.T) { + _ = test.WithHTTPMock(t) + + owner, repo := test.EnvOwnerRepo(t) + branch := "main" + + test.SetupOrgToken(owner, "test-token") + + // Use NoOp endpoints: new tree SHA == base tree SHA + baseRefURL, commitsURL, updateRefURL := test.MockGitHubWriteEndpointsNoOp(owner, repo, branch) + + files := []github.RepositoryContent{ + { + Name: github.Ptr("dir/example1.txt"), + Path: github.Ptr("dir/example1.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("hello 1"))), + }, + } + filesToUpload := map[types.UploadKey]types.UploadFileContent{ + {RepoName: repo, BranchPath: "refs/heads/" + branch}: { + TargetBranch: branch, + Content: files, + }, + } + + services.AddFilesToTargetRepos(context.Background(), test.TestConfig(), filesToUpload, nil, nil) + + info := httpmock.GetCallCountInfo() + // Should still fetch the ref and create the tree + require.Equal(t, 1, info["GET "+baseRefURL], "should GET base ref") + + // Should NOT create a commit or update the ref + require.Equal(t, 0, info["POST "+commitsURL], "should skip commit creation") + require.Equal(t, 0, info["PATCH "+updateRefURL], "should skip ref update") +} func TestAddFiles_DirectConflict_NonFastForward(t *testing.T) { _ = test.WithHTTPMock(t) @@ -398,10 +261,8 @@ func TestAddFiles_DirectConflict_NonFastForward(t *testing.T) { owner, repo := test.EnvOwnerRepo(t) branch := "main" - // Set up cached token for the org to bypass GitHub App auth test.SetupOrgToken(owner, "test-token") - // Mock standard direct write endpoints baseRefURL, commitsURL, updateRefURL := test.MockGitHubWriteEndpoints(owner, repo, branch) // Override UpdateRef to simulate 422 Unprocessable Entity (non-fast-forward) @@ -411,27 +272,24 @@ func TestAddFiles_DirectConflict_NonFastForward(t *testing.T) { files := []github.RepositoryContent{ { - Name: github.String("dir/example1.txt"), - Path: github.String("dir/example1.txt"), - Content: github.String(base64.StdEncoding.EncodeToString([]byte("hello 1"))), + Name: github.Ptr("dir/example1.txt"), + Path: github.Ptr("dir/example1.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("hello 1"))), }, } - services.FilesToUpload = map[types.UploadKey]types.UploadFileContent{ + filesToUpload := map[types.UploadKey]types.UploadFileContent{ {RepoName: repo, BranchPath: "refs/heads/" + branch}: { TargetBranch: branch, Content: files, }, } - // Run โ€“ should not panic; error is handled/logged internally. - services.AddFilesToTargetRepoBranch() + services.AddFilesToTargetRepos(context.Background(), test.TestConfig(), filesToUpload, nil, nil) info := httpmock.GetCallCountInfo() require.Equal(t, 1, info["GET "+baseRefURL]) require.Equal(t, 1, info["POST "+commitsURL]) require.Equal(t, 1, info["PATCH "+updateRefURL]) - - services.FilesToUpload = nil } func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { @@ -441,16 +299,16 @@ func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { owner, repo := test.EnvOwnerRepo(t) baseBranch := "main" - // Fresh token path - services.InstallationAccessToken = "" - test.MockGitHubAppTokenEndpoint(os.Getenv(configs.InstallationId)) - err := services.ConfigurePermissions() + cfg := test.TestConfig() + services.DefaultTokenManager().SetInstallationAccessToken("") + test.MockGitHubAppTokenEndpoint(cfg.InstallationId) + err := services.ConfigurePermissions(context.Background(), cfg) require.NoError(t, err, "ConfigurePermissions should succeed") - // Set up cached token for the org to bypass GitHub App auth test.SetupOrgToken(owner, "test-token") + test.MockListOpenPRs(owner, repo, nil) + test.MockGetCommit(owner, repo, "baseSha", "oldTreeSha") - // Base ref for creating temp branch httpmock.RegisterRegexpResponder("GET", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/ref/(?:refs/)?heads/`+baseBranch+`$`), httpmock.NewJsonResponderOrPanic(200, map[string]any{ @@ -459,7 +317,6 @@ func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { ) createRefURL := test.MockCreateRef(owner, repo) - // Temp branch interactions tempHead := `copier/\d{8}-\d{6}` httpmock.RegisterRegexpResponder("GET", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/ref/(?:refs/)?heads/`+tempHead+`$`), @@ -467,7 +324,6 @@ func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { "ref": "refs/heads/copier/20250101-000000", "object": map[string]any{"sha": "baseSha"}, }), ) - // Mock DELETE for existing temp branch cleanup httpmock.RegisterRegexpResponder("DELETE", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/refs/heads/`+tempHead+`$`), httpmock.NewStringResponder(204, ""), @@ -485,27 +341,22 @@ func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { httpmock.NewStringResponder(200, "{}"), ) - // PR create pr_number := 77 httpmock.RegisterResponder("POST", "https://api.github.com/repos/"+owner+"/"+repo+"/pulls", httpmock.NewJsonResponderOrPanic(201, map[string]any{"number": pr_number, "html_url": "https://github.com/" + owner + "/" + repo + "/pull/77"}), ) - // PR mergeability check returns dirty -> not mergeable httpmock.RegisterResponder("GET", "https://api.github.com/repos/"+owner+"/"+repo+"/pulls/77", httpmock.NewJsonResponderOrPanic(200, map[string]any{"mergeable": false, "mergeable_state": "dirty"}), ) - // Note: do NOT register PUT /merge to ensure it isn't called - // Also do NOT register DELETE for temp ref; conflict path returns early before cleanup - // Minimal file to write files := []github.RepositoryContent{{ - Name: github.String("f.txt"), - Path: github.String("f.txt"), - Content: github.String(base64.StdEncoding.EncodeToString([]byte("x"))), + Name: github.Ptr("f.txt"), + Path: github.Ptr("f.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("x"))), }} - services.FilesToUpload = map[types.UploadKey]types.UploadFileContent{ + filesToUpload := map[types.UploadKey]types.UploadFileContent{ {RepoName: repo, BranchPath: "refs/heads/" + baseBranch}: { TargetBranch: baseBranch, Content: files, @@ -513,22 +364,16 @@ func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { }, } - services.AddFilesToTargetRepoBranch() + services.AddFilesToTargetRepos(context.Background(), cfg, filesToUpload, nil, nil) - // Assertions info := httpmock.GetCallCountInfo() require.Equal(t, 1, info["POST "+createRefURL]) require.Equal(t, 1, test.CountByMethodAndURLRegexp("POST", regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/pulls$`))) - // No merge call should have been made require.Equal(t, 0, test.CountByMethodAndURLRegexp("PUT", regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/pulls/77/merge$`))) - // Only 1 DELETE call for initial cleanup of existing branch (before creating new one) - // No additional DELETE after merge conflict because we returned early require.Equal(t, 1, test.CountByMethodAndURLRegexp("DELETE", regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/git/refs/heads/copier/\d{8}-\d{6}$`))) - - services.FilesToUpload = nil } func TestPriority_Strategy_ConfigOverridesEnv_And_MessageFallbacks(t *testing.T) { @@ -537,22 +382,18 @@ func TestPriority_Strategy_ConfigOverridesEnv_And_MessageFallbacks(t *testing.T) owner, repo := test.EnvOwnerRepo(t) baseBranch := "main" - // Env specifies PR, but config will override to direct t.Setenv("COPIER_COMMIT_STRATEGY", "pr") - // Set up cached token for the org to bypass GitHub App auth test.SetupOrgToken(owner, "test-token") - // Mocks for direct flow baseRefURL, commitsURL, updateRefURL := test.MockGitHubWriteEndpoints(owner, repo, baseBranch) - // Intercept POST commit to assert commit message fallback when config empty but env default set wantMsg := "Env Default Commit Message" - t.Setenv(configs.DefaultCommitMessage, wantMsg) + testCfg := test.TestConfig() + testCfg.DefaultCommitMessage = wantMsg - // Replace commits responder with custom body assertion httpmock.RegisterResponder("POST", commitsURL, func(req *http.Request) (*http.Response, error) { - defer req.Body.Close() + defer func() { _ = req.Body.Close() }() b, _ := io.ReadAll(req.Body) if !strings.Contains(string(b), wantMsg) { t.Fatalf("commit body does not contain expected message: %s; body=%s", wantMsg, string(b)) @@ -561,32 +402,28 @@ func TestPriority_Strategy_ConfigOverridesEnv_And_MessageFallbacks(t *testing.T) }) files := []github.RepositoryContent{{ - Name: github.String("a.txt"), - Path: github.String("a.txt"), - Content: github.String(base64.StdEncoding.EncodeToString([]byte("x"))), + Name: github.Ptr("a.txt"), + Path: github.Ptr("a.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("x"))), }} - cfg := types.Configs{ + typeCfg := types.Configs{ TargetRepo: repo, TargetBranch: baseBranch, CopierCommitStrategy: "direct", // overrides env "pr" - // CommitMessage empty -> use env default } - services.FilesToUpload = map[types.UploadKey]types.UploadFileContent{ - {RepoName: repo, BranchPath: "refs/heads/" + baseBranch, CommitStrategy: cfg.CopierCommitStrategy}: {TargetBranch: baseBranch, Content: files}, + filesToUpload := map[types.UploadKey]types.UploadFileContent{ + {RepoName: repo, BranchPath: "refs/heads/" + baseBranch, CommitStrategy: typeCfg.CopierCommitStrategy}: {TargetBranch: baseBranch, Content: files}, } - services.AddFilesToTargetRepoBranch() // No longer takes parameters - uses FilesToUpload map + services.AddFilesToTargetRepos(context.Background(), testCfg, filesToUpload, nil, nil) info := httpmock.GetCallCountInfo() require.Equal(t, 1, info["GET "+baseRefURL]) require.Equal(t, 1, info["POST "+commitsURL]) require.Equal(t, 1, info["PATCH "+updateRefURL]) - // No PR endpoints should be called require.Equal(t, 0, test.CountByMethodAndURLRegexp("POST", regexp.MustCompile(`/pulls$`))) - - services.FilesToUpload = nil } func TestPriority_PRTitleDefaultsToCommitMessage_And_NoAutoMergeWhenConfigPresent(t *testing.T) { @@ -596,16 +433,16 @@ func TestPriority_PRTitleDefaultsToCommitMessage_And_NoAutoMergeWhenConfigPresen owner, repo := test.EnvOwnerRepo(t) baseBranch := "main" - // Token setup - services.InstallationAccessToken = "" - test.MockGitHubAppTokenEndpoint(os.Getenv(configs.InstallationId)) - err := services.ConfigurePermissions() + cfg := test.TestConfig() + services.DefaultTokenManager().SetInstallationAccessToken("") + test.MockGitHubAppTokenEndpoint(cfg.InstallationId) + err := services.ConfigurePermissions(context.Background(), cfg) require.NoError(t, err, "ConfigurePermissions should succeed") - // Set up cached token for the org to bypass GitHub App auth test.SetupOrgToken(owner, "test-token") + test.MockListOpenPRs(owner, repo, nil) + test.MockGetCommit(owner, repo, "baseSha", "oldTreeSha") - // Base ref and temp branch setup httpmock.RegisterRegexpResponder("GET", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/ref/(?:refs/)?heads/`+baseBranch+`$`), httpmock.NewJsonResponderOrPanic(200, map[string]any{"ref": "refs/heads/" + baseBranch, "object": map[string]any{"sha": "baseSha"}}), @@ -616,7 +453,6 @@ func TestPriority_PRTitleDefaultsToCommitMessage_And_NoAutoMergeWhenConfigPresen regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/ref/(?:refs/)?heads/`+tempHead+`$`), httpmock.NewJsonResponderOrPanic(200, map[string]any{"ref": "refs/heads/copier/20250101-000000", "object": map[string]any{"sha": "baseSha"}}), ) - // Mock DELETE for existing temp branch cleanup httpmock.RegisterRegexpResponder("DELETE", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/refs/heads/`+tempHead+`$`), httpmock.NewStringResponder(204, ""), @@ -627,7 +463,7 @@ func TestPriority_PRTitleDefaultsToCommitMessage_And_NoAutoMergeWhenConfigPresen ) commitsURL := "https://api.github.com/repos/" + owner + "/" + repo + "/git/commits" want := "Env Fallback Message" - t.Setenv(configs.DefaultCommitMessage, want) + cfg.DefaultCommitMessage = want httpmock.RegisterResponder("POST", commitsURL, func(req *http.Request) (*http.Response, error) { b, _ := io.ReadAll(req.Body) if !strings.Contains(string(b), want) { @@ -640,7 +476,6 @@ func TestPriority_PRTitleDefaultsToCommitMessage_And_NoAutoMergeWhenConfigPresen httpmock.NewStringResponder(200, "{}"), ) - // Assert PR title equals commit message when PRTitle empty httpmock.RegisterResponder("POST", "https://api.github.com/repos/"+owner+"/"+repo+"/pulls", func(req *http.Request) (*http.Response, error) { @@ -652,44 +487,197 @@ func TestPriority_PRTitleDefaultsToCommitMessage_And_NoAutoMergeWhenConfigPresen }, ) - // No merge; MergeWithoutReview=false when matching config present and not set to true - // If code attempted merge, there would be a 404 on PUT, failing the test via missing responder count. - files := []github.RepositoryContent{{ - Name: github.String("only.txt"), Path: github.String("only.txt"), - Content: github.String(base64.StdEncoding.EncodeToString([]byte("y"))), + Name: github.Ptr("only.txt"), Path: github.Ptr("only.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("y"))), }} - // cfg := types.Configs{TargetRepo: repo, TargetBranch: baseBranch /* MergeWithoutReview: false (zero value) */} - services.FilesToUpload = map[types.UploadKey]types.UploadFileContent{{RepoName: repo, BranchPath: "refs/heads/" + baseBranch, RuleName: "", CommitStrategy: "pr"}: {TargetBranch: baseBranch, Content: files, CommitStrategy: "pr"}} + filesToUpload := map[types.UploadKey]types.UploadFileContent{ + {RepoName: repo, BranchPath: "refs/heads/" + baseBranch, RuleName: "", CommitStrategy: "pr"}: {TargetBranch: baseBranch, Content: files, CommitStrategy: "pr"}, + } - services.AddFilesToTargetRepoBranch() // No longer takes parameters - uses FilesToUpload map + services.AddFilesToTargetRepos(context.Background(), cfg, filesToUpload, nil, nil) - // Ensure a PR was created but no merge occurred require.Equal(t, 1, test.CountByMethodAndURLRegexp("POST", regexp.MustCompile(`/pulls$`))) require.Equal(t, 0, test.CountByMethodAndURLRegexp("PUT", regexp.MustCompile(`/pulls/5/merge$`))) +} + +// TestAddFilesToTargetRepos_MixedStrategies_ProducesSeparateOperations verifies +// that two UploadKey entries for the same repo/branch but with different commit +// strategies (direct vs pull_request) produce independent write operations. +func TestAddFilesToTargetRepos_MixedStrategies_ProducesSeparateOperations(t *testing.T) { + _ = test.WithHTTPMock(t) - services.FilesToUpload = nil + owner, repo := test.EnvOwnerRepo(t) + baseBranch := "main" + + // Configure token / permissions + cfg := test.TestConfig() + services.DefaultTokenManager().SetInstallationAccessToken("") + test.MockGitHubAppTokenEndpoint(cfg.InstallationId) + err := services.ConfigurePermissions(context.Background(), cfg) + require.NoError(t, err, "ConfigurePermissions should succeed") + test.SetupOrgToken(owner, "test-token") + test.MockListOpenPRs(owner, repo, nil) + + // --- Mock direct-commit endpoints --- + baseRefURL, directCommitsURL, updateRefURL := test.MockGitHubWriteEndpoints(owner, repo, baseBranch) + + // --- Mock PR-strategy endpoints --- + createRefURL := test.MockCreateRef(owner, repo) + tempHead := `copier/\d{8}-\d{6}` + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/ref/(?:refs/)?heads/`+tempHead+`$`), + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "ref": "refs/heads/copier/20250101-000000", "object": map[string]any{"sha": "baseSha"}, + }), + ) + httpmock.RegisterRegexpResponder("PATCH", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/refs/heads/`+tempHead+`$`), + httpmock.NewStringResponder(200, "{}"), + ) + test.MockPullsAndMerge(owner, repo, 99) + test.MockDeleteTempRef(owner, repo) + + // --- Build two batches for the SAME repo/branch but different strategies --- + directFiles := []github.RepositoryContent{{ + Name: github.Ptr("direct-file.txt"), + Path: github.Ptr("direct-file.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("direct content"))), + }} + prFiles := []github.RepositoryContent{{ + Name: github.Ptr("pr-file.txt"), + Path: github.Ptr("pr-file.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("pr content"))), + }} + + filesToUpload := map[types.UploadKey]types.UploadFileContent{ + {RepoName: repo, BranchPath: "refs/heads/" + baseBranch, CommitStrategy: "direct"}: { + TargetBranch: baseBranch, + Content: directFiles, + CommitStrategy: "direct", + }, + {RepoName: repo, BranchPath: "refs/heads/" + baseBranch, CommitStrategy: "pull_request"}: { + TargetBranch: baseBranch, + Content: prFiles, + CommitStrategy: "pr", + AutoMergePR: true, + }, + } + + services.AddFilesToTargetRepos(context.Background(), cfg, filesToUpload, nil, nil) + + info := httpmock.GetCallCountInfo() + + // Direct-commit path should fire: GET base ref, POST commit, PATCH update ref + require.GreaterOrEqual(t, info["GET "+baseRefURL], 1, "direct path: GET base ref") + require.GreaterOrEqual(t, info["POST "+directCommitsURL], 1, "direct path: POST commit") + require.GreaterOrEqual(t, info["PATCH "+updateRefURL], 1, "direct path: PATCH update ref") + + // PR path should fire: POST create ref (temp branch) + POST pulls + require.GreaterOrEqual(t, info["POST "+createRefURL], 1, "PR path: POST create temp branch ref") + require.GreaterOrEqual(t, 1, test.CountByMethodAndURLRegexp("POST", + regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/pulls$`), + ), "PR path: POST create PR") +} + +// TestAddFilesViaPR_ReusesExistingCopierPR verifies that when an open PR from a +// copier/* branch already exists, the app pushes to that branch and updates the PR +// title/body instead of creating a duplicate PR. +func TestAddFilesViaPR_ReusesExistingCopierPR(t *testing.T) { + _ = test.WithHTTPMock(t) + t.Setenv("COPIER_COMMIT_STRATEGY", "pr") + + owner, repo := test.EnvOwnerRepo(t) + baseBranch := "main" + existingBranch := "copier/20260101-120000" + + cfg := test.TestConfig() + services.DefaultTokenManager().SetInstallationAccessToken("") + test.MockGitHubAppTokenEndpoint(cfg.InstallationId) + err := services.ConfigurePermissions(context.Background(), cfg) + require.NoError(t, err, "ConfigurePermissions should succeed") + + test.SetupOrgToken(owner, "test-token") + test.MockGetCommit(owner, repo, "existingSha", "oldTreeSha") + + // Return an existing open PR from a copier/* branch + test.MockListOpenPRs(owner, repo, []map[string]any{ + { + "number": 77, + "head": map[string]any{"ref": existingBranch}, + "base": map[string]any{"ref": baseBranch}, + }, + }) + + // Mock the existing copier branch ref (for commit push) + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/ref/(?:refs/)?heads/`+regexp.QuoteMeta(existingBranch)+`$`), + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "ref": "refs/heads/" + existingBranch, "object": map[string]any{"sha": "existingSha"}, + }), + ) + httpmock.RegisterRegexpResponder("POST", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/trees(\?.*)?$`), + httpmock.NewJsonResponderOrPanic(201, map[string]any{"sha": "newTreeSha"}), + ) + commitsURL := "https://api.github.com/repos/" + owner + "/" + repo + "/git/commits" + httpmock.RegisterResponder("POST", commitsURL, + httpmock.NewJsonResponderOrPanic(201, map[string]any{"sha": "newCommitSha"}), + ) + httpmock.RegisterRegexpResponder("PATCH", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/refs/heads/`+regexp.QuoteMeta(existingBranch)+`$`), + httpmock.NewStringResponder(200, "{}"), + ) + // Mock PR Edit (title/body update) + editPRURL := "https://api.github.com/repos/" + owner + "/" + repo + "/pulls/77" + httpmock.RegisterResponder("PATCH", editPRURL, + httpmock.NewJsonResponderOrPanic(200, map[string]any{"number": 77}), + ) + + files := []github.RepositoryContent{{ + Name: github.Ptr("updated-file.txt"), + Path: github.Ptr("updated-file.txt"), + Content: github.Ptr(base64.StdEncoding.EncodeToString([]byte("new content"))), + }} + filesToUpload := map[types.UploadKey]types.UploadFileContent{ + {RepoName: repo, BranchPath: "refs/heads/" + baseBranch, CommitStrategy: "pull_request"}: { + TargetBranch: baseBranch, + Content: files, + CommitStrategy: "pr", + }, + } + + services.AddFilesToTargetRepos(context.Background(), cfg, filesToUpload, nil, nil) + + info := httpmock.GetCallCountInfo() + + // Should NOT create a new branch or new PR + require.Equal(t, 0, test.CountByMethodAndURLRegexp("POST", + regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/git/refs$`), + ), "should not create a new branch ref") + require.Equal(t, 0, test.CountByMethodAndURLRegexp("POST", + regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/pulls$`), + ), "should not create a new PR") + + // Should commit to the existing branch and update the PR + require.Equal(t, 1, info["POST "+commitsURL], "should commit to existing branch") + require.Equal(t, 1, info["PATCH "+editPRURL], "should update PR title/body") } -// TestDeleteBranchIfExists_NilReference tests that deleteBranchIfExists handles nil references gracefully func TestDeleteBranchIfExists_NilReference(t *testing.T) { _ = test.WithHTTPMock(t) - // Force fresh token - services.InstallationAccessToken = "" - test.MockGitHubAppTokenEndpoint(os.Getenv(configs.InstallationId)) - err := services.ConfigurePermissions() + cfg := test.TestConfig() + services.DefaultTokenManager().SetInstallationAccessToken("") + test.MockGitHubAppTokenEndpoint(cfg.InstallationId) + err := services.ConfigurePermissions(context.Background(), cfg) require.NoError(t, err, "ConfigurePermissions should succeed") - // This should not panic or make any API calls when ref is nil - // We're testing that the function returns early without attempting to delete ctx := context.Background() client := services.GetRestClient() - // Call with nil reference - should return immediately without error - err = services.DeleteBranchIfExistsExported(ctx, client, "test-org/test-repo", nil) + err = services.DeleteBranchIfExistsExported(ctx, client, cfg.ConfigRepoOwner, "test-org/test-repo", nil) require.NoError(t, err, "DeleteBranchIfExistsExported should succeed with nil ref") - // Verify no DELETE requests were made (since ref was nil) require.Equal(t, 0, test.CountByMethodAndURLRegexp("DELETE", regexp.MustCompile(`/git/refs/`))) } diff --git a/services/health_metrics.go b/services/health_metrics.go index 104e4d2..4858149 100644 --- a/services/health_metrics.go +++ b/services/health_metrics.go @@ -1,10 +1,15 @@ package services import ( + "context" "encoding/json" "net/http" + "os" "sync" "time" + + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/types" ) // HealthStatus represents the health status of the application @@ -217,21 +222,21 @@ func (mc *MetricsCollector) RecordGitHubAPIError() { func (mc *MetricsCollector) GetFilesMatched() int { mc.mu.RLock() defer mc.mu.RUnlock() - return int(mc.filesMatched) + return int(mc.filesMatched) // #nosec G115 -- counter fits in int } // GetFilesUploaded returns the current files uploaded count func (mc *MetricsCollector) GetFilesUploaded() int { mc.mu.RLock() defer mc.mu.RUnlock() - return int(mc.filesUploaded) + return int(mc.filesUploaded) // #nosec G115 -- counter fits in int } // GetFilesUploadFailed returns the current files upload failed count func (mc *MetricsCollector) GetFilesUploadFailed() int { mc.mu.RLock() defer mc.mu.RUnlock() - return int(mc.filesUploadFailed) + return int(mc.filesUploadFailed) // #nosec G115 -- counter fits in int } // GetMetrics returns current metrics @@ -287,10 +292,7 @@ func (mc *MetricsCollector) GetMetrics(fileStateService FileStateService) Metric Calls: mc.githubAPICalls, Errors: mc.githubAPIErrors, ErrorRate: githubErrorRate, - RateLimit: RateLimitInfo{ - Remaining: 5000, // TODO: Get from GitHub API - ResetAt: time.Now().Add(1 * time.Hour), - }, + RateLimit: currentRateLimitInfo(), }, Queues: QueueMetrics{ UploadQueueSize: len(uploadQueue), @@ -341,31 +343,102 @@ func calculateStats(durations []time.Duration) ProcessingTimeStats { } } -// HealthHandler handles /health endpoint -func HealthHandler(fileStateService FileStateService, startTime time.Time) http.HandlerFunc { +// HealthHandler handles /health (liveness) endpoint. +// Returns 200 if the process is running. This is a lightweight check +// suitable for Cloud Run / Kubernetes liveness probes. +func HealthHandler(startTime time.Time, version string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - uploadQueue := fileStateService.GetFilesToUpload() - deprecationQueue := fileStateService.GetFilesToDeprecate() + health := map[string]interface{}{ + "status": "healthy", + "version": version, + "started": true, + "uptime": time.Since(startTime).String(), + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(health) + } +} + +// ReadinessHandler handles /ready endpoint. +// Checks actual dependency connectivity (GitHub API auth, MongoDB). +// Returns 200 if all dependencies are reachable, 503 otherwise. +// Suitable for Cloud Run / Kubernetes readiness probes. +func ReadinessHandler(container *ServiceContainer) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + status := "ready" + httpStatus := http.StatusOK + + // Check GitHub API: verify we have a valid authentication token + githubStatus := "healthy" + githubAuth := defaultTokenManager.GetInstallationAccessToken() != "" + if !githubAuth { + githubStatus = "not_authenticated" + } + // Check rate limit state + remaining, resetAt := GlobalRateLimitState.Get() + if remaining == 0 && time.Now().Before(resetAt) { + githubStatus = "rate_limited" + } + + // Check MongoDB (if audit logging is enabled) + auditStatus := "disabled" + auditConnected := false + if container.AuditLogger != nil { + if err := container.AuditLogger.Ping(ctx); err != nil { + auditStatus = "unavailable" + status = "degraded" + } else { + auditStatus = "connected" + auditConnected = true + } + } + + // If GitHub is not authenticated, we're not ready + if !githubAuth { + status = "not_ready" + httpStatus = http.StatusServiceUnavailable + } + + uploadQueue := container.FileStateService.GetFilesToUpload() + deprecationQueue := container.FileStateService.GetFilesToDeprecate() health := HealthStatus{ - Status: "healthy", + Status: status, Started: true, GitHub: GitHubHealthStatus{ - Status: "healthy", - Authenticated: true, + Status: githubStatus, + Authenticated: githubAuth, }, Queues: QueueHealthStatus{ UploadCount: len(uploadQueue), DeprecationCount: len(deprecationQueue), }, - Uptime: time.Since(startTime).String(), + AuditLogger: AuditLoggerHealthStatus{ + Status: auditStatus, + Connected: auditConnected, + }, + Uptime: time.Since(container.StartTime).String(), } w.Header().Set("Content-Type", "application/json") + w.WriteHeader(httpStatus) _ = json.NewEncoder(w).Encode(health) } } +// currentRateLimitInfo returns the most recently observed GitHub API rate limit info. +func currentRateLimitInfo() RateLimitInfo { + remaining, resetAt := GlobalRateLimitState.Get() + if remaining < 0 { + // No API calls made yet; return safe defaults + return RateLimitInfo{Remaining: -1, ResetAt: time.Time{}} + } + return RateLimitInfo{Remaining: remaining, ResetAt: resetAt} +} + // MetricsHandler handles /metrics endpoint func MetricsHandler(metricsCollector *MetricsCollector, fileStateService FileStateService) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -374,3 +447,143 @@ func MetricsHandler(metricsCollector *MetricsCollector, fileStateService FileSta _ = json.NewEncoder(w).Encode(metrics) } } + +// ConfigDiagnosticResponse is the JSON structure returned by the /config endpoint. +type ConfigDiagnosticResponse struct { + // Version is the build version (set via -ldflags at build time). + Version string `json:"version"` + + // Environment summarizes non-secret runtime configuration. + Environment ConfigEnvironment `json:"environment"` + + // Workflows contains the resolved workflow definitions (if loadable). + // Nil when the config could not be loaded (see LoadError). + Workflows []ConfigDiagnosticWorkflow `json:"workflows,omitempty"` + + // LoadError is set when the effective config file cannot be loaded or parsed. + LoadError string `json:"load_error,omitempty"` +} + +// ConfigEnvironment is a sanitised view of configs.Config. +// Secret fields are replaced with a presence indicator (e.g. "[SET]" / "[NOT SET]"). +type ConfigEnvironment struct { + Port string `json:"port"` + DryRun bool `json:"dry_run"` + UseMainConfig bool `json:"use_main_config"` + EffectiveConfig string `json:"effective_config_file"` + ConfigRepoOwner string `json:"config_repo_owner"` + ConfigRepoName string `json:"config_repo_name"` + ConfigRepoBranch string `json:"config_repo_branch"` + WebserverPath string `json:"webserver_path"` + + // Feature flags + AuditEnabled bool `json:"audit_enabled"` + MetricsEnabled bool `json:"metrics_enabled"` + SlackEnabled bool `json:"slack_enabled"` + + // Tuning + ConfigCacheTTLSeconds int `json:"config_cache_ttl_seconds"` + WebhookProcessingTimeoutSeconds int `json:"webhook_processing_timeout_seconds"` + WebhookMaxRetries int `json:"webhook_max_retries"` + GitHubAPIMaxRetries int `json:"github_api_max_retries"` + + // Secrets (presence only) + PEMKey string `json:"pem_key"` + WebhookSecret string `json:"webhook_secret"` + MongoURI string `json:"mongo_uri"` + SlackWebhook string `json:"slack_webhook_url"` +} + +// ConfigDiagnosticWorkflow is a compact summary of a resolved workflow. +type ConfigDiagnosticWorkflow struct { + Name string `json:"name"` + SourceRepo string `json:"source_repo"` + SourceBranch string `json:"source_branch"` + DestRepo string `json:"dest_repo"` + DestBranch string `json:"dest_branch"` + CommitStrategy string `json:"commit_strategy"` + Transforms int `json:"transforms"` + Exclude []string `json:"exclude,omitempty"` +} + +// ConfigDiagnosticHandler handles the GET /config endpoint. +// It returns a read-only view of the resolved runtime configuration with +// all secrets redacted, useful for debugging workflow-matching issues. +func ConfigDiagnosticHandler(container *ServiceContainer, version string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + resp := ConfigDiagnosticResponse{ + Version: version, + Environment: buildConfigEnvironment(container.Config), + } + + // Attempt to load and resolve the effective configuration. + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + yamlCfg, err := container.ConfigLoader.LoadConfig(ctx, container.Config) + if err != nil { + resp.LoadError = err.Error() + } else if yamlCfg != nil { + resp.Workflows = summariseWorkflows(yamlCfg) + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + } +} + +// secretPresence returns "[SET]" if s is non-empty, "[NOT SET]" otherwise. +func secretPresence(s string) string { + if s != "" { + return "[SET]" + } + return "[NOT SET]" +} + +func buildConfigEnvironment(cfg *configs.Config) ConfigEnvironment { + return ConfigEnvironment{ + Port: cfg.Port, + DryRun: cfg.DryRun, + UseMainConfig: cfg.UseMainConfig, + EffectiveConfig: cfg.EffectiveConfigFile(), + ConfigRepoOwner: cfg.ConfigRepoOwner, + ConfigRepoName: cfg.ConfigRepoName, + ConfigRepoBranch: cfg.ConfigRepoBranch, + WebserverPath: cfg.WebserverPath, + + AuditEnabled: cfg.AuditEnabled, + MetricsEnabled: cfg.MetricsEnabled, + SlackEnabled: cfg.SlackEnabled, + + ConfigCacheTTLSeconds: cfg.ConfigCacheTTLSeconds, + WebhookProcessingTimeoutSeconds: cfg.WebhookProcessingTimeoutSeconds, + WebhookMaxRetries: cfg.WebhookMaxRetries, + GitHubAPIMaxRetries: cfg.GitHubAPIMaxRetries, + + PEMKey: secretPresence(os.Getenv("GITHUB_APP_PRIVATE_KEY_B64")), + WebhookSecret: secretPresence(cfg.WebhookSecret), + MongoURI: secretPresence(cfg.MongoURI), + SlackWebhook: secretPresence(cfg.SlackWebhookURL), + } +} + +func summariseWorkflows(cfg *types.YAMLConfig) []ConfigDiagnosticWorkflow { + out := make([]ConfigDiagnosticWorkflow, 0, len(cfg.Workflows)) + for _, wf := range cfg.Workflows { + strategy := "direct" + if wf.CommitStrategy != nil { + strategy = string(wf.CommitStrategy.Type) + } + out = append(out, ConfigDiagnosticWorkflow{ + Name: wf.Name, + SourceRepo: wf.Source.Repo, + SourceBranch: wf.Source.Branch, + DestRepo: wf.Destination.Repo, + DestBranch: wf.Destination.Branch, + CommitStrategy: strategy, + Transforms: len(wf.Transformations), + Exclude: wf.Exclude, + }) + } + return out +} diff --git a/services/health_metrics_test.go b/services/health_metrics_test.go index c3ede8d..eb5e6d5 100644 --- a/services/health_metrics_test.go +++ b/services/health_metrics_test.go @@ -1,12 +1,14 @@ package services_test import ( + "context" "encoding/json" "net/http" "net/http/httptest" "testing" "time" + "github.com/grove-platform/github-copier/configs" "github.com/grove-platform/github-copier/services" "github.com/grove-platform/github-copier/types" "github.com/stretchr/testify/assert" @@ -136,10 +138,9 @@ func TestMetricsCollector_QueueSizes(t *testing.T) { } func TestHealthHandler(t *testing.T) { - fileStateService := services.NewFileStateService() startTime := time.Now().Add(-1 * time.Hour) - handler := services.HealthHandler(fileStateService, startTime) + handler := services.HealthHandler(startTime, "v0.0.0-test") req := httptest.NewRequest("GET", "/health", nil) w := httptest.NewRecorder() @@ -158,6 +159,70 @@ func TestHealthHandler(t *testing.T) { assert.NotNil(t, health["uptime"]) } +func TestReadinessHandler(t *testing.T) { + config := &configs.Config{ + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: false, + } + + container, err := services.NewServiceContainer(config) + require.NoError(t, err) + + // Clear any token set by previous tests so GitHub shows as not_authenticated + container.TokenManager.SetInstallationAccessToken("") + + handler := services.ReadinessHandler(container) + + req := httptest.NewRequest("GET", "/ready", nil) + w := httptest.NewRecorder() + + handler(w, req) + + assert.Equal(t, "application/json", w.Header().Get("Content-Type")) + + var health services.HealthStatus + err = json.Unmarshal(w.Body.Bytes(), &health) + require.NoError(t, err) + + // With no installation token set, should be not_ready + assert.Equal(t, "not_ready", health.Status) + assert.False(t, health.GitHub.Authenticated) + assert.Equal(t, "not_authenticated", health.GitHub.Status) + assert.True(t, health.Started) +} + +func TestReadinessHandler_WithAuth(t *testing.T) { + config := &configs.Config{ + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: false, + } + + container, err := services.NewServiceContainer(config) + require.NoError(t, err) + + // Set a token so GitHub shows as authenticated + container.TokenManager.SetInstallationAccessToken("test-token") + + handler := services.ReadinessHandler(container) + + req := httptest.NewRequest("GET", "/ready", nil) + w := httptest.NewRecorder() + + handler(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var health services.HealthStatus + err = json.Unmarshal(w.Body.Bytes(), &health) + require.NoError(t, err) + + assert.Equal(t, "ready", health.Status) + assert.True(t, health.GitHub.Authenticated) + assert.Equal(t, "healthy", health.GitHub.Status) +} + func TestMetricsHandler(t *testing.T) { collector := services.NewMetricsCollector() fileStateService := services.NewFileStateService() @@ -286,6 +351,161 @@ func TestMetricsCollector_SuccessRateCalculation(t *testing.T) { } } +func TestConfigDiagnosticHandler_EnvironmentFields(t *testing.T) { + config := &configs.Config{ + Port: "8080", + DryRun: true, + UseMainConfig: true, + MainConfigFile: ".copier/main.yaml", + ConfigFile: "copier-config.yaml", + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + ConfigRepoBranch: "main", + WebserverPath: "/events", + AuditEnabled: false, + MetricsEnabled: true, + SlackEnabled: true, + SlackWebhookURL: "https://hooks.slack.com/test", + WebhookSecret: "s3cret", + MongoURI: "", + ConfigCacheTTLSeconds: 60, + WebhookProcessingTimeoutSeconds: 300, + WebhookMaxRetries: 3, + GitHubAPIMaxRetries: 5, + } + + container, err := services.NewServiceContainer(config) + require.NoError(t, err) + + handler := services.ConfigDiagnosticHandler(container, "v1.2.3-test") + + req := httptest.NewRequest("GET", "/config", nil) + w := httptest.NewRecorder() + + handler(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "application/json", w.Header().Get("Content-Type")) + + var resp services.ConfigDiagnosticResponse + err = json.Unmarshal(w.Body.Bytes(), &resp) + require.NoError(t, err) + + // Verify version + assert.Equal(t, "v1.2.3-test", resp.Version) + + // Verify environment fields + env := resp.Environment + assert.Equal(t, "8080", env.Port) + assert.True(t, env.DryRun) + assert.True(t, env.UseMainConfig) + assert.Equal(t, ".copier/main.yaml", env.EffectiveConfig) + assert.Equal(t, "test-owner", env.ConfigRepoOwner) + assert.Equal(t, "test-repo", env.ConfigRepoName) + assert.Equal(t, "main", env.ConfigRepoBranch) + assert.Equal(t, "/events", env.WebserverPath) + assert.True(t, env.MetricsEnabled) + assert.True(t, env.SlackEnabled) + assert.False(t, env.AuditEnabled) + assert.Equal(t, 60, env.ConfigCacheTTLSeconds) + assert.Equal(t, 300, env.WebhookProcessingTimeoutSeconds) + assert.Equal(t, 3, env.WebhookMaxRetries) + assert.Equal(t, 5, env.GitHubAPIMaxRetries) + + // Secrets should be redacted + assert.Equal(t, "[SET]", env.WebhookSecret) + assert.Equal(t, "[SET]", env.SlackWebhook) + assert.Equal(t, "[NOT SET]", env.MongoURI) + + // Config loading will fail (no real GitHub client), but the endpoint still works + assert.NotEmpty(t, resp.LoadError) + assert.Nil(t, resp.Workflows) +} + +func TestConfigDiagnosticHandler_WorkflowSummary(t *testing.T) { + // Test the workflow summary with a mock config loader that returns a valid config + config := &configs.Config{ + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + } + + container, err := services.NewServiceContainer(config) + require.NoError(t, err) + + // Replace the config loader with one that returns test workflows + container.ConfigLoader = &mockConfigLoaderForDiagnostic{ + config: &types.YAMLConfig{ + Workflows: []types.Workflow{ + { + Name: "copy-go-examples", + Source: types.Source{Repo: "org/source", Branch: "main"}, + Destination: types.Destination{Repo: "org/dest", Branch: "main"}, + Transformations: []types.Transformation{ + {Move: &types.MoveTransform{From: "examples", To: "code"}}, + }, + CommitStrategy: &types.CommitStrategyConfig{Type: "pull_request"}, + }, + { + Name: "copy-js-examples", + Source: types.Source{Repo: "org/source", Branch: "main"}, + Destination: types.Destination{Repo: "org/dest-2", Branch: "develop"}, + Transformations: []types.Transformation{ + {Glob: &types.GlobTransform{Pattern: "**/*.js", Transform: "js/${relative_path}"}}, + {Glob: &types.GlobTransform{Pattern: "**/*.ts", Transform: "ts/${relative_path}"}}, + }, + Exclude: []string{"node_modules"}, + }, + }, + }, + } + + handler := services.ConfigDiagnosticHandler(container, "v0.0.0-test") + + req := httptest.NewRequest("GET", "/config", nil) + w := httptest.NewRecorder() + + handler(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var resp services.ConfigDiagnosticResponse + err = json.Unmarshal(w.Body.Bytes(), &resp) + require.NoError(t, err) + + assert.Empty(t, resp.LoadError) + require.Len(t, resp.Workflows, 2) + + wf1 := resp.Workflows[0] + assert.Equal(t, "copy-go-examples", wf1.Name) + assert.Equal(t, "org/source", wf1.SourceRepo) + assert.Equal(t, "main", wf1.SourceBranch) + assert.Equal(t, "org/dest", wf1.DestRepo) + assert.Equal(t, "main", wf1.DestBranch) + assert.Equal(t, "pull_request", wf1.CommitStrategy) + assert.Equal(t, 1, wf1.Transforms) + + wf2 := resp.Workflows[1] + assert.Equal(t, "copy-js-examples", wf2.Name) + assert.Equal(t, "org/dest-2", wf2.DestRepo) + assert.Equal(t, "develop", wf2.DestBranch) + assert.Equal(t, "direct", wf2.CommitStrategy) // nil commit_strategy defaults to "direct" + assert.Equal(t, 2, wf2.Transforms) + assert.Equal(t, []string{"node_modules"}, wf2.Exclude) +} + +// mockConfigLoaderForDiagnostic returns a static config for testing the diagnostic endpoint. +type mockConfigLoaderForDiagnostic struct { + config *types.YAMLConfig +} + +func (m *mockConfigLoaderForDiagnostic) LoadConfig(_ context.Context, _ *configs.Config) (*types.YAMLConfig, error) { + return m.config, nil +} + +func (m *mockConfigLoaderForDiagnostic) LoadConfigFromContent(_ string, _ string) (*types.YAMLConfig, error) { + return m.config, nil +} + func TestMetricsCollector_ConcurrentAccess(t *testing.T) { collector := services.NewMetricsCollector() fileStateService := services.NewFileStateService() diff --git a/services/integration_test.go b/services/integration_test.go new file mode 100644 index 0000000..cef22b1 --- /dev/null +++ b/services/integration_test.go @@ -0,0 +1,699 @@ +package services + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "regexp" + "testing" + + "github.com/google/go-github/v82/github" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/types" + "github.com/jarcoal/httpmock" +) + +// --- Mock ConfigLoader for integration tests --- + +type mockConfigLoader struct { + config *types.YAMLConfig + err error +} + +func (m *mockConfigLoader) LoadConfig(_ context.Context, _ *configs.Config) (*types.YAMLConfig, error) { + return m.config, m.err +} + +func (m *mockConfigLoader) LoadConfigFromContent(_ string, _ string) (*types.YAMLConfig, error) { + return m.config, m.err +} + +// --- Helper to build a signed merged-PR webhook request --- + +func buildMergedPRWebhook(t *testing.T, owner, repo, branch string, prNumber int, secret string) (*http.Request, []byte) { + t.Helper() + prEvent := &github.PullRequestEvent{ + Action: github.Ptr("closed"), + PullRequest: &github.PullRequest{ + Number: github.Ptr(prNumber), + Merged: github.Ptr(true), + MergeCommitSHA: github.Ptr("abc123def456"), + Base: &github.PullRequestBranch{ + Ref: github.Ptr(branch), + }, + }, + Repo: &github.Repository{ + Name: github.Ptr(repo), + Owner: &github.User{Login: github.Ptr(owner)}, + }, + } + payload, err := json.Marshal(prEvent) + if err != nil { + t.Fatalf("marshal PR event: %v", err) + } + + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) + req.Header.Set("X-GitHub-Event", "pull_request") + req.Header.Set("X-GitHub-Delivery", "integration-test-delivery-1") + + if secret != "" { + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write(payload) + req.Header.Set("X-Hub-Signature-256", "sha256="+hex.EncodeToString(mac.Sum(nil))) + } + + return req, payload +} + +// --- Integration test: full webhook โ†’ config โ†’ process โ†’ upload flow --- + +func TestIntegration_MergedPR_DirectCommit(t *testing.T) { + // This test verifies the complete webhook processing pipeline: + // webhook delivery โ†’ config load โ†’ workflow match โ†’ file fetch โ†’ process โ†’ commit to target + + owner := "test-org" + sourceRepo := "source-repo" + targetRepo := "target-repo" + branch := "main" + prNumber := 42 + + // 1. Set up global httpmock to intercept ALL HTTP calls (including GraphQL) + httpmock.Activate() + t.Cleanup(httpmock.DeactivateAndReset) + + // Use a fresh TokenManager + tm := NewTokenManager() + tm.SetInstallationAccessToken("test-token") + tm.SetTokenForOrgNoExpiry(owner, "test-token") + prev := defaultTokenManager + defaultTokenManager = tm + t.Cleanup(func() { defaultTokenManager = prev }) + + // 2. Mock GraphQL endpoint for GetFilesChangedInPr + httpmock.RegisterResponder("POST", "https://api.github.com/graphql", + func(req *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(200, map[string]any{ + "data": map[string]any{ + "repository": map[string]any{ + "pullRequest": map[string]any{ + "files": map[string]any{ + "edges": []map[string]any{ + { + "node": map[string]any{ + "path": "examples/hello.go", + "additions": 10, + "deletions": 2, + "changeType": "MODIFIED", + }, + }, + }, + "pageInfo": map[string]any{ + "hasNextPage": false, + "endCursor": "", + }, + }, + }, + }, + }, + }) + }, + ) + + // 3. Mock REST endpoints for retrieving source file content + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+sourceRepo+`/contents/examples/hello\.go`), + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "type": "file", + "name": "hello.go", + "path": "examples/hello.go", + "encoding": "base64", + "content": base64.StdEncoding.EncodeToString([]byte("package main\n\nfunc main() {}\n")), + }), + ) + + // 4. Mock REST endpoints for writing to target repo (direct commit) + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/ref/`), + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "ref": "refs/heads/" + branch, + "object": map[string]any{"sha": "base-sha-000"}, + }), + ) + // Mock GET commit for empty-commit detection + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/commits/base-sha-000$`), + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "sha": "base-sha-000", + "tree": map[string]any{"sha": "old-tree-sha"}, + }), + ) + httpmock.RegisterRegexpResponder("POST", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/trees`), + httpmock.NewJsonResponderOrPanic(201, map[string]any{"sha": "new-tree-sha"}), + ) + httpmock.RegisterResponder("POST", + "https://api.github.com/repos/"+owner+"/"+targetRepo+"/git/commits", + httpmock.NewJsonResponderOrPanic(201, map[string]any{"sha": "new-commit-sha"}), + ) + httpmock.RegisterRegexpResponder("PATCH", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/refs/heads/`+branch), + httpmock.NewStringResponder(200, `{}`), + ) + + // 5. Mock deprecation file endpoint (UpdateDeprecationFile reads from source repo) + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+sourceRepo+`/contents/deprecated`), + httpmock.NewStringResponder(404, `{"message":"Not Found"}`), + ) + + // 6. Set up mock ConfigLoader with a matching workflow + mockConfig := &types.YAMLConfig{ + Workflows: []types.Workflow{ + { + Name: "test-workflow", + Source: types.Source{ + Repo: owner + "/" + sourceRepo, + Branch: branch, + }, + Destination: types.Destination{ + Repo: owner + "/" + targetRepo, + Branch: branch, + }, + Transformations: []types.Transformation{ + { + Copy: &types.CopyTransform{ + From: "examples/hello.go", + To: "examples/hello.go", + }, + }, + }, + CommitStrategy: &types.CommitStrategyConfig{ + Type: "direct", + CommitMessage: "chore: sync from source", + }, + }, + }, + } + + // 7. Create container with mock config loader + config := configs.NewConfig() + config.ConfigRepoOwner = owner + config.ConfigRepoName = "config-repo" + config.ConfigRepoBranch = "main" + config.AuditEnabled = false + config.DefaultCommitMessage = "chore: sync files" + + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer: %v", err) + } + container.ConfigLoader = &mockConfigLoader{config: mockConfig} + + // 8. Send the webhook + req, _ := buildMergedPRWebhook(t, owner, sourceRepo, branch, prNumber, "") + + w := httptest.NewRecorder() + HandleWebhookWithContainer(w, req, config, container) + + // 9. Wait for background goroutine to complete + container.Wait() + + // 10. Verify HTTP response + if w.Code != http.StatusAccepted { + t.Errorf("status = %d, want %d", w.Code, http.StatusAccepted) + } + + // 11. Verify the GraphQL endpoint was called (file list) + info := httpmock.GetCallCountInfo() + graphqlCalls := info["POST https://api.github.com/graphql"] + if graphqlCalls < 1 { + t.Errorf("expected at least 1 GraphQL call, got %d", graphqlCalls) + } + + // 12. Verify the workflow processor queued files for upload + // RecordFileUploaded is called when the processor queues a file. + // If the source content mock wasn't hit or parsing failed, this will be 0. + filesUploaded := container.MetricsCollector.GetFilesUploaded() + t.Logf("files uploaded: %d", filesUploaded) + + // At minimum, verify the full pipeline ran (GraphQL + workflow processing) + if graphqlCalls < 1 { + t.Error("pipeline did not reach file retrieval stage") + } +} + +func TestIntegration_MergedPR_NoMatchingWorkflows(t *testing.T) { + // Test that a merged PR to a branch with no matching workflows + // is handled gracefully without panics or errors. + + owner := "test-org" + sourceRepo := "source-repo" + + httpmock.Activate() + t.Cleanup(httpmock.DeactivateAndReset) + + tm := NewTokenManager() + tm.SetInstallationAccessToken("test-token") + prev := defaultTokenManager + defaultTokenManager = tm + t.Cleanup(func() { defaultTokenManager = prev }) + + // Config with workflow for "main" branch only + mockConfig := &types.YAMLConfig{ + Workflows: []types.Workflow{ + { + Name: "main-only", + Source: types.Source{Repo: owner + "/" + sourceRepo, Branch: "main"}, + }, + }, + } + + config := &configs.Config{ + ConfigRepoOwner: owner, + ConfigRepoName: "config-repo", + AuditEnabled: false, + } + + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer: %v", err) + } + container.ConfigLoader = &mockConfigLoader{config: mockConfig} + + // Send webhook for "develop" branch โ€” no matching workflow + req, _ := buildMergedPRWebhook(t, owner, sourceRepo, "develop", 99, "") + + w := httptest.NewRecorder() + HandleWebhookWithContainer(w, req, config, container) + container.Wait() + + if w.Code != http.StatusAccepted { + t.Errorf("status = %d, want %d", w.Code, http.StatusAccepted) + } + + // Webhook should be recorded as failed (no matching workflows) + metrics := container.MetricsCollector.GetMetrics(container.FileStateService) + if metrics.Webhooks.Failed < 1 { + t.Error("expected webhook failed count >= 1 (no matching workflows)") + } +} + +func TestIntegration_MergedPR_ConfigLoadError(t *testing.T) { + // Test that a config load failure is handled gracefully. + + owner := "test-org" + sourceRepo := "source-repo" + + httpmock.Activate() + t.Cleanup(httpmock.DeactivateAndReset) + + tm := NewTokenManager() + tm.SetInstallationAccessToken("test-token") + prev := defaultTokenManager + defaultTokenManager = tm + t.Cleanup(func() { defaultTokenManager = prev }) + + config := &configs.Config{ + ConfigRepoOwner: owner, + ConfigRepoName: "config-repo", + AuditEnabled: false, + } + + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer: %v", err) + } + container.ConfigLoader = &mockConfigLoader{ + err: ErrConfigLoad, + } + + req, _ := buildMergedPRWebhook(t, owner, sourceRepo, "main", 50, "") + + w := httptest.NewRecorder() + HandleWebhookWithContainer(w, req, config, container) + container.Wait() + + if w.Code != http.StatusAccepted { + t.Errorf("status = %d, want %d", w.Code, http.StatusAccepted) + } + + metrics := container.MetricsCollector.GetMetrics(container.FileStateService) + if metrics.Webhooks.Failed < 1 { + t.Error("expected webhook failed count >= 1 (config load error)") + } +} + +func TestIntegration_WebhookSignatureVerification(t *testing.T) { + // Test end-to-end with signature verification enabled. + + secret := "integration-test-secret" + + config := &configs.Config{ + ConfigRepoOwner: "test-org", + ConfigRepoName: "config-repo", + WebhookSecret: secret, + AuditEnabled: false, + } + + t.Run("valid signature accepted", func(t *testing.T) { + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer: %v", err) + } + + req, _ := buildMergedPRWebhook(t, "test-org", "source-repo", "main", 1, secret) + w := httptest.NewRecorder() + HandleWebhookWithContainer(w, req, config, container) + container.Wait() + + // Should be accepted (202), not rejected + if w.Code == http.StatusUnauthorized { + t.Error("valid signature was rejected") + } + }) + + t.Run("invalid signature rejected", func(t *testing.T) { + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer: %v", err) + } + + // Build request signed with wrong secret + req, _ := buildMergedPRWebhook(t, "test-org", "source-repo", "main", 2, "wrong-secret") + w := httptest.NewRecorder() + HandleWebhookWithContainer(w, req, config, container) + + if w.Code != http.StatusUnauthorized { + t.Errorf("status = %d, want %d", w.Code, http.StatusUnauthorized) + } + }) +} + +// --- Integration test: target repo batching with mixed strategies --- + +func TestIntegration_TargetRepoBatching_MixedStrategies(t *testing.T) { + // Verifies that two workflows targeting the same repo but with different + // commit strategies (direct vs pull_request) produce separate write operations. + // Also verifies that two workflows with the same strategy are batched together. + + owner := "test-org" + sourceRepo := "source-repo" + targetRepo := "target-repo" + branch := "main" + prNumber := 55 + + httpmock.Activate() + t.Cleanup(httpmock.DeactivateAndReset) + + tm := NewTokenManager() + tm.SetInstallationAccessToken("test-token") + tm.SetTokenForOrgNoExpiry(owner, "test-token") + prev := defaultTokenManager + defaultTokenManager = tm + t.Cleanup(func() { defaultTokenManager = prev }) + + // GraphQL: return two changed files + httpmock.RegisterResponder("POST", "https://api.github.com/graphql", + func(req *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(200, map[string]any{ + "data": map[string]any{ + "repository": map[string]any{ + "pullRequest": map[string]any{ + "files": map[string]any{ + "edges": []map[string]any{ + {"node": map[string]any{"path": "docs/guide.md", "additions": 5, "deletions": 0, "changeType": "MODIFIED"}}, + {"node": map[string]any{"path": "examples/demo.go", "additions": 10, "deletions": 3, "changeType": "MODIFIED"}}, + }, + "pageInfo": map[string]any{"hasNextPage": false, "endCursor": ""}, + }, + }, + }, + }, + }) + }, + ) + + // Source file content mocks + for _, path := range []string{"docs/guide.md", "examples/demo.go"} { + p := path + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+sourceRepo+`/contents/`+regexp.QuoteMeta(p)), + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "type": "file", "name": p, "path": p, "encoding": "base64", + "content": base64.StdEncoding.EncodeToString([]byte("content of " + p)), + }), + ) + } + + // Target repo write endpoints (direct commit) + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/ref/heads/`+branch+`$`), + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "ref": "refs/heads/" + branch, "object": map[string]any{"sha": "base-sha"}, + }), + ) + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/commits/base-sha$`), + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "sha": "base-sha", "tree": map[string]any{"sha": "old-tree-sha"}, + }), + ) + directTreesURL := regexp.MustCompile(`^https://api\.github\.com/repos/` + owner + `/` + targetRepo + `/git/trees`) + httpmock.RegisterRegexpResponder("POST", directTreesURL, + httpmock.NewJsonResponderOrPanic(201, map[string]any{"sha": "new-tree-sha"}), + ) + directCommitsURL := "https://api.github.com/repos/" + owner + "/" + targetRepo + "/git/commits" + httpmock.RegisterResponder("POST", directCommitsURL, + httpmock.NewJsonResponderOrPanic(201, map[string]any{"sha": "new-commit-sha"}), + ) + httpmock.RegisterRegexpResponder("PATCH", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/refs/heads/`+branch), + httpmock.NewStringResponder(200, `{}`), + ) + + // Target repo PR endpoints (for pull_request strategy) + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/pulls\?`), + httpmock.NewJsonResponderOrPanic(200, []map[string]any{}), // no existing PRs + ) + httpmock.RegisterRegexpResponder("POST", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/refs$`), + httpmock.NewJsonResponderOrPanic(201, map[string]any{"ref": "refs/heads/copier/test", "object": map[string]any{"sha": "base-sha"}}), + ) + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/ref/(?:refs/)?heads/copier/`), + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "ref": "refs/heads/copier/test", "object": map[string]any{"sha": "base-sha"}, + }), + ) + httpmock.RegisterRegexpResponder("DELETE", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/refs/heads/copier/`), + httpmock.NewStringResponder(204, ""), + ) + httpmock.RegisterRegexpResponder("PATCH", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+targetRepo+`/git/refs/heads/copier/`), + httpmock.NewStringResponder(200, `{}`), + ) + prCreateURL := "https://api.github.com/repos/" + owner + "/" + targetRepo + "/pulls" + httpmock.RegisterResponder("POST", prCreateURL, + httpmock.NewJsonResponderOrPanic(201, map[string]any{"number": 99, "html_url": "https://github.com/" + owner + "/" + targetRepo + "/pull/99"}), + ) + + // Deprecation file mock (404 = no existing file) + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/config-repo/contents/`), + httpmock.NewStringResponder(404, `{"message":"Not Found"}`), + ) + + // Config: 3 workflows โ€” two direct (should batch), one PR (separate operation) + mockConfig := &types.YAMLConfig{ + Workflows: []types.Workflow{ + { + Name: "wf-direct-docs", + Source: types.Source{Repo: owner + "/" + sourceRepo, Branch: branch}, + Destination: types.Destination{Repo: owner + "/" + targetRepo, Branch: branch}, + Transformations: []types.Transformation{ + {Copy: &types.CopyTransform{From: "docs/guide.md", To: "docs/guide.md"}}, + }, + CommitStrategy: &types.CommitStrategyConfig{Type: "direct", CommitMessage: "sync docs"}, + }, + { + Name: "wf-direct-examples", + Source: types.Source{Repo: owner + "/" + sourceRepo, Branch: branch}, + Destination: types.Destination{Repo: owner + "/" + targetRepo, Branch: branch}, + Transformations: []types.Transformation{ + {Copy: &types.CopyTransform{From: "examples/demo.go", To: "examples/demo.go"}}, + }, + CommitStrategy: &types.CommitStrategyConfig{Type: "direct", CommitMessage: "sync examples"}, + }, + { + Name: "wf-pr-docs", + Source: types.Source{Repo: owner + "/" + sourceRepo, Branch: branch}, + Destination: types.Destination{Repo: owner + "/" + targetRepo, Branch: branch}, + Transformations: []types.Transformation{ + {Copy: &types.CopyTransform{From: "docs/guide.md", To: "pr-docs/guide.md"}}, + }, + CommitStrategy: &types.CommitStrategyConfig{ + Type: "pull_request", + CommitMessage: "sync via PR", + PRTitle: "Copier: sync docs", + }, + }, + }, + } + + config := configs.NewConfig() + config.ConfigRepoOwner = owner + config.ConfigRepoName = "config-repo" + config.ConfigRepoBranch = "main" + config.AuditEnabled = false + config.DefaultCommitMessage = "chore: sync" + + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer: %v", err) + } + container.ConfigLoader = &mockConfigLoader{config: mockConfig} + + req, _ := buildMergedPRWebhook(t, owner, sourceRepo, branch, prNumber, "") + w := httptest.NewRecorder() + HandleWebhookWithContainer(w, req, config, container) + container.Wait() + + if w.Code != http.StatusAccepted { + t.Errorf("status = %d, want %d", w.Code, http.StatusAccepted) + } + + info := httpmock.GetCallCountInfo() + + // Count PATCH to main branch ref (only direct commits update this) + directRefUpdateKey := "PATCH =~^https://api\\.github\\.com/repos/" + owner + "/" + targetRepo + "/git/refs/heads/" + branch + prCreateCalls := info["POST "+prCreateURL] + + // Find the direct ref update count from the call info map + directRefUpdates := 0 + for k, v := range info { + if k == directRefUpdateKey { + directRefUpdates = v + } + } + + t.Logf("Direct ref updates: %d, PR create calls: %d", directRefUpdates, prCreateCalls) + + // Direct commit: the two direct-strategy workflows should batch into 1 ref update + if directRefUpdates != 1 { + t.Errorf("expected 1 direct ref update (batched), got %d", directRefUpdates) + } + + // PR: separate operation should create 1 PR + if prCreateCalls != 1 { + t.Errorf("expected 1 PR created (separate strategy), got %d", prCreateCalls) + } +} + +// --- Unit tests for extracted helper functions --- + +func TestLoadAndMatchWorkflows_MatchesBranch(t *testing.T) { + config := &configs.Config{ + ConfigRepoOwner: "org", + ConfigRepoName: "config", + AuditEnabled: false, + } + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer: %v", err) + } + container.ConfigLoader = &mockConfigLoader{ + config: &types.YAMLConfig{ + Workflows: []types.Workflow{ + {Name: "main-wf", Source: types.Source{Repo: "org/repo", Branch: "main"}}, + {Name: "dev-wf", Source: types.Source{Repo: "org/repo", Branch: "develop"}}, + {Name: "other-wf", Source: types.Source{Repo: "other/repo", Branch: "main"}}, + }, + }, + } + + yamlConfig, err := loadAndMatchWorkflows(context.Background(), config, container, "org/repo", "main", 1) + if err != nil { + t.Fatalf("loadAndMatchWorkflows: %v", err) + } + if len(yamlConfig.Workflows) != 1 { + t.Fatalf("expected 1 matching workflow, got %d", len(yamlConfig.Workflows)) + } + if yamlConfig.Workflows[0].Name != "main-wf" { + t.Errorf("matched workflow = %q, want main-wf", yamlConfig.Workflows[0].Name) + } +} + +func TestLoadAndMatchWorkflows_NoMatch(t *testing.T) { + config := &configs.Config{ + ConfigRepoOwner: "org", + ConfigRepoName: "config", + AuditEnabled: false, + } + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer: %v", err) + } + container.ConfigLoader = &mockConfigLoader{ + config: &types.YAMLConfig{ + Workflows: []types.Workflow{ + {Name: "main-wf", Source: types.Source{Repo: "org/repo", Branch: "main"}}, + }, + }, + } + + _, err = loadAndMatchWorkflows(context.Background(), config, container, "org/repo", "develop", 1) + if err == nil { + t.Error("expected error for no matching workflows") + } +} + +func TestPollMergeability(t *testing.T) { + httpmock.Activate() + t.Cleanup(httpmock.DeactivateAndReset) + + httpmock.RegisterResponder("GET", + "https://api.github.com/repos/org/repo/pulls/10", + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "mergeable": true, + "mergeable_state": "clean", + }), + ) + + tm := NewTokenManager() + tm.SetTokenForOrgNoExpiry("org", "test-token") + prev := defaultTokenManager + defaultTokenManager = tm + t.Cleanup(func() { defaultTokenManager = prev }) + + client := newGitHubRESTClient("test-token", nil) + mergeable, state := pollMergeability(context.Background(), client, "org", "repo", 10, 3, 10) + + if mergeable == nil { + t.Fatal("expected mergeable to be computed") + } + if !*mergeable { + t.Error("expected mergeable = true") + } + if state != "clean" { + t.Errorf("state = %q, want clean", state) + } +} + +func TestRecordBatchFailure(t *testing.T) { + mc := NewMetricsCollector() + + recordBatchFailure(nil, 5) // should not panic + + recordBatchFailure(mc, 3) + if mc.GetFilesUploadFailed() != 3 { + t.Errorf("filesUploadFailed = %d, want 3", mc.GetFilesUploadFailed()) + } +} diff --git a/services/logger.go b/services/logger.go index e6c13b5..c09cf1f 100644 --- a/services/logger.go +++ b/services/logger.go @@ -2,9 +2,8 @@ package services import ( "context" - "encoding/json" "fmt" - "log" + "log/slog" "net/http" "os" "strings" @@ -20,53 +19,104 @@ type contextKey string // requestIDKey is the context key for request IDs const requestIDKey contextKey = "request_id" -var googleInfoLogger *log.Logger -var googleWarningLogger *log.Logger -var googleErrorLogger *log.Logger -var googleCriticalLogger *log.Logger +// LevelCritical is a custom slog level above Error for critical/fatal issues. +// slog defines Debug=-4, Info=0, Warn=4, Error=8; we use 12 for Critical. +const LevelCritical = slog.Level(12) -// keep a reference to allow flushing/closing and to avoid re-initialization +// keep a reference to allow flushing/closing var googleLoggingClient *logging.Client var gcpLoggingEnabled bool -// InitializeGoogleLogger sets up Google Cloud Logging level loggers if not disabled. -// It is safe to call multiple times; initialization will only occur once per process. -func InitializeGoogleLogger() { - // Allow disabling cloud logging for local/dev via env. +// googleLoggers maps slog levels to GCP Cloud Logging standard loggers. +// Only populated when GCP Cloud Logging is enabled. +var googleLoggers map[slog.Level]*logging.Logger + +// InitializeLogger sets up the slog-based logger with JSON output and optional +// GCP Cloud Logging integration. Call this once at startup. +func InitializeLogger(config *configs.Config) { + level := slog.LevelInfo + if isDebugEnabled() { + level = slog.LevelDebug + } + + opts := &slog.HandlerOptions{ + Level: level, + ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { + // Rename "level" to "severity" for Cloud Logging JSON compatibility. + // Cloud Run/GKE auto-parses severity from structured JSON on stdout. + if a.Key == slog.LevelKey { + a.Key = "severity" + lvl := a.Value.Any().(slog.Level) + switch { + case lvl >= LevelCritical: + a.Value = slog.StringValue("CRITICAL") + case lvl >= slog.LevelError: + a.Value = slog.StringValue("ERROR") + case lvl >= slog.LevelWarn: + a.Value = slog.StringValue("WARNING") + case lvl >= slog.LevelInfo: + a.Value = slog.StringValue("INFO") + default: + a.Value = slog.StringValue("DEBUG") + } + } + // Rename "msg" to "message" for Cloud Logging compatibility + if a.Key == slog.MessageKey { + a.Key = "message" + } + return a + }, + } + + handler := slog.NewJSONHandler(os.Stdout, opts) + slog.SetDefault(slog.New(handler)) + + // Optionally initialize GCP Cloud Logging API client for direct log ingestion + initGCPLogging(config) +} + +// initGCPLogging sets up the GCP Cloud Logging API client if configured. +// This is a secondary logging path โ€” the primary path is JSON to stdout which +// Cloud Run/GKE auto-ingests. The API client can be useful for non-Cloud Run +// deployments or when you need log entries with richer metadata. +func initGCPLogging(config *configs.Config) { if isCloudLoggingDisabled() { gcpLoggingEnabled = false return } if googleLoggingClient != nil { - // already initialized gcpLoggingEnabled = true return } - projectId := os.Getenv(configs.GoogleCloudProjectId) + projectId := config.GoogleCloudProjectId if projectId == "" { - log.Printf("[WARN] GOOGLE_CLOUD_PROJECT_ID not set, disabling cloud logging\n") + slog.Warn("GOOGLE_CLOUD_PROJECT_ID not set, disabling GCP Cloud Logging API client") gcpLoggingEnabled = false return } client, err := logging.NewClient(context.Background(), projectId) if err != nil { - log.Printf("[WARN] Failed to create Google logging client: %v\n", err) + slog.Warn("failed to create GCP Cloud Logging client, falling back to stdout only", + "error", err) gcpLoggingEnabled = false return } googleLoggingClient = client gcpLoggingEnabled = true - logName := os.Getenv(configs.CopierLogName) + logName := config.CopierLogName if logName == "" { - logName = "code-copier-log" // fallback default + logName = "code-copier-log" + } + + googleLoggers = map[slog.Level]*logging.Logger{ + slog.LevelInfo: client.Logger(logName), + slog.LevelWarn: client.Logger(logName), + slog.LevelError: client.Logger(logName), + LevelCritical: client.Logger(logName), } - googleInfoLogger = client.Logger(logName).StandardLogger(logging.Info) - googleWarningLogger = client.Logger(logName).StandardLogger(logging.Warning) - googleErrorLogger = client.Logger(logName).StandardLogger(logging.Error) - googleCriticalLogger = client.Logger(logName).StandardLogger(logging.Critical) } // CloseGoogleLogger flushes and closes the underlying Google logging client, if any. @@ -76,94 +126,115 @@ func CloseGoogleLogger() { } } -// LogDebug writes debug logs only when LOG_LEVEL=debug or COPIER_DEBUG=true. -func LogDebug(message string) { - if !isDebugEnabled() { - return - } - // Mirror to GCP as info if available, plus prefix to stdout - if googleInfoLogger != nil && gcpLoggingEnabled { - googleInfoLogger.Println("[DEBUG] " + message) +// gcpSeverity maps slog levels to GCP logging severity. +func gcpSeverity(level slog.Level) logging.Severity { + switch { + case level >= LevelCritical: + return logging.Critical + case level >= slog.LevelError: + return logging.Error + case level >= slog.LevelWarn: + return logging.Warning + default: + return logging.Info } - log.Println("[DEBUG] " + message) } -func LogInfo(message string) { - if googleInfoLogger != nil && gcpLoggingEnabled { - googleInfoLogger.Println(message) +// logToGCP sends a log entry to GCP Cloud Logging API if enabled. +func logToGCP(level slog.Level, msg string, attrs ...any) { + if !gcpLoggingEnabled || googleLoggers == nil { + return + } + logger := googleLoggers[slog.LevelInfo] // default + if l, ok := googleLoggers[level]; ok { + logger = l + } + if logger == nil { + return } - log.Println("[INFO] " + message) -} -func LogWarning(message string) { - if googleWarningLogger != nil && gcpLoggingEnabled { - googleWarningLogger.Println(message) + // Build payload as a map for structured GCP log entries + payload := map[string]any{"message": msg} + for i := 0; i+1 < len(attrs); i += 2 { + if key, ok := attrs[i].(string); ok { + payload[key] = attrs[i+1] + } } - log.Println("[WARN] " + message) + + logger.Log(logging.Entry{ + Severity: gcpSeverity(level), + Payload: payload, + }) } -func LogError(message string) { - if googleErrorLogger != nil && gcpLoggingEnabled { - googleErrorLogger.Println(message) +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Convenience logging functions +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +// LogDebug writes a debug-level log. Only emits when LOG_LEVEL=debug or COPIER_DEBUG=true. +func LogDebug(message string, args ...any) { + if !isDebugEnabled() { + return } - log.Println("[ERROR] " + message) + slog.Debug(message, args...) + logToGCP(slog.LevelDebug, message, args...) } -func LogCritical(message string) { - if googleCriticalLogger != nil && gcpLoggingEnabled { - googleCriticalLogger.Println(message) - } - log.Println("[CRITICAL] " + message) +// LogInfo writes an info-level log. +func LogInfo(message string, args ...any) { + slog.Info(message, args...) + logToGCP(slog.LevelInfo, message, args...) } -func isDebugEnabled() bool { - if strings.EqualFold(os.Getenv("LOG_LEVEL"), "debug") { - return true - } - return strings.EqualFold(os.Getenv("COPIER_DEBUG"), "true") +// LogWarning writes a warning-level log. +func LogWarning(message string, args ...any) { + slog.Warn(message, args...) // #nosec G706 -- structured logging; args are key-value pairs, not user input + logToGCP(slog.LevelWarn, message, args...) } -func isCloudLoggingDisabled() bool { - return strings.EqualFold(os.Getenv("COPIER_DISABLE_CLOUD_LOGGING"), "true") +// LogError writes an error-level log. +func LogError(message string, args ...any) { + slog.Error(message, args...) + logToGCP(slog.LevelError, message, args...) } -// Context-aware logging functions +// LogCritical writes a critical-level log (above Error). +func LogCritical(message string, args ...any) { + slog.Log(context.Background(), LevelCritical, message, args...) + logToGCP(LevelCritical, message, args...) +} -// LogInfoCtx logs an info message with context and additional fields +// LogInfoCtx writes an info-level log with context. func LogInfoCtx(ctx context.Context, message string, fields map[string]interface{}) { - msg := formatLogMessage(ctx, message, fields) - LogInfo(msg) + slog.InfoContext(ctx, message, mapToAttrs(fields)...) + logToGCP(slog.LevelInfo, message, mapToAttrs(fields)...) } -// LogWarningCtx logs a warning message with context and additional fields +// LogWarningCtx writes a warning-level log with context. func LogWarningCtx(ctx context.Context, message string, fields map[string]interface{}) { - msg := formatLogMessage(ctx, message, fields) - LogWarning(msg) + slog.WarnContext(ctx, message, mapToAttrs(fields)...) + logToGCP(slog.LevelWarn, message, mapToAttrs(fields)...) } -// LogErrorCtx logs an error message with context and additional fields +// LogErrorCtx writes an error-level log with context and an optional error. func LogErrorCtx(ctx context.Context, message string, err error, fields map[string]interface{}) { - if fields == nil { - fields = make(map[string]interface{}) - } + attrs := mapToAttrs(fields) if err != nil { - fields["error"] = err.Error() + attrs = append(attrs, slog.String("error", err.Error())) } - msg := formatLogMessage(ctx, message, fields) - LogError(msg) + slog.ErrorContext(ctx, message, attrs...) + logToGCP(slog.LevelError, message, attrs...) } -// LogWebhookOperation logs webhook-related operations +// LogWebhookOperation logs webhook-related operations. func LogWebhookOperation(ctx context.Context, operation string, message string, err error, fields ...map[string]interface{}) { allFields := make(map[string]interface{}) allFields["operation"] = operation - if len(fields) > 0 && fields[0] != nil { for k, v := range fields[0] { allFields[k] = v } } - if err != nil { LogErrorCtx(ctx, message, err, allFields) } else { @@ -171,7 +242,7 @@ func LogWebhookOperation(ctx context.Context, operation string, message string, } } -// LogFileOperation logs file-related operations +// LogFileOperation logs file-related operations. func LogFileOperation(ctx context.Context, operation string, sourcePath string, targetRepo string, message string, err error, fields ...map[string]interface{}) { allFields := make(map[string]interface{}) allFields["operation"] = operation @@ -179,13 +250,11 @@ func LogFileOperation(ctx context.Context, operation string, sourcePath string, if targetRepo != "" { allFields["target_repo"] = targetRepo } - if len(fields) > 0 && fields[0] != nil { for k, v := range fields[0] { allFields[k] = v } } - if err != nil { LogErrorCtx(ctx, message, err, allFields) } else { @@ -193,35 +262,43 @@ func LogFileOperation(ctx context.Context, operation string, sourcePath string, } } -// LogAndReturnError logs an error and returns +// LogAndReturnError logs an error and returns (convenience for early-return error paths). func LogAndReturnError(ctx context.Context, operation string, message string, err error) { LogErrorCtx(ctx, message, err, map[string]interface{}{ "operation": operation, }) } -// formatLogMessage formats a log message with context and fields -func formatLogMessage(ctx context.Context, message string, fields map[string]interface{}) string { +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Helpers +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +// mapToAttrs converts a map[string]interface{} to slog key-value pairs. +func mapToAttrs(fields map[string]interface{}) []any { if len(fields) == 0 { - return message + return nil } - - // Convert fields to JSON for structured logging - fieldsJSON, err := json.Marshal(fields) - if err != nil { - return fmt.Sprintf("%s | fields_error=%v", message, err) + attrs := make([]any, 0, len(fields)*2) + for k, v := range fields { + attrs = append(attrs, k, v) } - - return fmt.Sprintf("%s | %s", message, string(fieldsJSON)) + return attrs } -// WithRequestID adds a request ID to the context and returns both the context and the ID +// WithRequestID adds a request ID to the context and returns both the context and the ID. func WithRequestID(r *http.Request) (context.Context, string) { - // Generate a simple request ID requestID := fmt.Sprintf("%d", time.Now().UnixNano()) - - // Add to context using typed key to avoid collisions ctx := context.WithValue(r.Context(), requestIDKey, requestID) - return ctx, requestID } + +func isDebugEnabled() bool { + if strings.EqualFold(os.Getenv("LOG_LEVEL"), "debug") { + return true + } + return strings.EqualFold(os.Getenv("COPIER_DEBUG"), "true") +} + +func isCloudLoggingDisabled() bool { + return strings.EqualFold(os.Getenv("COPIER_DISABLE_CLOUD_LOGGING"), "true") +} diff --git a/services/logger_test.go b/services/logger_test.go index 19f148f..c87c660 100644 --- a/services/logger_test.go +++ b/services/logger_test.go @@ -3,14 +3,36 @@ package services import ( "bytes" "context" + "encoding/json" "fmt" - "log" + "log/slog" "net/http/httptest" "os" - "strings" "testing" ) +// setupTestLogger creates a JSON slog logger writing to the given buffer +// and sets it as the default. Returns a cleanup function. +func setupTestLogger(t *testing.T, buf *bytes.Buffer) func() { + t.Helper() + old := slog.Default() + handler := slog.NewJSONHandler(buf, &slog.HandlerOptions{ + Level: slog.LevelDebug, // capture all levels in tests + }) + slog.SetDefault(slog.New(handler)) + return func() { slog.SetDefault(old) } +} + +// parseLine unmarshals the first JSON object from a buffer. +func parseLine(t *testing.T, buf *bytes.Buffer) map[string]interface{} { + t.Helper() + var m map[string]interface{} + if err := json.Unmarshal(buf.Bytes(), &m); err != nil { + t.Fatalf("failed to parse JSON log line: %v\nbuf: %s", err, buf.String()) + } + return m +} + func TestLogDebug(t *testing.T) { tests := []struct { name string @@ -44,34 +66,36 @@ func TestLogDebug(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Set environment variables if tt.logLevel != "" { - os.Setenv("LOG_LEVEL", tt.logLevel) - defer os.Unsetenv("LOG_LEVEL") + _ = os.Setenv("LOG_LEVEL", tt.logLevel) + defer func() { _ = os.Unsetenv("LOG_LEVEL") }() } if tt.copierDebug != "" { - os.Setenv("COPIER_DEBUG", tt.copierDebug) - defer os.Unsetenv("COPIER_DEBUG") + _ = os.Setenv("COPIER_DEBUG", tt.copierDebug) + defer func() { _ = os.Unsetenv("COPIER_DEBUG") }() } - // Capture log output var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() LogDebug(tt.message) - output := buf.String() if tt.shouldLog { - if !strings.Contains(output, "[DEBUG]") { - t.Error("Expected [DEBUG] prefix in output") + if buf.Len() == 0 { + t.Error("Expected log output but got none") + return } - if !strings.Contains(output, tt.message) { - t.Errorf("Expected message %q in output", tt.message) + m := parseLine(t, &buf) + if m["msg"] != tt.message { + t.Errorf("Expected message %q, got %q", tt.message, m["msg"]) + } + if m["level"] != "DEBUG" { + t.Errorf("Expected level DEBUG, got %q", m["level"]) } } else { - if output != "" { - t.Errorf("Expected no output, got: %s", output) + if buf.Len() != 0 { + t.Errorf("Expected no output, got: %s", buf.String()) } } }) @@ -80,143 +104,157 @@ func TestLogDebug(t *testing.T) { func TestLogInfo(t *testing.T) { var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() - message := "test info message" - LogInfo(message) + LogInfo("test info message") - output := buf.String() - if !strings.Contains(output, "[INFO]") { - t.Error("Expected [INFO] prefix in output") + m := parseLine(t, &buf) + if m["msg"] != "test info message" { + t.Errorf("Expected message %q, got %q", "test info message", m["msg"]) } - if !strings.Contains(output, message) { - t.Errorf("Expected message %q in output", message) + if m["level"] != "INFO" { + t.Errorf("Expected level INFO, got %q", m["level"]) + } +} + +func TestLogInfoWithAttrs(t *testing.T) { + var buf bytes.Buffer + cleanup := setupTestLogger(t, &buf) + defer cleanup() + + LogInfo("server started", "port", 8080, "env", "prod") + + m := parseLine(t, &buf) + if m["msg"] != "server started" { + t.Errorf("Expected message %q, got %q", "server started", m["msg"]) + } + if m["port"] != float64(8080) { // JSON unmarshals numbers as float64 + t.Errorf("Expected port=8080, got %v", m["port"]) + } + if m["env"] != "prod" { + t.Errorf("Expected env=prod, got %v", m["env"]) } } func TestLogWarning(t *testing.T) { var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() - message := "test warning message" - LogWarning(message) + LogWarning("test warning message") - output := buf.String() - if !strings.Contains(output, "[WARN]") { - t.Error("Expected [WARN] prefix in output") + m := parseLine(t, &buf) + if m["msg"] != "test warning message" { + t.Errorf("Expected message %q, got %q", "test warning message", m["msg"]) } - if !strings.Contains(output, message) { - t.Errorf("Expected message %q in output", message) + if m["level"] != "WARN" { + t.Errorf("Expected level WARN, got %q", m["level"]) } } func TestLogError(t *testing.T) { var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() - message := "test error message" - LogError(message) + LogError("test error message") - output := buf.String() - if !strings.Contains(output, "[ERROR]") { - t.Error("Expected [ERROR] prefix in output") + m := parseLine(t, &buf) + if m["msg"] != "test error message" { + t.Errorf("Expected message %q, got %q", "test error message", m["msg"]) } - if !strings.Contains(output, message) { - t.Errorf("Expected message %q in output", message) + if m["level"] != "ERROR" { + t.Errorf("Expected level ERROR, got %q", m["level"]) } } func TestLogCritical(t *testing.T) { var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() - message := "test critical message" - LogCritical(message) + LogCritical("test critical message") - output := buf.String() - if !strings.Contains(output, "[CRITICAL]") { - t.Error("Expected [CRITICAL] prefix in output") + m := parseLine(t, &buf) + if m["msg"] != "test critical message" { + t.Errorf("Expected message %q, got %q", "test critical message", m["msg"]) } - if !strings.Contains(output, message) { - t.Errorf("Expected message %q in output", message) + // With default slog handler, custom level 12 shows as ERROR+4 + level, ok := m["level"].(string) + if !ok || level != "ERROR+4" { + t.Errorf("Expected level ERROR+4 (critical), got %q", m["level"]) } } func TestLogInfoCtx(t *testing.T) { var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() ctx := context.Background() - message := "test context message" fields := map[string]interface{}{ "key1": "value1", - "key2": 123, + "key2": float64(123), } - LogInfoCtx(ctx, message, fields) + LogInfoCtx(ctx, "test context message", fields) - output := buf.String() - if !strings.Contains(output, message) { - t.Errorf("Expected message %q in output", message) + m := parseLine(t, &buf) + if m["msg"] != "test context message" { + t.Errorf("Expected message %q, got %q", "test context message", m["msg"]) } - if !strings.Contains(output, "key1") { - t.Error("Expected field key1 in output") + if m["key1"] != "value1" { + t.Errorf("Expected key1=value1, got %v", m["key1"]) } - if !strings.Contains(output, "value1") { - t.Error("Expected field value1 in output") + if m["key2"] != float64(123) { + t.Errorf("Expected key2=123, got %v", m["key2"]) } } func TestLogWarningCtx(t *testing.T) { var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() ctx := context.Background() - message := "test warning context" fields := map[string]interface{}{ "warning_type": "test", } - LogWarningCtx(ctx, message, fields) + LogWarningCtx(ctx, "test warning context", fields) - output := buf.String() - if !strings.Contains(output, message) { - t.Errorf("Expected message %q in output", message) + m := parseLine(t, &buf) + if m["msg"] != "test warning context" { + t.Errorf("Expected message %q, got %q", "test warning context", m["msg"]) } - if !strings.Contains(output, "warning_type") { - t.Error("Expected field warning_type in output") + if m["warning_type"] != "test" { + t.Errorf("Expected warning_type=test, got %v", m["warning_type"]) } } func TestLogErrorCtx(t *testing.T) { var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() ctx := context.Background() - message := "test error context" err := fmt.Errorf("test error") fields := map[string]interface{}{ - "error_code": 500, + "error_code": float64(500), } - LogErrorCtx(ctx, message, err, fields) + LogErrorCtx(ctx, "test error context", err, fields) - output := buf.String() - if !strings.Contains(output, message) { - t.Errorf("Expected message %q in output", message) + m := parseLine(t, &buf) + if m["msg"] != "test error context" { + t.Errorf("Expected message %q, got %q", "test error context", m["msg"]) } - if !strings.Contains(output, "test error") { - t.Error("Expected error message in output") + if m["error"] != "test error" { + t.Errorf("Expected error=test error, got %v", m["error"]) } - if !strings.Contains(output, "error_code") { - t.Error("Expected field error_code in output") + if m["error_code"] != float64(500) { + t.Errorf("Expected error_code=500, got %v", m["error_code"]) } } @@ -233,35 +271,35 @@ func TestLogWebhookOperation(t *testing.T) { operation: "webhook_received", message: "webhook processed", err: nil, - wantLevel: "[INFO]", + wantLevel: "INFO", }, { name: "failed operation", operation: "webhook_parse", message: "failed to parse webhook", err: fmt.Errorf("parse error"), - wantLevel: "[ERROR]", + wantLevel: "ERROR", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() ctx := context.Background() LogWebhookOperation(ctx, tt.operation, tt.message, tt.err) - output := buf.String() - if !strings.Contains(output, tt.wantLevel) { - t.Errorf("Expected %s level in output", tt.wantLevel) + m := parseLine(t, &buf) + if m["level"] != tt.wantLevel { + t.Errorf("Expected level %s, got %q", tt.wantLevel, m["level"]) } - if !strings.Contains(output, tt.message) { - t.Errorf("Expected message %q in output", tt.message) + if m["msg"] != tt.message { + t.Errorf("Expected message %q, got %q", tt.message, m["msg"]) } - if !strings.Contains(output, tt.operation) { - t.Errorf("Expected operation %q in output", tt.operation) + if m["operation"] != tt.operation { + t.Errorf("Expected operation %q, got %v", tt.operation, m["operation"]) } }) } @@ -269,21 +307,21 @@ func TestLogWebhookOperation(t *testing.T) { func TestLogFileOperation(t *testing.T) { var buf bytes.Buffer - log.SetOutput(&buf) - defer log.SetOutput(os.Stderr) + cleanup := setupTestLogger(t, &buf) + defer cleanup() ctx := context.Background() LogFileOperation(ctx, "copy", "source/file.go", "target/repo", "file copied", nil) - output := buf.String() - if !strings.Contains(output, "copy") { - t.Error("Expected operation 'copy' in output") + m := parseLine(t, &buf) + if m["operation"] != "copy" { + t.Errorf("Expected operation=copy, got %v", m["operation"]) } - if !strings.Contains(output, "source/file.go") { - t.Error("Expected source path in output") + if m["source_path"] != "source/file.go" { + t.Errorf("Expected source_path=source/file.go, got %v", m["source_path"]) } - if !strings.Contains(output, "target/repo") { - t.Error("Expected target repo in output") + if m["target_repo"] != "target/repo" { + t.Errorf("Expected target_repo=target/repo, got %v", m["target_repo"]) } } @@ -296,58 +334,113 @@ func TestWithRequestID(t *testing.T) { t.Error("Expected non-empty request ID") } - // Check that request ID is in context using the typed key ctxValue := ctx.Value(requestIDKey) if ctxValue == nil { t.Error("Expected request_id in context") } - if ctxValue.(string) != requestID { t.Error("Context request_id doesn't match returned request ID") } } -func TestFormatLogMessage(t *testing.T) { +func TestMapToAttrs(t *testing.T) { tests := []struct { - name string - message string - fields map[string]interface{} - want []string + name string + fields map[string]interface{} + want int // expected number of resulting attrs (key-value pairs) }{ { - name: "no fields", - message: "test message", - fields: nil, - want: []string{"test message"}, + name: "nil fields", + fields: nil, + want: 0, }, { - name: "with fields", - message: "test message", + name: "empty fields", + fields: map[string]interface{}{}, + want: 0, + }, + { + name: "with fields", fields: map[string]interface{}{ "key1": "value1", "key2": 123, }, - want: []string{"test message", "key1", "value1"}, - }, - { - name: "empty fields", - message: "test message", - fields: map[string]interface{}{}, - want: []string{"test message"}, + want: 4, // 2 key-value pairs = 4 elements }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - result := formatLogMessage(ctx, tt.message, tt.fields) + result := mapToAttrs(tt.fields) + if len(result) != tt.want { + t.Errorf("mapToAttrs() returned %d elements, want %d", len(result), tt.want) + } + }) + } +} - for _, want := range tt.want { - if !strings.Contains(result, want) { - t.Errorf("formatLogMessage() missing %q in result: %s", want, result) +func TestInitializeLoggerSeverityMapping(t *testing.T) { + // Enable debug so LogDebug actually emits + t.Setenv("COPIER_DEBUG", "true") + + // Test that InitializeLogger sets up a handler that maps levels to severity strings + var buf bytes.Buffer + opts := &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { + if a.Key == slog.LevelKey { + a.Key = "severity" + lvl := a.Value.Any().(slog.Level) + switch { + case lvl >= LevelCritical: + a.Value = slog.StringValue("CRITICAL") + case lvl >= slog.LevelError: + a.Value = slog.StringValue("ERROR") + case lvl >= slog.LevelWarn: + a.Value = slog.StringValue("WARNING") + case lvl >= slog.LevelInfo: + a.Value = slog.StringValue("INFO") + default: + a.Value = slog.StringValue("DEBUG") } } - }) + if a.Key == slog.MessageKey { + a.Key = "message" + } + return a + }, + } + + handler := slog.NewJSONHandler(&buf, opts) + old := slog.Default() + slog.SetDefault(slog.New(handler)) + defer slog.SetDefault(old) + + tests := []struct { + logFunc func() + wantSeverity string + }{ + {func() { LogDebug("d") }, "DEBUG"}, + {func() { LogInfo("i") }, "INFO"}, + {func() { LogWarning("w") }, "WARNING"}, + {func() { LogError("e") }, "ERROR"}, + {func() { LogCritical("c") }, "CRITICAL"}, + } + + for _, tt := range tests { + buf.Reset() + tt.logFunc() + + var m map[string]interface{} + if err := json.Unmarshal(buf.Bytes(), &m); err != nil { + t.Fatalf("failed to parse JSON: %v", err) + } + if m["severity"] != tt.wantSeverity { + t.Errorf("Expected severity=%s, got %v", tt.wantSeverity, m["severity"]) + } + if m["message"] == nil { + t.Error("Expected 'message' key in JSON output") + } } } @@ -368,10 +461,8 @@ func TestIsDebugEnabled(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - os.Setenv("LOG_LEVEL", tt.logLevel) - os.Setenv("COPIER_DEBUG", tt.copierDebug) - defer os.Unsetenv("LOG_LEVEL") - defer os.Unsetenv("COPIER_DEBUG") + t.Setenv("LOG_LEVEL", tt.logLevel) + t.Setenv("COPIER_DEBUG", tt.copierDebug) got := isDebugEnabled() if got != tt.want { @@ -395,8 +486,7 @@ func TestIsCloudLoggingDisabled(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - os.Setenv("COPIER_DISABLE_CLOUD_LOGGING", tt.value) - defer os.Unsetenv("COPIER_DISABLE_CLOUD_LOGGING") + t.Setenv("COPIER_DISABLE_CLOUD_LOGGING", tt.value) got := isCloudLoggingDisabled() if got != tt.want { diff --git a/services/main_config_loader.go b/services/main_config_loader.go index 274663b..9189bd9 100644 --- a/services/main_config_loader.go +++ b/services/main_config_loader.go @@ -2,11 +2,12 @@ package services import ( "context" + "errors" "fmt" "path/filepath" "strings" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "gopkg.in/yaml.v3" "github.com/grove-platform/github-copier/configs" @@ -14,15 +15,11 @@ import ( ) // DefaultMainConfigLoader implements the ConfigLoader interface with main config support -type DefaultMainConfigLoader struct { - configLoader ConfigLoader -} +type DefaultMainConfigLoader struct{} // NewMainConfigLoader creates a new main config loader func NewMainConfigLoader() ConfigLoader { - return &DefaultMainConfigLoader{ - configLoader: NewConfigLoader(), - } + return &DefaultMainConfigLoader{} } // LoadConfig implements the ConfigLoader interface @@ -47,10 +44,7 @@ func (mcl *DefaultMainConfigLoader) LoadMainConfig(ctx context.Context, config * var err error // Determine which config file to load - configFile := config.ConfigFile - if config.MainConfigFile != "" { - configFile = config.MainConfigFile - } + configFile := config.EffectiveConfigFile() // Try to load from local file first (for testing) content, err = loadLocalConfigFile(configFile) @@ -62,6 +56,10 @@ func (mcl *DefaultMainConfigLoader) LoadMainConfig(ctx context.Context, config * // Fall back to fetching from repository content, err = retrieveConfigFileContent(ctx, configFile, config) if err != nil { + // Check if this is an authentication error and make it more prominent + if errors.Is(err, ErrAuthentication) { + return nil, fmt.Errorf("%w: unable to retrieve main config file. The GitHub App private key (PEM) may be invalid or expired. Please check the CODE_COPIER_PEM secret in GCP Secret Manager and redeploy the service. Original error: %v", ErrAuthentication, err) + } return nil, fmt.Errorf("failed to retrieve main config file: %w", err) } } @@ -72,19 +70,19 @@ func (mcl *DefaultMainConfigLoader) LoadMainConfig(ctx context.Context, config * // LoadMainConfigFromContent loads main configuration from a string and resolves references func (mcl *DefaultMainConfigLoader) LoadMainConfigFromContent(ctx context.Context, content string, config *configs.Config) (*types.YAMLConfig, error) { if content == "" { - return nil, fmt.Errorf("main config file is empty") + return nil, fmt.Errorf("%w: main config file is empty", ErrConfigLoad) } // Parse as MainConfig var mainConfig types.MainConfig err := yaml.Unmarshal([]byte(content), &mainConfig) if err != nil { - return nil, fmt.Errorf("failed to parse main config: %w", err) + return nil, fmt.Errorf("%w: failed to parse main config: %v", ErrConfigLoad, err) } // Validate that workflow_configs is present if len(mainConfig.WorkflowConfigs) == 0 { - return nil, fmt.Errorf("main config must have at least one workflow_config entry") + return nil, fmt.Errorf("%w: main config must have at least one workflow_config entry", ErrConfigValidation) } // Set defaults for main config @@ -92,7 +90,7 @@ func (mcl *DefaultMainConfigLoader) LoadMainConfigFromContent(ctx context.Contex // Validate main config if err := mainConfig.Validate(); err != nil { - return nil, fmt.Errorf("main config validation failed: %w", err) + return nil, fmt.Errorf("%w: main config: %v", ErrConfigValidation, err) } LogInfoCtx(ctx, "loaded main config with workflow references", map[string]interface{}{ @@ -174,16 +172,53 @@ func (mcl *DefaultMainConfigLoader) resolveWorkflowReferences(ctx context.Contex // Validate merged config if err := mergedConfig.Validate(); err != nil { - return nil, fmt.Errorf("merged config validation failed: %w", err) + return nil, fmt.Errorf("%w: merged config: %v", ErrConfigValidation, err) } LogInfoCtx(ctx, "successfully resolved all workflow references", map[string]interface{}{ "total_workflows": len(mergedConfig.Workflows), }) + // Warn when multiple workflows target the same repo/branch with different strategies. + warnConflictingStrategies(ctx, mergedConfig.Workflows) + return mergedConfig, nil } +// warnConflictingStrategies logs a warning when workflows share a destination +// repo and branch but use different commit strategies (e.g., "direct" vs "pull_request"). +// This is informational โ€” the app handles this correctly by creating separate +// operations per strategy โ€” but it may surprise operators who expect a single PR. +func warnConflictingStrategies(ctx context.Context, workflows []types.Workflow) { + // Group workflow names by (repo, branch, strategy) + type destKey struct{ repo, branch string } + strategies := make(map[destKey]map[string][]string) // destKey -> strategy -> []workflowName + + for _, wf := range workflows { + if wf.Destination.Repo == "" { + continue + } + dk := destKey{repo: wf.Destination.Repo, branch: wf.Destination.Branch} + if dk.branch == "" { + dk.branch = "main" + } + strategy := getCommitStrategyType(wf) + if strategies[dk] == nil { + strategies[dk] = make(map[string][]string) + } + strategies[dk][strategy] = append(strategies[dk][strategy], wf.Name) + } + + for dk, stratMap := range strategies { + if len(stratMap) > 1 { + LogWarningCtx(ctx, "workflows targeting the same repo use different commit strategies; files will be written separately per strategy", map[string]interface{}{ + "destination": dk.repo + ":" + dk.branch, + "strategies": stratMap, + }) + } + } +} + // loadWorkflowConfig loads a workflow config based on the reference type func (mcl *DefaultMainConfigLoader) loadWorkflowConfig(ctx context.Context, ref *types.WorkflowConfigRef, config *configs.Config) (*types.WorkflowConfig, error) { switch ref.Source { @@ -199,7 +234,7 @@ func (mcl *DefaultMainConfigLoader) loadWorkflowConfig(ctx context.Context, ref case "repo": // Remote file in a different repo - return mcl.loadRemoteWorkflowConfig(ctx, ref) + return mcl.loadRemoteWorkflowConfig(ctx, config, ref) default: return nil, fmt.Errorf("unsupported workflow config source: %s", ref.Source) @@ -221,7 +256,7 @@ func (mcl *DefaultMainConfigLoader) loadLocalWorkflowConfig(ctx context.Context, // Resolve $ref references baseRepo := fmt.Sprintf("%s/%s", config.ConfigRepoOwner, config.ConfigRepoName) - if err := mcl.resolveWorkflowFieldReferences(ctx, workflowConfig, baseRepo, config.ConfigRepoBranch, ref.Path); err != nil { + if err := mcl.resolveWorkflowFieldReferences(ctx, config, workflowConfig, baseRepo, config.ConfigRepoBranch, ref.Path); err != nil { return nil, err } @@ -229,7 +264,7 @@ func (mcl *DefaultMainConfigLoader) loadLocalWorkflowConfig(ctx context.Context, } // Fall back to fetching from config repo - client, err := GetRestClientForOrg(config.ConfigRepoOwner) + client, err := GetRestClientForOrg(ctx, config, config.ConfigRepoOwner) if err != nil { return nil, fmt.Errorf("failed to get GitHub client for org %s: %w", config.ConfigRepoOwner, err) } @@ -247,7 +282,7 @@ func (mcl *DefaultMainConfigLoader) loadLocalWorkflowConfig(ctx context.Context, return nil, fmt.Errorf("failed to get workflow config file: %w", err) } if fileContent == nil { - return nil, fmt.Errorf("workflow config file content is nil for path: %s", ref.Path) + return nil, fmt.Errorf("%w: workflow config at path: %s", ErrContentNil, ref.Path) } content, err = fileContent.GetContent() @@ -262,7 +297,7 @@ func (mcl *DefaultMainConfigLoader) loadLocalWorkflowConfig(ctx context.Context, // Resolve $ref references baseRepo := fmt.Sprintf("%s/%s", config.ConfigRepoOwner, config.ConfigRepoName) - if err := mcl.resolveWorkflowFieldReferences(ctx, workflowConfig, baseRepo, config.ConfigRepoBranch, ref.Path); err != nil { + if err := mcl.resolveWorkflowFieldReferences(ctx, config, workflowConfig, baseRepo, config.ConfigRepoBranch, ref.Path); err != nil { return nil, err } @@ -270,7 +305,7 @@ func (mcl *DefaultMainConfigLoader) loadLocalWorkflowConfig(ctx context.Context, } // loadRemoteWorkflowConfig loads a workflow config from a different repo -func (mcl *DefaultMainConfigLoader) loadRemoteWorkflowConfig(ctx context.Context, ref *types.WorkflowConfigRef) (*types.WorkflowConfig, error) { +func (mcl *DefaultMainConfigLoader) loadRemoteWorkflowConfig(ctx context.Context, config *configs.Config, ref *types.WorkflowConfigRef) (*types.WorkflowConfig, error) { // Parse repo owner and name parts := strings.Split(ref.Repo, "/") if len(parts) != 2 { @@ -280,7 +315,7 @@ func (mcl *DefaultMainConfigLoader) loadRemoteWorkflowConfig(ctx context.Context repo := parts[1] // Get GitHub client for the repo's org - client, err := GetRestClientForOrg(owner) + client, err := GetRestClientForOrg(ctx, config, owner) if err != nil { return nil, fmt.Errorf("failed to get GitHub client for org %s: %w", owner, err) } @@ -299,7 +334,7 @@ func (mcl *DefaultMainConfigLoader) loadRemoteWorkflowConfig(ctx context.Context return nil, fmt.Errorf("failed to get workflow config file from %s: %w", ref.Repo, err) } if fileContent == nil { - return nil, fmt.Errorf("workflow config file content is nil for path: %s in repo %s", ref.Path, ref.Repo) + return nil, fmt.Errorf("%w: workflow config at path: %s in repo %s", ErrContentNil, ref.Path, ref.Repo) } content, err := fileContent.GetContent() @@ -323,7 +358,7 @@ func (mcl *DefaultMainConfigLoader) loadRemoteWorkflowConfig(ctx context.Context workflowConfig.SourceBranch = ref.Branch // Resolve $ref references - if err := mcl.resolveWorkflowFieldReferences(ctx, workflowConfig, ref.Repo, ref.Branch, ref.Path); err != nil { + if err := mcl.resolveWorkflowFieldReferences(ctx, config, workflowConfig, ref.Repo, ref.Branch, ref.Path); err != nil { return nil, err } @@ -333,20 +368,20 @@ func (mcl *DefaultMainConfigLoader) loadRemoteWorkflowConfig(ctx context.Context // parseWorkflowConfig parses a workflow config from content func (mcl *DefaultMainConfigLoader) parseWorkflowConfig(content string, filename string) (*types.WorkflowConfig, error) { if content == "" { - return nil, fmt.Errorf("workflow config file is empty") + return nil, fmt.Errorf("%w: workflow config file is empty", ErrConfigLoad) } var workflowConfig types.WorkflowConfig err := yaml.Unmarshal([]byte(content), &workflowConfig) if err != nil { - return nil, fmt.Errorf("failed to parse workflow config file %s: %w", filename, err) + return nil, fmt.Errorf("%w: failed to parse workflow config file %s: %v", ErrConfigLoad, filename, err) } return &workflowConfig, nil } // resolveWorkflowFieldReferences resolves all $ref references in workflow fields (transformations, exclude, commit_strategy) -func (mcl *DefaultMainConfigLoader) resolveWorkflowFieldReferences(ctx context.Context, workflowConfig *types.WorkflowConfig, baseRepo string, baseBranch string, basePath string) error { +func (mcl *DefaultMainConfigLoader) resolveWorkflowFieldReferences(ctx context.Context, config *configs.Config, workflowConfig *types.WorkflowConfig, baseRepo string, baseBranch string, basePath string) error { for i := range workflowConfig.Workflows { workflow := &workflowConfig.Workflows[i] @@ -357,7 +392,7 @@ func (mcl *DefaultMainConfigLoader) resolveWorkflowFieldReferences(ctx context.C "ref": workflow.TransformationsRef, }) - content, err := mcl.resolveReference(ctx, workflow.TransformationsRef, baseRepo, baseBranch, basePath) + content, err := mcl.resolveReference(ctx, config, workflow.TransformationsRef, baseRepo, baseBranch, basePath) if err != nil { return fmt.Errorf("failed to resolve transformations $ref for workflow %s: %w", workflow.Name, err) } @@ -377,7 +412,7 @@ func (mcl *DefaultMainConfigLoader) resolveWorkflowFieldReferences(ctx context.C "ref": workflow.ExcludeRef, }) - content, err := mcl.resolveReference(ctx, workflow.ExcludeRef, baseRepo, baseBranch, basePath) + content, err := mcl.resolveReference(ctx, config, workflow.ExcludeRef, baseRepo, baseBranch, basePath) if err != nil { return fmt.Errorf("failed to resolve exclude $ref for workflow %s: %w", workflow.Name, err) } @@ -397,7 +432,7 @@ func (mcl *DefaultMainConfigLoader) resolveWorkflowFieldReferences(ctx context.C "ref": workflow.CommitStrategyRef, }) - content, err := mcl.resolveReference(ctx, workflow.CommitStrategyRef, baseRepo, baseBranch, basePath) + content, err := mcl.resolveReference(ctx, config, workflow.CommitStrategyRef, baseRepo, baseBranch, basePath) if err != nil { return fmt.Errorf("failed to resolve commit_strategy $ref for workflow %s: %w", workflow.Name, err) } @@ -416,7 +451,7 @@ func (mcl *DefaultMainConfigLoader) resolveWorkflowFieldReferences(ctx context.C // resolveReference resolves a $ref reference to actual content // This supports references in transformations, commit strategies, etc. -func (mcl *DefaultMainConfigLoader) resolveReference(ctx context.Context, ref string, baseRepo string, baseBranch string, basePath string) (string, error) { +func (mcl *DefaultMainConfigLoader) resolveReference(ctx context.Context, config *configs.Config, ref string, baseRepo string, baseBranch string, basePath string) (string, error) { // Parse reference format // Supports: // - Relative paths: "strategies/pr-strategy.yaml" @@ -424,15 +459,15 @@ func (mcl *DefaultMainConfigLoader) resolveReference(ctx context.Context, ref st if strings.HasPrefix(ref, "repo://") { // Remote repo reference - return mcl.resolveRemoteReference(ctx, ref) + return mcl.resolveRemoteReference(ctx, config, ref) } // Relative path reference - return mcl.resolveRelativeReference(ctx, ref, baseRepo, baseBranch, basePath) + return mcl.resolveRelativeReference(ctx, config, ref, baseRepo, baseBranch, basePath) } // resolveRemoteReference resolves a repo:// reference -func (mcl *DefaultMainConfigLoader) resolveRemoteReference(ctx context.Context, ref string) (string, error) { +func (mcl *DefaultMainConfigLoader) resolveRemoteReference(ctx context.Context, config *configs.Config, ref string) (string, error) { // Parse: repo://owner/repo/path/to/file.yaml@branch ref = strings.TrimPrefix(ref, "repo://") @@ -455,7 +490,7 @@ func (mcl *DefaultMainConfigLoader) resolveRemoteReference(ctx context.Context, filePath := pathParts[2] // Fetch file content - client, err := GetRestClientForOrg(owner) + client, err := GetRestClientForOrg(ctx, config, owner) if err != nil { return "", fmt.Errorf("failed to get GitHub client for org %s: %w", owner, err) } @@ -473,14 +508,14 @@ func (mcl *DefaultMainConfigLoader) resolveRemoteReference(ctx context.Context, return "", fmt.Errorf("failed to get referenced file: %w", err) } if fileContent == nil { - return "", fmt.Errorf("referenced file content is nil for path: %s", filePath) + return "", fmt.Errorf("%w: referenced file at path: %s", ErrContentNil, filePath) } return fileContent.GetContent() } // resolveRelativeReference resolves a relative path reference -func (mcl *DefaultMainConfigLoader) resolveRelativeReference(ctx context.Context, ref string, baseRepo string, baseBranch string, basePath string) (string, error) { +func (mcl *DefaultMainConfigLoader) resolveRelativeReference(ctx context.Context, config *configs.Config, ref string, baseRepo string, baseBranch string, basePath string) (string, error) { // Resolve relative to base path baseDir := filepath.Dir(basePath) resolvedPath := filepath.Join(baseDir, ref) @@ -494,7 +529,7 @@ func (mcl *DefaultMainConfigLoader) resolveRelativeReference(ctx context.Context repo := parts[1] // Fetch file content - client, err := GetRestClientForOrg(owner) + client, err := GetRestClientForOrg(ctx, config, owner) if err != nil { return "", fmt.Errorf("failed to get GitHub client for org %s: %w", owner, err) } @@ -512,7 +547,7 @@ func (mcl *DefaultMainConfigLoader) resolveRelativeReference(ctx context.Context return "", fmt.Errorf("failed to get referenced file: %w", err) } if fileContent == nil { - return "", fmt.Errorf("referenced file content is nil for path: %s", resolvedPath) + return "", fmt.Errorf("%w: referenced file at path: %s", ErrContentNil, resolvedPath) } return fileContent.GetContent() diff --git a/services/pattern_matcher.go b/services/pattern_matcher.go index ee3feb0..ea30503 100644 --- a/services/pattern_matcher.go +++ b/services/pattern_matcher.go @@ -10,6 +10,10 @@ import ( "github.com/grove-platform/github-copier/types" ) +// unreplacedVarRe matches ${var} placeholders that were not substituted. +// Compiled once at package level to avoid repeated compilation. +var unreplacedVarRe = regexp.MustCompile(`\$\{([^}]+)\}`) + // PatternMatcher handles pattern matching for file paths type PatternMatcher interface { Match(filePath string, pattern types.SourcePattern) types.MatchResult @@ -99,7 +103,7 @@ func (pm *DefaultPatternMatcher) matchGlob(filePath, pattern string) types.Match func (pm *DefaultPatternMatcher) matchRegex(filePath, pattern string) types.MatchResult { re, err := regexp.Compile(pattern) if err != nil { - LogInfo(fmt.Sprintf("REGEX COMPILE ERROR: pattern=%s, error=%v", pattern, err)) + LogInfo("REGEX COMPILE ERROR", "pattern", pattern, "error", err) return types.NewMatchResult(false, nil) } @@ -107,7 +111,7 @@ func (pm *DefaultPatternMatcher) matchRegex(filePath, pattern string) types.Matc if match == nil { // Log server file pattern attempts for debugging if strings.Contains(pattern, "server/") && strings.Contains(filePath, "server/") { - LogInfo(fmt.Sprintf("REGEX NO MATCH: file=%s, pattern=%s", filePath, pattern)) + LogInfo("REGEX NO MATCH", "file", filePath, "pattern", pattern) } return types.NewMatchResult(false, nil) } @@ -122,7 +126,7 @@ func (pm *DefaultPatternMatcher) matchRegex(filePath, pattern string) types.Matc // Log server file matches for debugging if strings.Contains(pattern, "server/") { - LogInfo(fmt.Sprintf("REGEX MATCH SUCCESS: file=%s, pattern=%s, variables=%v", filePath, pattern, variables)) + LogInfo("REGEX MATCH SUCCESS", "file", filePath, "pattern", pattern, "variables", variables) } return types.NewMatchResult(true, variables) @@ -169,8 +173,7 @@ func (pt *DefaultPathTransformer) Transform(sourcePath string, template string, // extractUnreplacedVars extracts variable names that weren't replaced func extractUnreplacedVars(s string) []string { var unreplaced []string - re := regexp.MustCompile(`\$\{([^}]+)\}`) - matches := re.FindAllStringSubmatch(s, -1) + matches := unreplacedVarRe.FindAllStringSubmatch(s, -1) for _, match := range matches { if len(match) > 1 { unreplaced = append(unreplaced, match[1]) diff --git a/services/pr_template_fetcher.go b/services/pr_template_fetcher.go index 2501056..47f6449 100644 --- a/services/pr_template_fetcher.go +++ b/services/pr_template_fetcher.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" ) // PRTemplateFetcher defines the interface for fetching PR templates from repositories @@ -51,14 +51,14 @@ func (f *DefaultPRTemplateFetcher) FetchPRTemplate(ctx context.Context, client * for _, path := range templatePaths { content, err := f.fetchFileContent(ctx, client, owner, repo, path, branch) if err == nil && content != "" { - LogInfo(fmt.Sprintf("Found PR template in %s/%s at %s", owner, repo, path)) + LogInfo("Found PR template", "owner", owner, "repo", repo, "path", path) return content, nil } // Continue to next location if not found } // No template found - LogDebug(fmt.Sprintf("No PR template found in %s/%s (checked %d locations)", owner, repo, len(templatePaths))) + LogDebug("No PR template found", "owner", owner, "repo", repo, "locations_checked", len(templatePaths)) return "", nil } @@ -101,4 +101,3 @@ func MergePRBodyWithTemplate(configuredBody, template string) string { // Merge: template first, then separator, then configured body return fmt.Sprintf("%s\n\n---\n\n%s", template, configuredBody) } - diff --git a/services/pr_template_fetcher_test.go b/services/pr_template_fetcher_test.go index ab4e24a..55bea32 100644 --- a/services/pr_template_fetcher_test.go +++ b/services/pr_template_fetcher_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/require" ) @@ -256,4 +256,3 @@ func TestPRTemplateFetcher_StopsAtFirstMatch(t *testing.T) { require.Equal(t, 0, info["GET https://api.github.com/repos/testowner/testrepo/contents/"+location]) } } - diff --git a/services/rate_limit.go b/services/rate_limit.go new file mode 100644 index 0000000..9d7b1b9 --- /dev/null +++ b/services/rate_limit.go @@ -0,0 +1,199 @@ +package services + +import ( + "context" + "net/http" + "strconv" + "sync" + "time" +) + +// RateLimitState holds the most recently observed GitHub API rate limit info. +// Updated atomically by rateLimitTransport on every API response. +type RateLimitState struct { + mu sync.RWMutex + remaining int + resetAt time.Time +} + +// Get returns the current rate limit state. +func (s *RateLimitState) Get() (remaining int, resetAt time.Time) { + s.mu.RLock() + defer s.mu.RUnlock() + return s.remaining, s.resetAt +} + +// update stores the latest rate limit values from response headers. +func (s *RateLimitState) update(remaining int, resetAt time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + s.remaining = remaining + s.resetAt = resetAt +} + +// GlobalRateLimitState is the shared rate limit state read by the health/metrics endpoints. +var GlobalRateLimitState = &RateLimitState{remaining: -1} // -1 = not yet observed + +// rateLimitTransport is an http.RoundTripper that: +// 1. Records rate limit headers from every GitHub API response. +// 2. On HTTP 403 (primary rate limit) or 429 (secondary/abuse rate limit), +// waits for the Retry-After / X-RateLimit-Reset period and retries once. +// 3. Respects context cancellation during the wait. +type rateLimitTransport struct { + base http.RoundTripper + metrics *MetricsCollector // optional, may be nil +} + +// newRateLimitTransport wraps a base transport with rate limit handling. +func newRateLimitTransport(base http.RoundTripper, metrics *MetricsCollector) *rateLimitTransport { + if base == nil { + base = http.DefaultTransport + } + return &rateLimitTransport{base: base, metrics: metrics} +} + +// RoundTrip implements http.RoundTripper. +func (t *rateLimitTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if t.metrics != nil { + t.metrics.RecordGitHubAPICall() + } + + resp, err := t.base.RoundTrip(req) + if err != nil { + if t.metrics != nil { + t.metrics.RecordGitHubAPIError() + } + return resp, err + } + + // Always record rate limit headers + t.recordRateLimit(resp) + + // Check for rate limiting (403 primary or 429 secondary/abuse) + if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests { + if isRateLimited(resp) { + waitDuration := retryAfterDuration(resp) + if waitDuration > 0 { + LogWarning("GitHub API rate limited, waiting before retry", + "status", resp.StatusCode, + "wait_seconds", waitDuration.Seconds(), + "url", req.URL.String(), + ) + + // Wait with context cancellation support + if err := waitWithContext(req.Context(), waitDuration); err != nil { + return resp, nil // context cancelled, return original response + } + + // Close the original response body before retrying + _ = resp.Body.Close() + + // Retry once + if t.metrics != nil { + t.metrics.RecordGitHubAPICall() + } + retryResp, retryErr := t.base.RoundTrip(req) + if retryErr != nil { + if t.metrics != nil { + t.metrics.RecordGitHubAPIError() + } + return retryResp, retryErr + } + t.recordRateLimit(retryResp) + return retryResp, nil + } + } + } + + if resp.StatusCode >= 400 && t.metrics != nil { + t.metrics.RecordGitHubAPIError() + } + + return resp, err +} + +// recordRateLimit extracts rate limit info from response headers and updates shared state. +func (t *rateLimitTransport) recordRateLimit(resp *http.Response) { + remaining := resp.Header.Get("X-RateLimit-Remaining") + reset := resp.Header.Get("X-RateLimit-Reset") + + if remaining == "" && reset == "" { + return + } + + rem, _ := strconv.Atoi(remaining) + resetUnix, _ := strconv.ParseInt(reset, 10, 64) + resetTime := time.Unix(resetUnix, 0) + + GlobalRateLimitState.update(rem, resetTime) +} + +// isRateLimited checks if a response indicates a rate limit condition. +func isRateLimited(resp *http.Response) bool { + // HTTP 429 is always a rate limit + if resp.StatusCode == http.StatusTooManyRequests { + return true + } + + // HTTP 403 with X-RateLimit-Remaining: 0 is a primary rate limit + if resp.StatusCode == http.StatusForbidden { + remaining := resp.Header.Get("X-RateLimit-Remaining") + if remaining == "0" { + return true + } + // Also check for Retry-After header (abuse/secondary rate limit) + if resp.Header.Get("Retry-After") != "" { + return true + } + } + + return false +} + +// retryAfterDuration determines how long to wait before retrying. +// It checks Retry-After header first, then falls back to X-RateLimit-Reset. +// Returns 0 if no retry info is available. Caps at 60 seconds. +func retryAfterDuration(resp *http.Response) time.Duration { + const maxWait = 60 * time.Second + + // Check Retry-After header (used by secondary/abuse rate limits) + if retryAfter := resp.Header.Get("Retry-After"); retryAfter != "" { + if seconds, err := strconv.Atoi(retryAfter); err == nil && seconds > 0 { + d := time.Duration(seconds) * time.Second + if d > maxWait { + return maxWait + } + return d + } + } + + // Fall back to X-RateLimit-Reset (Unix timestamp) + if reset := resp.Header.Get("X-RateLimit-Reset"); reset != "" { + if resetUnix, err := strconv.ParseInt(reset, 10, 64); err == nil { + resetTime := time.Unix(resetUnix, 0) + d := time.Until(resetTime) + if d <= 0 { + return 1 * time.Second // Already past, retry immediately with small buffer + } + if d > maxWait { + return maxWait + } + return d + } + } + + return 0 +} + +// waitWithContext sleeps for the given duration, returning early if ctx is cancelled. +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/services/rate_limit_test.go b/services/rate_limit_test.go new file mode 100644 index 0000000..ead08f9 --- /dev/null +++ b/services/rate_limit_test.go @@ -0,0 +1,388 @@ +package services + +import ( + "context" + "fmt" + "net/http" + "strconv" + "testing" + "time" +) + +// mockTransport is a configurable http.RoundTripper for testing. +type mockTransport struct { + handler func(req *http.Request) (*http.Response, error) + calls int +} + +func (m *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { + m.calls++ + return m.handler(req) +} + +func TestRateLimitTransport_RecordsRateLimitHeaders(t *testing.T) { + // Save and restore global state + origRemaining, origReset := GlobalRateLimitState.Get() + defer GlobalRateLimitState.update(origRemaining, origReset) + + resetTime := time.Now().Add(30 * time.Minute).Unix() + + mock := &mockTransport{ + handler: func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Header: http.Header{ + "X-Ratelimit-Remaining": []string{"4500"}, + "X-Ratelimit-Reset": []string{strconv.FormatInt(resetTime, 10)}, + }, + Body: http.NoBody, + }, nil + }, + } + + rt := newRateLimitTransport(mock, nil) + req, _ := http.NewRequest("GET", "https://api.github.com/repos", nil) + _, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip error: %v", err) + } + + remaining, reset := GlobalRateLimitState.Get() + if remaining != 4500 { + t.Errorf("expected remaining=4500, got %d", remaining) + } + if reset.Unix() != resetTime { + t.Errorf("expected resetAt=%d, got %d", resetTime, reset.Unix()) + } +} + +func TestRateLimitTransport_RetriesOn429(t *testing.T) { + callCount := 0 + mock := &mockTransport{ + handler: func(req *http.Request) (*http.Response, error) { + callCount++ + if callCount == 1 { + return &http.Response{ + StatusCode: 429, + Header: http.Header{ + "Retry-After": []string{"1"}, + }, + Body: http.NoBody, + }, nil + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{}, + Body: http.NoBody, + }, nil + }, + } + + rt := newRateLimitTransport(mock, nil) + req, _ := http.NewRequest("GET", "https://api.github.com/repos", nil) + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip error: %v", err) + } + + if resp.StatusCode != 200 { + t.Errorf("expected status 200 after retry, got %d", resp.StatusCode) + } + if callCount != 2 { + t.Errorf("expected 2 calls (original + retry), got %d", callCount) + } +} + +func TestRateLimitTransport_RetriesOn403WithRateLimitExhausted(t *testing.T) { + resetTime := time.Now().Add(1 * time.Second).Unix() + callCount := 0 + + mock := &mockTransport{ + handler: func(req *http.Request) (*http.Response, error) { + callCount++ + if callCount == 1 { + return &http.Response{ + StatusCode: 403, + Header: http.Header{ + "X-Ratelimit-Remaining": []string{"0"}, + "X-Ratelimit-Reset": []string{strconv.FormatInt(resetTime, 10)}, + }, + Body: http.NoBody, + }, nil + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{}, + Body: http.NoBody, + }, nil + }, + } + + rt := newRateLimitTransport(mock, nil) + req, _ := http.NewRequest("GET", "https://api.github.com/repos", nil) + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip error: %v", err) + } + + if resp.StatusCode != 200 { + t.Errorf("expected status 200 after retry, got %d", resp.StatusCode) + } + if callCount != 2 { + t.Errorf("expected 2 calls, got %d", callCount) + } +} + +func TestRateLimitTransport_NoRetryOnRegular403(t *testing.T) { + mock := &mockTransport{ + handler: func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 403, + Header: http.Header{ + "X-Ratelimit-Remaining": []string{"4999"}, // Not exhausted + }, + Body: http.NoBody, + }, nil + }, + } + + rt := newRateLimitTransport(mock, nil) + req, _ := http.NewRequest("GET", "https://api.github.com/repos", nil) + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip error: %v", err) + } + + if resp.StatusCode != 403 { + t.Errorf("expected status 403 (no retry), got %d", resp.StatusCode) + } + if mock.calls != 1 { + t.Errorf("expected 1 call (no retry), got %d", mock.calls) + } +} + +func TestRateLimitTransport_RespectsContextCancellation(t *testing.T) { + mock := &mockTransport{ + handler: func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 429, + Header: http.Header{ + "Retry-After": []string{"60"}, // 60 seconds + }, + Body: http.NoBody, + }, nil + }, + } + + rt := newRateLimitTransport(mock, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/repos", nil) + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip error: %v", err) + } + + // Should return the original 429 response since context cancelled during wait + if resp.StatusCode != 429 { + t.Errorf("expected original 429 response, got %d", resp.StatusCode) + } + // Should not have retried (only 1 call to mock) + if mock.calls != 1 { + t.Errorf("expected 1 call (no retry due to context), got %d", mock.calls) + } +} + +func TestRateLimitTransport_RecordsMetrics(t *testing.T) { + mc := NewMetricsCollector() + + mock := &mockTransport{ + handler: func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Header: http.Header{}, + Body: http.NoBody, + }, nil + }, + } + + rt := newRateLimitTransport(mock, mc) + req, _ := http.NewRequest("GET", "https://api.github.com/repos", nil) + _, _ = rt.RoundTrip(req) + + mc.mu.RLock() + defer mc.mu.RUnlock() + if mc.githubAPICalls != 1 { + t.Errorf("expected 1 API call recorded, got %d", mc.githubAPICalls) + } + if mc.githubAPIErrors != 0 { + t.Errorf("expected 0 API errors, got %d", mc.githubAPIErrors) + } +} + +func TestRateLimitTransport_RecordsErrorMetrics(t *testing.T) { + mc := NewMetricsCollector() + + mock := &mockTransport{ + handler: func(req *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("connection refused") + }, + } + + rt := newRateLimitTransport(mock, mc) + req, _ := http.NewRequest("GET", "https://api.github.com/repos", nil) + _, _ = rt.RoundTrip(req) + + mc.mu.RLock() + defer mc.mu.RUnlock() + if mc.githubAPICalls != 1 { + t.Errorf("expected 1 API call recorded, got %d", mc.githubAPICalls) + } + if mc.githubAPIErrors != 1 { + t.Errorf("expected 1 API error, got %d", mc.githubAPIErrors) + } +} + +func TestIsRateLimited(t *testing.T) { + tests := []struct { + name string + statusCode int + headers http.Header + want bool + }{ + { + name: "429 is always rate limited", + statusCode: 429, + headers: http.Header{}, + want: true, + }, + { + name: "403 with remaining 0", + statusCode: 403, + headers: http.Header{"X-Ratelimit-Remaining": []string{"0"}}, + want: true, + }, + { + name: "403 with Retry-After", + statusCode: 403, + headers: http.Header{"Retry-After": []string{"60"}}, + want: true, + }, + { + name: "403 with remaining > 0 (not rate limited)", + statusCode: 403, + headers: http.Header{"X-Ratelimit-Remaining": []string{"4999"}}, + want: false, + }, + { + name: "200 is never rate limited", + statusCode: 200, + headers: http.Header{}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp := &http.Response{StatusCode: tt.statusCode, Header: tt.headers} + if got := isRateLimited(resp); got != tt.want { + t.Errorf("isRateLimited() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRetryAfterDuration(t *testing.T) { + tests := []struct { + name string + headers http.Header + wantMin time.Duration + wantMax time.Duration + }{ + { + name: "Retry-After in seconds", + headers: http.Header{"Retry-After": []string{"5"}}, + wantMin: 5 * time.Second, + wantMax: 5 * time.Second, + }, + { + name: "Retry-After capped at 60s", + headers: http.Header{"Retry-After": []string{"120"}}, + wantMin: 60 * time.Second, + wantMax: 60 * time.Second, + }, + { + name: "X-RateLimit-Reset fallback", + headers: http.Header{ + "X-Ratelimit-Reset": []string{strconv.FormatInt(time.Now().Add(10*time.Second).Unix(), 10)}, + }, + wantMin: 8 * time.Second, // Allow some clock skew + wantMax: 12 * time.Second, // Allow some clock skew + }, + { + name: "no headers returns 0", + headers: http.Header{}, + wantMin: 0, + wantMax: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp := &http.Response{Header: tt.headers} + got := retryAfterDuration(resp) + if got < tt.wantMin || got > tt.wantMax { + t.Errorf("retryAfterDuration() = %v, want between %v and %v", got, tt.wantMin, tt.wantMax) + } + }) + } +} + +func TestRateLimitState_Concurrent(t *testing.T) { + state := &RateLimitState{remaining: -1} + + done := make(chan struct{}) + go func() { + for i := 0; i < 1000; i++ { + state.update(i, time.Now()) + } + close(done) + }() + + // Concurrent reads while writing + for i := 0; i < 1000; i++ { + state.Get() + } + + <-done // Wait for writer to finish +} + +func TestCurrentRateLimitInfo_DefaultState(t *testing.T) { + // Save and restore + origRemaining, origReset := GlobalRateLimitState.Get() + defer GlobalRateLimitState.update(origRemaining, origReset) + + GlobalRateLimitState.update(-1, time.Time{}) + info := currentRateLimitInfo() + if info.Remaining != -1 { + t.Errorf("expected remaining=-1 for default state, got %d", info.Remaining) + } +} + +func TestCurrentRateLimitInfo_WithData(t *testing.T) { + origRemaining, origReset := GlobalRateLimitState.Get() + defer GlobalRateLimitState.update(origRemaining, origReset) + + resetAt := time.Now().Add(30 * time.Minute) + GlobalRateLimitState.update(4500, resetAt) + + info := currentRateLimitInfo() + if info.Remaining != 4500 { + t.Errorf("expected remaining=4500, got %d", info.Remaining) + } + if info.ResetAt.Unix() != resetAt.Unix() { + t.Errorf("expected resetAt=%v, got %v", resetAt, info.ResetAt) + } +} diff --git a/services/service_container.go b/services/service_container.go index e9b6879..85c1dfb 100644 --- a/services/service_container.go +++ b/services/service_container.go @@ -13,6 +13,7 @@ import ( type ServiceContainer struct { Config *configs.Config FileStateService FileStateService + TokenManager *TokenManager // New services ConfigLoader ConfigLoader @@ -24,9 +25,15 @@ type ServiceContainer struct { MetricsCollector *MetricsCollector SlackNotifier SlackNotifier + // Webhook deduplication + DeliveryTracker *DeliveryTracker + // Server state StartTime time.Time + // Background goroutine tracking (for graceful shutdown and tests) + wg sync.WaitGroup + // Shutdown state closeOnce sync.Once closed bool @@ -37,15 +44,19 @@ func NewServiceContainer(config *configs.Config) (*ServiceContainer, error) { // Initialize file state service fileStateService := NewFileStateService() - // Initialize config loader based on configuration + // Initialize config loader based on configuration, wrapped with an optional TTL cache var configLoader ConfigLoader if config.UseMainConfig && config.MainConfigFile != "" { // Use main config loader for new format with workflow references (when USE_MAIN_CONFIG=true) configLoader = NewMainConfigLoader() } else { - // Use default config loader for singular config file (when USE_MAIN_CONFIG=false) + // Deprecated: the legacy single-file config path will be removed in a future release. + LogWarning("DEPRECATION: USE_MAIN_CONFIG is not set or MAIN_CONFIG_FILE is empty. "+ + "The legacy single-file config path will be removed in a future release. "+ + "Migrate to the main config format. Falling back to: %s", config.EffectiveConfigFile()) configLoader = NewConfigLoader() } + configLoader = NewCachedConfigLoader(configLoader, time.Duration(config.ConfigCacheTTLSeconds)*time.Second) patternMatcher := NewPatternMatcher() pathTransformer := NewPathTransformer() @@ -54,11 +65,14 @@ func NewServiceContainer(config *configs.Config) (*ServiceContainer, error) { metricsCollector := NewMetricsCollector() // Initialize Slack notifier - slackNotifier := NewSlackNotifier( + // Use plain text mode for Workflow Builder webhooks (they don't support attachments) + slackNotifier := NewSlackNotifierWithOptions( config.SlackWebhookURL, config.SlackChannel, config.SlackUsername, config.SlackIconEmoji, + config.SlackPlainText, + config.SlackMessageVariable, ) // Initialize audit logger @@ -77,6 +91,7 @@ func NewServiceContainer(config *configs.Config) (*ServiceContainer, error) { return &ServiceContainer{ Config: config, FileStateService: fileStateService, + TokenManager: defaultTokenManager, ConfigLoader: configLoader, PatternMatcher: patternMatcher, PathTransformer: pathTransformer, @@ -85,14 +100,23 @@ func NewServiceContainer(config *configs.Config) (*ServiceContainer, error) { AuditLogger: auditLogger, MetricsCollector: metricsCollector, SlackNotifier: slackNotifier, + DeliveryTracker: NewDeliveryTracker(1 * time.Hour), StartTime: time.Now(), }, nil } +// Wait blocks until all background goroutines tracked by this container have finished. +func (sc *ServiceContainer) Wait() { + sc.wg.Wait() +} + // Close cleans up resources. Safe to call multiple times. func (sc *ServiceContainer) Close(ctx context.Context) error { var closeErr error sc.closeOnce.Do(func() { + if sc.DeliveryTracker != nil { + sc.DeliveryTracker.Stop() + } if sc.AuditLogger != nil { closeErr = sc.AuditLogger.Close(ctx) } diff --git a/services/service_container_test.go b/services/service_container_test.go index ed8bcc7..96cebbc 100644 --- a/services/service_container_test.go +++ b/services/service_container_test.go @@ -110,6 +110,10 @@ func TestNewServiceContainer(t *testing.T) { t.Error("SlackNotifier is nil") } + if container.TokenManager == nil { + t.Error("TokenManager is nil") + } + // Check that StartTime is set if container.StartTime.IsZero() { t.Error("StartTime is zero") diff --git a/services/slack_notifier.go b/services/slack_notifier.go index c1614d8..95a0bfb 100644 --- a/services/slack_notifier.go +++ b/services/slack_notifier.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "time" ) @@ -13,79 +14,106 @@ import ( type SlackNotifier interface { // NotifyPRProcessed sends a notification when a PR is successfully processed NotifyPRProcessed(ctx context.Context, event *PRProcessedEvent) error - + // NotifyError sends a notification when an error occurs NotifyError(ctx context.Context, event *ErrorEvent) error - + // NotifyFilesCopied sends a notification when files are copied NotifyFilesCopied(ctx context.Context, event *FilesCopiedEvent) error - + // NotifyDeprecation sends a notification when files are deprecated NotifyDeprecation(ctx context.Context, event *DeprecationEvent) error - + // IsEnabled returns true if Slack notifications are enabled IsEnabled() bool } // PRProcessedEvent contains information about a processed PR type PRProcessedEvent struct { - PRNumber int - PRTitle string - PRURL string - SourceRepo string - FilesMatched int - FilesCopied int - FilesFailed int + PRNumber int + PRTitle string + PRURL string + SourceRepo string + TargetRepos []string // List of target repositories files were copied to + FilesMatched int + FilesCopied int + FilesFailed int ProcessingTime time.Duration } // ErrorEvent contains information about an error type ErrorEvent struct { - Operation string - Error error - PRNumber int - SourceRepo string + Operation string + Error error + PRNumber int + SourceRepo string + DeliveryID string // GitHub webhook delivery ID for tracing + Attempts int // number of processing attempts (0 = not set) AdditionalInfo map[string]interface{} } // FilesCopiedEvent contains information about copied files type FilesCopiedEvent struct { - PRNumber int - SourceRepo string - TargetRepo string - FileCount int - Files []string - RuleName string + PRNumber int + SourceRepo string + TargetRepo string + FileCount int + Files []string + RuleName string } // DeprecationEvent contains information about deprecated files type DeprecationEvent struct { - PRNumber int - SourceRepo string - FileCount int - Files []string + PRNumber int + SourceRepo string + FileCount int + Files []string } // DefaultSlackNotifier implements SlackNotifier using Slack webhooks type DefaultSlackNotifier struct { - webhookURL string - enabled bool - channel string - username string - iconEmoji string - httpClient *http.Client + webhookURL string + enabled bool + channel string + username string + iconEmoji string + httpClient *http.Client + plainTextOnly bool // If true, always use plain text (for Workflow Builder webhooks) + messageVariable string // Variable name for Workflow Builder webhooks (default: "text") } // NewSlackNotifier creates a new Slack notifier func NewSlackNotifier(webhookURL, channel, username, iconEmoji string) SlackNotifier { + return NewSlackNotifierWithOptions(webhookURL, channel, username, iconEmoji, false, "text") +} + +// NewSlackNotifierWithOptions creates a new Slack notifier with additional options +// plainTextOnly should be true when using Slack Workflow Builder webhooks, +// which don't support attachments or blocks +// messageVariable is the JSON key name for Workflow Builder webhooks (e.g., "text", "data", "message") +func NewSlackNotifierWithOptions(webhookURL, channel, username, iconEmoji string, plainTextOnly bool, messageVariable string) SlackNotifier { enabled := webhookURL != "" - + + // Auto-detect Workflow Builder webhooks (they use /triggers/ instead of /services/) + // and force plain text mode since they don't support attachments + if strings.Contains(webhookURL, "/triggers/") { + plainTextOnly = true + LogInfo("detected Slack Workflow Builder webhook, using plain text mode") + } + + // Default to "text" if not specified + if messageVariable == "" { + messageVariable = "text" + } + return &DefaultSlackNotifier{ - webhookURL: webhookURL, - enabled: enabled, - channel: channel, - username: username, - iconEmoji: iconEmoji, + webhookURL: webhookURL, + enabled: enabled, + channel: channel, + username: username, + iconEmoji: iconEmoji, + plainTextOnly: plainTextOnly, + messageVariable: messageVariable, httpClient: &http.Client{ Timeout: 10 * time.Second, }, @@ -102,22 +130,59 @@ func (sn *DefaultSlackNotifier) NotifyPRProcessed(ctx context.Context, event *PR if !sn.enabled { return nil } - + + // Plain text format for Workflow Builder webhooks + // Use status emoji based on success/failure + statusEmoji := "โœ…" + statusText := "Success" + if event.FilesFailed > 0 { + statusEmoji = "โš ๏ธ" + statusText = "Partial" + } + + plainText := fmt.Sprintf("%s *PR #%d* โ€” %s\n"+ + "*Source:* %s\n", + statusEmoji, event.PRNumber, statusText, + event.SourceRepo) + + // Add target repos if available + if len(event.TargetRepos) > 0 { + if len(event.TargetRepos) == 1 { + plainText += fmt.Sprintf("*Target:* %s\n", event.TargetRepos[0]) + } else { + plainText += fmt.Sprintf("*Targets:* %s\n", strings.Join(event.TargetRepos, ", ")) + } + } + + plainText += fmt.Sprintf("*Files:* %d copied", event.FilesCopied) + if event.FilesFailed > 0 { + plainText += fmt.Sprintf(", %d failed", event.FilesFailed) + } + + plainText += fmt.Sprintf("\n*Time:* %s\n<%s|View PR>", + formatDuration(event.ProcessingTime), + event.PRURL) + + if sn.plainTextOnly { + return sn.sendPlainText(ctx, plainText) + } + color := "good" // green if event.FilesFailed > 0 { color = "warning" // yellow } - + message := &SlackMessage{ Channel: sn.channel, Username: sn.username, IconEmoji: sn.iconEmoji, + Text: plainText, // Fallback text Attachments: []SlackAttachment{ { - Color: color, - Title: fmt.Sprintf("โœ… PR #%d Processed", event.PRNumber), - TitleLink: event.PRURL, - Text: event.PRTitle, + Color: color, + Title: fmt.Sprintf("โœ… PR #%d Processed", event.PRNumber), + TitleLink: event.PRURL, + Text: event.PRTitle, Fields: []SlackField{ {Title: "Repository", Value: event.SourceRepo, Short: true}, {Title: "Files Matched", Value: fmt.Sprintf("%d", event.FilesMatched), Short: true}, @@ -131,8 +196,8 @@ func (sn *DefaultSlackNotifier) NotifyPRProcessed(ctx context.Context, event *PR }, }, } - - return sn.sendMessage(ctx, message) + + return sn.sendMessageWithFallback(ctx, message, plainText) } // NotifyError sends a notification when an error occurs @@ -140,24 +205,53 @@ func (sn *DefaultSlackNotifier) NotifyError(ctx context.Context, event *ErrorEve if !sn.enabled { return nil } - + + // Build plain text format for Workflow Builder webhooks + plainText := fmt.Sprintf("โŒ *Error* โ€” %s", event.Operation) + + if event.PRNumber > 0 && event.SourceRepo != "" { + plainText += fmt.Sprintf("\n*PR:* <%s/pull/%d|#%d> in %s", + "https://github.com/"+event.SourceRepo, event.PRNumber, event.PRNumber, event.SourceRepo) + } else if event.SourceRepo != "" { + plainText += fmt.Sprintf("\n*Repo:* %s", event.SourceRepo) + } + + plainText += fmt.Sprintf("\n*Error:* %s", event.Error.Error()) + + if event.Attempts > 0 { + plainText += fmt.Sprintf(" (attempt %d)", event.Attempts) + } + + if sn.plainTextOnly { + return sn.sendPlainText(ctx, plainText) + } + fields := []SlackField{ {Title: "Operation", Value: event.Operation, Short: true}, {Title: "Error", Value: event.Error.Error(), Short: false}, } - + if event.SourceRepo != "" { fields = append(fields, SlackField{Title: "Repository", Value: event.SourceRepo, Short: true}) } - + if event.PRNumber > 0 { fields = append(fields, SlackField{Title: "PR Number", Value: fmt.Sprintf("#%d", event.PRNumber), Short: true}) } - + + if event.DeliveryID != "" { + fields = append(fields, SlackField{Title: "Delivery ID", Value: event.DeliveryID, Short: true}) + } + + if event.Attempts > 0 { + fields = append(fields, SlackField{Title: "Attempts", Value: fmt.Sprintf("%d", event.Attempts), Short: true}) + } + message := &SlackMessage{ Channel: sn.channel, Username: sn.username, IconEmoji: sn.iconEmoji, + Text: plainText, // Fallback text Attachments: []SlackAttachment{ { Color: "danger", // red @@ -170,8 +264,8 @@ func (sn *DefaultSlackNotifier) NotifyError(ctx context.Context, event *ErrorEve }, }, } - - return sn.sendMessage(ctx, message) + + return sn.sendMessageWithFallback(ctx, message, plainText) } // NotifyFilesCopied sends a notification when files are copied @@ -179,28 +273,51 @@ func (sn *DefaultSlackNotifier) NotifyFilesCopied(ctx context.Context, event *Fi if !sn.enabled { return nil } - + // Limit files shown to first 10 - filesText := "" displayFiles := event.Files + moreCount := 0 if len(displayFiles) > 10 { + moreCount = len(event.Files) - 10 displayFiles = displayFiles[:10] - filesText = fmt.Sprintf("```\n%s\n... and %d more```", - formatFileList(displayFiles), - len(event.Files)-10) + } + + // Plain text format for Workflow Builder webhooks + plainText := fmt.Sprintf("๐Ÿ“‹ *PR #%d* โ€” %d files copied\n"+ + "*Rule:* %s\n"+ + "*Target:* %s\n"+ + "%s", + event.PRNumber, + event.FileCount, + event.RuleName, + event.TargetRepo, + formatFileListCompact(displayFiles)) + if moreCount > 0 { + plainText += fmt.Sprintf("_...and %d more_", moreCount) + } + + if sn.plainTextOnly { + return sn.sendPlainText(ctx, plainText) + } + + filesText := "" + if moreCount > 0 { + filesText = fmt.Sprintf("```\n%s\n... and %d more```", + formatFileList(displayFiles), moreCount) } else { filesText = fmt.Sprintf("```\n%s```", formatFileList(displayFiles)) } - + message := &SlackMessage{ Channel: sn.channel, Username: sn.username, IconEmoji: sn.iconEmoji, + Text: plainText, // Fallback text Attachments: []SlackAttachment{ { - Color: "good", // green - Title: fmt.Sprintf("๐Ÿ“‹ Files Copied from PR #%d", event.PRNumber), - Text: filesText, + Color: "good", // green + Title: fmt.Sprintf("๐Ÿ“‹ Files Copied from PR #%d", event.PRNumber), + Text: filesText, Fields: []SlackField{ {Title: "Source", Value: event.SourceRepo, Short: true}, {Title: "Target", Value: event.TargetRepo, Short: true}, @@ -213,8 +330,8 @@ func (sn *DefaultSlackNotifier) NotifyFilesCopied(ctx context.Context, event *Fi }, }, } - - return sn.sendMessage(ctx, message) + + return sn.sendMessageWithFallback(ctx, message, plainText) } // NotifyDeprecation sends a notification when files are deprecated @@ -222,18 +339,31 @@ func (sn *DefaultSlackNotifier) NotifyDeprecation(ctx context.Context, event *De if !sn.enabled { return nil } - + + // Plain text format for Workflow Builder webhooks + plainText := fmt.Sprintf("โš ๏ธ *PR #%d* โ€” %d files deprecated\n"+ + "*Repo:* %s\n"+ + "%s", + event.PRNumber, event.FileCount, + event.SourceRepo, + formatFileListCompact(event.Files)) + + if sn.plainTextOnly { + return sn.sendPlainText(ctx, plainText) + } + filesText := fmt.Sprintf("```\n%s```", formatFileList(event.Files)) - + message := &SlackMessage{ Channel: sn.channel, Username: sn.username, IconEmoji: sn.iconEmoji, + Text: plainText, // Fallback text Attachments: []SlackAttachment{ { - Color: "warning", // yellow - Title: fmt.Sprintf("โš ๏ธ Files Deprecated from PR #%d", event.PRNumber), - Text: filesText, + Color: "warning", // yellow + Title: fmt.Sprintf("โš ๏ธ Files Deprecated from PR #%d", event.PRNumber), + Text: filesText, Fields: []SlackField{ {Title: "Repository", Value: event.SourceRepo, Short: true}, {Title: "File Count", Value: fmt.Sprintf("%d", event.FileCount), Short: true}, @@ -244,38 +374,66 @@ func (sn *DefaultSlackNotifier) NotifyDeprecation(ctx context.Context, event *De }, }, } - - return sn.sendMessage(ctx, message) + + return sn.sendMessageWithFallback(ctx, message, plainText) +} + +// sendPlainText sends a plain text message to Slack +// For Workflow Builder webhooks (/triggers/), this sends a simple object that +// the workflow can use as input variables +func (sn *DefaultSlackNotifier) sendPlainText(ctx context.Context, text string) error { + // For Workflow Builder webhooks, we send a simple key-value payload + // using the configured variable name (e.g., "text", "data", "message") + payload, err := json.Marshal(map[string]string{sn.messageVariable: text}) + if err != nil { + return fmt.Errorf("failed to marshal slack message: %w", err) + } + + return sn.sendPayload(ctx, payload) } -// sendMessage sends a message to Slack -func (sn *DefaultSlackNotifier) sendMessage(ctx context.Context, message *SlackMessage) error { +// sendMessageWithFallback tries to send a rich message first, then falls back to plain text +// if the webhook doesn't support attachments (e.g., Workflow Builder webhooks) +func (sn *DefaultSlackNotifier) sendMessageWithFallback(ctx context.Context, message *SlackMessage, plainText string) error { payload, err := json.Marshal(message) if err != nil { return fmt.Errorf("failed to marshal slack message: %w", err) } - + + err = sn.sendPayload(ctx, payload) + if err != nil { + // If the rich message failed, try plain text as fallback + // This handles Workflow Builder webhooks that don't support attachments + LogInfo("rich slack message failed, trying plain text fallback", "error", err.Error()) + return sn.sendPlainText(ctx, plainText) + } + + return nil +} + +// sendPayload sends the raw JSON payload to Slack +func (sn *DefaultSlackNotifier) sendPayload(ctx context.Context, payload []byte) error { req, err := http.NewRequestWithContext(ctx, "POST", sn.webhookURL, bytes.NewBuffer(payload)) if err != nil { return fmt.Errorf("failed to create slack request: %w", err) } - + req.Header.Set("Content-Type", "application/json") - - resp, err := sn.httpClient.Do(req) + + resp, err := sn.httpClient.Do(req) // #nosec G704 -- URL is the Slack webhook URL from trusted config if err != nil { return fmt.Errorf("failed to send slack message: %w", err) } - defer resp.Body.Close() - + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { return fmt.Errorf("slack returned non-200 status: %d", resp.StatusCode) } - + return nil } -// formatFileList formats a list of files for display +// formatFileList formats a list of files for display (verbose, with bullets) func formatFileList(files []string) string { result := "" for _, file := range files { @@ -284,13 +442,40 @@ func formatFileList(files []string) string { return result } +// formatFileListCompact formats files in a compact inline format for plain text messages +func formatFileListCompact(files []string) string { + if len(files) == 0 { + return "" + } + result := "`" + for i, file := range files { + if i > 0 { + result += "`, `" + } + result += file + } + result += "`\n" + return result +} + +// formatDuration formats a duration in a human-readable way +func formatDuration(d time.Duration) string { + if d < time.Second { + return fmt.Sprintf("%dms", d.Milliseconds()) + } + if d < time.Minute { + return fmt.Sprintf("%.1fs", d.Seconds()) + } + return fmt.Sprintf("%.1fm", d.Minutes()) +} + // SlackMessage represents a Slack message type SlackMessage struct { - Channel string `json:"channel,omitempty"` - Username string `json:"username,omitempty"` - IconEmoji string `json:"icon_emoji,omitempty"` - Text string `json:"text,omitempty"` - Attachments []SlackAttachment `json:"attachments,omitempty"` + Channel string `json:"channel,omitempty"` + Username string `json:"username,omitempty"` + IconEmoji string `json:"icon_emoji,omitempty"` + Text string `json:"text,omitempty"` + Attachments []SlackAttachment `json:"attachments,omitempty"` } // SlackAttachment represents a Slack message attachment @@ -311,4 +496,3 @@ type SlackField struct { Value string `json:"value"` Short bool `json:"short"` } - diff --git a/services/slack_notifier_test.go b/services/slack_notifier_test.go index 6dd9030..7627a80 100644 --- a/services/slack_notifier_test.go +++ b/services/slack_notifier_test.go @@ -316,6 +316,157 @@ func TestFormatFileList(t *testing.T) { } } +func TestSlackNotifier_PlainTextMode(t *testing.T) { + event := &PRProcessedEvent{ + PRNumber: 42, + PRTitle: "Test PR", + PRURL: "https://github.com/test/repo/pull/42", + SourceRepo: "test/repo", + FilesMatched: 10, + FilesCopied: 8, + FilesFailed: 2, + ProcessingTime: 5 * time.Second, + } + + // Plain text mode sends a simple map with the configured variable name + var receivedPayload map[string]string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &receivedPayload) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + // Create notifier with plain text mode enabled, using "text" as the variable name + notifier := NewSlackNotifierWithOptions(server.URL, "#test", "Test Bot", ":robot:", true, "text") + ctx := context.Background() + + err := notifier.NotifyPRProcessed(ctx, event) + if err != nil { + t.Errorf("NotifyPRProcessed() error = %v", err) + } + + if receivedPayload == nil { + t.Fatal("No payload received") + } + + // In plain text mode, should have the configured variable name with the message + textValue, ok := receivedPayload["text"] + if !ok || textValue == "" { + t.Error("Plain text payload should have 'text' field set") + } + + // Should only have the one key + if len(receivedPayload) != 1 { + t.Errorf("Plain text payload should have exactly 1 key, got %d", len(receivedPayload)) + } + + // Verify the text contains expected content + if !contains(textValue, "PR #42") { + t.Error("Plain text should contain PR number") + } + if !contains(textValue, "test/repo") { + t.Error("Plain text should contain repo name") + } +} + +func TestSlackNotifier_CustomMessageVariable(t *testing.T) { + event := &PRProcessedEvent{ + PRNumber: 42, + PRTitle: "Test PR", + PRURL: "https://github.com/test/repo/pull/42", + SourceRepo: "test/repo", + FilesMatched: 10, + FilesCopied: 8, + FilesFailed: 0, + ProcessingTime: 5 * time.Second, + } + + var receivedPayload map[string]string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &receivedPayload) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + // Create notifier with custom variable name "data" (like a real Workflow Builder setup) + notifier := NewSlackNotifierWithOptions(server.URL, "#test", "Test Bot", ":robot:", true, "data") + ctx := context.Background() + + err := notifier.NotifyPRProcessed(ctx, event) + if err != nil { + t.Errorf("NotifyPRProcessed() error = %v", err) + } + + if receivedPayload == nil { + t.Fatal("No payload received") + } + + // Should use the custom variable name "data" + dataValue, ok := receivedPayload["data"] + if !ok || dataValue == "" { + t.Error("Payload should have 'data' field set (custom variable name)") + } + + // Should NOT have the default "text" key + if _, hasText := receivedPayload["text"]; hasText { + t.Error("Payload should not have 'text' field when using custom variable name") + } + + // Verify content + if !contains(dataValue, "PR #42") { + t.Error("Message should contain PR number") + } +} + +func TestSlackNotifier_AutoDetectWorkflowBuilder(t *testing.T) { + // Test that /triggers/ URLs auto-enable plain text mode + var receivedPayload map[string]string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &receivedPayload) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + // Simulate a Workflow Builder URL (with /triggers/) + workflowURL := server.URL + "/triggers/test" + notifier := NewSlackNotifierWithOptions(workflowURL, "#test", "Test Bot", ":robot:", false, "text") + + event := &PRProcessedEvent{ + PRNumber: 42, + PRTitle: "Test PR", + PRURL: "https://github.com/test/repo/pull/42", + SourceRepo: "test/repo", + FilesMatched: 10, + FilesCopied: 8, + FilesFailed: 0, + ProcessingTime: 5 * time.Second, + } + + ctx := context.Background() + err := notifier.NotifyPRProcessed(ctx, event) + if err != nil { + t.Errorf("NotifyPRProcessed() error = %v", err) + } + + if receivedPayload == nil { + t.Fatal("No payload received") + } + + // Should auto-detect and use plain text format (simple map) + textValue, ok := receivedPayload["text"] + if !ok || textValue == "" { + t.Error("Workflow Builder webhook should send plain text payload with 'text' field") + } + + // Should only have the message variable, no attachments or other Slack fields + if len(receivedPayload) != 1 { + t.Errorf("Workflow Builder payload should have exactly 1 key, got %d keys: %v", len(receivedPayload), receivedPayload) + } +} + // Helper function func contains(s, substr string) bool { return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && (s[:len(substr)] == substr || s[len(s)-len(substr):] == substr || containsMiddle(s, substr))) diff --git a/services/token_manager.go b/services/token_manager.go new file mode 100644 index 0000000..0280d6e --- /dev/null +++ b/services/token_manager.go @@ -0,0 +1,134 @@ +package services + +import ( + "net/http" + "sync" + "time" +) + +// tokenEntry stores a cached installation token with its expiration time. +type tokenEntry struct { + Token string + ExpiresAt time.Time +} + +// TokenManager provides thread-safe management of GitHub App authentication tokens. +// It caches JWT tokens and per-org installation tokens with expiry tracking, +// and holds the HTTP client used for GitHub API calls. +type TokenManager struct { + mu sync.RWMutex + + // Default installation access token (set once at startup via ConfigurePermissions) + installationAccessToken string + + // Per-org installation token cache with expiry + installationTokenCache map[string]tokenEntry + + // Cached JWT token and its expiry + cachedJWT string + cachedJWTExpiry time.Time + + // HTTP client used for GitHub API calls (swappable for testing with httpmock) + httpClient *http.Client +} + +// NewTokenManager creates a new TokenManager instance. +func NewTokenManager() *TokenManager { + return &TokenManager{ + installationTokenCache: make(map[string]tokenEntry), + httpClient: http.DefaultClient, + } +} + +// GetInstallationAccessToken returns the default installation access token. +func (tm *TokenManager) GetInstallationAccessToken() string { + tm.mu.RLock() + defer tm.mu.RUnlock() + return tm.installationAccessToken +} + +// SetInstallationAccessToken sets the default installation access token. +func (tm *TokenManager) SetInstallationAccessToken(token string) { + tm.mu.Lock() + defer tm.mu.Unlock() + tm.installationAccessToken = token +} + +// GetHTTPClient returns the HTTP client used for GitHub API calls. +func (tm *TokenManager) GetHTTPClient() *http.Client { + tm.mu.RLock() + defer tm.mu.RUnlock() + if tm.httpClient == nil { + return http.DefaultClient + } + return tm.httpClient +} + +// SetHTTPClient sets the HTTP client used for GitHub API calls. +func (tm *TokenManager) SetHTTPClient(client *http.Client) { + tm.mu.Lock() + defer tm.mu.Unlock() + tm.httpClient = client +} + +// GetTokenForOrg returns a cached token for the given org if it exists and is still valid. +// Returns empty string and false if no valid token exists. +func (tm *TokenManager) GetTokenForOrg(org string) (string, bool) { + tm.mu.RLock() + defer tm.mu.RUnlock() + entry, ok := tm.installationTokenCache[org] + if !ok || entry.Token == "" { + return "", false + } + // Check if token is expired (with 5-minute buffer for safety) + if !entry.ExpiresAt.IsZero() && time.Now().After(entry.ExpiresAt.Add(-5*time.Minute)) { + return "", false + } + return entry.Token, true +} + +// SetTokenForOrg caches an installation token for an org with expiry. +func (tm *TokenManager) SetTokenForOrg(org, token string, expiresAt time.Time) { + tm.mu.Lock() + defer tm.mu.Unlock() + tm.installationTokenCache[org] = tokenEntry{ + Token: token, + ExpiresAt: expiresAt, + } +} + +// SetTokenForOrgNoExpiry caches an installation token for an org without expiry tracking. +// Primarily used in tests where token expiry is not relevant. +func (tm *TokenManager) SetTokenForOrgNoExpiry(org, token string) { + tm.mu.Lock() + defer tm.mu.Unlock() + tm.installationTokenCache[org] = tokenEntry{ + Token: token, + } +} + +// GetCachedJWT returns the cached JWT if it's still valid. +func (tm *TokenManager) GetCachedJWT() (string, bool) { + tm.mu.RLock() + defer tm.mu.RUnlock() + if tm.cachedJWT != "" && time.Now().Before(tm.cachedJWTExpiry) { + return tm.cachedJWT, true + } + return "", false +} + +// SetCachedJWT caches a JWT with its expiry time. +func (tm *TokenManager) SetCachedJWT(token string, expiry time.Time) { + tm.mu.Lock() + defer tm.mu.Unlock() + tm.cachedJWT = token + tm.cachedJWTExpiry = expiry +} + +// defaultTokenManager is the package-level TokenManager instance. +var defaultTokenManager = NewTokenManager() + +// DefaultTokenManager returns the package-level TokenManager instance. +func DefaultTokenManager() *TokenManager { + return defaultTokenManager +} diff --git a/services/webhook_handler_new.go b/services/webhook_handler_new.go index dd7ec04..399ebbc 100644 --- a/services/webhook_handler_new.go +++ b/services/webhook_handler_new.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/grove-platform/github-copier/configs" "github.com/grove-platform/github-copier/types" ) @@ -44,8 +44,12 @@ func simpleVerifySignature(sigHeader string, body, secret []byte) bool { } // RetrieveFileContentsWithConfigAndBranch fetches file contents from a specific branch -func RetrieveFileContentsWithConfigAndBranch(ctx context.Context, filePath string, branch string, repoOwner string, repoName string) (*github.RepositoryContent, error) { - client := GetRestClient() +func RetrieveFileContentsWithConfigAndBranch(ctx context.Context, config *configs.Config, filePath string, branch string, repoOwner string, repoName string) (*github.RepositoryContent, error) { + // Use org-specific client to ensure we have the right installation token + client, err := GetRestClientForOrg(ctx, config, repoOwner) + if err != nil { + return nil, fmt.Errorf("failed to get GitHub client for org %s: %w", repoOwner, err) + } fileContent, _, _, err := client.Repositories.GetContents( ctx, @@ -65,6 +69,12 @@ func RetrieveFileContentsWithConfigAndBranch(ctx context.Context, filePath strin // HandleWebhookWithContainer handles incoming GitHub webhook requests using the service container func HandleWebhookWithContainer(w http.ResponseWriter, r *http.Request, config *configs.Config, container *ServiceContainer) { + // GitHub webhooks are always POST + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + startTime := time.Now() ctx := r.Context() @@ -90,9 +100,23 @@ func HandleWebhookWithContainer(w http.ResponseWriter, r *http.Request, config * return } + // Check for duplicate delivery using X-GitHub-Delivery header + deliveryID := r.Header.Get("X-GitHub-Delivery") + if deliveryID != "" && container.DeliveryTracker != nil { + if !container.DeliveryTracker.TryRecord(deliveryID) { + LogInfoCtx(ctx, "duplicate webhook delivery, skipping", map[string]interface{}{ + "delivery_id": deliveryID, + "event_type": eventType, + }) + w.WriteHeader(http.StatusOK) + return + } + } + LogInfoCtx(ctx, "payload read", map[string]interface{}{ - "elapsed_ms": time.Since(startTime).Milliseconds(), - "size_bytes": len(payload), + "elapsed_ms": time.Since(startTime).Milliseconds(), + "size_bytes": len(payload), + "delivery_id": deliveryID, }) // Verify webhook signature @@ -107,6 +131,8 @@ func HandleWebhookWithContainer(w http.ResponseWriter, r *http.Request, config * LogInfoCtx(ctx, "signature verified", map[string]interface{}{ "elapsed_ms": time.Since(startTime).Milliseconds(), }) + } else { + LogWarningCtx(ctx, "webhook signature verification DISABLED - no webhook secret configured; set WEBHOOK_SECRET for production use", nil) } // Parse webhook event @@ -142,7 +168,7 @@ func HandleWebhookWithContainer(w http.ResponseWriter, r *http.Request, config * "merged": merged, }) - if !(action == "closed" && merged) { + if action != "closed" || !merged { LogInfoCtx(ctx, "skipping non-merged PR", map[string]interface{}{ "action": action, "merged": merged, @@ -174,6 +200,7 @@ func HandleWebhookWithContainer(w http.ResponseWriter, r *http.Request, config * "sha": sourceCommitSHA, "repo": fmt.Sprintf("%s/%s", repoOwner, repoName), "base_branch": baseBranch, + "delivery_id": deliveryID, "elapsed_ms": time.Since(startTime).Milliseconds(), }) @@ -184,7 +211,9 @@ func HandleWebhookWithContainer(w http.ResponseWriter, r *http.Request, config * w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) - _, _ = w.Write([]byte(`{"status":"accepted"}`)) + if _, err := w.Write([]byte(`{"status":"accepted"}`)); err != nil { + LogWarningCtx(ctx, "failed to write webhook response body", map[string]interface{}{"error": err.Error()}) + } LogInfoCtx(ctx, "response sent", map[string]interface{}{ "elapsed_ms": time.Since(startTime).Milliseconds(), @@ -198,125 +227,279 @@ func HandleWebhookWithContainer(w http.ResponseWriter, r *http.Request, config * }) } - // Process asynchronously in background with a new context - // Don't use the request context as it will be cancelled when the request completes + // Process asynchronously in background with a new context. + // Don't use the request context as it will be cancelled when the request completes. bgCtx := context.Background() - go handleMergedPRWithContainer(bgCtx, prNumber, sourceCommitSHA, repoOwner, repoName, baseBranch, config, container) + + // Apply a timeout to prevent stuck API calls from running indefinitely (#9). + if config.WebhookProcessingTimeoutSeconds > 0 { + var cancel context.CancelFunc + bgCtx, cancel = context.WithTimeout(bgCtx, time.Duration(config.WebhookProcessingTimeoutSeconds)*time.Second) + // cancel is called after processing completes (in the goroutine below) + _ = cancel // used in defer below + container.wg.Add(1) + go func() { + defer container.wg.Done() + defer cancel() + processWebhookWithRetry(bgCtx, prNumber, sourceCommitSHA, repoOwner, repoName, baseBranch, deliveryID, config, container) + }() + } else { + container.wg.Add(1) + go func() { + defer container.wg.Done() + processWebhookWithRetry(bgCtx, prNumber, sourceCommitSHA, repoOwner, repoName, baseBranch, deliveryID, config, container) + }() + } } -// handleMergedPRWithContainer processes a merged PR using the new pattern matching system -func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommitSHA string, repoOwner string, repoName string, baseBranch string, config *configs.Config, container *ServiceContainer) { +// processWebhookWithRetry wraps handleMergedPRWithContainer with panic recovery +// and exponential-backoff retries for transient failures (#7). +func processWebhookWithRetry(ctx context.Context, prNumber int, sourceCommitSHA string, repoOwner string, repoName string, baseBranch string, deliveryID string, config *configs.Config, container *ServiceContainer) { + webhookRepo := fmt.Sprintf("%s/%s", repoOwner, repoName) + maxAttempts := config.WebhookMaxRetries + 1 // retries + initial attempt + delay := time.Duration(config.WebhookRetryInitialDelay) * time.Second + + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + lastErr = runWithRecovery(ctx, prNumber, sourceCommitSHA, repoOwner, repoName, baseBranch, config, container) + if lastErr == nil { + return // success + } + + // Don't retry if context is done (timeout or cancellation) + if ctx.Err() != nil { + LogErrorCtx(ctx, "webhook processing failed โ€” context expired, skipping retry", lastErr, map[string]interface{}{ + "pr_number": prNumber, + "repo": webhookRepo, + "attempt": attempt, + }) + break + } + + // Don't retry permanent errors โ€” they won't resolve by retrying (#8) + if IsPermanentError(lastErr) { + LogErrorCtx(ctx, "webhook processing failed with permanent error, skipping retry", lastErr, map[string]interface{}{ + "pr_number": prNumber, + "repo": webhookRepo, + "attempt": attempt, + }) + break + } + + if attempt < maxAttempts { + LogWarningCtx(ctx, "webhook processing failed (transient), retrying", map[string]interface{}{ + "pr_number": prNumber, + "repo": webhookRepo, + "attempt": attempt, + "max_attempts": maxAttempts, + "retry_delay": delay.String(), + "error": lastErr.Error(), + }) + + select { + case <-time.After(delay): + delay *= 2 // exponential backoff + case <-ctx.Done(): + LogWarningCtx(ctx, "retry wait interrupted by context cancellation", map[string]interface{}{ + "pr_number": prNumber, + }) + break + } + } + } + + // Processing failed โ€” alert via Slack + operation := "webhook_processing_exhausted" + if IsPermanentError(lastErr) { + operation = "webhook_processing_permanent_error" + } + LogCritical("webhook processing failed", + "pr_number", prNumber, + "repo", webhookRepo, + "delivery_id", deliveryID, + "attempts", maxAttempts, + "permanent", IsPermanentError(lastErr), + "error", lastErr, + ) + container.MetricsCollector.RecordWebhookFailed() + if notifyErr := container.SlackNotifier.NotifyError(ctx, &ErrorEvent{ + Operation: operation, + Error: fmt.Errorf("failed after %d attempt(s): %w", maxAttempts, lastErr), + PRNumber: prNumber, + SourceRepo: webhookRepo, + DeliveryID: deliveryID, + Attempts: maxAttempts, + }); notifyErr != nil { + LogWarning("failed to send Slack error notification", "error", notifyErr) + } +} + +// runWithRecovery calls handleMergedPRWithContainer in a panic-safe wrapper, +// converting panics into errors. +func runWithRecovery(ctx context.Context, prNumber int, sourceCommitSHA string, repoOwner string, repoName string, baseBranch string, config *configs.Config, container *ServiceContainer) (retErr error) { + defer func() { + if r := recover(); r != nil { + retErr = fmt.Errorf("panic: %v", r) + LogCritical("panic in webhook handler", "pr_number", prNumber, "repo_owner", repoOwner, "repo_name", repoName, "recovered", r) + } + }() + return handleMergedPRWithContainer(ctx, prNumber, sourceCommitSHA, repoOwner, repoName, baseBranch, config, container) +} + +// handleMergedPRWithContainer orchestrates processing of a merged PR: +// auth โ†’ config โ†’ match workflows โ†’ fetch changed files โ†’ process โ†’ upload โ†’ notify. +// Returns an error if a retryable failure occurred (#6 โ€” per-workflow error tracking). +func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommitSHA string, repoOwner string, repoName string, baseBranch string, config *configs.Config, container *ServiceContainer) error { startTime := time.Now() + webhookRepo := fmt.Sprintf("%s/%s", repoOwner, repoName) - // Configure GitHub permissions - if InstallationAccessToken == "" { - if err := ConfigurePermissions(); err != nil { + // 1. Ensure GitHub auth + if defaultTokenManager.GetInstallationAccessToken() == "" { + if err := ConfigurePermissions(ctx, config); err != nil { LogAndReturnError(ctx, "auth", "failed to configure GitHub permissions", err) container.MetricsCollector.RecordWebhookFailed() - return + notifySlackError(ctx, container, "auth", err, prNumber, webhookRepo) + return fmt.Errorf("auth: %w", err) } } - // Load configuration using new loader - // Note: config.ConfigRepoOwner and config.ConfigRepoName are already set from env.yaml - // The webhook repoOwner/repoName are used for matching workflows, not for loading config + // 2. Load config and find matching workflows + yamlConfig, err := loadAndMatchWorkflows(ctx, config, container, webhookRepo, baseBranch, prNumber) + if err != nil { + return fmt.Errorf("config: %w", err) + } + + // 3. Fetch changed files from the source PR + changedFiles, err := fetchChangedFiles(ctx, config, container, repoOwner, repoName, prNumber, webhookRepo) + if err != nil { + return fmt.Errorf("fetch_files: %w", err) + } + + // 4. Snapshot metrics before processing + filesMatchedBefore := container.MetricsCollector.GetFilesMatched() + filesUploadedBefore := container.MetricsCollector.GetFilesUploaded() + filesFailedBefore := container.MetricsCollector.GetFilesUploadFailed() + + // 5. Process workflows independently, collecting per-workflow errors (#6) + workflowErrors := processFilesWithWorkflows(ctx, prNumber, sourceCommitSHA, changedFiles, yamlConfig, config, container) + uploadAndDeprecateFiles(ctx, config, container, repoOwner, repoName, baseBranch, prNumber) + + // 6. Collect unique target repos for notification + targetRepos := collectTargetRepos(yamlConfig) + + // 7. Report completion + reportCompletion(ctx, container, webhookRepo, prNumber, sourceCommitSHA, startTime, + filesMatchedBefore, filesUploadedBefore, filesFailedBefore, targetRepos) + + // Return an aggregate error if any workflows failed (enables retry for partial failures) + if len(workflowErrors) > 0 { + errMsgs := make([]string, 0, len(workflowErrors)) + for wfName, wfErr := range workflowErrors { + errMsgs = append(errMsgs, fmt.Sprintf("%s: %v", wfName, wfErr)) + } + return fmt.Errorf("%d workflow(s) failed: %s", len(workflowErrors), strings.Join(errMsgs, "; ")) + } + + return nil +} + +// loadAndMatchWorkflows loads the YAML config and filters to workflows matching +// the webhook's source repo and branch. Returns nil and logs/notifies on error. +func loadAndMatchWorkflows(ctx context.Context, config *configs.Config, container *ServiceContainer, webhookRepo string, baseBranch string, prNumber int) (*types.YAMLConfig, error) { yamlConfig, err := container.ConfigLoader.LoadConfig(ctx, config) if err != nil { LogAndReturnError(ctx, "config_load", "failed to load config", err) container.MetricsCollector.RecordWebhookFailed() - - // Send error notification to Slack - _ = container.SlackNotifier.NotifyError(ctx, &ErrorEvent{ - Operation: "config_load", - Error: err, - PRNumber: prNumber, - SourceRepo: fmt.Sprintf("%s/%s", repoOwner, repoName), - }) - return + notifySlackError(ctx, container, "config_load", err, prNumber, webhookRepo) + return nil, err } - // Find workflows matching this source repo and branch - webhookRepo := fmt.Sprintf("%s/%s", repoOwner, repoName) - var matchingWorkflows []types.Workflow - for _, workflow := range yamlConfig.Workflows { - // Match both repository and branch - if workflow.Source.Repo == webhookRepo && workflow.Source.Branch == baseBranch { - matchingWorkflows = append(matchingWorkflows, workflow) + var matching []types.Workflow + for _, wf := range yamlConfig.Workflows { + if wf.Source.Repo == webhookRepo && wf.Source.Branch == baseBranch { + matching = append(matching, wf) } } - if len(matchingWorkflows) == 0 { + if len(matching) == 0 { LogWarningCtx(ctx, "no workflows configured for source repository and branch", map[string]interface{}{ "webhook_repo": webhookRepo, "base_branch": baseBranch, "workflow_count": len(yamlConfig.Workflows), }) container.MetricsCollector.RecordWebhookFailed() - return + return nil, fmt.Errorf("no matching workflows") } LogInfoCtx(ctx, "found matching workflows", map[string]interface{}{ "webhook_repo": webhookRepo, "base_branch": baseBranch, - "matching_count": len(matchingWorkflows), + "matching_count": len(matching), }) - // Store matching workflows for processing - yamlConfig.Workflows = matchingWorkflows + yamlConfig.Workflows = matching + return yamlConfig, nil +} - // Get changed files from PR (from the source repository that triggered the webhook) - changedFiles, err := GetFilesChangedInPr(repoOwner, repoName, prNumber) +// fetchChangedFiles retrieves the files changed in a PR, logging and notifying on error. +func fetchChangedFiles(ctx context.Context, config *configs.Config, container *ServiceContainer, repoOwner string, repoName string, prNumber int, webhookRepo string) ([]types.ChangedFile, error) { + changedFiles, err := GetFilesChangedInPr(ctx, config, repoOwner, repoName, prNumber) if err != nil { LogAndReturnError(ctx, "get_files", "failed to get changed files", err) container.MetricsCollector.RecordWebhookFailed() - - // Send error notification to Slack - _ = container.SlackNotifier.NotifyError(ctx, &ErrorEvent{ - Operation: "get_files", - Error: err, - PRNumber: prNumber, - SourceRepo: webhookRepo, - }) - return + notifySlackError(ctx, container, "get_files", err, prNumber, webhookRepo) + return nil, err } LogInfoCtx(ctx, "retrieved changed files", map[string]interface{}{ "count": len(changedFiles), }) + return changedFiles, nil +} - // Track metrics before processing - filesMatchedBefore := container.MetricsCollector.GetFilesMatched() - filesUploadedBefore := container.MetricsCollector.GetFilesUploaded() - filesFailedBefore := container.MetricsCollector.GetFilesUploadFailed() - - // Process files with workflow processor - processFilesWithWorkflows(ctx, prNumber, sourceCommitSHA, changedFiles, yamlConfig, container) - +// uploadAndDeprecateFiles drains the file-state queues, uploading files to target +// repos and updating the deprecation file in the source repo. +func uploadAndDeprecateFiles(ctx context.Context, config *configs.Config, container *ServiceContainer, sourceRepoOwner, sourceRepoName, sourceBranch string, prNumber int) { // Upload queued files - FilesToUpload = container.FileStateService.GetFilesToUpload() - AddFilesToTargetRepoBranchWithFetcher(container.PRTemplateFetcher, container.MetricsCollector) + filesToUpload := container.FileStateService.GetFilesToUpload() + AddFilesToTargetRepos(ctx, config, filesToUpload, container.PRTemplateFetcher, container.MetricsCollector) container.FileStateService.ClearFilesToUpload() - // Update deprecation file - copy from FileStateService to global map for legacy function - // The deprecationMap is keyed by deprecation file name, with a slice of entries per file + // Build deprecation map and update file in the source repo deprecationMap := container.FileStateService.GetFilesToDeprecate() - FilesToDeprecate = make(map[string]types.Configs) + filesToDeprecate := make(map[string]types.Configs) + var deprecatedFiles []string for _, entries := range deprecationMap { - // Iterate over all entries for each deprecation file for _, entry := range entries { - FilesToDeprecate[entry.FileName] = types.Configs{ + filesToDeprecate[entry.FileName] = types.Configs{ TargetRepo: entry.Repo, TargetBranch: entry.Branch, } + deprecatedFiles = append(deprecatedFiles, entry.FileName) } } - UpdateDeprecationFile() + UpdateDeprecationFile(ctx, config, filesToDeprecate, sourceRepoOwner, sourceRepoName, sourceBranch) container.FileStateService.ClearFilesToDeprecate() - // Calculate metrics after processing - filesMatched := container.MetricsCollector.GetFilesMatched() - filesMatchedBefore - filesUploaded := container.MetricsCollector.GetFilesUploaded() - filesUploadedBefore - filesFailed := container.MetricsCollector.GetFilesUploadFailed() - filesFailedBefore + // Send Slack notification if files were deprecated + if len(deprecatedFiles) > 0 { + sourceRepo := fmt.Sprintf("%s/%s", sourceRepoOwner, sourceRepoName) + if notifyErr := container.SlackNotifier.NotifyDeprecation(ctx, &DeprecationEvent{ + PRNumber: prNumber, + SourceRepo: sourceRepo, + FileCount: len(deprecatedFiles), + Files: deprecatedFiles, + }); notifyErr != nil { + LogWarningCtx(ctx, "failed to send Slack deprecation notification", map[string]interface{}{"error": notifyErr.Error()}) + } + } +} + +// reportCompletion calculates processing metrics and sends a Slack notification. +func reportCompletion(ctx context.Context, container *ServiceContainer, webhookRepo string, prNumber int, sourceCommitSHA string, startTime time.Time, matchedBefore int, uploadedBefore int, failedBefore int, targetRepos []string) { + filesMatched := container.MetricsCollector.GetFilesMatched() - matchedBefore + filesUploaded := container.MetricsCollector.GetFilesUploaded() - uploadedBefore + filesFailed := container.MetricsCollector.GetFilesUploadFailed() - failedBefore processingTime := time.Since(startTime) LogInfoCtx(ctx, "--Done--", map[string]interface{}{ @@ -324,22 +507,56 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit "sha": sourceCommitSHA, }) - // Send success notification to Slack - _ = container.SlackNotifier.NotifyPRProcessed(ctx, &PRProcessedEvent{ + if notifyErr := container.SlackNotifier.NotifyPRProcessed(ctx, &PRProcessedEvent{ PRNumber: prNumber, - PRTitle: fmt.Sprintf("PR #%d", prNumber), // TODO: Get actual PR title from GitHub + PRTitle: fmt.Sprintf("PR #%d", prNumber), PRURL: fmt.Sprintf("https://github.com/%s/pull/%d", webhookRepo, prNumber), SourceRepo: webhookRepo, + TargetRepos: targetRepos, FilesMatched: filesMatched, FilesCopied: filesUploaded, FilesFailed: filesFailed, ProcessingTime: processingTime, - }) + }); notifyErr != nil { + LogWarningCtx(ctx, "failed to send Slack PR processed notification", map[string]interface{}{"error": notifyErr.Error()}) + } +} + +// collectTargetRepos extracts unique target repository names from workflows. +func collectTargetRepos(yamlConfig *types.YAMLConfig) []string { + if yamlConfig == nil { + return nil + } + + seen := make(map[string]bool) + var repos []string + for _, wf := range yamlConfig.Workflows { + repo := wf.Destination.Repo + if repo != "" && !seen[repo] { + seen[repo] = true + repos = append(repos, repo) + } + } + return repos +} + +// notifySlackError is a helper to send a Slack error notification, logging any failure. +func notifySlackError(ctx context.Context, container *ServiceContainer, operation string, err error, prNumber int, sourceRepo string) { + if notifyErr := container.SlackNotifier.NotifyError(ctx, &ErrorEvent{ + Operation: operation, + Error: err, + PRNumber: prNumber, + SourceRepo: sourceRepo, + }); notifyErr != nil { + LogWarningCtx(ctx, "failed to send Slack error notification", map[string]interface{}{"error": notifyErr.Error()}) + } } -// processFilesWithWorkflows processes changed files using the workflow system +// processFilesWithWorkflows processes changed files using the workflow system. +// Each workflow is processed independently (#6 โ€” graceful partial failure). +// Returns a map of workflow name โ†’ error for any that failed; nil if all succeeded. func processFilesWithWorkflows(ctx context.Context, prNumber int, sourceCommitSHA string, - changedFiles []types.ChangedFile, yamlConfig *types.YAMLConfig, container *ServiceContainer) { + changedFiles []types.ChangedFile, yamlConfig *types.YAMLConfig, config *configs.Config, container *ServiceContainer) map[string]error { LogInfoCtx(ctx, "processing files with workflows", map[string]interface{}{ "file_count": len(changedFiles), @@ -353,13 +570,18 @@ func processFilesWithWorkflows(ctx context.Context, prNumber int, sourceCommitSH container.FileStateService, container.MetricsCollector, container.MessageTemplater, + config, ) - // Process each workflow + workflowErrors := make(map[string]error) + + // Process each workflow independently for _, workflow := range yamlConfig.Workflows { if err := ctx.Err(); err != nil { LogWebhookOperation(ctx, "workflow_processing", "workflow processing cancelled", err) - return + // Mark remaining workflows as cancelled + workflowErrors[workflow.Name] = fmt.Errorf("cancelled: %w", err) + continue } err := workflowProcessor.ProcessWorkflow(ctx, workflow, changedFiles, prNumber, sourceCommitSHA) @@ -367,12 +589,19 @@ func processFilesWithWorkflows(ctx context.Context, prNumber int, sourceCommitSH LogErrorCtx(ctx, "failed to process workflow", err, map[string]interface{}{ "workflow_name": workflow.Name, }) - // Continue processing other workflows + workflowErrors[workflow.Name] = err + // Continue processing other workflows โ€” don't let one failure block the rest continue } } LogInfoCtx(ctx, "workflow processing complete", map[string]interface{}{ "workflow_count": len(yamlConfig.Workflows), + "failed_count": len(workflowErrors), }) + + if len(workflowErrors) == 0 { + return nil + } + return workflowErrors } diff --git a/services/webhook_handler_new_test.go b/services/webhook_handler_new_test.go index d488ac3..2467d2f 100644 --- a/services/webhook_handler_new_test.go +++ b/services/webhook_handler_new_test.go @@ -2,6 +2,7 @@ package services import ( "bytes" + "context" "crypto/hmac" "crypto/rand" "crypto/rsa" @@ -13,11 +14,11 @@ import ( "encoding/pem" "net/http" "net/http/httptest" - "os" "testing" - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/grove-platform/github-copier/configs" + "github.com/jarcoal/httpmock" ) func TestSimpleVerifySignature(t *testing.T) { @@ -83,6 +84,33 @@ func TestSimpleVerifySignature(t *testing.T) { } } +func TestHandleWebhookWithContainer_MethodNotAllowed(t *testing.T) { + config := &configs.Config{ + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: false, + } + + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer() error = %v", err) + } + + methods := []string{"GET", "PUT", "DELETE", "PATCH"} + for _, method := range methods { + t.Run(method, func(t *testing.T) { + req := httptest.NewRequest(method, "/events", nil) + w := httptest.NewRecorder() + + HandleWebhookWithContainer(w, req, config, container) + + if w.Code != http.StatusMethodNotAllowed { + t.Errorf("%s: status code = %d, want %d", method, w.Code, http.StatusMethodNotAllowed) + } + }) + } +} + func TestHandleWebhookWithContainer_MissingEventType(t *testing.T) { config := &configs.Config{ ConfigRepoOwner: "test-owner", @@ -97,7 +125,7 @@ func TestHandleWebhookWithContainer_MissingEventType(t *testing.T) { } payload := []byte(`{"action": "closed"}`) - req := httptest.NewRequest("POST", "/webhook", bytes.NewReader(payload)) + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) // Missing X-GitHub-Event header w := httptest.NewRecorder() @@ -128,7 +156,7 @@ func TestHandleWebhookWithContainer_InvalidSignature(t *testing.T) { } payload := []byte(`{"action": "closed"}`) - req := httptest.NewRequest("POST", "/webhook", bytes.NewReader(payload)) + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) req.Header.Set("X-GitHub-Event", "pull_request") req.Header.Set("X-Hub-Signature-256", "sha256=invalid") @@ -158,10 +186,10 @@ func TestHandleWebhookWithContainer_ValidSignature(t *testing.T) { // Create a valid pull_request event payload prEvent := &github.PullRequestEvent{ - Action: github.String("opened"), + Action: github.Ptr("opened"), PullRequest: &github.PullRequest{ - Number: github.Int(123), - Merged: github.Bool(false), + Number: github.Ptr(123), + Merged: github.Ptr(false), }, } @@ -172,7 +200,7 @@ func TestHandleWebhookWithContainer_ValidSignature(t *testing.T) { mac.Write(payload) signature := "sha256=" + hex.EncodeToString(mac.Sum(nil)) - req := httptest.NewRequest("POST", "/webhook", bytes.NewReader(payload)) + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) req.Header.Set("X-GitHub-Event", "pull_request") req.Header.Set("X-Hub-Signature-256", signature) @@ -205,7 +233,7 @@ func TestHandleWebhookWithContainer_NonPREvent(t *testing.T) { } payload, _ := json.Marshal(pushEvent) - req := httptest.NewRequest("POST", "/webhook", bytes.NewReader(payload)) + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) req.Header.Set("X-GitHub-Event", "push") w := httptest.NewRecorder() @@ -233,15 +261,15 @@ func TestHandleWebhookWithContainer_NonMergedPR(t *testing.T) { // Create a PR event that's not merged prEvent := &github.PullRequestEvent{ - Action: github.String("opened"), + Action: github.Ptr("opened"), PullRequest: &github.PullRequest{ - Number: github.Int(123), - Merged: github.Bool(false), + Number: github.Ptr(123), + Merged: github.Ptr(false), }, } payload, _ := json.Marshal(prEvent) - req := httptest.NewRequest("POST", "/webhook", bytes.NewReader(payload)) + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) req.Header.Set("X-GitHub-Event", "pull_request") w := httptest.NewRecorder() @@ -261,26 +289,22 @@ func TestHandleWebhookWithContainer_MergedPR(t *testing.T) { // the webhook handler returns the correct HTTP response. // Set up environment variables to prevent ConfigurePermissions from failing - // We don't clean these up because: - // 1. The background goroutine may still need them after the test completes - // 2. TestMain in github_write_to_target_test.go sets them up properly anyway - // 3. These are test values that won't affect other tests - os.Setenv(configs.AppId, "123456") - os.Setenv(configs.InstallationId, "789012") - os.Setenv(configs.ConfigRepoOwner, "test-owner") - os.Setenv(configs.ConfigRepoName, "test-repo") - os.Setenv("SKIP_SECRET_MANAGER", "true") + // t.Setenv auto-cleans up after the test + t.Setenv(configs.AppId, "123456") + t.Setenv(configs.InstallationId, "789012") + t.Setenv(configs.ConfigRepoOwner, "test-owner") + t.Setenv(configs.ConfigRepoName, "test-repo") + t.Setenv("SKIP_SECRET_MANAGER", "true") // Generate a valid RSA private key for testing key, _ := rsa.GenerateKey(rand.Reader, 1024) der := x509.MarshalPKCS1PrivateKey(key) pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der}) - os.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) - os.Setenv("GITHUB_APP_PRIVATE_KEY_B64", base64.StdEncoding.EncodeToString(pemBytes)) + t.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) + t.Setenv("GITHUB_APP_PRIVATE_KEY_B64", base64.StdEncoding.EncodeToString(pemBytes)) - // Set InstallationAccessToken to prevent ConfigurePermissions from being called - // We don't reset this because the background goroutine may still need it after the test completes - InstallationAccessToken = "test-token" + // Set installation access token to prevent ConfigurePermissions from being called + defaultTokenManager.SetInstallationAccessToken("test-token") config := &configs.Config{ ConfigRepoOwner: "test-owner", @@ -296,31 +320,34 @@ func TestHandleWebhookWithContainer_MergedPR(t *testing.T) { // Create a merged PR event prEvent := &github.PullRequestEvent{ - Action: github.String("closed"), + Action: github.Ptr("closed"), PullRequest: &github.PullRequest{ - Number: github.Int(123), - Merged: github.Bool(true), - MergeCommitSHA: github.String("abc123"), + Number: github.Ptr(123), + Merged: github.Ptr(true), + MergeCommitSHA: github.Ptr("abc123"), Base: &github.PullRequestBranch{ - Ref: github.String("main"), + Ref: github.Ptr("main"), }, }, Repo: &github.Repository{ - Name: github.String("test-repo"), + Name: github.Ptr("test-repo"), Owner: &github.User{ - Login: github.String("test-owner"), + Login: github.Ptr("test-owner"), }, }, } payload, _ := json.Marshal(prEvent) - req := httptest.NewRequest("POST", "/webhook", bytes.NewReader(payload)) + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) req.Header.Set("X-GitHub-Event", "pull_request") w := httptest.NewRecorder() HandleWebhookWithContainer(w, req, config, container) + // Wait for background goroutine to finish to avoid race conditions during cleanup + container.Wait() + // Should return 202 Accepted for merged PRs if w.Code != http.StatusAccepted { t.Errorf("Status code = %d, want %d", w.Code, http.StatusAccepted) @@ -332,9 +359,6 @@ func TestHandleWebhookWithContainer_MergedPR(t *testing.T) { if response["status"] != "accepted" { t.Errorf("Response status = %v, want accepted", response["status"]) } - - // Note: The background goroutine will continue running and will eventually fail - // when trying to access GitHub APIs. This is expected and doesn't affect the test result. } func TestHandleWebhookWithContainer_MergedPRToDevelopmentBranch(t *testing.T) { @@ -343,20 +367,20 @@ func TestHandleWebhookWithContainer_MergedPRToDevelopmentBranch(t *testing.T) { // (assuming workflows are configured for main branch only) // Set up environment variables - os.Setenv(configs.AppId, "123456") - os.Setenv(configs.InstallationId, "789012") - os.Setenv(configs.ConfigRepoOwner, "test-owner") - os.Setenv(configs.ConfigRepoName, "test-repo") - os.Setenv("SKIP_SECRET_MANAGER", "true") + t.Setenv(configs.AppId, "123456") + t.Setenv(configs.InstallationId, "789012") + t.Setenv(configs.ConfigRepoOwner, "test-owner") + t.Setenv(configs.ConfigRepoName, "test-repo") + t.Setenv("SKIP_SECRET_MANAGER", "true") // Generate a valid RSA private key for testing key, _ := rsa.GenerateKey(rand.Reader, 1024) der := x509.MarshalPKCS1PrivateKey(key) pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der}) - os.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) - os.Setenv("GITHUB_APP_PRIVATE_KEY_B64", base64.StdEncoding.EncodeToString(pemBytes)) + t.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) + t.Setenv("GITHUB_APP_PRIVATE_KEY_B64", base64.StdEncoding.EncodeToString(pemBytes)) - InstallationAccessToken = "test-token" + defaultTokenManager.SetInstallationAccessToken("test-token") config := &configs.Config{ ConfigRepoOwner: "test-owner", @@ -372,31 +396,34 @@ func TestHandleWebhookWithContainer_MergedPRToDevelopmentBranch(t *testing.T) { // Create a merged PR event to development branch prEvent := &github.PullRequestEvent{ - Action: github.String("closed"), + Action: github.Ptr("closed"), PullRequest: &github.PullRequest{ - Number: github.Int(456), - Merged: github.Bool(true), - MergeCommitSHA: github.String("def456"), + Number: github.Ptr(456), + Merged: github.Ptr(true), + MergeCommitSHA: github.Ptr("def456"), Base: &github.PullRequestBranch{ - Ref: github.String("development"), + Ref: github.Ptr("development"), }, }, Repo: &github.Repository{ - Name: github.String("test-repo"), + Name: github.Ptr("test-repo"), Owner: &github.User{ - Login: github.String("test-owner"), + Login: github.Ptr("test-owner"), }, }, } payload, _ := json.Marshal(prEvent) - req := httptest.NewRequest("POST", "/webhook", bytes.NewReader(payload)) + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) req.Header.Set("X-GitHub-Event", "pull_request") w := httptest.NewRecorder() HandleWebhookWithContainer(w, req, config, container) + // Wait for background goroutine to finish to avoid race conditions during cleanup + container.Wait() + // Should still return 202 Accepted (webhook accepts the event) if w.Code != http.StatusAccepted { t.Errorf("Status code = %d, want %d", w.Code, http.StatusAccepted) @@ -408,11 +435,6 @@ func TestHandleWebhookWithContainer_MergedPRToDevelopmentBranch(t *testing.T) { if response["status"] != "accepted" { t.Errorf("Response status = %v, want accepted", response["status"]) } - - // Note: The background goroutine will fail to find matching workflows - // because the workflow config specifies main branch, not development. - // This is the expected behavior - the webhook accepts the event but - // no workflows will be processed. } func TestHandleWebhookWithContainer_MergedPRWithDifferentBranches(t *testing.T) { @@ -447,19 +469,19 @@ func TestHandleWebhookWithContainer_MergedPRWithDifferentBranches(t *testing.T) } // Set up environment variables - os.Setenv(configs.AppId, "123456") - os.Setenv(configs.InstallationId, "789012") - os.Setenv(configs.ConfigRepoOwner, "test-owner") - os.Setenv(configs.ConfigRepoName, "test-repo") - os.Setenv("SKIP_SECRET_MANAGER", "true") + t.Setenv(configs.AppId, "123456") + t.Setenv(configs.InstallationId, "789012") + t.Setenv(configs.ConfigRepoOwner, "test-owner") + t.Setenv(configs.ConfigRepoName, "test-repo") + t.Setenv("SKIP_SECRET_MANAGER", "true") key, _ := rsa.GenerateKey(rand.Reader, 1024) der := x509.MarshalPKCS1PrivateKey(key) pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der}) - os.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) - os.Setenv("GITHUB_APP_PRIVATE_KEY_B64", base64.StdEncoding.EncodeToString(pemBytes)) + t.Setenv("GITHUB_APP_PRIVATE_KEY", string(pemBytes)) + t.Setenv("GITHUB_APP_PRIVATE_KEY_B64", base64.StdEncoding.EncodeToString(pemBytes)) - InstallationAccessToken = "test-token" + defaultTokenManager.SetInstallationAccessToken("test-token") for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -477,31 +499,34 @@ func TestHandleWebhookWithContainer_MergedPRWithDifferentBranches(t *testing.T) // Create a merged PR event with specific base branch prEvent := &github.PullRequestEvent{ - Action: github.String("closed"), + Action: github.Ptr("closed"), PullRequest: &github.PullRequest{ - Number: github.Int(tc.prNumber), - Merged: github.Bool(true), - MergeCommitSHA: github.String("abc123"), + Number: github.Ptr(tc.prNumber), + Merged: github.Ptr(true), + MergeCommitSHA: github.Ptr("abc123"), Base: &github.PullRequestBranch{ - Ref: github.String(tc.baseBranch), + Ref: github.Ptr(tc.baseBranch), }, }, Repo: &github.Repository{ - Name: github.String("test-repo"), + Name: github.Ptr("test-repo"), Owner: &github.User{ - Login: github.String("test-owner"), + Login: github.Ptr("test-owner"), }, }, } payload, _ := json.Marshal(prEvent) - req := httptest.NewRequest("POST", "/webhook", bytes.NewReader(payload)) + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) req.Header.Set("X-GitHub-Event", "pull_request") w := httptest.NewRecorder() HandleWebhookWithContainer(w, req, config, container) + // Wait for background goroutine to finish to avoid race conditions during cleanup + container.Wait() + // Should return 202 Accepted for all merged PRs if w.Code != http.StatusAccepted { t.Errorf("Status code = %d, want %d", w.Code, http.StatusAccepted) @@ -517,16 +542,143 @@ func TestHandleWebhookWithContainer_MergedPRWithDifferentBranches(t *testing.T) } } +func TestHandleWebhookWithContainer_DuplicateDelivery(t *testing.T) { + config := &configs.Config{ + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: false, + } + + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer() error = %v", err) + } + + // Create a push event payload (non-PR, returns 204) + pushEvent := map[string]interface{}{ + "ref": "refs/heads/main", + } + payload, _ := json.Marshal(pushEvent) + + deliveryID := "test-delivery-abc-123" + + // First request with this delivery ID should be processed normally + req1 := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) + req1.Header.Set("X-GitHub-Event", "push") + req1.Header.Set("X-GitHub-Delivery", deliveryID) + w1 := httptest.NewRecorder() + + HandleWebhookWithContainer(w1, req1, config, container) + + if w1.Code != http.StatusNoContent { + t.Errorf("First request: status code = %d, want %d", w1.Code, http.StatusNoContent) + } + + // Second request with the same delivery ID should be rejected as duplicate + req2 := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) + req2.Header.Set("X-GitHub-Event", "push") + req2.Header.Set("X-GitHub-Delivery", deliveryID) + w2 := httptest.NewRecorder() + + HandleWebhookWithContainer(w2, req2, config, container) + + if w2.Code != http.StatusOK { + t.Errorf("Duplicate request: status code = %d, want %d (duplicate should return 200 OK)", w2.Code, http.StatusOK) + } + + // Third request with a different delivery ID should be processed + req3 := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) + req3.Header.Set("X-GitHub-Event", "push") + req3.Header.Set("X-GitHub-Delivery", "different-delivery-456") + w3 := httptest.NewRecorder() + + HandleWebhookWithContainer(w3, req3, config, container) + + if w3.Code != http.StatusNoContent { + t.Errorf("Different delivery: status code = %d, want %d", w3.Code, http.StatusNoContent) + } +} + +func TestHandleWebhookWithContainer_NoDeliveryHeader(t *testing.T) { + config := &configs.Config{ + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: false, + } + + container, err := NewServiceContainer(config) + if err != nil { + t.Fatalf("NewServiceContainer() error = %v", err) + } + + // Request without X-GitHub-Delivery header should still be processed + pushEvent := map[string]interface{}{ + "ref": "refs/heads/main", + } + payload, _ := json.Marshal(pushEvent) + + req := httptest.NewRequest("POST", "/events", bytes.NewReader(payload)) + req.Header.Set("X-GitHub-Event", "push") + // No X-GitHub-Delivery header + + w := httptest.NewRecorder() + + HandleWebhookWithContainer(w, req, config, container) + + if w.Code != http.StatusNoContent { + t.Errorf("Status code = %d, want %d", w.Code, http.StatusNoContent) + } +} + func TestRetrieveFileContentsWithConfigAndBranch(t *testing.T) { - // This test would require mocking the GitHub client - // For now, we document the expected behavior - t.Skip("Skipping test that requires GitHub API mocking") - - // Expected behavior: - // - Should call client.Repositories.GetContents with correct parameters - // - Should use the specified branch in RepositoryContentGetOptions - // - Should return file content on success - // - Should return error on failure + // Use global httpmock since the REST client constructed inside the function + // creates its own http.Client transport chain. + httpmock.Activate() + t.Cleanup(httpmock.DeactivateAndReset) + + owner := "test-owner" + repo := "test-repo" + branch := "main" + filePath := "examples/hello.go" + expectedContent := "package main\n\nfunc main() {}\n" + + defaultTokenManager.SetInstallationAccessToken("test-token") + SetInstallationTokenForOrg(owner, "test-token") + + httpmock.RegisterResponder("GET", + "https://api.github.com/repos/"+owner+"/"+repo+"/contents/"+filePath, + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "type": "file", + "name": "hello.go", + "path": filePath, + "encoding": "base64", + "content": base64.StdEncoding.EncodeToString([]byte(expectedContent)), + }), + ) + + config := &configs.Config{ + ConfigRepoOwner: owner, + ConfigRepoName: repo, + } + + fileContent, err := RetrieveFileContentsWithConfigAndBranch( + context.Background(), config, filePath, branch, owner, repo, + ) + if err != nil { + t.Fatalf("RetrieveFileContentsWithConfigAndBranch: %v", err) + } + + if fileContent == nil { + t.Fatal("expected non-nil file content") + } + + content, err := fileContent.GetContent() + if err != nil { + t.Fatalf("GetContent: %v", err) + } + if content != expectedContent { + t.Errorf("content = %q, want %q", content, expectedContent) + } } func TestMaxWebhookBodyBytes(t *testing.T) { diff --git a/services/workflow_processor.go b/services/workflow_processor.go index a766907..c29d1b1 100644 --- a/services/workflow_processor.go +++ b/services/workflow_processor.go @@ -4,17 +4,21 @@ import ( "context" "fmt" "path/filepath" + "regexp" "strings" + "sync" "time" "github.com/bmatcuk/doublestar/v4" - "github.com/google/go-github/v48/github" - . "github.com/grove-platform/github-copier/types" + "github.com/google/go-github/v82/github" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/types" + "golang.org/x/sync/errgroup" ) // WorkflowProcessor processes workflows and applies transformations type WorkflowProcessor interface { - ProcessWorkflow(ctx context.Context, workflow Workflow, changedFiles []ChangedFile, prNumber int, sourceCommitSHA string) error + ProcessWorkflow(ctx context.Context, workflow types.Workflow, changedFiles []types.ChangedFile, prNumber int, sourceCommitSHA string) error } // workflowProcessor implements WorkflowProcessor @@ -24,6 +28,7 @@ type workflowProcessor struct { fileStateService FileStateService metricsCollector *MetricsCollector messageTemplater MessageTemplater + config *configs.Config } // NewWorkflowProcessor creates a new workflow processor @@ -33,6 +38,7 @@ func NewWorkflowProcessor( fileStateService FileStateService, metricsCollector *MetricsCollector, messageTemplater MessageTemplater, + config *configs.Config, ) WorkflowProcessor { return &workflowProcessor{ patternMatcher: patternMatcher, @@ -40,14 +46,33 @@ func NewWorkflowProcessor( fileStateService: fileStateService, metricsCollector: metricsCollector, messageTemplater: messageTemplater, + config: config, } } -// ProcessWorkflow processes a single workflow +// matchResult holds the outcome of the match phase for a single file. +type matchResult struct { + workflow types.Workflow + file types.ChangedFile + targetPath string + isDelete bool + prNumber int + sourceCommitSHA string + fileContent *github.RepositoryContent // populated by fetch phase +} + +// maxConcurrentFetches limits parallel GitHub API calls per workflow to avoid +// hitting secondary rate limits. +const maxConcurrentFetches = 5 + +// ProcessWorkflow processes a single workflow in three phases: +// 1. Match โ€” identify files that match transformations (fast, no I/O) +// 2. Fetch โ€” retrieve file contents from GitHub in parallel +// 3. Queue โ€” add fetched files to the upload queue (sequential, mutates shared state) func (wp *workflowProcessor) ProcessWorkflow( ctx context.Context, - workflow Workflow, - changedFiles []ChangedFile, + workflow types.Workflow, + changedFiles []types.ChangedFile, prNumber int, sourceCommitSHA string, ) error { @@ -58,26 +83,75 @@ func (wp *workflowProcessor) ProcessWorkflow( "file_count": len(changedFiles), }) - // Track files matched and skipped - filesMatched := 0 + // โ”€โ”€ Phase 1: Match โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + var matches []matchResult filesSkipped := 0 - // Process each changed file for _, file := range changedFiles { - matched, err := wp.processFileForWorkflow(ctx, workflow, file, prNumber, sourceCommitSHA) - if err != nil { - LogErrorCtx(ctx, "Failed to process file for workflow", err, map[string]interface{}{ - "workflow_name": workflow.Name, - "file_path": file.Path, - }) + mr, matched := wp.matchFile(ctx, workflow, file, prNumber, sourceCommitSHA) + if !matched { + filesSkipped++ continue } + matches = append(matches, mr) + } + + if len(matches) == 0 { + LogInfoCtx(ctx, "Workflow processing complete", map[string]interface{}{ + "workflow_name": workflow.Name, + "files_matched": 0, + "files_skipped": filesSkipped, + }) + return nil + } + + // โ”€โ”€ Phase 2: Fetch (parallel) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(maxConcurrentFetches) - if matched { + var fetchErrMu sync.Mutex + var fetchErrors []error + + for i := range matches { + if matches[i].isDelete { + continue + } + mr := &matches[i] + g.Go(func() error { + fc, err := wp.fetchFileContent(gctx, workflow, mr.file, mr.sourceCommitSHA) + if err != nil { + fetchErrMu.Lock() + fetchErrors = append(fetchErrors, fmt.Errorf("fetch %s: %w", mr.file.Path, err)) + fetchErrMu.Unlock() + return nil // don't abort sibling fetches + } + mr.fileContent = fc + return nil + }) + } + _ = g.Wait() + + for _, fe := range fetchErrors { + LogErrorCtx(ctx, "Failed to fetch file content", fe, map[string]interface{}{ + "workflow_name": workflow.Name, + }) + } + + // โ”€โ”€ Phase 3: Queue (sequential) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + filesMatched := 0 + for i := range matches { + mr := &matches[i] + if mr.isDelete { + wp.addToDeprecationMap(mr.workflow, mr.targetPath, mr.file.Path, mr.prNumber) filesMatched++ - } else { - filesSkipped++ + continue + } + if mr.fileContent == nil { + continue // fetch failed โ€” already logged } + mr.fileContent.Name = github.Ptr(mr.targetPath) + wp.queueUpload(ctx, mr.workflow, mr.fileContent, mr.targetPath, mr.prNumber, mr.sourceCommitSHA) + filesMatched++ } LogInfoCtx(ctx, "Workflow processing complete", map[string]interface{}{ @@ -89,35 +163,38 @@ func (wp *workflowProcessor) ProcessWorkflow( return nil } -// processFileForWorkflow processes a single file for a workflow -func (wp *workflowProcessor) processFileForWorkflow( +// matchFile checks exclusions and transformations for a single file. +// Returns a matchResult and true if the file matched a transformation. +func (wp *workflowProcessor) matchFile( ctx context.Context, - workflow Workflow, - file ChangedFile, + workflow types.Workflow, + file types.ChangedFile, prNumber int, sourceCommitSHA string, -) (bool, error) { - // Check if file is excluded +) (matchResult, bool) { if wp.isExcluded(file.Path, workflow.Exclude) { LogInfoCtx(ctx, "File excluded by workflow exclude patterns", map[string]interface{}{ "workflow_name": workflow.Name, "file_path": file.Path, }) - return false, nil + return matchResult{}, false } - // Try each transformation until one matches for i, transformation := range workflow.Transformations { matched, targetPath, err := wp.applyTransformation(ctx, workflow, transformation, file.Path) if err != nil { - return false, fmt.Errorf("transformation[%d]: %w", i, err) + LogErrorCtx(ctx, "Failed to apply transformation", err, map[string]interface{}{ + "workflow_name": workflow.Name, + "transformation_idx": i, + "file_path": file.Path, + }) + return matchResult{}, false } if !matched { continue } - // File matched this transformation LogInfoCtx(ctx, "File matched transformation", map[string]interface{}{ "workflow_name": workflow.Name, "transformation_idx": i, @@ -126,46 +203,53 @@ func (wp *workflowProcessor) processFileForWorkflow( "target_path": targetPath, }) - // Handle file based on status - // GitHub GraphQL API returns uppercase status: "DELETED", "ADDED", "MODIFIED", etc. - if file.Status == "DELETED" || file.Status == "removed" { - // Add to deprecation map - wp.addToDeprecationMap(workflow, targetPath) - } else { - // Add to upload queue - err := wp.addToUploadQueue(ctx, workflow, file, targetPath, prNumber, sourceCommitSHA) - if err != nil { - return false, fmt.Errorf("failed to queue file for upload: %w", err) - } - } - - return true, nil + isDelete := file.Status == "DELETED" || file.Status == "removed" + return matchResult{ + workflow: workflow, + file: file, + targetPath: targetPath, + isDelete: isDelete, + prNumber: prNumber, + sourceCommitSHA: sourceCommitSHA, + }, true } - // No transformation matched LogInfoCtx(ctx, "File did not match any transformation", map[string]interface{}{ "workflow_name": workflow.Name, "file_path": file.Path, }) + return matchResult{}, false +} - return false, nil +// fetchFileContent retrieves a single file's content from the source repository. +func (wp *workflowProcessor) fetchFileContent( + ctx context.Context, + workflow types.Workflow, + file types.ChangedFile, + sourceCommitSHA string, +) (*github.RepositoryContent, error) { + parts := strings.Split(workflow.Source.Repo, "/") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid source repo format: expected owner/repo, got: %s", workflow.Source.Repo) + } + return RetrieveFileContentsWithConfigAndBranch(ctx, wp.config, file.Path, sourceCommitSHA, parts[0], parts[1]) } // applyTransformation applies a transformation to a file path func (wp *workflowProcessor) applyTransformation( ctx context.Context, - workflow Workflow, - transformation Transformation, + workflow types.Workflow, + transformation types.Transformation, sourcePath string, ) (matched bool, targetPath string, err error) { switch transformation.GetType() { - case TransformationTypeMove: + case types.TransformationTypeMove: return wp.applyMoveTransformation(transformation.Move, sourcePath) - case TransformationTypeCopy: + case types.TransformationTypeCopy: return wp.applyCopyTransformation(transformation.Copy, sourcePath) - case TransformationTypeGlob: + case types.TransformationTypeGlob: return wp.applyGlobTransformation(transformation.Glob, sourcePath) - case TransformationTypeRegex: + case types.TransformationTypeRegex: return wp.applyRegexTransformation(transformation.Regex, sourcePath) default: return false, "", fmt.Errorf("unknown transformation type: %s", transformation.GetType()) @@ -174,7 +258,7 @@ func (wp *workflowProcessor) applyTransformation( // applyMoveTransformation applies a move transformation func (wp *workflowProcessor) applyMoveTransformation( - move *MoveTransform, + move *types.MoveTransform, sourcePath string, ) (matched bool, targetPath string, err error) { // Check if source path starts with the "from" prefix @@ -197,7 +281,7 @@ func (wp *workflowProcessor) applyMoveTransformation( // applyCopyTransformation applies a copy transformation func (wp *workflowProcessor) applyCopyTransformation( - copy *CopyTransform, + copy *types.CopyTransform, sourcePath string, ) (matched bool, targetPath string, err error) { // Copy only matches exact file path @@ -209,7 +293,7 @@ func (wp *workflowProcessor) applyCopyTransformation( // applyGlobTransformation applies a glob transformation func (wp *workflowProcessor) applyGlobTransformation( - glob *GlobTransform, + glob *types.GlobTransform, sourcePath string, ) (matched bool, targetPath string, err error) { // Use doublestar for glob matching @@ -235,12 +319,12 @@ func (wp *workflowProcessor) applyGlobTransformation( // applyRegexTransformation applies a regex transformation func (wp *workflowProcessor) applyRegexTransformation( - regex *RegexTransform, + regex *types.RegexTransform, sourcePath string, ) (matched bool, targetPath string, err error) { // Use existing pattern matcher for regex - sourcePattern := SourcePattern{ - Type: PatternTypeRegex, + sourcePattern := types.SourcePattern{ + Type: types.PatternTypeRegex, Pattern: regex.Pattern, } @@ -280,86 +364,99 @@ func (wp *workflowProcessor) extractGlobVariables(pattern, path string) map[stri return variables } -// isExcluded checks if a file path matches any exclude pattern +// isExcluded checks if a file path matches any exclude pattern (using regex) func (wp *workflowProcessor) isExcluded(path string, excludePatterns []string) bool { for _, pattern := range excludePatterns { - matched, err := doublestar.Match(pattern, path) + re, err := regexp.Compile(pattern) if err != nil { - LogWarning(fmt.Sprintf("Invalid exclude pattern: %s: %v", pattern, err)) + LogWarning("Invalid exclude regex pattern", "pattern", pattern, "error", err) continue } - if matched { + if re.MatchString(path) { return true } } return false } -// addToDeprecationMap adds a file to the deprecation map -func (wp *workflowProcessor) addToDeprecationMap(workflow Workflow, targetPath string) { +// addToDeprecationMap adds a file to the deprecation map if deprecation tracking is enabled +func (wp *workflowProcessor) addToDeprecationMap(workflow types.Workflow, targetPath string, sourcePath string, prNumber int) { + // Only track deprecations if explicitly enabled + if workflow.DeprecationCheck == nil || !workflow.DeprecationCheck.Enabled { + return + } + deprecationFile := "deprecated_examples.json" - if workflow.DeprecationCheck != nil && workflow.DeprecationCheck.File != "" { + if workflow.DeprecationCheck.File != "" { deprecationFile = workflow.DeprecationCheck.File } - entry := DeprecatedFileEntry{ - FileName: targetPath, - Repo: workflow.Destination.Repo, - Branch: workflow.Destination.Branch, + entry := types.DeprecatedFileEntry{ + FileName: targetPath, + Repo: workflow.Destination.Repo, + Branch: workflow.Destination.Branch, + SourcePath: sourcePath, + PRNumber: prNumber, } wp.fileStateService.AddFileToDeprecate(deprecationFile, entry) } -// addToUploadQueue adds a file to the upload queue -func (wp *workflowProcessor) addToUploadQueue( +// queueUpload adds a pre-fetched file to the upload queue. The fileContent must +// already have its Name set to the target path. +func (wp *workflowProcessor) queueUpload( ctx context.Context, - workflow Workflow, - file ChangedFile, + workflow types.Workflow, + fileContent *github.RepositoryContent, targetPath string, prNumber int, sourceCommitSHA string, -) error { - // Parse source repo owner/name - parts := strings.Split(workflow.Source.Repo, "/") - if len(parts) != 2 { - return fmt.Errorf("invalid source repo format: expected owner/repo, got: %s", workflow.Source.Repo) - } - sourceRepoOwner := parts[0] - sourceRepoName := parts[1] - - // Fetch file content from source repository - fileContent, err := RetrieveFileContentsWithConfigAndBranch(ctx, file.Path, sourceCommitSHA, sourceRepoOwner, sourceRepoName) - if err != nil { - return fmt.Errorf("failed to retrieve file content: %w", err) - } - - // Update file name to target path - fileContent.Name = github.String(targetPath) - - // Create upload key - key := UploadKey{ - RepoName: workflow.Destination.Repo, - BranchPath: workflow.Destination.Branch, +) { + + // Create upload key โ€” includes CommitStrategy so that workflows with + // different strategies targeting the same repo produce separate operations. + key := types.UploadKey{ + RepoName: workflow.Destination.Repo, + BranchPath: workflow.Destination.Branch, + CommitStrategy: getCommitStrategyType(workflow), } // Get existing entries from FileStateService filesToUpload := wp.fileStateService.GetFilesToUpload() content, exists := filesToUpload[key] if !exists { - content = UploadFileContent{ + content = types.UploadFileContent{ Content: []github.RepositoryContent{}, - CommitStrategy: CommitStrategy(getCommitStrategyType(workflow)), + CommitStrategy: types.CommitStrategy(getCommitStrategyType(workflow)), UsePRTemplate: getUsePRTemplate(workflow), AutoMergePR: getAutoMerge(workflow), } + } else { + // When batching multiple workflows, use AND logic for auto-merge (conservative): + // auto-merge is only enabled if ALL workflows in the batch want it. + // Log a warning when workflows have conflicting auto-merge settings. + workflowAutoMerge := getAutoMerge(workflow) + if workflowAutoMerge != content.AutoMergePR { + LogWarning("Workflows in batch have conflicting auto_merge settings; using AND logic (auto-merge disabled)", + "workflow", workflow.Name, + "target", key.RepoName, + "workflow_auto_merge", workflowAutoMerge, + "batch_auto_merge", content.AutoMergePR, + ) + // AND logic: if either is false, result is false + content.AutoMergePR = false + } + // For PR template, use OR logic - if any workflow wants it, use it + if getUsePRTemplate(workflow) && !content.UsePRTemplate { + content.UsePRTemplate = true + } } // Add file to content content.Content = append(content.Content, *fileContent) // Render templates with message context - msgCtx := NewMessageContext() + msgCtx := types.NewMessageContext() msgCtx.SourceRepo = workflow.Source.Repo msgCtx.SourceBranch = workflow.Source.Branch msgCtx.TargetRepo = workflow.Destination.Repo @@ -368,6 +465,10 @@ func (wp *workflowProcessor) addToUploadQueue( msgCtx.CommitSHA = sourceCommitSHA msgCtx.FileCount = len(content.Content) + // Track previous metadata so we can log when a later workflow overwrites it. + prevCommitMsg := content.CommitMessage + prevPRTitle := content.PRTitle + // Render commit message if workflow.CommitStrategy != nil && workflow.CommitStrategy.CommitMessage != "" { content.CommitMessage = wp.messageTemplater.RenderCommitMessage(workflow.CommitStrategy.CommitMessage, msgCtx) @@ -387,6 +488,24 @@ func (wp *workflowProcessor) addToUploadQueue( content.PRBody = wp.messageTemplater.RenderPRBody(workflow.CommitStrategy.PRBody, msgCtx) } + // Log when a subsequent workflow in the same batch overwrites PR metadata. + if exists && prevCommitMsg != "" && prevCommitMsg != content.CommitMessage { + LogInfo("Workflow overwrites batched commit message (last wins)", + "workflow", workflow.Name, + "target", workflow.Destination.Repo, + "prev_commit_message", prevCommitMsg, + "new_commit_message", content.CommitMessage, + ) + } + if exists && prevPRTitle != "" && prevPRTitle != content.PRTitle { + LogInfo("Workflow overwrites batched PR title (last wins)", + "workflow", workflow.Name, + "target", workflow.Destination.Repo, + "prev_pr_title", prevPRTitle, + "new_pr_title", content.PRTitle, + ) + } + // Add back to FileStateService wp.fileStateService.AddFileToUpload(key, content) @@ -394,27 +513,25 @@ func (wp *workflowProcessor) addToUploadQueue( if wp.metricsCollector != nil { wp.metricsCollector.RecordFileUploaded(0 * time.Second) } - - return nil } // Helper functions to extract config values -func getCommitStrategyType(workflow Workflow) string { +func getCommitStrategyType(workflow types.Workflow) string { if workflow.CommitStrategy != nil && workflow.CommitStrategy.Type != "" { return workflow.CommitStrategy.Type } return "pull_request" // default } -func getUsePRTemplate(workflow Workflow) bool { +func getUsePRTemplate(workflow types.Workflow) bool { if workflow.CommitStrategy != nil { return workflow.CommitStrategy.UsePRTemplate } return false } -func getAutoMerge(workflow Workflow) bool { +func getAutoMerge(workflow types.Workflow) bool { if workflow.CommitStrategy != nil { return workflow.CommitStrategy.AutoMerge } diff --git a/services/workflow_processor_test.go b/services/workflow_processor_test.go index 6ea30db..fb7f749 100644 --- a/services/workflow_processor_test.go +++ b/services/workflow_processor_test.go @@ -8,6 +8,8 @@ import ( "github.com/grove-platform/github-copier/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + test "github.com/grove-platform/github-copier/tests" ) // ============================================================================ @@ -57,6 +59,13 @@ func createTestWorkflow(name string, transformations []types.Transformation) typ } } +// createTestWorkflowWithDeprecation creates a test workflow with deprecation tracking enabled +func createTestWorkflowWithDeprecation(name string, transformations []types.Transformation) types.Workflow { + wf := createTestWorkflow(name, transformations) + wf.DeprecationCheck = &types.DeprecationConfig{Enabled: true} + return wf +} + func createMoveTransformation(from, to string) types.Transformation { return types.Transformation{ Move: &types.MoveTransform{From: from, To: to}, @@ -154,9 +163,10 @@ func TestWorkflowProcessor_MoveTransformation(t *testing.T) { fileStateService, nil, // metrics collector &mockMessageTemplater{}, + test.TestConfig(), ) - workflow := createTestWorkflow("test-workflow", []types.Transformation{ + workflow := createTestWorkflowWithDeprecation("test-workflow", []types.Transformation{ createMoveTransformation(tt.from, tt.to), }) @@ -223,9 +233,10 @@ func TestWorkflowProcessor_CopyTransformation(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) - workflow := createTestWorkflow("test-workflow", []types.Transformation{ + workflow := createTestWorkflowWithDeprecation("test-workflow", []types.Transformation{ createCopyTransformation(tt.from, tt.to), }) @@ -297,9 +308,10 @@ func TestWorkflowProcessor_GlobTransformation(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) - workflow := createTestWorkflow("test-workflow", []types.Transformation{ + workflow := createTestWorkflowWithDeprecation("test-workflow", []types.Transformation{ createGlobTransformation(tt.pattern, tt.transform), }) @@ -364,9 +376,10 @@ func TestWorkflowProcessor_RegexTransformation(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) - workflow := createTestWorkflow("test-workflow", []types.Transformation{ + workflow := createTestWorkflowWithDeprecation("test-workflow", []types.Transformation{ createRegexTransformation(tt.pattern, tt.transform), }) @@ -392,7 +405,7 @@ func TestWorkflowProcessor_RegexTransformation(t *testing.T) { // ============================================================================ func TestWorkflowProcessor_ExcludePatterns(t *testing.T) { - // Note: Exclude patterns use glob matching (doublestar), not regex + // Note: Exclude patterns use regex matching (consistent with documentation and SourcePattern.ExcludePatterns) tests := []struct { name string exclude []string @@ -400,47 +413,65 @@ func TestWorkflowProcessor_ExcludePatterns(t *testing.T) { wantExcluded bool }{ { - name: "exclude by extension glob", - exclude: []string{"**/*_test.go"}, + name: "exclude by extension regex", + exclude: []string{".*_test\\.go$"}, sourcePath: "src/main_test.go", wantExcluded: true, }, { - name: "exclude by directory glob", - exclude: []string{"vendor/**"}, + name: "exclude by directory regex", + exclude: []string{"^vendor/"}, sourcePath: "vendor/pkg/lib.go", wantExcluded: true, }, { - name: "exclude by filename glob", - exclude: []string{"**/.DS_Store"}, + name: "exclude by filename regex", + exclude: []string{"\\.DS_Store$"}, sourcePath: "src/.DS_Store", wantExcluded: true, }, { name: "not excluded - no match", - exclude: []string{"**/*_test.go"}, + exclude: []string{".*_test\\.go$"}, sourcePath: "src/main.go", wantExcluded: false, }, { name: "multiple exclude patterns - first matches", - exclude: []string{"**/*_test.go", "vendor/**"}, + exclude: []string{".*_test\\.go$", "^vendor/"}, sourcePath: "src/main_test.go", wantExcluded: true, }, { name: "multiple exclude patterns - second matches", - exclude: []string{"**/*_test.go", "vendor/**"}, + exclude: []string{".*_test\\.go$", "^vendor/"}, sourcePath: "vendor/lib.go", wantExcluded: true, }, { name: "multiple exclude patterns - none match", - exclude: []string{"**/*_test.go", "vendor/**"}, + exclude: []string{".*_test\\.go$", "^vendor/"}, sourcePath: "src/main.go", wantExcluded: false, }, + { + name: "exclude spec files", + exclude: []string{".*\\.spec\\.ts$"}, + sourcePath: "src/utils/logger.spec.ts", + wantExcluded: true, + }, + { + name: "exclude eslint config", + exclude: []string{"\\.eslintrc"}, + sourcePath: "examples/typescript/.eslintrc.json", + wantExcluded: true, + }, + { + name: "exclude node_modules directory", + exclude: []string{"node_modules/"}, + sourcePath: "examples/typescript/node_modules/pkg/index.js", + wantExcluded: true, + }, } for _, tt := range tests { @@ -452,6 +483,7 @@ func TestWorkflowProcessor_ExcludePatterns(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) workflow := types.Workflow{ @@ -468,7 +500,8 @@ func TestWorkflowProcessor_ExcludePatterns(t *testing.T) { createMoveTransformation("src", "dest"), createMoveTransformation("vendor", "vendor"), }, - Exclude: tt.exclude, + Exclude: tt.exclude, + DeprecationCheck: &types.DeprecationConfig{Enabled: true}, } changedFiles := []types.ChangedFile{ @@ -503,6 +536,7 @@ func TestWorkflowProcessor_MultipleTransformations(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) workflow := types.Workflow{ @@ -520,6 +554,7 @@ func TestWorkflowProcessor_MultipleTransformations(t *testing.T) { createMoveTransformation("docs", "documentation"), createCopyTransformation("README.md", "docs/README.md"), }, + DeprecationCheck: &types.DeprecationConfig{Enabled: true}, } changedFiles := []types.ChangedFile{ @@ -564,6 +599,7 @@ func TestWorkflowProcessor_EmptyChangedFiles(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) workflow := createTestWorkflow("test-workflow", []types.Transformation{ @@ -587,6 +623,7 @@ func TestWorkflowProcessor_NoTransformations(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) workflow := createTestWorkflow("test-workflow", []types.Transformation{}) @@ -614,6 +651,7 @@ func TestWorkflowProcessor_InvalidExcludePattern(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) workflow := types.Workflow{ @@ -630,7 +668,8 @@ func TestWorkflowProcessor_InvalidExcludePattern(t *testing.T) { createMoveTransformation("src", "dest"), }, // Invalid glob pattern - should be handled gracefully - Exclude: []string{"[invalid"}, + Exclude: []string{"[invalid"}, + DeprecationCheck: &types.DeprecationConfig{Enabled: true}, } changedFiles := []types.ChangedFile{ @@ -658,6 +697,7 @@ func TestWorkflowProcessor_CustomDeprecationFile(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) workflow := types.Workflow{ @@ -674,7 +714,8 @@ func TestWorkflowProcessor_CustomDeprecationFile(t *testing.T) { createMoveTransformation("src", "dest"), }, DeprecationCheck: &types.DeprecationConfig{ - File: "custom_deprecation.json", + Enabled: true, + File: "custom_deprecation.json", }, } @@ -690,6 +731,8 @@ func TestWorkflowProcessor_CustomDeprecationFile(t *testing.T) { assert.True(t, exists, "expected custom deprecation file to be used") require.Len(t, entries, 1, "expected one entry in custom deprecation file") assert.Equal(t, "dest/main.go", entries[0].FileName) + assert.Equal(t, "src/main.go", entries[0].SourcePath, "expected source path to be recorded") + assert.Equal(t, 1, entries[0].PRNumber, "expected PR number to be recorded") } // ============================================================================ @@ -733,11 +776,14 @@ func TestWorkflowProcessor_FileStatusHandling(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) workflow := createTestWorkflow("test-workflow", []types.Transformation{ createMoveTransformation("src", "dest"), }) + // Enable deprecation tracking for this test + workflow.DeprecationCheck = &types.DeprecationConfig{Enabled: true} changedFiles := []types.ChangedFile{ {Path: "src/main.go", Status: tt.status}, @@ -759,6 +805,45 @@ func TestWorkflowProcessor_FileStatusHandling(t *testing.T) { } } +func TestWorkflowProcessor_DeprecationDisabledByDefault(t *testing.T) { + // Test that deprecation tracking does NOT happen when DeprecationCheck.Enabled is false/unset + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + test.TestConfig(), + ) + + // Create workflow WITHOUT deprecation enabled + workflow := createTestWorkflow("test-workflow", []types.Transformation{ + createMoveTransformation("src", "dest"), + }) + // DeprecationCheck is nil - should not track deprecations + + changedFiles := []types.ChangedFile{ + {Path: "src/main.go", Status: "removed"}, + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + assert.Empty(t, deprecated, "expected NO deprecation tracking when DeprecationCheck is nil") + + // Also test when DeprecationCheck exists but Enabled is false + workflow.DeprecationCheck = &types.DeprecationConfig{Enabled: false, File: "test.json"} + fileStateService.ClearFilesToDeprecate() + + err = processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated = fileStateService.GetFilesToDeprecate() + assert.Empty(t, deprecated, "expected NO deprecation tracking when DeprecationCheck.Enabled is false") +} + // ============================================================================ // Tests for Path Transformation Edge Cases // ============================================================================ @@ -811,9 +896,10 @@ func TestWorkflowProcessor_PathTransformationEdgeCases(t *testing.T) { fileStateService, nil, &mockMessageTemplater{}, + test.TestConfig(), ) - workflow := createTestWorkflow("test-workflow", []types.Transformation{tt.transform}) + workflow := createTestWorkflowWithDeprecation("test-workflow", []types.Transformation{tt.transform}) changedFiles := []types.ChangedFile{ {Path: tt.sourcePath, Status: "removed"}, @@ -831,3 +917,59 @@ func TestWorkflowProcessor_PathTransformationEdgeCases(t *testing.T) { }) } } + +// ============================================================================ +// Tests for Auto-Merge Batching Behavior +// ============================================================================ + +// TestFileStateService_AutoMergeBatching_AllTrue tests that when all entries +// in a batch have auto_merge: true, the batch retains auto_merge: true. +func TestFileStateService_AutoMergeBatching_AllTrue(t *testing.T) { + fileStateService := services.NewFileStateService() + + key := types.UploadKey{ + RepoName: "test-org/dest-repo", + BranchPath: "main", + CommitStrategy: "pull_request", + } + + // First upload with auto_merge: true + fileStateService.AddFileToUpload(key, types.UploadFileContent{ + TargetBranch: "main", + CommitStrategy: "pull_request", + AutoMergePR: true, + }) + + uploads := fileStateService.GetFilesToUpload() + require.Len(t, uploads, 1, "expected one upload entry") + assert.True(t, uploads[key].AutoMergePR, "expected auto_merge to be true") +} + +// TestFileStateService_AutoMergeBatching_AllFalse tests that when all entries +// in a batch have auto_merge: false, the batch has auto_merge: false. +func TestFileStateService_AutoMergeBatching_AllFalse(t *testing.T) { + fileStateService := services.NewFileStateService() + + key := types.UploadKey{ + RepoName: "test-org/dest-repo", + BranchPath: "main", + CommitStrategy: "pull_request", + } + + // Upload with auto_merge: false + fileStateService.AddFileToUpload(key, types.UploadFileContent{ + TargetBranch: "main", + CommitStrategy: "pull_request", + AutoMergePR: false, + }) + + uploads := fileStateService.GetFilesToUpload() + require.Len(t, uploads, 1, "expected one upload entry") + assert.False(t, uploads[key].AutoMergePR, "expected auto_merge to be false") +} + +// Note: The actual conflict detection and AND logic happens in workflow_processor.go's +// queueUpload function when adding files to an existing batch. Testing this requires +// either mocking GitHub API calls or testing at the integration level. +// The conflict warning is logged when workflows with different auto_merge settings +// target the same repo, and AND logic is applied (any false results in false). diff --git a/testdata/README.md b/testdata/README.md index 8497088..cffc38d 100644 --- a/testdata/README.md +++ b/testdata/README.md @@ -1,253 +1,312 @@ -# Test Payloads +# Test Data -Test files for local integration testing of the github-copier app. - -## Quick Start (Isolated Testing) - -```bash -# 1. Copy and configure test environment -cp testdata/.env.test .env.test -# Edit .env.test with your GitHub App credentials - -# 2. Copy config to your test source repo (cbullinger/copier-app-source-test) -# Upload testdata/source-repo-files/.copier/main.yaml to .copier/main.yaml - -# 3. Start the app with test config -ENV_FILE=.env.test go run app.go - -# 4. In another terminal, start webhook tunnel -smee --url https://smee.io/YOUR_CHANNEL --target http://localhost:8080/webhook - -# 5. Create a PR in your test source repo and merge it -``` +Test fixtures for local and CI testing of the github-copier application. ## Directory Structure ``` testdata/ -โ”œโ”€โ”€ .env.test # Test environment variables (copy to .env.test) -โ”œโ”€โ”€ source-repo-files/ # Files to copy to your test source repo +โ”œโ”€โ”€ README.md # This file +โ”œโ”€โ”€ .env.test # Test environment template +โ”œโ”€โ”€ test-config.yaml # Example workflow config (inline format) +โ”œโ”€โ”€ source-repo-files/ # Files to copy to a test source repo โ”‚ โ””โ”€โ”€ .copier/ -โ”‚ โ””โ”€โ”€ main.yaml # Workflow config for test repos -โ”œโ”€โ”€ test-pr-merged.json # Sample webhook payload -โ”œโ”€โ”€ test-config.yaml # Example config (reference only) -โ””โ”€โ”€ example-pr-merged.json # Generic example payload +โ”‚ โ””โ”€โ”€ main.yaml # Main config format example +โ”‚ +โ”œโ”€โ”€ # Webhook Payloads - Merged PRs (should trigger processing) +โ”œโ”€โ”€ example-pr-merged.json # Generic merged PR example +โ”œโ”€โ”€ test-pr-merged.json # Test repo merged PR +โ”œโ”€โ”€ pr-multiple-workflows.json # Files matching multiple workflows +โ”œโ”€โ”€ pr-large-changeset.json # Large PR (35+ files) for pagination testing +โ”œโ”€โ”€ pr-renamed-files.json # Files with status: renamed +โ”œโ”€โ”€ pr-with-deprecations.json # Mix of added and removed files +โ”‚ +โ”œโ”€โ”€ # Webhook Payloads - Non-merged (should be ignored) +โ”œโ”€โ”€ pr-closed-not-merged.json # Closed without merge +โ”œโ”€โ”€ pr-opened.json # PR opened event +โ”œโ”€โ”€ pr-synchronize.json # PR updated with new commits +โ”œโ”€โ”€ pr-no-matching-files.json # Merged but no files match patterns +โ”‚ +โ””โ”€โ”€ push-to-main.json # Direct push event (if supported) ``` -## Test Repositories - -| Repo | Purpose | -|------|---------| -| `cbullinger/copier-app-source-test` | Source repo (receives PRs, triggers webhooks) | -| `cbullinger/copier-app-dest-1` | Destination for Go examples | -| `cbullinger/copier-app-dest-2` | Destination for Python examples | +## Quick Start -## Configured Workflows +### Option 1: Automated Integration Tests -The test config (`source-repo-files/.copier/main.yaml`) defines: - -| Workflow | Source Pattern | Destination | Transform | -|----------|---------------|-------------|-----------| -| `test-go-to-dest1` | `examples/go/**` | `copier-app-dest-1` | `examples/go/` โ†’ `go-examples/` | -| `test-python-to-dest2` | `examples/python/**` | `copier-app-dest-2` | `examples/python/` โ†’ `python-examples/` | -| `test-docs-to-dest1` | `docs/**` | `copier-app-dest-1` | `docs/` โ†’ `documentation/` | - -## Files +```bash +# Run full integration test suite +./scripts/integration-test.sh -### example-pr-merged.json -A complete example of a merged PR webhook payload with: -- Multiple file changes (added, modified, removed) -- Go and Python examples -- Database and auth categories -- Realistic file structure +# Quick smoke test only +./scripts/integration-test.sh --quick -## Usage +# Verbose output +./scripts/integration-test.sh --verbose +``` -### Option 1: Use Example Payload +### Option 2: Manual Testing ```bash -# Build the test tool -go build -o test-webhook ./cmd/test-webhook +# 1. Start the app in dry-run mode +make run-local-quick + +# 2. In another terminal, send a test webhook +make test-webhook-example -# Send example payload +# Or use the test-webhook CLI directly ./test-webhook -payload testdata/example-pr-merged.json ``` -### Option 2: Fetch Real PR Data +### Option 3: Test with Real GitHub Data ```bash -# Set GitHub token +# Set your GitHub token (see "GitHub Token Requirements" below) export GITHUB_TOKEN=ghp_your_token_here -# Test with real PR +# Fetch and send real PR data ./test-webhook -pr 123 -owner myorg -repo myrepo ``` -### Option 3: Use Helper Script +#### GitHub Token Requirements + +The token is used to fetch PR metadata and file lists from the GitHub API. + +**Classic Personal Access Token:** +- Go to https://github.com/settings/tokens +- Generate new token (classic) +- Required scope: `repo` (or `public_repo` for public repos only) + +**Fine-grained Personal Access Token:** +- Go to https://github.com/settings/tokens?type=beta +- Select the specific repositories you need +- Required permissions: + - Pull requests: **Read-only** + - Contents: **Read-only** + +## Test Payloads + +> **Important:** Test payloads contain fake PR numbers. The app will match workflows +> but fail when fetching files from GitHub (the PR doesn't exist). To test the full +> pipeline, use a **real PR number** with `./test-webhook -pr -owner -repo `. + +### Merged PRs (Trigger Processing) + +| File | Description | Expected Behavior | +|------|-------------|-------------------| +| `example-pr-merged.json` | Generic example with Go/Python files | Matches workflows, fails on file fetch (fake PR) | +| `test-pr-merged.json` | Test repo specific | Matches workflows, fails on file fetch (fake PR) | +| `pr-multiple-workflows.json` | Files for Go, Python, JS, and docs | Should trigger multiple workflows | +| `pr-large-changeset.json` | 35 files across multiple categories | Tests batch processing | +| `pr-renamed-files.json` | Files with `status: renamed` | Tests rename handling | +| `pr-with-deprecations.json` | Mix of added and removed files | Tests deprecation tracking | +### Non-Merged Events (Should Be Ignored) + +| File | Description | Expected Behavior | +|------|-------------|-------------------| +| `pr-closed-not-merged.json` | Closed without merge | Should be acknowledged but not processed | +| `pr-opened.json` | PR opened event | Should be ignored | +| `pr-synchronize.json` | PR updated | Should be ignored | +| `pr-no-matching-files.json` | Merged but only CI/config files | No files match patterns | + +## What You Can Test + +### With Fake Payloads (testdata/*.json) + +These test payloads use fake PR numbers, so they validate: +- โœ… Webhook signature verification +- โœ… Payload parsing +- โœ… Config loading and caching +- โœ… Workflow matching (source repo + branch) +- โœ… Slack error notifications +- โŒ File fetching (fails - PR doesn't exist) +- โŒ File processing and copying + +### With Real PRs (-pr flag) + +Using a real PR number tests the complete pipeline: ```bash -# Make script executable -chmod +x scripts/test-with-pr.sh +export GITHUB_TOKEN=ghp_... +./test-webhook -pr -owner cbullinger -repo copier-app-source-test -secret test-secret +``` -# Test with real PR (interactive) -./scripts/test-with-pr.sh 123 myorg myrepo +This validates: +- โœ… Everything above, plus: +- โœ… File fetching from GitHub +- โœ… Pattern matching on actual files +- โœ… File content retrieval +- โœ… Path transformations +- โœ… Commit/PR creation (unless DRY_RUN=true) +- โœ… Slack success notifications + +### Other Events + +| File | Description | Expected Behavior | +|------|-------------|-------------------| +| `push-to-main.json` | Direct push to main branch | Depends on app configuration | + +## Configuration Files + +### test-config.yaml + +Example workflow configuration using the **inline workflows format**: + +```yaml +workflows: + - name: "workflow-name" + source: + repo: "owner/repo" + branch: "main" + patterns: + - "examples/go/**" + destination: + repo: "owner/target-repo" + branch: "main" + transformations: + - move: + from: "examples/go/" + to: "go-examples/" + commit_strategy: + type: "pull_request" + pr_title: "Sync examples" ``` -## Testing Scenarios +### source-repo-files/.copier/main.yaml -### Test Pattern Matching +Example main config using the **workflow_configs format** (for source repos): -Create custom payloads to test specific patterns: +```yaml +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false -**Test Regex Pattern:** -```json -{ - "files": [ - { - "filename": "examples/go/database/connect.go", - "status": "added" - } - ] -} +workflow_configs: + - source: "inline" + workflows: + - name: "workflow-name" + # ... workflow definition ``` -**Test Glob Pattern:** -```json -{ - "files": [ - { - "filename": "examples/go/main.go", - "status": "added" - }, - { - "filename": "examples/python/main.py", - "status": "added" - } - ] -} -``` +## Environment Configuration -### Test Deprecation +Copy `.env.test` to the project root and configure: -```json -{ - "files": [ - { - "filename": "examples/deprecated-example.go", - "status": "removed" - } - ] -} +```bash +cp testdata/.env.test .env.test +# Edit .env.test with your values ``` -### Test Multiple Languages +Key settings for testing: + +| Variable | Description | Test Value | +|----------|-------------|------------| +| `DRY_RUN` | Skip actual writes | `true` for safe testing | +| `COPIER_DISABLE_CLOUD_LOGGING` | Use stdout logging | `true` | +| `LOG_LEVEL` | Logging verbosity | `debug` for troubleshooting | +| `AUDIT_ENABLED` | MongoDB audit logging | `false` (no MongoDB needed) | + +## Creating Custom Payloads + +1. Copy an existing payload as a template +2. Modify the `files` array to test specific patterns +3. Update PR metadata as needed + +Example for testing a specific pattern: ```json { + "action": "closed", + "pull_request": { + "merged": true, + "merge_commit_sha": "abc123" + }, + "repository": { + "full_name": "cbullinger/copier-app-source-test" + }, "files": [ { - "filename": "examples/go/database/connect.go", - "status": "added" - }, - { - "filename": "examples/python/database/connect.py", - "status": "added" - }, - { - "filename": "examples/javascript/database/connect.js", + "filename": "examples/go/your-test-file.go", "status": "added" } ] } ``` -## Creating Custom Payloads +## Validating Results -1. Copy `example-pr-merged.json` -2. Modify the `files` array to match your test case -3. Update PR metadata as needed -4. Save with descriptive name (e.g., `test-go-examples.json`) +### Check Application Logs -## Testing with Dry-Run Mode +```bash +# Logs go to stdout in local mode +# Look for: +# - "webhook received" - payload arrived +# - "PR merged" - event recognized +# - "found matching workflows" - config loaded +# - "File matched transformation" - pattern matched +# - "[DRY-RUN] Would upload" - files would be written +``` -Test without making actual commits: +### Check Metrics ```bash -# Start app in dry-run mode -DRY_RUN=true ./github-copier & +curl http://localhost:8080/metrics | jq +``` -# Send test webhook -./test-webhook -payload testdata/example-pr-merged.json +### Check Health -# Check logs for pattern matching and transformations +```bash +curl http://localhost:8080/health | jq ``` -## Validating Results +## Test Repositories -After sending a test webhook: +For isolated end-to-end testing, you can set up dedicated test repositories: -1. **Check Application Logs** - ```bash - # Local - tail -f logs/app.log - - # GCP - gcloud app logs tail -s default - ``` +| Repo | Purpose | +|------|---------| +| `your-org/copier-source-test` | Source repo that triggers webhooks | +| `your-org/copier-dest-test-1` | First destination repo | +| `your-org/copier-dest-test-2` | Second destination repo | -2. **Check Metrics** - ```bash - curl http://localhost:8080/metrics | jq - ``` +Configure these in your test workflow config and `.env.test`. -3. **Check Audit Logs** (if enabled) - ```javascript - db.audit_events.find().sort({timestamp: -1}).limit(10) - ``` +## Troubleshooting -4. **Verify Pattern Matching** - - Check which files were matched - - Verify path transformations - - Confirm message templating +### Webhook returns 401 -## Common Test Cases +Signature verification failed. Either: +- Set `WEBHOOK_SECRET` to match your test secret +- Clear `WEBHOOK_SECRET` to skip verification in testing -### Test Case 1: New Go Examples -```bash -./test-webhook -payload testdata/example-pr-merged.json -``` -Expected: Files copied to target repo with transformed paths +### No files matched -### Test Case 2: Real PR from Production -```bash -export GITHUB_TOKEN=ghp_... -./scripts/test-with-pr.sh 456 mongodb docs-realm -``` -Expected: Real PR data fetched and processed +Use `config-validator` to test patterns: -### Test Case 3: Dry-Run Validation ```bash -DRY_RUN=true ./github-copier & -./test-webhook -payload testdata/example-pr-merged.json +./config-validator test-pattern \ + -type glob \ + -pattern 'examples/go/**' \ + -file 'examples/go/main.go' ``` -Expected: Processing logged but no commits made -## Troubleshooting +### App won't start + +Check for port conflicts: -### Webhook Returns 401 -- Check webhook secret matches -- Verify signature generation +```bash +lsof -i :8080 +``` -### Files Not Matched -- Check pattern in copier-config.yaml -- Use `config-validator test-pattern` to debug +Check logs: -### Path Transformation Wrong -- Use `config-validator test-transform` to debug -- Check variable names in template +```bash +cat /tmp/integration-test-app.log +``` -### No Response -- Verify app is running -- Check webhook URL is correct -- Review application logs +## See Also +- [LOCAL-TESTING.md](../docs/LOCAL-TESTING.md) - Full local testing guide +- [WEBHOOK-TESTING.md](../docs/WEBHOOK-TESTING.md) - Webhook-specific testing +- [CONFIG-REFERENCE.md](../docs/CONFIG-REFERENCE.md) - Configuration schema diff --git a/testdata/example-pr-merged.json b/testdata/example-pr-merged.json index b492c8e..ab60d15 100644 --- a/testdata/example-pr-merged.json +++ b/testdata/example-pr-merged.json @@ -11,23 +11,23 @@ "ref": "add-go-examples", "sha": "abc123def456", "repo": { - "name": "source-repo", - "full_name": "myorg/source-repo" + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" } }, "base": { "ref": "main", "repo": { - "name": "source-repo", - "full_name": "myorg/source-repo" + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" } } }, "repository": { - "name": "source-repo", - "full_name": "myorg/source-repo", + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", "owner": { - "login": "myorg" + "login": "cbullinger" } }, "files": [ diff --git a/testdata/pr-closed-not-merged.json b/testdata/pr-closed-not-merged.json new file mode 100644 index 0000000..20dcfc1 --- /dev/null +++ b/testdata/pr-closed-not-merged.json @@ -0,0 +1,45 @@ +{ + "action": "closed", + "number": 15, + "pull_request": { + "number": 15, + "state": "closed", + "merged": false, + "merge_commit_sha": null, + "title": "Draft: experimental feature (closed without merge)", + "head": { + "ref": "feature/experimental", + "sha": "def456abc789", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + }, + "base": { + "ref": "main", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + } + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + } + }, + "installation": { + "id": 12345678 + }, + "files": [ + { + "filename": "examples/go/experimental.go", + "status": "added", + "additions": 100, + "deletions": 0, + "changes": 100 + } + ] +} diff --git a/testdata/pr-large-changeset.json b/testdata/pr-large-changeset.json new file mode 100644 index 0000000..27e4eb4 --- /dev/null +++ b/testdata/pr-large-changeset.json @@ -0,0 +1,73 @@ +{ + "action": "closed", + "number": 40, + "pull_request": { + "number": 40, + "state": "closed", + "merged": true, + "merge_commit_sha": "large123changeset456", + "title": "Major refactor: reorganize all examples", + "head": { + "ref": "refactor/reorganize-examples", + "sha": "refactor789sha", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + }, + "base": { + "ref": "main", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + } + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + } + }, + "installation": { + "id": 12345678 + }, + "files": [ + {"filename": "examples/go/basics/hello.go", "status": "added", "additions": 15, "deletions": 0, "changes": 15}, + {"filename": "examples/go/basics/variables.go", "status": "added", "additions": 25, "deletions": 0, "changes": 25}, + {"filename": "examples/go/basics/functions.go", "status": "added", "additions": 35, "deletions": 0, "changes": 35}, + {"filename": "examples/go/basics/structs.go", "status": "added", "additions": 40, "deletions": 0, "changes": 40}, + {"filename": "examples/go/basics/interfaces.go", "status": "added", "additions": 45, "deletions": 0, "changes": 45}, + {"filename": "examples/go/concurrency/goroutines.go", "status": "added", "additions": 30, "deletions": 0, "changes": 30}, + {"filename": "examples/go/concurrency/channels.go", "status": "added", "additions": 50, "deletions": 0, "changes": 50}, + {"filename": "examples/go/concurrency/select.go", "status": "added", "additions": 35, "deletions": 0, "changes": 35}, + {"filename": "examples/go/concurrency/mutex.go", "status": "added", "additions": 40, "deletions": 0, "changes": 40}, + {"filename": "examples/go/concurrency/waitgroup.go", "status": "added", "additions": 25, "deletions": 0, "changes": 25}, + {"filename": "examples/go/database/connect.go", "status": "modified", "additions": 20, "deletions": 10, "changes": 30}, + {"filename": "examples/go/database/query.go", "status": "modified", "additions": 25, "deletions": 15, "changes": 40}, + {"filename": "examples/go/database/transactions.go", "status": "added", "additions": 60, "deletions": 0, "changes": 60}, + {"filename": "examples/go/database/migrations.go", "status": "added", "additions": 80, "deletions": 0, "changes": 80}, + {"filename": "examples/go/http/server.go", "status": "added", "additions": 55, "deletions": 0, "changes": 55}, + {"filename": "examples/go/http/client.go", "status": "added", "additions": 45, "deletions": 0, "changes": 45}, + {"filename": "examples/go/http/middleware.go", "status": "added", "additions": 70, "deletions": 0, "changes": 70}, + {"filename": "examples/go/http/handlers.go", "status": "added", "additions": 90, "deletions": 0, "changes": 90}, + {"filename": "examples/go/testing/unit_test.go", "status": "added", "additions": 50, "deletions": 0, "changes": 50}, + {"filename": "examples/go/testing/integration_test.go", "status": "added", "additions": 75, "deletions": 0, "changes": 75}, + {"filename": "examples/python/basics/hello.py", "status": "added", "additions": 10, "deletions": 0, "changes": 10}, + {"filename": "examples/python/basics/variables.py", "status": "added", "additions": 20, "deletions": 0, "changes": 20}, + {"filename": "examples/python/basics/functions.py", "status": "added", "additions": 30, "deletions": 0, "changes": 30}, + {"filename": "examples/python/basics/classes.py", "status": "added", "additions": 45, "deletions": 0, "changes": 45}, + {"filename": "examples/python/async/coroutines.py", "status": "added", "additions": 40, "deletions": 0, "changes": 40}, + {"filename": "examples/python/async/asyncio.py", "status": "added", "additions": 55, "deletions": 0, "changes": 55}, + {"filename": "examples/python/database/sqlalchemy.py", "status": "added", "additions": 70, "deletions": 0, "changes": 70}, + {"filename": "examples/python/database/pymongo.py", "status": "added", "additions": 60, "deletions": 0, "changes": 60}, + {"filename": "examples/python/http/flask_app.py", "status": "added", "additions": 80, "deletions": 0, "changes": 80}, + {"filename": "examples/python/http/fastapi_app.py", "status": "added", "additions": 90, "deletions": 0, "changes": 90}, + {"filename": "examples/old/legacy_example.go", "status": "removed", "additions": 0, "deletions": 100, "changes": 100}, + {"filename": "examples/old/deprecated.py", "status": "removed", "additions": 0, "deletions": 50, "changes": 50}, + {"filename": "docs/go-examples.md", "status": "added", "additions": 150, "deletions": 0, "changes": 150}, + {"filename": "docs/python-examples.md", "status": "added", "additions": 120, "deletions": 0, "changes": 120}, + {"filename": "docs/getting-started.md", "status": "modified", "additions": 30, "deletions": 20, "changes": 50} + ] +} diff --git a/testdata/pr-multiple-workflows.json b/testdata/pr-multiple-workflows.json new file mode 100644 index 0000000..1a5b560 --- /dev/null +++ b/testdata/pr-multiple-workflows.json @@ -0,0 +1,80 @@ +{ + "action": "closed", + "number": 30, + "pull_request": { + "number": 30, + "state": "closed", + "merged": true, + "merge_commit_sha": "multi123workflow456", + "title": "Add examples for multiple languages", + "head": { + "ref": "feature/multi-language", + "sha": "head789sha", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + }, + "base": { + "ref": "main", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + } + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + } + }, + "installation": { + "id": 12345678 + }, + "files": [ + { + "filename": "examples/go/database/connection.go", + "status": "added", + "additions": 60, + "deletions": 0, + "changes": 60 + }, + { + "filename": "examples/go/database/query.go", + "status": "added", + "additions": 80, + "deletions": 0, + "changes": 80 + }, + { + "filename": "examples/python/database/connection.py", + "status": "added", + "additions": 45, + "deletions": 0, + "changes": 45 + }, + { + "filename": "examples/python/database/query.py", + "status": "added", + "additions": 55, + "deletions": 0, + "changes": 55 + }, + { + "filename": "docs/database-guide.md", + "status": "added", + "additions": 120, + "deletions": 0, + "changes": 120 + }, + { + "filename": "examples/javascript/database/connection.js", + "status": "added", + "additions": 40, + "deletions": 0, + "changes": 40 + } + ] +} diff --git a/testdata/pr-no-matching-files.json b/testdata/pr-no-matching-files.json new file mode 100644 index 0000000..18ab8cd --- /dev/null +++ b/testdata/pr-no-matching-files.json @@ -0,0 +1,59 @@ +{ + "action": "closed", + "number": 35, + "pull_request": { + "number": 35, + "state": "closed", + "merged": true, + "merge_commit_sha": "nomatch123sha456", + "title": "Update CI configuration", + "head": { + "ref": "chore/update-ci", + "sha": "ci789sha", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + }, + "base": { + "ref": "main", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + } + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + } + }, + "installation": { + "id": 12345678 + }, + "files": [ + { + "filename": ".github/workflows/ci.yml", + "status": "modified", + "additions": 10, + "deletions": 5, + "changes": 15 + }, + { + "filename": ".github/dependabot.yml", + "status": "added", + "additions": 20, + "deletions": 0, + "changes": 20 + }, + { + "filename": "Makefile", + "status": "modified", + "additions": 5, + "deletions": 2, + "changes": 7 + } + ] +} diff --git a/testdata/pr-opened.json b/testdata/pr-opened.json new file mode 100644 index 0000000..91896a0 --- /dev/null +++ b/testdata/pr-opened.json @@ -0,0 +1,46 @@ +{ + "action": "opened", + "number": 20, + "pull_request": { + "number": 20, + "state": "open", + "merged": false, + "merge_commit_sha": null, + "title": "Add new Python examples", + "draft": false, + "head": { + "ref": "feature/python-examples", + "sha": "abc123new456", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + }, + "base": { + "ref": "main", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + } + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + } + }, + "installation": { + "id": 12345678 + }, + "files": [ + { + "filename": "examples/python/new_feature.py", + "status": "added", + "additions": 50, + "deletions": 0, + "changes": 50 + } + ] +} diff --git a/testdata/pr-renamed-files.json b/testdata/pr-renamed-files.json new file mode 100644 index 0000000..d932760 --- /dev/null +++ b/testdata/pr-renamed-files.json @@ -0,0 +1,70 @@ +{ + "action": "closed", + "number": 45, + "pull_request": { + "number": 45, + "state": "closed", + "merged": true, + "merge_commit_sha": "rename123files456", + "title": "Rename examples to follow naming convention", + "head": { + "ref": "chore/rename-examples", + "sha": "rename789sha", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + }, + "base": { + "ref": "main", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + } + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + } + }, + "installation": { + "id": 12345678 + }, + "files": [ + { + "filename": "examples/go/database/db_connect.go", + "status": "renamed", + "previous_filename": "examples/go/database/connect.go", + "additions": 5, + "deletions": 5, + "changes": 10 + }, + { + "filename": "examples/go/database/db_query.go", + "status": "renamed", + "previous_filename": "examples/go/database/query.go", + "additions": 3, + "deletions": 3, + "changes": 6 + }, + { + "filename": "examples/python/database/py_connect.py", + "status": "renamed", + "previous_filename": "examples/python/database/connect.py", + "additions": 2, + "deletions": 2, + "changes": 4 + }, + { + "filename": "examples/go/auth/auth_login.go", + "status": "renamed", + "previous_filename": "examples/go/auth/login.go", + "additions": 0, + "deletions": 0, + "changes": 0 + } + ] +} diff --git a/testdata/pr-synchronize.json b/testdata/pr-synchronize.json new file mode 100644 index 0000000..9d67e78 --- /dev/null +++ b/testdata/pr-synchronize.json @@ -0,0 +1,54 @@ +{ + "action": "synchronize", + "number": 25, + "before": "old123sha456", + "after": "new789sha012", + "pull_request": { + "number": 25, + "state": "open", + "merged": false, + "merge_commit_sha": null, + "title": "Update Go examples with new patterns", + "head": { + "ref": "feature/update-go-examples", + "sha": "new789sha012", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + }, + "base": { + "ref": "main", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + } + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + } + }, + "installation": { + "id": 12345678 + }, + "files": [ + { + "filename": "examples/go/patterns/singleton.go", + "status": "modified", + "additions": 20, + "deletions": 5, + "changes": 25 + }, + { + "filename": "examples/go/patterns/factory.go", + "status": "added", + "additions": 45, + "deletions": 0, + "changes": 45 + } + ] +} diff --git a/testdata/pr-with-deprecations.json b/testdata/pr-with-deprecations.json new file mode 100644 index 0000000..3e25973 --- /dev/null +++ b/testdata/pr-with-deprecations.json @@ -0,0 +1,80 @@ +{ + "action": "closed", + "number": 50, + "pull_request": { + "number": 50, + "state": "closed", + "merged": true, + "merge_commit_sha": "deprecate123files456", + "title": "Remove deprecated examples and add replacements", + "head": { + "ref": "chore/cleanup-deprecated", + "sha": "deprecate789sha", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + }, + "base": { + "ref": "main", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + } + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + } + }, + "installation": { + "id": 12345678 + }, + "files": [ + { + "filename": "examples/go/legacy/old_auth.go", + "status": "removed", + "additions": 0, + "deletions": 75, + "changes": 75 + }, + { + "filename": "examples/go/legacy/old_database.go", + "status": "removed", + "additions": 0, + "deletions": 120, + "changes": 120 + }, + { + "filename": "examples/python/deprecated/old_api.py", + "status": "removed", + "additions": 0, + "deletions": 90, + "changes": 90 + }, + { + "filename": "examples/go/auth/modern_auth.go", + "status": "added", + "additions": 85, + "deletions": 0, + "changes": 85 + }, + { + "filename": "examples/go/database/modern_db.go", + "status": "added", + "additions": 130, + "deletions": 0, + "changes": 130 + }, + { + "filename": "examples/python/api/modern_api.py", + "status": "added", + "additions": 100, + "deletions": 0, + "changes": 100 + } + ] +} diff --git a/testdata/push-to-main.json b/testdata/push-to-main.json new file mode 100644 index 0000000..f082c23 --- /dev/null +++ b/testdata/push-to-main.json @@ -0,0 +1,76 @@ +{ + "ref": "refs/heads/main", + "before": "abc123before", + "after": "def456after", + "created": false, + "deleted": false, + "forced": false, + "base_ref": null, + "compare": "https://github.com/cbullinger/copier-app-source-test/compare/abc123before...def456after", + "commits": [ + { + "id": "def456after", + "tree_id": "tree123", + "distinct": true, + "message": "Direct push: fix typo in example", + "timestamp": "2025-02-16T10:30:00-08:00", + "url": "https://github.com/cbullinger/copier-app-source-test/commit/def456after", + "author": { + "name": "Test User", + "email": "test@example.com", + "username": "testuser" + }, + "committer": { + "name": "Test User", + "email": "test@example.com", + "username": "testuser" + }, + "added": [], + "removed": [], + "modified": [ + "examples/go/main.go" + ] + } + ], + "head_commit": { + "id": "def456after", + "tree_id": "tree123", + "distinct": true, + "message": "Direct push: fix typo in example", + "timestamp": "2025-02-16T10:30:00-08:00", + "url": "https://github.com/cbullinger/copier-app-source-test/commit/def456after", + "author": { + "name": "Test User", + "email": "test@example.com", + "username": "testuser" + }, + "committer": { + "name": "Test User", + "email": "test@example.com", + "username": "testuser" + }, + "added": [], + "removed": [], + "modified": [ + "examples/go/main.go" + ] + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + }, + "default_branch": "main" + }, + "pusher": { + "name": "testuser", + "email": "test@example.com" + }, + "sender": { + "login": "testuser" + }, + "installation": { + "id": 12345678 + } +} diff --git a/testdata/test-config.yaml b/testdata/test-config.yaml index 181c191..7827576 100644 --- a/testdata/test-config.yaml +++ b/testdata/test-config.yaml @@ -1,6 +1,11 @@ # Test configuration for local integration testing +# This file demonstrates the current workflow schema for reference. +# # Source: cbullinger/copier-app-source-test # Destinations: cbullinger/copier-app-dest-1, cbullinger/copier-app-dest-2 +# +# NOTE: This is the inline workflow format. For the main config format +# (with workflow_configs), see source-repo-files/.copier/main.yaml workflows: # Workflow 1: Copy Go files to dest-1 @@ -9,18 +14,17 @@ workflows: repo: "cbullinger/copier-app-source-test" branch: "main" patterns: - - type: glob - pattern: "examples/go/**" + - "examples/go/**" destination: repo: "cbullinger/copier-app-dest-1" branch: "main" transformations: - - type: move - from: "examples/go/" - to: "go-examples/" - commit: - strategy: pr - message: "Sync Go examples from source" + - move: + from: "examples/go/" + to: "go-examples/" + commit_strategy: + type: "pull_request" + commit_message: "Sync Go examples from source" pr_title: "[Test] Sync Go examples" auto_merge: false @@ -30,37 +34,55 @@ workflows: repo: "cbullinger/copier-app-source-test" branch: "main" patterns: - - type: glob - pattern: "examples/python/**" + - "examples/python/**" destination: repo: "cbullinger/copier-app-dest-2" branch: "main" transformations: - - type: move - from: "examples/python/" - to: "python-examples/" - commit: - strategy: pr - message: "Sync Python examples from source" + - move: + from: "examples/python/" + to: "python-examples/" + commit_strategy: + type: "pull_request" + commit_message: "Sync Python examples from source" pr_title: "[Test] Sync Python examples" auto_merge: false - # Workflow 3: Copy docs to both destinations + # Workflow 3: Copy docs to dest-1 (direct commit) - name: "test-docs-to-dest1" source: repo: "cbullinger/copier-app-source-test" branch: "main" patterns: - - type: glob - pattern: "docs/**" + - "docs/**" destination: repo: "cbullinger/copier-app-dest-1" branch: "main" transformations: - - type: move - from: "docs/" - to: "documentation/" - commit: - strategy: direct - message: "Sync docs from source" + - move: + from: "docs/" + to: "documentation/" + commit_strategy: + type: "direct" + commit_message: "Sync docs from source" + + # Workflow 4: Copy JavaScript files to dest-2 (demonstrates additional language) + - name: "test-js-to-dest2" + source: + repo: "cbullinger/copier-app-source-test" + branch: "main" + patterns: + - "examples/javascript/**" + destination: + repo: "cbullinger/copier-app-dest-2" + branch: "main" + transformations: + - move: + from: "examples/javascript/" + to: "js-examples/" + commit_strategy: + type: "pull_request" + commit_message: "Sync JavaScript examples from source" + pr_title: "[Test] Sync JavaScript examples" + auto_merge: false diff --git a/tests/utils.go b/tests/utils.go index 4bc2ff2..7a4d01f 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -30,20 +30,38 @@ func EnvOwnerRepo(t testing.TB) (string, string) { return owner, repo } +// TestConfig returns a *configs.Config populated from the current environment variables. +// This mirrors the values set in TestMain for test suites. +func TestConfig() *configs.Config { + cfg := configs.NewConfig() + cfg.ConfigRepoOwner = os.Getenv(configs.ConfigRepoOwner) + cfg.ConfigRepoName = os.Getenv(configs.ConfigRepoName) + cfg.InstallationId = os.Getenv(configs.InstallationId) + cfg.AppId = os.Getenv(configs.AppId) + cfg.AppClientId = os.Getenv(configs.AppClientId) + cfg.ConfigRepoBranch = os.Getenv(configs.ConfigRepoBranch) + if cfg.ConfigRepoBranch == "" { + cfg.ConfigRepoBranch = "main" + } + return cfg +} + // // HTTP/test wiring helpers // -// WithHTTPMock wraps a test in `httpmock` activation on a dedicated http.Client and routes services.HTTPClient through it. -// Used in any test that needs multiple mock endpoints. Wrap t.Run blocks to avoid leftover mocks affecting other tests. +// WithHTTPMock wraps a test in `httpmock` activation on a dedicated http.Client +// and routes the TokenManager's HTTP client through it. func WithHTTPMock(t testing.TB) *http.Client { t.Helper() c := &http.Client{} httpmock.ActivateNonDefault(c) t.Cleanup(func() { httpmock.DeactivateAndReset() }) - prev := services.HTTPClient - services.HTTPClient = c - t.Cleanup(func() { services.HTTPClient = prev }) + + tm := services.DefaultTokenManager() + prev := tm.GetHTTPClient() + tm.SetHTTPClient(c) + t.Cleanup(func() { tm.SetHTTPClient(prev) }) return c } @@ -60,7 +78,6 @@ func DumpHttpmockCalls(t testing.TB) { // // MockGitHubAppTokenEndpoint mocks the GitHub App installation token endpoint with a fixed fake token. -// Used in to simulate any auth-triggered flow without needing a real installation ID. func MockGitHubAppTokenEndpoint(installationID string) { httpmock.RegisterResponder("POST", "https://api.github.com/app/installations/"+installationID+"/access_tokens", @@ -69,7 +86,6 @@ func MockGitHubAppTokenEndpoint(installationID string) { } // MockGitHubAppInstallations mocks the GitHub App installations list endpoint. -// Used to simulate fetching installation IDs for organizations. func MockGitHubAppInstallations(orgToInstallationID map[string]string) { installations := []map[string]any{} for org, installID := range orgToInstallationID { @@ -93,9 +109,10 @@ func SetupOrgToken(org, token string) { services.SetInstallationTokenForOrg(org, token) } -// MockGitHubWriteEndpoints mocks the full direct-commit flow endpoints for a single branch: GET base ref, POST trees, POST commits, PATCH ref. -// Used to simulate writing to a GitHub repo without creating a PR. +// MockGitHubWriteEndpoints mocks the full direct-commit flow endpoints for a single branch. // Returns the URLs for the base ref, commits, and update ref endpoints. +// The base commit's tree SHA is "oldTreeSha" (different from the new tree "newTreeSha"), +// so commits will proceed normally. Use MockGitHubWriteEndpointsNoOp to simulate no changes. func MockGitHubWriteEndpoints(owner, repo, branch string) (baseRefURL, commitsURL, updateRefURL string) { baseRefURL = "https://api.github.com/repos/" + owner + "/" + repo + "/git/ref/heads/" + branch httpmock.RegisterResponder("GET", baseRefURL, @@ -107,6 +124,16 @@ func MockGitHubWriteEndpoints(owner, repo, branch string) (baseRefURL, commitsUR }), ) + // Mock GET commit to return the base commit's tree SHA + getCommitRe := regexp.MustCompile(`^https://api\.github\.com/repos/` + regexp.QuoteMeta(owner) + `/` + + regexp.QuoteMeta(repo) + `/git/commits/baseSha$`) + httpmock.RegisterRegexpResponder("GET", getCommitRe, + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "sha": "baseSha", + "tree": map[string]any{"sha": "oldTreeSha"}, + }), + ) + treesRe := regexp.MustCompile(`^https://api\.github\.com/repos/` + regexp.QuoteMeta(owner) + `/` + regexp.QuoteMeta(repo) + `/git/trees(\?.*)?$`) httpmock.RegisterRegexpResponder("POST", treesRe, @@ -130,8 +157,50 @@ func MockGitHubWriteEndpoints(owner, repo, branch string) (baseRefURL, commitsUR return } +// MockGitHubWriteEndpointsNoOp is like MockGitHubWriteEndpoints but the new tree SHA +// equals the base commit's tree SHA, simulating a no-op (duplicate) commit. +func MockGitHubWriteEndpointsNoOp(owner, repo, branch string) (baseRefURL, commitsURL, updateRefURL string) { + baseRefURL = "https://api.github.com/repos/" + owner + "/" + repo + "/git/ref/heads/" + branch + httpmock.RegisterResponder("GET", baseRefURL, + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "ref": "refs/heads/" + branch, + "object": map[string]any{"sha": "baseSha"}, + }), + ) + + getCommitRe := regexp.MustCompile(`^https://api\.github\.com/repos/` + regexp.QuoteMeta(owner) + `/` + + regexp.QuoteMeta(repo) + `/git/commits/baseSha$`) + httpmock.RegisterRegexpResponder("GET", getCommitRe, + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "sha": "baseSha", + "tree": map[string]any{"sha": "sameTreeSha"}, + }), + ) + + treesRe := regexp.MustCompile(`^https://api\.github\.com/repos/` + regexp.QuoteMeta(owner) + `/` + + regexp.QuoteMeta(repo) + `/git/trees(\?.*)?$`) + httpmock.RegisterRegexpResponder("POST", treesRe, + httpmock.NewJsonResponderOrPanic(201, map[string]any{ + "sha": "sameTreeSha", // same as base โ€” no real changes + }), + ) + + commitsURL = "https://api.github.com/repos/" + owner + "/" + repo + "/git/commits" + httpmock.RegisterResponder("POST", commitsURL, + httpmock.NewJsonResponderOrPanic(201, map[string]any{ + "sha": "newCommitSha", + }), + ) + + updateRefURL = "https://api.github.com/repos/" + owner + "/" + repo + "/git/refs/heads/" + branch + httpmock.RegisterResponder("PATCH", updateRefURL, + httpmock.NewStringResponder(200, `{}`), + ) + + return +} + // MockContentsEndpoint mocks GET file contents for a given path/ref. -// Used to simulate reading a file from a GitHub repo. func MockContentsEndpoint(owner, repo, path, contentB64 string) { re := regexp.MustCompile( `^https://api\.github\.com/repos/` + regexp.QuoteMeta(owner) + `/` + @@ -149,7 +218,6 @@ func MockContentsEndpoint(owner, repo, path, contentB64 string) { } // MockCreateRef mocks POST to create a new temp branch ref. Returns the exact URL for call-count asserts. -// Used to simulate creating a new branch for writing files without actually pushing to GitHub. func MockCreateRef(owner, repo string) string { url := "https://api.github.com/repos/" + owner + "/" + repo + "/git/refs" httpmock.RegisterResponder("POST", url, @@ -161,8 +229,33 @@ func MockCreateRef(owner, repo string) string { return url } +// MockGetCommit mocks GET /repos/{owner}/{repo}/git/commits/{sha} for tree comparison. +// baseTreeSHA is the tree SHA of the base commit; use a value different from the new tree +// to allow commits, or the same value to simulate a no-op. +func MockGetCommit(owner, repo, commitSHA, baseTreeSHA string) { + re := regexp.MustCompile(`^https://api\.github\.com/repos/` + regexp.QuoteMeta(owner) + `/` + + regexp.QuoteMeta(repo) + `/git/commits/` + regexp.QuoteMeta(commitSHA) + `$`) + httpmock.RegisterRegexpResponder("GET", re, + httpmock.NewJsonResponderOrPanic(200, map[string]any{ + "sha": commitSHA, + "tree": map[string]any{"sha": baseTreeSHA}, + }), + ) +} + +// MockListOpenPRs mocks the "list open PRs" endpoint, returning the supplied PRs. +// Pass nil or empty to simulate no existing open PRs. +func MockListOpenPRs(owner, repo string, prs []map[string]any) { + if prs == nil { + prs = []map[string]any{} + } + httpmock.RegisterRegexpResponder("GET", + regexp.MustCompile(`^https://api\.github\.com/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/pulls\?`), + httpmock.NewJsonResponderOrPanic(200, prs), + ) +} + // MockPullsAndMerge mocks creating and merging a PR. -// Used to simulate the full PR flow for functions that create a PR and then merge it func MockPullsAndMerge(owner, repo string, number int) { httpmock.RegisterResponder("POST", "https://api.github.com/repos/"+owner+"/"+repo+"/pulls", @@ -175,7 +268,6 @@ func MockPullsAndMerge(owner, repo string, number int) { } // MockDeleteTempRef mocks DELETE to remove a temporary branch ref. -// Used to simulate cleaning up after writing files without actually deleting a branch on GitHub. func MockDeleteTempRef(owner, repo string) { re := regexp.MustCompile( `^https://api\.github\.com/repos/` + regexp.QuoteMeta(owner) + `/` + @@ -188,7 +280,7 @@ func MockDeleteTempRef(owner, repo string) { // Staging/assertion helpers // -// NormalizeUpload flattens FilesToUpload to UploadKey -> []names for simpler comparisons. +// NormalizeUpload flattens a FilesToUpload map to UploadKey -> []names for simpler comparisons. func NormalizeUpload(in map[types.UploadKey]types.UploadFileContent) map[types.UploadKey][]string { out := make(map[types.UploadKey][]string, len(in)) for k, v := range in { @@ -206,103 +298,12 @@ func MakeChanged(status, path string) types.ChangedFile { return types.ChangedFile{Status: status, Path: path} } -// ResetGlobals clears FilesToUpload and FilesToDeprecate. -func ResetGlobals() { - services.FilesToUpload = nil - services.FilesToDeprecate = nil -} - -// AssertUploadedPaths asserts that the staged filenames match the want for the given repo/branch (order-insensitive). -func AssertUploadedPaths(t *testing.T, repo, branch string, want []string) { - t.Helper() - key := types.UploadKey{RepoName: repo, BranchPath: "refs/heads/" + branch} - got, ok := services.FilesToUpload[key] - if !ok { - t.Fatalf("expected FilesToUpload to contain key for %s/%s", repo, branch) - } - - var names []string - for _, c := range got.Content { - n := c.GetName() - if n == "" { - n = c.GetPath() // fallback: some code paths populate only Path - } - names = append(names, n) - } - - // exact, order-insensitive comparison - if len(want) == 0 && len(names) == 0 { - return - } - if len(want) != len(names) { - t.Fatalf("staged names length mismatch: got=%v want=%v", names, want) - } - wantSet := map[string]struct{}{} - for _, w := range want { - wantSet[w] = struct{}{} - } - for _, n := range names { - if _, ok := wantSet[n]; !ok { - t.Fatalf("unexpected staged path %q; got=%v want=%v", n, names, want) - } - } -} - -// AssertUploadedPathsFromConfig converts staged source paths to target paths using cfg, -// then compares against want - i.e. target paths (order-insensitive). -// Used when the staged files are from a config that specifies source/target directories. -func AssertUploadedPathsFromConfig(t *testing.T, cfg types.Configs, want []string) { - t.Helper() - key := types.UploadKey{RepoName: cfg.TargetRepo, BranchPath: "refs/heads/" + cfg.TargetBranch} - got, ok := services.FilesToUpload[key] - if !ok { - t.Fatalf("expected FilesToUpload to contain key for %s/%s", cfg.TargetRepo, cfg.TargetBranch) - } - var names []string - for _, c := range got.Content { - // Prefer Name if present - n := c.GetName() - if n == "" { - n = c.GetPath() // usually the *source* path (e.g. examples/โ€ฆ) - } - // If the staged name looks like a source path, rewrite to target - if cfg.SourceDirectory != "" && strings.HasPrefix(n, cfg.SourceDirectory) { - rel := strings.TrimPrefix(n, cfg.SourceDirectory) - rel = strings.TrimPrefix(rel, "/") - n = cfg.TargetDirectory - if rel != "" { - n = cfg.TargetDirectory + "/" + rel - } - } - names = append(names, n) - } - - // order-insensitive compare - if len(want) == 0 && len(names) == 0 { - return - } - if len(want) != len(names) { - t.Fatalf("staged names length mismatch: got=%v want=%v", names, want) - } - wantSet := map[string]struct{}{} - for _, w := range want { - wantSet[w] = struct{}{} - } - for _, n := range names { - if _, ok = wantSet[n]; !ok { - t.Fatalf("unexpected staged path %q; got=%v want=%v", n, names, want) - } - } -} - // CountByMethodAndURLRegexp adds up call counts for a given METHOD whose stored httpmock key's URL matches urlRE. -// Works for both exact and regex-registered responders. -// Used to assert that a specific endpoint was called a certain number of times. func CountByMethodAndURLRegexp(method string, urlRE *regexp.Regexp) int { info := httpmock.GetCallCountInfo() total := 0 for k, v := range info { - if !(strings.HasPrefix(k, method+" ") || strings.HasPrefix(k, method+"=~")) { + if !strings.HasPrefix(k, method+" ") && !strings.HasPrefix(k, method+"=~") { continue } var urlish string @@ -324,7 +325,7 @@ func CountByMethodAndURLRegexp(method string, urlRE *regexp.Regexp) int { } // GetRefGetCount counts GET calls to /git/ref/(refs/)?heads/ -// for the given owner/repo/branch. Used to assert that a ref was fetched. +// for the given owner/repo/branch. func GetRefGetCount(owner, repo, branch string) int { re := regexp.MustCompile(`/repos/` + regexp.QuoteMeta(owner) + `/` + regexp.QuoteMeta(repo) + `/git/ref/(?:refs/)?heads/` + regexp.QuoteMeta(branch) + `$`) diff --git a/types/config.go b/types/config.go index 0a07cce..de4ec27 100644 --- a/types/config.go +++ b/types/config.go @@ -69,17 +69,17 @@ type YAMLConfig struct { // MainConfig represents the central configuration file that references workflow configs type MainConfig struct { - Defaults *Defaults `yaml:"defaults,omitempty" json:"defaults,omitempty"` + Defaults *Defaults `yaml:"defaults,omitempty" json:"defaults,omitempty"` WorkflowConfigs []WorkflowConfigRef `yaml:"workflow_configs" json:"workflow_configs"` } // WorkflowConfigRef references a workflow configuration file type WorkflowConfigRef struct { - Source string `yaml:"source" json:"source"` // "local", "repo", or "inline" - Path string `yaml:"path,omitempty" json:"path,omitempty"` // Path to config file - Repo string `yaml:"repo,omitempty" json:"repo,omitempty"` // Repository (for source="repo") - Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` // Branch (for source="repo") - Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` // Whether this workflow config is enabled (default: true) + Source string `yaml:"source" json:"source"` // "local", "repo", or "inline" + Path string `yaml:"path,omitempty" json:"path,omitempty"` // Path to config file + Repo string `yaml:"repo,omitempty" json:"repo,omitempty"` // Repository (for source="repo") + Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` // Branch (for source="repo") + Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` // Whether this workflow config is enabled (default: true) Workflows []Workflow `yaml:"workflows,omitempty" json:"workflows,omitempty"` // Inline workflows (for source="inline") } @@ -121,9 +121,9 @@ func (r *RefOrValue[T]) GetValue() *T { // TransformationsOrRef can be either inline transformations or a $ref type TransformationsOrRef struct { - Ref string `yaml:"-" json:"-"` - Transformations []Transformation `yaml:"-" json:"-"` - isRef bool + Ref string `yaml:"-" json:"-"` + Transformations []Transformation `yaml:"-" json:"-"` + isRef bool } // UnmarshalYAML implements custom YAML unmarshaling for transformations @@ -299,14 +299,14 @@ type Workflow struct { // Source defines the source repository and branch type Source struct { Repo string `yaml:"repo" json:"repo"` - Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` // defaults to "main" + Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` // defaults to "main" InstallationID string `yaml:"installation_id,omitempty" json:"installation_id,omitempty"` // optional override } // Destination defines the destination repository and branch type Destination struct { Repo string `yaml:"repo" json:"repo"` - Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` // defaults to "main" + Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` // defaults to "main" InstallationID string `yaml:"installation_id,omitempty" json:"installation_id,omitempty"` // optional override } @@ -343,14 +343,14 @@ type CopyTransform struct { // GlobTransform uses glob patterns with path transformation type GlobTransform struct { - Pattern string `yaml:"pattern" json:"pattern"` // Glob pattern (e.g., "mflix/server/**/*.js") - Transform string `yaml:"transform" json:"transform"` // Path transform template (e.g., "server/${relative_path}") + Pattern string `yaml:"pattern" json:"pattern"` // Glob pattern (e.g., "mflix/server/**/*.js") + Transform string `yaml:"transform" json:"transform"` // Path transform template (e.g., "server/${relative_path}") } // RegexTransform uses regex patterns with named capture groups type RegexTransform struct { - Pattern string `yaml:"pattern" json:"pattern"` // Regex pattern with named groups - Transform string `yaml:"transform" json:"transform"` // Path transform template using captured groups + Pattern string `yaml:"pattern" json:"pattern"` // Regex pattern with named groups + Transform string `yaml:"transform" json:"transform"` // Path transform template using captured groups } // Validate validates the YAML configuration @@ -658,7 +658,7 @@ func NewTransformContext(sourcePath string, variables map[string]string) *Transf // AddBuiltInVariables adds built-in variables like ${path}, ${filename}, ${dir}, ${ext} func (tc *TransformContext) AddBuiltInVariables() { tc.Variables["path"] = tc.SourcePath - + // Extract filename lastSlash := strings.LastIndex(tc.SourcePath, "/") if lastSlash >= 0 { @@ -668,7 +668,7 @@ func (tc *TransformContext) AddBuiltInVariables() { tc.Variables["filename"] = tc.SourcePath tc.Variables["dir"] = "" } - + // Extract extension filename := tc.Variables["filename"] lastDot := strings.LastIndex(filename, ".") @@ -681,15 +681,15 @@ func (tc *TransformContext) AddBuiltInVariables() { // MessageContext holds context for message template rendering type MessageContext struct { - RuleName string // Name of the copy rule - SourceRepo string // Source repository - TargetRepo string // Target repository - SourceBranch string // Source branch - TargetBranch string // Target branch - FileCount int // Number of files being copied - PRNumber int // PR number that triggered the copy - CommitSHA string // Commit SHA - Variables map[string]string // Variables from pattern matching + RuleName string // Name of the copy rule + SourceRepo string // Source repository + TargetRepo string // Target repository + SourceBranch string // Source branch + TargetBranch string // Target branch + FileCount int // Number of files being copied + PRNumber int // PR number that triggered the copy + CommitSHA string // Commit SHA + Variables map[string]string // Variables from pattern matching } // NewMessageContext creates a new message context @@ -707,13 +707,13 @@ func NewMessageContext() *MessageContext { func (w *Workflow) UnmarshalYAML(unmarshal func(interface{}) error) error { // Create a temporary struct with the same fields but using OrRef types type workflowAlias struct { - Name string `yaml:"name"` - Source Source `yaml:"source"` - Destination Destination `yaml:"destination"` - Transformations TransformationsOrRef `yaml:"transformations"` - Exclude ExcludeOrRef `yaml:"exclude,omitempty"` - CommitStrategy CommitStrategyOrRef `yaml:"commit_strategy,omitempty"` - DeprecationCheck *DeprecationConfig `yaml:"deprecation_check,omitempty"` + Name string `yaml:"name"` + Source Source `yaml:"source"` + Destination Destination `yaml:"destination"` + Transformations TransformationsOrRef `yaml:"transformations"` + Exclude ExcludeOrRef `yaml:"exclude,omitempty"` + CommitStrategy CommitStrategyOrRef `yaml:"commit_strategy,omitempty"` + DeprecationCheck *DeprecationConfig `yaml:"deprecation_check,omitempty"` } var alias workflowAlias @@ -909,4 +909,3 @@ func (t *Transformation) GetType() TransformationType { } return "" } - diff --git a/types/config_test.go b/types/config_test.go index aab0f2f..9d92d6c 100644 --- a/types/config_test.go +++ b/types/config_test.go @@ -11,15 +11,15 @@ import ( // fields are properly merged from defaults when a workflow has a partial commit_strategy func TestYAMLConfig_SetDefaults_CommitStrategyMerging(t *testing.T) { tests := []struct { - name string - defaults *Defaults - workflowCommitStrategy *CommitStrategyConfig - expectedType string - expectedCommitMessage string - expectedPRTitle string - expectedPRBody string - expectedUsePRTemplate bool - expectedAutoMerge bool + name string + defaults *Defaults + workflowCommitStrategy *CommitStrategyConfig + expectedType string + expectedCommitMessage string + expectedPRTitle string + expectedPRBody string + expectedUsePRTemplate bool + expectedAutoMerge bool }{ { name: "workflow with only pr_title should inherit commit_message from defaults", @@ -269,4 +269,3 @@ func TestWorkflowConfig_SetDefaults_CommitStrategyMerging(t *testing.T) { assert.True(t, workflow2.CommitStrategy.UsePRTemplate) assert.False(t, workflow2.CommitStrategy.AutoMerge) } - diff --git a/types/types.go b/types/types.go index 9cb346f..5f1a3c3 100644 --- a/types/types.go +++ b/types/types.go @@ -1,7 +1,7 @@ package types import ( - "github.com/google/go-github/v48/github" + "github.com/google/go-github/v82/github" "github.com/shurcooL/githubv4" ) @@ -92,10 +92,12 @@ type Configs struct { } type DeprecationFile []DeprecatedFileEntry type DeprecatedFileEntry struct { - FileName string `json:"filename"` - Repo string `json:"repo"` - Branch string `json:"branch"` - DeletedOn string `json:"deleted_on"` + FileName string `json:"filename"` + Repo string `json:"repo"` + Branch string `json:"branch"` + DeletedOn string `json:"deleted_on"` + SourcePath string `json:"source_path,omitempty"` // Original source file path + PRNumber int `json:"pr_number,omitempty"` // PR that caused the deletion } // **** UPLOAD TYPES **** // @@ -103,8 +105,8 @@ type DeprecatedFileEntry struct { type UploadKey struct { RepoName string `json:"repo_name"` BranchPath string `json:"branch_path"` - RuleName string `json:"rule_name"` // Include rule name to allow multiple rules targeting same repo/branch - CommitStrategy string `json:"commit_strategy"` // Include strategy to differentiate direct vs PR + RuleName string `json:"rule_name"` // Include rule name to allow multiple rules targeting same repo/branch + CommitStrategy string `json:"commit_strategy"` // Include strategy to differentiate direct vs PR } type UploadFileContent struct { @@ -114,7 +116,7 @@ type UploadFileContent struct { CommitMessage string `json:"commit_message,omitempty"` PRTitle string `json:"pr_title,omitempty"` PRBody string `json:"pr_body,omitempty"` - UsePRTemplate bool `json:"use_pr_template,omitempty"` // If true, fetch and merge PR template from target repo + UsePRTemplate bool `json:"use_pr_template,omitempty"` // If true, fetch and merge PR template from target repo AutoMergePR bool `json:"auto_merge_pr,omitempty"` }