From c852bdc811cbab7ab4c427b63468a47a4331c66e Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Fri, 13 Feb 2026 09:17:59 -0500 Subject: [PATCH 01/11] TRT-1989: add partition management --- pkg/db/partitions/README.md | 1021 ++++++++++++++++++++++++++ pkg/db/partitions/examples.go | 608 +++++++++++++++ pkg/db/partitions/partitions.go | 1011 +++++++++++++++++++++++++ pkg/db/partitions/partitions_test.go | 97 +++ 4 files changed, 2737 insertions(+) create mode 100644 pkg/db/partitions/README.md create mode 100644 pkg/db/partitions/examples.go create mode 100644 pkg/db/partitions/partitions.go create mode 100644 pkg/db/partitions/partitions_test.go diff --git a/pkg/db/partitions/README.md b/pkg/db/partitions/README.md new file mode 100644 index 000000000..1480c3cd5 --- /dev/null +++ b/pkg/db/partitions/README.md @@ -0,0 +1,1021 @@ +# Partition Management APIs + +This package provides GORM-based APIs for managing PostgreSQL table partitions, specifically for `test_analysis_by_job_by_dates`. + +## Overview + +The partition management APIs provide read-only analysis and write operations (with dry-run support) for managing the lifecycle of table partitions based on retention policies. + +**Based on**: [partition-retention-management-guide.md](../../../.claude/partition-retention-management-guide.md) + +## Features + +- ✅ List all partitions with metadata +- ✅ Get partition statistics and summaries +- ✅ Identify partitions for removal based on retention policy +- ✅ Analyze partitions by age groups and time periods +- ✅ Validate retention policies (safety checks) +- ✅ Dry-run support for all destructive operations +- ✅ Comprehensive logging +- ✅ SQL injection protection + +## API Reference + +### Read-Only Operations + +#### ListTablePartitions +Returns all partitions for a given table with metadata. + +```go +partitions, err := partitions.ListTablePartitions(dbc, "test_analysis_by_job_by_dates") +if err != nil { + log.WithError(err).Error("failed to list partitions") +} + +for _, p := range partitions { + fmt.Printf("%s: %s, Age: %d days, Size: %s\n", + p.TableName, p.PartitionDate, p.Age, p.SizePretty) +} +``` + +**Parameters**: +- `tableName` - Name of the partitioned parent table + +**Returns**: `[]PartitionInfo` containing: +- `TableName` - Partition table name +- `SchemaName` - Schema (always "public") +- `PartitionDate` - Date the partition represents +- `Age` - Days since partition date +- `SizeBytes` - Storage in bytes +- `SizePretty` - Human-readable size +- `RowEstimate` - Estimated row count + +--- + +#### GetPartitionStats +Returns aggregate statistics about all partitions. + +```go +stats, err := partitions.GetPartitionStats(dbc) +if err != nil { + log.WithError(err).Error("failed to get stats") +} + +fmt.Printf("Total: %d partitions, %s\n", + stats.TotalPartitions, stats.TotalSizePretty) +fmt.Printf("Range: %s to %s\n", + stats.OldestDate.Format("2006-01-02"), + stats.NewestDate.Format("2006-01-02")) +``` + +**Returns**: `*PartitionStats` containing: +- `TotalPartitions` - Total partition count +- `TotalSizeBytes` / `TotalSizePretty` - Total storage +- `OldestDate` / `NewestDate` - Date range +- `AvgSizeBytes` / `AvgSizePretty` - Average partition size + +--- + +#### GetPartitionsForRemoval +Identifies partitions older than the retention period. + +```go +partitions, err := partitions.GetPartitionsForRemoval(dbc, 180) +if err != nil { + log.WithError(err).Error("failed to get partitions for removal") +} + +fmt.Printf("Found %d partitions older than 180 days\n", len(partitions)) +``` + +**Parameters**: +- `retentionDays` - Retention period in days + +**Returns**: `[]PartitionInfo` for partitions older than retention period (can be deleted or detached) + +--- + +#### GetRetentionSummary +Provides a summary of what would be affected by a retention policy. + +```go +summary, err := partitions.GetRetentionSummary(dbc, 180) +if err != nil { + log.WithError(err).Error("failed to get summary") +} + +fmt.Printf("Would delete %d partitions, reclaiming %s\n", + summary.PartitionsToRemove, summary.StoragePretty) +``` + +**Parameters**: +- `retentionDays` - Retention period in days + +**Returns**: `*RetentionSummary` containing: +- `RetentionDays` - Policy retention period +- `CutoffDate` - Date cutoff for removal +- `PartitionsToRemove` - Count of partitions to remove +- `StorageToReclaim` / `StoragePretty` - Storage to be freed +- `OldestPartition` / `NewestPartition` - Range of affected partitions + +--- + +#### GetPartitionsByAgeGroup +Returns partition counts and sizes grouped by age buckets. + +```go +groups, err := partitions.GetPartitionsByAgeGroup(dbc) +if err != nil { + log.WithError(err).Error("failed to get age groups") +} + +for _, group := range groups { + fmt.Printf("%s: %d partitions, %s (%.2f%%)\n", + group["age_bucket"], + group["partition_count"], + group["total_size"], + group["percentage"]) +} +``` + +**Age Buckets**: +- Future (dates in the future) +- 0-30 days +- 30-90 days +- 90-180 days +- 180-365 days +- 365+ days + +--- + +#### GetPartitionsByMonth +Returns partition counts and sizes grouped by month. + +```go +months, err := partitions.GetPartitionsByMonth(dbc) +if err != nil { + log.WithError(err).Error("failed to get monthly data") +} +``` + +**Returns**: Monthly aggregates with partition counts and sizes + +--- + +#### ValidateRetentionPolicy +Validates that a retention policy is safe to apply. + +```go +err := partitions.ValidateRetentionPolicy(dbc, "test_analysis_by_job_by_dates", 180) +if err != nil { + log.WithError(err).Error("retention policy is not safe") +} +``` + +**Parameters**: +- `tableName` - Name of the partitioned parent table +- `retentionDays` - Retention period in days + +**Safety Checks**: +- Minimum 90 days retention +- Maximum 75% of attached partitions deleted +- Maximum 80% of attached storage deleted + +**Important**: Only considers **attached partitions** when validating thresholds. Detached partitions are excluded from calculations to ensure the policy is safe for active data. + +**Returns**: Error if policy would be unsafe + +--- + +### Write Operations (Require Write Access) + +⚠️ **Warning**: All write operations require database write access. Read-only users will get permission errors. + +#### DropPartition +Drops a single partition. + +```go +// Dry run (safe) +err := partitions.DropPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", true) + +// Actual drop (DESTRUCTIVE) +err := partitions.DropPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", false) +``` + +**Parameters**: +- `partitionName` - Full partition table name +- `dryRun` - If true, only logs what would happen + +**Safety Features**: +- Validates partition name format +- Prevents SQL injection +- Logs all operations + +--- + +#### DetachPartition +Detaches a partition from the parent table (safer alternative to DROP). + +```go +// Dry run +err := partitions.DetachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", true) + +// Actual detach +err := partitions.DetachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", false) +``` + +**Use When**: +- You want to archive data before deletion +- You want a reversible operation (can reattach if needed) + +--- + +#### ListAttachedPartitions +Lists all partitions currently attached to the parent table. + +```go +attached, err := partitions.ListAttachedPartitions(dbc, "test_analysis_by_job_by_dates") +if err != nil { + log.WithError(err).Error("failed to list attached partitions") +} + +for _, p := range attached { + fmt.Printf("%s: %s, Size: %s\n", p.TableName, p.PartitionDate, p.SizePretty) +} +``` + +**Parameters**: +- `tableName` - Name of the partitioned parent table + +**Returns**: `[]PartitionInfo` for attached partitions only + +**How It Works**: +- Queries `pg_inherits` to find partitions in the inheritance hierarchy +- Returns only partitions that are currently attached to the parent table + +**Use When**: +- You need to analyze only active partitions +- You want to distinguish between attached and detached partitions +- You need to check the current state of the partitioned table + +--- + +#### ListDetachedPartitions +Lists all partitions that have been detached from the parent table. + +```go +detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") +if err != nil { + log.WithError(err).Error("failed to list detached partitions") +} + +for _, p := range detached { + fmt.Printf("%s: %s, Size: %s\n", p.TableName, p.PartitionDate, p.SizePretty) +} +``` + +**Parameters**: +- `tableName` - Name of the partitioned parent table + +**Returns**: `[]PartitionInfo` for detached partitions + +**How It Works**: +- Queries `pg_inherits` to find attached partitions +- Returns tables matching the naming pattern but NOT in the inheritance hierarchy + +--- + +#### GetAttachedPartitionStats +Returns statistics about attached partitions only. + +```go +stats, err := partitions.GetAttachedPartitionStats(dbc, "test_analysis_by_job_by_dates") +if err != nil { + log.WithError(err).Error("failed to get attached stats") +} + +fmt.Printf("Attached: %d partitions (%s)\n", + stats.TotalPartitions, stats.TotalSizePretty) +``` + +**Parameters**: +- `tableName` - Name of the partitioned parent table + +**Returns**: `*PartitionStats` with aggregate statistics for attached partitions only + +**Use When**: +- Validating retention policies (should only consider active partitions) +- Analyzing current active storage usage +- Monitoring production partition health + +--- + +#### GetDetachedPartitionStats +Returns statistics about detached partitions. + +```go +stats, err := partitions.GetDetachedPartitionStats(dbc, "test_analysis_by_job_by_dates") +if err != nil { + log.WithError(err).Error("failed to get detached stats") +} + +fmt.Printf("Detached: %d partitions (%s)\n", + stats.TotalPartitions, stats.TotalSizePretty) +``` + +**Returns**: `*PartitionStats` for detached partitions only + +--- + +#### IsPartitionAttached +Checks if a specific partition is currently attached to the parent table. + +```go +isAttached, err := partitions.IsPartitionAttached(dbc, "test_analysis_by_job_by_dates_2024_10_29") +if err != nil { + log.WithError(err).Error("check failed") +} + +if isAttached { + fmt.Println("Partition is part of the parent table") +} else { + fmt.Println("Partition is detached (standalone table)") +} +``` + +**Returns**: `bool` indicating attachment status + +--- + +#### ReattachPartition +Reattaches a previously detached partition back to the parent table. + +```go +// Dry run +err := partitions.ReattachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", true) + +// Actual reattach +err := partitions.ReattachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", false) +``` + +**Use When**: +- You need to restore archived data +- You detached a partition by mistake +- Historical analysis requires old data + +**Note**: Automatically calculates the date range from the partition name + +--- + +#### DetachOldPartitions +Bulk operation to detach all partitions older than retention period. + +```go +// Dry run +detached, err := partitions.DetachOldPartitions(dbc, 180, true) +fmt.Printf("Would detach %d partitions\n", detached) + +// Actual detach +detached, err := partitions.DetachOldPartitions(dbc, 180, false) +fmt.Printf("Detached %d partitions\n", detached) +``` + +**Parameters**: +- `retentionDays` - Retention period in days +- `dryRun` - If true, only simulates the operation + +**Features**: +- Validates retention policy before execution +- Processes partitions in order (oldest first) +- Logs each partition detachment +- Returns count of partitions detached + +--- + +#### DropOldPartitions +Bulk operation to drop all partitions older than retention period. + +```go +// Dry run - see what would happen +dropped, err := partitions.DropOldPartitions(dbc, 180, true) +fmt.Printf("Would drop %d partitions\n", dropped) + +// Actual cleanup (DESTRUCTIVE) +dropped, err := partitions.DropOldPartitions(dbc, 180, false) +fmt.Printf("Dropped %d partitions\n", dropped) +``` + +**Parameters**: +- `retentionDays` - Retention period in days +- `dryRun` - If true, only simulates the operation + +**Features**: +- Validates retention policy before execution +- Processes partitions in order (oldest first) +- Logs each partition drop +- Returns count of partitions dropped + +--- + +#### DropOldDetachedPartitions +Bulk operation to drop detached partitions older than retention period. + +```go +// Dry run - see what would happen +dropped, err := partitions.DropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180, true) +fmt.Printf("Would drop %d detached partitions\n", dropped) + +// Actual cleanup (DESTRUCTIVE) +dropped, err := partitions.DropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180, false) +fmt.Printf("Dropped %d detached partitions\n", dropped) +``` + +**Parameters**: +- `tableName` - Name of the parent table +- `retentionDays` - Retention period in days +- `dryRun` - If true, only simulates the operation + +**Use When**: +- You have detached partitions that have been archived +- You want to clean up old detached partitions no longer needed +- You need to reclaim storage from detached partitions + +**Features**: +- Lists all detached partitions first +- Filters by retention period +- Processes partitions in order (oldest first) +- Logs each partition drop +- Returns count of partitions dropped + +**Note**: Unlike `DropOldPartitions`, this only affects detached partitions. Attached partitions remain untouched. + +--- + +## Usage Examples + +### Example 1: Analyze Current State + +```go +import "github.com/openshift/sippy/pkg/db/partitions" + +func analyzePartitions(dbc *db.DB) { + // Get overall statistics + stats, err := partitions.GetPartitionStats(dbc) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Total: %d partitions (%s)\n", + stats.TotalPartitions, stats.TotalSizePretty) + + // Analyze by age groups + groups, err := partitions.GetPartitionsByAgeGroup(dbc) + if err != nil { + log.Fatal(err) + } + + for _, group := range groups { + fmt.Printf("%s: %s\n", group["age_bucket"], group["total_size"]) + } +} +``` + +### Example 2: Dry Run Cleanup + +```go +func dryRunCleanup(dbc *db.DB, retentionDays int) { + // Validate policy + if err := partitions.ValidateRetentionPolicy(dbc, retentionDays); err != nil { + log.Fatalf("Policy validation failed: %v", err) + } + + // Get summary + summary, err := partitions.GetRetentionSummary(dbc, retentionDays) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Would delete %d partitions, reclaiming %s\n", + summary.PartitionsToRemove, summary.StoragePretty) + + // Perform dry run + dropped, err := partitions.DropOldPartitions(dbc, retentionDays, true) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Dry run complete: %d partitions would be dropped\n", dropped) +} +``` + +### Example 3: Execute Cleanup (Production) + +```go +func executeCleanup(dbc *db.DB, retentionDays int) { + // Always validate first + if err := partitions.ValidateRetentionPolicy(dbc, retentionDays); err != nil { + return fmt.Errorf("retention policy failed validation: %w", err) + } + + // Get summary for logging + summary, err := partitions.GetRetentionSummary(dbc, retentionDays) + if err != nil { + return err + } + + log.WithFields(log.Fields{ + "retention_days": retentionDays, + "partitions_to_delete": summary.PartitionsToRemove, + "storage_to_reclaim": summary.StoragePretty, + }).Info("starting partition cleanup") + + // Execute cleanup (NOT a dry run) + dropped, err := partitions.DropOldPartitions(dbc, retentionDays, false) + if err != nil { + return fmt.Errorf("cleanup failed: %w", err) + } + + log.WithField("dropped", dropped).Info("partition cleanup completed") + return nil +} +``` + +### Example 4: Detach Instead of Drop (Safer) + +```go +func detachForArchival(dbc *db.DB, retentionDays int) error { + // Validate policy + if err := partitions.ValidateRetentionPolicy(dbc, retentionDays); err != nil { + return err + } + + // Detach old partitions instead of dropping + detached, err := partitions.DetachOldPartitions(dbc, retentionDays, false) + if err != nil { + return fmt.Errorf("detach failed: %w", err) + } + + log.WithField("detached", detached).Info("partitions detached for archival") + + // Now archive the detached partitions (external process) + // archiveDetachedPartitions(dbc) + + return nil +} +``` + +### Example 5: Compare Attached vs Detached Partitions + +```go +func comparePartitionState(dbc *db.DB, tableName string) error { + // Get all partitions (attached + detached) + allPartitions, err := partitions.ListTablePartitions(dbc, tableName) + if err != nil { + return err + } + + // Get only attached partitions + attached, err := partitions.ListAttachedPartitions(dbc, tableName) + if err != nil { + return err + } + + // Get only detached partitions + detached, err := partitions.ListDetachedPartitions(dbc, tableName) + if err != nil { + return err + } + + // Display summary + fmt.Printf("Partition State for %s:\n", tableName) + fmt.Printf(" Total: %d partitions\n", len(allPartitions)) + fmt.Printf(" Attached: %d partitions\n", len(attached)) + fmt.Printf(" Detached: %d partitions\n", len(detached)) + + // Calculate storage breakdown + var attachedSize, detachedSize int64 + for _, p := range attached { + attachedSize += p.SizeBytes + } + for _, p := range detached { + detachedSize += p.SizeBytes + } + + fmt.Printf("\nStorage Breakdown:\n") + fmt.Printf(" Attached: %d bytes\n", attachedSize) + fmt.Printf(" Detached: %d bytes\n", detachedSize) + fmt.Printf(" Total: %d bytes\n", attachedSize+detachedSize) + + return nil +} +``` + +--- + +### Example 6: Working with Detached Partitions + +```go +func manageDetachedPartitions(dbc *db.DB) error { + // List all detached partitions + detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") + if err != nil { + return err + } + + fmt.Printf("Found %d detached partitions\n", len(detached)) + + // Get statistics + stats, err := partitions.GetDetachedPartitionStats(dbc, "test_analysis_by_job_by_dates") + if err != nil { + return err + } + + fmt.Printf("Detached partitions total: %s\n", stats.TotalSizePretty) + + // Check if specific partition is detached + for _, p := range detached { + isAttached, err := partitions.IsPartitionAttached(dbc, p.TableName) + if err != nil { + continue + } + + if !isAttached { + fmt.Printf("%s is detached and ready for archival\n", p.TableName) + // Archive this partition to S3, compress, etc. + } + } + + return nil +} +``` + +--- + +### Example 7: Reattach Archived Data + +```go +func restoreArchivedPartition(dbc *db.DB, partitionName string) error { + // Check current status + isAttached, err := partitions.IsPartitionAttached(dbc, partitionName) + if err != nil { + return err + } + + if isAttached { + return fmt.Errorf("partition %s is already attached", partitionName) + } + + log.WithField("partition", partitionName).Info("reattaching partition") + + // Reattach the partition + err = partitions.ReattachPartition(dbc, partitionName, false) + if err != nil { + return fmt.Errorf("reattach failed: %w", err) + } + + log.Info("partition reattached successfully") + return nil +} +``` + +### Example 4: Complete Workflow + +See [examples.go](./examples.go) for a complete workflow demonstration including: +- Current state analysis +- Age distribution +- Retention policy comparison +- Dry run execution + +--- + +## Integration with Automation + +### Option 1: Kubernetes CronJob + +```go +// In your scheduled job +func scheduledCleanup() { + dbc := db.New(...) + + // 180-day retention policy + dropped, err := partitions.DropOldPartitions(dbc, 180, false) + if err != nil { + log.WithError(err).Error("scheduled cleanup failed") + return + } + + log.WithField("dropped", dropped).Info("scheduled cleanup completed") +} +``` + +### Option 2: CLI Command + +```go +func main() { + retentionDays := flag.Int("retention-days", 180, "Retention period in days") + dryRun := flag.Bool("dry-run", true, "Perform dry run only") + flag.Parse() + + dbc := db.New(...) + + dropped, err := partitions.DropOldPartitions(dbc, *retentionDays, *dryRun) + if err != nil { + log.Fatal(err) + } + + if *dryRun { + fmt.Printf("DRY RUN: Would drop %d partitions\n", dropped) + } else { + fmt.Printf("Dropped %d partitions\n", dropped) + } +} +``` + +--- + +## Safety Features + +### Input Validation +- Partition names are validated against expected format +- SQL injection protection through parameterized queries +- Minimum retention period enforcement (30 days) + +### Threshold Checks +- Maximum 75% of partitions can be deleted +- Maximum 80% of storage can be deleted +- Policy must be validated before execution + +### Dry Run Support +- All destructive operations support dry-run mode +- Dry runs log what would happen without making changes +- Always test with dry-run first + +### Comprehensive Logging +- All operations are logged with structured fields +- Errors include context for debugging +- Timing information for performance monitoring + +--- + +## Error Handling + +All functions return errors that should be checked: + +```go +partitions, err := partitions.ListTablePartitions(dbc, "test_analysis_by_job_by_dates") +if err != nil { + log.WithError(err).Error("failed to list partitions") + return err +} +``` + +Common error scenarios: +- Database connection issues +- Permission denied (read-only user attempting writes) +- Invalid retention policy +- Partition name validation failures + +--- + +## Testing + +Run the test suite: + +```bash +go test ./pkg/db/partitions/... +``` + +Test coverage includes: +- Partition name validation +- Struct initialization +- Edge cases and invalid inputs + +--- + +## Detach/Archive Workflow + +### Understanding Detached Partitions + +When a partition is **detached**, it: +1. Becomes a standalone table (no longer part of the partitioned table) +2. Keeps all its data intact +3. Can still be queried directly by table name +4. Can be archived, compressed, or exported +5. Can be reattached if needed +6. Doesn't show up in queries against the parent table + +### How to Find Detached Partitions + +PostgreSQL tracks partition relationships in `pg_inherits`. Detached partitions: +- Still exist as tables in `pg_tables` +- Are NOT in the `pg_inherits` hierarchy +- Match the partition naming pattern + +**Query to find them:** +```go +detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") +// Returns all tables matching naming pattern but not attached +``` + +### Typical Detach/Archive Workflow + +#### Step 1: Detach Old Partitions +```go +// Detach partitions older than 180 days +detached, err := partitions.DetachOldPartitions(dbc, 180, false) +log.Printf("Detached %d partitions\n", detached) +``` + +**Result**: Partitions are now standalone tables + +#### Step 2: List Detached Partitions +```go +// Find all detached partitions +detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") + +for _, p := range detached { + fmt.Printf("Detached: %s (%s)\n", p.TableName, p.SizePretty) +} +``` + +#### Step 3: Archive Detached Partitions +External archival process (examples): + +**Option A: Export to CSV/Parquet** +```bash +# Export to compressed CSV +psql $SIPPY_DSN -c " +COPY test_analysis_by_job_by_dates_2024_10_29 +TO STDOUT CSV HEADER +" | gzip > partition_2024_10_29.csv.gz + +# Upload to S3 +aws s3 cp partition_2024_10_29.csv.gz s3://sippy-archive/ +``` + +**Option B: Use pg_dump** +```bash +pg_dump $SIPPY_DSN \ + -t test_analysis_by_job_by_dates_2024_10_29 \ + --format=custom \ + | gzip > partition_2024_10_29.pgdump.gz +``` + +**Option C: Direct S3 export (requires aws_s3 extension)** +```sql +SELECT aws_s3.query_export_to_s3( + 'SELECT * FROM test_analysis_by_job_by_dates_2024_10_29', + aws_commons.create_s3_uri('sippy-archive', 'partitions/2024_10_29.parquet', 'us-east-1'), + options := 'FORMAT PARQUET' +); +``` + +#### Step 4: Verify Archive +```bash +# Verify archive exists and is readable +aws s3 ls s3://sippy-archive/partition_2024_10_29.csv.gz +# Check file size matches expected +``` + +#### Step 5: Drop Detached Partitions + +**Option A: Bulk drop old detached partitions (recommended)** +```go +// Drop all detached partitions older than 180 days +// (Assumes they have already been archived) + +// Dry run first +dropped, err := partitions.DropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180, true) +fmt.Printf("Would drop %d detached partitions\n", dropped) + +// Actual drop +dropped, err = partitions.DropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180, false) +fmt.Printf("Dropped %d detached partitions\n", dropped) +``` + +**Option B: Selective drop with archive verification** +```go +// After successful archive, drop detached partitions +detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") + +for _, p := range detached { + // Verify this partition has been archived + if isArchived(p.TableName) { + err := partitions.DropPartition(dbc, p.TableName, false) + if err != nil { + log.WithError(err).Error("failed to drop detached partition") + } + } +} +``` + +#### Step 6: Restore if Needed +If you need to restore archived data: + +1. **Restore from archive**: +```bash +# Restore table from pg_dump +gunzip -c partition_2024_10_29.pgdump.gz | pg_restore -d $SIPPY_DSN +``` + +2. **Reattach partition**: +```go +err := partitions.ReattachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", false) +``` + +### Advantages of Detach vs. DROP + +| Aspect | DETACH | DROP | +|--------|--------|------| +| **Reversible** | ✅ Yes (can reattach) | ❌ No (permanent) | +| **Data preserved** | ✅ Yes (in detached table) | ❌ No (deleted) | +| **Immediate space** | ❌ No (table still exists) | ✅ Yes (storage freed) | +| **Archive time** | ✅ After detach | ⚠️ Before drop | +| **Risk** | 🟢 Low | 🔴 High | +| **Speed** | ⚡ Fast | ⚡ Fast | +| **Query detached data** | ✅ Yes (by table name) | ❌ No (gone) | + +### Complete Automation Example + +```go +func automatedArchiveCleanup(dbc *db.DB, archiver Archiver) error { + retentionDays := 180 + + // 1. Detach old partitions + detached, err := partitions.DetachOldPartitions(dbc, retentionDays, false) + if err != nil { + return err + } + + log.Printf("Detached %d partitions\n", detached) + + // 2. Get list of detached partitions + detachedList, err := partitions.ListDetachedPartitions(dbc) + if err != nil { + return err + } + + // 3. Archive each detached partition + for _, p := range detachedList { + // Archive to S3 + err := archiver.Archive(p.TableName) + if err != nil { + log.WithError(err).WithField("partition", p.TableName).Error("archive failed") + continue + } + + // Verify archive + if !archiver.Verify(p.TableName) { + log.WithField("partition", p.TableName).Error("archive verification failed") + continue + } + + // Drop detached partition + err = partitions.DropPartition(dbc, p.TableName, false) + if err != nil { + log.WithError(err).WithField("partition", p.TableName).Error("drop failed") + continue + } + + log.WithField("partition", p.TableName).Info("archived and dropped successfully") + } + + return nil +} +``` + +--- + +## Related Documentation + +- [Partition Retention Management Guide](../../../.claude/partition-retention-management-guide.md) - Complete guide with SQL examples +- [Database Schema Analysis](../../../.claude/db-schema-analysis.md) - Overall database structure +- [Database Analysis Index](../../../.claude/db-analysis-index.md) - Navigation to all analysis docs + +--- + +## Recommended Retention Policies + +Based on analysis in the retention management guide: + +| Policy | Retention | Storage | Use Case | +|--------|-----------|---------|----------| +| Conservative | 365 days | ~900 GB | Full year of data, Y-o-Y comparisons | +| **Recommended** | **180 days** | **~450 GB** | **6 months, covers release cycles** | +| Aggressive | 90 days | ~225 GB | Recent CI health only, max savings | + +**Current recommendation**: **180-day retention** +- Balances historical data access with storage efficiency +- Covers typical OpenShift release cycles +- Would reclaim ~160 GB immediately +- Stabilizes storage at ~450 GB + +--- + +## Notes + +- All operations require `*db.DB` instance (GORM wrapper) +- Read-only operations are safe with read-only database credentials +- Write operations require admin credentials +- Partition format: `test_analysis_by_job_by_dates_YYYY_MM_DD` +- Only `test_analysis_by_job_by_dates` partitions are supported currently diff --git a/pkg/db/partitions/examples.go b/pkg/db/partitions/examples.go new file mode 100644 index 000000000..7062878f5 --- /dev/null +++ b/pkg/db/partitions/examples.go @@ -0,0 +1,608 @@ +package partitions + +import ( + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/openshift/sippy/pkg/db" +) + +// ExampleListPartitionedTables demonstrates how to list all partitioned tables +// +// Usage: +// +// ExampleListPartitionedTables(dbc) +func ExampleListPartitionedTables(dbc *db.DB) { + tables, err := ListPartitionedTables(dbc) + if err != nil { + log.WithError(err).Error("failed to list partitioned tables") + return + } + + fmt.Printf("Found %d partitioned tables:\n", len(tables)) + for _, t := range tables { + fmt.Printf(" %s: %d partitions, Strategy: %s\n", + t.TableName, t.PartitionCount, t.PartitionStrategy) + } +} + +// ExampleListPartitions demonstrates how to list partitions for a table +// If retentionDays > 0, only shows partitions older than that value +// If retentionDays <= 0, shows all partitions +// +// Usage: +// +// ExampleListPartitions(dbc, "test_analysis_by_job_by_dates", 180) // Show partitions older than 180 days +// ExampleListPartitions(dbc, "test_analysis_by_job_by_dates", 0) // Show all partitions +func ExampleListPartitions(dbc *db.DB, tableName string, retentionDays int) { + partitions, err := ListTablePartitions(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to list partitions") + return + } + + if retentionDays > 0 { + fmt.Printf("Partitions older than %d days for %s:\n", retentionDays, tableName) + } else { + fmt.Printf("All partitions for %s:\n", tableName) + } + + for _, p := range partitions { + if p.Age > retentionDays || retentionDays < 1 { + fmt.Printf(" %s - Date: %s, Age: %d days, Size: %s\n", + p.TableName, p.PartitionDate.Format("2006-01-02"), p.Age, p.SizePretty) + } + } +} + +// ExampleGetStats demonstrates how to get partition statistics +// +// Usage: +// +// ExampleGetStats(dbc, "test_analysis_by_job_by_dates") +func ExampleGetStats(dbc *db.DB, tableName string) { + stats, err := GetPartitionStats(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to get stats") + return + } + + fmt.Printf("\nPartition Statistics for %s:\n", tableName) + fmt.Printf(" Total Partitions: %d\n", stats.TotalPartitions) + fmt.Printf(" Total Size: %s\n", stats.TotalSizePretty) + fmt.Printf(" Average Size: %s\n", stats.AvgSizePretty) + fmt.Printf(" Date Range: %s to %s\n", + stats.OldestDate.Format("2006-01-02"), + stats.NewestDate.Format("2006-01-02")) +} + +// ExampleComparePartitionStats demonstrates comparing attached vs detached partition statistics +// +// Usage: +// +// ExampleComparePartitionStats(dbc, "test_analysis_by_job_by_dates") +func ExampleComparePartitionStats(dbc *db.DB, tableName string) { + fmt.Printf("\n=== Partition Statistics Comparison for %s ===\n", tableName) + + // Get all partition stats + allStats, err := GetPartitionStats(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to get all partition stats") + return + } + + // Get attached partition stats + attachedStats, err := GetAttachedPartitionStats(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to get attached partition stats") + return + } + + // Get detached partition stats + detachedStats, err := GetDetachedPartitionStats(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to get detached partition stats") + return + } + + fmt.Printf("\nAll Partitions (Attached + Detached):\n") + fmt.Printf(" Total: %d partitions (%s)\n", allStats.TotalPartitions, allStats.TotalSizePretty) + + fmt.Printf("\nAttached Partitions:\n") + fmt.Printf(" Total: %d partitions (%s)\n", attachedStats.TotalPartitions, attachedStats.TotalSizePretty) + if attachedStats.TotalPartitions > 0 { + fmt.Printf(" Range: %s to %s\n", + attachedStats.OldestDate.Format("2006-01-02"), + attachedStats.NewestDate.Format("2006-01-02")) + } + + fmt.Printf("\nDetached Partitions:\n") + fmt.Printf(" Total: %d partitions (%s)\n", detachedStats.TotalPartitions, detachedStats.TotalSizePretty) + if detachedStats.TotalPartitions > 0 { + fmt.Printf(" Range: %s to %s\n", + detachedStats.OldestDate.Format("2006-01-02"), + detachedStats.NewestDate.Format("2006-01-02")) + } + + // Calculate percentages + if allStats.TotalPartitions > 0 { + attachedPct := float64(attachedStats.TotalPartitions) / float64(allStats.TotalPartitions) * 100 + detachedPct := float64(detachedStats.TotalPartitions) / float64(allStats.TotalPartitions) * 100 + fmt.Printf("\nDistribution:\n") + fmt.Printf(" Attached: %.1f%%\n", attachedPct) + fmt.Printf(" Detached: %.1f%%\n", detachedPct) + } +} + +// ExampleCheckRetentionPolicy demonstrates how to check what a retention policy would affect +// +// Usage: +// +// ExampleCheckRetentionPolicy(dbc, "test_analysis_by_job_by_dates", 180) +func ExampleCheckRetentionPolicy(dbc *db.DB, tableName string, retentionDays int) { + // First validate the policy + if err := ValidateRetentionPolicy(dbc, tableName, retentionDays); err != nil { + log.WithError(err).Error("retention policy validation failed") + return + } + + // Get summary of what would be affected + summary, err := GetRetentionSummary(dbc, tableName, retentionDays) + if err != nil { + log.WithError(err).Error("failed to get retention summary") + return + } + + fmt.Printf("\nRetention Policy Analysis for %s (%d days):\n", tableName, retentionDays) + fmt.Printf(" Cutoff Date: %s\n", summary.CutoffDate.Format("2006-01-02")) + fmt.Printf(" Partitions to Remove: %d\n", summary.PartitionsToRemove) + fmt.Printf(" Storage to Reclaim: %s\n", summary.StoragePretty) + if summary.PartitionsToRemove > 0 { + fmt.Printf(" Oldest: %s\n", summary.OldestPartition) + fmt.Printf(" Newest: %s\n", summary.NewestPartition) + } + + // Get detailed list of partitions that would be removed + partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays) + if err != nil { + log.WithError(err).Error("failed to get partitions for removal") + return + } + + if len(partitions) > 0 { + fmt.Printf("\nPartitions that would be removed (showing first 10):\n") + for i, p := range partitions { + if i < 10 { + fmt.Printf(" %s - %s ago, Size: %s\n", + p.TableName, p.PartitionDate.Format("2006-01-02"), p.SizePretty) + } + } + if len(partitions) > 10 { + fmt.Printf(" ... and %d more\n", len(partitions)-10) + } + } +} + +// ExampleAgeGroupAnalysis demonstrates how to analyze partitions by age +// +// Usage: +// +// ExampleAgeGroupAnalysis(dbc, "test_analysis_by_job_by_dates") +func ExampleAgeGroupAnalysis(dbc *db.DB, tableName string) { + groups, err := GetPartitionsByAgeGroup(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to get age groups") + return + } + + fmt.Printf("\nPartitions by Age Group for %s:\n", tableName) + for _, group := range groups { + fmt.Printf(" %s: %d partitions, %s (%.2f%%)\n", + group["age_bucket"], + group["partition_count"], + group["total_size"], + group["percentage"]) + } +} + +// ExampleMonthlyAnalysis demonstrates how to analyze partitions by month +// +// Usage: +// +// ExampleMonthlyAnalysis(dbc, "test_analysis_by_job_by_dates") +func ExampleMonthlyAnalysis(dbc *db.DB, tableName string) { + months, err := GetPartitionsByMonth(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to get monthly breakdown") + return + } + + fmt.Printf("\nPartitions by Month for %s (recent):\n", tableName) + for i, month := range months { + if i < 6 { // Show last 6 months + fmt.Printf(" %v: %d partitions, Total: %s, Avg: %s\n", + month["month"], + month["partition_count"], + month["total_size"], + month["avg_partition_size"]) + } + } +} + +// ExampleDryRunCleanup demonstrates a dry-run cleanup operation +// +// Usage: +// +// ExampleDryRunCleanup(dbc, "test_analysis_by_job_by_dates", 180) +func ExampleDryRunCleanup(dbc *db.DB, tableName string, retentionDays int) { + fmt.Printf("\n=== DRY RUN: Partition Cleanup for %s (%d day retention) ===\n", tableName, retentionDays) + + // Validate policy + if err := ValidateRetentionPolicy(dbc, tableName, retentionDays); err != nil { + log.WithError(err).Error("retention policy failed validation") + return + } + + // Get summary + summary, err := GetRetentionSummary(dbc, tableName, retentionDays) + if err != nil { + log.WithError(err).Error("failed to get summary") + return + } + + if summary.PartitionsToRemove == 0 { + fmt.Println("No partitions to delete") + return + } + + fmt.Printf("Would delete %d partitions, reclaiming %s\n", + summary.PartitionsToRemove, summary.StoragePretty) + + // Perform dry run + dropped, err := DropOldPartitions(dbc, tableName, retentionDays, true) // true = dry run + if err != nil { + log.WithError(err).Error("dry run failed") + return + } + + fmt.Printf("Dry run completed: would drop %d partitions\n", dropped) +} + +// ExampleDetachedPartitions demonstrates working with detached partitions for a table +// +// Usage: +// +// ExampleDetachedPartitions(dbc, "test_analysis_by_job_by_dates") +func ExampleDetachedPartitions(dbc *db.DB, tableName string) { + fmt.Printf("\n=== Detached Partitions for %s ===\n", tableName) + + // List detached partitions + detached, err := ListDetachedPartitions(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to list detached partitions") + return + } + + if len(detached) == 0 { + fmt.Println("No detached partitions found") + return + } + + fmt.Printf("Found %d detached partitions:\n", len(detached)) + for i, p := range detached { + if i < 5 { + fmt.Printf(" %s - Date: %s, Size: %s\n", + p.TableName, p.PartitionDate.Format("2006-01-02"), p.SizePretty) + } + } + + // Get statistics about detached partitions + stats, err := GetDetachedPartitionStats(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to get detached stats") + return + } + + fmt.Printf("\nDetached Partition Statistics:\n") + fmt.Printf(" Total: %d partitions (%s)\n", stats.TotalPartitions, stats.TotalSizePretty) + if stats.TotalPartitions > 0 { + fmt.Printf(" Range: %s to %s\n", + stats.OldestDate.Format("2006-01-02"), + stats.NewestDate.Format("2006-01-02")) + } +} + +// ExampleAttachedPartitions demonstrates working with attached partitions for a table +// +// Usage: +// +// ExampleAttachedPartitions(dbc, "test_analysis_by_job_by_dates") +func ExampleAttachedPartitions(dbc *db.DB, tableName string) { + fmt.Printf("\n=== Attached Partitions for %s ===\n", tableName) + + // List attached partitions + attached, err := ListAttachedPartitions(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to list attached partitions") + return + } + + if len(attached) == 0 { + fmt.Println("No attached partitions found") + return + } + + fmt.Printf("Found %d attached partitions:\n", len(attached)) + for i, p := range attached { + if i < 10 { + fmt.Printf(" %s - Date: %s, Age: %d days, Size: %s\n", + p.TableName, p.PartitionDate.Format("2006-01-02"), p.Age, p.SizePretty) + } + } + + if len(attached) > 10 { + fmt.Printf(" ... and %d more\n", len(attached)-10) + } + + // Calculate total size + var totalSize int64 + for _, p := range attached { + totalSize += p.SizeBytes + } + + fmt.Printf("\nAttached Partition Summary:\n") + fmt.Printf(" Total: %d partitions\n", len(attached)) + fmt.Printf(" Total Size: %d bytes\n", totalSize) + if len(attached) > 0 { + fmt.Printf(" Range: %s to %s\n", + attached[0].PartitionDate.Format("2006-01-02"), + attached[len(attached)-1].PartitionDate.Format("2006-01-02")) + } +} + +// ExampleDropOldDetachedPartitions demonstrates dropping old detached partitions +// +// Usage: +// +// ExampleDropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180) +func ExampleDropOldDetachedPartitions(dbc *db.DB, tableName string, retentionDays int) { + fmt.Printf("\n=== Drop Old Detached Partitions for %s (%d days) ===\n", tableName, retentionDays) + + // 1. Check what detached partitions exist + detached, err := ListDetachedPartitions(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to list detached partitions") + return + } + + if len(detached) == 0 { + fmt.Println("No detached partitions found") + return + } + + fmt.Printf("Found %d detached partitions\n", len(detached)) + + // 2. Show which ones would be dropped + cutoffDate := time.Now().AddDate(0, 0, -retentionDays) + fmt.Printf("Cutoff date: %s\n", cutoffDate.Format("2006-01-02")) + + toRemove := 0 + var totalSize int64 + for _, p := range detached { + if p.PartitionDate.Before(cutoffDate) { + toRemove++ + totalSize += p.SizeBytes + if toRemove <= 5 { + fmt.Printf(" Would drop: %s (Age: %d days, Size: %s)\n", + p.TableName, p.Age, p.SizePretty) + } + } + } + + if toRemove > 5 { + fmt.Printf(" ... and %d more\n", toRemove-5) + } + + if toRemove == 0 { + fmt.Println("No detached partitions older than retention period") + return + } + + fmt.Printf("\nTotal to remove: %d partitions\n", toRemove) + + // 3. Dry run + fmt.Println("\nRunning dry run...") + dropped, err := DropOldDetachedPartitions(dbc, tableName, retentionDays, true) + if err != nil { + log.WithError(err).Error("dry run failed") + return + } + + fmt.Printf("Dry run completed: would drop %d detached partitions\n", dropped) + + // 4. Actual drop (commented out for safety) + // fmt.Println("\nActual drop (uncomment to execute):") + // dropped, err = DropOldDetachedPartitions(dbc, tableName, retentionDays, false) + // if err != nil { + // log.WithError(err).Error("drop failed") + // return + // } + // fmt.Printf("Dropped %d detached partitions\n", dropped) +} + +// ExampleDetachWorkflow demonstrates the detach/archive workflow +// +// Usage: +// +// ExampleDetachWorkflow(dbc, "test_analysis_by_job_by_dates", 180) +func ExampleDetachWorkflow(dbc *db.DB, tableName string, retentionDays int) { + fmt.Printf("\n=== Detach Workflow for %s (%d days) ===\n", tableName, retentionDays) + + // 1. Check what would be detached + summary, err := GetRetentionSummary(dbc, tableName, retentionDays) + if err != nil { + log.WithError(err).Error("failed to get summary") + return + } + + fmt.Printf("1. Would detach %d partitions (%s)\n", + summary.PartitionsToRemove, summary.StoragePretty) + + // 2. Detach partitions (dry run) + detached, err := DetachOldPartitions(dbc, tableName, retentionDays, true) + if err != nil { + log.WithError(err).Error("dry run failed") + return + } + + fmt.Printf("2. Dry run: would detach %d partitions\n", detached) + + // 3. Actual detach (commented out - requires admin) + // detached, err = DetachOldPartitions(dbc, tableName, retentionDays, false) + // fmt.Printf("3. Detached %d partitions\n", detached) + + // 4. Check detached partitions + fmt.Println("\n4. After detachment, you can:") + fmt.Println(" - Archive to S3 using external scripts") + fmt.Println(" - Compress and store offline") + fmt.Println(" - Query detached tables directly if needed") + fmt.Println(" - Reattach if data is needed again") + fmt.Println(" - Drop when ready to free storage") +} + +// ExampleReattachPartition demonstrates reattaching a detached partition +// +// Usage: +// +// ExampleReattachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29") +func ExampleReattachPartition(dbc *db.DB, partitionName string) { + fmt.Printf("\n=== Reattach Partition: %s ===\n", partitionName) + + // 1. Check if partition is attached + isAttached, err := IsPartitionAttached(dbc, partitionName) + if err != nil { + log.WithError(err).Error("failed to check partition status") + return + } + + fmt.Printf("1. Partition attached: %v\n", isAttached) + + if isAttached { + fmt.Println("Partition is already attached, no action needed") + return + } + + // 2. Reattach (dry run) + err = ReattachPartition(dbc, partitionName, true) + if err != nil { + log.WithError(err).Error("dry run failed") + return + } + + fmt.Println("2. Dry run successful") + + // 3. Actual reattach (commented out - requires admin) + // err = ReattachPartition(dbc, partitionName, false) + // if err != nil { + // log.WithError(err).Error("reattach failed") + // return + // } + // fmt.Println("3. Partition reattached successfully") +} + +// ExampleWorkflowForAnyTable demonstrates managing partitions for any table +// +// Usage: +// +// ExampleWorkflowForAnyTable(dbc) +func ExampleWorkflowForAnyTable(dbc *db.DB) { + fmt.Println("=== Managing Partitions for Any Table ===") + + // 1. List all partitioned tables + fmt.Println("\n1. Discovering partitioned tables:") + tables, err := ListPartitionedTables(dbc) + if err != nil { + log.WithError(err).Error("failed to list partitioned tables") + return + } + + for _, table := range tables { + fmt.Printf(" - %s: %d partitions (%s)\n", + table.TableName, table.PartitionCount, table.PartitionStrategy) + } + + // 2. For each table, analyze retention + fmt.Println("\n2. Analyzing retention policies:") + for _, table := range tables { + fmt.Printf("\nTable: %s\n", table.TableName) + + // Get current stats + stats, err := GetPartitionStats(dbc, table.TableName) + if err != nil { + log.WithError(err).WithField("table", table.TableName).Error("failed to get stats") + continue + } + + fmt.Printf(" Total: %d partitions (%s)\n", + stats.TotalPartitions, stats.TotalSizePretty) + fmt.Printf(" Range: %s to %s\n", + stats.OldestDate.Format("2006-01-02"), + stats.NewestDate.Format("2006-01-02")) + + // Check 180-day retention policy + summary, err := GetRetentionSummary(dbc, table.TableName, 180) + if err != nil { + log.WithError(err).WithField("table", table.TableName).Error("failed to get summary") + continue + } + + if summary.PartitionsToRemove > 0 { + fmt.Printf(" 180-day policy: Would remove %d partitions (%s)\n", + summary.PartitionsToRemove, summary.StoragePretty) + } else { + fmt.Println(" 180-day policy: No partitions to remove") + } + } +} + +// ExampleCompleteWorkflow demonstrates a complete partition management workflow for a specific table +// +// Usage: +// +// ExampleCompleteWorkflow(dbc, "test_analysis_by_job_by_dates") +func ExampleCompleteWorkflow(dbc *db.DB, tableName string) { + fmt.Printf("=== Partition Management Workflow for %s ===\n", tableName) + + // 1. Get current state + fmt.Println("\n1. Current State:") + ExampleGetStats(dbc, tableName) + + // 2. Analyze by age + fmt.Println("\n2. Age Distribution:") + ExampleAgeGroupAnalysis(dbc, tableName) + + // 3. Check various retention policies + for _, days := range []int{90, 180, 365} { + fmt.Printf("\n3. Analyzing %d-day retention policy:\n", days) + ExampleCheckRetentionPolicy(dbc, tableName, days) + } + + // 4. Recommended: 180-day retention dry run + fmt.Println("\n4. Recommended Policy (180 days):") + ExampleDryRunCleanup(dbc, tableName, 180) + + // 5. Check for detached partitions + fmt.Println("\n5. Detached Partitions:") + ExampleDetachedPartitions(dbc, tableName) + + fmt.Println("\n=== Workflow Complete ===") + fmt.Println("Options for cleanup:") + fmt.Printf(" 1. DROP immediately:\n") + fmt.Printf(" dropped, err := partitions.DropOldPartitions(dbc, \"%s\", 180, false)\n", tableName) + fmt.Printf(" 2. DETACH for archival:\n") + fmt.Printf(" detached, err := partitions.DetachOldPartitions(dbc, \"%s\", 180, false)\n", tableName) + fmt.Println(" // Archive detached partitions to S3") + fmt.Println(" // Drop detached partitions when archived") +} diff --git a/pkg/db/partitions/partitions.go b/pkg/db/partitions/partitions.go new file mode 100644 index 000000000..ec1786c08 --- /dev/null +++ b/pkg/db/partitions/partitions.go @@ -0,0 +1,1011 @@ +package partitions + +import ( + "database/sql" + "fmt" + "strings" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/openshift/sippy/pkg/db" +) + +// PartitionInfo holds metadata about a partition +type PartitionInfo struct { + TableName string `gorm:"column:tablename"` + SchemaName string `gorm:"column:schemaname"` + PartitionDate time.Time `gorm:"column:partition_date"` + Age int `gorm:"column:age_days"` + SizeBytes int64 `gorm:"column:size_bytes"` + SizePretty string `gorm:"column:size_pretty"` + RowEstimate int64 `gorm:"column:row_estimate"` +} + +// PartitionedTableInfo holds metadata about a partitioned parent table +type PartitionedTableInfo struct { + TableName string `gorm:"column:tablename"` + SchemaName string `gorm:"column:schemaname"` + PartitionCount int `gorm:"column:partition_count"` + PartitionStrategy string `gorm:"column:partition_strategy"` +} + +// PartitionStats holds aggregate statistics about partitions +type PartitionStats struct { + TotalPartitions int + TotalSizeBytes int64 + TotalSizePretty string + OldestDate time.Time + NewestDate time.Time + AvgSizeBytes int64 + AvgSizePretty string +} + +// RetentionSummary provides a summary of what would be affected by a retention policy +type RetentionSummary struct { + RetentionDays int + CutoffDate time.Time + PartitionsToRemove int + StorageToReclaim int64 + StoragePretty string + OldestPartition string + NewestPartition string +} + +// ListPartitionedTables returns all partitioned parent tables in the database +func ListPartitionedTables(dbc *db.DB) ([]PartitionedTableInfo, error) { + start := time.Now() + var tables []PartitionedTableInfo + + query := ` + SELECT + c.relname AS tablename, + n.nspname AS schemaname, + COUNT(i.inhrelid)::INT AS partition_count, + CASE pp.partstrat + WHEN 'r' THEN 'RANGE' + WHEN 'l' THEN 'LIST' + WHEN 'h' THEN 'HASH' + ELSE 'UNKNOWN' + END AS partition_strategy + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_partitioned_table pp ON pp.partrelid = c.oid + LEFT JOIN pg_inherits i ON i.inhparent = c.oid + WHERE n.nspname = 'public' + GROUP BY c.relname, n.nspname, pp.partstrat + ORDER BY c.relname + ` + + result := dbc.DB.Raw(query).Scan(&tables) + if result.Error != nil { + log.WithError(result.Error).Error("failed to list partitioned tables") + return nil, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "count": len(tables), + "elapsed": elapsed, + }).Info("listed partitioned tables") + + return tables, nil +} + +// ListTablePartitions returns all partitions for a given table +func ListTablePartitions(dbc *db.DB, tableName string) ([]PartitionInfo, error) { + start := time.Now() + var partitions []PartitionInfo + + query := ` + SELECT + tablename, + 'public' as schemaname, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + (CURRENT_DATE - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD'))::INT AS age_days, + pg_total_relation_size('public.'||tablename) AS size_bytes, + pg_size_pretty(pg_total_relation_size('public.'||tablename)) AS size_pretty, + COALESCE(n_live_tup, 0) AS row_estimate + FROM pg_tables + LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename + AND pg_stat_user_tables.schemaname = pg_tables.schemaname + WHERE pg_tables.schemaname = 'public' + AND pg_tables.tablename LIKE @table_pattern + ORDER BY partition_date ASC + ` + + tablePattern := tableName + "_20%" + result := dbc.DB.Raw(query, sql.Named("table_pattern", tablePattern)).Scan(&partitions) + if result.Error != nil { + log.WithError(result.Error).WithField("table", tableName).Error("failed to list table partitions") + return nil, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "count": len(partitions), + "elapsed": elapsed, + }).Info("listed table partitions") + + return partitions, nil +} + +// GetPartitionStats returns aggregate statistics about partitions for a given table +func GetPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, error) { + start := time.Now() + var stats PartitionStats + + query := ` + WITH partition_info AS ( + SELECT + tablename, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + pg_total_relation_size('public.'||tablename) AS size_bytes + FROM pg_tables + WHERE schemaname = 'public' + AND tablename LIKE @table_pattern + ) + SELECT + COUNT(*)::INT AS total_partitions, + SUM(size_bytes)::BIGINT AS total_size_bytes, + pg_size_pretty(SUM(size_bytes)) AS total_size_pretty, + MIN(partition_date) AS oldest_date, + MAX(partition_date) AS newest_date, + AVG(size_bytes)::BIGINT AS avg_size_bytes, + pg_size_pretty(AVG(size_bytes)::BIGINT) AS avg_size_pretty + FROM partition_info + ` + + tablePattern := tableName + "_20%" + result := dbc.DB.Raw(query, sql.Named("table_pattern", tablePattern)).Scan(&stats) + if result.Error != nil { + log.WithError(result.Error).WithField("table", tableName).Error("failed to get partition statistics") + return nil, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "total_partitions": stats.TotalPartitions, + "total_size": stats.TotalSizePretty, + "elapsed": elapsed, + }).Info("retrieved partition statistics") + + return &stats, nil +} + +// GetPartitionsForRemoval identifies partitions older than the retention period for a given table +// This is a read-only operation (dry-run) that shows what would be removed (deleted or detached) +func GetPartitionsForRemoval(dbc *db.DB, tableName string, retentionDays int) ([]PartitionInfo, error) { + start := time.Now() + var partitions []PartitionInfo + + cutoffDate := time.Now().AddDate(0, 0, -retentionDays) + + query := ` + SELECT + tablename, + 'public' as schemaname, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + (CURRENT_DATE - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD'))::INT AS age_days, + pg_total_relation_size('public.'||tablename) AS size_bytes, + pg_size_pretty(pg_total_relation_size('public.'||tablename)) AS size_pretty, + COALESCE(n_live_tup, 0) AS row_estimate + FROM pg_tables + LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename + AND pg_stat_user_tables.schemaname = pg_tables.schemaname + WHERE pg_tables.schemaname = 'public' + AND pg_tables.tablename LIKE @table_pattern + AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date + ORDER BY partition_date ASC + ` + + tablePattern := tableName + "_20%" + result := dbc.DB.Raw(query, + sql.Named("table_pattern", tablePattern), + sql.Named("cutoff_date", cutoffDate)).Scan(&partitions) + if result.Error != nil { + log.WithError(result.Error).WithField("table", tableName).Error("failed to get partitions for removal") + return nil, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "retention_days": retentionDays, + "cutoff_date": cutoffDate.Format("2006-01-02"), + "count": len(partitions), + "elapsed": elapsed, + }).Info("identified partitions for removal") + + return partitions, nil +} + +// GetRetentionSummary provides a summary of what would be affected by a retention policy for a given table +func GetRetentionSummary(dbc *db.DB, tableName string, retentionDays int) (*RetentionSummary, error) { + start := time.Now() + + cutoffDate := time.Now().AddDate(0, 0, -retentionDays) + + var summary RetentionSummary + summary.RetentionDays = retentionDays + summary.CutoffDate = cutoffDate + + query := ` + SELECT + COUNT(*)::INT AS partitions_to_remove, + COALESCE(SUM(pg_total_relation_size('public.'||tablename)), 0)::BIGINT AS storage_to_reclaim, + COALESCE(pg_size_pretty(SUM(pg_total_relation_size('public.'||tablename))), '0 bytes') AS storage_pretty, + MIN(tablename) AS oldest_partition, + MAX(tablename) AS newest_partition + FROM pg_tables + WHERE schemaname = 'public' + AND tablename LIKE @table_pattern + AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date + ` + + tablePattern := tableName + "_20%" + result := dbc.DB.Raw(query, + sql.Named("table_pattern", tablePattern), + sql.Named("cutoff_date", cutoffDate)).Scan(&summary) + if result.Error != nil { + log.WithError(result.Error).WithField("table", tableName).Error("failed to get retention summary") + return nil, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "retention_days": retentionDays, + "partitions_to_remove": summary.PartitionsToRemove, + "storage_to_reclaim": summary.StoragePretty, + "elapsed": elapsed, + }).Info("calculated retention summary") + + return &summary, nil +} + +// GetPartitionsByAgeGroup returns partition counts and sizes grouped by age buckets for a given table +func GetPartitionsByAgeGroup(dbc *db.DB, tableName string) ([]map[string]interface{}, error) { + start := time.Now() + + query := ` + WITH partition_ages AS ( + SELECT + tablename, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + (CURRENT_DATE - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD'))::INT AS age_days, + pg_total_relation_size('public.'||tablename) AS size_bytes + FROM pg_tables + WHERE schemaname = 'public' + AND tablename LIKE @table_pattern + ) + SELECT + CASE + WHEN age_days < 0 THEN 'Future' + WHEN age_days < 30 THEN '0-30 days' + WHEN age_days < 90 THEN '30-90 days' + WHEN age_days < 180 THEN '90-180 days' + WHEN age_days < 365 THEN '180-365 days' + ELSE '365+ days' + END AS age_bucket, + COUNT(*)::INT AS partition_count, + SUM(size_bytes)::BIGINT AS total_size_bytes, + pg_size_pretty(SUM(size_bytes)) AS total_size, + ROUND(SUM(size_bytes) * 100.0 / SUM(SUM(size_bytes)) OVER (), 2) AS percentage + FROM partition_ages + GROUP BY age_bucket + ORDER BY MIN(age_days) + ` + + tablePattern := tableName + "_20%" + var results []map[string]interface{} + err := dbc.DB.Raw(query, sql.Named("table_pattern", tablePattern)).Scan(&results).Error + if err != nil { + log.WithError(err).WithField("table", tableName).Error("failed to get partitions by age group") + return nil, err + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "groups": len(results), + "elapsed": elapsed, + }).Info("retrieved partitions by age group") + + return results, nil +} + +// GetPartitionsByMonth returns partition counts and sizes grouped by month for a given table +func GetPartitionsByMonth(dbc *db.DB, tableName string) ([]map[string]interface{}, error) { + start := time.Now() + + query := ` + SELECT + DATE_TRUNC('month', TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD')) AS month, + COUNT(*)::INT AS partition_count, + pg_size_pretty(SUM(pg_total_relation_size('public.'||tablename))) AS total_size, + pg_size_pretty(AVG(pg_total_relation_size('public.'||tablename))::BIGINT) AS avg_partition_size, + MIN(tablename) AS first_partition, + MAX(tablename) AS last_partition + FROM pg_tables + WHERE schemaname = 'public' + AND tablename LIKE @table_pattern + GROUP BY DATE_TRUNC('month', TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD')) + ORDER BY month DESC + ` + + tablePattern := tableName + "_20%" + var results []map[string]interface{} + err := dbc.DB.Raw(query, sql.Named("table_pattern", tablePattern)).Scan(&results).Error + if err != nil { + log.WithError(err).WithField("table", tableName).Error("failed to get partitions by month") + return nil, err + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "months": len(results), + "elapsed": elapsed, + }).Info("retrieved partitions by month") + + return results, nil +} + +// ValidateRetentionPolicy checks if a retention policy would be safe to apply for a given table +// Returns an error if the policy would delete critical data or too much data +// Only considers attached partitions when validating thresholds +func ValidateRetentionPolicy(dbc *db.DB, tableName string, retentionDays int) error { + // Minimum retention is 90 days + if retentionDays < 90 { + return fmt.Errorf("retention policy too aggressive: minimum 90 days required, got %d", retentionDays) + } + + summary, err := GetRetentionSummary(dbc, tableName, retentionDays) + if err != nil { + return fmt.Errorf("failed to get retention summary: %w", err) + } + + // Get stats for attached partitions only (detached partitions are not considered) + stats, err := GetAttachedPartitionStats(dbc, tableName) + if err != nil { + return fmt.Errorf("failed to get attached partition stats: %w", err) + } + + // Check if we'd delete more than 75% of attached partitions + if stats.TotalPartitions > 0 { + deletePercentage := float64(summary.PartitionsToRemove) / float64(stats.TotalPartitions) * 100 + if deletePercentage > 75 { + return fmt.Errorf("retention policy would delete %.1f%% of attached partitions (%d of %d) - exceeds 75%% safety threshold", + deletePercentage, summary.PartitionsToRemove, stats.TotalPartitions) + } + } + + // Check if we'd delete more than 80% of storage from attached partitions + if stats.TotalSizeBytes > 0 { + deletePercentage := float64(summary.StorageToReclaim) / float64(stats.TotalSizeBytes) * 100 + if deletePercentage > 80 { + return fmt.Errorf("retention policy would delete %.1f%% of attached storage (%s of %s) - exceeds 80%% safety threshold", + deletePercentage, summary.StoragePretty, stats.TotalSizePretty) + } + } + + log.WithFields(log.Fields{ + "table": tableName, + "retention_days": retentionDays, + "partitions_to_remove": summary.PartitionsToRemove, + "attached_partitions": stats.TotalPartitions, + "attached_storage": stats.TotalSizePretty, + "storage_to_reclaim": summary.StoragePretty, + }).Info("retention policy validated") + + return nil +} + +// DropPartition drops a single partition (DESTRUCTIVE - requires write access) +// This is a wrapper around DROP TABLE for safety and logging +func DropPartition(dbc *db.DB, partitionName string, dryRun bool) error { + start := time.Now() + + // Extract table name from partition name + tableName, err := extractTableNameFromPartition(partitionName) + if err != nil { + return fmt.Errorf("invalid partition name: %w", err) + } + + // Validate partition name format for safety + if !isValidPartitionName(tableName, partitionName) { + return fmt.Errorf("invalid partition name: %s - must match %s_YYYY_MM_DD", partitionName, tableName) + } + + if dryRun { + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + }).Info("[DRY RUN] would drop partition") + return nil + } + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", partitionName) + result := dbc.DB.Exec(query) + if result.Error != nil { + log.WithError(result.Error).WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + }).Error("failed to drop partition") + return result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + "elapsed": elapsed, + }).Info("dropped partition") + + return nil +} + +// DetachPartition detaches a partition from the parent table (safer alternative to DROP) +// The detached table can be archived or dropped later +func DetachPartition(dbc *db.DB, partitionName string, dryRun bool) error { + start := time.Now() + + // Extract table name from partition name + tableName, err := extractTableNameFromPartition(partitionName) + if err != nil { + return fmt.Errorf("invalid partition name: %w", err) + } + + // Validate partition name format for safety + if !isValidPartitionName(tableName, partitionName) { + return fmt.Errorf("invalid partition name: %s - must match %s_YYYY_MM_DD", partitionName, tableName) + } + + if dryRun { + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + }).Info("[DRY RUN] would detach partition") + return nil + } + + query := fmt.Sprintf("ALTER TABLE %s DETACH PARTITION %s", tableName, partitionName) + result := dbc.DB.Exec(query) + if result.Error != nil { + log.WithError(result.Error).WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + }).Error("failed to detach partition") + return result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + "elapsed": elapsed, + }).Info("detached partition") + + return nil +} + +// DropOldPartitions drops all partitions older than the retention period for a given table +// This is a bulk operation wrapper that calls DropPartition for each old partition +func DropOldPartitions(dbc *db.DB, tableName string, retentionDays int, dryRun bool) (int, error) { + start := time.Now() + + // Validate retention policy first + if err := ValidateRetentionPolicy(dbc, tableName, retentionDays); err != nil { + return 0, fmt.Errorf("retention policy validation failed: %w", err) + } + + partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays) + if err != nil { + return 0, fmt.Errorf("failed to get partitions for removal: %w", err) + } + + if len(partitions) == 0 { + log.WithField("table", tableName).Info("no partitions to delete") + return 0, nil + } + + droppedCount := 0 + var totalSize int64 + + for _, partition := range partitions { + if err := DropPartition(dbc, partition.TableName, dryRun); err != nil { + log.WithError(err).WithField("partition", partition.TableName).Error("failed to drop partition") + continue + } + droppedCount++ + totalSize += partition.SizeBytes + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "retention_days": retentionDays, + "total_dropped": droppedCount, + "storage_reclaimed": fmt.Sprintf("%d bytes", totalSize), + "dry_run": dryRun, + "elapsed": elapsed, + }).Info("completed dropping old partitions") + + return droppedCount, nil +} + +// DropOldDetachedPartitions drops detached partitions older than retentionDays (DESTRUCTIVE) +// This removes detached partitions that are no longer needed +// Use this after archiving detached partitions or when you're sure the data is no longer needed +func DropOldDetachedPartitions(dbc *db.DB, tableName string, retentionDays int, dryRun bool) (int, error) { + start := time.Now() + + // Get all detached partitions + detached, err := ListDetachedPartitions(dbc, tableName) + if err != nil { + return 0, fmt.Errorf("failed to list detached partitions: %w", err) + } + + if len(detached) == 0 { + log.WithField("table", tableName).Info("no detached partitions found") + return 0, nil + } + + // Filter by retention period + cutoffDate := time.Now().AddDate(0, 0, -retentionDays) + var toRemove []PartitionInfo + + for _, partition := range detached { + if partition.PartitionDate.Before(cutoffDate) { + toRemove = append(toRemove, partition) + } + } + + if len(toRemove) == 0 { + log.WithFields(log.Fields{ + "table": tableName, + "retention_days": retentionDays, + "cutoff_date": cutoffDate.Format("2006-01-02"), + }).Info("no detached partitions older than retention period") + return 0, nil + } + + // Drop each old detached partition + droppedCount := 0 + var totalSize int64 + + for _, partition := range toRemove { + if err := DropPartition(dbc, partition.TableName, dryRun); err != nil { + log.WithError(err).WithField("partition", partition.TableName).Error("failed to drop detached partition") + continue + } + droppedCount++ + totalSize += partition.SizeBytes + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "retention_days": retentionDays, + "total_dropped": droppedCount, + "storage_reclaimed": fmt.Sprintf("%d bytes", totalSize), + "dry_run": dryRun, + "elapsed": elapsed, + }).Info("completed dropping old detached partitions") + + return droppedCount, nil +} + +// ListDetachedPartitions returns partitions that have been detached from the parent table +// Detached partitions are standalone tables that match the naming pattern but are no longer +// part of the partitioned table hierarchy +func ListDetachedPartitions(dbc *db.DB, tableName string) ([]PartitionInfo, error) { + start := time.Now() + var partitions []PartitionInfo + + query := ` + WITH attached_partitions AS ( + -- Get all currently attached partitions using pg_inherits + SELECT c.relname AS tablename + FROM pg_inherits i + JOIN pg_class c ON i.inhrelid = c.oid + JOIN pg_class p ON i.inhparent = p.oid + WHERE p.relname = @table_name + ) + SELECT + tablename, + 'public' as schemaname, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + (CURRENT_DATE - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD'))::INT AS age_days, + pg_total_relation_size('public.'||tablename) AS size_bytes, + pg_size_pretty(pg_total_relation_size('public.'||tablename)) AS size_pretty, + COALESCE(n_live_tup, 0) AS row_estimate + FROM pg_tables + LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename + AND pg_stat_user_tables.schemaname = pg_tables.schemaname + WHERE pg_tables.schemaname = 'public' + AND pg_tables.tablename LIKE @table_pattern + AND pg_tables.tablename NOT IN (SELECT tablename FROM attached_partitions) + ORDER BY partition_date ASC + ` + + tablePattern := tableName + "_20%" + result := dbc.DB.Raw(query, + sql.Named("table_name", tableName), + sql.Named("table_pattern", tablePattern)).Scan(&partitions) + if result.Error != nil { + log.WithError(result.Error).WithField("table", tableName).Error("failed to list detached partitions") + return nil, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "count": len(partitions), + "elapsed": elapsed, + }).Info("listed detached partitions") + + return partitions, nil +} + +// ListAttachedPartitions returns partitions that are currently attached to the parent table +// These are partitions that are part of the active partitioned table hierarchy +func ListAttachedPartitions(dbc *db.DB, tableName string) ([]PartitionInfo, error) { + start := time.Now() + var partitions []PartitionInfo + + query := ` + WITH attached_partitions AS ( + -- Get all currently attached partitions using pg_inherits + SELECT c.relname AS tablename + FROM pg_inherits i + JOIN pg_class c ON i.inhrelid = c.oid + JOIN pg_class p ON i.inhparent = p.oid + WHERE p.relname = @table_name + ) + SELECT + tablename, + 'public' as schemaname, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + (CURRENT_DATE - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD'))::INT AS age_days, + pg_total_relation_size('public.'||tablename) AS size_bytes, + pg_size_pretty(pg_total_relation_size('public.'||tablename)) AS size_pretty, + COALESCE(n_live_tup, 0) AS row_estimate + FROM pg_tables + LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename + AND pg_stat_user_tables.schemaname = pg_tables.schemaname + WHERE pg_tables.schemaname = 'public' + AND pg_tables.tablename IN (SELECT tablename FROM attached_partitions) + ORDER BY partition_date ASC + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&partitions) + if result.Error != nil { + log.WithError(result.Error).WithField("table", tableName).Error("failed to list attached partitions") + return nil, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "count": len(partitions), + "elapsed": elapsed, + }).Info("listed attached partitions") + + return partitions, nil +} + +// GetAttachedPartitionStats returns statistics about attached partitions for a given table +func GetAttachedPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, error) { + start := time.Now() + var stats PartitionStats + + query := ` + WITH attached_partitions AS ( + SELECT c.relname AS tablename + FROM pg_inherits i + JOIN pg_class c ON i.inhrelid = c.oid + JOIN pg_class p ON i.inhparent = p.oid + WHERE p.relname = @table_name + ), + attached_info AS ( + SELECT + tablename, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + pg_total_relation_size('public.'||tablename) AS size_bytes + FROM pg_tables + WHERE schemaname = 'public' + AND tablename IN (SELECT tablename FROM attached_partitions) + ) + SELECT + COALESCE(COUNT(*), 0)::INT AS total_partitions, + COALESCE(SUM(size_bytes), 0)::BIGINT AS total_size_bytes, + pg_size_pretty(COALESCE(SUM(size_bytes), 0)) AS total_size_pretty, + MIN(partition_date) AS oldest_date, + MAX(partition_date) AS newest_date, + COALESCE(AVG(size_bytes), 0)::BIGINT AS avg_size_bytes, + pg_size_pretty(COALESCE(AVG(size_bytes), 0)::BIGINT) AS avg_size_pretty + FROM attached_info + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&stats) + if result.Error != nil { + log.WithError(result.Error).WithField("table", tableName).Error("failed to get attached partition statistics") + return nil, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "total_partitions": stats.TotalPartitions, + "total_size": stats.TotalSizePretty, + "elapsed": elapsed, + }).Info("retrieved attached partition statistics") + + return &stats, nil +} + +// GetDetachedPartitionStats returns statistics about detached partitions for a given table +func GetDetachedPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, error) { + start := time.Now() + var stats PartitionStats + + query := ` + WITH attached_partitions AS ( + SELECT c.relname AS tablename + FROM pg_inherits i + JOIN pg_class c ON i.inhrelid = c.oid + JOIN pg_class p ON i.inhparent = p.oid + WHERE p.relname = @table_name + ), + detached_info AS ( + SELECT + tablename, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + pg_total_relation_size('public.'||tablename) AS size_bytes + FROM pg_tables + WHERE schemaname = 'public' + AND tablename LIKE @table_pattern + AND tablename NOT IN (SELECT tablename FROM attached_partitions) + ) + SELECT + COUNT(*)::INT AS total_partitions, + COALESCE(SUM(size_bytes), 0)::BIGINT AS total_size_bytes, + COALESCE(pg_size_pretty(SUM(size_bytes)), '0 bytes') AS total_size_pretty, + MIN(partition_date) AS oldest_date, + MAX(partition_date) AS newest_date, + COALESCE(AVG(size_bytes), 0)::BIGINT AS avg_size_bytes, + COALESCE(pg_size_pretty(AVG(size_bytes)::BIGINT), '0 bytes') AS avg_size_pretty + FROM detached_info + ` + + tablePattern := tableName + "_20%" + result := dbc.DB.Raw(query, + sql.Named("table_name", tableName), + sql.Named("table_pattern", tablePattern)).Scan(&stats) + if result.Error != nil { + log.WithError(result.Error).WithField("table", tableName).Error("failed to get detached partition statistics") + return nil, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "total_partitions": stats.TotalPartitions, + "total_size": stats.TotalSizePretty, + "elapsed": elapsed, + }).Info("retrieved detached partition statistics") + + return &stats, nil +} + +// ReattachPartition reattaches a previously detached partition back to the parent table +// This is useful if a partition was detached for archival but needs to be restored +func ReattachPartition(dbc *db.DB, partitionName string, dryRun bool) error { + start := time.Now() + + // Extract table name from partition name + tableName, err := extractTableNameFromPartition(partitionName) + if err != nil { + return fmt.Errorf("invalid partition name: %w", err) + } + + // Validate partition name format for safety + if !isValidPartitionName(tableName, partitionName) { + return fmt.Errorf("invalid partition name: %s - must match %s_YYYY_MM_DD", partitionName, tableName) + } + + // Extract date from partition name + prefix := tableName + "_" + dateStr := partitionName[len(prefix):] + partitionDate, err := time.Parse("2006_01_02", dateStr) + if err != nil { + return fmt.Errorf("invalid partition date format: %w", err) + } + + // Calculate date range for the partition + startDate := partitionDate.Format("2006-01-02") + endDate := partitionDate.AddDate(0, 0, 1).Format("2006-01-02") + + if dryRun { + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + "start_date": startDate, + "end_date": endDate, + }).Info("[DRY RUN] would reattach partition") + return nil + } + + // Reattach the partition with FOR VALUES clause + query := fmt.Sprintf( + "ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM ('%s') TO ('%s')", + tableName, + partitionName, + startDate, + endDate, + ) + + result := dbc.DB.Exec(query) + if result.Error != nil { + log.WithError(result.Error).WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + }).Error("failed to reattach partition") + return result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + "elapsed": elapsed, + }).Info("reattached partition") + + return nil +} + +// IsPartitionAttached checks if a partition is currently attached to the parent table +func IsPartitionAttached(dbc *db.DB, partitionName string) (bool, error) { + start := time.Now() + + // Extract table name from partition name + tableName, err := extractTableNameFromPartition(partitionName) + if err != nil { + return false, fmt.Errorf("invalid partition name: %w", err) + } + + // Validate partition name format for safety + if !isValidPartitionName(tableName, partitionName) { + return false, fmt.Errorf("invalid partition name: %s", partitionName) + } + + var isAttached bool + query := ` + SELECT EXISTS( + SELECT 1 + FROM pg_inherits i + JOIN pg_class c ON i.inhrelid = c.oid + JOIN pg_class p ON i.inhparent = p.oid + WHERE p.relname = @table_name + AND c.relname = @partition_name + ) AS is_attached + ` + + result := dbc.DB.Raw(query, + sql.Named("table_name", tableName), + sql.Named("partition_name", partitionName)).Scan(&isAttached) + if result.Error != nil { + log.WithError(result.Error).WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + }).Error("failed to check partition status") + return false, result.Error + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + "is_attached": isAttached, + "elapsed": elapsed, + }).Debug("checked partition attachment status") + + return isAttached, nil +} + +// DetachOldPartitions detaches all partitions older than the retention period for a given table +// This is safer than dropping as partitions can be reattached if needed +func DetachOldPartitions(dbc *db.DB, tableName string, retentionDays int, dryRun bool) (int, error) { + start := time.Now() + + // Validate retention policy first + if err := ValidateRetentionPolicy(dbc, tableName, retentionDays); err != nil { + return 0, fmt.Errorf("retention policy validation failed: %w", err) + } + + partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays) + if err != nil { + return 0, fmt.Errorf("failed to get partitions for removal: %w", err) + } + + if len(partitions) == 0 { + log.WithField("table", tableName).Info("no partitions to detach") + return 0, nil + } + + detachedCount := 0 + var totalSize int64 + + for _, partition := range partitions { + if err := DetachPartition(dbc, partition.TableName, dryRun); err != nil { + log.WithError(err).WithField("partition", partition.TableName).Error("failed to detach partition") + continue + } + detachedCount++ + totalSize += partition.SizeBytes + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "retention_days": retentionDays, + "total_detached": detachedCount, + "storage_affected": fmt.Sprintf("%d bytes", totalSize), + "dry_run": dryRun, + "elapsed": elapsed, + }).Info("completed detaching old partitions") + + return detachedCount, nil +} + +// extractTableNameFromPartition extracts the table name from a partition name +// Partition format: {tablename}_YYYY_MM_DD +func extractTableNameFromPartition(partitionName string) (string, error) { + // Must end with _YYYY_MM_DD (10 characters + 1 underscore = 11) + if len(partitionName) < 12 { + return "", fmt.Errorf("partition name too short: %s", partitionName) + } + + // Extract the date portion (last 10 characters should be YYYY_MM_DD) + dateStr := partitionName[len(partitionName)-10:] + + // Validate date format + _, err := time.Parse("2006_01_02", dateStr) + if err != nil { + return "", fmt.Errorf("invalid date format in partition name: %s", partitionName) + } + + // Table name is everything except the last 11 characters (_YYYY_MM_DD) + tableName := partitionName[:len(partitionName)-11] + + return tableName, nil +} + +// isValidPartitionName validates that a partition name matches the expected format for a given table +// This is a safety check to prevent SQL injection and accidental drops +func isValidPartitionName(tableName, partitionName string) bool { + expectedPrefix := tableName + "_" + expectedLen := len(expectedPrefix) + 10 // prefix + "YYYY_MM_DD" + + if len(partitionName) != expectedLen { + return false + } + + if !strings.HasPrefix(partitionName, expectedPrefix) { + return false + } + + // Must start with 20xx (year 2000-2099) + if len(partitionName) < len(expectedPrefix)+2 || partitionName[len(expectedPrefix):len(expectedPrefix)+2] != "20" { + return false + } + + // Validate date format by parsing + dateStr := partitionName[len(expectedPrefix):] // YYYY_MM_DD format + _, err := time.Parse("2006_01_02", dateStr) + return err == nil +} diff --git a/pkg/db/partitions/partitions_test.go b/pkg/db/partitions/partitions_test.go new file mode 100644 index 000000000..6d5bada34 --- /dev/null +++ b/pkg/db/partitions/partitions_test.go @@ -0,0 +1,97 @@ +package partitions + +import ( + "testing" + "time" +) + +func TestIsValidTestAnalysisPartitionName(t *testing.T) { + tests := []struct { + name string + partition string + want bool + }{ + { + name: "valid partition name", + partition: "test_analysis_by_job_by_dates_2024_10_29", + want: true, + }, + { + name: "valid partition name 2026", + partition: "test_analysis_by_job_by_dates_2026_01_15", + want: true, + }, + { + name: "invalid - too short", + partition: "test_analysis_by_job_by_dates", + want: false, + }, + { + name: "invalid - wrong prefix", + partition: "wrong_analysis_by_job_by_dates_2024_10_29", + want: false, + }, + { + name: "invalid - wrong date format", + partition: "test_analysis_by_job_by_dates_2024_13_40", + want: false, + }, + { + name: "invalid - SQL injection attempt", + partition: "test_analysis_by_job_by_dates_2024_10_29; DROP TABLE prow_jobs;", + want: false, + }, + { + name: "invalid - missing date", + partition: "test_analysis_by_job_by_dates_", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isValidPartitionName("test_analysis_by_job_by_dates", tt.partition) + if got != tt.want { + t.Errorf("isValidTestAnalysisPartitionName(%q) = %v, want %v", tt.partition, got, tt.want) + } + }) + } +} + +func TestPartitionInfo(t *testing.T) { + // Test that PartitionInfo struct can be instantiated + partition := PartitionInfo{ + TableName: "test_analysis_by_job_by_dates_2024_10_29", + SchemaName: "public", + PartitionDate: time.Date(2024, 10, 29, 0, 0, 0, 0, time.UTC), + Age: 100, + SizeBytes: 1073741824, // 1 GB + SizePretty: "1 GB", + RowEstimate: 1000000, + } + + if partition.TableName != "test_analysis_by_job_by_dates_2024_10_29" { + t.Errorf("unexpected table name: %s", partition.TableName) + } +} + +func TestRetentionSummary(t *testing.T) { + // Test that RetentionSummary struct can be instantiated + summary := RetentionSummary{ + RetentionDays: 180, + CutoffDate: time.Now().AddDate(0, 0, -180), + PartitionsToRemove: 50, + StorageToReclaim: 53687091200, // ~50 GB + StoragePretty: "50 GB", + OldestPartition: "test_analysis_by_job_by_dates_2024_10_29", + NewestPartition: "test_analysis_by_job_by_dates_2024_12_17", + } + + if summary.RetentionDays != 180 { + t.Errorf("unexpected retention days: %d", summary.RetentionDays) + } + + if summary.PartitionsToRemove != 50 { + t.Errorf("unexpected partitions to remove: %d", summary.PartitionsToRemove) + } +} From 7a6fd7d78b9578f8b353d64c7e30a20ad2611bba Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Tue, 17 Feb 2026 20:08:04 -0500 Subject: [PATCH 02/11] TRT-1989: update partition management --- pkg/dataloader/prowloader/prow.go | 94 ++- pkg/db/UTILS_README.md | 836 +++++++++++++++++++ pkg/db/partitions/README.md | 711 ++++++++++++++++- pkg/db/partitions/examples.go | 172 +++- pkg/db/partitions/partitions.go | 1101 +++++++++++++++++++++++++- pkg/db/partitions/partitions_test.go | 445 +++++++++++ pkg/db/utils.go | 771 ++++++++++++++++++ pkg/db/utils_example.go | 687 ++++++++++++++++ pkg/db/utils_test.go | 303 +++++++ 9 files changed, 5057 insertions(+), 63 deletions(-) create mode 100644 pkg/db/UTILS_README.md create mode 100644 pkg/db/utils.go create mode 100644 pkg/db/utils_example.go create mode 100644 pkg/db/utils_test.go diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index 21be20d4d..1bd44eec3 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -11,7 +11,6 @@ import ( "reflect" "regexp" "strconv" - "strings" "sync" "sync/atomic" "time" @@ -22,6 +21,7 @@ import ( "github.com/jackc/pgtype" "github.com/lib/pq" "github.com/openshift/sippy/pkg/bigquery/bqlabel" + "github.com/openshift/sippy/pkg/db/partitions" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -235,9 +235,21 @@ func (pl *ProwLoader) Load() { pl.errors = append(pl.errors, err) } + // detach and drop older partitions + err := pl.agePartitionsForDailyTestAnalysisByJob() + if err != nil { + pl.errors = append(pl.errors, errors.Wrap(err, "error aging daily test analysis by job table")) + } + + // create any partitions needed for new data to be imported + err = pl.preparePartitionsForDailyTestAnalysisByJob() + if err != nil { + pl.errors = append(pl.errors, errors.Wrap(err, "error preparing partitions for daily test analysis by job table")) + } + // load the test analysis by job data into tables partitioned by day, letting bigquery do the // heavy lifting for us. - err := pl.loadDailyTestAnalysisByJob(pl.ctx) + err = pl.loadDailyTestAnalysisByJob(pl.ctx) if err != nil { pl.errors = append(pl.errors, errors.Wrap(err, "error updating daily test analysis by job")) } @@ -346,6 +358,54 @@ func NextDay(dateStr string) (string, error) { return nextDay.Format("2006-01-02"), nil } +func (pl *ProwLoader) agePartitionsForDailyTestAnalysisByJob() error { + tableName := "test_analysis_by_job_by_dates" + detached, err := partitions.DetachOldPartitions(pl.dbc, tableName, 90, false) + if err != nil { + log.WithError(err).Errorf("error detaching partitions for %s", tableName) + } else { + log.Infof("detached %d partitions from %s", detached, tableName) + } + dropped, err := partitions.DropOldDetachedPartitions(pl.dbc, tableName, 100, false) + if err != nil { + log.WithError(err).Errorf("error detaching partitions for %s", tableName) + return err + } + log.Infof("dropped %d detached partitions from %s", dropped, tableName) + + return nil +} + +func (pl *ProwLoader) preparePartitionsForDailyTestAnalysisByJob() error { + tableName := "test_analysis_by_job_by_dates" + + log.Infof("preparing partitions for %s", tableName) + stats, err := partitions.GetAttachedPartitionStats(pl.dbc, tableName) + + if err != nil { + log.WithError(err).Errorf("error detaching partitions for %s", tableName) + return err + } + fmt.Printf(" Total: %d partitions (%s)\n", stats.TotalPartitions, stats.TotalSizePretty) + // when we initialize a new table the query goes back 14 days, go back one more + mostRecentDate := time.Now().Add(-15 * 24 * time.Hour) + if stats.TotalPartitions > 0 { + fmt.Printf(" Range: %s to %s\n", + stats.OldestDate.Format("2006-01-02"), + stats.NewestDate.Format("2006-01-02")) + mostRecentDate = stats.NewestDate + } + + created, err := partitions.CreateMissingPartitions(pl.dbc, tableName, mostRecentDate, time.Now().Add(48*time.Hour), false) + if err != nil { + log.WithError(err).Errorf("error creating partitions for %s", tableName) + return err + } + + log.Infof("created %d partitions for %s", created, tableName) + return nil +} + // loadDailyTestAnalysisByJob loads test analysis data into partitioned tables in postgres, one per // day. The data is calculated by querying bigquery to do the heavy lifting for us. Each day is committed // transactionally so the process is safe to interrupt and resume later. The process takes about 20 minutes @@ -382,21 +442,24 @@ func (pl *ProwLoader) loadDailyTestAnalysisByJob(ctx context.Context) error { dLog := log.WithField("date", dateToImport) dLog.Infof("Loading test analysis by job daily summaries") - nextDay, err := NextDay(dateToImport) - if err != nil { - return errors.Wrapf(err, "error parsing next day from %s", dateToImport) - } - // create a partition for this date - partitionSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS test_analysis_by_job_by_dates_%s PARTITION OF test_analysis_by_job_by_dates - FOR VALUES FROM ('%s') TO ('%s');`, strings.ReplaceAll(dateToImport, "-", "_"), dateToImport, nextDay) - dLog.Info(partitionSQL) + // partition creation now managed by preparePartitionsForDailyTestAnalysisByJob - if res := pl.dbc.DB.Exec(partitionSQL); res.Error != nil { - log.WithError(res.Error).Error("error creating partition") - return res.Error - } - dLog.Warnf("partition created for releases %v", pl.releases) + //nextDay, err := NextDay(dateToImport) + //if err != nil { + // return errors.Wrapf(err, "error parsing next day from %s", dateToImport) + //} + + // create a partition for this date + //partitionSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS test_analysis_by_job_by_dates_%s PARTITION OF test_analysis_by_job_by_dates + // FOR VALUES FROM ('%s') TO ('%s');`, strings.ReplaceAll(dateToImport, "-", "_"), dateToImport, nextDay) + //dLog.Info(partitionSQL) + // + //if res := pl.dbc.DB.Exec(partitionSQL); res.Error != nil { + // log.WithError(res.Error).Error("error creating partition") + // return res.Error + //} + //dLog.Warnf("partition created for releases %v", pl.releases) q := pl.bigQueryClient.Query(ctx, bqlabel.ProwLoaderTestAnalysis, fmt.Sprintf(`WITH deduped_testcases AS ( @@ -1241,6 +1304,7 @@ func (pl *ProwLoader) extractTestCases(suite *junit.TestSuite, suiteID *uint, te continue } + // interesting that we rely on created_at here which is when we imported the test, not when the test ran testCases[testCacheKey] = &models.ProwJobRunTest{ TestID: testID, SuiteID: suiteID, diff --git a/pkg/db/UTILS_README.md b/pkg/db/UTILS_README.md new file mode 100644 index 000000000..df3911ef6 --- /dev/null +++ b/pkg/db/UTILS_README.md @@ -0,0 +1,836 @@ +# Database Utilities + +This package provides utility functions for database operations including schema verification and data migration. + +## Overview + +The utilities in `utils.go` provide safe, validated operations for working with database tables, particularly useful for: +- Schema migration and validation +- Data migration between tables +- Partition management workflows +- Table consolidation and archival + +## Functions + +### VerifyTablesHaveSameColumns + +Verifies that two tables have identical column definitions with configurable verification options. + +```go +// Full verification (default) - checks all aspects +err := dbc.VerifyTablesHaveSameColumns("source_table", "target_table", DefaultColumnVerificationOptions()) +if err != nil { + log.WithError(err).Error("tables have different schemas") +} + +// Data migration verification - only checks names and types +err := dbc.VerifyTablesHaveSameColumns("source_table", "target_table", DataMigrationColumnVerificationOptions()) +if err != nil { + log.WithError(err).Error("incompatible schemas for migration") +} +``` + +**Verification Options:** + +| Option | DefaultColumnVerificationOptions | DataMigrationColumnVerificationOptions | +|--------|----------------------------------|---------------------------------------| +| Column names | ✓ | ✓ | +| Data types | ✓ | ✓ | +| NOT NULL constraints | ✓ | ✗ | +| DEFAULT values | ✓ | ✗ | +| Column ordering | ✓ | ✓ | + +**Custom Options:** +```go +opts := ColumnVerificationOptions{ + CheckNullable: true, // Verify NOT NULL constraints match + CheckDefaults: false, // Skip default value comparison + CheckOrder: true, // Verify column order matches +} +err := dbc.VerifyTablesHaveSameColumns("table1", "table2", opts) +``` + +**Always Compared:** +- Column names (always required to match) +- Data types (with normalization, always required to match) + +**Optionally Compared:** +- NOT NULL constraints (controlled by `CheckNullable`) +- DEFAULT values (controlled by `CheckDefaults`) +- Column ordering/position (controlled by `CheckOrder`) + +**Returns:** +- `nil` if tables have compatible schemas +- Error with detailed description of differences + +**Use Cases:** +- **Full verification** (`DefaultColumnVerificationOptions`): + - Verifying partitions match parent table exactly + - Ensuring replicas have identical structures + - Validating table clones or backups + +- **Migration verification** (`DataMigrationColumnVerificationOptions`): + - Pre-migration schema validation + - Verifying data can be copied between tables + - Checking compatibility for INSERT INTO ... SELECT operations + +**Why Skip Nullable/Defaults for Migrations?** + +When migrating data with `INSERT INTO target SELECT * FROM source`, PostgreSQL only requires that: +- Column names exist in both tables +- Data types are compatible + +Nullable constraints and default values don't affect the data copy itself, so checking them is optional for migrations. + +--- + +### MigrateTableData + +Migrates all data from one table to another after verifying schemas match. + +```go +// Dry run first +rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", true) + +// Actual migration +rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", false) +if err != nil { + log.WithError(err).Error("migration failed") +} +``` + +**Process:** +1. Verifies schemas match using `VerifyTablesHaveSameColumns` +2. Checks row counts in both tables +3. Performs `INSERT INTO target SELECT * FROM source` +4. Verifies row counts after migration +5. Logs all steps with detailed metrics + +**Parameters:** +- `sourceTable` - Table to copy data from +- `targetTable` - Table to copy data to +- `dryRun` - If true, only verifies without copying data + +**Returns:** +- `rowsMigrated` - Number of rows successfully migrated (0 if dry run) +- `error` - Any error encountered during migration + +**Features:** +- Atomic operation (single INSERT statement) +- Dry-run support for safety +- Pre and post verification +- Comprehensive logging +- Handles empty source tables gracefully + +**Safety:** +- DOES NOT truncate target table (appends data) +- DOES NOT drop source table +- Fails fast if schemas don't match +- Warns on row count mismatches + +**Use Cases:** +- Migrating detached partitions to archive tables +- Consolidating multiple tables into one +- Moving data between environments +- Table restructuring workflows + +--- + +### MigrateTableDataRange + +Migrates data within a specific date range from one table to another after verifying schemas match. + +```go +startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) + +// Dry run first +rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, true) + +// Actual migration +rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, false) +if err != nil { + log.WithError(err).Error("migration failed") +} +``` + +**Process:** +1. Validates date range (endDate must be after startDate) +2. Verifies schemas match using `VerifyTablesHaveSameColumns` +3. Checks if target table is RANGE partitioned and verifies all necessary partitions exist for the date range +4. Counts rows in source table within date range +5. Performs `INSERT INTO target SELECT * FROM source WHERE date_column >= start AND date_column < end` +6. Verifies row counts after migration +7. Logs all steps with detailed metrics + +**Parameters:** +- `sourceTable` - Table to copy data from +- `targetTable` - Table to copy data to +- `dateColumn` - Column name to filter by date (e.g., "created_at") +- `startDate` - Start of date range (inclusive, >=) +- `endDate` - End of date range (exclusive, <) +- `dryRun` - If true, only verifies without copying data + +**Returns:** +- `rowsMigrated` - Number of rows successfully migrated (0 if dry run) +- `error` - Any error encountered during migration + +**Features:** +- Atomic operation (single INSERT statement) +- Dry-run support for safety +- Pre and post verification +- Comprehensive logging +- Handles empty date ranges gracefully +- Date range validation +- Automatic partition coverage verification for RANGE partitioned tables +- Prevents migration failures due to missing partitions + +**Safety:** +- DOES NOT truncate target table (appends data) +- DOES NOT drop source table +- Fails fast if schemas don't match +- Warns on row count mismatches +- Validates date range before execution + +**Use Cases:** +- Migrating large tables incrementally (month by month, year by year) +- Testing migrations with a subset of data before full migration +- Moving specific time periods to archive tables +- Backfilling historical data into partitioned tables +- Reducing lock contention by migrating in smaller batches +- Being able to pause and resume large migrations + +**Example - Incremental Monthly Migration:** +```go +// Migrate data month by month for 2024 +for month := 1; month <= 12; month++ { + startDate := time.Date(2024, time.Month(month), 1, 0, 0, 0, 0, time.UTC) + endDate := startDate.AddDate(0, 1, 0) + + rows, err := dbc.MigrateTableDataRange("orders", "orders_new", "order_date", startDate, endDate, false) + if err != nil { + log.WithError(err).WithField("month", month).Error("failed") + continue + } + log.WithField("rows", rows).Info("month migrated") +} +``` + +--- + +### GetTableRowCount + +Returns the number of rows in a table. + +```go +count, err := dbc.GetTableRowCount("table_name") +if err != nil { + log.WithError(err).Error("failed to get row count") +} +log.WithField("count", count).Info("table row count") +``` + +**Use Cases:** +- Pre-migration verification +- Monitoring table growth +- Validating migration success +- Capacity planning + +--- + +### SyncIdentityColumn + +Synchronizes the IDENTITY sequence for a column to match the current maximum value in the table. + +```go +err := dbc.SyncIdentityColumn("table_name", "id") +if err != nil { + log.WithError(err).Error("failed to sync identity column") +} +``` + +**How It Works**: +1. Queries the current maximum value of the column: `SELECT MAX(column) FROM table` +2. Calculates the next value (max + 1, or 1 if table is empty/all NULL) +3. Executes `ALTER TABLE table_name ALTER COLUMN column_name RESTART WITH next_value` +4. Logs the operation with the new sequence value + +**Returns**: Error if the operation fails + +**Use Cases**: +- After migrating data to a partitioned table with IDENTITY columns +- After bulk inserting data with explicit ID values +- When the IDENTITY sequence is out of sync with actual data +- After using `MigrateTableData` to copy data between tables + +**Example Workflow**: +```go +// Migrate data from old table to new partitioned table +rows, err := dbc.MigrateTableData("old_table", "new_partitioned_table", false) +if err != nil { + log.Fatal(err) +} + +// Sync the IDENTITY sequence so new inserts start at the correct value +err = dbc.SyncIdentityColumn("new_partitioned_table", "id") +if err != nil { + log.Fatal(err) +} + +log.Info("Migration complete - sequence synchronized") +``` + +**Important Notes**: +- The column must be an IDENTITY column (created with `GENERATED BY DEFAULT AS IDENTITY`) +- This does NOT work with traditional PostgreSQL sequences created separately +- For traditional sequences, use: `SELECT setval('sequence_name', (SELECT MAX(id) FROM table))` +- Safe to run multiple times - idempotent operation + +--- + +### GetPartitionStrategy + +Checks if a table is partitioned and returns its partition strategy. + +```go +strategy, err := dbc.GetPartitionStrategy("table_name") +if err != nil { + log.WithError(err).Error("failed to check partition strategy") +} + +if strategy == "" { + log.Info("table is not partitioned") +} else if strategy == db.PartitionStrategyRange { + log.Info("table uses RANGE partitioning") +} +``` + +**Returns**: +- Empty string `""` if table is not partitioned +- `PartitionStrategyRange`, `PartitionStrategyList`, `PartitionStrategyHash`, or `"UNKNOWN"` if partitioned + +**Constants**: +```go +db.PartitionStrategyRange // "RANGE" +db.PartitionStrategyList // "LIST" +db.PartitionStrategyHash // "HASH" +``` + +**Use Cases**: +- Before migrations, check if target table is partitioned +- Determine which partition management operations are applicable +- Validate table structure before data operations + +**Example**: +```go +strategy, err := dbc.GetPartitionStrategy("orders") +if err != nil { + log.Fatal(err) +} + +switch strategy { +case db.PartitionStrategyRange: + log.Info("table uses RANGE partitioning") +case db.PartitionStrategyList: + log.Info("table uses LIST partitioning") +case db.PartitionStrategyHash: + log.Info("table uses HASH partitioning") +case "": + log.Info("table is not partitioned") +} +``` + +--- + +### VerifyPartitionCoverage + +Verifies that all necessary partitions exist for a date range. + +```go +startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) + +err := dbc.VerifyPartitionCoverage("orders", startDate, endDate) +if err != nil { + // Prints: missing partitions for dates: [2024-01-15 2024-01-16] + log.WithError(err).Error("partition coverage check failed") +} +``` + +**How It Works**: +1. Queries all existing partitions for the table +2. Checks that a partition exists for each day in the range [startDate, endDate) +3. Returns error listing all missing partition dates +4. Logs successful verification with partition count + +**Assumptions**: +- Daily partitions with naming convention: `tablename_YYYY_MM_DD` +- Partitions are created for each calendar day +- Date range uses same convention as other functions (startDate inclusive, endDate exclusive) + +**Returns**: Error if any partitions are missing, nil if all exist + +**Use Cases**: +- Before migrating data to partitioned tables +- Verifying partition creation scripts completed successfully +- Pre-flight checks before bulk data operations +- Automated partition management validation + +**Example - Create missing partitions**: +```go +import "github.com/openshift/sippy/pkg/db/partitions" + +// Check if partitions exist +err := dbc.VerifyPartitionCoverage("orders", startDate, endDate) +if err != nil { + log.WithError(err).Warn("missing partitions - creating them") + + // Create missing partitions using partitions package + count, err := partitions.CreateMissingPartitions(dbc, "orders", startDate, endDate, false) + if err != nil { + log.Fatal(err) + } + log.WithField("created", count).Info("created missing partitions") +} + +// Now verify again +if err := dbc.VerifyPartitionCoverage("orders", startDate, endDate); err != nil { + log.Fatal("still missing partitions after creation") +} +``` + +--- + +## Helper Types + +### ColumnInfo + +Represents metadata about a database column. + +```go +type ColumnInfo struct { + ColumnName string + DataType string + IsNullable string + ColumnDefault sql.NullString + OrdinalPos int +} +``` + +--- + +### PartitionStrategy + +Defines the partitioning strategy type for PostgreSQL partitioned tables. + +```go +type PartitionStrategy string + +const ( + PartitionStrategyRange PartitionStrategy = "RANGE" + PartitionStrategyList PartitionStrategy = "LIST" + PartitionStrategyHash PartitionStrategy = "HASH" +) +``` + +**Usage**: +- Returned by `GetPartitionStrategy()` to indicate table's partitioning type +- Used by the `partitions` package in `PartitionConfig.Strategy` +- Can be compared directly with constants or used in switch statements + +**Example**: +```go +strategy, err := dbc.GetPartitionStrategy("orders") +if err != nil { + return err +} + +switch strategy { +case PartitionStrategyRange: + // Handle RANGE partitioned table +case PartitionStrategyList: + // Handle LIST partitioned table +case PartitionStrategyHash: + // Handle HASH partitioned table +case "": + // Table is not partitioned +} +``` + +--- + +### ColumnVerificationOptions + +Controls which aspects of column definitions to verify when comparing tables. + +```go +type ColumnVerificationOptions struct { + CheckNullable bool // Verify that columns have matching nullable constraints + CheckDefaults bool // Verify that columns have matching default values + CheckOrder bool // Verify that columns are in the same ordinal position +} +``` + +**Predefined Options:** + +```go +// DefaultColumnVerificationOptions - Full verification (all checks enabled) +opts := DefaultColumnVerificationOptions() +// Returns: ColumnVerificationOptions{CheckNullable: true, CheckDefaults: true, CheckOrder: true} + +// DataMigrationColumnVerificationOptions - Minimal verification for migrations +opts := DataMigrationColumnVerificationOptions() +// Returns: ColumnVerificationOptions{CheckNullable: false, CheckDefaults: false, CheckOrder: true} +``` + +**Usage**: +- Used by `VerifyTablesHaveSameColumns()` to control verification behavior +- Column names and data types are **always** verified regardless of options +- Optional checks allow flexibility for different use cases + +**Example - Custom Options**: +```go +// Custom verification: check types and nullability, skip defaults and order +opts := ColumnVerificationOptions{ + CheckNullable: true, + CheckDefaults: false, + CheckOrder: false, +} +err := dbc.VerifyTablesHaveSameColumns("table1", "table2", opts) +``` + +**When to Use Each Option:** + +| Scenario | Recommended Options | +|----------|-------------------| +| Verifying partition matches parent | `DefaultColumnVerificationOptions()` | +| Pre-migration compatibility check | `DataMigrationColumnVerificationOptions()` | +| Validating table replicas | `DefaultColumnVerificationOptions()` | +| Testing table clones | `DefaultColumnVerificationOptions()` | + +--- + +## Data Type Normalization + +The utilities normalize PostgreSQL data type names for accurate comparison: + +| PostgreSQL Type | Normalized | +|----------------|------------| +| `character varying` | `varchar` | +| `integer`, `int4` | `int` | +| `int8`, `bigserial` | `bigint` | +| `serial` | `int` | +| `timestamp without time zone` | `timestamp` | +| `timestamp with time zone` | `timestamptz` | +| `double precision` | `float8` | +| `boolean` | `bool` | + +This ensures that functionally equivalent types are treated as identical during comparison. + +--- + +## Usage Examples + +### Basic Migration + +```go +// Step 1: Verify schemas match +err := dbc.VerifyTablesHaveSameColumns("source_table", "target_table") +if err != nil { + log.Fatal(err) +} + +// Step 2: Dry run +_, err = dbc.MigrateTableData("source_table", "target_table", true) +if err != nil { + log.Fatal(err) +} + +// Step 3: Actual migration +rows, err := dbc.MigrateTableData("source_table", "target_table", false) +log.WithField("rows", rows).Info("migration completed") +``` + +--- + +### Partition to Archive Migration + +```go +// Migrate detached partition to archive table +partition := "test_analysis_by_job_by_dates_2024_01_15" +archive := "test_analysis_archive" + +rows, err := dbc.MigrateTableData(partition, archive, false) +if err != nil { + log.WithError(err).Error("migration failed") + return +} + +log.WithFields(log.Fields{ + "partition": partition, + "rows": rows, +}).Info("partition migrated to archive - safe to drop") +``` + +--- + +### Batch Migration + +```go +partitions := []string{ + "table_2024_01_15", + "table_2024_01_16", + "table_2024_01_17", +} + +var totalRows int64 +for _, partition := range partitions { + rows, err := dbc.MigrateTableData(partition, "archive_table", false) + if err != nil { + log.WithError(err).WithField("partition", partition).Error("failed") + continue + } + totalRows += rows +} + +log.WithField("total_rows", totalRows).Info("batch migration completed") +``` + +--- + +### Migration with Backup + +```go +// Create backup before migration +_, err := dbc.MigrateTableData("target_table", "backup_table", false) +if err != nil { + log.Fatal("backup failed") +} + +// Perform migration +rows, err := dbc.MigrateTableData("source_table", "target_table", false) +if err != nil { + log.Error("migration failed - restore from backup if needed") + return +} + +log.Info("migration successful - backup can be dropped") +``` + +--- + +### Incremental Migration by Date Range + +```go +// Migrate large table incrementally by month to reduce lock contention +for month := 1; month <= 12; month++ { + startDate := time.Date(2024, time.Month(month), 1, 0, 0, 0, 0, time.UTC) + endDate := startDate.AddDate(0, 1, 0) // First day of next month + + log.WithFields(log.Fields{ + "month": time.Month(month).String(), + "start": startDate.Format("2006-01-02"), + "end": endDate.Format("2006-01-02"), + }).Info("migrating month") + + rows, err := dbc.MigrateTableDataRange("large_table", "large_table_new", "created_at", startDate, endDate, false) + if err != nil { + log.WithError(err).WithField("month", month).Error("migration failed") + continue + } + + log.WithFields(log.Fields{ + "month": month, + "rows": rows, + }).Info("month migrated successfully") +} +``` + +--- + +### Migrate Specific Date Range to Archive + +```go +// Move Q1 2024 data to archive table +startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +endDate := time.Date(2024, 4, 1, 0, 0, 0, 0, time.UTC) + +// Dry run first +_, err := dbc.MigrateTableDataRange("orders", "orders_archive", "order_date", startDate, endDate, true) +if err != nil { + log.Fatal(err) +} + +// Actual migration +rows, err := dbc.MigrateTableDataRange("orders", "orders_archive", "order_date", startDate, endDate, false) +log.WithFields(log.Fields{ + "rows": rows, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), +}).Info("Q1 2024 data archived") +``` + +--- + +## Best Practices + +### Always Use Dry Run First + +```go +// GOOD: Verify before executing +_, err := dbc.MigrateTableData(source, target, true) +if err != nil { + return err +} +rows, err := dbc.MigrateTableData(source, target, false) + +// BAD: Direct migration without verification +rows, err := dbc.MigrateTableData(source, target, false) +``` + +### Verify Schemas Explicitly + +```go +// GOOD: Explicit verification with clear error handling +if err := dbc.VerifyTablesHaveSameColumns(source, target); err != nil { + log.WithError(err).Error("schema mismatch - cannot proceed") + return err +} + +// Migration happens in MigrateTableData, but explicit check is clearer +``` + +### Check Row Counts + +```go +// GOOD: Verify counts before and after +sourceBefore, _ := dbc.GetTableRowCount(source) +targetBefore, _ := dbc.GetTableRowCount(target) + +rows, err := dbc.MigrateTableData(source, target, false) + +targetAfter, _ := dbc.GetTableRowCount(target) +expected := targetBefore + sourceBefore +if targetAfter != expected { + log.Error("row count mismatch!") +} +``` + +### Use Transactions for Multiple Operations + +When performing multiple related operations, use database transactions: + +```go +tx := dbc.DB.Begin() + +// Perform operations +// ... + +if err != nil { + tx.Rollback() + return err +} + +tx.Commit() +``` + +--- + +## Error Handling + +All functions return detailed errors: + +```go +err := dbc.VerifyTablesHaveSameColumns("table1", "table2") +if err != nil { + // Error contains specific differences: + // "column name mismatch: columns in table1 but not in table2: [col1, col2]" + // "column definition mismatches: column foo: type mismatch (table1: int vs table2: bigint)" +} +``` + +Common errors: +- **Schema mismatch**: Tables have different columns or types +- **Table not found**: One or both tables don't exist +- **Permission denied**: Insufficient database privileges +- **Row count mismatch**: Data integrity issue after migration + +--- + +## Testing + +Unit tests cover: +- Data type normalization +- ColumnInfo struct +- Parameter validation + +Run tests: +```bash +go test ./pkg/db -v +``` + +Integration tests require a live database and are in separate test suites. + +--- + +## Logging + +All functions use structured logging with relevant fields: + +```go +log.WithFields(log.Fields{ + "source": sourceTable, + "target": targetTable, + "rows": rowsMigrated, +}).Info("migration completed") +``` + +Log levels: +- **Debug**: Column-level comparisons +- **Info**: Operation start/completion, row counts +- **Warn**: Row count mismatches (non-fatal) +- **Error**: Schema mismatches, migration failures + +--- + +## Integration with Partition Management + +These utilities work seamlessly with the partition management APIs in `pkg/db/partitions`: + +```go +import "github.com/openshift/sippy/pkg/db/partitions" + +// Detach old partitions +detached, _ := partitions.DetachOldPartitions(dbc, "parent_table", 180, false) + +// Migrate detached partitions to archive +for _, partition := range detachedPartitions { + dbc.MigrateTableData(partition.TableName, "archive_table", false) +} + +// Drop old partitions +partitions.DropOldDetachedPartitions(dbc, "parent_table", 180, false) +``` + +--- + +## Performance Considerations + +- **Single INSERT statement**: Migration uses `INSERT INTO ... SELECT` for efficiency +- **No row-by-row operations**: Bulk operation handled by PostgreSQL +- **Network efficiency**: Single round-trip for data transfer +- **Index usage**: PostgreSQL optimizer handles query execution + +For very large tables (millions of rows): +- Consider migrating in batches using WHERE clauses +- Monitor transaction log growth +- Use `ANALYZE` after migration +- Consider `VACUUM` on target table + +--- + +## See Also + +- [Partition Management APIs](./partitions/README.md) - For partition-specific operations +- [Database Schema](../../.claude/db-schema-analysis.md) - For schema documentation +- Examples in `utils_example.go` - For detailed usage patterns diff --git a/pkg/db/partitions/README.md b/pkg/db/partitions/README.md index 1480c3cd5..9c17ffdb7 100644 --- a/pkg/db/partitions/README.md +++ b/pkg/db/partitions/README.md @@ -80,18 +80,28 @@ fmt.Printf("Range: %s to %s\n", Identifies partitions older than the retention period. ```go -partitions, err := partitions.GetPartitionsForRemoval(dbc, 180) +// Get all partitions (attached + detached) older than 180 days +partitions, err := partitions.GetPartitionsForRemoval(dbc, "test_analysis_by_job_by_dates", 180, false) if err != nil { log.WithError(err).Error("failed to get partitions for removal") } fmt.Printf("Found %d partitions older than 180 days\n", len(partitions)) + +// Get only attached partitions older than 180 days +attachedPartitions, err := partitions.GetPartitionsForRemoval(dbc, "test_analysis_by_job_by_dates", 180, true) ``` **Parameters**: +- `tableName` - Name of the partitioned parent table - `retentionDays` - Retention period in days +- `attachedOnly` - If true, only returns attached partitions; if false, returns all partitions -**Returns**: `[]PartitionInfo` for partitions older than retention period (can be deleted or detached) +**Returns**: `[]PartitionInfo` for partitions older than retention period + +**Use When**: +- `attachedOnly = true`: Before detaching partitions (can only detach what's attached) +- `attachedOnly = false`: Before dropping partitions (can drop both attached and detached) --- @@ -99,17 +109,23 @@ fmt.Printf("Found %d partitions older than 180 days\n", len(partitions)) Provides a summary of what would be affected by a retention policy. ```go -summary, err := partitions.GetRetentionSummary(dbc, 180) +// Get summary for all partitions (attached + detached) +summary, err := partitions.GetRetentionSummary(dbc, "test_analysis_by_job_by_dates", 180, false) if err != nil { log.WithError(err).Error("failed to get summary") } fmt.Printf("Would delete %d partitions, reclaiming %s\n", summary.PartitionsToRemove, summary.StoragePretty) + +// Get summary for attached partitions only +attachedSummary, err := partitions.GetRetentionSummary(dbc, "test_analysis_by_job_by_dates", 180, true) ``` **Parameters**: +- `tableName` - Name of the partitioned parent table - `retentionDays` - Retention period in days +- `attachedOnly` - If true, only considers attached partitions; if false, considers all partitions **Returns**: `*RetentionSummary` containing: - `RetentionDays` - Policy retention period @@ -118,6 +134,10 @@ fmt.Printf("Would delete %d partitions, reclaiming %s\n", - `StorageToReclaim` / `StoragePretty` - Storage to be freed - `OldestPartition` / `NewestPartition` - Range of affected partitions +**Use When**: +- `attachedOnly = true`: Before detaching partitions or when validating against active data only +- `attachedOnly = false`: Before dropping partitions or when showing complete impact + --- #### GetPartitionsByAgeGroup @@ -191,6 +211,286 @@ if err != nil { ⚠️ **Warning**: All write operations require database write access. Read-only users will get permission errors. +#### CreatePartitionedTable +Creates a new partitioned table from a GORM model struct with a specified partitioning strategy. + +```go +// Define your model (or use an existing one) +type MyModel struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time `gorm:"index"` + Name string + Data string +} + +// RANGE partitioning (most common - for dates, timestamps) +config := partitions.NewRangePartitionConfig("created_at") + +// Dry run - see the SQL that would be executed +sql, err := partitions.CreatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", config, true) +if err != nil { + log.WithError(err).Error("dry run failed") +} +// Prints the CREATE TABLE statement with PARTITION BY RANGE clause + +// Actual creation +sql, err = partitions.CreatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", config, false) +``` + +**Parameters**: +- `model` - GORM model struct (must be a pointer, e.g., `&models.MyModel{}`) +- `tableName` - Name for the partitioned table +- `config` - Partition configuration (strategy, columns, etc.) +- `dryRun` - If true, prints SQL without executing + +**Partition Strategies**: + +1. **RANGE Partitioning** (for dates, timestamps, sequential values): +```go +config := partitions.NewRangePartitionConfig("created_at") +// Generates: PARTITION BY RANGE (created_at) +``` + +2. **LIST Partitioning** (for discrete categories): +```go +config := partitions.NewListPartitionConfig("region") +// Generates: PARTITION BY LIST (region) +``` + +3. **HASH Partitioning** (for load distribution): +```go +config := partitions.NewHashPartitionConfig(4, "user_id") +// Generates: PARTITION BY HASH (user_id) +// Modulus = 4 means 4 hash partitions will be needed +``` + +**How It Works**: +1. Validates partition configuration +2. Checks if table already exists (returns without error if it does) +3. Parses the GORM model to extract schema information +4. **Converts GORM/Go types to PostgreSQL types** (see Data Type Mapping below) +5. Generates `CREATE TABLE` statement with columns and data types +6. **Adds PRIMARY KEY constraint** (automatically includes partition columns if not already in primary key) +7. Adds `PARTITION BY [RANGE|LIST|HASH] (columns)` clause +8. Creates indexes (skips unique indexes without all partition keys) +9. In dry-run mode, prints SQL; otherwise executes it + +**Data Type Mapping**: +The function automatically converts Go/GORM types to PostgreSQL types: +- `uint`, `uint32`, `uint64`, `int` → `bigint` +- `uint8`, `int8`, `int16` → `smallint` +- `uint16`, `int32` → `integer` +- `int64` → `bigint` +- `float`, `float64` → `double precision` +- `float32` → `real` +- `string` → `text` +- `bool` → `boolean` +- `time.Time` → `timestamp with time zone` +- `[]byte` → `bytea` + +This ensures your GORM models with Go types like `uint` work correctly with PostgreSQL. + +**Important Notes**: +- **Primary keys**: Automatically generated with `PRIMARY KEY (columns)` constraint + - If your model's primary key doesn't include partition columns, they are automatically added + - For example, if you have `ID` as primary key and partition by `created_at`, the constraint will be `PRIMARY KEY (id, created_at)` + - This is a PostgreSQL requirement for partitioned tables +- **Primary key NOT NULL**: Automatically adds NOT NULL to primary key columns +- **Auto-increment fields**: Fields marked with `gorm:"autoIncrement"` are implemented using `GENERATED BY DEFAULT AS IDENTITY` + - IDENTITY columns are automatically NOT NULL (PostgreSQL requirement) + - Supports `autoIncrementIncrement` for custom increment values (e.g., `gorm:"autoIncrement;autoIncrementIncrement:10"` generates `IDENTITY (INCREMENT BY 10)`) + - Example: `ID uint \`gorm:"primaryKey;autoIncrement"\`` generates `id bigint GENERATED BY DEFAULT AS IDENTITY` +- **Column deduplication**: Automatically deduplicates columns to prevent the same column from appearing multiple times + - GORM can include duplicate fields in `stmt.Schema.Fields` (e.g., from embedded structs like `gorm.Model`) + - First occurrence of each column is used, subsequent duplicates are skipped with debug logging +- **Unique indexes**: Must include ALL partition columns (PostgreSQL requirement) +- **After creation**: Create actual partitions based on strategy +- Table creation is a one-time operation (cannot easily modify schema after) +- **Data types**: Automatically converted from Go types to PostgreSQL types + +**Example Models**: + +```go +// Basic model with auto-increment primary key +type MyModel struct { + ID uint `gorm:"primaryKey;autoIncrement"` + Name string `gorm:"not null"` + CreatedAt time.Time `gorm:"index"` +} +// Generated SQL: +// id bigint GENERATED BY DEFAULT AS IDENTITY +// PRIMARY KEY (id, created_at) -- includes partition column + +// Model with custom increment value +type CustomIncrement struct { + ID uint `gorm:"primaryKey;autoIncrement;autoIncrementIncrement:10"` + Data string + CreatedAt time.Time +} +// Generated SQL: +// id bigint GENERATED BY DEFAULT AS IDENTITY (INCREMENT BY 10) +``` + +**Complete Workflows**: + +**RANGE Partitioning (Date-based)**: +```go +// 1. Create the partitioned table structure +config := partitions.NewRangePartitionConfig("created_at") +_, err := partitions.CreatePartitionedTable(dbc, &models.MyModel{}, "my_table", config, false) + +// 2. Create partitions for date range +startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +endDate := time.Now() +created, err := partitions.CreateMissingPartitions(dbc, "my_table", startDate, endDate, false) +``` + +**HASH Partitioning (Load Distribution)**: +```go +// 1. Create the partitioned table structure +config := partitions.NewHashPartitionConfig(4, "user_id") +_, err := partitions.CreatePartitionedTable(dbc, &models.MyModel{}, "my_table", config, false) + +// 2. Create hash partitions manually +for i := 0; i < 4; i++ { + partName := fmt.Sprintf("my_table_%d", i) + sql := fmt.Sprintf("CREATE TABLE %s PARTITION OF my_table FOR VALUES WITH (MODULUS 4, REMAINDER %d)", partName, i) + dbc.DB.Exec(sql) +} +``` + +**LIST Partitioning (Category-based)**: +```go +// 1. Create the partitioned table structure +config := partitions.NewListPartitionConfig("region") +_, err := partitions.CreatePartitionedTable(dbc, &models.MyModel{}, "my_table", config, false) + +// 2. Create list partitions manually +regions := []string{"us-east", "us-west", "eu-central"} +for _, region := range regions { + partName := fmt.Sprintf("my_table_%s", region) + sql := fmt.Sprintf("CREATE TABLE %s PARTITION OF my_table FOR VALUES IN ('%s')", partName, region) + dbc.DB.Exec(sql) +} +``` + +--- + + +#### UpdatePartitionedTable +Updates an existing partitioned table schema to match a GORM model. + +```go +// Define your updated model +type MyModel struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time `gorm:"index"` + Name string + Data string + NewField string `gorm:"index"` // New field added + // OldField removed +} + +// Dry run - see what changes would be made +sql, err := partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", true) +if err != nil { + log.WithError(err).Error("dry run failed") +} +// Prints all ALTER TABLE statements that would be executed + +// Actual update +sql, err = partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", false) +``` + +**Parameters**: +- `model` - GORM model struct with desired schema (must be a pointer, e.g., `&models.MyModel{}`) +- `tableName` - Name of the existing partitioned table +- `dryRun` - If true, prints SQL without executing + +**How It Works**: +1. Checks if the table exists +2. Parses the GORM model to get desired schema +3. Queries database for current schema (columns, indexes, partition keys) +4. Compares schemas and generates ALTER statements for: + - **New columns**: `ALTER TABLE ADD COLUMN` + - **Modified columns**: `ALTER COLUMN TYPE`, `SET/DROP NOT NULL`, `SET/DROP DEFAULT` + - **Removed columns**: `ALTER TABLE DROP COLUMN` + - **New indexes**: `CREATE INDEX` + - **Modified indexes**: `DROP INDEX` + `CREATE INDEX` + - **Removed indexes**: `DROP INDEX` +5. In dry-run mode, prints SQL; otherwise executes it + +**Important Notes**: +- **Cannot change partition keys**: Partition columns cannot be modified after creation +- **Unique indexes**: Must include ALL partition columns (PostgreSQL requirement) +- **Primary key indexes**: Skipped (named `_pkey` by convention) +- **Primary key NOT NULL**: Automatically adds NOT NULL to primary key columns (PostgreSQL requirement) +- **Data types**: Automatically converted from Go types to PostgreSQL types (same as CreatePartitionedTable) +- **Type changes**: Use caution with data type changes that could cause data loss +- **Column removal**: Destructive operation - ensure data is not needed +- Always run dry-run first to preview changes + +**Schema Changes Detected**: + +1. **Column Changes**: + - New columns added with appropriate data type, NOT NULL, and DEFAULT + - Primary key columns automatically get NOT NULL constraint + - Type changes detected through normalized comparison (uses converted PostgreSQL types) + - NULL constraint changes + - DEFAULT value changes + - Removed columns + +2. **Index Changes**: + - New indexes created + - Modified indexes (column list changes) dropped and recreated + - Removed indexes dropped + - Validates unique indexes include partition keys + +**Use When**: +- Your GORM model schema has evolved +- Adding new fields to track additional data +- Modifying column types or constraints +- Adding or removing indexes +- Schema migrations in production + +**Safety Features**: +- Dry-run mode to preview all changes +- Validates unique indexes include partition keys +- Skips primary key indexes (prevents accidental modification) +- Comprehensive logging for each change +- Returns all SQL executed for audit trail + +**Example Workflow**: +```go +// 1. Update your GORM model +type TestResults struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time `gorm:"index"` + TestName string `gorm:"index"` + NewMetric float64 // Added field + // RemovedField deleted +} + +// 2. Dry run to see changes +sql, err := partitions.UpdatePartitionedTable(dbc, &TestResults{}, "test_results", true) +fmt.Println("Would execute:", sql) + +// 3. Review changes, then apply +sql, err = partitions.UpdatePartitionedTable(dbc, &TestResults{}, "test_results", false) +if err != nil { + log.Fatal(err) +} +``` + +**Limitations**: +- Cannot modify partition strategy (RANGE to LIST, etc.) +- Cannot change partition columns +- Cannot split or merge partitions +- Type conversions must be PostgreSQL-compatible +- For major schema changes, consider creating a new table and migrating data + +--- + #### DropPartition Drops a single partition. @@ -367,6 +667,51 @@ err := partitions.ReattachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_ --- +#### CreateMissingPartitions +Creates missing partitions for a date range. + +```go +startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +endDate := time.Date(2024, 1, 31, 0, 0, 0, 0, time.UTC) + +// Dry run - see what would be created +created, err := partitions.CreateMissingPartitions(dbc, "test_analysis_by_job_by_dates", startDate, endDate, true) +fmt.Printf("Would create %d partitions\n", created) + +// Actual creation +created, err = partitions.CreateMissingPartitions(dbc, "test_analysis_by_job_by_dates", startDate, endDate, false) +fmt.Printf("Created %d partitions\n", created) +``` + +**Parameters**: +- `tableName` - Name of the partitioned parent table +- `startDate` - Start of date range (inclusive) +- `endDate` - End of date range (inclusive) +- `dryRun` - If true, only simulates the operation + +**How It Works**: +1. Lists all existing partitions (attached + detached) +2. Generates list of dates in range that don't have partitions +3. For each missing partition: + - Creates table with same structure as parent (CREATE TABLE ... LIKE) + - Attaches partition with appropriate date range (FOR VALUES FROM ... TO ...) +4. Skips partitions that already exist +5. Returns count of partitions created + +**Use When**: +- Setting up a new partitioned table with historical dates +- Backfilling missing partitions after data gaps +- Preparing partitions in advance for future dates +- Recovering from partition management issues + +**Safety Features**: +- Checks for existing partitions before creating +- Dry-run mode to preview what will be created +- Automatically cleans up if attachment fails +- Comprehensive logging for each partition + +--- + #### DetachOldPartitions Bulk operation to detach all partitions older than retention period. @@ -678,7 +1023,365 @@ func restoreArchivedPartition(dbc *db.DB, partitionName string) error { } ``` -### Example 4: Complete Workflow +--- + +### Example 8: Create Missing Partitions for Date Range + +```go +func ensurePartitionsExist(dbc *db.DB, tableName string, startDate, endDate time.Time) error { + // Check what partitions would be created + created, err := partitions.CreateMissingPartitions(dbc, tableName, startDate, endDate, true) + if err != nil { + return fmt.Errorf("dry run failed: %w", err) + } + + if created == 0 { + log.Info("all partitions already exist") + return nil + } + + log.WithFields(log.Fields{ + "table": tableName, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + "to_create": created, + }).Info("creating missing partitions") + + // Create the missing partitions + created, err = partitions.CreateMissingPartitions(dbc, tableName, startDate, endDate, false) + if err != nil { + return fmt.Errorf("partition creation failed: %w", err) + } + + log.WithField("created", created).Info("partitions created successfully") + return nil +} + +// Example: Prepare partitions for next month +func prepareNextMonthPartitions(dbc *db.DB) error { + now := time.Now() + startOfNextMonth := time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, time.UTC) + endOfNextMonth := startOfNextMonth.AddDate(0, 1, -1) + + return ensurePartitionsExist(dbc, "test_analysis_by_job_by_dates", startOfNextMonth, endOfNextMonth) +} + +// Example: Backfill missing partitions for last 90 days +func backfillRecentPartitions(dbc *db.DB) error { + endDate := time.Now() + startDate := endDate.AddDate(0, 0, -90) + + return ensurePartitionsExist(dbc, "test_analysis_by_job_by_dates", startDate, endDate) +} +``` + +--- + +### Example 9: Create a New Partitioned Table from GORM Model + +```go +package main + +import ( + "time" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/partitions" +) + +// Define your model +type TestResults struct { + ID uint `gorm:"primaryKey"` + TestName string `gorm:"index"` + JobName string `gorm:"index"` + Result string + CreatedAt time.Time `gorm:"index"` // This will be the partition column + TestOutput string + Duration int +} + +func setupPartitionedTestResults(dbc *db.DB) error { + tableName := "test_results_partitioned" + + // Configure RANGE partitioning by created_at + config := partitions.NewRangePartitionConfig("created_at") + + // Step 1: Create the partitioned table (dry-run first) + sql, err := partitions.CreatePartitionedTable( + dbc, + &TestResults{}, + tableName, + config, + true, // dry-run + ) + if err != nil { + return fmt.Errorf("dry run failed: %w", err) + } + + log.Info("Would execute SQL:") + log.Info(sql) + + // The generated SQL will look like: + // CREATE TABLE IF NOT EXISTS test_results_partitioned ( + // id bigint NOT NULL, + // test_name text, + // job_name text, + // result text, + // created_at timestamp with time zone NOT NULL, + // test_output text, + // duration bigint, + // PRIMARY KEY (id, created_at) + // ) PARTITION BY RANGE (created_at) + // + // Note: created_at is automatically added to the primary key + // because it's the partition column (PostgreSQL requirement) + + // Step 2: Create the table for real + _, err = partitions.CreatePartitionedTable( + dbc, + &TestResults{}, + tableName, + config, + false, // execute + ) + if err != nil { + return fmt.Errorf("table creation failed: %w", err) + } + + log.WithField("table", tableName).Info("partitioned table created") + + // Step 3: Create partitions for the last 90 days + endDate := time.Now() + startDate := endDate.AddDate(0, 0, -90) + + created, err := partitions.CreateMissingPartitions( + dbc, + tableName, + startDate, + endDate, + false, + ) + if err != nil { + return fmt.Errorf("partition creation failed: %w", err) + } + + log.WithFields(log.Fields{ + "table": tableName, + "partitions": created, + }).Info("created partitions") + + return nil +} + +// You can now use the table normally with GORM +func insertTestResult(dbc *db.DB) error { + result := TestResults{ + TestName: "test-api-health", + JobName: "periodic-ci-test", + Result: "passed", + CreatedAt: time.Now(), + TestOutput: "All checks passed", + Duration: 125, + } + + // GORM will automatically route to the correct partition based on created_at + return dbc.DB.Create(&result).Error +} +``` + +**Key Points**: +- Model must have the partition column (e.g., `created_at`) +- PRIMARY KEY constraint is automatically generated +- Partition columns are automatically added to the primary key (PostgreSQL requirement) +- In the example above, `PRIMARY KEY (id, created_at)` is generated even though only `id` is marked as primaryKey +- Unique indexes must include the partition column +- Data is automatically routed to correct partition by PostgreSQL + +--- + +### Example 10: Update Partitioned Table Schema + +```go +package main + +import ( + "time" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/partitions" +) + +// Original model (what was created initially) +type TestResultsV1 struct { + ID uint `gorm:"primaryKey"` + TestName string `gorm:"index"` + JobName string `gorm:"index"` + Result string + CreatedAt time.Time `gorm:"index"` + TestOutput string + Duration int +} + +// Updated model with schema changes +type TestResultsV2 struct { + ID uint `gorm:"primaryKey"` + TestName string `gorm:"index"` + JobName string `gorm:"index"` + Result string + CreatedAt time.Time `gorm:"index"` + TestOutput string + Duration int + // New fields + TestSuite string `gorm:"index"` // Added: track test suite + ErrorCount int // Added: count of errors + // Removed: RemovedField no longer needed +} + +func updateTestResultsSchema(dbc *db.DB) error { + tableName := "test_results_partitioned" + + log.Info("Updating table schema to match new model...") + + // Step 1: Dry run to see what would change + sql, err := partitions.UpdatePartitionedTable( + dbc, + &TestResultsV2{}, + tableName, + true, // dry-run + ) + if err != nil { + return fmt.Errorf("dry run failed: %w", err) + } + + log.Info("Schema changes that would be applied:") + log.Info(sql) + + // Step 2: Review the changes and confirm + fmt.Println("\nReview the changes above.") + fmt.Print("Apply these changes? (yes/no): ") + var response string + fmt.Scanln(&response) + + if response != "yes" { + log.Info("Schema update cancelled") + return nil + } + + // Step 3: Apply the changes + sql, err = partitions.UpdatePartitionedTable( + dbc, + &TestResultsV2{}, + tableName, + false, // execute + ) + if err != nil { + return fmt.Errorf("schema update failed: %w", err) + } + + log.WithFields(log.Fields{ + "table": tableName, + "changes": sql, + }).Info("schema updated successfully") + + return nil +} + +// Automated schema migration (for CI/CD) +func automatedSchemaMigration(dbc *db.DB) error { + tableName := "test_results_partitioned" + + // Check what changes would be made + sql, err := partitions.UpdatePartitionedTable( + dbc, + &TestResultsV2{}, + tableName, + true, + ) + if err != nil { + return fmt.Errorf("schema check failed: %w", err) + } + + if sql == "" { + log.Info("Schema is up to date, no changes needed") + return nil + } + + // Log the planned changes + log.WithField("sql", sql).Info("applying schema changes") + + // Apply changes + sql, err = partitions.UpdatePartitionedTable( + dbc, + &TestResultsV2{}, + tableName, + false, + ) + if err != nil { + return fmt.Errorf("schema migration failed: %w", err) + } + + log.Info("schema migration completed successfully") + return nil +} + +// Example: Gradual schema evolution +func evolveSchema(dbc *db.DB) error { + tableName := "test_results_partitioned" + + // Phase 1: Add nullable columns first (safe) + type PhaseOne struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time `gorm:"index"` + TestName string + TestSuite string // New, nullable + } + + log.Info("Phase 1: Adding nullable columns") + _, err := partitions.UpdatePartitionedTable(dbc, &PhaseOne{}, tableName, false) + if err != nil { + return err + } + + // Phase 2: Populate new columns with data + log.Info("Phase 2: Populating new columns") + // (Application code populates test_suite from test_name) + + // Phase 3: Add indexes after data is populated + type PhaseTwo struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time `gorm:"index"` + TestName string + TestSuite string `gorm:"index"` // Now indexed + } + + log.Info("Phase 3: Adding indexes") + _, err = partitions.UpdatePartitionedTable(dbc, &PhaseTwo{}, tableName, false) + if err != nil { + return err + } + + log.Info("Schema evolution completed") + return nil +} +``` + +**Key Scenarios**: + +1. **Adding Columns**: New fields in the model are added to the table +2. **Removing Columns**: Fields removed from model are dropped (use caution) +3. **Changing Types**: Data type changes are detected and applied +4. **Adding Indexes**: New `gorm:"index"` tags create indexes +5. **Modifying Constraints**: NOT NULL and DEFAULT changes + +**Best Practices**: +- Always run dry-run first to preview changes +- Review generated SQL before applying +- Test schema changes in a development environment first +- For production, consider gradual evolution (add nullable, populate, add constraints) +- Back up data before major type conversions +- Monitor query performance after index changes + +--- + +### Example 11: Complete Workflow See [examples.go](./examples.go) for a complete workflow demonstration including: - Current state analysis diff --git a/pkg/db/partitions/examples.go b/pkg/db/partitions/examples.go index 7062878f5..6b6c4a0c4 100644 --- a/pkg/db/partitions/examples.go +++ b/pkg/db/partitions/examples.go @@ -2,6 +2,7 @@ package partitions import ( "fmt" + "strings" "time" log "github.com/sirupsen/logrus" @@ -148,8 +149,8 @@ func ExampleCheckRetentionPolicy(dbc *db.DB, tableName string, retentionDays int return } - // Get summary of what would be affected - summary, err := GetRetentionSummary(dbc, tableName, retentionDays) + // Get summary of what would be affected (all partitions) + summary, err := GetRetentionSummary(dbc, tableName, retentionDays, false) if err != nil { log.WithError(err).Error("failed to get retention summary") return @@ -164,8 +165,8 @@ func ExampleCheckRetentionPolicy(dbc *db.DB, tableName string, retentionDays int fmt.Printf(" Newest: %s\n", summary.NewestPartition) } - // Get detailed list of partitions that would be removed - partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays) + // Get detailed list of partitions that would be removed (both attached and detached) + partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays, false) if err != nil { log.WithError(err).Error("failed to get partitions for removal") return @@ -245,8 +246,8 @@ func ExampleDryRunCleanup(dbc *db.DB, tableName string, retentionDays int) { return } - // Get summary - summary, err := GetRetentionSummary(dbc, tableName, retentionDays) + // Get summary (all partitions, matching DropOldPartitions behavior) + summary, err := GetRetentionSummary(dbc, tableName, retentionDays, false) if err != nil { log.WithError(err).Error("failed to get summary") return @@ -440,8 +441,8 @@ func ExampleDropOldDetachedPartitions(dbc *db.DB, tableName string, retentionDay func ExampleDetachWorkflow(dbc *db.DB, tableName string, retentionDays int) { fmt.Printf("\n=== Detach Workflow for %s (%d days) ===\n", tableName, retentionDays) - // 1. Check what would be detached - summary, err := GetRetentionSummary(dbc, tableName, retentionDays) + // 1. Check what would be detached (only attached partitions) + summary, err := GetRetentionSummary(dbc, tableName, retentionDays, true) if err != nil { log.WithError(err).Error("failed to get summary") return @@ -512,6 +513,157 @@ func ExampleReattachPartition(dbc *db.DB, partitionName string) { // fmt.Println("3. Partition reattached successfully") } +// ExampleCreateMissingPartitions demonstrates creating missing partitions for a date range +// +// Usage: +// +// ExampleCreateMissingPartitions(dbc, "test_analysis_by_job_by_dates", "2024-01-01", "2024-01-31") +func ExampleCreateMissingPartitions(dbc *db.DB, tableName, startDateStr, endDateStr string) { + fmt.Printf("\n=== Create Missing Partitions for %s ===\n", tableName) + + // Parse dates + startDate, err := time.Parse("2006-01-02", startDateStr) + if err != nil { + log.WithError(err).Error("failed to parse start date") + return + } + + endDate, err := time.Parse("2006-01-02", endDateStr) + if err != nil { + log.WithError(err).Error("failed to parse end date") + return + } + + fmt.Printf("Date range: %s to %s\n", startDateStr, endDateStr) + + // Calculate expected number of partitions + days := int(endDate.Sub(startDate).Hours()/24) + 1 + fmt.Printf("Expected partitions: %d (one per day)\n", days) + + // Check current partitions + existing, err := ListTablePartitions(dbc, tableName) + if err != nil { + log.WithError(err).Error("failed to list existing partitions") + return + } + fmt.Printf("Existing partitions: %d\n", len(existing)) + + // Dry run to see what would be created + fmt.Println("\nDry run...") + created, err := CreateMissingPartitions(dbc, tableName, startDate, endDate, true) + if err != nil { + log.WithError(err).Error("dry run failed") + return + } + + if created == 0 { + fmt.Println("All partitions already exist - no action needed") + return + } + + fmt.Printf("Would create %d missing partitions\n", created) + + // Actual creation (commented out for safety) + // fmt.Println("\nCreating partitions...") + // created, err = CreateMissingPartitions(dbc, tableName, startDate, endDate, false) + // if err != nil { + // log.WithError(err).Error("creation failed") + // return + // } + // fmt.Printf("Successfully created %d partitions\n", created) +} + +// ExampleCreatePartitionedTable demonstrates creating a new partitioned table from a GORM model +// +// Usage: +// +// config := partitions.NewRangePartitionConfig("created_at") +// ExampleCreatePartitionedTable(dbc, &models.MyModel{}, "my_partitioned_table", config) +func ExampleCreatePartitionedTable(dbc *db.DB, model interface{}, tableName string, config PartitionConfig) { + fmt.Printf("\n=== Create Partitioned Table: %s ===\n", tableName) + fmt.Printf("Strategy: %s\n", config.Strategy) + fmt.Printf("Partition by: %s\n", strings.Join(config.Columns, ", ")) + + // Dry run to see the SQL that would be executed + fmt.Println("\nDry run - SQL that would be executed:") + sql, err := CreatePartitionedTable(dbc, model, tableName, config, true) + if err != nil { + log.WithError(err).Error("dry run failed") + return + } + + if sql == "" { + fmt.Println("Table already exists - no action needed") + return + } + + // Actual creation (commented out for safety) + // fmt.Println("\nCreating partitioned table...") + // _, err = CreatePartitionedTable(dbc, model, tableName, config, false) + // if err != nil { + // log.WithError(err).Error("table creation failed") + // return + // } + // fmt.Printf("Successfully created partitioned table: %s\n", tableName) + // + // // For RANGE partitions, create partitions for your date range + // if config.Strategy == db.PartitionStrategyRange { + // startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + // endDate := time.Now() + // created, err := CreateMissingPartitions(dbc, tableName, startDate, endDate, false) + // if err != nil { + // log.WithError(err).Error("partition creation failed") + // return + // } + // fmt.Printf("Created %d partitions\n", created) + // } + // + // // For HASH partitions, create the required number of partitions + // if config.Strategy == db.PartitionStrategyHash { + // for i := 0; i < config.Modulus; i++ { + // partName := fmt.Sprintf("%s_%d", tableName, i) + // sql := fmt.Sprintf("CREATE TABLE %s PARTITION OF %s FOR VALUES WITH (MODULUS %d, REMAINDER %d)", + // partName, tableName, config.Modulus, i) + // if err := dbc.DB.Exec(sql).Error; err != nil { + // log.WithError(err).Errorf("failed to create partition %s", partName) + // } + // } + // } +} + +// ExampleUpdatePartitionedTable demonstrates updating an existing partitioned table schema +// +// Usage: +// +// ExampleUpdatePartitionedTable(dbc, &models.MyModel{}, "my_partitioned_table") +func ExampleUpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string) { + fmt.Printf("\n=== Update Partitioned Table Schema: %s ===\n", tableName) + + // Dry run to see what changes would be made + fmt.Println("\nDry run - checking for schema changes:") + sql, err := UpdatePartitionedTable(dbc, model, tableName, true) + if err != nil { + log.WithError(err).Error("dry run failed") + return + } + + if sql == "" { + fmt.Println("Schema is up to date - no changes needed") + return + } + + fmt.Printf("\nChanges detected:\n%s\n", sql) + + // Actual update (commented out for safety) + // fmt.Println("\nApplying schema changes...") + // _, err = UpdatePartitionedTable(dbc, model, tableName, false) + // if err != nil { + // log.WithError(err).Error("schema update failed") + // return + // } + // fmt.Printf("Successfully updated table schema: %s\n", tableName) +} + // ExampleWorkflowForAnyTable demonstrates managing partitions for any table // // Usage: @@ -551,8 +703,8 @@ func ExampleWorkflowForAnyTable(dbc *db.DB) { stats.OldestDate.Format("2006-01-02"), stats.NewestDate.Format("2006-01-02")) - // Check 180-day retention policy - summary, err := GetRetentionSummary(dbc, table.TableName, 180) + // Check 180-day retention policy (all partitions) + summary, err := GetRetentionSummary(dbc, table.TableName, 180, false) if err != nil { log.WithError(err).WithField("table", table.TableName).Error("failed to get summary") continue diff --git a/pkg/db/partitions/partitions.go b/pkg/db/partitions/partitions.go index ec1786c08..7cd1d7d6c 100644 --- a/pkg/db/partitions/partitions.go +++ b/pkg/db/partitions/partitions.go @@ -7,6 +7,7 @@ import ( "time" log "github.com/sirupsen/logrus" + "gorm.io/gorm" "github.com/openshift/sippy/pkg/db" ) @@ -52,6 +53,88 @@ type RetentionSummary struct { NewestPartition string } +// PartitionConfig defines the configuration for creating a partitioned table +type PartitionConfig struct { + // Strategy is the partitioning strategy (RANGE, LIST, or HASH) + Strategy db.PartitionStrategy + + // Columns are the column(s) to partition by + // For RANGE and LIST: typically one column (e.g., "date", "created_at") + // For HASH: can be one or more columns + Columns []string + + // Modulus is required for HASH partitioning (number of partitions) + // Not used for RANGE or LIST + Modulus int +} + +// NewRangePartitionConfig creates a partition config for RANGE partitioning +func NewRangePartitionConfig(column string) PartitionConfig { + return PartitionConfig{ + Strategy: db.PartitionStrategyRange, + Columns: []string{column}, + } +} + +// NewListPartitionConfig creates a partition config for LIST partitioning +func NewListPartitionConfig(column string) PartitionConfig { + return PartitionConfig{ + Strategy: db.PartitionStrategyList, + Columns: []string{column}, + } +} + +// NewHashPartitionConfig creates a partition config for HASH partitioning +func NewHashPartitionConfig(modulus int, columns ...string) PartitionConfig { + return PartitionConfig{ + Strategy: db.PartitionStrategyHash, + Columns: columns, + Modulus: modulus, + } +} + +// Validate checks if the partition configuration is valid +func (pc PartitionConfig) Validate() error { + if pc.Strategy == "" { + return fmt.Errorf("partition strategy must be specified") + } + + if len(pc.Columns) == 0 { + return fmt.Errorf("at least one partition column must be specified") + } + + switch pc.Strategy { + case db.PartitionStrategyRange, db.PartitionStrategyList: + if len(pc.Columns) != 1 { + return fmt.Errorf("%s partitioning requires exactly one column, got %d", pc.Strategy, len(pc.Columns)) + } + case db.PartitionStrategyHash: + if pc.Modulus <= 0 { + return fmt.Errorf("HASH partitioning requires modulus > 0, got %d", pc.Modulus) + } + default: + return fmt.Errorf("unknown partition strategy: %s (valid: RANGE, LIST, HASH)", pc.Strategy) + } + + return nil +} + +// ToSQL generates the PARTITION BY clause for the CREATE TABLE statement +func (pc PartitionConfig) ToSQL() string { + columnList := strings.Join(pc.Columns, ", ") + + switch pc.Strategy { + case db.PartitionStrategyRange: + return fmt.Sprintf("PARTITION BY RANGE (%s)", columnList) + case db.PartitionStrategyList: + return fmt.Sprintf("PARTITION BY LIST (%s)", columnList) + case db.PartitionStrategyHash: + return fmt.Sprintf("PARTITION BY HASH (%s)", columnList) + default: + return "" + } +} + // ListPartitionedTables returns all partitioned parent tables in the database func ListPartitionedTables(dbc *db.DB) ([]PartitionedTableInfo, error) { start := time.Now() @@ -177,32 +260,66 @@ func GetPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, error) { // GetPartitionsForRemoval identifies partitions older than the retention period for a given table // This is a read-only operation (dry-run) that shows what would be removed (deleted or detached) -func GetPartitionsForRemoval(dbc *db.DB, tableName string, retentionDays int) ([]PartitionInfo, error) { +// If attachedOnly is true, only returns attached partitions (useful for detach operations) +// If attachedOnly is false, returns all partitions (useful for drop operations on both attached and detached) +func GetPartitionsForRemoval(dbc *db.DB, tableName string, retentionDays int, attachedOnly bool) ([]PartitionInfo, error) { start := time.Now() var partitions []PartitionInfo cutoffDate := time.Now().AddDate(0, 0, -retentionDays) - query := ` - SELECT - tablename, - 'public' as schemaname, - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, - (CURRENT_DATE - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD'))::INT AS age_days, - pg_total_relation_size('public.'||tablename) AS size_bytes, - pg_size_pretty(pg_total_relation_size('public.'||tablename)) AS size_pretty, - COALESCE(n_live_tup, 0) AS row_estimate - FROM pg_tables - LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename - AND pg_stat_user_tables.schemaname = pg_tables.schemaname - WHERE pg_tables.schemaname = 'public' - AND pg_tables.tablename LIKE @table_pattern - AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date - ORDER BY partition_date ASC - ` + var query string + if attachedOnly { + // Only return attached partitions + query = ` + WITH attached_partitions AS ( + SELECT c.relname AS tablename + FROM pg_inherits i + JOIN pg_class c ON i.inhrelid = c.oid + JOIN pg_class p ON i.inhparent = p.oid + WHERE p.relname = @table_name + ) + SELECT + tablename, + 'public' as schemaname, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + (CURRENT_DATE - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD'))::INT AS age_days, + pg_total_relation_size('public.'||tablename) AS size_bytes, + pg_size_pretty(pg_total_relation_size('public.'||tablename)) AS size_pretty, + COALESCE(n_live_tup, 0) AS row_estimate + FROM pg_tables + LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename + AND pg_stat_user_tables.schemaname = pg_tables.schemaname + WHERE pg_tables.schemaname = 'public' + AND pg_tables.tablename LIKE @table_pattern + AND pg_tables.tablename IN (SELECT tablename FROM attached_partitions) + AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date + ORDER BY partition_date ASC + ` + } else { + // Return all partitions (attached + detached) + query = ` + SELECT + tablename, + 'public' as schemaname, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, + (CURRENT_DATE - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD'))::INT AS age_days, + pg_total_relation_size('public.'||tablename) AS size_bytes, + pg_size_pretty(pg_total_relation_size('public.'||tablename)) AS size_pretty, + COALESCE(n_live_tup, 0) AS row_estimate + FROM pg_tables + LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename + AND pg_stat_user_tables.schemaname = pg_tables.schemaname + WHERE pg_tables.schemaname = 'public' + AND pg_tables.tablename LIKE @table_pattern + AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date + ORDER BY partition_date ASC + ` + } tablePattern := tableName + "_20%" result := dbc.DB.Raw(query, + sql.Named("table_name", tableName), sql.Named("table_pattern", tablePattern), sql.Named("cutoff_date", cutoffDate)).Scan(&partitions) if result.Error != nil { @@ -215,6 +332,7 @@ func GetPartitionsForRemoval(dbc *db.DB, tableName string, retentionDays int) ([ "table": tableName, "retention_days": retentionDays, "cutoff_date": cutoffDate.Format("2006-01-02"), + "attached_only": attachedOnly, "count": len(partitions), "elapsed": elapsed, }).Info("identified partitions for removal") @@ -223,7 +341,9 @@ func GetPartitionsForRemoval(dbc *db.DB, tableName string, retentionDays int) ([ } // GetRetentionSummary provides a summary of what would be affected by a retention policy for a given table -func GetRetentionSummary(dbc *db.DB, tableName string, retentionDays int) (*RetentionSummary, error) { +// If attachedOnly is true, only considers attached partitions (useful for detach operations) +// If attachedOnly is false, considers all partitions (useful for drop operations on both attached and detached) +func GetRetentionSummary(dbc *db.DB, tableName string, retentionDays int, attachedOnly bool) (*RetentionSummary, error) { start := time.Now() cutoffDate := time.Now().AddDate(0, 0, -retentionDays) @@ -232,21 +352,48 @@ func GetRetentionSummary(dbc *db.DB, tableName string, retentionDays int) (*Rete summary.RetentionDays = retentionDays summary.CutoffDate = cutoffDate - query := ` - SELECT - COUNT(*)::INT AS partitions_to_remove, - COALESCE(SUM(pg_total_relation_size('public.'||tablename)), 0)::BIGINT AS storage_to_reclaim, - COALESCE(pg_size_pretty(SUM(pg_total_relation_size('public.'||tablename))), '0 bytes') AS storage_pretty, - MIN(tablename) AS oldest_partition, - MAX(tablename) AS newest_partition - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE @table_pattern - AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date - ` + var query string + if attachedOnly { + // Only consider attached partitions + query = ` + WITH attached_partitions AS ( + SELECT c.relname AS tablename + FROM pg_inherits i + JOIN pg_class c ON i.inhrelid = c.oid + JOIN pg_class p ON i.inhparent = p.oid + WHERE p.relname = @table_name + ) + SELECT + COUNT(*)::INT AS partitions_to_remove, + COALESCE(SUM(pg_total_relation_size('public.'||tablename)), 0)::BIGINT AS storage_to_reclaim, + COALESCE(pg_size_pretty(SUM(pg_total_relation_size('public.'||tablename))), '0 bytes') AS storage_pretty, + MIN(tablename) AS oldest_partition, + MAX(tablename) AS newest_partition + FROM pg_tables + WHERE schemaname = 'public' + AND tablename LIKE @table_pattern + AND tablename IN (SELECT tablename FROM attached_partitions) + AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date + ` + } else { + // Consider all partitions (attached + detached) + query = ` + SELECT + COUNT(*)::INT AS partitions_to_remove, + COALESCE(SUM(pg_total_relation_size('public.'||tablename)), 0)::BIGINT AS storage_to_reclaim, + COALESCE(pg_size_pretty(SUM(pg_total_relation_size('public.'||tablename))), '0 bytes') AS storage_pretty, + MIN(tablename) AS oldest_partition, + MAX(tablename) AS newest_partition + FROM pg_tables + WHERE schemaname = 'public' + AND tablename LIKE @table_pattern + AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date + ` + } tablePattern := tableName + "_20%" result := dbc.DB.Raw(query, + sql.Named("table_name", tableName), sql.Named("table_pattern", tablePattern), sql.Named("cutoff_date", cutoffDate)).Scan(&summary) if result.Error != nil { @@ -258,6 +405,7 @@ func GetRetentionSummary(dbc *db.DB, tableName string, retentionDays int) (*Rete log.WithFields(log.Fields{ "table": tableName, "retention_days": retentionDays, + "attached_only": attachedOnly, "partitions_to_remove": summary.PartitionsToRemove, "storage_to_reclaim": summary.StoragePretty, "elapsed": elapsed, @@ -363,7 +511,8 @@ func ValidateRetentionPolicy(dbc *db.DB, tableName string, retentionDays int) er return fmt.Errorf("retention policy too aggressive: minimum 90 days required, got %d", retentionDays) } - summary, err := GetRetentionSummary(dbc, tableName, retentionDays) + // Get summary for attached partitions only to match stats below + summary, err := GetRetentionSummary(dbc, tableName, retentionDays, true) if err != nil { return fmt.Errorf("failed to get retention summary: %w", err) } @@ -502,7 +651,8 @@ func DropOldPartitions(dbc *db.DB, tableName string, retentionDays int, dryRun b return 0, fmt.Errorf("retention policy validation failed: %w", err) } - partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays) + // Get all partitions for removal (both attached and detached) + partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays, false) if err != nil { return 0, fmt.Errorf("failed to get partitions for removal: %w", err) } @@ -748,6 +898,888 @@ func GetAttachedPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, e return &stats, nil } +// CreateMissingPartitions creates partitions for a date range if they don't already exist +// Assumes daily partitions (one partition per day) based on the naming convention: tablename_YYYY_MM_DD +// Each partition covers a 24-hour period from midnight to midnight +// +// Workflow: +// 1. Lists all existing partitions (both attached and detached) +// 2. Generates list of missing dates in the specified range +// 3. For each missing date: creates table and attaches it as partition +// 4. Skips dates that already have partitions (attached or detached) +// +// Parameters: +// - tableName: Name of the partitioned parent table +// - startDate: Start of date range (inclusive) +// - endDate: End of date range (inclusive) +// - dryRun: If true, logs what would be created without executing +// +// Returns: Count of partitions created (or would be created in dry-run mode) +func CreateMissingPartitions(dbc *db.DB, tableName string, startDate, endDate time.Time, dryRun bool) (int, error) { + start := time.Now() + + // Validate date range + if endDate.Before(startDate) { + return 0, fmt.Errorf("end date (%s) cannot be before start date (%s)", + endDate.Format("2006-01-02"), startDate.Format("2006-01-02")) + } + + // Get list of all existing partitions (attached + detached) + existingPartitions, err := ListTablePartitions(dbc, tableName) + if err != nil { + return 0, fmt.Errorf("failed to list existing partitions: %w", err) + } + + // Create a map of existing partition dates for quick lookup + existingDates := make(map[string]bool) + for _, p := range existingPartitions { + dateStr := p.PartitionDate.Format("2006_01_02") + existingDates[dateStr] = true + } + + // Generate list of partitions to create + var partitionsToCreate []time.Time + currentDate := startDate + for !currentDate.After(endDate) { + dateStr := currentDate.Format("2006_01_02") + if !existingDates[dateStr] { + partitionsToCreate = append(partitionsToCreate, currentDate) + } + currentDate = currentDate.AddDate(0, 0, 1) // Move to next day + } + + if len(partitionsToCreate) == 0 { + log.WithFields(log.Fields{ + "table": tableName, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("no missing partitions to create") + return 0, nil + } + + createdCount := 0 + for _, partitionDate := range partitionsToCreate { + partitionName := fmt.Sprintf("%s_%s", tableName, partitionDate.Format("2006_01_02")) + rangeStart := partitionDate.Format("2006-01-02") + rangeEnd := partitionDate.AddDate(0, 0, 1).Format("2006-01-02") + + if dryRun { + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + "range_start": rangeStart, + "range_end": rangeEnd, + }).Info("[DRY RUN] would create partition") + createdCount++ + continue + } + + // Create the partition table with same structure as parent + createTableQuery := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (LIKE %s INCLUDING ALL)", partitionName, tableName) + result := dbc.DB.Exec(createTableQuery) + if result.Error != nil { + log.WithError(result.Error).WithField("partition", partitionName).Error("failed to create partition table") + continue + } + + // Attach the partition to the parent table + attachQuery := fmt.Sprintf( + "ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM ('%s') TO ('%s')", + tableName, + partitionName, + rangeStart, + rangeEnd, + ) + result = dbc.DB.Exec(attachQuery) + if result.Error != nil { + // If attach fails, try to clean up the created table + log.WithError(result.Error).WithField("partition", partitionName).Error("failed to attach partition") + dbc.DB.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partitionName)) + continue + } + + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + "range_start": rangeStart, + "range_end": rangeEnd, + }).Info("created and attached partition") + createdCount++ + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + "created": createdCount, + "total_days": len(partitionsToCreate), + "dry_run": dryRun, + "elapsed": elapsed, + }).Info("completed creating missing partitions") + + return createdCount, nil +} + +// gormTypeToPostgresType converts GORM/Go data types to PostgreSQL types +func gormTypeToPostgresType(dataType string) string { + dataType = strings.ToLower(strings.TrimSpace(dataType)) + + // Map GORM/Go types to PostgreSQL types + typeMap := map[string]string{ + // Integer types + "uint": "bigint", + "uint8": "smallint", + "uint16": "integer", + "uint32": "bigint", + "uint64": "bigint", + "int": "bigint", + "int8": "smallint", + "int16": "smallint", + "int32": "integer", + "int64": "bigint", + "integer": "integer", + "bigint": "bigint", + + // Float types + "float": "double precision", + "float32": "real", + "float64": "double precision", + + // String types + "string": "text", + "text": "text", + + // Boolean + "bool": "boolean", + "boolean": "boolean", + + // Time types + "time.time": "timestamp with time zone", + "time": "timestamp with time zone", + "timestamp": "timestamp with time zone", + "date": "date", + + // Binary + "[]byte": "bytea", + "bytes": "bytea", + "bytea": "bytea", + + // JSON + "json": "jsonb", + "jsonb": "jsonb", + + // UUID + "uuid": "uuid", + } + + // Check if we have a direct mapping + if pgType, exists := typeMap[dataType]; exists { + return pgType + } + + // If it's already a PostgreSQL type, return as-is + // Common PostgreSQL types that might pass through + postgresTypes := []string{ + "varchar", "character varying", + "smallint", "bigserial", "serial", + "numeric", "decimal", "real", "double precision", + "timestamptz", "timestamp without time zone", + "interval", "money", + "inet", "cidr", "macaddr", + "point", "line", "lseg", "box", "path", "polygon", "circle", + "xml", "array", + } + + for _, pgType := range postgresTypes { + if strings.Contains(dataType, pgType) { + return dataType + } + } + + // If we can't map it, log a warning and return as-is + // This allows for custom types or types we haven't mapped yet + log.WithField("data_type", dataType).Warn("unmapped data type - using as-is (may cause PostgreSQL error)") + return dataType +} + +// CreatePartitionedTable creates a new partitioned table based on a GORM model struct +// If the table already exists, it returns without error +// +// Parameters: +// - model: GORM model struct (must be a pointer, e.g., &models.MyModel{}) +// - tableName: Name for the partitioned table +// - config: Partition configuration (strategy, columns, etc.) +// - dryRun: If true, prints SQL without executing +// +// Returns: The SQL statement that was (or would be) executed +// +// Example: +// +// config := partitions.NewRangePartitionConfig("created_at") +// sql, err := partitions.CreatePartitionedTable(dbc, &MyModel{}, "my_table", config, true) +func CreatePartitionedTable(dbc *db.DB, model interface{}, tableName string, config PartitionConfig, dryRun bool) (string, error) { + start := time.Now() + + // Validate partition configuration + if err := config.Validate(); err != nil { + return "", fmt.Errorf("invalid partition config: %w", err) + } + + // Check if table already exists + if dbc.DB.Migrator().HasTable(tableName) { + log.WithField("table", tableName).Info("partitioned table already exists, skipping creation") + return "", nil + } + + // Use GORM statement parser to get the table structure from the model + stmt := &gorm.Statement{DB: dbc.DB} + if err := stmt.Parse(model); err != nil { + return "", fmt.Errorf("failed to parse model: %w", err) + } + + // Build the CREATE TABLE statement manually from the GORM schema + var columns []string + var primaryKeyColumns []string + + // Create a map of fields with default database values for quick lookup + hasDefaultDBValue := make(map[string]bool) + for _, field := range stmt.Schema.FieldsWithDefaultDBValue { + hasDefaultDBValue[field.Name] = true + } + + // Track which columns we've already added to prevent duplicates + addedColumns := make(map[string]bool) + + for _, field := range stmt.Schema.Fields { + // Skip fields that shouldn't be in the database + if field.IgnoreMigration { + continue + } + + // Skip fields with empty DBName or DataType + if field.DBName == "" || field.DataType == "" { + log.WithFields(log.Fields{ + "table": tableName, + "field": field.Name, + "db_name": field.DBName, + "data_type": field.DataType, + }).Warn("skipping field with empty DBName or DataType") + continue + } + + // Skip duplicate columns (GORM can include same field multiple times) + if addedColumns[field.DBName] { + log.WithFields(log.Fields{ + "table": tableName, + "column": field.DBName, + "field": field.Name, + }).Debug("skipping duplicate column") + continue + } + addedColumns[field.DBName] = true + + // Convert GORM/Go type to PostgreSQL type + pgType := gormTypeToPostgresType(string(field.DataType)) + columnDef := fmt.Sprintf("%s %s", field.DBName, pgType) + + // Handle AUTO_INCREMENT using GENERATED BY DEFAULT AS IDENTITY + // This must be done before NOT NULL and DEFAULT clauses + if field.AutoIncrement { + // IDENTITY columns are always NOT NULL, so we add GENERATED BY DEFAULT AS IDENTITY + if field.AutoIncrementIncrement > 0 { + columnDef += fmt.Sprintf(" GENERATED BY DEFAULT AS IDENTITY (INCREMENT BY %d)", field.AutoIncrementIncrement) + } else { + columnDef += " GENERATED BY DEFAULT AS IDENTITY" + } + // IDENTITY columns are automatically NOT NULL, no need to add it explicitly + } else { + // Add NOT NULL constraint if applicable + // Primary keys are always NOT NULL in PostgreSQL + if field.PrimaryKey || field.NotNull { + columnDef += " NOT NULL" + } + + // Add DEFAULT if specified + // Check both field.DefaultValue and if field is in FieldsWithDefaultDBValue + if field.DefaultValue != "" { + columnDef += fmt.Sprintf(" DEFAULT %s", field.DefaultValue) + } else if hasDefaultDBValue[field.Name] && field.DefaultValueInterface != nil { + // Field has a database-level default value + columnDef += fmt.Sprintf(" DEFAULT %v", field.DefaultValueInterface) + } + } + + columns = append(columns, columnDef) + + // Track primary key columns + if field.PrimaryKey { + primaryKeyColumns = append(primaryKeyColumns, field.DBName) + } + } + + // Add PRIMARY KEY constraint if we have primary keys + // For partitioned tables, the primary key must include all partition columns + if len(primaryKeyColumns) > 0 { + // Check if primary key includes all partition columns + pkMap := make(map[string]bool) + for _, pk := range primaryKeyColumns { + pkMap[pk] = true + } + + // Add missing partition columns to primary key + missingPartCols := []string{} + for _, partCol := range config.Columns { + if !pkMap[partCol] { + missingPartCols = append(missingPartCols, partCol) + } + } + + if len(missingPartCols) > 0 { + log.WithFields(log.Fields{ + "table": tableName, + "primary_keys": primaryKeyColumns, + "partition_columns": config.Columns, + "missing_in_pk": missingPartCols, + }).Warn("primary key must include all partition columns - adding partition columns to primary key") + primaryKeyColumns = append(primaryKeyColumns, missingPartCols...) + } + + primaryKeyConstraint := fmt.Sprintf("PRIMARY KEY (%s)", strings.Join(primaryKeyColumns, ", ")) + columns = append(columns, primaryKeyConstraint) + } + + // Build the CREATE TABLE statement with partition strategy + partitionClause := config.ToSQL() + createTableSQL := fmt.Sprintf( + "CREATE TABLE IF NOT EXISTS %s (\n %s\n) %s", + tableName, + strings.Join(columns, ",\n "), + partitionClause, + ) + + // Create a map of partition columns for easy lookup + partitionColMap := make(map[string]bool) + for _, col := range config.Columns { + partitionColMap[col] = true + } + + // Add indexes if they exist in the schema + var indexSQL strings.Builder + for _, idx := range stmt.Schema.ParseIndexes() { + // Skip unique indexes that don't include ALL partition keys + // (they're not allowed in partitioned tables) + if idx.Class == "UNIQUE" { + hasAllPartitionKeys := true + for _, partCol := range config.Columns { + found := false + for _, field := range idx.Fields { + if field.DBName == partCol { + found = true + break + } + } + if !found { + hasAllPartitionKeys = false + break + } + } + if !hasAllPartitionKeys { + log.WithFields(log.Fields{ + "table": tableName, + "index": idx.Name, + "partition_keys": config.Columns, + }).Warn("skipping unique index without all partition keys (not allowed on partitioned tables)") + continue + } + } + + indexSQL.WriteString("\n") + if idx.Class == "UNIQUE" { + indexSQL.WriteString(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (", idx.Name, tableName)) + } else { + indexSQL.WriteString(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (", idx.Name, tableName)) + } + + var fieldNames []string + for _, field := range idx.Fields { + fieldNames = append(fieldNames, field.DBName) + } + indexSQL.WriteString(strings.Join(fieldNames, ", ")) + indexSQL.WriteString(");") + } + + fullSQL := createTableSQL + ";" + indexSQL.String() + + if dryRun { + log.WithField("table", tableName).Info("[DRY RUN] would execute SQL:") + fmt.Println("\n" + strings.Repeat("-", 80)) + fmt.Println(fullSQL) + fmt.Println(strings.Repeat("-", 80) + "\n") + return fullSQL, nil + } + + // Execute the CREATE TABLE statement + result := dbc.DB.Exec(createTableSQL) + if result.Error != nil { + return "", fmt.Errorf("failed to create partitioned table: %w", result.Error) + } + + // Execute index creation statements + if indexSQL.Len() > 0 { + result = dbc.DB.Exec(indexSQL.String()) + if result.Error != nil { + log.WithError(result.Error).Warn("some indexes may have failed to create") + } + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "partition_strategy": string(config.Strategy), + "partition_columns": strings.Join(config.Columns, ", "), + "elapsed": elapsed, + }).Info("created partitioned table") + + return fullSQL, nil +} + +// indexInfo holds information about a database index +type indexInfo struct { + IndexName string + IsUnique bool + Columns []string +} + +// UpdatePartitionedTable updates an existing partitioned table schema based on a GORM model +// Detects differences between the model and current database schema and generates ALTER statements +// +// Parameters: +// - model: GORM model struct (must be a pointer, e.g., &models.MyModel{}) +// - tableName: Name of the existing partitioned table +// - dryRun: If true, prints SQL without executing +// +// Returns: The SQL statements that were (or would be) executed +// +// Example: +// +// sql, err := partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_table", true) +// +// Note: Cannot modify partition keys or add unique constraints without partition keys +func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dryRun bool) (string, error) { + start := time.Now() + + // Check if table exists + if !dbc.DB.Migrator().HasTable(tableName) { + return "", fmt.Errorf("table %s does not exist", tableName) + } + + // Parse the GORM model to get desired schema + stmt := &gorm.Statement{DB: dbc.DB} + if err := stmt.Parse(model); err != nil { + return "", fmt.Errorf("failed to parse model: %w", err) + } + + // Get current schema from database + currentColumns, err := dbc.GetTableColumns(tableName) + if err != nil { + return "", fmt.Errorf("failed to get current columns: %w", err) + } + + currentIndexes, err := getCurrentIndexes(dbc, tableName) + if err != nil { + return "", fmt.Errorf("failed to get current indexes: %w", err) + } + + // Get partition columns to validate unique indexes + partitionColumns, err := getPartitionColumns(dbc, tableName) + if err != nil { + return "", fmt.Errorf("failed to get partition columns: %w", err) + } + + // Build maps for comparison + currentColMap := make(map[string]db.ColumnInfo) + for _, col := range currentColumns { + currentColMap[col.ColumnName] = col + } + + currentIdxMap := make(map[string]indexInfo) + for _, idx := range currentIndexes { + currentIdxMap[idx.IndexName] = idx + } + + // Create a map of fields with default database values for quick lookup + hasDefaultDBValue := make(map[string]bool) + for _, field := range stmt.Schema.FieldsWithDefaultDBValue { + hasDefaultDBValue[field.Name] = true + } + + // Track which columns we've already processed to prevent duplicates + processedColumns := make(map[string]bool) + + // Generate ALTER statements + var alterStatements []string + + // Check for new or modified columns + for _, field := range stmt.Schema.Fields { + if field.IgnoreMigration { + continue + } + + // Skip fields with empty DBName or DataType + if field.DBName == "" || field.DataType == "" { + log.WithFields(log.Fields{ + "table": tableName, + "field": field.Name, + "db_name": field.DBName, + "data_type": field.DataType, + }).Warn("skipping field with empty DBName or DataType") + continue + } + + // Skip duplicate columns (GORM can include same field multiple times) + if processedColumns[field.DBName] { + log.WithFields(log.Fields{ + "table": tableName, + "column": field.DBName, + "field": field.Name, + }).Debug("skipping duplicate column") + continue + } + processedColumns[field.DBName] = true + + currentCol, exists := currentColMap[field.DBName] + + // Convert GORM/Go type to PostgreSQL type + pgType := gormTypeToPostgresType(string(field.DataType)) + + if !exists { + // New column - add it + columnDef := fmt.Sprintf("%s %s", field.DBName, pgType) + + // Handle AUTO_INCREMENT using GENERATED BY DEFAULT AS IDENTITY + if field.AutoIncrement { + // IDENTITY columns are always NOT NULL, so we add GENERATED BY DEFAULT AS IDENTITY + if field.AutoIncrementIncrement > 0 { + columnDef += fmt.Sprintf(" GENERATED BY DEFAULT AS IDENTITY (INCREMENT BY %d)", field.AutoIncrementIncrement) + } else { + columnDef += " GENERATED BY DEFAULT AS IDENTITY" + } + // IDENTITY columns are automatically NOT NULL, no need to add it explicitly + } else { + // Primary keys are always NOT NULL in PostgreSQL + if field.PrimaryKey || field.NotNull { + columnDef += " NOT NULL" + } + // Add DEFAULT if specified + // Check both field.DefaultValue and if field is in FieldsWithDefaultDBValue + if field.DefaultValue != "" { + columnDef += fmt.Sprintf(" DEFAULT %s", field.DefaultValue) + } else if hasDefaultDBValue[field.Name] && field.DefaultValueInterface != nil { + // Field has a database-level default value + columnDef += fmt.Sprintf(" DEFAULT %v", field.DefaultValueInterface) + } + } + + alterStatements = append(alterStatements, + fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s", tableName, columnDef)) + } else { + // Existing column - check for modifications + modifications := []string{} + + // Check data type + if !strings.EqualFold(normalizeDataType(currentCol.DataType), normalizeDataType(pgType)) { + modifications = append(modifications, + fmt.Sprintf("ALTER COLUMN %s TYPE %s", field.DBName, pgType)) + } + + // Check NOT NULL constraint + // Primary keys are always NOT NULL in PostgreSQL + currentNotNull := currentCol.IsNullable == "NO" + desiredNotNull := field.PrimaryKey || field.NotNull + if desiredNotNull != currentNotNull { + if desiredNotNull { + modifications = append(modifications, + fmt.Sprintf("ALTER COLUMN %s SET NOT NULL", field.DBName)) + } else { + modifications = append(modifications, + fmt.Sprintf("ALTER COLUMN %s DROP NOT NULL", field.DBName)) + } + } + + // Check DEFAULT value + currentDefault := "" + if currentCol.ColumnDefault.Valid { + currentDefault = currentCol.ColumnDefault.String + } + if field.DefaultValue != currentDefault { + if field.DefaultValue != "" { + modifications = append(modifications, + fmt.Sprintf("ALTER COLUMN %s SET DEFAULT %s", field.DBName, field.DefaultValue)) + } else if currentDefault != "" { + modifications = append(modifications, + fmt.Sprintf("ALTER COLUMN %s DROP DEFAULT", field.DBName)) + } + } + + // Add modifications as separate ALTER TABLE statements + for _, mod := range modifications { + alterStatements = append(alterStatements, + fmt.Sprintf("ALTER TABLE %s %s", tableName, mod)) + } + } + + // Remove from map to track processed columns + delete(currentColMap, field.DBName) + } + + // Remaining columns in map should be dropped + for colName := range currentColMap { + alterStatements = append(alterStatements, + fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", tableName, colName)) + } + + // Check indexes + partitionColMap := make(map[string]bool) + for _, col := range partitionColumns { + partitionColMap[col] = true + } + + for _, idx := range stmt.Schema.ParseIndexes() { + // Skip unique indexes that don't include all partition keys + if idx.Class == "UNIQUE" { + hasAllPartitionKeys := true + for _, partCol := range partitionColumns { + found := false + for _, field := range idx.Fields { + if field.DBName == partCol { + found = true + break + } + } + if !found { + hasAllPartitionKeys = false + break + } + } + if !hasAllPartitionKeys { + log.WithFields(log.Fields{ + "table": tableName, + "index": idx.Name, + "partition_keys": partitionColumns, + }).Warn("skipping unique index without all partition keys") + continue + } + } + + currentIdx, exists := currentIdxMap[idx.Name] + if !exists { + // New index - create it + var fieldNames []string + for _, field := range idx.Fields { + fieldNames = append(fieldNames, field.DBName) + } + + if idx.Class == "UNIQUE" { + alterStatements = append(alterStatements, + fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", + idx.Name, tableName, strings.Join(fieldNames, ", "))) + } else { + alterStatements = append(alterStatements, + fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", + idx.Name, tableName, strings.Join(fieldNames, ", "))) + } + } else { + // Index exists - check if it needs to be recreated + var desiredCols []string + for _, field := range idx.Fields { + desiredCols = append(desiredCols, field.DBName) + } + + colsMatch := len(currentIdx.Columns) == len(desiredCols) + if colsMatch { + for i, col := range currentIdx.Columns { + if col != desiredCols[i] { + colsMatch = false + break + } + } + } + + uniqueMatch := (idx.Class == "UNIQUE") == currentIdx.IsUnique + + if !colsMatch || !uniqueMatch { + // Drop and recreate index + alterStatements = append(alterStatements, + fmt.Sprintf("DROP INDEX IF EXISTS %s", idx.Name)) + + if idx.Class == "UNIQUE" { + alterStatements = append(alterStatements, + fmt.Sprintf("CREATE UNIQUE INDEX %s ON %s (%s)", + idx.Name, tableName, strings.Join(desiredCols, ", "))) + } else { + alterStatements = append(alterStatements, + fmt.Sprintf("CREATE INDEX %s ON %s (%s)", + idx.Name, tableName, strings.Join(desiredCols, ", "))) + } + } + } + + delete(currentIdxMap, idx.Name) + } + + // Drop indexes that are no longer in the model + for idxName := range currentIdxMap { + // Skip primary key and system indexes + if strings.HasSuffix(idxName, "_pkey") { + continue + } + alterStatements = append(alterStatements, + fmt.Sprintf("DROP INDEX IF EXISTS %s", idxName)) + } + + // If no changes, return early + if len(alterStatements) == 0 { + log.WithField("table", tableName).Info("schema is up to date, no changes needed") + return "", nil + } + + fullSQL := strings.Join(alterStatements, ";\n") + ";" + + if dryRun { + log.WithField("table", tableName).Info("[DRY RUN] would execute SQL:") + fmt.Println("\n" + strings.Repeat("-", 80)) + fmt.Println(fullSQL) + fmt.Println(strings.Repeat("-", 80) + "\n") + return fullSQL, nil + } + + // Execute ALTER statements + successCount := 0 + for _, stmt := range alterStatements { + result := dbc.DB.Exec(stmt) + if result.Error != nil { + log.WithError(result.Error).WithField("statement", stmt).Error("failed to execute ALTER statement") + continue + } + successCount++ + } + + elapsed := time.Since(start) + log.WithFields(log.Fields{ + "table": tableName, + "statements": len(alterStatements), + "successful": successCount, + "elapsed": elapsed, + }).Info("updated partitioned table schema") + + return fullSQL, nil +} + +// getCurrentColumns retrieves the current column schema from the database +// getCurrentIndexes retrieves the current indexes from the database +func getCurrentIndexes(dbc *db.DB, tableName string) ([]indexInfo, error) { + type indexRow struct { + IndexName string + IsUnique bool + Column string + } + + var rows []indexRow + + query := ` + SELECT + i.indexname AS index_name, + ix.indisunique AS is_unique, + a.attname AS column + FROM pg_indexes i + JOIN pg_class c ON c.relname = i.indexname + JOIN pg_index ix ON ix.indexrelid = c.oid + JOIN pg_attribute a ON a.attrelid = ix.indrelid AND a.attnum = ANY(ix.indkey) + WHERE i.schemaname = 'public' + AND i.tablename = $1 + ORDER BY i.indexname, a.attnum + ` + + result := dbc.DB.Raw(query, tableName).Scan(&rows) + if result.Error != nil { + return nil, result.Error + } + + // Group by index name + indexMap := make(map[string]*indexInfo) + for _, row := range rows { + if idx, exists := indexMap[row.IndexName]; exists { + idx.Columns = append(idx.Columns, row.Column) + } else { + indexMap[row.IndexName] = &indexInfo{ + IndexName: row.IndexName, + IsUnique: row.IsUnique, + Columns: []string{row.Column}, + } + } + } + + var indexes []indexInfo + for _, idx := range indexMap { + indexes = append(indexes, *idx) + } + + return indexes, nil +} + +// getPartitionColumns retrieves the partition key columns for a table +func getPartitionColumns(dbc *db.DB, tableName string) ([]string, error) { + var columns []string + + query := ` + SELECT a.attname + FROM pg_class c + JOIN pg_partitioned_table pt ON pt.partrelid = c.oid + JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = ANY(pt.partattrs) + WHERE c.relname = $1 + AND c.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public') + ORDER BY array_position(pt.partattrs, a.attnum) + ` + + result := dbc.DB.Raw(query, tableName).Scan(&columns) + if result.Error != nil { + return nil, result.Error + } + + return columns, nil +} + +// normalizeDataType normalizes data type strings for comparison +func normalizeDataType(dataType string) string { + // Convert to lowercase and remove common variations + normalized := strings.ToLower(strings.TrimSpace(dataType)) + + // Handle common type mappings + replacements := map[string]string{ + "character varying": "varchar", + "integer": "int", + "bigint": "int8", + "smallint": "int2", + "boolean": "bool", + "timestamp without time zone": "timestamp", + "timestamp with time zone": "timestamptz", + "double precision": "float8", + "real": "float4", + "character": "char", + "time without time zone": "time", + "time with time zone": "timetz", + } + + for old, new := range replacements { + if strings.Contains(normalized, old) { + normalized = strings.ReplaceAll(normalized, old, new) + } + } + + return normalized +} + // GetDetachedPartitionStats returns statistics about detached partitions for a given table func GetDetachedPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, error) { start := time.Now() @@ -927,7 +1959,8 @@ func DetachOldPartitions(dbc *db.DB, tableName string, retentionDays int, dryRun return 0, fmt.Errorf("retention policy validation failed: %w", err) } - partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays) + // Get only attached partitions for removal (can only detach what's attached) + partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays, true) if err != nil { return 0, fmt.Errorf("failed to get partitions for removal: %w", err) } diff --git a/pkg/db/partitions/partitions_test.go b/pkg/db/partitions/partitions_test.go index 6d5bada34..02738b866 100644 --- a/pkg/db/partitions/partitions_test.go +++ b/pkg/db/partitions/partitions_test.go @@ -3,6 +3,8 @@ package partitions import ( "testing" "time" + + "github.com/openshift/sippy/pkg/db" ) func TestIsValidTestAnalysisPartitionName(t *testing.T) { @@ -95,3 +97,446 @@ func TestRetentionSummary(t *testing.T) { t.Errorf("unexpected partitions to remove: %d", summary.PartitionsToRemove) } } + +func TestExtractTableNameFromPartition(t *testing.T) { + tests := []struct { + name string + partitionName string + wantTableName string + wantError bool + }{ + { + name: "valid partition", + partitionName: "test_analysis_by_job_by_dates_2024_10_29", + wantTableName: "test_analysis_by_job_by_dates", + wantError: false, + }, + { + name: "different table", + partitionName: "prow_job_runs_2024_01_15", + wantTableName: "prow_job_runs", + wantError: false, + }, + { + name: "too short", + partitionName: "short", + wantTableName: "", + wantError: true, + }, + { + name: "invalid date", + partitionName: "table_name_invalid_date", + wantTableName: "", + wantError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := extractTableNameFromPartition(tt.partitionName) + if (err != nil) != tt.wantError { + t.Errorf("extractTableNameFromPartition() error = %v, wantError %v", err, tt.wantError) + return + } + if got != tt.wantTableName { + t.Errorf("extractTableNameFromPartition() = %v, want %v", got, tt.wantTableName) + } + }) + } +} + +func TestIsValidPartitionName(t *testing.T) { + tests := []struct { + name string + tableName string + partitionName string + want bool + }{ + { + name: "valid partition", + tableName: "test_table", + partitionName: "test_table_2024_10_29", + want: true, + }, + { + name: "wrong table name", + tableName: "test_table", + partitionName: "other_table_2024_10_29", + want: false, + }, + { + name: "invalid date", + tableName: "test_table", + partitionName: "test_table_2024_13_40", + want: false, + }, + { + name: "wrong length", + tableName: "test_table", + partitionName: "test_table_2024_10", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isValidPartitionName(tt.tableName, tt.partitionName) + if got != tt.want { + t.Errorf("isValidPartitionName() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPartitionConfigValidation(t *testing.T) { + tests := []struct { + name string + config PartitionConfig + wantErr bool + }{ + { + name: "valid RANGE config", + config: NewRangePartitionConfig("created_at"), + wantErr: false, + }, + { + name: "valid LIST config", + config: NewListPartitionConfig("region"), + wantErr: false, + }, + { + name: "valid HASH config", + config: NewHashPartitionConfig(4, "user_id"), + wantErr: false, + }, + { + name: "invalid - no strategy", + config: PartitionConfig{ + Columns: []string{"created_at"}, + }, + wantErr: true, + }, + { + name: "invalid - no columns", + config: PartitionConfig{ + Strategy: db.PartitionStrategyRange, + Columns: []string{}, + }, + wantErr: true, + }, + { + name: "invalid - RANGE with multiple columns", + config: PartitionConfig{ + Strategy: db.PartitionStrategyRange, + Columns: []string{"col1", "col2"}, + }, + wantErr: true, + }, + { + name: "invalid - HASH with no modulus", + config: PartitionConfig{ + Strategy: db.PartitionStrategyHash, + Columns: []string{"user_id"}, + Modulus: 0, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("PartitionConfig.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestPartitionConfigToSQL(t *testing.T) { + tests := []struct { + name string + config PartitionConfig + expected string + }{ + { + name: "RANGE partition", + config: NewRangePartitionConfig("created_at"), + expected: "PARTITION BY RANGE (created_at)", + }, + { + name: "LIST partition", + config: NewListPartitionConfig("region"), + expected: "PARTITION BY LIST (region)", + }, + { + name: "HASH partition single column", + config: NewHashPartitionConfig(4, "user_id"), + expected: "PARTITION BY HASH (user_id)", + }, + { + name: "HASH partition multiple columns", + config: NewHashPartitionConfig(8, "user_id", "tenant_id"), + expected: "PARTITION BY HASH (user_id, tenant_id)", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.config.ToSQL() + if got != tt.expected { + t.Errorf("PartitionConfig.ToSQL() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestPrimaryKeyConstraint(t *testing.T) { + // This test documents that primary keys should get PRIMARY KEY constraint + // and NOT NULL constraint in the generated SQL + + // In CreatePartitionedTable: + // 1. Collect all primary key columns + // 2. Add NOT NULL to each primary key column definition + // 3. Add PRIMARY KEY (col1, col2, ...) constraint + // 4. For partitioned tables, ensure partition columns are in the primary key + + type TestModel struct { + ID uint `gorm:"primaryKey"` // Should get NOT NULL and be in PRIMARY KEY constraint + Name string `gorm:"not null"` // Should get NOT NULL from explicit tag + Age int // Should NOT get NOT NULL + CreatedAt string // For partition column + } + + // Verify the struct can be instantiated + var model TestModel + model.ID = 1 + + if model.ID != 1 { + t.Error("model instantiation failed") + } + + // The expected SQL should contain: + // - id bigint NOT NULL + // - PRIMARY KEY (id, created_at) -- includes partition column + // This is verified in integration tests with actual database +} + +func TestAutoIncrementHandling(t *testing.T) { + // This test documents that AutoIncrement fields should get GENERATED BY DEFAULT AS IDENTITY + // and AutoIncrementIncrement should be respected + + // In CreatePartitionedTable: + // 1. Check if field.AutoIncrement is true + // 2. If yes, add GENERATED BY DEFAULT AS IDENTITY + // 3. If AutoIncrementIncrement > 0, add INCREMENT BY clause + // 4. IDENTITY columns are automatically NOT NULL + + type TestModelWithAutoIncrement struct { + ID uint `gorm:"primaryKey;autoIncrement"` // Should get GENERATED BY DEFAULT AS IDENTITY + Name string `gorm:"not null"` + CreatedAt string // For partition column + } + + type TestModelWithIncrementBy struct { + ID uint `gorm:"primaryKey;autoIncrement;autoIncrementIncrement:10"` // Should get INCREMENT BY 10 + Name string `gorm:"not null"` + CreatedAt string + } + + // Verify the structs can be instantiated + var model1 TestModelWithAutoIncrement + model1.Name = "test" + + var model2 TestModelWithIncrementBy + model2.Name = "test" + + if model1.Name != "test" || model2.Name != "test" { + t.Error("model instantiation failed") + } + + // The expected SQL should contain: + // For TestModelWithAutoIncrement: + // - id bigint GENERATED BY DEFAULT AS IDENTITY + // + // For TestModelWithIncrementBy: + // - id bigint GENERATED BY DEFAULT AS IDENTITY (INCREMENT BY 10) + // + // This is verified in integration tests with actual database +} + +func TestGormTypeToPostgresType(t *testing.T) { + tests := []struct { + name string + gormType string + expected string + }{ + // Integer types + { + name: "uint to bigint", + gormType: "uint", + expected: "bigint", + }, + { + name: "uint8 to smallint", + gormType: "uint8", + expected: "smallint", + }, + { + name: "uint16 to integer", + gormType: "uint16", + expected: "integer", + }, + { + name: "uint32 to bigint", + gormType: "uint32", + expected: "bigint", + }, + { + name: "uint64 to bigint", + gormType: "uint64", + expected: "bigint", + }, + { + name: "int to bigint", + gormType: "int", + expected: "bigint", + }, + { + name: "int64 to bigint", + gormType: "int64", + expected: "bigint", + }, + // Float types + { + name: "float to double precision", + gormType: "float", + expected: "double precision", + }, + { + name: "float32 to real", + gormType: "float32", + expected: "real", + }, + { + name: "float64 to double precision", + gormType: "float64", + expected: "double precision", + }, + // String types + { + name: "string to text", + gormType: "string", + expected: "text", + }, + // Boolean + { + name: "bool to boolean", + gormType: "bool", + expected: "boolean", + }, + // Time types + { + name: "time.time to timestamptz", + gormType: "time.time", + expected: "timestamp with time zone", + }, + { + name: "time to timestamptz", + gormType: "time", + expected: "timestamp with time zone", + }, + // Binary + { + name: "[]byte to bytea", + gormType: "[]byte", + expected: "bytea", + }, + // JSON + { + name: "json to jsonb", + gormType: "json", + expected: "jsonb", + }, + // PostgreSQL types should pass through + { + name: "varchar remains varchar", + gormType: "varchar", + expected: "varchar", + }, + { + name: "character varying remains", + gormType: "character varying", + expected: "character varying", + }, + { + name: "timestamptz remains", + gormType: "timestamptz", + expected: "timestamptz", + }, + // Case insensitive + { + name: "UINT to bigint", + gormType: "UINT", + expected: "bigint", + }, + { + name: "String to text", + gormType: "String", + expected: "text", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := gormTypeToPostgresType(tt.gormType) + if got != tt.expected { + t.Errorf("gormTypeToPostgresType(%q) = %q, want %q", tt.gormType, got, tt.expected) + } + }) + } +} + +func TestColumnDeduplication(t *testing.T) { + // This test documents that CreatePartitionedTable and UpdatePartitionedTable + // deduplicate columns to prevent the same column from appearing multiple times + // in the generated SQL. + + // GORM's stmt.Schema.Fields can contain duplicate fields in certain cases: + // - Embedded structs with same field names + // - Field tags that create virtual fields + // - Polymorphic associations + // - Custom scanners/valuers + + // Example GORM model that might produce duplicates: + // type Model struct { + // gorm.Model // Contains CreatedAt, UpdatedAt, DeletedAt + // CreatedAt time.Time `gorm:"index"` // Duplicate! + // DeletedAt gorm.DeletedAt + // } + + // Without deduplication, this would generate: + // CREATE TABLE (... + // created_at timestamp with time zone, + // updated_at timestamp with time zone, + // deleted_at timestamp with time zone, + // created_at timestamp with time zone, -- DUPLICATE! + // deleted_at timestamp with time zone, -- DUPLICATE! + // ... + // ) + + // With deduplication (current implementation): + // CREATE TABLE (... + // created_at timestamp with time zone, + // updated_at timestamp with time zone, + // deleted_at timestamp with time zone, + // ... // No duplicates + // ) + + // The deduplication logic uses a map to track which columns have been added: + // - addedColumns[field.DBName] tracks columns in CreatePartitionedTable + // - processedColumns[field.DBName] tracks columns in UpdatePartitionedTable + // - First occurrence of a column is used, subsequent duplicates are skipped + + t.Log("Column deduplication documented - prevents duplicate columns in generated SQL") +} diff --git a/pkg/db/utils.go b/pkg/db/utils.go new file mode 100644 index 000000000..52e6d2a64 --- /dev/null +++ b/pkg/db/utils.go @@ -0,0 +1,771 @@ +package db + +import ( + "database/sql" + "errors" + "fmt" + "strings" + "time" + + log "github.com/sirupsen/logrus" +) + +// ColumnInfo represents metadata about a database column +type ColumnInfo struct { + ColumnName string + DataType string + IsNullable string + ColumnDefault sql.NullString + OrdinalPos int +} + +// PartitionStrategy defines the partitioning strategy type +type PartitionStrategy string + +const ( + // PartitionStrategyRange partitions by value ranges (e.g., date ranges) + PartitionStrategyRange PartitionStrategy = "RANGE" + // PartitionStrategyList partitions by discrete value lists + PartitionStrategyList PartitionStrategy = "LIST" + // PartitionStrategyHash partitions by hash of partition key + PartitionStrategyHash PartitionStrategy = "HASH" +) + +// ColumnVerificationOptions controls which aspects of column definitions to verify +type ColumnVerificationOptions struct { + // CheckNullable verifies that columns have matching nullable constraints + CheckNullable bool + // CheckDefaults verifies that columns have matching default values + CheckDefaults bool + // CheckOrder verifies that columns are in the same ordinal position + CheckOrder bool +} + +// DefaultColumnVerificationOptions returns options with all checks enabled +func DefaultColumnVerificationOptions() ColumnVerificationOptions { + return ColumnVerificationOptions{ + CheckNullable: true, + CheckDefaults: true, + CheckOrder: true, + } +} + +// DataMigrationColumnVerificationOptions returns options suitable for data migrations +// (only checks column names and types, not constraints or defaults) +func DataMigrationColumnVerificationOptions() ColumnVerificationOptions { + return ColumnVerificationOptions{ + CheckNullable: false, + CheckDefaults: false, + CheckOrder: true, + } +} + +// VerifyTablesHaveSameColumns verifies that two tables have identical column definitions +// Returns nil if the tables have the same columns, or an error describing the differences +// +// This function checks column names and data types by default. Use options parameter +// to control whether nullable constraints, default values, and column order are verified. +func (dbc *DB) VerifyTablesHaveSameColumns(table1, table2 string, opts ColumnVerificationOptions) error { + log.WithFields(log.Fields{ + "table1": table1, + "table2": table2, + }).Debug("verifying tables have same columns") + + // Get columns for both tables + cols1, err := dbc.GetTableColumns(table1) + if err != nil { + return fmt.Errorf("failed to get columns for table %s: %w", table1, err) + } + + cols2, err := dbc.GetTableColumns(table2) + if err != nil { + return fmt.Errorf("failed to get columns for table %s: %w", table2, err) + } + + // Check if column counts match + if len(cols1) != len(cols2) { + return fmt.Errorf("column count mismatch: %s has %d columns, %s has %d columns", + table1, len(cols1), table2, len(cols2)) + } + + // Create maps for easier comparison + cols1Map := make(map[string]ColumnInfo) + for _, col := range cols1 { + cols1Map[col.ColumnName] = col + } + + cols2Map := make(map[string]ColumnInfo) + for _, col := range cols2 { + cols2Map[col.ColumnName] = col + } + + // Check for missing columns + var missingInTable2 []string + for colName := range cols1Map { + if _, exists := cols2Map[colName]; !exists { + missingInTable2 = append(missingInTable2, colName) + } + } + + var missingInTable1 []string + for colName := range cols2Map { + if _, exists := cols1Map[colName]; !exists { + missingInTable1 = append(missingInTable1, colName) + } + } + + if len(missingInTable1) > 0 || len(missingInTable2) > 0 { + var errMsg strings.Builder + errMsg.WriteString("column name mismatch:") + if len(missingInTable2) > 0 { + errMsg.WriteString(fmt.Sprintf(" columns in %s but not in %s: %v;", + table1, table2, missingInTable2)) + } + if len(missingInTable1) > 0 { + errMsg.WriteString(fmt.Sprintf(" columns in %s but not in %s: %v", + table2, table1, missingInTable1)) + } + return errors.New(errMsg.String()) + } + + // Compare column definitions for matching columns + var differences []string + for colName, col1 := range cols1Map { + col2 := cols2Map[colName] + + // Normalize data types for comparison + type1 := normalizeDataType(col1.DataType) + type2 := normalizeDataType(col2.DataType) + + if !strings.EqualFold(type1, type2) { + differences = append(differences, + fmt.Sprintf("column %s: type mismatch (%s: %s vs %s: %s)", + colName, table1, col1.DataType, table2, col2.DataType)) + } + + // Optional: Check nullable constraints + if opts.CheckNullable && col1.IsNullable != col2.IsNullable { + differences = append(differences, + fmt.Sprintf("column %s: nullable mismatch (%s: %s vs %s: %s)", + colName, table1, col1.IsNullable, table2, col2.IsNullable)) + } + + // Optional: Compare defaults + if opts.CheckDefaults { + default1 := "" + if col1.ColumnDefault.Valid { + default1 = col1.ColumnDefault.String + } + default2 := "" + if col2.ColumnDefault.Valid { + default2 = col2.ColumnDefault.String + } + + if default1 != default2 { + differences = append(differences, + fmt.Sprintf("column %s: default mismatch (%s: %q vs %s: %q)", + colName, table1, default1, table2, default2)) + } + } + + // Optional: Check ordinal position (column order) + if opts.CheckOrder && col1.OrdinalPos != col2.OrdinalPos { + differences = append(differences, + fmt.Sprintf("column %s: position mismatch (%s: pos %d vs %s: pos %d)", + colName, table1, col1.OrdinalPos, table2, col2.OrdinalPos)) + } + } + + if len(differences) > 0 { + return fmt.Errorf("column definition mismatches:\n - %s", + strings.Join(differences, "\n - ")) + } + + log.WithFields(log.Fields{ + "table1": table1, + "table2": table2, + "count": len(cols1), + }).Info("tables have identical columns") + + return nil +} + +// GetTableColumns retrieves column information for a table from information_schema +func (dbc *DB) GetTableColumns(tableName string) ([]ColumnInfo, error) { + var columns []ColumnInfo + + query := ` + SELECT + column_name, + data_type, + is_nullable, + column_default, + ordinal_position + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = @table_name + ORDER BY ordinal_position + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&columns) + if result.Error != nil { + return nil, fmt.Errorf("failed to query columns for table %s: %w", tableName, result.Error) + } + + if len(columns) == 0 { + return nil, fmt.Errorf("table %s does not exist or has no columns", tableName) + } + + return columns, nil +} + +// normalizeDataType normalizes PostgreSQL data type names for comparison +func normalizeDataType(dataType string) string { + dataType = strings.ToLower(strings.TrimSpace(dataType)) + + // Map common type variations to standard forms + typeMap := map[string]string{ + "character varying": "varchar", + "integer": "int", + "int4": "int", + "int8": "bigint", + "bigserial": "bigint", + "serial": "int", + "timestamp without time zone": "timestamp", + "timestamp with time zone": "timestamptz", + "double precision": "float8", + "boolean": "bool", + } + + if normalized, exists := typeMap[dataType]; exists { + return normalized + } + + return dataType +} + +// MigrateTableData migrates all data from sourceTable to targetTable after verifying schemas match +// This function performs the following steps: +// 1. Verifies that both tables have identical column definitions +// 2. Checks row counts in both tables +// 3. Copies all data from source to target using INSERT INTO ... SELECT +// 4. Verifies row counts after migration +// +// Parameters: +// - sourceTable: The table to copy data from +// - targetTable: The table to copy data to +// - dryRun: If true, only verifies schemas and reports what would be migrated without actually copying data +// +// Returns: +// - rowsMigrated: The number of rows successfully migrated (0 if dryRun is true) +// - error: Any error encountered during migration +func (dbc *DB) MigrateTableData(sourceTable, targetTable string, dryRun bool) (int64, error) { + log.WithFields(log.Fields{ + "source": sourceTable, + "target": targetTable, + "dry_run": dryRun, + }).Info("starting table data migration") + + // Step 1: Verify schemas match + // For data migration, we only need to verify column names and types + // Nullable constraints and defaults don't affect the migration itself + if err := dbc.VerifyTablesHaveSameColumns(sourceTable, targetTable, DataMigrationColumnVerificationOptions()); err != nil { + return 0, fmt.Errorf("schema verification failed: %w", err) + } + + log.Info("schema verification passed - tables have identical column definitions") + + // Step 2: Get row counts before migration + sourceCount, err := dbc.GetTableRowCount(sourceTable) + if err != nil { + return 0, fmt.Errorf("failed to get source table row count: %w", err) + } + + targetCountBefore, err := dbc.GetTableRowCount(targetTable) + if err != nil { + return 0, fmt.Errorf("failed to get target table row count: %w", err) + } + + log.WithFields(log.Fields{ + "source_rows": sourceCount, + "target_rows": targetCountBefore, + }).Info("row counts before migration") + + if sourceCount == 0 { + log.Warn("source table is empty - nothing to migrate") + return 0, nil + } + + // Step 3: Dry run - report what would be migrated + if dryRun { + log.WithFields(log.Fields{ + "source_table": sourceTable, + "target_table": targetTable, + "rows_to_copy": sourceCount, + "target_current": targetCountBefore, + }).Info("[DRY RUN] would migrate data") + return 0, nil + } + + // Step 4: Get column names for the INSERT statement + columns, err := dbc.GetTableColumns(sourceTable) + if err != nil { + return 0, fmt.Errorf("failed to get column list: %w", err) + } + + var columnNames []string + for _, col := range columns { + columnNames = append(columnNames, col.ColumnName) + } + + // Step 5: Perform the migration using INSERT INTO ... SELECT + // This is done in a single statement for efficiency and atomicity + insertSQL := fmt.Sprintf( + "INSERT INTO %s (%s) SELECT %s FROM %s", + targetTable, + strings.Join(columnNames, ", "), + strings.Join(columnNames, ", "), + sourceTable, + ) + + log.WithFields(log.Fields{ + "source": sourceTable, + "target": targetTable, + "rows": sourceCount, + }).Info("migrating data") + + result := dbc.DB.Exec(insertSQL) + if result.Error != nil { + return 0, fmt.Errorf("data migration failed: %w", result.Error) + } + + rowsAffected := result.RowsAffected + + // Step 6: Verify migration success + targetCountAfter, err := dbc.GetTableRowCount(targetTable) + if err != nil { + return rowsAffected, fmt.Errorf("migration completed but failed to verify: %w", err) + } + + expectedCount := targetCountBefore + sourceCount + if targetCountAfter != expectedCount { + log.WithFields(log.Fields{ + "expected": expectedCount, + "actual": targetCountAfter, + "source": sourceCount, + "target": targetCountBefore, + }).Warn("row count mismatch after migration") + } + + log.WithFields(log.Fields{ + "source_table": sourceTable, + "target_table": targetTable, + "rows_migrated": rowsAffected, + "target_count_before": targetCountBefore, + "target_count_after": targetCountAfter, + }).Info("data migration completed successfully") + + return rowsAffected, nil +} + +// MigrateTableDataRange migrates data within a specific date range from sourceTable to targetTable +// This function performs the following steps: +// 1. Verifies that both tables have identical column definitions +// 2. Checks if target table is partitioned and verifies partition coverage for the date range +// 3. Counts rows in the date range +// 4. Copies data within the date range from source to target using INSERT INTO ... SELECT ... WHERE +// 5. Verifies row counts after migration +// +// If the target table is RANGE partitioned, the function automatically verifies that all necessary +// partitions exist for the date range being migrated. This prevents migration failures due to missing partitions. +// +// Parameters: +// - sourceTable: The table to copy data from +// - targetTable: The table to copy data to +// - dateColumn: The column name to filter by date range (e.g., "created_at") +// - startDate: Start of date range (inclusive) +// - endDate: End of date range (exclusive) +// - dryRun: If true, only verifies schemas and reports what would be migrated without actually copying data +// +// Returns: +// - rowsMigrated: The number of rows successfully migrated (0 if dryRun is true) +// - error: Any error encountered during migration +// +// Example: +// +// startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +// endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) +// rows, err := dbc.MigrateTableDataRange("old_table", "new_table", "created_at", startDate, endDate, false) +func (dbc *DB) MigrateTableDataRange(sourceTable, targetTable, dateColumn string, startDate, endDate time.Time, dryRun bool) (int64, error) { + log.WithFields(log.Fields{ + "source": sourceTable, + "target": targetTable, + "date_column": dateColumn, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + "dry_run": dryRun, + }).Info("starting table data migration for date range") + + // Validate date range + if endDate.Before(startDate) { + return 0, fmt.Errorf("end date (%s) cannot be before start date (%s)", + endDate.Format("2006-01-02"), startDate.Format("2006-01-02")) + } + + // Step 1: Verify schemas match + // For data migration, we only need to verify column names and types + // Nullable constraints and defaults don't affect the migration itself + if err := dbc.VerifyTablesHaveSameColumns(sourceTable, targetTable, DataMigrationColumnVerificationOptions()); err != nil { + return 0, fmt.Errorf("schema verification failed: %w", err) + } + + log.Info("schema verification passed - tables have identical column definitions") + + // Step 2: Check if target table is partitioned and verify partition coverage + partitionStrategy, err := dbc.GetPartitionStrategy(targetTable) + if err != nil { + return 0, fmt.Errorf("failed to check if target table is partitioned: %w", err) + } + + if partitionStrategy != "" { + log.WithFields(log.Fields{ + "table": targetTable, + "strategy": partitionStrategy, + }).Info("target table is partitioned - verifying partition coverage") + + // For RANGE partitioned tables, verify that partitions exist for the date range + if partitionStrategy == PartitionStrategyRange { + if err := dbc.VerifyPartitionCoverage(targetTable, startDate, endDate); err != nil { + return 0, fmt.Errorf("partition coverage verification failed: %w", err) + } + log.Info("partition coverage verified - all required partitions exist") + } else { + log.WithField("strategy", partitionStrategy).Warn("target table uses non-RANGE partitioning - skipping partition coverage check") + } + } + + // Step 3: Count rows in the date range in source table + var sourceCount int64 + countQuery := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s >= @start_date AND %s < @end_date", + sourceTable, dateColumn, dateColumn) + result := dbc.DB.Raw(countQuery, sql.Named("start_date", startDate), sql.Named("end_date", endDate)).Scan(&sourceCount) + if result.Error != nil { + return 0, fmt.Errorf("failed to count rows in date range: %w", result.Error) + } + + // Get total target row count before migration + targetCountBefore, err := dbc.GetTableRowCount(targetTable) + if err != nil { + return 0, fmt.Errorf("failed to get target table row count: %w", err) + } + + log.WithFields(log.Fields{ + "source_rows_in_range": sourceCount, + "target_rows": targetCountBefore, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("row counts before migration") + + if sourceCount == 0 { + log.Warn("no rows in date range - nothing to migrate") + return 0, nil + } + + // Step 4: Dry run - report what would be migrated + if dryRun { + log.WithFields(log.Fields{ + "source_table": sourceTable, + "target_table": targetTable, + "rows_to_copy": sourceCount, + "target_current": targetCountBefore, + "date_range": fmt.Sprintf("%s to %s", startDate.Format("2006-01-02"), endDate.Format("2006-01-02")), + }).Info("[DRY RUN] would migrate data") + return 0, nil + } + + // Step 5: Get column names for the INSERT statement + columns, err := dbc.GetTableColumns(sourceTable) + if err != nil { + return 0, fmt.Errorf("failed to get column list: %w", err) + } + + var columnNames []string + for _, col := range columns { + columnNames = append(columnNames, col.ColumnName) + } + + // Step 6: Perform the migration using INSERT INTO ... SELECT ... WHERE + // This is done in a single statement for efficiency and atomicity + insertSQL := fmt.Sprintf( + "INSERT INTO %s (%s) SELECT %s FROM %s WHERE %s >= @start_date AND %s < @end_date", + targetTable, + strings.Join(columnNames, ", "), + strings.Join(columnNames, ", "), + sourceTable, + dateColumn, + dateColumn, + ) + + log.WithFields(log.Fields{ + "source": sourceTable, + "target": targetTable, + "rows": sourceCount, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("migrating data in date range") + + result = dbc.DB.Exec(insertSQL, sql.Named("start_date", startDate), sql.Named("end_date", endDate)) + if result.Error != nil { + return 0, fmt.Errorf("data migration failed: %w", result.Error) + } + + rowsAffected := result.RowsAffected + + // Step 7: Verify migration success + targetCountAfter, err := dbc.GetTableRowCount(targetTable) + if err != nil { + return rowsAffected, fmt.Errorf("migration completed but failed to verify: %w", err) + } + + expectedCount := targetCountBefore + sourceCount + if targetCountAfter != expectedCount { + log.WithFields(log.Fields{ + "expected": expectedCount, + "actual": targetCountAfter, + "source_in_range": sourceCount, + "target_before": targetCountBefore, + "rows_actually_copied": rowsAffected, + }).Warn("row count mismatch after migration") + } + + log.WithFields(log.Fields{ + "source_table": sourceTable, + "target_table": targetTable, + "rows_migrated": rowsAffected, + "target_count_before": targetCountBefore, + "target_count_after": targetCountAfter, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("data migration completed successfully") + + return rowsAffected, nil +} + +// GetPartitionStrategy checks if a table is partitioned and returns its partition strategy +// Returns empty string ("") if table is not partitioned +// Returns PartitionStrategyRange, PartitionStrategyList, PartitionStrategyHash, or "UNKNOWN" if partitioned +// +// Example: +// +// strategy, err := dbc.GetPartitionStrategy("orders") +// if err != nil { +// return err +// } +// if strategy == PartitionStrategyRange { +// // Handle RANGE partitioned table +// } +func (dbc *DB) GetPartitionStrategy(tableName string) (PartitionStrategy, error) { + var strategy string + + query := ` + SELECT + CASE pp.partstrat + WHEN 'r' THEN 'RANGE' + WHEN 'l' THEN 'LIST' + WHEN 'h' THEN 'HASH' + ELSE 'UNKNOWN' + END AS partition_strategy + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_partitioned_table pp ON pp.partrelid = c.oid + WHERE n.nspname = 'public' + AND c.relname = @table_name + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&strategy) + if result.Error != nil { + return "", fmt.Errorf("failed to check partition strategy: %w", result.Error) + } + + // If no rows returned, table is not partitioned + if result.RowsAffected == 0 { + return "", nil + } + + return PartitionStrategy(strategy), nil +} + +// partitionDateInfo holds date range information for a partition +type partitionDateInfo struct { + PartitionName string + PartitionDate time.Time +} + +// getPartitionsInDateRange returns all partitions that cover a date range +// Assumes daily partitions with naming convention: tablename_YYYY_MM_DD +func (dbc *DB) getPartitionsInDateRange(tableName string, startDate, endDate time.Time) ([]partitionDateInfo, error) { + var partitions []partitionDateInfo + + // Prepare patterns in Go code since named parameters can't be concatenated in SQL + likePattern := tableName + "_%" + regexPattern := tableName + "_\\d{4}_\\d{2}_\\d{2}$" + + query := ` + SELECT + tablename AS partition_name, + TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date + FROM pg_tables + WHERE schemaname = 'public' + AND tablename LIKE @like_pattern + AND tablename ~ @regex_pattern + ORDER BY partition_date + ` + + result := dbc.DB.Raw(query, + sql.Named("like_pattern", likePattern), + sql.Named("regex_pattern", regexPattern), + ).Scan(&partitions) + if result.Error != nil { + return nil, fmt.Errorf("failed to query partitions: %w", result.Error) + } + + // Filter to only partitions in the date range + var filtered []partitionDateInfo + for _, p := range partitions { + if (p.PartitionDate.Equal(startDate) || p.PartitionDate.After(startDate)) && p.PartitionDate.Before(endDate) { + filtered = append(filtered, p) + } + } + + return filtered, nil +} + +// VerifyPartitionCoverage verifies that all necessary partitions exist for a date range +// Assumes daily partitions with naming convention: tablename_YYYY_MM_DD +// +// This function is useful before migrating data to partitioned tables to ensure +// all required partitions exist, preventing INSERT failures. +func (dbc *DB) VerifyPartitionCoverage(tableName string, startDate, endDate time.Time) error { + partitions, err := dbc.getPartitionsInDateRange(tableName, startDate, endDate) + if err != nil { + return fmt.Errorf("failed to get partitions: %w", err) + } + + // Create a map of existing partition dates for quick lookup + existingDates := make(map[string]bool) + for _, p := range partitions { + dateStr := p.PartitionDate.Format("2006-01-02") + existingDates[dateStr] = true + } + + // Check that we have a partition for each day in the range + var missingDates []string + currentDate := startDate + for currentDate.Before(endDate) { + dateStr := currentDate.Format("2006-01-02") + if !existingDates[dateStr] { + missingDates = append(missingDates, dateStr) + } + currentDate = currentDate.AddDate(0, 0, 1) // Move to next day + } + + if len(missingDates) > 0 { + return fmt.Errorf("missing partitions for dates: %v", missingDates) + } + + log.WithFields(log.Fields{ + "table": tableName, + "partition_count": len(partitions), + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("verified partition coverage for date range") + + return nil +} + +// GetTableRowCount returns the number of rows in a table +// This is useful for: +// - Verifying table size before operations +// - Comparing source and target tables during migration +// - Monitoring table growth +func (dbc *DB) GetTableRowCount(tableName string) (int64, error) { + var count int64 + + query := fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName) + result := dbc.DB.Raw(query).Scan(&count) + if result.Error != nil { + return 0, fmt.Errorf("failed to count rows in table %s: %w", tableName, result.Error) + } + + return count, nil +} + +// SyncIdentityColumn synchronizes the IDENTITY sequence for a column to match the current maximum value +// This is useful after migrating data to a partitioned table that uses IDENTITY columns +// +// NOTE: PostgreSQL does not have a SYNC IDENTITY command. Instead, this function uses +// ALTER TABLE ... ALTER COLUMN ... RESTART WITH, which is the standard PostgreSQL syntax +// for resetting an IDENTITY column's sequence to a specific value. +// +// Parameters: +// - tableName: Name of the table containing the IDENTITY column +// - columnName: Name of the IDENTITY column to sync (typically "id") +// +// The function executes: ALTER TABLE table_name ALTER COLUMN column_name RESTART WITH (max_value + 1) +// where max_value is the current maximum value in the column. +// +// Use cases: +// - After migrating data from a non-partitioned table to a partitioned table +// - After bulk inserting data with explicit IDs +// - When IDENTITY sequence is out of sync with actual data +// +// Example: +// +// err := dbc.SyncIdentityColumn("my_table", "id") +// if err != nil { +// log.WithError(err).Error("failed to sync identity column") +// } +func (dbc *DB) SyncIdentityColumn(tableName, columnName string) error { + log.WithFields(log.Fields{ + "table": tableName, + "column": columnName, + }).Info("synchronizing identity column") + + // Get the current maximum value + var maxValue sql.NullInt64 + query := fmt.Sprintf("SELECT MAX(%s) FROM %s", columnName, tableName) + result := dbc.DB.Raw(query).Scan(&maxValue) + if result.Error != nil { + return fmt.Errorf("failed to get max value for %s.%s: %w", tableName, columnName, result.Error) + } + + // If table is empty or column has all NULL values, start at 1 + nextValue := int64(1) + if maxValue.Valid { + nextValue = maxValue.Int64 + 1 + } + + log.WithFields(log.Fields{ + "table": tableName, + "column": columnName, + "max_value": maxValue.Int64, + "next_value": nextValue, + }).Debug("restarting identity sequence") + + // Restart the identity sequence + // NOTE: PostgreSQL requires "RESTART WITH" for IDENTITY columns, not "SYNC IDENTITY" + // This is the standard way to synchronize an IDENTITY sequence in PostgreSQL + alterSQL := fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s RESTART WITH %d", tableName, columnName, nextValue) + result = dbc.DB.Exec(alterSQL) + if result.Error != nil { + return fmt.Errorf("failed to sync identity for %s.%s: %w", tableName, columnName, result.Error) + } + + log.WithFields(log.Fields{ + "table": tableName, + "column": columnName, + "next_value": nextValue, + }).Info("identity column synchronized successfully") + + return nil +} diff --git a/pkg/db/utils_example.go b/pkg/db/utils_example.go new file mode 100644 index 000000000..bae3e9810 --- /dev/null +++ b/pkg/db/utils_example.go @@ -0,0 +1,687 @@ +package db + +import ( + "time" + + log "github.com/sirupsen/logrus" +) + +// ExampleVerifyTablesHaveSameColumns demonstrates how to verify that two tables have identical columns +// +// This is useful for: +// - Verifying partition tables match the parent table structure +// - Ensuring schema consistency before data migration +// - Validating table clones or backups +// +// Usage: +// +// err := dbc.VerifyTablesHaveSameColumns("source_table", "target_table", DefaultColumnVerificationOptions()) +// if err != nil { +// log.WithError(err).Error("tables have different schemas") +// } +func ExampleVerifyTablesHaveSameColumns(dbc *DB, sourceTable, targetTable string) { + log.WithFields(log.Fields{ + "source": sourceTable, + "target": targetTable, + }).Info("verifying tables have identical columns") + + // Use default options to verify all aspects: names, types, nullable, defaults, and order + err := dbc.VerifyTablesHaveSameColumns(sourceTable, targetTable, DefaultColumnVerificationOptions()) + if err != nil { + log.WithError(err).Error("table schema verification failed") + return + } + + log.Info("tables have identical column definitions") +} + +// ExampleVerifyPartitionMatchesParent demonstrates verifying a partition matches its parent table +// +// This is particularly useful when: +// - Creating new partitions +// - Reattaching detached partitions +// - Validating partition structure after schema changes +// +// Usage: +// +// parentTable := "test_analysis_by_job_by_dates" +// partition := "test_analysis_by_job_by_dates_2024_01_15" +// ExampleVerifyPartitionMatchesParent(dbc, parentTable, partition) +func ExampleVerifyPartitionMatchesParent(dbc *DB, parentTable, partition string) { + log.WithFields(log.Fields{ + "parent": parentTable, + "partition": partition, + }).Info("verifying partition matches parent table structure") + + // Use default options to ensure partition fully matches parent table + err := dbc.VerifyTablesHaveSameColumns(parentTable, partition, DefaultColumnVerificationOptions()) + if err != nil { + log.WithError(err).Error("partition schema does not match parent table") + log.Error("this partition may have been created with an old schema or manually modified") + return + } + + log.Info("partition structure matches parent table - safe to attach") +} + +// ExampleVerifyBeforeMigration demonstrates verification before data migration +// +// Before migrating data from one table to another, it's critical to ensure +// the schemas match to avoid data loss or type conversion errors. +// +// Usage: +// +// ExampleVerifyBeforeMigration(dbc, "old_table", "new_table") +func ExampleVerifyBeforeMigration(dbc *DB, sourceTable, targetTable string) { + log.Info("preparing data migration") + + // Step 1: Verify schemas match + // For data migration, we only need column names and types to match + // Nullable and default constraints don't affect the data copy + err := dbc.VerifyTablesHaveSameColumns(sourceTable, targetTable, DataMigrationColumnVerificationOptions()) + if err != nil { + log.WithError(err).Error("cannot migrate: schema mismatch detected") + log.Error("resolve schema differences before proceeding with migration") + return + } + + log.Info("schema verification passed - safe to proceed with migration") + + // Step 2: Proceed with migration + // (migration code would go here) +} + +// ExampleVerifyMultipleTables demonstrates checking multiple tables against a reference +// +// This is useful for: +// - Verifying all partitions match the parent table +// - Checking multiple replicas or shards have identical schemas +// - Validating a set of tables after schema updates +// +// Usage: +// +// ExampleVerifyMultipleTables(dbc, "parent_table", []string{"partition_1", "partition_2", "partition_3"}) +func ExampleVerifyMultipleTables(dbc *DB, referenceTable string, tablesToCheck []string) { + log.WithFields(log.Fields{ + "reference": referenceTable, + "count": len(tablesToCheck), + }).Info("verifying multiple tables against reference") + + var failures []string + for _, table := range tablesToCheck { + // Use default options to fully verify schema consistency + err := dbc.VerifyTablesHaveSameColumns(referenceTable, table, DefaultColumnVerificationOptions()) + if err != nil { + log.WithError(err).WithField("table", table).Error("schema mismatch detected") + failures = append(failures, table) + } else { + log.WithField("table", table).Debug("schema matches reference") + } + } + + if len(failures) > 0 { + log.WithFields(log.Fields{ + "total": len(tablesToCheck), + "failures": len(failures), + "failed": failures, + }).Error("schema verification completed with failures") + } else { + log.WithField("count", len(tablesToCheck)).Info("all tables match reference schema") + } +} + +// ExampleMigrateTableData demonstrates basic table data migration +// +// This function: +// - Verifies schemas match before migration +// - Copies all data from source to target +// - Supports dry-run mode for safety +// - Verifies migration success +// +// Usage: +// +// rowsMigrated, err := dbc.MigrateTableData("old_table", "new_table", false) +func ExampleMigrateTableData(dbc *DB, sourceTable, targetTable string) { + log.WithFields(log.Fields{ + "source": sourceTable, + "target": targetTable, + }).Info("preparing table migration") + + // Step 1: Dry run first to verify and preview + log.Info("performing dry run") + _, err := dbc.MigrateTableData(sourceTable, targetTable, true) + if err != nil { + log.WithError(err).Error("dry run failed - cannot proceed with migration") + return + } + + log.Info("dry run successful - proceeding with actual migration") + + // Step 2: Perform actual migration + rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, false) + if err != nil { + log.WithError(err).Error("migration failed") + return + } + + log.WithField("rows", rowsMigrated).Info("migration completed successfully") +} + +// ExampleMigratePartitionData demonstrates migrating data from a detached partition to a new table +// +// Use case: You have a detached partition with old data that needs to be migrated +// to a new table structure or archive table. +// +// Usage: +// +// ExampleMigratePartitionData(dbc, "test_table_2024_01_15", "archive_table") +func ExampleMigratePartitionData(dbc *DB, detachedPartition, archiveTable string) { + log.WithFields(log.Fields{ + "partition": detachedPartition, + "archive": archiveTable, + }).Info("migrating detached partition to archive") + + // Verify the partition is actually detached (optional safety check) + // This would use functions from pkg/db/partitions if available + + // Migrate the data + rowsMigrated, err := dbc.MigrateTableData(detachedPartition, archiveTable, false) + if err != nil { + log.WithError(err).Error("partition migration failed") + return + } + + log.WithFields(log.Fields{ + "partition": detachedPartition, + "archive": archiveTable, + "rows": rowsMigrated, + }).Info("partition data migrated to archive - safe to drop partition") +} + +// ExampleMigrateWithBackup demonstrates migrating data with a backup strategy +// +// Best practice: Create a backup before migration in case something goes wrong +// +// Usage: +// +// ExampleMigrateWithBackup(dbc, "source_table", "target_table", "backup_table") +func ExampleMigrateWithBackup(dbc *DB, sourceTable, targetTable, backupTable string) { + log.Info("migration with backup strategy") + + // Step 1: Create backup of target table + log.WithField("backup", backupTable).Info("creating backup of target table") + _, err := dbc.MigrateTableData(targetTable, backupTable, false) + if err != nil { + log.WithError(err).Error("backup creation failed - aborting migration") + return + } + + log.Info("backup created successfully") + + // Step 2: Perform migration + log.Info("performing migration") + rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, false) + if err != nil { + log.WithError(err).Error("migration failed - restore from backup if needed") + log.WithField("backup", backupTable).Info("backup table is available for restoration") + return + } + + log.WithField("rows", rowsMigrated).Info("migration completed successfully") + log.WithField("backup", backupTable).Info("backup table can be dropped if no longer needed") +} + +// ExampleBatchMigratePartitions demonstrates migrating multiple partitions +// +// Use case: You have multiple detached partitions that need to be migrated +// to an archive table or consolidated into a single table. +// +// Usage: +// +// partitions := []string{"table_2024_01_15", "table_2024_01_16", "table_2024_01_17"} +// ExampleBatchMigratePartitions(dbc, partitions, "archive_table") +func ExampleBatchMigratePartitions(dbc *DB, partitions []string, targetTable string) { + log.WithFields(log.Fields{ + "partitions": len(partitions), + "target": targetTable, + }).Info("batch migrating partitions") + + var totalRows int64 + var successCount int + var failures []string + + for _, partition := range partitions { + log.WithField("partition", partition).Info("migrating partition") + + rows, err := dbc.MigrateTableData(partition, targetTable, false) + if err != nil { + log.WithError(err).WithField("partition", partition).Error("partition migration failed") + failures = append(failures, partition) + continue + } + + totalRows += rows + successCount++ + log.WithFields(log.Fields{ + "partition": partition, + "rows": rows, + }).Info("partition migrated successfully") + } + + log.WithFields(log.Fields{ + "total_partitions": len(partitions), + "successful": successCount, + "failed": len(failures), + "total_rows": totalRows, + }).Info("batch migration completed") + + if len(failures) > 0 { + log.WithField("failed_partitions", failures).Warn("some partitions failed to migrate") + } +} + +// ExampleMigrateAndVerify demonstrates migration with comprehensive verification +// +// This example shows best practices for production migrations: +// - Dry run first +// - Verify schemas +// - Perform migration +// - Verify row counts +// - Log all steps +// +// Usage: +// +// ExampleMigrateAndVerify(dbc, "source_table", "target_table") +func ExampleMigrateAndVerify(dbc *DB, sourceTable, targetTable string) { + log.Info("production migration workflow") + + // Step 1: Verify schemas match + log.Info("step 1: verifying schema compatibility") + // For migration, we only need column names and types to match + if err := dbc.VerifyTablesHaveSameColumns(sourceTable, targetTable, DataMigrationColumnVerificationOptions()); err != nil { + log.WithError(err).Error("schema verification failed") + return + } + log.Info("schema verification passed") + + // Step 2: Get pre-migration counts + log.Info("step 2: getting pre-migration row counts") + sourceCount, err := dbc.GetTableRowCount(sourceTable) + if err != nil { + log.WithError(err).Error("failed to get source count") + return + } + targetCountBefore, err := dbc.GetTableRowCount(targetTable) + if err != nil { + log.WithError(err).Error("failed to get target count") + return + } + + log.WithFields(log.Fields{ + "source_rows": sourceCount, + "target_rows": targetCountBefore, + }).Info("pre-migration row counts") + + // Step 3: Dry run + log.Info("step 3: performing dry run") + _, err = dbc.MigrateTableData(sourceTable, targetTable, true) + if err != nil { + log.WithError(err).Error("dry run failed") + return + } + log.Info("dry run successful") + + // Step 4: Actual migration + log.Info("step 4: performing actual migration") + rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, false) + if err != nil { + log.WithError(err).Error("migration failed") + return + } + + // Step 5: Verify results + log.Info("step 5: verifying migration results") + targetCountAfter, err := dbc.GetTableRowCount(targetTable) + if err != nil { + log.WithError(err).Error("failed to verify final count") + return + } + + expectedCount := targetCountBefore + sourceCount + if targetCountAfter != expectedCount { + log.WithFields(log.Fields{ + "expected": expectedCount, + "actual": targetCountAfter, + }).Error("row count mismatch detected!") + return + } + + log.WithFields(log.Fields{ + "source_table": sourceTable, + "target_table": targetTable, + "rows_migrated": rowsMigrated, + "target_before": targetCountBefore, + "target_after": targetCountAfter, + "verification": "passed", + }).Info("migration completed and verified successfully") +} + +// ExampleSyncIdentityColumn demonstrates synchronizing an IDENTITY column sequence +// +// This is useful after migrating data to a table with IDENTITY columns, +// ensuring the sequence starts at the correct value. +// +// Usage: +// +// ExampleSyncIdentityColumn(dbc, "my_table", "id") +func ExampleSyncIdentityColumn(dbc *DB, tableName, columnName string) { + log.WithFields(log.Fields{ + "table": tableName, + "column": columnName, + }).Info("synchronizing identity column") + + // Sync the identity sequence to match the current max value + err := dbc.SyncIdentityColumn(tableName, columnName) + if err != nil { + log.WithError(err).Error("failed to sync identity column") + return + } + + log.Info("identity column synchronized successfully") +} + +// ExampleMigrateToPartitionedTable demonstrates the complete workflow for +// migrating from a non-partitioned table to a partitioned table +// +// Usage: +// +// ExampleMigrateToPartitionedTable(dbc, "orders", "orders_partitioned") +func ExampleMigrateToPartitionedTable(dbc *DB, sourceTable, partitionedTable string) { + log.Info("Complete workflow: Migrating to partitioned table") + + // Assume partitioned table was created using CreatePartitionedTable + // and partitions were created using CreateMissingPartitions + + // Step 1: Migrate the data + log.Info("Step 1: Migrating data") + rows, err := dbc.MigrateTableData(sourceTable, partitionedTable, false) + if err != nil { + log.WithError(err).Error("data migration failed") + return + } + + log.WithField("rows", rows).Info("data migrated successfully") + + // Step 2: Sync the IDENTITY column + log.Info("Step 2: Synchronizing IDENTITY sequence") + err = dbc.SyncIdentityColumn(partitionedTable, "id") + if err != nil { + log.WithError(err).Error("failed to sync identity column") + return + } + + // Step 3: Verify row counts match + log.Info("Step 3: Verifying row counts") + sourceCount, _ := dbc.GetTableRowCount(sourceTable) + targetCount, _ := dbc.GetTableRowCount(partitionedTable) + + if sourceCount != targetCount { + log.WithFields(log.Fields{ + "source": sourceCount, + "target": targetCount, + }).Error("row count mismatch!") + return + } + + log.WithFields(log.Fields{ + "source_table": sourceTable, + "partitioned_table": partitionedTable, + "rows": rows, + }).Info("migration to partitioned table completed successfully") + + // Next steps (manual): + // 1. Test the partitioned table thoroughly + // 2. Update application to use new table + // 3. After verification, drop the old table +} + +// ExampleMigrateTableDataRange demonstrates migrating data for a specific date range +// +// This is useful when: +// - Migrating data incrementally in smaller batches +// - Testing migrations with a subset of data +// - Moving specific time periods to archive tables +// - Migrating data to date-partitioned tables partition by partition +// +// Usage: +// +// startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +// endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) +// ExampleMigrateTableDataRange(dbc, "orders", "orders_archive", "created_at", startDate, endDate) +func ExampleMigrateTableDataRange(dbc *DB, sourceTable, targetTable, dateColumn string, startDate, endDate time.Time) { + log.WithFields(log.Fields{ + "source": sourceTable, + "target": targetTable, + "date_column": dateColumn, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("migrating data for date range") + + // Step 1: Dry run first to verify and preview + log.Info("performing dry run") + _, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, true) + if err != nil { + log.WithError(err).Error("dry run failed - cannot proceed with migration") + return + } + + log.Info("dry run successful - proceeding with actual migration") + + // Step 2: Perform actual migration + rowsMigrated, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, false) + if err != nil { + log.WithError(err).Error("migration failed") + return + } + + log.WithFields(log.Fields{ + "rows": rowsMigrated, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("migration completed successfully") +} + +// ExampleIncrementalMigrationByMonth demonstrates migrating data month by month +// +// This approach is useful for: +// - Large tables where migrating all at once would be too slow +// - Reducing lock contention by migrating in smaller batches +// - Being able to pause and resume migrations +// - Easier rollback if issues are detected +// +// Usage: +// +// ExampleIncrementalMigrationByMonth(dbc, "large_table", "large_table_new", "created_at", 2024) +func ExampleIncrementalMigrationByMonth(dbc *DB, sourceTable, targetTable, dateColumn string, year int) { + log.WithFields(log.Fields{ + "source": sourceTable, + "target": targetTable, + "year": year, + }).Info("starting incremental migration by month") + + var totalMigrated int64 + var failedMonths []string + + // Migrate data month by month + for month := 1; month <= 12; month++ { + startDate := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC) + endDate := startDate.AddDate(0, 1, 0) // First day of next month + + log.WithFields(log.Fields{ + "month": time.Month(month).String(), + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("migrating month") + + rows, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, false) + if err != nil { + log.WithError(err).WithField("month", time.Month(month).String()).Error("month migration failed") + failedMonths = append(failedMonths, time.Month(month).String()) + continue + } + + totalMigrated += rows + log.WithFields(log.Fields{ + "month": time.Month(month).String(), + "rows": rows, + }).Info("month migrated successfully") + } + + log.WithFields(log.Fields{ + "total_rows": totalMigrated, + "total_months": 12, + "failed_months": len(failedMonths), + }).Info("incremental migration completed") + + if len(failedMonths) > 0 { + log.WithField("failed_months", failedMonths).Warn("some months failed to migrate") + } +} + +// ExampleMigrateToPartitionByDateRange demonstrates migrating data to a specific partition +// +// This workflow is useful when: +// - You have a non-partitioned table and want to migrate to a partitioned structure +// - You want to populate partitions incrementally +// - You're backfilling historical data into partitions +// +// Important: MigrateTableDataRange automatically verifies that all necessary partitions +// exist for the date range being migrated. If the target table is RANGE partitioned and +// partitions are missing, the function will return an error before attempting migration. +// +// Usage: +// +// ExampleMigrateToPartitionByDateRange(dbc, "orders", "orders_partitioned", "order_date") +func ExampleMigrateToPartitionByDateRange(dbc *DB, sourceTable, partitionedTable, dateColumn string) { + log.Info("migrating data to partitioned table by date range") + + // Example: Migrate January 2024 data to the partition + startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) + + // Step 1: Migrate the data for this date range + // The function will automatically verify that partitions exist for all dates + // in the range [2024-01-01, 2024-02-01) before attempting the migration + log.WithFields(log.Fields{ + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("migrating date range to partition") + + rows, err := dbc.MigrateTableDataRange(sourceTable, partitionedTable, dateColumn, startDate, endDate, false) + if err != nil { + log.WithError(err).Error("migration failed") + return + } + + log.WithField("rows", rows).Info("data migrated to partition") + + // Step 2: Verify the data landed in the expected partition + // This would use partition-specific queries to verify + log.Info("verifying data distribution across partitions") + + // Step 3: Repeat for other date ranges as needed + log.Info("migration to partition completed - repeat for additional date ranges as needed") +} + +// ExampleGetPartitionStrategy demonstrates checking if a table is partitioned +// +// This is useful before performing operations that differ between partitioned +// and non-partitioned tables. +// +// Usage: +// +// ExampleGetPartitionStrategy(dbc, "orders") +func ExampleGetPartitionStrategy(dbc *DB, tableName string) { + log.WithField("table", tableName).Info("checking partition strategy") + + strategy, err := dbc.GetPartitionStrategy(tableName) + if err != nil { + log.WithError(err).Error("failed to check partition strategy") + return + } + + if strategy == "" { + log.Info("table is not partitioned") + // Proceed with normal table operations + } else { + log.WithField("strategy", strategy).Info("table is partitioned") + + switch strategy { + case PartitionStrategyRange: + log.Info("table uses RANGE partitioning - can use date-based partition operations") + case PartitionStrategyList: + log.Info("table uses LIST partitioning - partitioned by discrete values") + case PartitionStrategyHash: + log.Info("table uses HASH partitioning - partitioned by hash function") + default: + log.Warn("unknown partition strategy") + } + } +} + +// ExampleVerifyPartitionCoverage demonstrates verifying partition coverage before migration +// +// This workflow ensures all necessary partitions exist before attempting a data migration, +// preventing runtime failures due to missing partitions. +// +// Usage: +// +// ExampleVerifyPartitionCoverage(dbc, "orders", startDate, endDate) +func ExampleVerifyPartitionCoverage(dbc *DB, tableName string, startDate, endDate time.Time) { + log.WithFields(log.Fields{ + "table": tableName, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }).Info("verifying partition coverage") + + // Verify that all necessary partitions exist + err := dbc.VerifyPartitionCoverage(tableName, startDate, endDate) + if err != nil { + log.WithError(err).Error("partition coverage verification failed") + log.Error("missing partitions detected - cannot proceed with migration") + log.Info("create missing partitions using partitions.CreateMissingPartitions before retrying") + return + } + + log.Info("partition coverage verified - all required partitions exist") + log.Info("safe to proceed with data migration") +} + +// ExampleCheckAndCreatePartitions demonstrates checking for missing partitions and creating them +// +// This workflow combines partition verification with automatic creation of missing partitions. +// +// Note: This example shows the pattern but doesn't import the partitions package +// to avoid circular dependencies in the example file. +// +// Usage: +// +// ExampleCheckAndCreatePartitions(dbc, "orders", startDate, endDate) +func ExampleCheckAndCreatePartitions(dbc *DB, tableName string, startDate, endDate time.Time) { + log.Info("checking partition coverage and creating missing partitions") + + // Step 1: Check if partitions exist + err := dbc.VerifyPartitionCoverage(tableName, startDate, endDate) + if err != nil { + log.WithError(err).Warn("missing partitions detected") + + // Step 2: In actual usage, you would create missing partitions using: + // import "github.com/openshift/sippy/pkg/db/partitions" + // count, err := partitions.CreateMissingPartitions(dbc, tableName, startDate, endDate, false) + + log.Info("would create missing partitions here using partitions.CreateMissingPartitions") + return + } + + log.Info("all partitions exist - ready for operations") +} diff --git a/pkg/db/utils_test.go b/pkg/db/utils_test.go new file mode 100644 index 000000000..e34b00f4c --- /dev/null +++ b/pkg/db/utils_test.go @@ -0,0 +1,303 @@ +package db + +import ( + "database/sql" + "testing" +) + +func TestNormalizeDataType(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "character varying to varchar", + input: "character varying", + expected: "varchar", + }, + { + name: "integer to int", + input: "integer", + expected: "int", + }, + { + name: "int4 to int", + input: "int4", + expected: "int", + }, + { + name: "int8 to bigint", + input: "int8", + expected: "bigint", + }, + { + name: "bigserial to bigint", + input: "bigserial", + expected: "bigint", + }, + { + name: "timestamp without time zone", + input: "timestamp without time zone", + expected: "timestamp", + }, + { + name: "timestamp with time zone to timestamptz", + input: "timestamp with time zone", + expected: "timestamptz", + }, + { + name: "double precision to float8", + input: "double precision", + expected: "float8", + }, + { + name: "boolean to bool", + input: "boolean", + expected: "bool", + }, + { + name: "text remains text", + input: "text", + expected: "text", + }, + { + name: "uppercase INTEGER to int", + input: "INTEGER", + expected: "int", + }, + { + name: "mixed case Boolean to bool", + input: "Boolean", + expected: "bool", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := normalizeDataType(tt.input) + if got != tt.expected { + t.Errorf("normalizeDataType(%q) = %q, want %q", tt.input, got, tt.expected) + } + }) + } +} + +func TestColumnInfo(t *testing.T) { + // Test that ColumnInfo struct can be instantiated + col := ColumnInfo{ + ColumnName: "test_column", + DataType: "varchar", + IsNullable: "NO", + ColumnDefault: sql.NullString{String: "default_value", Valid: true}, + OrdinalPos: 1, + } + + if col.ColumnName != "test_column" { + t.Errorf("unexpected column name: %s", col.ColumnName) + } + + if col.DataType != "varchar" { + t.Errorf("unexpected data type: %s", col.DataType) + } + + if col.IsNullable != "NO" { + t.Errorf("unexpected nullable: %s", col.IsNullable) + } + + if !col.ColumnDefault.Valid || col.ColumnDefault.String != "default_value" { + t.Errorf("unexpected default: %v", col.ColumnDefault) + } + + if col.OrdinalPos != 1 { + t.Errorf("unexpected ordinal position: %d", col.OrdinalPos) + } +} + +// Note: Integration tests for MigrateTableData require a live database connection +// and would be in a separate integration test suite. Unit tests verify the +// basic structure and flow of the function. + +func TestMigrateTableDataValidation(t *testing.T) { + // This test documents the expected behavior and parameters + // Actual migration testing requires database fixtures + + type testCase struct { + name string + sourceTable string + targetTable string + dryRun bool + expectError bool + errorContains string + } + + tests := []testCase{ + { + name: "dry run mode", + sourceTable: "source_table", + targetTable: "target_table", + dryRun: true, + expectError: false, + }, + { + name: "actual migration", + sourceTable: "source_table", + targetTable: "target_table", + dryRun: false, + expectError: false, + }, + } + + // Document expected behavior for each test case + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test validates structure and parameters are correct + // Actual database testing would be done in integration tests + if tt.sourceTable == "" { + t.Error("source table should not be empty") + } + if tt.targetTable == "" { + t.Error("target table should not be empty") + } + }) + } +} + +func TestSyncIdentityColumn(t *testing.T) { + // This test documents the expected behavior of SyncIdentityColumn + // which synchronizes the IDENTITY sequence for a column to match the current maximum value + + // The function should: + // 1. Get the current maximum value from the column + // 2. Calculate the next value (max + 1, or 1 if table is empty) + // 3. Execute ALTER TABLE ... ALTER COLUMN ... RESTART WITH next_value + // 4. Log the operation with appropriate fields + + // Use cases: + // - After migrating data from non-partitioned to partitioned table + // - After bulk inserting data with explicit IDs + // - When IDENTITY sequence is out of sync + + // Example usage: + // err := dbc.SyncIdentityColumn("my_table", "id") + // if err != nil { + // log.WithError(err).Error("failed to sync identity column") + // } + + // Expected SQL for a table with max(id) = 100: + // ALTER TABLE my_table ALTER COLUMN id RESTART WITH 101 + + // Expected SQL for an empty table: + // ALTER TABLE my_table ALTER COLUMN id RESTART WITH 1 + + // This is a documentation test - actual functionality requires a live database + // and is tested in integration tests + t.Log("SyncIdentityColumn documented - integration tests required for full validation") +} + +func TestMigrateTableDataRange(t *testing.T) { + // This test documents the expected behavior of MigrateTableDataRange + // which migrates data within a specific date range from one table to another + + // The function should: + // 1. Verify schemas match between source and target tables + // 2. Check if target table is RANGE partitioned and verify partition coverage for the date range + // 3. Count rows in the source table within the date range + // 4. Execute INSERT INTO target SELECT * FROM source WHERE date_column >= start AND date_column < end + // 5. Verify row counts after migration + // 6. Support dry-run mode for testing + + // Use cases: + // - Migrating data incrementally in smaller batches + // - Testing migrations with a subset of data + // - Moving specific time periods to archive tables + // - Migrating data to date-partitioned tables partition by partition + + // Example usage: + // startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + // endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) + // rows, err := dbc.MigrateTableDataRange("orders", "orders_archive", "created_at", startDate, endDate, false) + // if err != nil { + // log.WithError(err).Error("migration failed") + // } + + // Expected behavior: + // - startDate is inclusive (>=) + // - endDate is exclusive (<) + // - Returns error if endDate is before startDate + // - Returns 0 rows if no data in range + // - Dry run mode returns 0 rows but validates everything else + // - If target is RANGE partitioned, verifies all partitions exist for the date range + // - Returns error if target is partitioned and partitions are missing for the date range + // - Skips partition check for non-RANGE partitioned tables (LIST, HASH) + + // This is a documentation test - actual functionality requires a live database + // and is tested in integration tests + t.Log("MigrateTableDataRange documented - integration tests required for full validation") +} + +func TestGetPartitionStrategy(t *testing.T) { + // This test documents the expected behavior of GetPartitionStrategy + // which checks if a table is partitioned and returns its partition strategy + + // The function should: + // 1. Query PostgreSQL system catalogs (pg_partitioned_table) + // 2. Return empty string ("") if table is not partitioned + // 3. Return PartitionStrategyRange, PartitionStrategyList, PartitionStrategyHash, or "UNKNOWN" + // 4. Handle non-existent tables gracefully + + // Example usage: + // strategy, err := dbc.GetPartitionStrategy("orders") + // if err != nil { + // log.WithError(err).Error("failed to check partition strategy") + // } + // if strategy == PartitionStrategyRange { + // // Table uses RANGE partitioning + // } + + // Expected behavior: + // - Returns "" for non-partitioned tables + // - Returns PartitionStrategyRange for RANGE partitioned tables (partstrat = 'r') + // - Returns PartitionStrategyList for LIST partitioned tables (partstrat = 'l') + // - Returns PartitionStrategyHash for HASH partitioned tables (partstrat = 'h') + // - Returns "UNKNOWN" for other partition strategies + // - Constants defined in pkg/db: PartitionStrategyRange, PartitionStrategyList, PartitionStrategyHash + + // This is a documentation test - actual functionality requires a live database + // and is tested in integration tests + t.Log("GetPartitionStrategy documented - integration tests required for full validation") +} + +func TestVerifyPartitionCoverage(t *testing.T) { + // This test documents the expected behavior of VerifyPartitionCoverage + // which verifies that all necessary partitions exist for a date range + + // The function should: + // 1. Query all partitions for the table + // 2. Check that a partition exists for each day in [startDate, endDate) + // 3. Return error listing missing partition dates if any are missing + // 4. Return nil if all partitions exist + // 5. Log successful verification with partition count + + // Assumptions: + // - Daily partitions with naming: tablename_YYYY_MM_DD + // - Partitions cover single calendar days + // - startDate is inclusive, endDate is exclusive + + // Example usage: + // err := dbc.VerifyPartitionCoverage("orders", startDate, endDate) + // if err != nil { + // // Error message: "missing partitions for dates: [2024-01-15 2024-01-16]" + // log.WithError(err).Error("partition coverage check failed") + // } + + // Expected behavior: + // - Returns nil if all partitions exist for the date range + // - Returns error if any partitions are missing + // - Error message includes list of missing dates + // - Useful before data migrations to partitioned tables + + // This is a documentation test - actual functionality requires a live database + // and is tested in integration tests + t.Log("VerifyPartitionCoverage documented - integration tests required for full validation") +} From 55418aae6539873cde89d43268fbc524ee667677 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Thu, 12 Mar 2026 14:55:52 -0400 Subject: [PATCH 03/11] TRT-1989: update table migration --- pkg/dataloader/prowloader/prow.go | 106 ++- pkg/db/UTILS_README.md | 1277 ++++++++++++++++++++++++++++- pkg/db/partitions/partitions.go | 152 +++- pkg/db/utils.go | 1096 ++++++++++++++++++++++++- pkg/db/utils_example.go | 376 ++++++++- pkg/db/utils_test.go | 177 ++++ 6 files changed, 3091 insertions(+), 93 deletions(-) diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index 1bd44eec3..8af429191 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -167,11 +167,64 @@ func (pl *ProwLoader) Errors() []error { return pl.errors } +// PartitionManagementConfig defines partition lifecycle settings for a table +type PartitionManagementConfig struct { + TableName string // Name of the partitioned table + FuturePartitionWindow time.Duration // How far in the future to create partitions + DetachAfter int // Detach partitions older than this many days + DropDetachedAfter int // Drop detached partitions older than this many days + InitialLookbackDays int // Days to look back when initializing a new table +} + +var partitionConfigs = []PartitionManagementConfig{ + { + TableName: "test_analysis_by_job_by_dates", + FuturePartitionWindow: 48 * time.Hour, + DetachAfter: 90, + DropDetachedAfter: 100, + InitialLookbackDays: 15, + }, + { + TableName: "prow_job_run_tests", + FuturePartitionWindow: 48 * time.Hour, + DetachAfter: 90, + DropDetachedAfter: 100, + InitialLookbackDays: 15, + }, +} + +func (pl *ProwLoader) updatePartitions(config PartitionManagementConfig) error { + err := pl.agePartitions(config) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("error aging %s", config.TableName)) + } + + err = pl.preparePartitions(config) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("error preparing %s", config.TableName)) + } + + return nil +} + func (pl *ProwLoader) Load() { start := time.Now() log.Infof("started loading prow jobs to DB...") + for _, config := range partitionConfigs { + err := pl.updatePartitions(config) + if err != nil { + pl.errors = append(pl.errors, err) + + // if we have errors with partition management we can't be sure that we have created + // the necessary partitions to proceed with loading + // we could possibly differentiate between removing old and creating new but for now + // any failures here block any loading + return + } + } + // Update unmerged PR statuses in case any have merged if err := pl.syncPRStatus(); err != nil { pl.errors = append(pl.errors, errors.Wrap(err, "error in syncPRStatus")) @@ -235,21 +288,9 @@ func (pl *ProwLoader) Load() { pl.errors = append(pl.errors, err) } - // detach and drop older partitions - err := pl.agePartitionsForDailyTestAnalysisByJob() - if err != nil { - pl.errors = append(pl.errors, errors.Wrap(err, "error aging daily test analysis by job table")) - } - - // create any partitions needed for new data to be imported - err = pl.preparePartitionsForDailyTestAnalysisByJob() - if err != nil { - pl.errors = append(pl.errors, errors.Wrap(err, "error preparing partitions for daily test analysis by job table")) - } - // load the test analysis by job data into tables partitioned by day, letting bigquery do the // heavy lifting for us. - err = pl.loadDailyTestAnalysisByJob(pl.ctx) + err := pl.loadDailyTestAnalysisByJob(pl.ctx) if err != nil { pl.errors = append(pl.errors, errors.Wrap(err, "error updating daily test analysis by job")) } @@ -358,37 +399,37 @@ func NextDay(dateStr string) (string, error) { return nextDay.Format("2006-01-02"), nil } -func (pl *ProwLoader) agePartitionsForDailyTestAnalysisByJob() error { - tableName := "test_analysis_by_job_by_dates" - detached, err := partitions.DetachOldPartitions(pl.dbc, tableName, 90, false) +// agePartitions detaches and drops old partitions based on configuration +func (pl *ProwLoader) agePartitions(config PartitionManagementConfig) error { + detached, err := partitions.DetachOldPartitions(pl.dbc, config.TableName, config.DetachAfter, false) if err != nil { - log.WithError(err).Errorf("error detaching partitions for %s", tableName) + log.WithError(err).Errorf("error detaching partitions for %s", config.TableName) } else { - log.Infof("detached %d partitions from %s", detached, tableName) + log.Infof("detached %d partitions from %s", detached, config.TableName) } - dropped, err := partitions.DropOldDetachedPartitions(pl.dbc, tableName, 100, false) + dropped, err := partitions.DropOldDetachedPartitions(pl.dbc, config.TableName, config.DropDetachedAfter, false) if err != nil { - log.WithError(err).Errorf("error detaching partitions for %s", tableName) + log.WithError(err).Errorf("error dropping detached partitions for %s", config.TableName) return err } - log.Infof("dropped %d detached partitions from %s", dropped, tableName) + log.Infof("dropped %d detached partitions from %s", dropped, config.TableName) return nil } -func (pl *ProwLoader) preparePartitionsForDailyTestAnalysisByJob() error { - tableName := "test_analysis_by_job_by_dates" - - log.Infof("preparing partitions for %s", tableName) - stats, err := partitions.GetAttachedPartitionStats(pl.dbc, tableName) +// preparePartitions creates missing partitions for future data based on configuration +func (pl *ProwLoader) preparePartitions(config PartitionManagementConfig) error { + log.Infof("preparing partitions for %s", config.TableName) + stats, err := partitions.GetAttachedPartitionStats(pl.dbc, config.TableName) if err != nil { - log.WithError(err).Errorf("error detaching partitions for %s", tableName) + log.WithError(err).Errorf("error getting partition stats for %s", config.TableName) return err } fmt.Printf(" Total: %d partitions (%s)\n", stats.TotalPartitions, stats.TotalSizePretty) - // when we initialize a new table the query goes back 14 days, go back one more - mostRecentDate := time.Now().Add(-15 * 24 * time.Hour) + + // When initializing a new table, look back the configured number of days + mostRecentDate := time.Now().Add(-time.Duration(config.InitialLookbackDays) * 24 * time.Hour) if stats.TotalPartitions > 0 { fmt.Printf(" Range: %s to %s\n", stats.OldestDate.Format("2006-01-02"), @@ -396,13 +437,14 @@ func (pl *ProwLoader) preparePartitionsForDailyTestAnalysisByJob() error { mostRecentDate = stats.NewestDate } - created, err := partitions.CreateMissingPartitions(pl.dbc, tableName, mostRecentDate, time.Now().Add(48*time.Hour), false) + futureDate := time.Now().Add(config.FuturePartitionWindow) + created, err := partitions.CreateMissingPartitions(pl.dbc, config.TableName, mostRecentDate, futureDate, false) if err != nil { - log.WithError(err).Errorf("error creating partitions for %s", tableName) + log.WithError(err).Errorf("error creating partitions for %s", config.TableName) return err } - log.Infof("created %d partitions for %s", created, tableName) + log.Infof("created %d partitions for %s", created, config.TableName) return nil } diff --git a/pkg/db/UTILS_README.md b/pkg/db/UTILS_README.md index df3911ef6..6c2c8ae7c 100644 --- a/pkg/db/UTILS_README.md +++ b/pkg/db/UTILS_README.md @@ -7,9 +7,42 @@ This package provides utility functions for database operations including schema The utilities in `utils.go` provide safe, validated operations for working with database tables, particularly useful for: - Schema migration and validation - Data migration between tables +- Atomic table renames and swaps +- Sequence management and auditing - Partition management workflows - Table consolidation and archival +## Quick Function Reference + +**Schema Verification:** +- `VerifyTablesHaveSameColumns` - Compare table schemas +- `GetTableColumns` - Get column metadata for a table + +**Data Migration:** +- `MigrateTableData` - Copy all data between tables +- `MigrateTableDataRange` - Copy data for specific date range +- `GetTableRowCount` - Count rows in a table + +**Table Renaming:** +- `RenameTables` - Atomically rename tables, sequences, and partitions + +**Sequence Management:** +- `GetSequenceMetadata` - Get detailed linkage info (SERIAL vs IDENTITY) +- `GetTableSequences` - List sequences for a specific table +- `ListAllTableSequences` - List sequences for all tables +- `SyncIdentityColumn` - Sync IDENTITY sequence after data migration + +**Partition Information:** +- `GetTablePartitions` - List partitions for a specific table +- `GetPartitionStrategy` - Check if table is partitioned (RANGE/LIST/HASH) +- `VerifyPartitionCoverage` - Verify all partitions exist for date range + +**Constraint Information:** +- `GetTableConstraints` - List constraints for a specific table + +**Index Information:** +- `GetTableIndexes` - List indexes for a specific table + ## Functions ### VerifyTablesHaveSameColumns @@ -90,10 +123,16 @@ Migrates all data from one table to another after verifying schemas match. ```go // Dry run first -rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", true) +rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", nil, true) // Actual migration -rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", false) +rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", nil, false) +if err != nil { + log.WithError(err).Error("migration failed") +} + +// Migrate with omitting columns (e.g., to use target's auto-increment for id) +rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", []string{"id"}, false) if err != nil { log.WithError(err).Error("migration failed") } @@ -109,6 +148,7 @@ if err != nil { **Parameters:** - `sourceTable` - Table to copy data from - `targetTable` - Table to copy data to +- `omitColumns` - List of column names to omit from migration (e.g., `[]string{"id"}` to use target's auto-increment). Pass `nil` to copy all columns. - `dryRun` - If true, only verifies without copying data **Returns:** @@ -145,10 +185,16 @@ startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) // Dry run first -rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, true) +rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, nil, true) // Actual migration -rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, false) +rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, nil, false) +if err != nil { + log.WithError(err).Error("migration failed") +} + +// Migrate with omitting columns (e.g., to use target's auto-increment for id) +rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, []string{"id"}, false) if err != nil { log.WithError(err).Error("migration failed") } @@ -169,6 +215,7 @@ if err != nil { - `dateColumn` - Column name to filter by date (e.g., "created_at") - `startDate` - Start of date range (inclusive, >=) - `endDate` - End of date range (exclusive, <) +- `omitColumns` - List of column names to omit from migration (e.g., `[]string{"id"}` to use target's auto-increment). Pass `nil` to copy all columns. - `dryRun` - If true, only verifies without copying data **Returns:** @@ -238,6 +285,427 @@ log.WithField("count", count).Info("table row count") --- +### RenameTables + +Renames multiple tables atomically in a single transaction. + +```go +renames := map[string]string{ + "orders_old": "orders_backup", + "orders_new": "orders", +} + +// Dry run first (renameSequences=true, renamePartitions=true, renameConstraints=true, renameIndexes=true) +_, err := dbc.RenameTables(renames, true, true, true, true, true) +if err != nil { + log.WithError(err).Error("validation failed") +} + +// Execute renames (renameSequences=true, renamePartitions=true, renameConstraints=true, renameIndexes=true) +count, err := dbc.RenameTables(renames, true, true, true, true, false) +if err != nil { + log.WithError(err).Error("rename failed") +} +log.WithField("renamed", count).Info("tables, partitions, sequences, constraints, and indexes renamed") +``` + +**How It Works**: +1. Validates that all source tables exist +2. Checks for conflicts (target table already exists, unless it's also being renamed) +3. Executes all `ALTER TABLE ... RENAME TO ...` statements in a single transaction +4. Either all renames succeed or all are rolled back + +**Parameters**: +- `tableRenames`: Map of source table names to target names (from -> to) +- `renameSequences`: If true, also renames sequences owned by table columns (SERIAL, BIGSERIAL, IDENTITY) +- `renamePartitions`: If true, also renames child partitions of partitioned tables +- `renameConstraints`: If true, also renames table constraints (primary keys, foreign keys, unique, check) +- `renameIndexes`: If true, also renames table indexes (including those backing constraints) +- `dryRun`: If true, only validates without executing + +**Returns**: +- `renamedCount`: Number of tables successfully renamed (0 if dry run) +- `error`: Any error encountered + +**Features**: +- **Atomic operation**: All renames happen in one transaction +- **Validation**: Checks source tables exist and no conflicts +- **Dry-run support**: Test before executing +- **Fast**: PostgreSQL only updates metadata, not data +- **Safe**: Views, indexes, and foreign keys are automatically updated + +**Use Cases**: +- Swapping partitioned tables with non-partitioned tables +- Renaming related tables together for consistency +- Atomic schema migrations +- Creating backups before migrations +- Rolling back failed migrations + +**Important Notes**: +- All renames must succeed or all will fail (atomic) +- Table swap scenarios are detected and allowed (when target is also a source) +- Extremely fast - only metadata is updated +- PostgreSQL automatically updates dependent object **references** (views, FKs) but NOT their names +- **Sequences are NOT automatically renamed by PostgreSQL** - use `renameSequences=true` to rename them +- **Partitions are NOT automatically renamed by PostgreSQL** - use `renamePartitions=true` to rename them +- **Constraints are NOT automatically renamed by PostgreSQL** - use `renameConstraints=true` to rename them +- **Indexes are NOT automatically renamed by PostgreSQL** - use `renameIndexes=true` to rename them +- **Rename order matters** - sequences/constraints/indexes are processed in sorted order to avoid naming conflicts during table swaps + +**Understanding SERIAL vs IDENTITY:** + +Both create auto-increment columns, but with different syntax and internal linkage: + +```sql +-- Old way: SERIAL (still widely used) +CREATE TABLE orders ( + id SERIAL PRIMARY KEY, + name TEXT +); +-- Creates sequence: orders_id_seq +-- Linkage: pg_depend (deptype='a') + column DEFAULT nextval('orders_id_seq') + +-- Modern way: IDENTITY (SQL standard, recommended) +CREATE TABLE orders ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name TEXT +); +-- Creates sequence: orders_id_seq +-- Linkage: pg_depend (deptype='i') + pg_attribute.attidentity +``` + +**Key Differences:** + +| Aspect | SERIAL | IDENTITY | +|--------|--------|----------| +| SQL Standard | No (PostgreSQL-specific) | Yes (SQL:2003 standard) | +| Dependency Type | `'a'` (auto) | `'i'` (internal) | +| Column Default | `nextval('seq_name')` (name-based) | None (OID-based internally) | +| Rename Safety | Default uses sequence NAME | Fully OID-based, safer | +| PostgreSQL Tables | `pg_depend` + `pg_attrdef` | `pg_depend` + `pg_attribute` | + +**How Sequences Are Linked to Columns:** + +PostgreSQL uses multiple mechanisms to link sequences to columns: + +1. **`pg_depend`** - Dependency tracking (OID-based, survives renames) + - SERIAL: `deptype = 'a'` (auto dependency) + - IDENTITY: `deptype = 'i'` (internal dependency) + +2. **Column Metadata:** + - SERIAL: Column default = `nextval('sequence_name')` (stored as text!) + - IDENTITY: `pg_attribute.attidentity` = `'d'` or `'a'` (uses OID reference) + +3. **Sequence Ownership:** + - Both: `pg_sequence` records which table.column owns the sequence + +**Why Our RenameTables Function Works Safely:** + +When we execute `ALTER SEQUENCE old_seq RENAME TO new_seq`: + +✅ **IDENTITY columns (safe):** +- `pg_depend` uses OID, not name → automatically updated +- `pg_attribute.attidentity` uses OID → no change needed +- Column has NO default expression → nothing to update +- **Result: Fully automatic, zero risk** + +⚠️ **SERIAL columns (mostly safe):** +- `pg_depend` uses OID, not name → automatically updated +- BUT: Column default `nextval('old_seq')` is stored as TEXT +- PostgreSQL does NOT automatically update the default expression +- **However**: `nextval()` resolves the sequence name at runtime, and PostgreSQL's search path finds the renamed sequence +- **Result: Works in practice, but default text is stale** + +**Both are captured by:** +- `GetTableSequences` / `ListAllTableSequences` +- `RenameTables(renameSequences=true)` +- `SyncIdentityColumn` + +**About Sequence Renaming:** + +When you rename a table in PostgreSQL, **sequences are NOT automatically renamed**. This can lead to naming inconsistencies: + +```sql +-- Before rename: +-- Table: orders +-- Sequence: orders_id_seq + +ALTER TABLE orders RENAME TO orders_old; + +-- After rename: +-- Table: orders_old +-- Sequence: orders_id_seq (still has old name!) +``` + +To keep sequence names consistent with table names, use `renameSequences=true`: +- Finds all sequences owned by table columns (SERIAL, BIGSERIAL, IDENTITY) +- Renames them to match new table name: `newtable_columnname_seq` +- All renames (tables + sequences) happen in one atomic transaction +- If any rename fails, all are rolled back + +**When to use `renameSequences=true`:** +- ✅ When swapping production tables (keeps naming consistent) +- ✅ When table names are part of your naming conventions +- ✅ When you want clean, matching names for monitoring/debugging +- ❌ When sequences are shared or manually managed +- ❌ When you don't care about sequence naming consistency + +**About Partition Renaming:** + +When you rename a partitioned table in PostgreSQL, **child partitions are NOT automatically renamed**: + +```sql +-- Before rename: +-- Parent table: orders +-- Partitions: orders_2024_01_01, orders_2024_01_02, etc. + +ALTER TABLE orders RENAME TO orders_old; + +-- After rename: +-- Parent table: orders_old +-- Partitions: orders_2024_01_01, orders_2024_01_02, etc. (still have old prefix!) +``` + +To keep partition names consistent with the parent table, use `renamePartitions=true`: +- Finds all child partitions using PostgreSQL's inheritance system +- Extracts the suffix from each partition name (e.g., `_2024_01_01`) +- Renames to match new parent: `newtable_2024_01_01` +- All renames (tables + partitions + sequences) happen in one atomic transaction +- If any rename fails, all are rolled back + +**How Partition Renaming Works:** +``` +Old table: orders +Old partitions: orders_2024_01_01, orders_2024_01_02 + +New table: orders_old +New partitions: orders_old_2024_01_01, orders_old_2024_01_02 + +Suffix extraction: _2024_01_01, _2024_01_02 +New naming: newtable + suffix +``` + +**When to use `renamePartitions=true`:** +- ✅ When swapping partitioned tables in production +- ✅ When partition naming follows table name prefix convention +- ✅ When you want consistent naming for all related objects +- ✅ When monitoring/debugging relies on naming patterns +- ❌ When partitions use custom naming unrelated to table name +- ❌ When partitions are manually managed with specific names + +**Renaming Partition Sequences, Constraints, and Indexes:** + +When `renamePartitions=true`, the function will **also** rename sequences, constraints, and indexes on those partition tables if the respective flags are enabled: + +- `renamePartitions=true` + `renameSequences=true` → Renames sequences on both parent table AND partition tables +- `renamePartitions=true` + `renameConstraints=true` → Renames constraints on both parent table AND partition tables +- `renamePartitions=true` + `renameIndexes=true` → Renames indexes on both parent table AND partition tables + +Example: +```go +renames := map[string]string{ + "orders": "orders_v2", +} + +// Rename table, partitions, and all their sequences/constraints/indexes +count, err := dbc.RenameTables(renames, true, true, true, true, false) +// ↑ ↑ ↑ ↑ +// sequences ──┘ │ │ │ +// partitions ────────┘ │ │ +// constraints ──────────────┘ │ +// indexes ────────────────────┘ + +// Result: +// Parent table: +// - orders_v2 +// - orders_v2_id_seq +// - orders_v2_pkey +// - orders_v2_pkey (index) +// +// Partitions: +// - orders_v2_2024_01 +// - orders_v2_2024_01_pkey +// - orders_v2_2024_01_pkey (index) +// - orders_v2_2024_02 +// - orders_v2_2024_02_pkey +// - orders_v2_2024_02_pkey (index) +``` + +This ensures complete naming consistency across the entire partitioned table hierarchy. + +**About Constraint Renaming:** + +When you rename a table in PostgreSQL, **constraints are NOT automatically renamed**: + +```sql +-- Before rename: +-- Table: orders +-- Constraints: orders_pkey, orders_email_key, orders_customer_id_fkey + +ALTER TABLE orders RENAME TO orders_old; + +-- After rename: +-- Table: orders_old +-- Constraints: orders_pkey, orders_email_key, orders_customer_id_fkey (still have old names!) +``` + +To keep constraint names consistent with table names, use `renameConstraints=true`: +- Finds all constraints for the table (primary keys, foreign keys, unique, check, exclusion) +- Extracts the suffix from each constraint name (e.g., `_pkey`, `_email_key`) +- Renames to match new table: `newtable_pkey`, `newtable_email_key` +- All renames (tables + partitions + sequences + constraints) happen in one atomic transaction +- If any rename fails, all are rolled back + +**How Constraint Renaming Works:** +``` +Old table: orders +Old constraints: orders_pkey, orders_email_key, orders_customer_id_fkey + +New table: orders_old +New constraints: orders_old_pkey, orders_old_email_key, orders_old_customer_id_fkey + +Suffix extraction: _pkey, _email_key, _customer_id_fkey +New naming: newtable + suffix +``` + +**Constraint Types Renamed:** +- Primary keys (`p`) - e.g., `tablename_pkey` +- Foreign keys (`f`) - e.g., `tablename_column_fkey` +- Unique constraints (`u`) - e.g., `tablename_column_key` +- Check constraints (`c`) - e.g., `tablename_column_check` +- Exclusion constraints (`x`) - e.g., `tablename_excl` + +**Important Note about Indexes:** +Renaming a constraint does NOT rename the backing index. Indexes are separate objects in PostgreSQL and must be renamed separately if needed. The `RenameTables` function currently only renames constraints, not indexes. + +**When to use `renameConstraints=true`:** +- ✅ When swapping tables in production (keeps naming consistent) +- ✅ When constraint names follow table name prefix convention +- ✅ When you want clean, matching names for schema documentation +- ✅ When monitoring/debugging relies on naming patterns +- ❌ When constraints use custom naming unrelated to table name +- ❌ When constraints are manually managed with specific names + +**About Index Renaming:** + +When you rename a table in PostgreSQL, **indexes are NOT automatically renamed**: + +```sql +-- Before rename: +-- Table: orders +-- Indexes: orders_pkey, orders_email_key, orders_customer_id_idx + +ALTER TABLE orders RENAME TO orders_old; + +-- After rename: +-- Table: orders_old +-- Indexes: orders_pkey, orders_email_key, orders_customer_id_idx (still have old names!) +``` + +To keep index names consistent with table names, use `renameIndexes=true`: +- Finds all indexes for the table (including those backing constraints) +- Extracts the suffix from each index name (e.g., `_pkey`, `_email_key`, `_customer_id_idx`) +- Renames to match new table: `newtable_pkey`, `newtable_email_key`, `newtable_customer_id_idx` +- All renames (tables + partitions + sequences + constraints + indexes) happen in one atomic transaction +- If any rename fails, all are rolled back + +**How Index Renaming Works:** +``` +Old table: orders +Old indexes: orders_pkey, orders_email_key, orders_customer_id_idx + +New table: orders_old +New indexes: orders_old_pkey, orders_old_email_key, orders_old_customer_id_idx + +Suffix extraction: _pkey, _email_key, _customer_id_idx +New naming: newtable + suffix +``` + +**Index Types Renamed:** +- Primary key indexes - e.g., `tablename_pkey` +- Unique indexes - e.g., `tablename_column_key` +- Regular indexes (B-tree, GIN, GiST, etc.) - e.g., `tablename_column_idx` +- Partial indexes - Any index following the naming pattern + +**Important: Indexes vs Constraints** + +Indexes and constraints are separate objects in PostgreSQL: +- Renaming a constraint does NOT rename the backing index +- Renaming an index does NOT rename the constraint +- When you create a primary key, PostgreSQL creates both a constraint AND an index with the same name +- **Recommendation:** Use both `renameConstraints=true` and `renameIndexes=true` together to keep names consistent + +**Performance Note:** +Index renaming is extremely fast - it only updates metadata in PostgreSQL system catalogs, without touching the actual index data structure. However, it does require a brief `ACCESS EXCLUSIVE` lock on the index. + +**When to use `renameIndexes=true`:** +- ✅ When swapping tables in production (keeps naming consistent) +- ✅ When index names follow table name prefix convention +- ✅ When you want clean, matching names for performance analysis +- ✅ When monitoring/debugging relies on naming patterns +- ✅ When renaming constraints (to keep constraint and index names aligned) +- ❌ When indexes use custom naming unrelated to table name +- ❌ When indexes are manually managed with specific names + +**Rename Order Handling:** + +When swapping tables (e.g., `A -> B, C -> A`), the order of operations matters to avoid naming conflicts: + +```go +renames := map[string]string{ + "table_base": "table_old", // Free up "table_base" namespace + "table_new": "table_base", // Now safe to use "table_base" +} +``` + +Without proper ordering, index/constraint renames could fail: +```sql +-- Wrong order (if processed randomly): +ALTER INDEX table_new_pkey RENAME TO table_base_pkey; -- ERROR! table_base_pkey already exists + +-- Correct order (sorted): +ALTER INDEX table_base_pkey RENAME TO table_old_pkey; -- Frees up "table_base_pkey" +ALTER INDEX table_new_pkey RENAME TO table_base_pkey; -- Now safe +``` + +**How we handle it:** +- Tables are renamed first (in transaction) +- Sequences/constraints/indexes are renamed in **sorted order** (alphabetically by name) +- Sorting ensures `table_base` is processed before `table_new`, avoiding conflicts +- All operations are deterministic - same input always produces same execution order + +**Example - Table Swap**: +```go +// Swap old table with new partitioned table atomically +renames := map[string]string{ + "orders": "orders_old", // Save current table + "orders_partitioned": "orders", // New table becomes production +} + +// Rename sequences, partitions, constraints, and indexes too +count, err := dbc.RenameTables(renames, true, true, true, true, false) +if err != nil { + // If any rename fails, all are rolled back + log.Fatal(err) +} +``` + +**Example - Three-Way Swap**: +```go +// Rotate three tables: production -> backup, new -> production, backup -> archive +renames := map[string]string{ + "orders": "orders_backup", // Production becomes backup + "orders_new": "orders", // New becomes production + "orders_backup": "orders_archive", // Old backup becomes archive +} + +// Rename sequences, partitions, constraints, and indexes too +count, err := dbc.RenameTables(renames, true, true, true, true, false) +// All renames happen atomically (tables + partitions + sequences + constraints + indexes) +``` + +--- + ### SyncIdentityColumn Synchronizes the IDENTITY sequence for a column to match the current maximum value in the table. @@ -401,6 +869,404 @@ if err := dbc.VerifyPartitionCoverage("orders", startDate, endDate); err != nil --- +### GetSequenceMetadata + +Returns detailed metadata about how sequences are linked to columns in a table. + +```go +metadata, err := dbc.GetSequenceMetadata("orders") +if err != nil { + log.WithError(err).Error("failed to get metadata") +} + +for _, m := range metadata { + linkageType := "SERIAL" + if m.IsIdentityColumn { + linkageType = "IDENTITY" + } + + log.WithFields(log.Fields{ + "column": m.ColumnName, + "sequence": m.SequenceName, + "linkage_type": linkageType, + "dep_type": m.DependencyType, + "owner": m.SequenceOwner, + }).Info("sequence linkage") +} +``` + +**Returns**: List of `SequenceMetadata` structs containing: +- `SequenceName`: Name of the sequence +- `TableName`: Name of the table +- `ColumnName`: Name of the column +- `DependencyType`: `'a'` (SERIAL) or `'i'` (IDENTITY) +- `IsIdentityColumn`: `true` if column uses GENERATED AS IDENTITY +- `SequenceOwner`: Owner in format `table.column` + +**Use Cases**: +- Understanding the internal linkage mechanism (OID vs name-based) +- Debugging why a sequence rename might cause issues +- Determining if columns use SERIAL or IDENTITY +- Validating sequence ownership before renames +- Educational/documentation purposes + +**Example - Compare SERIAL vs IDENTITY Linkage**: +```go +metadata, _ := dbc.GetSequenceMetadata("orders") +for _, m := range metadata { + if m.IsIdentityColumn { + fmt.Printf("%s: IDENTITY (OID-based, safe to rename)\n", m.ColumnName) + } else { + fmt.Printf("%s: SERIAL (default uses name, usually safe)\n", m.ColumnName) + } +} +``` + +--- + +### GetTableSequences + +Returns all sequences owned by columns in a specific table (SERIAL, BIGSERIAL, IDENTITY). + +```go +sequences, err := dbc.GetTableSequences("orders") +if err != nil { + log.WithError(err).Error("failed to get sequences") +} + +for _, seq := range sequences { + log.WithFields(log.Fields{ + "sequence": seq.SequenceName, + "table": seq.TableName, + "column": seq.ColumnName, + }).Info("found sequence") +} +``` + +**Returns**: List of `SequenceInfo` structs containing: +- `SequenceName`: Name of the sequence +- `TableName`: Name of the table owning the sequence +- `ColumnName`: Name of the column using the sequence + +**Sequence Types Captured:** +- **SERIAL/BIGSERIAL**: Creates a sequence like `tablename_columnname_seq` +- **IDENTITY**: Creates an internal sequence like `tablename_columnname_seq` (GENERATED BY DEFAULT AS IDENTITY) + +**Use Cases**: +- Checking which sequences will be renamed +- Auditing sequence ownership for a specific table +- Debugging sequence-related issues +- Understanding table dependencies before renames + +**Example - Check Before Rename**: +```go +// Check what sequences exist before renaming +sequences, _ := dbc.GetTableSequences("orders_old") +if len(sequences) > 0 { + log.WithField("count", len(sequences)).Info("found sequences - will rename with table") + + // Use renameSequences=true to keep them consistent + renames := map[string]string{"orders_old": "orders"} + dbc.RenameTables(renames, true, false, false) +} else { + // No sequences to worry about + renames := map[string]string{"orders_old": "orders"} + dbc.RenameTables(renames, false, false, false) +} +``` + +--- + +### ListAllTableSequences + +Returns all sequences owned by table columns across the entire database (public schema). + +```go +allSequences, err := dbc.ListAllTableSequences() +if err != nil { + log.WithError(err).Error("failed to list sequences") +} + +for tableName, sequences := range allSequences { + log.WithFields(log.Fields{ + "table": tableName, + "count": len(sequences), + }).Info("table sequences") + + for _, seq := range sequences { + log.WithFields(log.Fields{ + "sequence": seq.SequenceName, + "column": seq.ColumnName, + }).Debug("sequence detail") + } +} +``` + +**Returns**: Map where: +- **Key**: Table name +- **Value**: List of `SequenceInfo` structs for that table + +**Use Cases**: +- Database-wide sequence auditing +- Understanding auto-increment usage patterns +- Finding all sequences that need syncing after bulk operations +- Generating database documentation +- Preparing for bulk table renames +- Identifying orphaned sequences + +**Example - Audit All Sequences**: +```go +allSequences, err := dbc.ListAllTableSequences() +if err != nil { + log.Fatal(err) +} + +log.WithField("tables", len(allSequences)).Info("tables with sequences") + +// Show summary +totalSequences := 0 +for tableName, sequences := range allSequences { + totalSequences += len(sequences) + fmt.Printf("Table: %s has %d sequence(s)\n", tableName, len(sequences)) + for _, seq := range sequences { + fmt.Printf(" - %s.%s → %s\n", seq.TableName, seq.ColumnName, seq.SequenceName) + } +} + +log.WithField("total_sequences", totalSequences).Info("audit complete") +``` + +**Example - Find Tables Without Sequences**: +```go +// Get all tables +allTables := []string{"orders", "items", "users", "logs"} + +// Get tables with sequences +tablesWithSequences, _ := dbc.ListAllTableSequences() + +// Find tables without sequences +for _, table := range allTables { + if _, hasSequence := tablesWithSequences[table]; !hasSequence { + log.WithField("table", table).Info("table has no sequences - using explicit IDs") + } +} +``` + +**Example - Sync All Identity Sequences**: +```go +// Get all tables with sequences +allSequences, err := dbc.ListAllTableSequences() +if err != nil { + log.Fatal(err) +} + +// Sync identity column for each table with sequences +for tableName, sequences := range allSequences { + for _, seq := range sequences { + // Only sync if column looks like an ID column + if seq.ColumnName == "id" { + err := dbc.SyncIdentityColumn(tableName, seq.ColumnName) + if err != nil { + log.WithError(err).WithField("table", tableName).Error("sync failed") + } else { + log.WithField("table", tableName).Info("synced identity") + } + } + } +} +``` + +--- + +### GetTablePartitions + +Returns all child partitions of a partitioned table. + +```go +partitions, err := dbc.GetTablePartitions("orders") +if err != nil { + log.WithError(err).Error("failed to get partitions") +} + +for _, part := range partitions { + log.WithFields(log.Fields{ + "partition": part.PartitionName, + "parent": part.ParentTable, + }).Info("found partition") +} +``` + +**Returns**: List of `PartitionTableInfo` structs containing: +- `PartitionName`: Name of the partition +- `ParentTable`: Name of the parent partitioned table + +**Use Cases**: +- Checking which partitions will be renamed +- Auditing partition structure +- Understanding table dependencies before renames +- Verifying partition naming conventions + +**Example - Check Partitions Before Rename**: +```go +// Check what partitions exist before renaming +partitions, _ := dbc.GetTablePartitions("orders_old") +log.WithField("count", len(partitions)).Info("found partitions") + +for _, part := range partitions { + // Extract suffix to see naming pattern + suffix := strings.TrimPrefix(part.PartitionName, "orders_old") + log.WithFields(log.Fields{ + "partition": part.PartitionName, + "suffix": suffix, + }).Info("partition details") +} + +// If partitions follow naming convention, rename them too +if len(partitions) > 0 { + renames := map[string]string{"orders_old": "orders"} + dbc.RenameTables(renames, true, true, true, false) // renamePartitions=true +} +``` + +--- + +### GetTableConstraints + +Returns all constraints for a table (primary keys, foreign keys, unique, check, exclusion). + +```go +constraints, err := dbc.GetTableConstraints("orders") +if err != nil { + log.WithError(err).Error("failed to get constraints") +} + +for _, cons := range constraints { + log.WithFields(log.Fields{ + "constraint": cons.ConstraintName, + "type": cons.ConstraintType, + "definition": cons.Definition, + }).Info("found constraint") +} +``` + +**Returns**: List of `ConstraintInfo` structs containing: +- `ConstraintName`: Name of the constraint (e.g., "orders_pkey") +- `TableName`: Name of the table +- `ConstraintType`: Single character type code: + - `'p'` - Primary key + - `'f'` - Foreign key + - `'u'` - Unique + - `'c'` - Check + - `'x'` - Exclusion +- `Definition`: SQL definition of the constraint (e.g., "PRIMARY KEY (id)") + +**Use Cases**: +- Checking which constraints will be renamed +- Auditing constraint naming conventions +- Understanding table dependencies before renames +- Verifying constraint structure + +**Example - Check Constraints Before Rename**: +```go +// Check what constraints exist before renaming +constraints, _ := dbc.GetTableConstraints("orders_old") +log.WithField("count", len(constraints)).Info("found constraints") + +for _, cons := range constraints { + // Extract suffix to see naming pattern + suffix := strings.TrimPrefix(cons.ConstraintName, "orders_old") + typeNames := map[string]string{ + "p": "PRIMARY KEY", + "f": "FOREIGN KEY", + "u": "UNIQUE", + "c": "CHECK", + "x": "EXCLUSION", + } + + log.WithFields(log.Fields{ + "constraint": cons.ConstraintName, + "suffix": suffix, + "type": typeNames[cons.ConstraintType], + }).Info("constraint details") +} + +// If constraints follow naming convention, rename them too +if len(constraints) > 0 { + renames := map[string]string{"orders_old": "orders"} + dbc.RenameTables(renames, true, true, true, true, false) // renameConstraints=true, renameIndexes=true +} +``` + +--- + +### GetTableIndexes + +Returns all indexes for a table (including those backing constraints). + +```go +indexes, err := dbc.GetTableIndexes("orders") +if err != nil { + log.WithError(err).Error("failed to get indexes") +} + +for _, idx := range indexes { + log.WithFields(log.Fields{ + "index": idx.IndexName, + "is_primary": idx.IsPrimary, + "is_unique": idx.IsUnique, + }).Info("found index") +} +``` + +**Returns**: List of `IndexInfo` structs containing: +- `IndexName`: Name of the index (e.g., "orders_pkey") +- `TableName`: Name of the table +- `Definition`: Full CREATE INDEX statement +- `IsPrimary`: true if this is a primary key index +- `IsUnique`: true if this is a unique index + +**Use Cases**: +- Checking which indexes will be renamed +- Auditing index naming conventions +- Understanding table performance characteristics +- Verifying index structure before operations + +**Important Note**: +Indexes and constraints are separate objects. An index backing a primary key or unique constraint has the same name as the constraint, but they are different objects. Renaming one does NOT rename the other. + +**Example - Check Indexes Before Rename**: +```go +// Check what indexes exist before renaming +indexes, _ := dbc.GetTableIndexes("orders_old") +log.WithField("count", len(indexes)).Info("found indexes") + +for _, idx := range indexes { + // Extract suffix to see naming pattern + suffix := strings.TrimPrefix(idx.IndexName, "orders_old") + + indexType := "REGULAR" + if idx.IsPrimary { + indexType = "PRIMARY KEY" + } else if idx.IsUnique { + indexType = "UNIQUE" + } + + log.WithFields(log.Fields{ + "index": idx.IndexName, + "suffix": suffix, + "type": indexType, + }).Info("index details") +} + +// If indexes follow naming convention, rename them too +if len(indexes) > 0 { + renames := map[string]string{"orders_old": "orders"} + dbc.RenameTables(renames, true, true, true, true, false) // renameIndexes=true +} +``` + +--- + ## Helper Types ### ColumnInfo @@ -419,6 +1285,66 @@ type ColumnInfo struct { --- +### SequenceInfo + +Represents basic information about a sequence associated with a table column. + +```go +type SequenceInfo struct { + SequenceName string + TableName string + ColumnName string +} +``` + +**Usage**: +- Returned by `GetTableSequences()` to show sequences owned by table columns +- Used internally by `RenameTables()` when `renameSequences=true` +- Includes sequences from SERIAL, BIGSERIAL, and IDENTITY columns + +--- + +### SequenceMetadata + +Represents detailed metadata about how a sequence is linked to a column. + +```go +type SequenceMetadata struct { + SequenceName string + TableName string + ColumnName string + DependencyType string // 'a' = auto (SERIAL), 'i' = internal (IDENTITY) + IsIdentityColumn bool // true if column uses GENERATED AS IDENTITY + SequenceOwner string // Table.Column that owns this sequence +} +``` + +**Usage**: +- Returned by `GetSequenceMetadata()` to show detailed linkage information +- Helps understand the difference between SERIAL and IDENTITY columns +- Shows PostgreSQL's internal dependency mechanism (OID-based vs name-based) +- Useful for debugging and educational purposes + +--- + +### PartitionTableInfo + +Represents information about a table partition. + +```go +type PartitionTableInfo struct { + PartitionName string + ParentTable string +} +``` + +**Usage**: +- Returned by `GetTablePartitions()` to show child partitions of a table +- Used internally by `RenameTables()` when `renamePartitions=true` +- Works with any partition type (RANGE, LIST, HASH) + +--- + ### PartitionStrategy Defines the partitioning strategy type for PostgreSQL partitioned tables. @@ -529,6 +1455,62 @@ This ensures that functionally equivalent types are treated as identical during --- +## Omitting Columns During Migration + +Both `MigrateTableData` and `MigrateTableDataRange` support omitting specific columns during migration. This is useful when: + +- **Auto-increment columns**: The target table has an `id` column with `GENERATED BY DEFAULT AS IDENTITY` and you want new IDs to be generated instead of copying from source +- **Computed columns**: The target table has columns that should be calculated rather than copied +- **Different schemas**: Some columns exist in the source but shouldn't be migrated to the target + +### Example: Omitting ID Column + +```go +// Migrate data but let target table generate new IDs +rows, err := dbc.MigrateTableData( + "old_table", + "new_table", + []string{"id"}, // Omit the id column + false, +) +if err != nil { + log.WithError(err).Error("migration failed") + return +} + +log.WithField("rows", rows).Info("migrated with new IDs generated") +``` + +### Example: Omitting Multiple Columns + +```go +// Omit multiple columns during range migration +rows, err := dbc.MigrateTableDataRange( + "source", + "target", + "created_at", + startDate, + endDate, + []string{"id", "updated_at", "version"}, // Omit these columns + false, +) +``` + +### How It Works + +When you specify `omitColumns`: +1. The function retrieves all columns from the source table +2. Filters out any columns in the `omitColumns` list +3. Generates `INSERT INTO target (col1, col2, ...) SELECT col1, col2, ... FROM source` +4. Only the non-omitted columns are included in both the INSERT and SELECT clauses + +**Important Notes:** +- If you omit a `NOT NULL` column without a default, the migration will fail +- Omitted columns in the target table must either be nullable or have default values +- Pass `nil` or `[]string{}` to copy all columns (default behavior) + +--- + ## Usage Examples ### Basic Migration @@ -541,13 +1523,13 @@ if err != nil { } // Step 2: Dry run -_, err = dbc.MigrateTableData("source_table", "target_table", true) +_, err = dbc.MigrateTableData("source_table", "target_table", nil, true) if err != nil { log.Fatal(err) } // Step 3: Actual migration -rows, err := dbc.MigrateTableData("source_table", "target_table", false) +rows, err := dbc.MigrateTableData("source_table", "target_table", nil, false) log.WithField("rows", rows).Info("migration completed") ``` @@ -560,7 +1542,7 @@ log.WithField("rows", rows).Info("migration completed") partition := "test_analysis_by_job_by_dates_2024_01_15" archive := "test_analysis_archive" -rows, err := dbc.MigrateTableData(partition, archive, false) +rows, err := dbc.MigrateTableData(partition, archive, nil, false) if err != nil { log.WithError(err).Error("migration failed") return @@ -585,7 +1567,7 @@ partitions := []string{ var totalRows int64 for _, partition := range partitions { - rows, err := dbc.MigrateTableData(partition, "archive_table", false) + rows, err := dbc.MigrateTableData(partition, "archive_table", nil, false) if err != nil { log.WithError(err).WithField("partition", partition).Error("failed") continue @@ -602,13 +1584,13 @@ log.WithField("total_rows", totalRows).Info("batch migration completed") ```go // Create backup before migration -_, err := dbc.MigrateTableData("target_table", "backup_table", false) +_, err := dbc.MigrateTableData("target_table", "backup_table", nil, false) if err != nil { log.Fatal("backup failed") } // Perform migration -rows, err := dbc.MigrateTableData("source_table", "target_table", false) +rows, err := dbc.MigrateTableData("source_table", "target_table", nil, false) if err != nil { log.Error("migration failed - restore from backup if needed") return @@ -633,7 +1615,7 @@ for month := 1; month <= 12; month++ { "end": endDate.Format("2006-01-02"), }).Info("migrating month") - rows, err := dbc.MigrateTableDataRange("large_table", "large_table_new", "created_at", startDate, endDate, false) + rows, err := dbc.MigrateTableDataRange("large_table", "large_table_new", "created_at", startDate, endDate, nil, false) if err != nil { log.WithError(err).WithField("month", month).Error("migration failed") continue @@ -648,6 +1630,136 @@ for month := 1; month <= 12; month++ { --- +### Swap Partitioned Table with Non-Partitioned Table + +```go +// Complete workflow: Migrate to partitioned table and swap atomically + +oldTable := "orders" +newPartitionedTable := "orders_partitioned" + +// Step 1: Verify data was migrated successfully +oldCount, _ := dbc.GetTableRowCount(oldTable) +newCount, _ := dbc.GetTableRowCount(newPartitionedTable) + +if oldCount != newCount { + log.Fatal("row count mismatch - cannot swap") +} + +// Step 2: Perform atomic table swap +renames := map[string]string{ + "orders": "orders_old", // Save current table + "orders_partitioned": "orders", // New table becomes production +} + +// Dry run first +_, err := dbc.RenameTables(renames, true, true, true) +if err != nil { + log.Fatal(err) +} + +// Execute swap (rename sequences and partitions too) +count, err := dbc.RenameTables(renames, true, true, false) +if err != nil { + log.Fatal(err) +} + +log.WithFields(log.Fields{ + "renamed": count, + "old_table": "orders_old", + "new_table": "orders", + "partitioned": true, +}).Info("tables swapped - partitioned table is now active") + +// If something goes wrong, you can easily rollback: +// rollback := map[string]string{ +// "orders": "orders_partitioned", +// "orders_old": "orders", +// } +// dbc.RenameTables(rollback, true, true, false) +``` + +--- + +### Three-Way Table Rotation + +```go +// Rotate tables: archive old backup, current becomes backup, new becomes current + +renames := map[string]string{ + "orders": "orders_backup", // Current production -> backup + "orders_new": "orders", // New table -> production + "orders_backup": "orders_archive", // Old backup -> archive +} + +// All three renames happen atomically in one transaction (rename sequences and partitions too) +count, err := dbc.RenameTables(renames, true, true, false) +if err != nil { + log.WithError(err).Error("rotation failed - no changes made") + return +} + +log.WithField("renamed", count).Info("three-way rotation completed") + +// Result: +// - orders (was orders_new) - now in production +// - orders_backup (was orders) - current backup +// - orders_archive (was orders_backup) - archived +``` + +--- + +### Migrate with Auto-Generated IDs + +```go +// When migrating to a table with auto-increment ID, omit the id column +// so the target table generates new sequential IDs + +sourceTable := "prow_job_run_tests" +targetTable := "prow_job_run_tests_partitioned" +startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) + +// Dry run first to verify +_, err := dbc.MigrateTableDataRange( + sourceTable, + targetTable, + "created_at", + startDate, + endDate, + []string{"id"}, // Omit id column - target will auto-generate + true, +) +if err != nil { + log.Fatal(err) +} + +// Actual migration +rows, err := dbc.MigrateTableDataRange( + sourceTable, + targetTable, + "created_at", + startDate, + endDate, + []string{"id"}, // Omit id column + false, +) +if err != nil { + log.Fatal(err) +} + +log.WithFields(log.Fields{ + "rows": rows, + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), +}).Info("data migrated with new IDs generated") + +// Note: No need to sync identity column since we're omitting id +// The target table's auto-increment will continue from its current value +``` + +--- + ### Migrate Specific Date Range to Archive ```go @@ -656,13 +1768,13 @@ startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) endDate := time.Date(2024, 4, 1, 0, 0, 0, 0, time.UTC) // Dry run first -_, err := dbc.MigrateTableDataRange("orders", "orders_archive", "order_date", startDate, endDate, true) +_, err := dbc.MigrateTableDataRange("orders", "orders_archive", "order_date", startDate, endDate, nil, true) if err != nil { log.Fatal(err) } // Actual migration -rows, err := dbc.MigrateTableDataRange("orders", "orders_archive", "order_date", startDate, endDate, false) +rows, err := dbc.MigrateTableDataRange("orders", "orders_archive", "order_date", startDate, endDate, nil, false) log.WithFields(log.Fields{ "rows": rows, "start_date": startDate.Format("2006-01-02"), @@ -672,20 +1784,80 @@ log.WithFields(log.Fields{ --- +### Complete Partitioned Table Migration Workflow + +```go +// End-to-end example: Migrate from non-partitioned to partitioned table + +// Step 1: Create partitioned table (using partitions package) +// import "github.com/openshift/sippy/pkg/db/partitions" +// partitionConfig := partitions.NewRangePartitionConfig("created_at") +// _, err := partitions.CreatePartitionedTableFromExisting(dbc, "orders", "orders_partitioned", partitionConfig, false) + +// Step 2: Create necessary partitions +startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +endDate := time.Now() +// _, err := partitions.CreateMissingPartitions(dbc, "orders_partitioned", startDate, endDate, false) + +// Step 3: Migrate data (omit id to use auto-increment) +rows, err := dbc.MigrateTableDataRange( + "orders", + "orders_partitioned", + "created_at", + startDate, + endDate, + []string{"id"}, // Omit id - let target generate new IDs + false, +) +if err != nil { + log.Fatal(err) +} +log.WithField("rows", rows).Info("data migrated") + +// Step 4: Verify row counts match +oldCount, _ := dbc.GetTableRowCount("orders") +newCount, _ := dbc.GetTableRowCount("orders_partitioned") +if oldCount != newCount { + log.Fatal("row count mismatch!") +} + +// Step 5: Atomically swap tables +renames := map[string]string{ + "orders": "orders_old", + "orders_partitioned": "orders", +} + +count, err := dbc.RenameTables(renames, true, true, false) +if err != nil { + log.Fatal(err) +} + +log.WithFields(log.Fields{ + "renamed": count, + "rows": rows, + "partitioned": true, +}).Info("migration completed - partitioned table is now active") + +// Step 6: After verification period, drop old table +// DROP TABLE orders_old; +``` + +--- + ## Best Practices ### Always Use Dry Run First ```go // GOOD: Verify before executing -_, err := dbc.MigrateTableData(source, target, true) +_, err := dbc.MigrateTableData(source, target, nil, true) if err != nil { return err } -rows, err := dbc.MigrateTableData(source, target, false) +rows, err := dbc.MigrateTableData(source, target, nil, false) // BAD: Direct migration without verification -rows, err := dbc.MigrateTableData(source, target, false) +rows, err := dbc.MigrateTableData(source, target, nil, false) ``` ### Verify Schemas Explicitly @@ -707,7 +1879,7 @@ if err := dbc.VerifyTablesHaveSameColumns(source, target); err != nil { sourceBefore, _ := dbc.GetTableRowCount(source) targetBefore, _ := dbc.GetTableRowCount(target) -rows, err := dbc.MigrateTableData(source, target, false) +rows, err := dbc.MigrateTableData(source, target, nil, false) targetAfter, _ := dbc.GetTableRowCount(target) expected := targetBefore + sourceBefore @@ -734,6 +1906,75 @@ if err != nil { tx.Commit() ``` +### Test Table Renames with Dry Run + +```go +// GOOD: Always dry run before renaming +renames := map[string]string{ + "orders_old": "orders_backup", + "orders_new": "orders", +} + +_, err := dbc.RenameTables(renames, true, true, true) +if err != nil { + log.WithError(err).Error("validation failed") + return +} + +count, err := dbc.RenameTables(renames, true, true, false) + +// BAD: Direct rename without validation +count, err := dbc.RenameTables(renames, true, true, false) +``` + +### Verify Before Swapping Tables + +```go +// GOOD: Verify data integrity before swapping +oldCount, _ := dbc.GetTableRowCount("orders") +newCount, _ := dbc.GetTableRowCount("orders_partitioned") + +if oldCount != newCount { + log.Error("cannot swap - row counts don't match") + return +} + +// Now safe to swap +dbc.RenameTables(map[string]string{ + "orders": "orders_old", + "orders_partitioned": "orders", +}, true, true, false) + +// BAD: Swap without verifying data +dbc.RenameTables(renames, true, true, false) +``` + +### Keep Rollback Plans Ready + +```go +// GOOD: Define rollback before making changes +renames := map[string]string{ + "orders": "orders_old", + "orders_new": "orders", +} + +// Define rollback upfront +rollback := map[string]string{ + "orders": "orders_new", + "orders_old": "orders", +} + +// Execute rename +_, err := dbc.RenameTables(renames, true, true, false) +if err != nil { + log.Error("rename failed - no rollback needed") + return +} + +// If issues found after rename, easy to rollback +// dbc.RenameTables(rollback, true, true, false) +``` + --- ## Error Handling @@ -805,7 +2046,7 @@ detached, _ := partitions.DetachOldPartitions(dbc, "parent_table", 180, false) // Migrate detached partitions to archive for _, partition := range detachedPartitions { - dbc.MigrateTableData(partition.TableName, "archive_table", false) + dbc.MigrateTableData(partition.TableName, "archive_table", nil, false) } // Drop old partitions diff --git a/pkg/db/partitions/partitions.go b/pkg/db/partitions/partitions.go index 7cd1d7d6c..4e3a7c97a 100644 --- a/pkg/db/partitions/partitions.go +++ b/pkg/db/partitions/partitions.go @@ -8,6 +8,7 @@ import ( log "github.com/sirupsen/logrus" "gorm.io/gorm" + "gorm.io/gorm/schema" "github.com/openshift/sippy/pkg/db" ) @@ -1285,20 +1286,26 @@ func CreatePartitionedTable(dbc *db.DB, model interface{}, tableName string, con } } if !hasAllPartitionKeys { + // Generate table-specific name for logging + indexName := makeTableSpecificIndexName(tableName, idx.Fields) log.WithFields(log.Fields{ "table": tableName, - "index": idx.Name, + "index": indexName, + "model_index": idx.Name, "partition_keys": config.Columns, }).Warn("skipping unique index without all partition keys (not allowed on partitioned tables)") continue } } + // Generate table-specific index name to avoid conflicts when creating multiple tables from same model + indexName := makeTableSpecificIndexName(tableName, idx.Fields) + indexSQL.WriteString("\n") if idx.Class == "UNIQUE" { - indexSQL.WriteString(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (", idx.Name, tableName)) + indexSQL.WriteString(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (", indexName, tableName)) } else { - indexSQL.WriteString(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (", idx.Name, tableName)) + indexSQL.WriteString(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (", indexName, tableName)) } var fieldNames []string @@ -1397,6 +1404,30 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry return "", fmt.Errorf("failed to get partition columns: %w", err) } + // Get primary key columns to prevent dropping NOT NULL from them + primaryKeyColumns, err := getPrimaryKeyColumns(dbc, tableName) + if err != nil { + return "", fmt.Errorf("failed to get primary key columns: %w", err) + } + + log.WithFields(log.Fields{ + "table": tableName, + "partition_columns": partitionColumns, + "primary_key_columns": primaryKeyColumns, + }).Info("table structure for update validation") + + // Build a set of partition columns for quick lookup + partitionColMap := make(map[string]bool) + for _, col := range partitionColumns { + partitionColMap[col] = true + } + + // Build a set of primary key columns for quick lookup + primaryKeyColMap := make(map[string]bool) + for _, col := range primaryKeyColumns { + primaryKeyColMap[col] = true + } + // Build maps for comparison currentColMap := make(map[string]db.ColumnInfo) for _, col := range currentColumns { @@ -1494,16 +1525,37 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry } // Check NOT NULL constraint - // Primary keys are always NOT NULL in PostgreSQL + // Primary keys, partition keys, and columns in primary key are always NOT NULL in PostgreSQL currentNotNull := currentCol.IsNullable == "NO" - desiredNotNull := field.PrimaryKey || field.NotNull + + // Check if column is part of primary key or partition key + isPartOfPrimaryKey := primaryKeyColMap[field.DBName] + isPartOfPartitionKey := partitionColMap[field.DBName] + + // Desired NOT NULL state: explicit primary key, explicit not null, or part of actual primary key + desiredNotNull := field.PrimaryKey || field.NotNull || isPartOfPrimaryKey + if desiredNotNull != currentNotNull { if desiredNotNull { modifications = append(modifications, fmt.Sprintf("ALTER COLUMN %s SET NOT NULL", field.DBName)) } else { - modifications = append(modifications, - fmt.Sprintf("ALTER COLUMN %s DROP NOT NULL", field.DBName)) + // Cannot drop NOT NULL from primary key or partition key columns + if isPartOfPrimaryKey { + log.WithFields(log.Fields{ + "table": tableName, + "column": field.DBName, + }).Warn("cannot drop NOT NULL from primary key column - skipping") + } else if isPartOfPartitionKey { + log.WithFields(log.Fields{ + "table": tableName, + "column": field.DBName, + }).Warn("cannot drop NOT NULL from partition key column - skipping") + } else { + // Safe to drop NOT NULL + modifications = append(modifications, + fmt.Sprintf("ALTER COLUMN %s DROP NOT NULL", field.DBName)) + } } } @@ -1540,11 +1592,7 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry } // Check indexes - partitionColMap := make(map[string]bool) - for _, col := range partitionColumns { - partitionColMap[col] = true - } - + // (partitionColMap already created earlier) for _, idx := range stmt.Schema.ParseIndexes() { // Skip unique indexes that don't include all partition keys if idx.Class == "UNIQUE" { @@ -1563,16 +1611,22 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry } } if !hasAllPartitionKeys { + // Generate table-specific name for logging + indexName := makeTableSpecificIndexName(tableName, idx.Fields) log.WithFields(log.Fields{ "table": tableName, - "index": idx.Name, + "index": indexName, + "model_index": idx.Name, "partition_keys": partitionColumns, }).Warn("skipping unique index without all partition keys") continue } } - currentIdx, exists := currentIdxMap[idx.Name] + // Generate table-specific index name to avoid conflicts when creating multiple tables from same model + indexName := makeTableSpecificIndexName(tableName, idx.Fields) + + currentIdx, exists := currentIdxMap[indexName] if !exists { // New index - create it var fieldNames []string @@ -1583,11 +1637,11 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry if idx.Class == "UNIQUE" { alterStatements = append(alterStatements, fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", - idx.Name, tableName, strings.Join(fieldNames, ", "))) + indexName, tableName, strings.Join(fieldNames, ", "))) } else { alterStatements = append(alterStatements, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", - idx.Name, tableName, strings.Join(fieldNames, ", "))) + indexName, tableName, strings.Join(fieldNames, ", "))) } } else { // Index exists - check if it needs to be recreated @@ -1611,21 +1665,21 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry if !colsMatch || !uniqueMatch { // Drop and recreate index alterStatements = append(alterStatements, - fmt.Sprintf("DROP INDEX IF EXISTS %s", idx.Name)) + fmt.Sprintf("DROP INDEX IF EXISTS %s", indexName)) if idx.Class == "UNIQUE" { alterStatements = append(alterStatements, fmt.Sprintf("CREATE UNIQUE INDEX %s ON %s (%s)", - idx.Name, tableName, strings.Join(desiredCols, ", "))) + indexName, tableName, strings.Join(desiredCols, ", "))) } else { alterStatements = append(alterStatements, fmt.Sprintf("CREATE INDEX %s ON %s (%s)", - idx.Name, tableName, strings.Join(desiredCols, ", "))) + indexName, tableName, strings.Join(desiredCols, ", "))) } } } - delete(currentIdxMap, idx.Name) + delete(currentIdxMap, indexName) } // Drop indexes that are no longer in the model @@ -1697,11 +1751,11 @@ func getCurrentIndexes(dbc *db.DB, tableName string) ([]indexInfo, error) { JOIN pg_index ix ON ix.indexrelid = c.oid JOIN pg_attribute a ON a.attrelid = ix.indrelid AND a.attnum = ANY(ix.indkey) WHERE i.schemaname = 'public' - AND i.tablename = $1 + AND i.tablename = @table_name ORDER BY i.indexname, a.attnum ` - result := dbc.DB.Raw(query, tableName).Scan(&rows) + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&rows) if result.Error != nil { return nil, result.Error } @@ -1737,12 +1791,62 @@ func getPartitionColumns(dbc *db.DB, tableName string) ([]string, error) { FROM pg_class c JOIN pg_partitioned_table pt ON pt.partrelid = c.oid JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = ANY(pt.partattrs) - WHERE c.relname = $1 + WHERE c.relname = @table_name AND c.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public') ORDER BY array_position(pt.partattrs, a.attnum) ` - result := dbc.DB.Raw(query, tableName).Scan(&columns) + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&columns) + if result.Error != nil { + return nil, result.Error + } + + return columns, nil +} + +// makeTableSpecificIndexName creates a table-specific index name from index fields +// This prevents index name collisions when creating multiple tables from the same model +// +// GORM often generates index names like "idx_{original_table_name}_{column_names}" +// (e.g., "idx_prow_job_run_tests_suite_id"). When creating a different table like +// "prow_job_run_tests_copy_1", we need to avoid the redundant name like +// "prow_job_run_tests_copy_1_idx_prow_job_run_tests_suite_id". +// +// Instead, we extract the column names from the index fields and generate: +// "{table_name}_idx_{column_names}" +// +// Examples: +// - Index fields: [suite_id] -> "prow_job_run_tests_copy_1_idx_suite_id" +// - Index fields: [created_at] -> "orders_copy_1_idx_created_at" +// - Index fields: [user_id, org_id] -> "users_backup_idx_user_id_org_id" +func makeTableSpecificIndexName(tableName string, indexFields []schema.IndexOption) string { + // Extract column names from index fields + var columnNames []string + for _, field := range indexFields { + columnNames = append(columnNames, field.DBName) + } + + // Generate table-specific index name: {table}_idx_{columns} + columnSuffix := strings.Join(columnNames, "_") + return fmt.Sprintf("%s_idx_%s", tableName, columnSuffix) +} + +// getPrimaryKeyColumns retrieves the columns that are part of the primary key for a table +func getPrimaryKeyColumns(dbc *db.DB, tableName string) ([]string, error) { + var columns []string + + query := ` + SELECT a.attname + FROM pg_index i + JOIN pg_class c ON c.oid = i.indrelid + JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = ANY(i.indkey) + WHERE c.relname = @table_name + AND c.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public') + AND i.indisprimary = true + ORDER BY array_position(i.indkey, a.attnum) + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&columns) if result.Error != nil { return nil, result.Error } diff --git a/pkg/db/utils.go b/pkg/db/utils.go index 52e6d2a64..3ee8a3d59 100644 --- a/pkg/db/utils.go +++ b/pkg/db/utils.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "sort" "strings" "time" @@ -254,12 +255,13 @@ func normalizeDataType(dataType string) string { // Parameters: // - sourceTable: The table to copy data from // - targetTable: The table to copy data to +// - omitColumns: List of column names to omit from migration (e.g., ["id"] to use target's auto-increment) // - dryRun: If true, only verifies schemas and reports what would be migrated without actually copying data // // Returns: // - rowsMigrated: The number of rows successfully migrated (0 if dryRun is true) // - error: Any error encountered during migration -func (dbc *DB) MigrateTableData(sourceTable, targetTable string, dryRun bool) (int64, error) { +func (dbc *DB) MigrateTableData(sourceTable, targetTable string, omitColumns []string, dryRun bool) (int64, error) { log.WithFields(log.Fields{ "source": sourceTable, "target": targetTable, @@ -313,9 +315,22 @@ func (dbc *DB) MigrateTableData(sourceTable, targetTable string, dryRun bool) (i return 0, fmt.Errorf("failed to get column list: %w", err) } + // Create a map of columns to omit for quick lookup + omitMap := make(map[string]bool) + for _, col := range omitColumns { + omitMap[col] = true + } + + // Build column list, excluding omitted columns var columnNames []string for _, col := range columns { - columnNames = append(columnNames, col.ColumnName) + if !omitMap[col.ColumnName] { + columnNames = append(columnNames, col.ColumnName) + } + } + + if len(columnNames) == 0 { + return 0, fmt.Errorf("no columns to migrate after omitting %v", omitColumns) } // Step 5: Perform the migration using INSERT INTO ... SELECT @@ -385,6 +400,7 @@ func (dbc *DB) MigrateTableData(sourceTable, targetTable string, dryRun bool) (i // - dateColumn: The column name to filter by date range (e.g., "created_at") // - startDate: Start of date range (inclusive) // - endDate: End of date range (exclusive) +// - omitColumns: List of column names to omit from migration (e.g., ["id"] to use target's auto-increment) // - dryRun: If true, only verifies schemas and reports what would be migrated without actually copying data // // Returns: @@ -395,8 +411,8 @@ func (dbc *DB) MigrateTableData(sourceTable, targetTable string, dryRun bool) (i // // startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) // endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) -// rows, err := dbc.MigrateTableDataRange("old_table", "new_table", "created_at", startDate, endDate, false) -func (dbc *DB) MigrateTableDataRange(sourceTable, targetTable, dateColumn string, startDate, endDate time.Time, dryRun bool) (int64, error) { +// rows, err := dbc.MigrateTableDataRange("old_table", "new_table", "created_at", startDate, endDate, nil, false) +func (dbc *DB) MigrateTableDataRange(sourceTable, targetTable, dateColumn string, startDate, endDate time.Time, omitColumns []string, dryRun bool) (int64, error) { log.WithFields(log.Fields{ "source": sourceTable, "target": targetTable, @@ -489,9 +505,22 @@ func (dbc *DB) MigrateTableDataRange(sourceTable, targetTable, dateColumn string return 0, fmt.Errorf("failed to get column list: %w", err) } + // Create a map of columns to omit for quick lookup + omitMap := make(map[string]bool) + for _, col := range omitColumns { + omitMap[col] = true + } + + // Build column list, excluding omitted columns var columnNames []string for _, col := range columns { - columnNames = append(columnNames, col.ColumnName) + if !omitMap[col.ColumnName] { + columnNames = append(columnNames, col.ColumnName) + } + } + + if len(columnNames) == 0 { + return 0, fmt.Errorf("no columns to migrate after omitting %v", omitColumns) } // Step 6: Perform the migration using INSERT INTO ... SELECT ... WHERE @@ -700,6 +729,1063 @@ func (dbc *DB) GetTableRowCount(tableName string) (int64, error) { return count, nil } +// SequenceInfo represents information about a sequence associated with a table column +type SequenceInfo struct { + SequenceName string + TableName string + ColumnName string +} + +// PartitionTableInfo represents information about a table partition +type PartitionTableInfo struct { + PartitionName string + ParentTable string +} + +// ConstraintInfo represents information about a table constraint +type ConstraintInfo struct { + ConstraintName string + TableName string + ConstraintType string // 'p'=primary key, 'f'=foreign key, 'u'=unique, 'c'=check, 'x'=exclusion + Definition string // Full constraint definition +} + +// GetTableConstraints returns all constraints for a table +// This includes primary keys, foreign keys, unique constraints, check constraints, and exclusion constraints +// +// Constraint types: +// - 'p' = Primary key +// - 'f' = Foreign key +// - 'u' = Unique +// - 'c' = Check +// - 'x' = Exclusion +// +// Example: +// +// constraints, err := dbc.GetTableConstraints("orders") +// if err != nil { +// log.WithError(err).Error("failed to get constraints") +// } +// for _, c := range constraints { +// log.WithFields(log.Fields{ +// "constraint": c.ConstraintName, +// "type": c.ConstraintType, +// }).Info("found constraint") +// } +func (dbc *DB) GetTableConstraints(tableName string) ([]ConstraintInfo, error) { + var constraints []ConstraintInfo + + query := ` + SELECT + con.conname AS constraint_name, + t.relname AS table_name, + con.contype AS constraint_type, + pg_get_constraintdef(con.oid) AS definition + FROM pg_constraint con + JOIN pg_class t ON con.conrelid = t.oid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE t.relname = @table_name + AND n.nspname = 'public' + ORDER BY con.contype, con.conname + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&constraints) + if result.Error != nil { + return nil, fmt.Errorf("failed to get constraints for table %s: %w", tableName, result.Error) + } + + return constraints, nil +} + +// IndexInfo represents metadata about a table index +type IndexInfo struct { + IndexName string + TableName string + Definition string // Index definition (CREATE INDEX statement) + IsPrimary bool // true if this is a primary key index + IsUnique bool // true if this is a unique index +} + +// GetTableIndexes returns all indexes for a table +// This includes indexes created explicitly and indexes backing constraints (primary keys, unique constraints) +// +// Note: Indexes backing constraints may have the same name as the constraint, +// but they are separate objects. Renaming a constraint does NOT rename the index. +// +// Example: +// +// indexes, err := dbc.GetTableIndexes("orders") +// if err != nil { +// log.WithError(err).Error("failed to get indexes") +// } +// for _, idx := range indexes { +// log.WithFields(log.Fields{ +// "index": idx.IndexName, +// "is_primary": idx.IsPrimary, +// "is_unique": idx.IsUnique, +// }).Info("found index") +// } +func (dbc *DB) GetTableIndexes(tableName string) ([]IndexInfo, error) { + var indexes []IndexInfo + + query := ` + SELECT + i.indexname AS index_name, + i.tablename AS table_name, + i.indexdef AS definition, + ix.indisprimary AS is_primary, + ix.indisunique AS is_unique + FROM pg_indexes i + JOIN pg_class c ON c.relname = i.indexname + JOIN pg_index ix ON ix.indexrelid = c.oid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE i.tablename = @table_name + AND i.schemaname = 'public' + AND n.nspname = 'public' + ORDER BY i.indexname + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&indexes) + if result.Error != nil { + return nil, fmt.Errorf("failed to get indexes for table %s: %w", tableName, result.Error) + } + + return indexes, nil +} + +// GetTablePartitions returns all partitions of a partitioned table +// Uses PostgreSQL's partition inheritance system to find child partitions +func (dbc *DB) GetTablePartitions(tableName string) ([]PartitionTableInfo, error) { + var partitions []PartitionTableInfo + + query := ` + SELECT + child.relname AS partition_name, + parent.relname AS parent_table + FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace + JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace + WHERE parent.relname = @table_name + AND nmsp_parent.nspname = 'public' + AND nmsp_child.nspname = 'public' + ORDER BY child.relname + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&partitions) + if result.Error != nil { + return nil, fmt.Errorf("failed to get partitions for table %s: %w", tableName, result.Error) + } + + return partitions, nil +} + +// SequenceMetadata represents detailed metadata about how a sequence is linked to a column +type SequenceMetadata struct { + SequenceName string + TableName string + ColumnName string + DependencyType string // 'a' = auto (SERIAL), 'i' = internal (IDENTITY) + IsIdentityColumn bool // true if column uses GENERATED AS IDENTITY + SequenceOwner string // Table.Column that owns this sequence +} + +// GetSequenceMetadata returns detailed metadata about how a sequence is linked to a column +// This shows the internal PostgreSQL mechanisms that link IDENTITY/SERIAL columns to sequences: +// +// For IDENTITY columns, PostgreSQL uses: +// 1. pg_depend: Creates an internal dependency (deptype='i') linking sequence to column +// 2. pg_attribute.attidentity: Marks column as identity ('d' or 'a') +// 3. pg_sequence: Stores sequence ownership information +// +// For SERIAL columns, PostgreSQL uses: +// 1. pg_depend: Creates an auto dependency (deptype='a') linking sequence to column +// 2. Column default: Uses nextval('sequence_name') +// +// When you rename a sequence using ALTER SEQUENCE...RENAME: +// - PostgreSQL automatically updates pg_depend (OID-based, not name-based) +// - For SERIAL: You must also update the column default expression (name-based!) +// - For IDENTITY: No additional updates needed (uses OID internally) +// +// This is why our RenameTables function just renames sequences - PostgreSQL handles the rest +// for IDENTITY columns, but SERIAL columns may have stale defaults if renamed outside ALTER TABLE. +// +// Example: +// +// metadata, err := dbc.GetSequenceMetadata("orders") +// for _, m := range metadata { +// log.WithFields(log.Fields{ +// "sequence": m.SequenceName, +// "column": m.ColumnName, +// "dep_type": m.DependencyType, +// "is_identity": m.IsIdentityColumn, +// }).Info("sequence linkage") +// } +func (dbc *DB) GetSequenceMetadata(tableName string) ([]SequenceMetadata, error) { + var metadata []SequenceMetadata + + query := ` + SELECT + s.relname AS sequence_name, + t.relname AS table_name, + a.attname AS column_name, + d.deptype AS dependency_type, + CASE WHEN a.attidentity IN ('a', 'd') THEN true ELSE false END AS is_identity_column, + t.relname || '.' || a.attname AS sequence_owner + FROM pg_class s + JOIN pg_depend d ON d.objid = s.oid + JOIN pg_class t ON d.refobjid = t.oid + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid + JOIN pg_namespace n ON n.oid = s.relnamespace + WHERE s.relkind = 'S' + AND t.relname = @table_name + AND n.nspname = 'public' + AND d.deptype IN ('a', 'i') + ORDER BY a.attnum + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&metadata) + if result.Error != nil { + return nil, fmt.Errorf("failed to get sequence metadata for table %s: %w", tableName, result.Error) + } + + return metadata, nil +} + +// GetTableSequences returns all sequences owned by columns in the specified table +// This includes sequences from: +// - SERIAL/BIGSERIAL columns (dependency type 'a' - auto) +// - IDENTITY columns (dependency type 'i' - internal, e.g., GENERATED BY DEFAULT AS IDENTITY) +// +// Example: +// +// sequences, err := dbc.GetTableSequences("orders") +// if err != nil { +// log.WithError(err).Error("failed to get sequences") +// } +// for _, seq := range sequences { +// log.WithFields(log.Fields{ +// "sequence": seq.SequenceName, +// "table": seq.TableName, +// "column": seq.ColumnName, +// }).Info("found sequence") +// } +func (dbc *DB) GetTableSequences(tableName string) ([]SequenceInfo, error) { + var sequences []SequenceInfo + + query := ` + SELECT + s.relname AS sequence_name, + t.relname AS table_name, + a.attname AS column_name + FROM pg_class s + JOIN pg_depend d ON d.objid = s.oid + JOIN pg_class t ON d.refobjid = t.oid + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid + JOIN pg_namespace n ON n.oid = s.relnamespace + WHERE s.relkind = 'S' + AND t.relname = @table_name + AND n.nspname = 'public' + AND d.deptype IN ('a', 'i') + ORDER BY a.attnum + ` + + result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&sequences) + if result.Error != nil { + return nil, fmt.Errorf("failed to get sequences for table %s: %w", tableName, result.Error) + } + + return sequences, nil +} + +// ListAllTableSequences returns all sequences owned by table columns in the public schema +// This includes sequences from: +// - SERIAL/BIGSERIAL columns (dependency type 'a' - auto) +// - IDENTITY columns (dependency type 'i' - internal, e.g., GENERATED BY DEFAULT AS IDENTITY) +// +// This is useful for: +// - Auditing sequence ownership across the entire database +// - Understanding which tables use auto-increment columns +// - Finding sequences that may need to be renamed or synced +// - Database documentation and inventory +// +// # Returns a map where keys are table names and values are lists of sequences +// +// Example: +// +// allSequences, err := dbc.ListAllTableSequences() +// if err != nil { +// log.WithError(err).Error("failed to list sequences") +// } +// for tableName, sequences := range allSequences { +// log.WithFields(log.Fields{ +// "table": tableName, +// "count": len(sequences), +// }).Info("table sequences") +// for _, seq := range sequences { +// log.WithFields(log.Fields{ +// "sequence": seq.SequenceName, +// "column": seq.ColumnName, +// }).Debug("sequence detail") +// } +// } +func (dbc *DB) ListAllTableSequences() (map[string][]SequenceInfo, error) { + var allSequences []SequenceInfo + + query := ` + SELECT + s.relname AS sequence_name, + t.relname AS table_name, + a.attname AS column_name + FROM pg_class s + JOIN pg_depend d ON d.objid = s.oid + JOIN pg_class t ON d.refobjid = t.oid + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid + JOIN pg_namespace n ON n.oid = s.relnamespace + WHERE s.relkind = 'S' + AND n.nspname = 'public' + AND d.deptype IN ('a', 'i') + ORDER BY t.relname, a.attnum + ` + + result := dbc.DB.Raw(query).Scan(&allSequences) + if result.Error != nil { + return nil, fmt.Errorf("failed to list all table sequences: %w", result.Error) + } + + // Group sequences by table name + sequencesByTable := make(map[string][]SequenceInfo) + for _, seq := range allSequences { + sequencesByTable[seq.TableName] = append(sequencesByTable[seq.TableName], seq) + } + + log.WithFields(log.Fields{ + "tables": len(sequencesByTable), + "sequences": len(allSequences), + }).Info("listed all table sequences") + + return sequencesByTable, nil +} + +// topologicalSortRenames orders rename operations to avoid naming conflicts +// If renaming A->B and B->C, we must do B->C first (to free up "B" namespace) +// Returns sorted list of old names in the order they should be processed +// +// Example: +// - Renames: {table_b: table_old, table_a: table_b} +// - Dependencies: table_a depends on table_b (because table_a->table_b and table_b exists) +// - Result: [table_b, table_a] (process table_b first to free up the name) +func topologicalSortRenames(renames map[string]string) []string { + // Build dependency graph + // If we're renaming X->Y, and Y is also being renamed (Y->Z), + // then X depends on Y (Y must be renamed first to free up the name) + dependencies := make(map[string][]string) // map[node][]dependencies + allNodes := make(map[string]bool) + + for oldName := range renames { + allNodes[oldName] = true + dependencies[oldName] = []string{} + } + + for oldName, newName := range renames { + // If newName matches another oldName, we depend on that being renamed first + if allNodes[newName] { + dependencies[oldName] = append(dependencies[oldName], newName) + } + } + + // Topological sort using Kahn's algorithm + // inDegree[node] = number of things that node depends on + inDegree := make(map[string]int) + for node, deps := range dependencies { + inDegree[node] = len(deps) + } + + // Start with nodes that have no dependencies + queue := []string{} + for node := range allNodes { + if inDegree[node] == 0 { + queue = append(queue, node) + } + } + + // Sort queue for deterministic ordering + sort.Strings(queue) + + result := []string{} + for len(queue) > 0 { + // Pop from queue + current := queue[0] + queue = queue[1:] + result = append(result, current) + + // Reduce in-degree of dependent nodes + for node, deps := range dependencies { + for _, dep := range deps { + if dep == current { + inDegree[node]-- + if inDegree[node] == 0 { + queue = append(queue, node) + sort.Strings(queue) // Keep queue sorted for deterministic order + } + } + } + } + } + + // Check for cycles (shouldn't happen with table renames, but be safe) + if len(result) != len(allNodes) { + log.Warn("circular dependency detected in renames, using alphabetical order as fallback") + result = []string{} + for node := range allNodes { + result = append(result, node) + } + sort.Strings(result) + } + + return result +} + +// RenameTables renames multiple tables atomically in a single transaction +// This function is useful for: +// - Swapping partitioned tables with non-partitioned tables +// - Renaming related tables together to maintain consistency +// - Performing atomic schema migrations +// +// Parameters: +// - tableRenames: Map of source table names to target table names (from -> to) +// - renameSequences: If true, also renames sequences owned by table columns (e.g., SERIAL, IDENTITY) +// - renamePartitions: If true, also renames child partitions of partitioned tables +// - renameConstraints: If true, also renames table constraints (primary keys, foreign keys, unique, check) +// - renameIndexes: If true, also renames table indexes (including those backing constraints) +// - dryRun: If true, only validates the operation without executing it +// +// Returns: +// - renamedCount: Number of tables successfully renamed (0 if dryRun is true) +// - error: Any error encountered during the operation +// +// Example: +// +// renames := map[string]string{ +// "orders_old": "orders_backup", +// "orders_new": "orders", +// "orders_archive": "orders_old_archive", +// } +// count, err := dbc.RenameTables(renames, true, true, true, true, false) +// if err != nil { +// log.WithError(err).Error("table rename failed") +// } +// +// Important Notes: +// - All renames are executed in a single transaction - either all succeed or all fail +// - The function validates that all source tables exist before attempting renames +// - The function checks for conflicts (target table already exists) +// - Views, indexes, and foreign keys are automatically updated by PostgreSQL +// - Renaming is extremely fast - PostgreSQL only updates metadata, not data +// - When renameSequences=true, sequences follow naming pattern: newtablename_columnname_seq +// - Sequences owned by SERIAL, BIGSERIAL, and IDENTITY columns will be renamed +// - When renamePartitions=true, child partitions follow naming pattern: newtablename_suffix +// - Partition renaming extracts suffix from old name and applies to new table name +// - When renamePartitions=true AND renameSequences/Constraints/Indexes=true, partition sequences/constraints/indexes are also renamed +// - When renameConstraints=true, constraints follow naming pattern: newtablename_suffix +// - Constraint renaming applies to primary keys, foreign keys, unique, check, and exclusion constraints +// - When renameIndexes=true, indexes follow naming pattern: newtablename_suffix +// - Index renaming applies to all indexes including those backing constraints +// - Indexes with the same name as constraints are skipped (they're renamed automatically with the constraint) +// - Renames are processed in sorted order to avoid naming conflicts during table swaps (e.g., A->B, B->C) +// - All operations are deterministic - same input always produces same execution order +func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool, renamePartitions bool, renameConstraints bool, renameIndexes bool, dryRun bool) (int, error) { + if len(tableRenames) == 0 { + return 0, fmt.Errorf("no tables to rename") + } + + log.WithFields(log.Fields{ + "count": len(tableRenames), + "dry_run": dryRun, + }).Info("starting table rename operation") + + // Step 1: Validate all source tables exist and check for conflicts + var sourceNames []string + var targetNames []string + for source, target := range tableRenames { + sourceNames = append(sourceNames, source) + targetNames = append(targetNames, target) + } + + // Check that all source tables exist + for source := range tableRenames { + var exists bool + query := ` + SELECT EXISTS ( + SELECT 1 FROM pg_tables + WHERE schemaname = 'public' AND tablename = @table_name + ) + ` + result := dbc.DB.Raw(query, sql.Named("table_name", source)).Scan(&exists) + if result.Error != nil { + return 0, fmt.Errorf("failed to check if table %s exists: %w", source, result.Error) + } + if !exists { + return 0, fmt.Errorf("source table %s does not exist", source) + } + } + + // Check for conflicts - ensure no target tables already exist + // (unless they're also being renamed as part of this operation) + for source, target := range tableRenames { + // Skip check if this target is also a source (table swap scenario) + if _, isAlsoSource := tableRenames[target]; isAlsoSource { + log.WithFields(log.Fields{ + "source": source, + "target": target, + }).Debug("target is also a source - table swap detected") + continue + } + + var exists bool + query := ` + SELECT EXISTS ( + SELECT 1 FROM pg_tables + WHERE schemaname = 'public' AND tablename = @table_name + ) + ` + result := dbc.DB.Raw(query, sql.Named("table_name", target)).Scan(&exists) + if result.Error != nil { + return 0, fmt.Errorf("failed to check if target table %s exists: %w", target, result.Error) + } + if exists { + return 0, fmt.Errorf("target table %s already exists (conflict with rename from %s)", target, source) + } + } + + log.WithFields(log.Fields{ + "sources": sourceNames, + "targets": targetNames, + }).Info("validation passed - all source tables exist and no conflicts detected") + + // Step 2: Find sequences that need to be renamed (if requested) + sequenceRenames := make(map[string]string) + if renameSequences { + for source, target := range tableRenames { + sequences, err := dbc.GetTableSequences(source) + if err != nil { + return 0, fmt.Errorf("failed to get sequences for table %s: %w", source, err) + } + + for _, seq := range sequences { + // Generate new sequence name following PostgreSQL convention + // old: oldtable_columnname_seq -> new: newtable_columnname_seq + newSeqName := fmt.Sprintf("%s_%s_seq", target, seq.ColumnName) + sequenceRenames[seq.SequenceName] = newSeqName + + log.WithFields(log.Fields{ + "table": source, + "column": seq.ColumnName, + "old_sequence": seq.SequenceName, + "new_sequence": newSeqName, + }).Debug("will rename sequence") + } + } + + if len(sequenceRenames) > 0 { + log.WithField("count", len(sequenceRenames)).Info("found sequences to rename") + } + } + + // Step 2b: Find partitions that need to be renamed (if requested) + partitionRenames := make(map[string]string) + if renamePartitions { + for source, target := range tableRenames { + partitions, err := dbc.GetTablePartitions(source) + if err != nil { + return 0, fmt.Errorf("failed to get partitions for table %s: %w", source, err) + } + + for _, part := range partitions { + // Extract suffix from old partition name + // old: oldtable_2024_01_01 -> suffix: _2024_01_01 + // new: newtable_2024_01_01 + suffix := strings.TrimPrefix(part.PartitionName, source) + if suffix == part.PartitionName { + // Partition name doesn't start with parent table name - skip + log.WithFields(log.Fields{ + "partition": part.PartitionName, + "parent": source, + }).Warn("partition name doesn't start with parent table name - skipping") + continue + } + + newPartName := target + suffix + partitionRenames[part.PartitionName] = newPartName + + log.WithFields(log.Fields{ + "parent": source, + "old_partition": part.PartitionName, + "new_partition": newPartName, + "suffix": suffix, + }).Debug("will rename partition") + } + } + + if len(partitionRenames) > 0 { + log.WithField("count", len(partitionRenames)).Info("found partitions to rename") + + // Also find sequences/constraints/indexes for partition tables + // This allows renaming them when the partition is renamed + if renameSequences { + for oldPartName, newPartName := range partitionRenames { + partSeqs, err := dbc.GetTableSequences(oldPartName) + if err != nil { + return 0, fmt.Errorf("failed to get sequences for partition %s: %w", oldPartName, err) + } + + for _, seq := range partSeqs { + newSeqName := fmt.Sprintf("%s_%s_seq", newPartName, seq.ColumnName) + sequenceRenames[seq.SequenceName] = newSeqName + + log.WithFields(log.Fields{ + "partition": oldPartName, + "column": seq.ColumnName, + "old_sequence": seq.SequenceName, + "new_sequence": newSeqName, + }).Debug("will rename partition sequence") + } + } + } + } + } + + // Step 2c: Find constraints that need to be renamed (if requested) + constraintRenames := make(map[string]map[string]string) // map[tableName]map[oldConstraint]newConstraint + if renameConstraints { + for source, target := range tableRenames { + constraints, err := dbc.GetTableConstraints(source) + if err != nil { + return 0, fmt.Errorf("failed to get constraints for table %s: %w", source, err) + } + + for _, cons := range constraints { + // Extract suffix from old constraint name if it starts with the table name + // old: oldtable_pkey -> suffix: _pkey + // new: newtable_pkey + suffix := strings.TrimPrefix(cons.ConstraintName, source) + if suffix == cons.ConstraintName { + // Constraint name doesn't start with table name - skip + log.WithFields(log.Fields{ + "constraint": cons.ConstraintName, + "table": source, + }).Debug("constraint name doesn't start with table name - skipping") + continue + } + + newConsName := target + suffix + + // Initialize map for this table if needed + if constraintRenames[source] == nil { + constraintRenames[source] = make(map[string]string) + } + constraintRenames[source][cons.ConstraintName] = newConsName + + log.WithFields(log.Fields{ + "table": source, + "old_constraint": cons.ConstraintName, + "new_constraint": newConsName, + "type": cons.ConstraintType, + "suffix": suffix, + }).Debug("will rename constraint") + } + } + + totalConstraints := 0 + for _, consMap := range constraintRenames { + totalConstraints += len(consMap) + } + if totalConstraints > 0 { + log.WithField("count", totalConstraints).Info("found constraints to rename") + } + + // Also find constraints for partition tables + if renamePartitions && len(partitionRenames) > 0 { + for oldPartName, newPartName := range partitionRenames { + partCons, err := dbc.GetTableConstraints(oldPartName) + if err != nil { + return 0, fmt.Errorf("failed to get constraints for partition %s: %w", oldPartName, err) + } + + for _, cons := range partCons { + suffix := strings.TrimPrefix(cons.ConstraintName, oldPartName) + if suffix == cons.ConstraintName { + log.WithFields(log.Fields{ + "constraint": cons.ConstraintName, + "partition": oldPartName, + }).Debug("constraint name doesn't start with partition name - skipping") + continue + } + + newConsName := newPartName + suffix + + if constraintRenames[oldPartName] == nil { + constraintRenames[oldPartName] = make(map[string]string) + } + constraintRenames[oldPartName][cons.ConstraintName] = newConsName + + log.WithFields(log.Fields{ + "partition": oldPartName, + "old_constraint": cons.ConstraintName, + "new_constraint": newConsName, + "type": cons.ConstraintType, + "suffix": suffix, + }).Debug("will rename partition constraint") + } + } + } + } + + // Step 2d: Find indexes that need to be renamed (if requested) + indexRenames := make(map[string]map[string]string) // map[tableName]map[oldIndex]newIndex + if renameIndexes { + for source, target := range tableRenames { + indexes, err := dbc.GetTableIndexes(source) + if err != nil { + return 0, fmt.Errorf("failed to get indexes for table %s: %w", source, err) + } + + for _, idx := range indexes { + // Extract suffix from old index name if it starts with the table name + // old: oldtable_pkey -> suffix: _pkey + // new: newtable_pkey + suffix := strings.TrimPrefix(idx.IndexName, source) + if suffix == idx.IndexName { + // Index name doesn't start with table name - skip + log.WithFields(log.Fields{ + "index": idx.IndexName, + "table": source, + }).Debug("index name doesn't start with table name - skipping") + continue + } + + newIdxName := target + suffix + + // Initialize map for this table if needed + if indexRenames[source] == nil { + indexRenames[source] = make(map[string]string) + } + indexRenames[source][idx.IndexName] = newIdxName + + log.WithFields(log.Fields{ + "table": source, + "old_index": idx.IndexName, + "new_index": newIdxName, + "is_unique": idx.IsUnique, + "is_primary": idx.IsPrimary, + "suffix": suffix, + }).Debug("will rename index") + } + } + + totalIndexes := 0 + for _, idxMap := range indexRenames { + totalIndexes += len(idxMap) + } + if totalIndexes > 0 { + log.WithField("count", totalIndexes).Info("found indexes to rename") + } + + // Also find indexes for partition tables + if renamePartitions && len(partitionRenames) > 0 { + for oldPartName, newPartName := range partitionRenames { + partIdxs, err := dbc.GetTableIndexes(oldPartName) + if err != nil { + return 0, fmt.Errorf("failed to get indexes for partition %s: %w", oldPartName, err) + } + + for _, idx := range partIdxs { + suffix := strings.TrimPrefix(idx.IndexName, oldPartName) + if suffix == idx.IndexName { + log.WithFields(log.Fields{ + "index": idx.IndexName, + "partition": oldPartName, + }).Debug("index name doesn't start with partition name - skipping") + continue + } + + newIdxName := newPartName + suffix + + if indexRenames[oldPartName] == nil { + indexRenames[oldPartName] = make(map[string]string) + } + indexRenames[oldPartName][idx.IndexName] = newIdxName + + log.WithFields(log.Fields{ + "partition": oldPartName, + "old_index": idx.IndexName, + "new_index": newIdxName, + "is_unique": idx.IsUnique, + "is_primary": idx.IsPrimary, + "suffix": suffix, + }).Debug("will rename partition index") + } + } + } + } + + // Step 3: Dry run - report what would be renamed + if dryRun { + log.Info("[DRY RUN] would rename the following tables:") + for source, target := range tableRenames { + log.WithFields(log.Fields{ + "from": source, + "to": target, + }).Info("[DRY RUN] table rename") + } + + if len(partitionRenames) > 0 { + log.Info("[DRY RUN] would rename the following partitions:") + for oldPart, newPart := range partitionRenames { + log.WithFields(log.Fields{ + "from": oldPart, + "to": newPart, + }).Info("[DRY RUN] partition rename") + } + } + + if len(sequenceRenames) > 0 { + log.Info("[DRY RUN] would rename the following sequences:") + for oldSeq, newSeq := range sequenceRenames { + log.WithFields(log.Fields{ + "from": oldSeq, + "to": newSeq, + }).Info("[DRY RUN] sequence rename") + } + } + + totalConstraints := 0 + for _, consMap := range constraintRenames { + totalConstraints += len(consMap) + } + if totalConstraints > 0 { + log.Info("[DRY RUN] would rename the following constraints:") + for tableName, consMap := range constraintRenames { + for oldCons, newCons := range consMap { + log.WithFields(log.Fields{ + "table": tableName, + "from": oldCons, + "to": newCons, + }).Info("[DRY RUN] constraint rename") + } + } + } + + totalIndexes := 0 + for _, idxMap := range indexRenames { + totalIndexes += len(idxMap) + } + if totalIndexes > 0 { + log.Info("[DRY RUN] would rename the following indexes:") + for tableName, idxMap := range indexRenames { + for oldIdx, newIdx := range idxMap { + log.WithFields(log.Fields{ + "table": tableName, + "from": oldIdx, + "to": newIdx, + }).Info("[DRY RUN] index rename") + } + } + } + + return 0, nil + } + + // Step 4: Execute all renames in a single transaction + tx := dbc.DB.Begin() + if tx.Error != nil { + return 0, fmt.Errorf("failed to begin transaction: %w", tx.Error) + } + + // Use defer to handle rollback on error + committed := false + defer func() { + if !committed { + tx.Rollback() + } + }() + + // Execute each table rename + renamedCount := 0 + for source, target := range tableRenames { + renameSQL := fmt.Sprintf("ALTER TABLE %s RENAME TO %s", source, target) + + log.WithFields(log.Fields{ + "from": source, + "to": target, + }).Info("renaming table") + + result := tx.Exec(renameSQL) + if result.Error != nil { + return 0, fmt.Errorf("failed to rename table %s to %s: %w", source, target, result.Error) + } + + renamedCount++ + } + + // Execute each partition rename + partitionsRenamed := 0 + for oldPart, newPart := range partitionRenames { + renameSQL := fmt.Sprintf("ALTER TABLE %s RENAME TO %s", oldPart, newPart) + + log.WithFields(log.Fields{ + "from": oldPart, + "to": newPart, + }).Info("renaming partition") + + result := tx.Exec(renameSQL) + if result.Error != nil { + return 0, fmt.Errorf("failed to rename partition %s to %s: %w", oldPart, newPart, result.Error) + } + + partitionsRenamed++ + } + + // Execute each sequence rename + // Use topological sort to avoid naming conflicts + // (e.g., table_b_id_seq -> table_old_id_seq must happen before table_a_id_seq -> table_b_id_seq) + sequencesRenamed := 0 + sortedSeqNames := topologicalSortRenames(sequenceRenames) + + for _, oldSeq := range sortedSeqNames { + newSeq := sequenceRenames[oldSeq] + renameSQL := fmt.Sprintf("ALTER SEQUENCE %s RENAME TO %s", oldSeq, newSeq) + + log.WithFields(log.Fields{ + "from": oldSeq, + "to": newSeq, + }).Info("renaming sequence") + + result := tx.Exec(renameSQL) + if result.Error != nil { + return 0, fmt.Errorf("failed to rename sequence %s to %s: %w", oldSeq, newSeq, result.Error) + } + + sequencesRenamed++ + } + + // Execute each constraint rename + // Use topological sort to handle rename dependencies + constraintsRenamed := 0 + + // Flatten constraint renames into a simple map for topological sort + flatConstraintRenames := make(map[string]string) + constraintToTable := make(map[string]string) + for tableName, consMap := range constraintRenames { + for oldCons, newCons := range consMap { + flatConstraintRenames[oldCons] = newCons + constraintToTable[oldCons] = tableName + } + } + + sortedConsNames := topologicalSortRenames(flatConstraintRenames) + + for _, oldCons := range sortedConsNames { + newCons := flatConstraintRenames[oldCons] + tableName := constraintToTable[oldCons] + + // Get the new table name (in case table or partition was renamed) + newTableName := tableName + if renamed, exists := tableRenames[tableName]; exists { + newTableName = renamed + } else if renamed, exists := partitionRenames[tableName]; exists { + newTableName = renamed + } + + renameSQL := fmt.Sprintf("ALTER TABLE %s RENAME CONSTRAINT %s TO %s", newTableName, oldCons, newCons) + + log.WithFields(log.Fields{ + "table": newTableName, + "from": oldCons, + "to": newCons, + }).Info("renaming constraint") + + result := tx.Exec(renameSQL) + if result.Error != nil { + return 0, fmt.Errorf("failed to rename constraint %s to %s on table %s: %w", oldCons, newCons, newTableName, result.Error) + } + + constraintsRenamed++ + } + + // Build a set of constraint names that were renamed + // (to skip indexes with the same name, as they're renamed automatically with the constraint) + renamedConstraintNames := make(map[string]bool) + for _, consMap := range constraintRenames { + for oldCons := range consMap { + renamedConstraintNames[oldCons] = true + } + } + + // Execute each index rename + // Use topological sort to handle rename dependencies + indexesRenamed := 0 + + // Flatten index renames into a simple map for topological sort + flatIndexRenames := make(map[string]string) + indexToTable := make(map[string]string) + for tableName, idxMap := range indexRenames { + for oldIdx, newIdx := range idxMap { + flatIndexRenames[oldIdx] = newIdx + indexToTable[oldIdx] = tableName + } + } + + sortedIdxNames := topologicalSortRenames(flatIndexRenames) + + for _, oldIdx := range sortedIdxNames { + newIdx := flatIndexRenames[oldIdx] + tableName := indexToTable[oldIdx] + + // Skip if this index has the same name as a constraint we renamed + // PostgreSQL automatically renames the backing index when renaming PRIMARY KEY or UNIQUE constraints + if renamedConstraintNames[oldIdx] { + log.WithFields(log.Fields{ + "table": tableName, + "index": oldIdx, + }).Debug("skipping index - already renamed as part of constraint rename") + continue + } + + renameSQL := fmt.Sprintf("ALTER INDEX %s RENAME TO %s", oldIdx, newIdx) + + log.WithFields(log.Fields{ + "table": tableName, + "from": oldIdx, + "to": newIdx, + }).Info("renaming index") + + result := tx.Exec(renameSQL) + if result.Error != nil { + return 0, fmt.Errorf("failed to rename index %s to %s: %w", oldIdx, newIdx, result.Error) + } + + indexesRenamed++ + } + + // Commit the transaction + if err := tx.Commit().Error; err != nil { + return 0, fmt.Errorf("failed to commit transaction: %w", err) + } + committed = true + + log.WithFields(log.Fields{ + "renamed_tables": renamedCount, + "renamed_partitions": partitionsRenamed, + "renamed_sequences": sequencesRenamed, + "renamed_constraints": constraintsRenamed, + "renamed_indexes": indexesRenamed, + }).Info("rename operation completed successfully") + + return renamedCount, nil +} + // SyncIdentityColumn synchronizes the IDENTITY sequence for a column to match the current maximum value // This is useful after migrating data to a partitioned table that uses IDENTITY columns // diff --git a/pkg/db/utils_example.go b/pkg/db/utils_example.go index bae3e9810..439b233a8 100644 --- a/pkg/db/utils_example.go +++ b/pkg/db/utils_example.go @@ -1,6 +1,7 @@ package db import ( + "fmt" "time" log "github.com/sirupsen/logrus" @@ -140,7 +141,9 @@ func ExampleVerifyMultipleTables(dbc *DB, referenceTable string, tablesToCheck [ // // Usage: // -// rowsMigrated, err := dbc.MigrateTableData("old_table", "new_table", false) +// rowsMigrated, err := dbc.MigrateTableData("old_table", "new_table", nil, false) +// // Or omit id column to use target's auto-increment: +// rowsMigrated, err := dbc.MigrateTableData("old_table", "new_table", []string{"id"}, false) func ExampleMigrateTableData(dbc *DB, sourceTable, targetTable string) { log.WithFields(log.Fields{ "source": sourceTable, @@ -149,7 +152,7 @@ func ExampleMigrateTableData(dbc *DB, sourceTable, targetTable string) { // Step 1: Dry run first to verify and preview log.Info("performing dry run") - _, err := dbc.MigrateTableData(sourceTable, targetTable, true) + _, err := dbc.MigrateTableData(sourceTable, targetTable, nil, true) if err != nil { log.WithError(err).Error("dry run failed - cannot proceed with migration") return @@ -158,7 +161,7 @@ func ExampleMigrateTableData(dbc *DB, sourceTable, targetTable string) { log.Info("dry run successful - proceeding with actual migration") // Step 2: Perform actual migration - rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, false) + rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, nil, false) if err != nil { log.WithError(err).Error("migration failed") return @@ -185,7 +188,7 @@ func ExampleMigratePartitionData(dbc *DB, detachedPartition, archiveTable string // This would use functions from pkg/db/partitions if available // Migrate the data - rowsMigrated, err := dbc.MigrateTableData(detachedPartition, archiveTable, false) + rowsMigrated, err := dbc.MigrateTableData(detachedPartition, archiveTable, nil, false) if err != nil { log.WithError(err).Error("partition migration failed") return @@ -210,7 +213,7 @@ func ExampleMigrateWithBackup(dbc *DB, sourceTable, targetTable, backupTable str // Step 1: Create backup of target table log.WithField("backup", backupTable).Info("creating backup of target table") - _, err := dbc.MigrateTableData(targetTable, backupTable, false) + _, err := dbc.MigrateTableData(targetTable, backupTable, nil, false) if err != nil { log.WithError(err).Error("backup creation failed - aborting migration") return @@ -220,7 +223,7 @@ func ExampleMigrateWithBackup(dbc *DB, sourceTable, targetTable, backupTable str // Step 2: Perform migration log.Info("performing migration") - rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, false) + rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, nil, false) if err != nil { log.WithError(err).Error("migration failed - restore from backup if needed") log.WithField("backup", backupTable).Info("backup table is available for restoration") @@ -253,7 +256,7 @@ func ExampleBatchMigratePartitions(dbc *DB, partitions []string, targetTable str for _, partition := range partitions { log.WithField("partition", partition).Info("migrating partition") - rows, err := dbc.MigrateTableData(partition, targetTable, false) + rows, err := dbc.MigrateTableData(partition, targetTable, nil, false) if err != nil { log.WithError(err).WithField("partition", partition).Error("partition migration failed") failures = append(failures, partition) @@ -324,7 +327,7 @@ func ExampleMigrateAndVerify(dbc *DB, sourceTable, targetTable string) { // Step 3: Dry run log.Info("step 3: performing dry run") - _, err = dbc.MigrateTableData(sourceTable, targetTable, true) + _, err = dbc.MigrateTableData(sourceTable, targetTable, nil, true) if err != nil { log.WithError(err).Error("dry run failed") return @@ -333,7 +336,7 @@ func ExampleMigrateAndVerify(dbc *DB, sourceTable, targetTable string) { // Step 4: Actual migration log.Info("step 4: performing actual migration") - rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, false) + rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, nil, false) if err != nil { log.WithError(err).Error("migration failed") return @@ -404,7 +407,7 @@ func ExampleMigrateToPartitionedTable(dbc *DB, sourceTable, partitionedTable str // Step 1: Migrate the data log.Info("Step 1: Migrating data") - rows, err := dbc.MigrateTableData(sourceTable, partitionedTable, false) + rows, err := dbc.MigrateTableData(sourceTable, partitionedTable, nil, false) if err != nil { log.WithError(err).Error("data migration failed") return @@ -469,7 +472,7 @@ func ExampleMigrateTableDataRange(dbc *DB, sourceTable, targetTable, dateColumn // Step 1: Dry run first to verify and preview log.Info("performing dry run") - _, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, true) + _, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, nil, true) if err != nil { log.WithError(err).Error("dry run failed - cannot proceed with migration") return @@ -478,7 +481,7 @@ func ExampleMigrateTableDataRange(dbc *DB, sourceTable, targetTable, dateColumn log.Info("dry run successful - proceeding with actual migration") // Step 2: Perform actual migration - rowsMigrated, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, false) + rowsMigrated, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, nil, false) if err != nil { log.WithError(err).Error("migration failed") return @@ -523,7 +526,7 @@ func ExampleIncrementalMigrationByMonth(dbc *DB, sourceTable, targetTable, dateC "end_date": endDate.Format("2006-01-02"), }).Info("migrating month") - rows, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, false) + rows, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, nil, false) if err != nil { log.WithError(err).WithField("month", time.Month(month).String()).Error("month migration failed") failedMonths = append(failedMonths, time.Month(month).String()) @@ -577,7 +580,7 @@ func ExampleMigrateToPartitionByDateRange(dbc *DB, sourceTable, partitionedTable "end_date": endDate.Format("2006-01-02"), }).Info("migrating date range to partition") - rows, err := dbc.MigrateTableDataRange(sourceTable, partitionedTable, dateColumn, startDate, endDate, false) + rows, err := dbc.MigrateTableDataRange(sourceTable, partitionedTable, dateColumn, startDate, endDate, nil, false) if err != nil { log.WithError(err).Error("migration failed") return @@ -657,6 +660,173 @@ func ExampleVerifyPartitionCoverage(dbc *DB, tableName string, startDate, endDat log.Info("safe to proceed with data migration") } +// ExampleSequenceMetadata demonstrates examining how sequences are linked to columns +// +// # This shows the internal PostgreSQL mechanisms for SERIAL vs IDENTITY columns +// +// Usage: +// +// ExampleSequenceMetadata(dbc, "orders") +func ExampleSequenceMetadata(dbc *DB, tableName string) { + log.WithField("table", tableName).Info("examining sequence metadata") + + metadata, err := dbc.GetSequenceMetadata(tableName) + if err != nil { + log.WithError(err).Error("failed to get sequence metadata") + return + } + + if len(metadata) == 0 { + log.Info("no sequences found for this table") + return + } + + log.WithField("count", len(metadata)).Info("found sequences") + + for _, m := range metadata { + linkageType := "SERIAL" + if m.IsIdentityColumn { + linkageType = "IDENTITY" + } + + log.WithFields(log.Fields{ + "column": m.ColumnName, + "sequence": m.SequenceName, + "linkage_type": linkageType, + "dep_type": m.DependencyType, + "owner": m.SequenceOwner, + }).Info("sequence linkage details") + + // Explain the linkage mechanism + if m.IsIdentityColumn { + log.WithField("column", m.ColumnName).Debug( + "IDENTITY: Linked via pg_depend (OID-based) + pg_attribute.attidentity. " + + "Renaming sequence is safe - PostgreSQL uses OID internally, not name.") + } else { + log.WithField("column", m.ColumnName).Debug( + "SERIAL: Linked via pg_depend + column DEFAULT nextval('seq_name'). " + + "Column default uses sequence NAME, but pg_depend uses OID.") + } + } +} + +// ExampleListTableSequences demonstrates listing sequences for a specific table +// +// This is useful for: +// - Understanding which columns use auto-increment +// - Checking sequence names before table renames +// - Debugging sequence-related issues +// - Auditing table structure +// +// Usage: +// +// ExampleListTableSequences(dbc, "orders") +func ExampleListTableSequences(dbc *DB, tableName string) { + log.WithField("table", tableName).Info("listing sequences for table") + + sequences, err := dbc.GetTableSequences(tableName) + if err != nil { + log.WithError(err).Error("failed to get sequences") + return + } + + if len(sequences) == 0 { + log.Info("no sequences found for this table") + return + } + + log.WithField("count", len(sequences)).Info("found sequences") + for _, seq := range sequences { + log.WithFields(log.Fields{ + "sequence": seq.SequenceName, + "column": seq.ColumnName, + }).Info("sequence detail") + } +} + +// ExampleListAllTableSequences demonstrates listing sequences for all tables +// +// This is useful for: +// - Database auditing and inventory +// - Understanding auto-increment usage across tables +// - Finding sequences that may need syncing +// - Preparing for bulk table operations +// +// Usage: +// +// ExampleListAllTableSequences(dbc) +func ExampleListAllTableSequences(dbc *DB) { + log.Info("listing all table sequences in database") + + allSequences, err := dbc.ListAllTableSequences() + if err != nil { + log.WithError(err).Error("failed to list all sequences") + return + } + + log.WithField("tables_with_sequences", len(allSequences)).Info("found tables with sequences") + + for tableName, sequences := range allSequences { + log.WithFields(log.Fields{ + "table": tableName, + "sequence_count": len(sequences), + }).Info("table sequences") + + for _, seq := range sequences { + log.WithFields(log.Fields{ + "table": tableName, + "sequence": seq.SequenceName, + "column": seq.ColumnName, + }).Debug("sequence detail") + } + } +} + +// ExampleCheckSequencesBeforeRename demonstrates checking sequences before renaming tables +// +// This workflow helps you understand what will be renamed when using renameSequences=true +// +// Usage: +// +// ExampleCheckSequencesBeforeRename(dbc, "orders_old", "orders") +func ExampleCheckSequencesBeforeRename(dbc *DB, oldTableName, newTableName string) { + log.Info("checking sequences before table rename") + + // Get sequences for the old table + sequences, err := dbc.GetTableSequences(oldTableName) + if err != nil { + log.WithError(err).Error("failed to get sequences") + return + } + + if len(sequences) == 0 { + log.Info("no sequences to rename") + return + } + + log.WithField("count", len(sequences)).Info("sequences will be renamed") + + // Show what the new sequence names will be + for _, seq := range sequences { + newSeqName := fmt.Sprintf("%s_%s_seq", newTableName, seq.ColumnName) + log.WithFields(log.Fields{ + "old_sequence": seq.SequenceName, + "new_sequence": newSeqName, + "column": seq.ColumnName, + }).Info("planned rename") + } + + // Now perform the rename + renames := map[string]string{oldTableName: newTableName} + count, err := dbc.RenameTables(renames, true, false, false, false, false) + if err != nil { + log.WithError(err).Error("rename failed") + return + } + + log.WithField("renamed", count).Info("table and sequences renamed") +} + // ExampleCheckAndCreatePartitions demonstrates checking for missing partitions and creating them // // This workflow combines partition verification with automatic creation of missing partitions. @@ -685,3 +855,181 @@ func ExampleCheckAndCreatePartitions(dbc *DB, tableName string, startDate, endDa log.Info("all partitions exist - ready for operations") } + +// ExampleRenameTables demonstrates renaming multiple tables atomically +// +// This is useful for: +// - Swapping a new partitioned table with an old non-partitioned table +// - Renaming related tables together to maintain consistency +// - Performing atomic schema migrations +// +// Usage: +// +// ExampleRenameTables(dbc) +func ExampleRenameTables(dbc *DB) { + log.Info("renaming multiple tables atomically") + + // Define table renames + renames := map[string]string{ + "orders_old": "orders_backup", + "orders_new": "orders", + } + + // Dry run first to verify + _, err := dbc.RenameTables(renames, true, true, true, true, true) + if err != nil { + log.WithError(err).Error("dry run failed") + return + } + + // Execute the renames (rename sequences and partitions too) + count, err := dbc.RenameTables(renames, true, true, true, true, false) + if err != nil { + log.WithError(err).Error("rename failed") + return + } + + log.WithField("renamed", count).Info("tables renamed successfully") +} + +// ExampleSwapPartitionedTable demonstrates swapping a partitioned table with a non-partitioned table +// +// This is a common workflow when migrating from non-partitioned to partitioned tables: +// 1. Create new partitioned table +// 2. Migrate data to partitioned table +// 3. Swap the tables atomically so the partitioned table becomes the active one +// +// Usage: +// +// ExampleSwapPartitionedTable(dbc, "orders", "orders_partitioned") +func ExampleSwapPartitionedTable(dbc *DB, oldTable, newPartitionedTable string) { + log.Info("swapping non-partitioned table with partitioned table") + + // Step 1: Verify data was migrated successfully + oldCount, _ := dbc.GetTableRowCount(oldTable) + newCount, _ := dbc.GetTableRowCount(newPartitionedTable) + + if oldCount != newCount { + log.WithFields(log.Fields{ + "old_count": oldCount, + "new_count": newCount, + }).Error("row count mismatch - cannot swap tables") + return + } + + log.Info("row counts match - proceeding with table swap") + + // Step 2: Perform atomic rename to swap tables + // orders -> orders_old + // orders_partitioned -> orders + renames := map[string]string{ + oldTable: oldTable + "_old", + newPartitionedTable: oldTable, + } + + // Rename sequences and partitions too so they match the new table names + count, err := dbc.RenameTables(renames, true, true, true, true, false) + if err != nil { + log.WithError(err).Error("table swap failed") + return + } + + log.WithFields(log.Fields{ + "renamed": count, + "old_table": oldTable + "_old", + "new_table": oldTable, + "partitioned": true, + }).Info("tables swapped successfully - partitioned table is now active") + + // Next steps: + // 1. Test the new partitioned table thoroughly + // 2. After verification period, drop the old table +} + +// ExampleThreeWayTableSwap demonstrates a three-way table swap +// +// This pattern is useful when you want to: +// - Keep a backup of the current production table +// - Swap in a new table +// - Archive the old backup +// +// Usage: +// +// ExampleThreeWayTableSwap(dbc) +func ExampleThreeWayTableSwap(dbc *DB) { + log.Info("performing three-way table swap") + + // Scenario: + // orders (current production) + // orders_new (migrated data, ready to go live) + // orders_backup (previous backup to archive) + + // Step 1: First rename the old backup to archive + // Step 2: Rename current production to backup + // Step 3: Rename new table to production + // + // All in one atomic transaction: + // orders -> orders_backup + // orders_new -> orders + // orders_backup -> orders_archive + + renames := map[string]string{ + "orders": "orders_backup", + "orders_new": "orders", + "orders_backup": "orders_archive", + } + + // Dry run first (also check sequence renames) + _, err := dbc.RenameTables(renames, true, true, true, true, true) + if err != nil { + log.WithError(err).Error("dry run validation failed") + return + } + + // Execute the swap (rename sequences and partitions too) + count, err := dbc.RenameTables(renames, true, true, true, true, false) + if err != nil { + log.WithError(err).Error("table swap failed - no changes made") + return + } + + log.WithFields(log.Fields{ + "renamed": count, + "production": "orders (was orders_new)", + "backup": "orders_backup (was orders)", + "archive": "orders_archive (was orders_backup)", + }).Info("three-way swap completed successfully") +} + +// ExampleRollbackTableSwap demonstrates rolling back a table swap +// +// # If you swapped tables but need to revert, you can use RenameTables again +// +// Usage: +// +// ExampleRollbackTableSwap(dbc) +func ExampleRollbackTableSwap(dbc *DB) { + log.Info("rolling back table swap") + + // Assume we previously did: + // orders -> orders_old + // orders_partitioned -> orders + // + // To rollback: + // orders -> orders_partitioned (restore original name) + // orders_old -> orders (restore to production) + + rollbackRenames := map[string]string{ + "orders": "orders_partitioned", + "orders_old": "orders", + } + + // Rename sequences and partitions back too + count, err := dbc.RenameTables(rollbackRenames, true, true, true, true, false) + if err != nil { + log.WithError(err).Error("rollback failed") + return + } + + log.WithField("renamed", count).Info("rollback completed - original table restored") +} diff --git a/pkg/db/utils_test.go b/pkg/db/utils_test.go index e34b00f4c..efdc4a362 100644 --- a/pkg/db/utils_test.go +++ b/pkg/db/utils_test.go @@ -126,6 +126,7 @@ func TestMigrateTableDataValidation(t *testing.T) { name string sourceTable string targetTable string + omitColumns []string dryRun bool expectError bool errorContains string @@ -136,6 +137,7 @@ func TestMigrateTableDataValidation(t *testing.T) { name: "dry run mode", sourceTable: "source_table", targetTable: "target_table", + omitColumns: nil, dryRun: true, expectError: false, }, @@ -143,6 +145,23 @@ func TestMigrateTableDataValidation(t *testing.T) { name: "actual migration", sourceTable: "source_table", targetTable: "target_table", + omitColumns: nil, + dryRun: false, + expectError: false, + }, + { + name: "migration with omitted id column", + sourceTable: "source_table", + targetTable: "target_table", + omitColumns: []string{"id"}, + dryRun: false, + expectError: false, + }, + { + name: "migration with multiple omitted columns", + sourceTable: "source_table", + targetTable: "target_table", + omitColumns: []string{"id", "updated_at", "version"}, dryRun: false, expectError: false, }, @@ -163,6 +182,72 @@ func TestMigrateTableDataValidation(t *testing.T) { } } +func TestOmitColumnsLogic(t *testing.T) { + // Test the omit columns filtering logic + columns := []ColumnInfo{ + {ColumnName: "id", DataType: "bigint"}, + {ColumnName: "name", DataType: "varchar"}, + {ColumnName: "created_at", DataType: "timestamp"}, + {ColumnName: "updated_at", DataType: "timestamp"}, + } + + tests := []struct { + name string + omitColumns []string + expectedCols []string + }{ + { + name: "no columns omitted", + omitColumns: nil, + expectedCols: []string{"id", "name", "created_at", "updated_at"}, + }, + { + name: "omit id column", + omitColumns: []string{"id"}, + expectedCols: []string{"name", "created_at", "updated_at"}, + }, + { + name: "omit multiple columns", + omitColumns: []string{"id", "updated_at"}, + expectedCols: []string{"name", "created_at"}, + }, + { + name: "omit all columns results in error case", + omitColumns: []string{"id", "name", "created_at", "updated_at"}, + expectedCols: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a map of columns to omit for quick lookup + omitMap := make(map[string]bool) + for _, col := range tt.omitColumns { + omitMap[col] = true + } + + // Build column list, excluding omitted columns + var columnNames []string + for _, col := range columns { + if !omitMap[col.ColumnName] { + columnNames = append(columnNames, col.ColumnName) + } + } + + // Verify result matches expected + if len(columnNames) != len(tt.expectedCols) { + t.Errorf("expected %d columns, got %d", len(tt.expectedCols), len(columnNames)) + } + + for i, colName := range columnNames { + if i >= len(tt.expectedCols) || colName != tt.expectedCols[i] { + t.Errorf("column mismatch at position %d: expected %s, got %s", i, tt.expectedCols[i], colName) + } + } + }) + } +} + func TestSyncIdentityColumn(t *testing.T) { // This test documents the expected behavior of SyncIdentityColumn // which synchronizes the IDENTITY sequence for a column to match the current maximum value @@ -301,3 +386,95 @@ func TestVerifyPartitionCoverage(t *testing.T) { // and is tested in integration tests t.Log("VerifyPartitionCoverage documented - integration tests required for full validation") } + +func TestRenameTables(t *testing.T) { + // This test documents the expected behavior of RenameTables + // which renames multiple tables atomically in a single transaction + + // The function should: + // 1. Validate that all source tables exist + // 2. Check for conflicts (target table already exists) + // 3. Allow table swaps (where target is also a source) + // 4. Execute all renames in a single transaction + // 5. Rollback all renames if any fail + // 6. Support dry-run mode + + // Example usage: + // renames := map[string]string{ + // "orders_old": "orders_backup", + // "orders_new": "orders", + // } + // count, err := dbc.RenameTables(renames, false) + // if err != nil { + // log.WithError(err).Error("rename failed") + // } + + // Expected behavior: + // - Returns error if any source table doesn't exist + // - Returns error if target table exists (unless it's also a source - table swap) + // - All renames happen atomically (all succeed or all fail) + // - PostgreSQL automatically updates views, indexes, and foreign keys + // - Very fast operation (only metadata update) + // - Dry run returns 0 count but validates everything + + // Test cases for validation logic + tests := []struct { + name string + renames map[string]string + expectError bool + errorContains string + }{ + { + name: "simple rename", + renames: map[string]string{ + "table_old": "table_new", + }, + expectError: false, + }, + { + name: "table swap", + renames: map[string]string{ + "table_a": "table_b", + "table_b": "table_a", + }, + expectError: false, // Allowed - swap scenario + }, + { + name: "multiple renames", + renames: map[string]string{ + "orders_old": "orders_backup", + "orders_new": "orders", + "items_old": "items_backup", + }, + expectError: false, + }, + { + name: "empty map", + renames: map[string]string{}, + expectError: true, + }, + { + name: "three-way swap", + renames: map[string]string{ + "orders": "orders_backup", + "orders_new": "orders", + "orders_backup": "orders_archive", + }, + expectError: false, // Complex swap allowed + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Validate structure + if tt.expectError && len(tt.renames) > 0 { + // Should expect error for valid reason + t.Logf("Expected error case: %s", tt.name) + } + }) + } + + // This is a documentation test - actual functionality requires a live database + // and is tested in integration tests + t.Log("RenameTables documented - integration tests required for full validation") +} From 3cf6503915e077694693a95f8bde4dd730c034c7 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Thu, 12 Mar 2026 15:00:32 -0400 Subject: [PATCH 04/11] TRT-1989: remove deprecated code --- pkg/dataloader/prowloader/prow.go | 33 ------------------------------- 1 file changed, 33 deletions(-) diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index 8af429191..0522b408f 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -384,21 +384,6 @@ func DaysBetween(start, end time.Time) []string { return days } -// NextDay takes a date string in YYYY-MM-DD format and returns the date string for the following day. -func NextDay(dateStr string) (string, error) { - // Parse the input date string - date, err := time.Parse("2006-01-02", dateStr) - if err != nil { - return "", fmt.Errorf("invalid date format: %v", err) - } - - // Add one day to the parsed date - nextDay := date.Add(24 * time.Hour) - - // Format the next day back to YYYY-MM-DD - return nextDay.Format("2006-01-02"), nil -} - // agePartitions detaches and drops old partitions based on configuration func (pl *ProwLoader) agePartitions(config PartitionManagementConfig) error { detached, err := partitions.DetachOldPartitions(pl.dbc, config.TableName, config.DetachAfter, false) @@ -485,24 +470,6 @@ func (pl *ProwLoader) loadDailyTestAnalysisByJob(ctx context.Context) error { dLog.Infof("Loading test analysis by job daily summaries") - // partition creation now managed by preparePartitionsForDailyTestAnalysisByJob - - //nextDay, err := NextDay(dateToImport) - //if err != nil { - // return errors.Wrapf(err, "error parsing next day from %s", dateToImport) - //} - - // create a partition for this date - //partitionSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS test_analysis_by_job_by_dates_%s PARTITION OF test_analysis_by_job_by_dates - // FOR VALUES FROM ('%s') TO ('%s');`, strings.ReplaceAll(dateToImport, "-", "_"), dateToImport, nextDay) - //dLog.Info(partitionSQL) - // - //if res := pl.dbc.DB.Exec(partitionSQL); res.Error != nil { - // log.WithError(res.Error).Error("error creating partition") - // return res.Error - //} - //dLog.Warnf("partition created for releases %v", pl.releases) - q := pl.bigQueryClient.Query(ctx, bqlabel.ProwLoaderTestAnalysis, fmt.Sprintf(`WITH deduped_testcases AS ( SELECT From 277edabe6acdecd347ae2bc5d7b63f6d81bfd723 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Thu, 12 Mar 2026 15:24:54 -0400 Subject: [PATCH 05/11] TRT-1989: identifier quoting --- pkg/db/partitions/README.md | 2 +- pkg/db/partitions/partitions.go | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/db/partitions/README.md b/pkg/db/partitions/README.md index 9c17ffdb7..4ebf21913 100644 --- a/pkg/db/partitions/README.md +++ b/pkg/db/partitions/README.md @@ -1441,7 +1441,7 @@ func main() { ### Input Validation - Partition names are validated against expected format - SQL injection protection through parameterized queries -- Minimum retention period enforcement (30 days) +- Minimum retention period enforcement (90 days) ### Threshold Checks - Maximum 75% of partitions can be deleted diff --git a/pkg/db/partitions/partitions.go b/pkg/db/partitions/partitions.go index 4e3a7c97a..11127cd44 100644 --- a/pkg/db/partitions/partitions.go +++ b/pkg/db/partitions/partitions.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/lib/pq" log "github.com/sirupsen/logrus" "gorm.io/gorm" "gorm.io/gorm/schema" @@ -578,7 +579,7 @@ func DropPartition(dbc *db.DB, partitionName string, dryRun bool) error { return nil } - query := fmt.Sprintf("DROP TABLE IF EXISTS %s", partitionName) + query := "DROP TABLE IF EXISTS " + pq.QuoteIdentifier(partitionName) result := dbc.DB.Exec(query) if result.Error != nil { log.WithError(result.Error).WithFields(log.Fields{ @@ -622,7 +623,7 @@ func DetachPartition(dbc *db.DB, partitionName string, dryRun bool) error { return nil } - query := fmt.Sprintf("ALTER TABLE %s DETACH PARTITION %s", tableName, partitionName) + query := fmt.Sprintf("ALTER TABLE %s DETACH PARTITION %s", pq.QuoteIdentifier(tableName), pq.QuoteIdentifier(partitionName)) result := dbc.DB.Exec(query) if result.Error != nil { log.WithError(result.Error).WithFields(log.Fields{ @@ -976,7 +977,7 @@ func CreateMissingPartitions(dbc *db.DB, tableName string, startDate, endDate ti } // Create the partition table with same structure as parent - createTableQuery := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (LIKE %s INCLUDING ALL)", partitionName, tableName) + createTableQuery := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (LIKE %s INCLUDING ALL)", pq.QuoteIdentifier(partitionName), pq.QuoteIdentifier(tableName)) result := dbc.DB.Exec(createTableQuery) if result.Error != nil { log.WithError(result.Error).WithField("partition", partitionName).Error("failed to create partition table") @@ -986,8 +987,8 @@ func CreateMissingPartitions(dbc *db.DB, tableName string, startDate, endDate ti // Attach the partition to the parent table attachQuery := fmt.Sprintf( "ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM ('%s') TO ('%s')", - tableName, - partitionName, + pq.QuoteIdentifier(tableName), + pq.QuoteIdentifier(partitionName), rangeStart, rangeEnd, ) @@ -995,7 +996,7 @@ func CreateMissingPartitions(dbc *db.DB, tableName string, startDate, endDate ti if result.Error != nil { // If attach fails, try to clean up the created table log.WithError(result.Error).WithField("partition", partitionName).Error("failed to attach partition") - dbc.DB.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partitionName)) + dbc.DB.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", pq.QuoteIdentifier(partitionName))) continue } @@ -1254,7 +1255,7 @@ func CreatePartitionedTable(dbc *db.DB, model interface{}, tableName string, con partitionClause := config.ToSQL() createTableSQL := fmt.Sprintf( "CREATE TABLE IF NOT EXISTS %s (\n %s\n) %s", - tableName, + pq.QuoteIdentifier(tableName), strings.Join(columns, ",\n "), partitionClause, ) @@ -1513,7 +1514,7 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry } alterStatements = append(alterStatements, - fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s", tableName, columnDef)) + fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s", pq.QuoteIdentifier(tableName), columnDef)) } else { // Existing column - check for modifications modifications := []string{} @@ -1577,7 +1578,7 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry // Add modifications as separate ALTER TABLE statements for _, mod := range modifications { alterStatements = append(alterStatements, - fmt.Sprintf("ALTER TABLE %s %s", tableName, mod)) + fmt.Sprintf("ALTER TABLE %s %s", pq.QuoteIdentifier(tableName), mod)) } } @@ -1588,7 +1589,7 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry // Remaining columns in map should be dropped for colName := range currentColMap { alterStatements = append(alterStatements, - fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", tableName, colName)) + fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", pq.QuoteIdentifier(tableName), colName)) } // Check indexes @@ -1979,8 +1980,8 @@ func ReattachPartition(dbc *db.DB, partitionName string, dryRun bool) error { // Reattach the partition with FOR VALUES clause query := fmt.Sprintf( "ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM ('%s') TO ('%s')", - tableName, - partitionName, + pq.QuoteIdentifier(tableName), + pq.QuoteIdentifier(partitionName), startDate, endDate, ) From 99fc72d84b81f3f54a8a9ac389967fc76dc4a194 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Thu, 12 Mar 2026 17:20:18 -0400 Subject: [PATCH 06/11] TRT-1989: refactor --- pkg/dataloader/prowloader/prow.go | 6 +- pkg/db/partitions/partitions.go | 39 ++++- pkg/db/utils.go | 276 ++++++++++-------------------- pkg/db/utils_example.go | 42 +++-- 4 files changed, 147 insertions(+), 216 deletions(-) diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index 0522b408f..3d1a6f049 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -414,16 +414,16 @@ func (pl *ProwLoader) preparePartitions(config PartitionManagementConfig) error fmt.Printf(" Total: %d partitions (%s)\n", stats.TotalPartitions, stats.TotalSizePretty) // When initializing a new table, look back the configured number of days - mostRecentDate := time.Now().Add(-time.Duration(config.InitialLookbackDays) * 24 * time.Hour) + oldestDate := time.Now().Add(-time.Duration(config.InitialLookbackDays) * 24 * time.Hour) if stats.TotalPartitions > 0 { fmt.Printf(" Range: %s to %s\n", stats.OldestDate.Format("2006-01-02"), stats.NewestDate.Format("2006-01-02")) - mostRecentDate = stats.NewestDate + oldestDate = stats.OldestDate } futureDate := time.Now().Add(config.FuturePartitionWindow) - created, err := partitions.CreateMissingPartitions(pl.dbc, config.TableName, mostRecentDate, futureDate, false) + created, err := partitions.CreateMissingPartitions(pl.dbc, config.TableName, oldestDate, futureDate, false) if err != nil { log.WithError(err).Errorf("error creating partitions for %s", config.TableName) return err diff --git a/pkg/db/partitions/partitions.go b/pkg/db/partitions/partitions.go index 11127cd44..c58530278 100644 --- a/pkg/db/partitions/partitions.go +++ b/pkg/db/partitions/partitions.go @@ -654,7 +654,7 @@ func DropOldPartitions(dbc *db.DB, tableName string, retentionDays int, dryRun b } // Get all partitions for removal (both attached and detached) - partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays, false) + partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays, true) if err != nil { return 0, fmt.Errorf("failed to get partitions for removal: %w", err) } @@ -1709,22 +1709,43 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry return fullSQL, nil } - // Execute ALTER statements - successCount := 0 - for _, stmt := range alterStatements { - result := dbc.DB.Exec(stmt) + // Execute ALTER statements in a transaction for atomicity + tx := dbc.DB.Begin() + if tx.Error != nil { + return "", fmt.Errorf("failed to begin transaction: %w", tx.Error) + } + + // Ensure transaction is properly handled + committed := false + defer func() { + if !committed { + tx.Rollback() + } + }() + + for i, stmt := range alterStatements { + result := tx.Exec(stmt) if result.Error != nil { - log.WithError(result.Error).WithField("statement", stmt).Error("failed to execute ALTER statement") - continue + log.WithError(result.Error).WithFields(log.Fields{ + "table": tableName, + "statement": stmt, + "index": i + 1, + "total": len(alterStatements), + }).Error("failed to execute ALTER statement") + return "", fmt.Errorf("failed to execute ALTER statement %d of %d: %w", i+1, len(alterStatements), result.Error) } - successCount++ } + // Commit the transaction + if err := tx.Commit().Error; err != nil { + return "", fmt.Errorf("failed to commit transaction: %w", err) + } + committed = true + elapsed := time.Since(start) log.WithFields(log.Fields{ "table": tableName, "statements": len(alterStatements), - "successful": successCount, "elapsed": elapsed, }).Info("updated partitioned table schema") diff --git a/pkg/db/utils.go b/pkg/db/utils.go index 3ee8a3d59..9133bd4b7 100644 --- a/pkg/db/utils.go +++ b/pkg/db/utils.go @@ -4,10 +4,10 @@ import ( "database/sql" "errors" "fmt" - "sort" "strings" "time" + "github.com/lib/pq" log "github.com/sirupsen/logrus" ) @@ -57,7 +57,7 @@ func DataMigrationColumnVerificationOptions() ColumnVerificationOptions { return ColumnVerificationOptions{ CheckNullable: false, CheckDefaults: false, - CheckOrder: true, + CheckOrder: false, } } @@ -337,10 +337,10 @@ func (dbc *DB) MigrateTableData(sourceTable, targetTable string, omitColumns []s // This is done in a single statement for efficiency and atomicity insertSQL := fmt.Sprintf( "INSERT INTO %s (%s) SELECT %s FROM %s", - targetTable, - strings.Join(columnNames, ", "), - strings.Join(columnNames, ", "), - sourceTable, + pq.QuoteIdentifier(targetTable), + pq.QuoteIdentifier(strings.Join(columnNames, ", ")), + pq.QuoteIdentifier(strings.Join(columnNames, ", ")), + pq.QuoteIdentifier(sourceTable), ) log.WithFields(log.Fields{ @@ -463,7 +463,7 @@ func (dbc *DB) MigrateTableDataRange(sourceTable, targetTable, dateColumn string // Step 3: Count rows in the date range in source table var sourceCount int64 countQuery := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s >= @start_date AND %s < @end_date", - sourceTable, dateColumn, dateColumn) + pq.QuoteIdentifier(sourceTable), pq.QuoteIdentifier(dateColumn), pq.QuoteIdentifier(dateColumn)) result := dbc.DB.Raw(countQuery, sql.Named("start_date", startDate), sql.Named("end_date", endDate)).Scan(&sourceCount) if result.Error != nil { return 0, fmt.Errorf("failed to count rows in date range: %w", result.Error) @@ -527,12 +527,12 @@ func (dbc *DB) MigrateTableDataRange(sourceTable, targetTable, dateColumn string // This is done in a single statement for efficiency and atomicity insertSQL := fmt.Sprintf( "INSERT INTO %s (%s) SELECT %s FROM %s WHERE %s >= @start_date AND %s < @end_date", - targetTable, - strings.Join(columnNames, ", "), - strings.Join(columnNames, ", "), - sourceTable, - dateColumn, - dateColumn, + pq.QuoteIdentifier(targetTable), + pq.QuoteIdentifier(strings.Join(columnNames, ", ")), + pq.QuoteIdentifier(strings.Join(columnNames, ", ")), + pq.QuoteIdentifier(sourceTable), + pq.QuoteIdentifier(dateColumn), + pq.QuoteIdentifier(dateColumn), ) log.WithFields(log.Fields{ @@ -720,7 +720,7 @@ func (dbc *DB) VerifyPartitionCoverage(tableName string, startDate, endDate time func (dbc *DB) GetTableRowCount(tableName string) (int64, error) { var count int64 - query := fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName) + query := fmt.Sprintf("SELECT COUNT(*) FROM %s", pq.QuoteIdentifier(tableName)) result := dbc.DB.Raw(query).Scan(&count) if result.Error != nil { return 0, fmt.Errorf("failed to count rows in table %s: %w", tableName, result.Error) @@ -1068,83 +1068,10 @@ func (dbc *DB) ListAllTableSequences() (map[string][]SequenceInfo, error) { return sequencesByTable, nil } -// topologicalSortRenames orders rename operations to avoid naming conflicts -// If renaming A->B and B->C, we must do B->C first (to free up "B" namespace) -// Returns sorted list of old names in the order they should be processed -// -// Example: -// - Renames: {table_b: table_old, table_a: table_b} -// - Dependencies: table_a depends on table_b (because table_a->table_b and table_b exists) -// - Result: [table_b, table_a] (process table_b first to free up the name) -func topologicalSortRenames(renames map[string]string) []string { - // Build dependency graph - // If we're renaming X->Y, and Y is also being renamed (Y->Z), - // then X depends on Y (Y must be renamed first to free up the name) - dependencies := make(map[string][]string) // map[node][]dependencies - allNodes := make(map[string]bool) - - for oldName := range renames { - allNodes[oldName] = true - dependencies[oldName] = []string{} - } - - for oldName, newName := range renames { - // If newName matches another oldName, we depend on that being renamed first - if allNodes[newName] { - dependencies[oldName] = append(dependencies[oldName], newName) - } - } - - // Topological sort using Kahn's algorithm - // inDegree[node] = number of things that node depends on - inDegree := make(map[string]int) - for node, deps := range dependencies { - inDegree[node] = len(deps) - } - - // Start with nodes that have no dependencies - queue := []string{} - for node := range allNodes { - if inDegree[node] == 0 { - queue = append(queue, node) - } - } - - // Sort queue for deterministic ordering - sort.Strings(queue) - - result := []string{} - for len(queue) > 0 { - // Pop from queue - current := queue[0] - queue = queue[1:] - result = append(result, current) - - // Reduce in-degree of dependent nodes - for node, deps := range dependencies { - for _, dep := range deps { - if dep == current { - inDegree[node]-- - if inDegree[node] == 0 { - queue = append(queue, node) - sort.Strings(queue) // Keep queue sorted for deterministic order - } - } - } - } - } - - // Check for cycles (shouldn't happen with table renames, but be safe) - if len(result) != len(allNodes) { - log.Warn("circular dependency detected in renames, using alphabetical order as fallback") - result = []string{} - for node := range allNodes { - result = append(result, node) - } - sort.Strings(result) - } - - return result +// TableRename represents a single table rename operation +type TableRename struct { + From string // Source table name + To string // Target table name } // RenameTables renames multiple tables atomically in a single transaction @@ -1154,7 +1081,7 @@ func topologicalSortRenames(renames map[string]string) []string { // - Performing atomic schema migrations // // Parameters: -// - tableRenames: Map of source table names to target table names (from -> to) +// - tableRenames: Ordered list of table renames to execute (executed in the order provided) // - renameSequences: If true, also renames sequences owned by table columns (e.g., SERIAL, IDENTITY) // - renamePartitions: If true, also renames child partitions of partitioned tables // - renameConstraints: If true, also renames table constraints (primary keys, foreign keys, unique, check) @@ -1167,10 +1094,10 @@ func topologicalSortRenames(renames map[string]string) []string { // // Example: // -// renames := map[string]string{ -// "orders_old": "orders_backup", -// "orders_new": "orders", -// "orders_archive": "orders_old_archive", +// renames := []db.TableRename{ +// {From: "orders_old", To: "orders_backup"}, // Execute first +// {From: "orders_new", To: "orders"}, // Execute second +// {From: "orders_archive", To: "orders_old"}, // Execute third // } // count, err := dbc.RenameTables(renames, true, true, true, true, false) // if err != nil { @@ -1193,9 +1120,9 @@ func topologicalSortRenames(renames map[string]string) []string { // - When renameIndexes=true, indexes follow naming pattern: newtablename_suffix // - Index renaming applies to all indexes including those backing constraints // - Indexes with the same name as constraints are skipped (they're renamed automatically with the constraint) -// - Renames are processed in sorted order to avoid naming conflicts during table swaps (e.g., A->B, B->C) -// - All operations are deterministic - same input always produces same execution order -func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool, renamePartitions bool, renameConstraints bool, renameIndexes bool, dryRun bool) (int, error) { +// - Renames are executed in the order provided - caller is responsible for dependency ordering +// - For table swaps (A->B, B->C), ensure B->C comes before A->B in the array +func (dbc *DB) RenameTables(tableRenames []TableRename, renameSequences bool, renamePartitions bool, renameConstraints bool, renameIndexes bool, dryRun bool) (int, error) { if len(tableRenames) == 0 { return 0, fmt.Errorf("no tables to rename") } @@ -1205,16 +1132,26 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool "dry_run": dryRun, }).Info("starting table rename operation") - // Step 1: Validate all source tables exist and check for conflicts + // Convert to map for easier lookups during validation and discovery + tableRenameMap := make(map[string]string) var sourceNames []string var targetNames []string - for source, target := range tableRenames { - sourceNames = append(sourceNames, source) - targetNames = append(targetNames, target) + for _, rename := range tableRenames { + if rename.From == "" || rename.To == "" { + return 0, fmt.Errorf("invalid rename: both From and To must be specified") + } + if _, exists := tableRenameMap[rename.From]; exists { + return 0, fmt.Errorf("duplicate source table: %s", rename.From) + } + tableRenameMap[rename.From] = rename.To + sourceNames = append(sourceNames, rename.From) + targetNames = append(targetNames, rename.To) } + // Step 1: Validate all source tables exist and check for conflicts + // Check that all source tables exist - for source := range tableRenames { + for source := range tableRenameMap { var exists bool query := ` SELECT EXISTS ( @@ -1233,9 +1170,9 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool // Check for conflicts - ensure no target tables already exist // (unless they're also being renamed as part of this operation) - for source, target := range tableRenames { + for source, target := range tableRenameMap { // Skip check if this target is also a source (table swap scenario) - if _, isAlsoSource := tableRenames[target]; isAlsoSource { + if _, isAlsoSource := tableRenameMap[target]; isAlsoSource { log.WithFields(log.Fields{ "source": source, "target": target, @@ -1267,7 +1204,7 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool // Step 2: Find sequences that need to be renamed (if requested) sequenceRenames := make(map[string]string) if renameSequences { - for source, target := range tableRenames { + for source, target := range tableRenameMap { sequences, err := dbc.GetTableSequences(source) if err != nil { return 0, fmt.Errorf("failed to get sequences for table %s: %w", source, err) @@ -1296,7 +1233,7 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool // Step 2b: Find partitions that need to be renamed (if requested) partitionRenames := make(map[string]string) if renamePartitions { - for source, target := range tableRenames { + for source, target := range tableRenameMap { partitions, err := dbc.GetTablePartitions(source) if err != nil { return 0, fmt.Errorf("failed to get partitions for table %s: %w", source, err) @@ -1359,7 +1296,7 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool // Step 2c: Find constraints that need to be renamed (if requested) constraintRenames := make(map[string]map[string]string) // map[tableName]map[oldConstraint]newConstraint if renameConstraints { - for source, target := range tableRenames { + for source, target := range tableRenameMap { constraints, err := dbc.GetTableConstraints(source) if err != nil { return 0, fmt.Errorf("failed to get constraints for table %s: %w", source, err) @@ -1445,7 +1382,7 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool // Step 2d: Find indexes that need to be renamed (if requested) indexRenames := make(map[string]map[string]string) // map[tableName]map[oldIndex]newIndex if renameIndexes { - for source, target := range tableRenames { + for source, target := range tableRenameMap { indexes, err := dbc.GetTableIndexes(source) if err != nil { return 0, fmt.Errorf("failed to get indexes for table %s: %w", source, err) @@ -1611,19 +1548,19 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool } }() - // Execute each table rename + // Execute each table rename in the order provided renamedCount := 0 - for source, target := range tableRenames { - renameSQL := fmt.Sprintf("ALTER TABLE %s RENAME TO %s", source, target) + for _, rename := range tableRenames { + renameSQL := fmt.Sprintf("ALTER TABLE %s RENAME TO %s", pq.QuoteIdentifier(rename.From), pq.QuoteIdentifier(rename.To)) log.WithFields(log.Fields{ - "from": source, - "to": target, + "from": rename.From, + "to": rename.To, }).Info("renaming table") result := tx.Exec(renameSQL) if result.Error != nil { - return 0, fmt.Errorf("failed to rename table %s to %s: %w", source, target, result.Error) + return 0, fmt.Errorf("failed to rename table %s to %s: %w", rename.From, rename.To, result.Error) } renamedCount++ @@ -1632,7 +1569,7 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool // Execute each partition rename partitionsRenamed := 0 for oldPart, newPart := range partitionRenames { - renameSQL := fmt.Sprintf("ALTER TABLE %s RENAME TO %s", oldPart, newPart) + renameSQL := fmt.Sprintf("ALTER TABLE %s RENAME TO %s", pq.QuoteIdentifier(oldPart), pq.QuoteIdentifier(newPart)) log.WithFields(log.Fields{ "from": oldPart, @@ -1648,14 +1585,11 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool } // Execute each sequence rename - // Use topological sort to avoid naming conflicts - // (e.g., table_b_id_seq -> table_old_id_seq must happen before table_a_id_seq -> table_b_id_seq) + // Sequences are renamed in the order discovered (matching table rename order) sequencesRenamed := 0 - sortedSeqNames := topologicalSortRenames(sequenceRenames) - for _, oldSeq := range sortedSeqNames { - newSeq := sequenceRenames[oldSeq] - renameSQL := fmt.Sprintf("ALTER SEQUENCE %s RENAME TO %s", oldSeq, newSeq) + for oldSeq, newSeq := range sequenceRenames { + renameSQL := fmt.Sprintf("ALTER SEQUENCE %s RENAME TO %s", pq.QuoteIdentifier(oldSeq), pq.QuoteIdentifier(newSeq)) log.WithFields(log.Fields{ "from": oldSeq, @@ -1671,47 +1605,33 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool } // Execute each constraint rename - // Use topological sort to handle rename dependencies constraintsRenamed := 0 - // Flatten constraint renames into a simple map for topological sort - flatConstraintRenames := make(map[string]string) - constraintToTable := make(map[string]string) for tableName, consMap := range constraintRenames { - for oldCons, newCons := range consMap { - flatConstraintRenames[oldCons] = newCons - constraintToTable[oldCons] = tableName - } - } - - sortedConsNames := topologicalSortRenames(flatConstraintRenames) - - for _, oldCons := range sortedConsNames { - newCons := flatConstraintRenames[oldCons] - tableName := constraintToTable[oldCons] - // Get the new table name (in case table or partition was renamed) newTableName := tableName - if renamed, exists := tableRenames[tableName]; exists { + if renamed, exists := tableRenameMap[tableName]; exists { newTableName = renamed } else if renamed, exists := partitionRenames[tableName]; exists { newTableName = renamed } - renameSQL := fmt.Sprintf("ALTER TABLE %s RENAME CONSTRAINT %s TO %s", newTableName, oldCons, newCons) + for oldCons, newCons := range consMap { + renameSQL := fmt.Sprintf("ALTER TABLE %s RENAME CONSTRAINT %s TO %s", pq.QuoteIdentifier(newTableName), pq.QuoteIdentifier(oldCons), pq.QuoteIdentifier(newCons)) - log.WithFields(log.Fields{ - "table": newTableName, - "from": oldCons, - "to": newCons, - }).Info("renaming constraint") + log.WithFields(log.Fields{ + "table": newTableName, + "from": oldCons, + "to": newCons, + }).Info("renaming constraint") + + result := tx.Exec(renameSQL) + if result.Error != nil { + return 0, fmt.Errorf("failed to rename constraint %s to %s on table %s: %w", oldCons, newCons, newTableName, result.Error) + } - result := tx.Exec(renameSQL) - if result.Error != nil { - return 0, fmt.Errorf("failed to rename constraint %s to %s on table %s: %w", oldCons, newCons, newTableName, result.Error) + constraintsRenamed++ } - - constraintsRenamed++ } // Build a set of constraint names that were renamed @@ -1724,49 +1644,35 @@ func (dbc *DB) RenameTables(tableRenames map[string]string, renameSequences bool } // Execute each index rename - // Use topological sort to handle rename dependencies indexesRenamed := 0 - // Flatten index renames into a simple map for topological sort - flatIndexRenames := make(map[string]string) - indexToTable := make(map[string]string) for tableName, idxMap := range indexRenames { for oldIdx, newIdx := range idxMap { - flatIndexRenames[oldIdx] = newIdx - indexToTable[oldIdx] = tableName - } - } - - sortedIdxNames := topologicalSortRenames(flatIndexRenames) + // Skip if this index has the same name as a constraint we renamed + // PostgreSQL automatically renames the backing index when renaming PRIMARY KEY or UNIQUE constraints + if renamedConstraintNames[oldIdx] { + log.WithFields(log.Fields{ + "table": tableName, + "index": oldIdx, + }).Debug("skipping index - already renamed as part of constraint rename") + continue + } - for _, oldIdx := range sortedIdxNames { - newIdx := flatIndexRenames[oldIdx] - tableName := indexToTable[oldIdx] + renameSQL := fmt.Sprintf("ALTER INDEX %s RENAME TO %s", pq.QuoteIdentifier(oldIdx), pq.QuoteIdentifier(newIdx)) - // Skip if this index has the same name as a constraint we renamed - // PostgreSQL automatically renames the backing index when renaming PRIMARY KEY or UNIQUE constraints - if renamedConstraintNames[oldIdx] { log.WithFields(log.Fields{ "table": tableName, - "index": oldIdx, - }).Debug("skipping index - already renamed as part of constraint rename") - continue - } + "from": oldIdx, + "to": newIdx, + }).Info("renaming index") - renameSQL := fmt.Sprintf("ALTER INDEX %s RENAME TO %s", oldIdx, newIdx) - - log.WithFields(log.Fields{ - "table": tableName, - "from": oldIdx, - "to": newIdx, - }).Info("renaming index") + result := tx.Exec(renameSQL) + if result.Error != nil { + return 0, fmt.Errorf("failed to rename index %s to %s: %w", oldIdx, newIdx, result.Error) + } - result := tx.Exec(renameSQL) - if result.Error != nil { - return 0, fmt.Errorf("failed to rename index %s to %s: %w", oldIdx, newIdx, result.Error) + indexesRenamed++ } - - indexesRenamed++ } // Commit the transaction @@ -1819,7 +1725,7 @@ func (dbc *DB) SyncIdentityColumn(tableName, columnName string) error { // Get the current maximum value var maxValue sql.NullInt64 - query := fmt.Sprintf("SELECT MAX(%s) FROM %s", columnName, tableName) + query := fmt.Sprintf("SELECT MAX(%s) FROM %s", pq.QuoteIdentifier(columnName), pq.QuoteIdentifier(tableName)) result := dbc.DB.Raw(query).Scan(&maxValue) if result.Error != nil { return fmt.Errorf("failed to get max value for %s.%s: %w", tableName, columnName, result.Error) @@ -1841,7 +1747,7 @@ func (dbc *DB) SyncIdentityColumn(tableName, columnName string) error { // Restart the identity sequence // NOTE: PostgreSQL requires "RESTART WITH" for IDENTITY columns, not "SYNC IDENTITY" // This is the standard way to synchronize an IDENTITY sequence in PostgreSQL - alterSQL := fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s RESTART WITH %d", tableName, columnName, nextValue) + alterSQL := fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s RESTART WITH %d", pq.QuoteIdentifier(tableName), pq.QuoteIdentifier(columnName), nextValue) result = dbc.DB.Exec(alterSQL) if result.Error != nil { return fmt.Errorf("failed to sync identity for %s.%s: %w", tableName, columnName, result.Error) diff --git a/pkg/db/utils_example.go b/pkg/db/utils_example.go index 439b233a8..1beefb068 100644 --- a/pkg/db/utils_example.go +++ b/pkg/db/utils_example.go @@ -817,7 +817,7 @@ func ExampleCheckSequencesBeforeRename(dbc *DB, oldTableName, newTableName strin } // Now perform the rename - renames := map[string]string{oldTableName: newTableName} + renames := []TableRename{{From: oldTableName, To: newTableName}} count, err := dbc.RenameTables(renames, true, false, false, false, false) if err != nil { log.WithError(err).Error("rename failed") @@ -869,10 +869,10 @@ func ExampleCheckAndCreatePartitions(dbc *DB, tableName string, startDate, endDa func ExampleRenameTables(dbc *DB) { log.Info("renaming multiple tables atomically") - // Define table renames - renames := map[string]string{ - "orders_old": "orders_backup", - "orders_new": "orders", + // Define table renames (order matters for dependencies) + renames := []TableRename{ + {From: "orders_old", To: "orders_backup"}, + {From: "orders_new", To: "orders"}, } // Dry run first to verify @@ -920,11 +920,11 @@ func ExampleSwapPartitionedTable(dbc *DB, oldTable, newPartitionedTable string) log.Info("row counts match - proceeding with table swap") // Step 2: Perform atomic rename to swap tables - // orders -> orders_old - // orders_partitioned -> orders - renames := map[string]string{ - oldTable: oldTable + "_old", - newPartitionedTable: oldTable, + // Order matters: rename orders -> orders_old first to free up the "orders" name + // Then rename orders_partitioned -> orders + renames := []TableRename{ + {From: oldTable, To: oldTable + "_old"}, + {From: newPartitionedTable, To: oldTable}, } // Rename sequences and partitions too so they match the new table names @@ -973,10 +973,14 @@ func ExampleThreeWayTableSwap(dbc *DB) { // orders_new -> orders // orders_backup -> orders_archive - renames := map[string]string{ - "orders": "orders_backup", - "orders_new": "orders", - "orders_backup": "orders_archive", + // Order matters for three-way swap - must free up names in the right order: + // 1. orders_backup -> orders_archive (frees up "orders_backup") + // 2. orders -> orders_backup (frees up "orders") + // 3. orders_new -> orders (swaps in new production table) + renames := []TableRename{ + {From: "orders_backup", To: "orders_archive"}, + {From: "orders", To: "orders_backup"}, + {From: "orders_new", To: "orders"}, } // Dry run first (also check sequence renames) @@ -1016,12 +1020,12 @@ func ExampleRollbackTableSwap(dbc *DB) { // orders_partitioned -> orders // // To rollback: - // orders -> orders_partitioned (restore original name) - // orders_old -> orders (restore to production) + // 1. orders -> orders_partitioned (frees up "orders") + // 2. orders_old -> orders (restore to production) - rollbackRenames := map[string]string{ - "orders": "orders_partitioned", - "orders_old": "orders", + rollbackRenames := []TableRename{ + {From: "orders", To: "orders_partitioned"}, + {From: "orders_old", To: "orders"}, } // Rename sequences and partitions back too From 0ffc9f0ffb613f31b357f292d304c570b279f4b6 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Thu, 12 Mar 2026 17:21:25 -0400 Subject: [PATCH 07/11] TRT-1989: remove examples --- pkg/db/partitions/examples.go | 760 ------------------------ pkg/db/utils_example.go | 1039 --------------------------------- 2 files changed, 1799 deletions(-) delete mode 100644 pkg/db/partitions/examples.go delete mode 100644 pkg/db/utils_example.go diff --git a/pkg/db/partitions/examples.go b/pkg/db/partitions/examples.go deleted file mode 100644 index 6b6c4a0c4..000000000 --- a/pkg/db/partitions/examples.go +++ /dev/null @@ -1,760 +0,0 @@ -package partitions - -import ( - "fmt" - "strings" - "time" - - log "github.com/sirupsen/logrus" - - "github.com/openshift/sippy/pkg/db" -) - -// ExampleListPartitionedTables demonstrates how to list all partitioned tables -// -// Usage: -// -// ExampleListPartitionedTables(dbc) -func ExampleListPartitionedTables(dbc *db.DB) { - tables, err := ListPartitionedTables(dbc) - if err != nil { - log.WithError(err).Error("failed to list partitioned tables") - return - } - - fmt.Printf("Found %d partitioned tables:\n", len(tables)) - for _, t := range tables { - fmt.Printf(" %s: %d partitions, Strategy: %s\n", - t.TableName, t.PartitionCount, t.PartitionStrategy) - } -} - -// ExampleListPartitions demonstrates how to list partitions for a table -// If retentionDays > 0, only shows partitions older than that value -// If retentionDays <= 0, shows all partitions -// -// Usage: -// -// ExampleListPartitions(dbc, "test_analysis_by_job_by_dates", 180) // Show partitions older than 180 days -// ExampleListPartitions(dbc, "test_analysis_by_job_by_dates", 0) // Show all partitions -func ExampleListPartitions(dbc *db.DB, tableName string, retentionDays int) { - partitions, err := ListTablePartitions(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to list partitions") - return - } - - if retentionDays > 0 { - fmt.Printf("Partitions older than %d days for %s:\n", retentionDays, tableName) - } else { - fmt.Printf("All partitions for %s:\n", tableName) - } - - for _, p := range partitions { - if p.Age > retentionDays || retentionDays < 1 { - fmt.Printf(" %s - Date: %s, Age: %d days, Size: %s\n", - p.TableName, p.PartitionDate.Format("2006-01-02"), p.Age, p.SizePretty) - } - } -} - -// ExampleGetStats demonstrates how to get partition statistics -// -// Usage: -// -// ExampleGetStats(dbc, "test_analysis_by_job_by_dates") -func ExampleGetStats(dbc *db.DB, tableName string) { - stats, err := GetPartitionStats(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to get stats") - return - } - - fmt.Printf("\nPartition Statistics for %s:\n", tableName) - fmt.Printf(" Total Partitions: %d\n", stats.TotalPartitions) - fmt.Printf(" Total Size: %s\n", stats.TotalSizePretty) - fmt.Printf(" Average Size: %s\n", stats.AvgSizePretty) - fmt.Printf(" Date Range: %s to %s\n", - stats.OldestDate.Format("2006-01-02"), - stats.NewestDate.Format("2006-01-02")) -} - -// ExampleComparePartitionStats demonstrates comparing attached vs detached partition statistics -// -// Usage: -// -// ExampleComparePartitionStats(dbc, "test_analysis_by_job_by_dates") -func ExampleComparePartitionStats(dbc *db.DB, tableName string) { - fmt.Printf("\n=== Partition Statistics Comparison for %s ===\n", tableName) - - // Get all partition stats - allStats, err := GetPartitionStats(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to get all partition stats") - return - } - - // Get attached partition stats - attachedStats, err := GetAttachedPartitionStats(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to get attached partition stats") - return - } - - // Get detached partition stats - detachedStats, err := GetDetachedPartitionStats(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to get detached partition stats") - return - } - - fmt.Printf("\nAll Partitions (Attached + Detached):\n") - fmt.Printf(" Total: %d partitions (%s)\n", allStats.TotalPartitions, allStats.TotalSizePretty) - - fmt.Printf("\nAttached Partitions:\n") - fmt.Printf(" Total: %d partitions (%s)\n", attachedStats.TotalPartitions, attachedStats.TotalSizePretty) - if attachedStats.TotalPartitions > 0 { - fmt.Printf(" Range: %s to %s\n", - attachedStats.OldestDate.Format("2006-01-02"), - attachedStats.NewestDate.Format("2006-01-02")) - } - - fmt.Printf("\nDetached Partitions:\n") - fmt.Printf(" Total: %d partitions (%s)\n", detachedStats.TotalPartitions, detachedStats.TotalSizePretty) - if detachedStats.TotalPartitions > 0 { - fmt.Printf(" Range: %s to %s\n", - detachedStats.OldestDate.Format("2006-01-02"), - detachedStats.NewestDate.Format("2006-01-02")) - } - - // Calculate percentages - if allStats.TotalPartitions > 0 { - attachedPct := float64(attachedStats.TotalPartitions) / float64(allStats.TotalPartitions) * 100 - detachedPct := float64(detachedStats.TotalPartitions) / float64(allStats.TotalPartitions) * 100 - fmt.Printf("\nDistribution:\n") - fmt.Printf(" Attached: %.1f%%\n", attachedPct) - fmt.Printf(" Detached: %.1f%%\n", detachedPct) - } -} - -// ExampleCheckRetentionPolicy demonstrates how to check what a retention policy would affect -// -// Usage: -// -// ExampleCheckRetentionPolicy(dbc, "test_analysis_by_job_by_dates", 180) -func ExampleCheckRetentionPolicy(dbc *db.DB, tableName string, retentionDays int) { - // First validate the policy - if err := ValidateRetentionPolicy(dbc, tableName, retentionDays); err != nil { - log.WithError(err).Error("retention policy validation failed") - return - } - - // Get summary of what would be affected (all partitions) - summary, err := GetRetentionSummary(dbc, tableName, retentionDays, false) - if err != nil { - log.WithError(err).Error("failed to get retention summary") - return - } - - fmt.Printf("\nRetention Policy Analysis for %s (%d days):\n", tableName, retentionDays) - fmt.Printf(" Cutoff Date: %s\n", summary.CutoffDate.Format("2006-01-02")) - fmt.Printf(" Partitions to Remove: %d\n", summary.PartitionsToRemove) - fmt.Printf(" Storage to Reclaim: %s\n", summary.StoragePretty) - if summary.PartitionsToRemove > 0 { - fmt.Printf(" Oldest: %s\n", summary.OldestPartition) - fmt.Printf(" Newest: %s\n", summary.NewestPartition) - } - - // Get detailed list of partitions that would be removed (both attached and detached) - partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays, false) - if err != nil { - log.WithError(err).Error("failed to get partitions for removal") - return - } - - if len(partitions) > 0 { - fmt.Printf("\nPartitions that would be removed (showing first 10):\n") - for i, p := range partitions { - if i < 10 { - fmt.Printf(" %s - %s ago, Size: %s\n", - p.TableName, p.PartitionDate.Format("2006-01-02"), p.SizePretty) - } - } - if len(partitions) > 10 { - fmt.Printf(" ... and %d more\n", len(partitions)-10) - } - } -} - -// ExampleAgeGroupAnalysis demonstrates how to analyze partitions by age -// -// Usage: -// -// ExampleAgeGroupAnalysis(dbc, "test_analysis_by_job_by_dates") -func ExampleAgeGroupAnalysis(dbc *db.DB, tableName string) { - groups, err := GetPartitionsByAgeGroup(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to get age groups") - return - } - - fmt.Printf("\nPartitions by Age Group for %s:\n", tableName) - for _, group := range groups { - fmt.Printf(" %s: %d partitions, %s (%.2f%%)\n", - group["age_bucket"], - group["partition_count"], - group["total_size"], - group["percentage"]) - } -} - -// ExampleMonthlyAnalysis demonstrates how to analyze partitions by month -// -// Usage: -// -// ExampleMonthlyAnalysis(dbc, "test_analysis_by_job_by_dates") -func ExampleMonthlyAnalysis(dbc *db.DB, tableName string) { - months, err := GetPartitionsByMonth(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to get monthly breakdown") - return - } - - fmt.Printf("\nPartitions by Month for %s (recent):\n", tableName) - for i, month := range months { - if i < 6 { // Show last 6 months - fmt.Printf(" %v: %d partitions, Total: %s, Avg: %s\n", - month["month"], - month["partition_count"], - month["total_size"], - month["avg_partition_size"]) - } - } -} - -// ExampleDryRunCleanup demonstrates a dry-run cleanup operation -// -// Usage: -// -// ExampleDryRunCleanup(dbc, "test_analysis_by_job_by_dates", 180) -func ExampleDryRunCleanup(dbc *db.DB, tableName string, retentionDays int) { - fmt.Printf("\n=== DRY RUN: Partition Cleanup for %s (%d day retention) ===\n", tableName, retentionDays) - - // Validate policy - if err := ValidateRetentionPolicy(dbc, tableName, retentionDays); err != nil { - log.WithError(err).Error("retention policy failed validation") - return - } - - // Get summary (all partitions, matching DropOldPartitions behavior) - summary, err := GetRetentionSummary(dbc, tableName, retentionDays, false) - if err != nil { - log.WithError(err).Error("failed to get summary") - return - } - - if summary.PartitionsToRemove == 0 { - fmt.Println("No partitions to delete") - return - } - - fmt.Printf("Would delete %d partitions, reclaiming %s\n", - summary.PartitionsToRemove, summary.StoragePretty) - - // Perform dry run - dropped, err := DropOldPartitions(dbc, tableName, retentionDays, true) // true = dry run - if err != nil { - log.WithError(err).Error("dry run failed") - return - } - - fmt.Printf("Dry run completed: would drop %d partitions\n", dropped) -} - -// ExampleDetachedPartitions demonstrates working with detached partitions for a table -// -// Usage: -// -// ExampleDetachedPartitions(dbc, "test_analysis_by_job_by_dates") -func ExampleDetachedPartitions(dbc *db.DB, tableName string) { - fmt.Printf("\n=== Detached Partitions for %s ===\n", tableName) - - // List detached partitions - detached, err := ListDetachedPartitions(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to list detached partitions") - return - } - - if len(detached) == 0 { - fmt.Println("No detached partitions found") - return - } - - fmt.Printf("Found %d detached partitions:\n", len(detached)) - for i, p := range detached { - if i < 5 { - fmt.Printf(" %s - Date: %s, Size: %s\n", - p.TableName, p.PartitionDate.Format("2006-01-02"), p.SizePretty) - } - } - - // Get statistics about detached partitions - stats, err := GetDetachedPartitionStats(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to get detached stats") - return - } - - fmt.Printf("\nDetached Partition Statistics:\n") - fmt.Printf(" Total: %d partitions (%s)\n", stats.TotalPartitions, stats.TotalSizePretty) - if stats.TotalPartitions > 0 { - fmt.Printf(" Range: %s to %s\n", - stats.OldestDate.Format("2006-01-02"), - stats.NewestDate.Format("2006-01-02")) - } -} - -// ExampleAttachedPartitions demonstrates working with attached partitions for a table -// -// Usage: -// -// ExampleAttachedPartitions(dbc, "test_analysis_by_job_by_dates") -func ExampleAttachedPartitions(dbc *db.DB, tableName string) { - fmt.Printf("\n=== Attached Partitions for %s ===\n", tableName) - - // List attached partitions - attached, err := ListAttachedPartitions(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to list attached partitions") - return - } - - if len(attached) == 0 { - fmt.Println("No attached partitions found") - return - } - - fmt.Printf("Found %d attached partitions:\n", len(attached)) - for i, p := range attached { - if i < 10 { - fmt.Printf(" %s - Date: %s, Age: %d days, Size: %s\n", - p.TableName, p.PartitionDate.Format("2006-01-02"), p.Age, p.SizePretty) - } - } - - if len(attached) > 10 { - fmt.Printf(" ... and %d more\n", len(attached)-10) - } - - // Calculate total size - var totalSize int64 - for _, p := range attached { - totalSize += p.SizeBytes - } - - fmt.Printf("\nAttached Partition Summary:\n") - fmt.Printf(" Total: %d partitions\n", len(attached)) - fmt.Printf(" Total Size: %d bytes\n", totalSize) - if len(attached) > 0 { - fmt.Printf(" Range: %s to %s\n", - attached[0].PartitionDate.Format("2006-01-02"), - attached[len(attached)-1].PartitionDate.Format("2006-01-02")) - } -} - -// ExampleDropOldDetachedPartitions demonstrates dropping old detached partitions -// -// Usage: -// -// ExampleDropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180) -func ExampleDropOldDetachedPartitions(dbc *db.DB, tableName string, retentionDays int) { - fmt.Printf("\n=== Drop Old Detached Partitions for %s (%d days) ===\n", tableName, retentionDays) - - // 1. Check what detached partitions exist - detached, err := ListDetachedPartitions(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to list detached partitions") - return - } - - if len(detached) == 0 { - fmt.Println("No detached partitions found") - return - } - - fmt.Printf("Found %d detached partitions\n", len(detached)) - - // 2. Show which ones would be dropped - cutoffDate := time.Now().AddDate(0, 0, -retentionDays) - fmt.Printf("Cutoff date: %s\n", cutoffDate.Format("2006-01-02")) - - toRemove := 0 - var totalSize int64 - for _, p := range detached { - if p.PartitionDate.Before(cutoffDate) { - toRemove++ - totalSize += p.SizeBytes - if toRemove <= 5 { - fmt.Printf(" Would drop: %s (Age: %d days, Size: %s)\n", - p.TableName, p.Age, p.SizePretty) - } - } - } - - if toRemove > 5 { - fmt.Printf(" ... and %d more\n", toRemove-5) - } - - if toRemove == 0 { - fmt.Println("No detached partitions older than retention period") - return - } - - fmt.Printf("\nTotal to remove: %d partitions\n", toRemove) - - // 3. Dry run - fmt.Println("\nRunning dry run...") - dropped, err := DropOldDetachedPartitions(dbc, tableName, retentionDays, true) - if err != nil { - log.WithError(err).Error("dry run failed") - return - } - - fmt.Printf("Dry run completed: would drop %d detached partitions\n", dropped) - - // 4. Actual drop (commented out for safety) - // fmt.Println("\nActual drop (uncomment to execute):") - // dropped, err = DropOldDetachedPartitions(dbc, tableName, retentionDays, false) - // if err != nil { - // log.WithError(err).Error("drop failed") - // return - // } - // fmt.Printf("Dropped %d detached partitions\n", dropped) -} - -// ExampleDetachWorkflow demonstrates the detach/archive workflow -// -// Usage: -// -// ExampleDetachWorkflow(dbc, "test_analysis_by_job_by_dates", 180) -func ExampleDetachWorkflow(dbc *db.DB, tableName string, retentionDays int) { - fmt.Printf("\n=== Detach Workflow for %s (%d days) ===\n", tableName, retentionDays) - - // 1. Check what would be detached (only attached partitions) - summary, err := GetRetentionSummary(dbc, tableName, retentionDays, true) - if err != nil { - log.WithError(err).Error("failed to get summary") - return - } - - fmt.Printf("1. Would detach %d partitions (%s)\n", - summary.PartitionsToRemove, summary.StoragePretty) - - // 2. Detach partitions (dry run) - detached, err := DetachOldPartitions(dbc, tableName, retentionDays, true) - if err != nil { - log.WithError(err).Error("dry run failed") - return - } - - fmt.Printf("2. Dry run: would detach %d partitions\n", detached) - - // 3. Actual detach (commented out - requires admin) - // detached, err = DetachOldPartitions(dbc, tableName, retentionDays, false) - // fmt.Printf("3. Detached %d partitions\n", detached) - - // 4. Check detached partitions - fmt.Println("\n4. After detachment, you can:") - fmt.Println(" - Archive to S3 using external scripts") - fmt.Println(" - Compress and store offline") - fmt.Println(" - Query detached tables directly if needed") - fmt.Println(" - Reattach if data is needed again") - fmt.Println(" - Drop when ready to free storage") -} - -// ExampleReattachPartition demonstrates reattaching a detached partition -// -// Usage: -// -// ExampleReattachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29") -func ExampleReattachPartition(dbc *db.DB, partitionName string) { - fmt.Printf("\n=== Reattach Partition: %s ===\n", partitionName) - - // 1. Check if partition is attached - isAttached, err := IsPartitionAttached(dbc, partitionName) - if err != nil { - log.WithError(err).Error("failed to check partition status") - return - } - - fmt.Printf("1. Partition attached: %v\n", isAttached) - - if isAttached { - fmt.Println("Partition is already attached, no action needed") - return - } - - // 2. Reattach (dry run) - err = ReattachPartition(dbc, partitionName, true) - if err != nil { - log.WithError(err).Error("dry run failed") - return - } - - fmt.Println("2. Dry run successful") - - // 3. Actual reattach (commented out - requires admin) - // err = ReattachPartition(dbc, partitionName, false) - // if err != nil { - // log.WithError(err).Error("reattach failed") - // return - // } - // fmt.Println("3. Partition reattached successfully") -} - -// ExampleCreateMissingPartitions demonstrates creating missing partitions for a date range -// -// Usage: -// -// ExampleCreateMissingPartitions(dbc, "test_analysis_by_job_by_dates", "2024-01-01", "2024-01-31") -func ExampleCreateMissingPartitions(dbc *db.DB, tableName, startDateStr, endDateStr string) { - fmt.Printf("\n=== Create Missing Partitions for %s ===\n", tableName) - - // Parse dates - startDate, err := time.Parse("2006-01-02", startDateStr) - if err != nil { - log.WithError(err).Error("failed to parse start date") - return - } - - endDate, err := time.Parse("2006-01-02", endDateStr) - if err != nil { - log.WithError(err).Error("failed to parse end date") - return - } - - fmt.Printf("Date range: %s to %s\n", startDateStr, endDateStr) - - // Calculate expected number of partitions - days := int(endDate.Sub(startDate).Hours()/24) + 1 - fmt.Printf("Expected partitions: %d (one per day)\n", days) - - // Check current partitions - existing, err := ListTablePartitions(dbc, tableName) - if err != nil { - log.WithError(err).Error("failed to list existing partitions") - return - } - fmt.Printf("Existing partitions: %d\n", len(existing)) - - // Dry run to see what would be created - fmt.Println("\nDry run...") - created, err := CreateMissingPartitions(dbc, tableName, startDate, endDate, true) - if err != nil { - log.WithError(err).Error("dry run failed") - return - } - - if created == 0 { - fmt.Println("All partitions already exist - no action needed") - return - } - - fmt.Printf("Would create %d missing partitions\n", created) - - // Actual creation (commented out for safety) - // fmt.Println("\nCreating partitions...") - // created, err = CreateMissingPartitions(dbc, tableName, startDate, endDate, false) - // if err != nil { - // log.WithError(err).Error("creation failed") - // return - // } - // fmt.Printf("Successfully created %d partitions\n", created) -} - -// ExampleCreatePartitionedTable demonstrates creating a new partitioned table from a GORM model -// -// Usage: -// -// config := partitions.NewRangePartitionConfig("created_at") -// ExampleCreatePartitionedTable(dbc, &models.MyModel{}, "my_partitioned_table", config) -func ExampleCreatePartitionedTable(dbc *db.DB, model interface{}, tableName string, config PartitionConfig) { - fmt.Printf("\n=== Create Partitioned Table: %s ===\n", tableName) - fmt.Printf("Strategy: %s\n", config.Strategy) - fmt.Printf("Partition by: %s\n", strings.Join(config.Columns, ", ")) - - // Dry run to see the SQL that would be executed - fmt.Println("\nDry run - SQL that would be executed:") - sql, err := CreatePartitionedTable(dbc, model, tableName, config, true) - if err != nil { - log.WithError(err).Error("dry run failed") - return - } - - if sql == "" { - fmt.Println("Table already exists - no action needed") - return - } - - // Actual creation (commented out for safety) - // fmt.Println("\nCreating partitioned table...") - // _, err = CreatePartitionedTable(dbc, model, tableName, config, false) - // if err != nil { - // log.WithError(err).Error("table creation failed") - // return - // } - // fmt.Printf("Successfully created partitioned table: %s\n", tableName) - // - // // For RANGE partitions, create partitions for your date range - // if config.Strategy == db.PartitionStrategyRange { - // startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) - // endDate := time.Now() - // created, err := CreateMissingPartitions(dbc, tableName, startDate, endDate, false) - // if err != nil { - // log.WithError(err).Error("partition creation failed") - // return - // } - // fmt.Printf("Created %d partitions\n", created) - // } - // - // // For HASH partitions, create the required number of partitions - // if config.Strategy == db.PartitionStrategyHash { - // for i := 0; i < config.Modulus; i++ { - // partName := fmt.Sprintf("%s_%d", tableName, i) - // sql := fmt.Sprintf("CREATE TABLE %s PARTITION OF %s FOR VALUES WITH (MODULUS %d, REMAINDER %d)", - // partName, tableName, config.Modulus, i) - // if err := dbc.DB.Exec(sql).Error; err != nil { - // log.WithError(err).Errorf("failed to create partition %s", partName) - // } - // } - // } -} - -// ExampleUpdatePartitionedTable demonstrates updating an existing partitioned table schema -// -// Usage: -// -// ExampleUpdatePartitionedTable(dbc, &models.MyModel{}, "my_partitioned_table") -func ExampleUpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string) { - fmt.Printf("\n=== Update Partitioned Table Schema: %s ===\n", tableName) - - // Dry run to see what changes would be made - fmt.Println("\nDry run - checking for schema changes:") - sql, err := UpdatePartitionedTable(dbc, model, tableName, true) - if err != nil { - log.WithError(err).Error("dry run failed") - return - } - - if sql == "" { - fmt.Println("Schema is up to date - no changes needed") - return - } - - fmt.Printf("\nChanges detected:\n%s\n", sql) - - // Actual update (commented out for safety) - // fmt.Println("\nApplying schema changes...") - // _, err = UpdatePartitionedTable(dbc, model, tableName, false) - // if err != nil { - // log.WithError(err).Error("schema update failed") - // return - // } - // fmt.Printf("Successfully updated table schema: %s\n", tableName) -} - -// ExampleWorkflowForAnyTable demonstrates managing partitions for any table -// -// Usage: -// -// ExampleWorkflowForAnyTable(dbc) -func ExampleWorkflowForAnyTable(dbc *db.DB) { - fmt.Println("=== Managing Partitions for Any Table ===") - - // 1. List all partitioned tables - fmt.Println("\n1. Discovering partitioned tables:") - tables, err := ListPartitionedTables(dbc) - if err != nil { - log.WithError(err).Error("failed to list partitioned tables") - return - } - - for _, table := range tables { - fmt.Printf(" - %s: %d partitions (%s)\n", - table.TableName, table.PartitionCount, table.PartitionStrategy) - } - - // 2. For each table, analyze retention - fmt.Println("\n2. Analyzing retention policies:") - for _, table := range tables { - fmt.Printf("\nTable: %s\n", table.TableName) - - // Get current stats - stats, err := GetPartitionStats(dbc, table.TableName) - if err != nil { - log.WithError(err).WithField("table", table.TableName).Error("failed to get stats") - continue - } - - fmt.Printf(" Total: %d partitions (%s)\n", - stats.TotalPartitions, stats.TotalSizePretty) - fmt.Printf(" Range: %s to %s\n", - stats.OldestDate.Format("2006-01-02"), - stats.NewestDate.Format("2006-01-02")) - - // Check 180-day retention policy (all partitions) - summary, err := GetRetentionSummary(dbc, table.TableName, 180, false) - if err != nil { - log.WithError(err).WithField("table", table.TableName).Error("failed to get summary") - continue - } - - if summary.PartitionsToRemove > 0 { - fmt.Printf(" 180-day policy: Would remove %d partitions (%s)\n", - summary.PartitionsToRemove, summary.StoragePretty) - } else { - fmt.Println(" 180-day policy: No partitions to remove") - } - } -} - -// ExampleCompleteWorkflow demonstrates a complete partition management workflow for a specific table -// -// Usage: -// -// ExampleCompleteWorkflow(dbc, "test_analysis_by_job_by_dates") -func ExampleCompleteWorkflow(dbc *db.DB, tableName string) { - fmt.Printf("=== Partition Management Workflow for %s ===\n", tableName) - - // 1. Get current state - fmt.Println("\n1. Current State:") - ExampleGetStats(dbc, tableName) - - // 2. Analyze by age - fmt.Println("\n2. Age Distribution:") - ExampleAgeGroupAnalysis(dbc, tableName) - - // 3. Check various retention policies - for _, days := range []int{90, 180, 365} { - fmt.Printf("\n3. Analyzing %d-day retention policy:\n", days) - ExampleCheckRetentionPolicy(dbc, tableName, days) - } - - // 4. Recommended: 180-day retention dry run - fmt.Println("\n4. Recommended Policy (180 days):") - ExampleDryRunCleanup(dbc, tableName, 180) - - // 5. Check for detached partitions - fmt.Println("\n5. Detached Partitions:") - ExampleDetachedPartitions(dbc, tableName) - - fmt.Println("\n=== Workflow Complete ===") - fmt.Println("Options for cleanup:") - fmt.Printf(" 1. DROP immediately:\n") - fmt.Printf(" dropped, err := partitions.DropOldPartitions(dbc, \"%s\", 180, false)\n", tableName) - fmt.Printf(" 2. DETACH for archival:\n") - fmt.Printf(" detached, err := partitions.DetachOldPartitions(dbc, \"%s\", 180, false)\n", tableName) - fmt.Println(" // Archive detached partitions to S3") - fmt.Println(" // Drop detached partitions when archived") -} diff --git a/pkg/db/utils_example.go b/pkg/db/utils_example.go deleted file mode 100644 index 1beefb068..000000000 --- a/pkg/db/utils_example.go +++ /dev/null @@ -1,1039 +0,0 @@ -package db - -import ( - "fmt" - "time" - - log "github.com/sirupsen/logrus" -) - -// ExampleVerifyTablesHaveSameColumns demonstrates how to verify that two tables have identical columns -// -// This is useful for: -// - Verifying partition tables match the parent table structure -// - Ensuring schema consistency before data migration -// - Validating table clones or backups -// -// Usage: -// -// err := dbc.VerifyTablesHaveSameColumns("source_table", "target_table", DefaultColumnVerificationOptions()) -// if err != nil { -// log.WithError(err).Error("tables have different schemas") -// } -func ExampleVerifyTablesHaveSameColumns(dbc *DB, sourceTable, targetTable string) { - log.WithFields(log.Fields{ - "source": sourceTable, - "target": targetTable, - }).Info("verifying tables have identical columns") - - // Use default options to verify all aspects: names, types, nullable, defaults, and order - err := dbc.VerifyTablesHaveSameColumns(sourceTable, targetTable, DefaultColumnVerificationOptions()) - if err != nil { - log.WithError(err).Error("table schema verification failed") - return - } - - log.Info("tables have identical column definitions") -} - -// ExampleVerifyPartitionMatchesParent demonstrates verifying a partition matches its parent table -// -// This is particularly useful when: -// - Creating new partitions -// - Reattaching detached partitions -// - Validating partition structure after schema changes -// -// Usage: -// -// parentTable := "test_analysis_by_job_by_dates" -// partition := "test_analysis_by_job_by_dates_2024_01_15" -// ExampleVerifyPartitionMatchesParent(dbc, parentTable, partition) -func ExampleVerifyPartitionMatchesParent(dbc *DB, parentTable, partition string) { - log.WithFields(log.Fields{ - "parent": parentTable, - "partition": partition, - }).Info("verifying partition matches parent table structure") - - // Use default options to ensure partition fully matches parent table - err := dbc.VerifyTablesHaveSameColumns(parentTable, partition, DefaultColumnVerificationOptions()) - if err != nil { - log.WithError(err).Error("partition schema does not match parent table") - log.Error("this partition may have been created with an old schema or manually modified") - return - } - - log.Info("partition structure matches parent table - safe to attach") -} - -// ExampleVerifyBeforeMigration demonstrates verification before data migration -// -// Before migrating data from one table to another, it's critical to ensure -// the schemas match to avoid data loss or type conversion errors. -// -// Usage: -// -// ExampleVerifyBeforeMigration(dbc, "old_table", "new_table") -func ExampleVerifyBeforeMigration(dbc *DB, sourceTable, targetTable string) { - log.Info("preparing data migration") - - // Step 1: Verify schemas match - // For data migration, we only need column names and types to match - // Nullable and default constraints don't affect the data copy - err := dbc.VerifyTablesHaveSameColumns(sourceTable, targetTable, DataMigrationColumnVerificationOptions()) - if err != nil { - log.WithError(err).Error("cannot migrate: schema mismatch detected") - log.Error("resolve schema differences before proceeding with migration") - return - } - - log.Info("schema verification passed - safe to proceed with migration") - - // Step 2: Proceed with migration - // (migration code would go here) -} - -// ExampleVerifyMultipleTables demonstrates checking multiple tables against a reference -// -// This is useful for: -// - Verifying all partitions match the parent table -// - Checking multiple replicas or shards have identical schemas -// - Validating a set of tables after schema updates -// -// Usage: -// -// ExampleVerifyMultipleTables(dbc, "parent_table", []string{"partition_1", "partition_2", "partition_3"}) -func ExampleVerifyMultipleTables(dbc *DB, referenceTable string, tablesToCheck []string) { - log.WithFields(log.Fields{ - "reference": referenceTable, - "count": len(tablesToCheck), - }).Info("verifying multiple tables against reference") - - var failures []string - for _, table := range tablesToCheck { - // Use default options to fully verify schema consistency - err := dbc.VerifyTablesHaveSameColumns(referenceTable, table, DefaultColumnVerificationOptions()) - if err != nil { - log.WithError(err).WithField("table", table).Error("schema mismatch detected") - failures = append(failures, table) - } else { - log.WithField("table", table).Debug("schema matches reference") - } - } - - if len(failures) > 0 { - log.WithFields(log.Fields{ - "total": len(tablesToCheck), - "failures": len(failures), - "failed": failures, - }).Error("schema verification completed with failures") - } else { - log.WithField("count", len(tablesToCheck)).Info("all tables match reference schema") - } -} - -// ExampleMigrateTableData demonstrates basic table data migration -// -// This function: -// - Verifies schemas match before migration -// - Copies all data from source to target -// - Supports dry-run mode for safety -// - Verifies migration success -// -// Usage: -// -// rowsMigrated, err := dbc.MigrateTableData("old_table", "new_table", nil, false) -// // Or omit id column to use target's auto-increment: -// rowsMigrated, err := dbc.MigrateTableData("old_table", "new_table", []string{"id"}, false) -func ExampleMigrateTableData(dbc *DB, sourceTable, targetTable string) { - log.WithFields(log.Fields{ - "source": sourceTable, - "target": targetTable, - }).Info("preparing table migration") - - // Step 1: Dry run first to verify and preview - log.Info("performing dry run") - _, err := dbc.MigrateTableData(sourceTable, targetTable, nil, true) - if err != nil { - log.WithError(err).Error("dry run failed - cannot proceed with migration") - return - } - - log.Info("dry run successful - proceeding with actual migration") - - // Step 2: Perform actual migration - rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, nil, false) - if err != nil { - log.WithError(err).Error("migration failed") - return - } - - log.WithField("rows", rowsMigrated).Info("migration completed successfully") -} - -// ExampleMigratePartitionData demonstrates migrating data from a detached partition to a new table -// -// Use case: You have a detached partition with old data that needs to be migrated -// to a new table structure or archive table. -// -// Usage: -// -// ExampleMigratePartitionData(dbc, "test_table_2024_01_15", "archive_table") -func ExampleMigratePartitionData(dbc *DB, detachedPartition, archiveTable string) { - log.WithFields(log.Fields{ - "partition": detachedPartition, - "archive": archiveTable, - }).Info("migrating detached partition to archive") - - // Verify the partition is actually detached (optional safety check) - // This would use functions from pkg/db/partitions if available - - // Migrate the data - rowsMigrated, err := dbc.MigrateTableData(detachedPartition, archiveTable, nil, false) - if err != nil { - log.WithError(err).Error("partition migration failed") - return - } - - log.WithFields(log.Fields{ - "partition": detachedPartition, - "archive": archiveTable, - "rows": rowsMigrated, - }).Info("partition data migrated to archive - safe to drop partition") -} - -// ExampleMigrateWithBackup demonstrates migrating data with a backup strategy -// -// Best practice: Create a backup before migration in case something goes wrong -// -// Usage: -// -// ExampleMigrateWithBackup(dbc, "source_table", "target_table", "backup_table") -func ExampleMigrateWithBackup(dbc *DB, sourceTable, targetTable, backupTable string) { - log.Info("migration with backup strategy") - - // Step 1: Create backup of target table - log.WithField("backup", backupTable).Info("creating backup of target table") - _, err := dbc.MigrateTableData(targetTable, backupTable, nil, false) - if err != nil { - log.WithError(err).Error("backup creation failed - aborting migration") - return - } - - log.Info("backup created successfully") - - // Step 2: Perform migration - log.Info("performing migration") - rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, nil, false) - if err != nil { - log.WithError(err).Error("migration failed - restore from backup if needed") - log.WithField("backup", backupTable).Info("backup table is available for restoration") - return - } - - log.WithField("rows", rowsMigrated).Info("migration completed successfully") - log.WithField("backup", backupTable).Info("backup table can be dropped if no longer needed") -} - -// ExampleBatchMigratePartitions demonstrates migrating multiple partitions -// -// Use case: You have multiple detached partitions that need to be migrated -// to an archive table or consolidated into a single table. -// -// Usage: -// -// partitions := []string{"table_2024_01_15", "table_2024_01_16", "table_2024_01_17"} -// ExampleBatchMigratePartitions(dbc, partitions, "archive_table") -func ExampleBatchMigratePartitions(dbc *DB, partitions []string, targetTable string) { - log.WithFields(log.Fields{ - "partitions": len(partitions), - "target": targetTable, - }).Info("batch migrating partitions") - - var totalRows int64 - var successCount int - var failures []string - - for _, partition := range partitions { - log.WithField("partition", partition).Info("migrating partition") - - rows, err := dbc.MigrateTableData(partition, targetTable, nil, false) - if err != nil { - log.WithError(err).WithField("partition", partition).Error("partition migration failed") - failures = append(failures, partition) - continue - } - - totalRows += rows - successCount++ - log.WithFields(log.Fields{ - "partition": partition, - "rows": rows, - }).Info("partition migrated successfully") - } - - log.WithFields(log.Fields{ - "total_partitions": len(partitions), - "successful": successCount, - "failed": len(failures), - "total_rows": totalRows, - }).Info("batch migration completed") - - if len(failures) > 0 { - log.WithField("failed_partitions", failures).Warn("some partitions failed to migrate") - } -} - -// ExampleMigrateAndVerify demonstrates migration with comprehensive verification -// -// This example shows best practices for production migrations: -// - Dry run first -// - Verify schemas -// - Perform migration -// - Verify row counts -// - Log all steps -// -// Usage: -// -// ExampleMigrateAndVerify(dbc, "source_table", "target_table") -func ExampleMigrateAndVerify(dbc *DB, sourceTable, targetTable string) { - log.Info("production migration workflow") - - // Step 1: Verify schemas match - log.Info("step 1: verifying schema compatibility") - // For migration, we only need column names and types to match - if err := dbc.VerifyTablesHaveSameColumns(sourceTable, targetTable, DataMigrationColumnVerificationOptions()); err != nil { - log.WithError(err).Error("schema verification failed") - return - } - log.Info("schema verification passed") - - // Step 2: Get pre-migration counts - log.Info("step 2: getting pre-migration row counts") - sourceCount, err := dbc.GetTableRowCount(sourceTable) - if err != nil { - log.WithError(err).Error("failed to get source count") - return - } - targetCountBefore, err := dbc.GetTableRowCount(targetTable) - if err != nil { - log.WithError(err).Error("failed to get target count") - return - } - - log.WithFields(log.Fields{ - "source_rows": sourceCount, - "target_rows": targetCountBefore, - }).Info("pre-migration row counts") - - // Step 3: Dry run - log.Info("step 3: performing dry run") - _, err = dbc.MigrateTableData(sourceTable, targetTable, nil, true) - if err != nil { - log.WithError(err).Error("dry run failed") - return - } - log.Info("dry run successful") - - // Step 4: Actual migration - log.Info("step 4: performing actual migration") - rowsMigrated, err := dbc.MigrateTableData(sourceTable, targetTable, nil, false) - if err != nil { - log.WithError(err).Error("migration failed") - return - } - - // Step 5: Verify results - log.Info("step 5: verifying migration results") - targetCountAfter, err := dbc.GetTableRowCount(targetTable) - if err != nil { - log.WithError(err).Error("failed to verify final count") - return - } - - expectedCount := targetCountBefore + sourceCount - if targetCountAfter != expectedCount { - log.WithFields(log.Fields{ - "expected": expectedCount, - "actual": targetCountAfter, - }).Error("row count mismatch detected!") - return - } - - log.WithFields(log.Fields{ - "source_table": sourceTable, - "target_table": targetTable, - "rows_migrated": rowsMigrated, - "target_before": targetCountBefore, - "target_after": targetCountAfter, - "verification": "passed", - }).Info("migration completed and verified successfully") -} - -// ExampleSyncIdentityColumn demonstrates synchronizing an IDENTITY column sequence -// -// This is useful after migrating data to a table with IDENTITY columns, -// ensuring the sequence starts at the correct value. -// -// Usage: -// -// ExampleSyncIdentityColumn(dbc, "my_table", "id") -func ExampleSyncIdentityColumn(dbc *DB, tableName, columnName string) { - log.WithFields(log.Fields{ - "table": tableName, - "column": columnName, - }).Info("synchronizing identity column") - - // Sync the identity sequence to match the current max value - err := dbc.SyncIdentityColumn(tableName, columnName) - if err != nil { - log.WithError(err).Error("failed to sync identity column") - return - } - - log.Info("identity column synchronized successfully") -} - -// ExampleMigrateToPartitionedTable demonstrates the complete workflow for -// migrating from a non-partitioned table to a partitioned table -// -// Usage: -// -// ExampleMigrateToPartitionedTable(dbc, "orders", "orders_partitioned") -func ExampleMigrateToPartitionedTable(dbc *DB, sourceTable, partitionedTable string) { - log.Info("Complete workflow: Migrating to partitioned table") - - // Assume partitioned table was created using CreatePartitionedTable - // and partitions were created using CreateMissingPartitions - - // Step 1: Migrate the data - log.Info("Step 1: Migrating data") - rows, err := dbc.MigrateTableData(sourceTable, partitionedTable, nil, false) - if err != nil { - log.WithError(err).Error("data migration failed") - return - } - - log.WithField("rows", rows).Info("data migrated successfully") - - // Step 2: Sync the IDENTITY column - log.Info("Step 2: Synchronizing IDENTITY sequence") - err = dbc.SyncIdentityColumn(partitionedTable, "id") - if err != nil { - log.WithError(err).Error("failed to sync identity column") - return - } - - // Step 3: Verify row counts match - log.Info("Step 3: Verifying row counts") - sourceCount, _ := dbc.GetTableRowCount(sourceTable) - targetCount, _ := dbc.GetTableRowCount(partitionedTable) - - if sourceCount != targetCount { - log.WithFields(log.Fields{ - "source": sourceCount, - "target": targetCount, - }).Error("row count mismatch!") - return - } - - log.WithFields(log.Fields{ - "source_table": sourceTable, - "partitioned_table": partitionedTable, - "rows": rows, - }).Info("migration to partitioned table completed successfully") - - // Next steps (manual): - // 1. Test the partitioned table thoroughly - // 2. Update application to use new table - // 3. After verification, drop the old table -} - -// ExampleMigrateTableDataRange demonstrates migrating data for a specific date range -// -// This is useful when: -// - Migrating data incrementally in smaller batches -// - Testing migrations with a subset of data -// - Moving specific time periods to archive tables -// - Migrating data to date-partitioned tables partition by partition -// -// Usage: -// -// startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) -// endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) -// ExampleMigrateTableDataRange(dbc, "orders", "orders_archive", "created_at", startDate, endDate) -func ExampleMigrateTableDataRange(dbc *DB, sourceTable, targetTable, dateColumn string, startDate, endDate time.Time) { - log.WithFields(log.Fields{ - "source": sourceTable, - "target": targetTable, - "date_column": dateColumn, - "start_date": startDate.Format("2006-01-02"), - "end_date": endDate.Format("2006-01-02"), - }).Info("migrating data for date range") - - // Step 1: Dry run first to verify and preview - log.Info("performing dry run") - _, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, nil, true) - if err != nil { - log.WithError(err).Error("dry run failed - cannot proceed with migration") - return - } - - log.Info("dry run successful - proceeding with actual migration") - - // Step 2: Perform actual migration - rowsMigrated, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, nil, false) - if err != nil { - log.WithError(err).Error("migration failed") - return - } - - log.WithFields(log.Fields{ - "rows": rowsMigrated, - "start_date": startDate.Format("2006-01-02"), - "end_date": endDate.Format("2006-01-02"), - }).Info("migration completed successfully") -} - -// ExampleIncrementalMigrationByMonth demonstrates migrating data month by month -// -// This approach is useful for: -// - Large tables where migrating all at once would be too slow -// - Reducing lock contention by migrating in smaller batches -// - Being able to pause and resume migrations -// - Easier rollback if issues are detected -// -// Usage: -// -// ExampleIncrementalMigrationByMonth(dbc, "large_table", "large_table_new", "created_at", 2024) -func ExampleIncrementalMigrationByMonth(dbc *DB, sourceTable, targetTable, dateColumn string, year int) { - log.WithFields(log.Fields{ - "source": sourceTable, - "target": targetTable, - "year": year, - }).Info("starting incremental migration by month") - - var totalMigrated int64 - var failedMonths []string - - // Migrate data month by month - for month := 1; month <= 12; month++ { - startDate := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC) - endDate := startDate.AddDate(0, 1, 0) // First day of next month - - log.WithFields(log.Fields{ - "month": time.Month(month).String(), - "start_date": startDate.Format("2006-01-02"), - "end_date": endDate.Format("2006-01-02"), - }).Info("migrating month") - - rows, err := dbc.MigrateTableDataRange(sourceTable, targetTable, dateColumn, startDate, endDate, nil, false) - if err != nil { - log.WithError(err).WithField("month", time.Month(month).String()).Error("month migration failed") - failedMonths = append(failedMonths, time.Month(month).String()) - continue - } - - totalMigrated += rows - log.WithFields(log.Fields{ - "month": time.Month(month).String(), - "rows": rows, - }).Info("month migrated successfully") - } - - log.WithFields(log.Fields{ - "total_rows": totalMigrated, - "total_months": 12, - "failed_months": len(failedMonths), - }).Info("incremental migration completed") - - if len(failedMonths) > 0 { - log.WithField("failed_months", failedMonths).Warn("some months failed to migrate") - } -} - -// ExampleMigrateToPartitionByDateRange demonstrates migrating data to a specific partition -// -// This workflow is useful when: -// - You have a non-partitioned table and want to migrate to a partitioned structure -// - You want to populate partitions incrementally -// - You're backfilling historical data into partitions -// -// Important: MigrateTableDataRange automatically verifies that all necessary partitions -// exist for the date range being migrated. If the target table is RANGE partitioned and -// partitions are missing, the function will return an error before attempting migration. -// -// Usage: -// -// ExampleMigrateToPartitionByDateRange(dbc, "orders", "orders_partitioned", "order_date") -func ExampleMigrateToPartitionByDateRange(dbc *DB, sourceTable, partitionedTable, dateColumn string) { - log.Info("migrating data to partitioned table by date range") - - // Example: Migrate January 2024 data to the partition - startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) - endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) - - // Step 1: Migrate the data for this date range - // The function will automatically verify that partitions exist for all dates - // in the range [2024-01-01, 2024-02-01) before attempting the migration - log.WithFields(log.Fields{ - "start_date": startDate.Format("2006-01-02"), - "end_date": endDate.Format("2006-01-02"), - }).Info("migrating date range to partition") - - rows, err := dbc.MigrateTableDataRange(sourceTable, partitionedTable, dateColumn, startDate, endDate, nil, false) - if err != nil { - log.WithError(err).Error("migration failed") - return - } - - log.WithField("rows", rows).Info("data migrated to partition") - - // Step 2: Verify the data landed in the expected partition - // This would use partition-specific queries to verify - log.Info("verifying data distribution across partitions") - - // Step 3: Repeat for other date ranges as needed - log.Info("migration to partition completed - repeat for additional date ranges as needed") -} - -// ExampleGetPartitionStrategy demonstrates checking if a table is partitioned -// -// This is useful before performing operations that differ between partitioned -// and non-partitioned tables. -// -// Usage: -// -// ExampleGetPartitionStrategy(dbc, "orders") -func ExampleGetPartitionStrategy(dbc *DB, tableName string) { - log.WithField("table", tableName).Info("checking partition strategy") - - strategy, err := dbc.GetPartitionStrategy(tableName) - if err != nil { - log.WithError(err).Error("failed to check partition strategy") - return - } - - if strategy == "" { - log.Info("table is not partitioned") - // Proceed with normal table operations - } else { - log.WithField("strategy", strategy).Info("table is partitioned") - - switch strategy { - case PartitionStrategyRange: - log.Info("table uses RANGE partitioning - can use date-based partition operations") - case PartitionStrategyList: - log.Info("table uses LIST partitioning - partitioned by discrete values") - case PartitionStrategyHash: - log.Info("table uses HASH partitioning - partitioned by hash function") - default: - log.Warn("unknown partition strategy") - } - } -} - -// ExampleVerifyPartitionCoverage demonstrates verifying partition coverage before migration -// -// This workflow ensures all necessary partitions exist before attempting a data migration, -// preventing runtime failures due to missing partitions. -// -// Usage: -// -// ExampleVerifyPartitionCoverage(dbc, "orders", startDate, endDate) -func ExampleVerifyPartitionCoverage(dbc *DB, tableName string, startDate, endDate time.Time) { - log.WithFields(log.Fields{ - "table": tableName, - "start_date": startDate.Format("2006-01-02"), - "end_date": endDate.Format("2006-01-02"), - }).Info("verifying partition coverage") - - // Verify that all necessary partitions exist - err := dbc.VerifyPartitionCoverage(tableName, startDate, endDate) - if err != nil { - log.WithError(err).Error("partition coverage verification failed") - log.Error("missing partitions detected - cannot proceed with migration") - log.Info("create missing partitions using partitions.CreateMissingPartitions before retrying") - return - } - - log.Info("partition coverage verified - all required partitions exist") - log.Info("safe to proceed with data migration") -} - -// ExampleSequenceMetadata demonstrates examining how sequences are linked to columns -// -// # This shows the internal PostgreSQL mechanisms for SERIAL vs IDENTITY columns -// -// Usage: -// -// ExampleSequenceMetadata(dbc, "orders") -func ExampleSequenceMetadata(dbc *DB, tableName string) { - log.WithField("table", tableName).Info("examining sequence metadata") - - metadata, err := dbc.GetSequenceMetadata(tableName) - if err != nil { - log.WithError(err).Error("failed to get sequence metadata") - return - } - - if len(metadata) == 0 { - log.Info("no sequences found for this table") - return - } - - log.WithField("count", len(metadata)).Info("found sequences") - - for _, m := range metadata { - linkageType := "SERIAL" - if m.IsIdentityColumn { - linkageType = "IDENTITY" - } - - log.WithFields(log.Fields{ - "column": m.ColumnName, - "sequence": m.SequenceName, - "linkage_type": linkageType, - "dep_type": m.DependencyType, - "owner": m.SequenceOwner, - }).Info("sequence linkage details") - - // Explain the linkage mechanism - if m.IsIdentityColumn { - log.WithField("column", m.ColumnName).Debug( - "IDENTITY: Linked via pg_depend (OID-based) + pg_attribute.attidentity. " + - "Renaming sequence is safe - PostgreSQL uses OID internally, not name.") - } else { - log.WithField("column", m.ColumnName).Debug( - "SERIAL: Linked via pg_depend + column DEFAULT nextval('seq_name'). " + - "Column default uses sequence NAME, but pg_depend uses OID.") - } - } -} - -// ExampleListTableSequences demonstrates listing sequences for a specific table -// -// This is useful for: -// - Understanding which columns use auto-increment -// - Checking sequence names before table renames -// - Debugging sequence-related issues -// - Auditing table structure -// -// Usage: -// -// ExampleListTableSequences(dbc, "orders") -func ExampleListTableSequences(dbc *DB, tableName string) { - log.WithField("table", tableName).Info("listing sequences for table") - - sequences, err := dbc.GetTableSequences(tableName) - if err != nil { - log.WithError(err).Error("failed to get sequences") - return - } - - if len(sequences) == 0 { - log.Info("no sequences found for this table") - return - } - - log.WithField("count", len(sequences)).Info("found sequences") - for _, seq := range sequences { - log.WithFields(log.Fields{ - "sequence": seq.SequenceName, - "column": seq.ColumnName, - }).Info("sequence detail") - } -} - -// ExampleListAllTableSequences demonstrates listing sequences for all tables -// -// This is useful for: -// - Database auditing and inventory -// - Understanding auto-increment usage across tables -// - Finding sequences that may need syncing -// - Preparing for bulk table operations -// -// Usage: -// -// ExampleListAllTableSequences(dbc) -func ExampleListAllTableSequences(dbc *DB) { - log.Info("listing all table sequences in database") - - allSequences, err := dbc.ListAllTableSequences() - if err != nil { - log.WithError(err).Error("failed to list all sequences") - return - } - - log.WithField("tables_with_sequences", len(allSequences)).Info("found tables with sequences") - - for tableName, sequences := range allSequences { - log.WithFields(log.Fields{ - "table": tableName, - "sequence_count": len(sequences), - }).Info("table sequences") - - for _, seq := range sequences { - log.WithFields(log.Fields{ - "table": tableName, - "sequence": seq.SequenceName, - "column": seq.ColumnName, - }).Debug("sequence detail") - } - } -} - -// ExampleCheckSequencesBeforeRename demonstrates checking sequences before renaming tables -// -// This workflow helps you understand what will be renamed when using renameSequences=true -// -// Usage: -// -// ExampleCheckSequencesBeforeRename(dbc, "orders_old", "orders") -func ExampleCheckSequencesBeforeRename(dbc *DB, oldTableName, newTableName string) { - log.Info("checking sequences before table rename") - - // Get sequences for the old table - sequences, err := dbc.GetTableSequences(oldTableName) - if err != nil { - log.WithError(err).Error("failed to get sequences") - return - } - - if len(sequences) == 0 { - log.Info("no sequences to rename") - return - } - - log.WithField("count", len(sequences)).Info("sequences will be renamed") - - // Show what the new sequence names will be - for _, seq := range sequences { - newSeqName := fmt.Sprintf("%s_%s_seq", newTableName, seq.ColumnName) - log.WithFields(log.Fields{ - "old_sequence": seq.SequenceName, - "new_sequence": newSeqName, - "column": seq.ColumnName, - }).Info("planned rename") - } - - // Now perform the rename - renames := []TableRename{{From: oldTableName, To: newTableName}} - count, err := dbc.RenameTables(renames, true, false, false, false, false) - if err != nil { - log.WithError(err).Error("rename failed") - return - } - - log.WithField("renamed", count).Info("table and sequences renamed") -} - -// ExampleCheckAndCreatePartitions demonstrates checking for missing partitions and creating them -// -// This workflow combines partition verification with automatic creation of missing partitions. -// -// Note: This example shows the pattern but doesn't import the partitions package -// to avoid circular dependencies in the example file. -// -// Usage: -// -// ExampleCheckAndCreatePartitions(dbc, "orders", startDate, endDate) -func ExampleCheckAndCreatePartitions(dbc *DB, tableName string, startDate, endDate time.Time) { - log.Info("checking partition coverage and creating missing partitions") - - // Step 1: Check if partitions exist - err := dbc.VerifyPartitionCoverage(tableName, startDate, endDate) - if err != nil { - log.WithError(err).Warn("missing partitions detected") - - // Step 2: In actual usage, you would create missing partitions using: - // import "github.com/openshift/sippy/pkg/db/partitions" - // count, err := partitions.CreateMissingPartitions(dbc, tableName, startDate, endDate, false) - - log.Info("would create missing partitions here using partitions.CreateMissingPartitions") - return - } - - log.Info("all partitions exist - ready for operations") -} - -// ExampleRenameTables demonstrates renaming multiple tables atomically -// -// This is useful for: -// - Swapping a new partitioned table with an old non-partitioned table -// - Renaming related tables together to maintain consistency -// - Performing atomic schema migrations -// -// Usage: -// -// ExampleRenameTables(dbc) -func ExampleRenameTables(dbc *DB) { - log.Info("renaming multiple tables atomically") - - // Define table renames (order matters for dependencies) - renames := []TableRename{ - {From: "orders_old", To: "orders_backup"}, - {From: "orders_new", To: "orders"}, - } - - // Dry run first to verify - _, err := dbc.RenameTables(renames, true, true, true, true, true) - if err != nil { - log.WithError(err).Error("dry run failed") - return - } - - // Execute the renames (rename sequences and partitions too) - count, err := dbc.RenameTables(renames, true, true, true, true, false) - if err != nil { - log.WithError(err).Error("rename failed") - return - } - - log.WithField("renamed", count).Info("tables renamed successfully") -} - -// ExampleSwapPartitionedTable demonstrates swapping a partitioned table with a non-partitioned table -// -// This is a common workflow when migrating from non-partitioned to partitioned tables: -// 1. Create new partitioned table -// 2. Migrate data to partitioned table -// 3. Swap the tables atomically so the partitioned table becomes the active one -// -// Usage: -// -// ExampleSwapPartitionedTable(dbc, "orders", "orders_partitioned") -func ExampleSwapPartitionedTable(dbc *DB, oldTable, newPartitionedTable string) { - log.Info("swapping non-partitioned table with partitioned table") - - // Step 1: Verify data was migrated successfully - oldCount, _ := dbc.GetTableRowCount(oldTable) - newCount, _ := dbc.GetTableRowCount(newPartitionedTable) - - if oldCount != newCount { - log.WithFields(log.Fields{ - "old_count": oldCount, - "new_count": newCount, - }).Error("row count mismatch - cannot swap tables") - return - } - - log.Info("row counts match - proceeding with table swap") - - // Step 2: Perform atomic rename to swap tables - // Order matters: rename orders -> orders_old first to free up the "orders" name - // Then rename orders_partitioned -> orders - renames := []TableRename{ - {From: oldTable, To: oldTable + "_old"}, - {From: newPartitionedTable, To: oldTable}, - } - - // Rename sequences and partitions too so they match the new table names - count, err := dbc.RenameTables(renames, true, true, true, true, false) - if err != nil { - log.WithError(err).Error("table swap failed") - return - } - - log.WithFields(log.Fields{ - "renamed": count, - "old_table": oldTable + "_old", - "new_table": oldTable, - "partitioned": true, - }).Info("tables swapped successfully - partitioned table is now active") - - // Next steps: - // 1. Test the new partitioned table thoroughly - // 2. After verification period, drop the old table -} - -// ExampleThreeWayTableSwap demonstrates a three-way table swap -// -// This pattern is useful when you want to: -// - Keep a backup of the current production table -// - Swap in a new table -// - Archive the old backup -// -// Usage: -// -// ExampleThreeWayTableSwap(dbc) -func ExampleThreeWayTableSwap(dbc *DB) { - log.Info("performing three-way table swap") - - // Scenario: - // orders (current production) - // orders_new (migrated data, ready to go live) - // orders_backup (previous backup to archive) - - // Step 1: First rename the old backup to archive - // Step 2: Rename current production to backup - // Step 3: Rename new table to production - // - // All in one atomic transaction: - // orders -> orders_backup - // orders_new -> orders - // orders_backup -> orders_archive - - // Order matters for three-way swap - must free up names in the right order: - // 1. orders_backup -> orders_archive (frees up "orders_backup") - // 2. orders -> orders_backup (frees up "orders") - // 3. orders_new -> orders (swaps in new production table) - renames := []TableRename{ - {From: "orders_backup", To: "orders_archive"}, - {From: "orders", To: "orders_backup"}, - {From: "orders_new", To: "orders"}, - } - - // Dry run first (also check sequence renames) - _, err := dbc.RenameTables(renames, true, true, true, true, true) - if err != nil { - log.WithError(err).Error("dry run validation failed") - return - } - - // Execute the swap (rename sequences and partitions too) - count, err := dbc.RenameTables(renames, true, true, true, true, false) - if err != nil { - log.WithError(err).Error("table swap failed - no changes made") - return - } - - log.WithFields(log.Fields{ - "renamed": count, - "production": "orders (was orders_new)", - "backup": "orders_backup (was orders)", - "archive": "orders_archive (was orders_backup)", - }).Info("three-way swap completed successfully") -} - -// ExampleRollbackTableSwap demonstrates rolling back a table swap -// -// # If you swapped tables but need to revert, you can use RenameTables again -// -// Usage: -// -// ExampleRollbackTableSwap(dbc) -func ExampleRollbackTableSwap(dbc *DB) { - log.Info("rolling back table swap") - - // Assume we previously did: - // orders -> orders_old - // orders_partitioned -> orders - // - // To rollback: - // 1. orders -> orders_partitioned (frees up "orders") - // 2. orders_old -> orders (restore to production) - - rollbackRenames := []TableRename{ - {From: "orders", To: "orders_partitioned"}, - {From: "orders_old", To: "orders"}, - } - - // Rename sequences and partitions back too - count, err := dbc.RenameTables(rollbackRenames, true, true, true, true, false) - if err != nil { - log.WithError(err).Error("rollback failed") - return - } - - log.WithField("renamed", count).Info("rollback completed - original table restored") -} From c73ada6e26e901b674eb69e25450c5b92e7da02a Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Thu, 12 Mar 2026 20:06:56 -0400 Subject: [PATCH 08/11] TRT-1989: additional refactor --- pkg/db/UTILS_README.md | 162 +++++++++++++++++--------------- pkg/db/partitions/partitions.go | 55 ++++++++--- pkg/db/utils.go | 98 +++++++++++++------ 3 files changed, 195 insertions(+), 120 deletions(-) diff --git a/pkg/db/UTILS_README.md b/pkg/db/UTILS_README.md index 6c2c8ae7c..f3f1d0d51 100644 --- a/pkg/db/UTILS_README.md +++ b/pkg/db/UTILS_README.md @@ -290,9 +290,10 @@ log.WithField("count", count).Info("table row count") Renames multiple tables atomically in a single transaction. ```go -renames := map[string]string{ - "orders_old": "orders_backup", - "orders_new": "orders", +// Order matters - renames are executed in the order provided +renames := []db.TableRename{ + {From: "orders_old", To: "orders_backup"}, + {From: "orders_new", To: "orders"}, } // Dry run first (renameSequences=true, renamePartitions=true, renameConstraints=true, renameIndexes=true) @@ -312,11 +313,11 @@ log.WithField("renamed", count).Info("tables, partitions, sequences, constraints **How It Works**: 1. Validates that all source tables exist 2. Checks for conflicts (target table already exists, unless it's also being renamed) -3. Executes all `ALTER TABLE ... RENAME TO ...` statements in a single transaction -4. Either all renames succeed or all are rolled back +3. Executes all `ALTER TABLE ... RENAME TO ...` statements in the order provided +4. Either all renames succeed or all are rolled back in a single transaction **Parameters**: -- `tableRenames`: Map of source table names to target names (from -> to) +- `tableRenames`: Ordered slice of TableRename structs specifying renames to execute - `renameSequences`: If true, also renames sequences owned by table columns (SERIAL, BIGSERIAL, IDENTITY) - `renamePartitions`: If true, also renames child partitions of partitioned tables - `renameConstraints`: If true, also renames table constraints (primary keys, foreign keys, unique, check) @@ -327,6 +328,8 @@ log.WithField("renamed", count).Info("tables, partitions, sequences, constraints - `renamedCount`: Number of tables successfully renamed (0 if dry run) - `error`: Any error encountered +**Note**: Caller is responsible for ordering renames correctly to avoid naming conflicts. For table swaps (A→B, B→C), ensure B→C comes before A→B in the array. + **Features**: - **Atomic operation**: All renames happen in one transaction - **Validation**: Checks source tables exist and no conflicts @@ -503,8 +506,8 @@ When `renamePartitions=true`, the function will **also** rename sequences, const Example: ```go -renames := map[string]string{ - "orders": "orders_v2", +renames := []db.TableRename{ + {From: "orders", To: "orders_v2"}, } // Rename table, partitions, and all their sequences/constraints/indexes @@ -652,34 +655,36 @@ Index renaming is extremely fast - it only updates metadata in PostgreSQL system When swapping tables (e.g., `A -> B, C -> A`), the order of operations matters to avoid naming conflicts: ```go -renames := map[string]string{ - "table_base": "table_old", // Free up "table_base" namespace - "table_new": "table_base", // Now safe to use "table_base" +// Order matters - rename table_base first to free up the name +renames := []db.TableRename{ + {From: "table_base", To: "table_old"}, // Free up "table_base" namespace + {From: "table_new", To: "table_base"}, // Now safe to use "table_base" } ``` -Without proper ordering, index/constraint renames could fail: +Without proper ordering, renames could fail: ```sql --- Wrong order (if processed randomly): -ALTER INDEX table_new_pkey RENAME TO table_base_pkey; -- ERROR! table_base_pkey already exists +-- Wrong order (if table_new renamed first): +ALTER TABLE table_new RENAME TO table_base; -- ERROR! table_base already exists --- Correct order (sorted): -ALTER INDEX table_base_pkey RENAME TO table_old_pkey; -- Frees up "table_base_pkey" -ALTER INDEX table_new_pkey RENAME TO table_base_pkey; -- Now safe +-- Correct order (as specified in array): +ALTER TABLE table_base RENAME TO table_old; -- Frees up "table_base" +ALTER TABLE table_new RENAME TO table_base; -- Now safe ``` -**How we handle it:** -- Tables are renamed first (in transaction) -- Sequences/constraints/indexes are renamed in **sorted order** (alphabetically by name) -- Sorting ensures `table_base` is processed before `table_new`, avoiding conflicts -- All operations are deterministic - same input always produces same execution order +**How it works:** +- Tables are renamed in the order specified in the array +- Each rename happens within a single transaction +- Caller is responsible for specifying correct order to avoid conflicts +- All operations are deterministic - renames execute in array order **Example - Table Swap**: ```go // Swap old table with new partitioned table atomically -renames := map[string]string{ - "orders": "orders_old", // Save current table - "orders_partitioned": "orders", // New table becomes production +// Order matters: rename orders first to free up the name +renames := []db.TableRename{ + {From: "orders", To: "orders_old"}, // Save current table + {From: "orders_partitioned", To: "orders"}, // New table becomes production } // Rename sequences, partitions, constraints, and indexes too @@ -693,10 +698,11 @@ if err != nil { **Example - Three-Way Swap**: ```go // Rotate three tables: production -> backup, new -> production, backup -> archive -renames := map[string]string{ - "orders": "orders_backup", // Production becomes backup - "orders_new": "orders", // New becomes production - "orders_backup": "orders_archive", // Old backup becomes archive +// Order matters - must free up names in the right order: +renames := []db.TableRename{ + {From: "orders_backup", To: "orders_archive"}, // Free up "orders_backup" + {From: "orders", To: "orders_backup"}, // Free up "orders" + {From: "orders_new", To: "orders"}, // New becomes production } // Rename sequences, partitions, constraints, and indexes too @@ -966,12 +972,12 @@ if len(sequences) > 0 { log.WithField("count", len(sequences)).Info("found sequences - will rename with table") // Use renameSequences=true to keep them consistent - renames := map[string]string{"orders_old": "orders"} - dbc.RenameTables(renames, true, false, false) + renames := []db.TableRename{{From: "orders_old", To: "orders"}} + dbc.RenameTables(renames, true, false, false, false, false) } else { // No sequences to worry about - renames := map[string]string{"orders_old": "orders"} - dbc.RenameTables(renames, false, false, false) + renames := []db.TableRename{{From: "orders_old", To: "orders"}} + dbc.RenameTables(renames, false, false, false, false, false) } ``` @@ -1123,8 +1129,8 @@ for _, part := range partitions { // If partitions follow naming convention, rename them too if len(partitions) > 0 { - renames := map[string]string{"orders_old": "orders"} - dbc.RenameTables(renames, true, true, true, false) // renamePartitions=true + renames := []db.TableRename{{From: "orders_old", To: "orders"}} + dbc.RenameTables(renames, true, true, true, false, false) // renamePartitions=true } ``` @@ -1192,7 +1198,7 @@ for _, cons := range constraints { // If constraints follow naming convention, rename them too if len(constraints) > 0 { - renames := map[string]string{"orders_old": "orders"} + renames := []db.TableRename{{From: "orders_old", To: "orders"}} dbc.RenameTables(renames, true, true, true, true, false) // renameConstraints=true, renameIndexes=true } ``` @@ -1260,7 +1266,7 @@ for _, idx := range indexes { // If indexes follow naming convention, rename them too if len(indexes) > 0 { - renames := map[string]string{"orders_old": "orders"} + renames := []db.TableRename{{From: "orders_old", To: "orders"}} dbc.RenameTables(renames, true, true, true, true, false) // renameIndexes=true } ``` @@ -1647,19 +1653,20 @@ if oldCount != newCount { } // Step 2: Perform atomic table swap -renames := map[string]string{ - "orders": "orders_old", // Save current table - "orders_partitioned": "orders", // New table becomes production +// Order matters: rename orders first to free up the name +renames := []db.TableRename{ + {From: "orders", To: "orders_old"}, // Save current table + {From: "orders_partitioned", To: "orders"}, // New table becomes production } // Dry run first -_, err := dbc.RenameTables(renames, true, true, true) +_, err := dbc.RenameTables(renames, true, true, true, false, true) if err != nil { log.Fatal(err) } // Execute swap (rename sequences and partitions too) -count, err := dbc.RenameTables(renames, true, true, false) +count, err := dbc.RenameTables(renames, true, true, true, false, false) if err != nil { log.Fatal(err) } @@ -1672,11 +1679,11 @@ log.WithFields(log.Fields{ }).Info("tables swapped - partitioned table is now active") // If something goes wrong, you can easily rollback: -// rollback := map[string]string{ -// "orders": "orders_partitioned", -// "orders_old": "orders", +// rollback := []db.TableRename{ +// {From: "orders", To: "orders_partitioned"}, +// {From: "orders_old", To: "orders"}, // } -// dbc.RenameTables(rollback, true, true, false) +// dbc.RenameTables(rollback, true, true, true, false, false) ``` --- @@ -1685,15 +1692,15 @@ log.WithFields(log.Fields{ ```go // Rotate tables: archive old backup, current becomes backup, new becomes current - -renames := map[string]string{ - "orders": "orders_backup", // Current production -> backup - "orders_new": "orders", // New table -> production - "orders_backup": "orders_archive", // Old backup -> archive +// Order matters - must free up names in the right order: +renames := []db.TableRename{ + {From: "orders_backup", To: "orders_archive"}, // Free up "orders_backup" + {From: "orders", To: "orders_backup"}, // Free up "orders" + {From: "orders_new", To: "orders"}, // New becomes production } // All three renames happen atomically in one transaction (rename sequences and partitions too) -count, err := dbc.RenameTables(renames, true, true, false) +count, err := dbc.RenameTables(renames, true, true, false, false, false) if err != nil { log.WithError(err).Error("rotation failed - no changes made") return @@ -1821,13 +1828,13 @@ if oldCount != newCount { log.Fatal("row count mismatch!") } -// Step 5: Atomically swap tables -renames := map[string]string{ - "orders": "orders_old", - "orders_partitioned": "orders", +// Step 5: Atomically swap tables (order matters) +renames := []db.TableRename{ + {From: "orders", To: "orders_old"}, + {From: "orders_partitioned", To: "orders"}, } -count, err := dbc.RenameTables(renames, true, true, false) +count, err := dbc.RenameTables(renames, true, true, false, false, false) if err != nil { log.Fatal(err) } @@ -1910,21 +1917,21 @@ tx.Commit() ```go // GOOD: Always dry run before renaming -renames := map[string]string{ - "orders_old": "orders_backup", - "orders_new": "orders", +renames := []db.TableRename{ + {From: "orders_old", To: "orders_backup"}, + {From: "orders_new", To: "orders"}, } -_, err := dbc.RenameTables(renames, true, true, true) +_, err := dbc.RenameTables(renames, true, true, true, false, true) if err != nil { log.WithError(err).Error("validation failed") return } -count, err := dbc.RenameTables(renames, true, true, false) +count, err := dbc.RenameTables(renames, true, true, true, false, false) // BAD: Direct rename without validation -count, err := dbc.RenameTables(renames, true, true, false) +count, err := dbc.RenameTables(renames, true, true, true, false, false) ``` ### Verify Before Swapping Tables @@ -1940,39 +1947,39 @@ if oldCount != newCount { } // Now safe to swap -dbc.RenameTables(map[string]string{ - "orders": "orders_old", - "orders_partitioned": "orders", -}, true, true, false) +dbc.RenameTables([]db.TableRename{ + {From: "orders", To: "orders_old"}, + {From: "orders_partitioned", To: "orders"}, +}, true, true, false, false, false) // BAD: Swap without verifying data -dbc.RenameTables(renames, true, true, false) +dbc.RenameTables(renames, true, true, false, false, false) ``` ### Keep Rollback Plans Ready ```go // GOOD: Define rollback before making changes -renames := map[string]string{ - "orders": "orders_old", - "orders_new": "orders", +renames := []db.TableRename{ + {From: "orders", To: "orders_old"}, + {From: "orders_new", To: "orders"}, } -// Define rollback upfront -rollback := map[string]string{ - "orders": "orders_new", - "orders_old": "orders", +// Define rollback upfront (reverse order) +rollback := []db.TableRename{ + {From: "orders", To: "orders_new"}, + {From: "orders_old", To: "orders"}, } // Execute rename -_, err := dbc.RenameTables(renames, true, true, false) +_, err := dbc.RenameTables(renames, true, true, false, false, false) if err != nil { log.Error("rename failed - no rollback needed") return } // If issues found after rename, easy to rollback -// dbc.RenameTables(rollback, true, true, false) +// dbc.RenameTables(rollback, true, true, false, false, false) ``` --- @@ -2074,4 +2081,3 @@ For very large tables (millions of rows): - [Partition Management APIs](./partitions/README.md) - For partition-specific operations - [Database Schema](../../.claude/db-schema-analysis.md) - For schema documentation -- Examples in `utils_example.go` - For detailed usage patterns diff --git a/pkg/db/partitions/partitions.go b/pkg/db/partitions/partitions.go index c58530278..1c2b85f19 100644 --- a/pkg/db/partitions/partitions.go +++ b/pkg/db/partitions/partitions.go @@ -1327,20 +1327,40 @@ func CreatePartitionedTable(dbc *db.DB, model interface{}, tableName string, con return fullSQL, nil } + // Execute table creation and index creation in a transaction + tx := dbc.DB.Begin() + if tx.Error != nil { + return "", fmt.Errorf("failed to begin transaction: %w", tx.Error) + } + + // Ensure transaction is properly handled + committed := false + defer func() { + if !committed { + tx.Rollback() + } + }() + // Execute the CREATE TABLE statement - result := dbc.DB.Exec(createTableSQL) + result := tx.Exec(createTableSQL) if result.Error != nil { return "", fmt.Errorf("failed to create partitioned table: %w", result.Error) } // Execute index creation statements if indexSQL.Len() > 0 { - result = dbc.DB.Exec(indexSQL.String()) + result = tx.Exec(indexSQL.String()) if result.Error != nil { - log.WithError(result.Error).Warn("some indexes may have failed to create") + return "", fmt.Errorf("failed to create indexes: %w", result.Error) } } + // Commit the transaction + if err := tx.Commit().Error; err != nil { + return "", fmt.Errorf("failed to commit transaction: %w", err) + } + committed = true + elapsed := time.Since(start) log.WithFields(log.Fields{ "table": tableName, @@ -1771,10 +1791,11 @@ func getCurrentIndexes(dbc *db.DB, tableName string) ([]indexInfo, error) { FROM pg_indexes i JOIN pg_class c ON c.relname = i.indexname JOIN pg_index ix ON ix.indexrelid = c.oid - JOIN pg_attribute a ON a.attrelid = ix.indrelid AND a.attnum = ANY(ix.indkey) + JOIN unnest(ix.indkey) WITH ORDINALITY AS u(attnum, ord) ON true + JOIN pg_attribute a ON a.attrelid = ix.indrelid AND a.attnum = u.attnum WHERE i.schemaname = 'public' AND i.tablename = @table_name - ORDER BY i.indexname, a.attnum + ORDER BY i.indexname, u.ord ` result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&rows) @@ -1877,15 +1898,23 @@ func getPrimaryKeyColumns(dbc *db.DB, tableName string) ([]string, error) { } // normalizeDataType normalizes data type strings for comparison +// Preserves type modifiers (length, precision, scale) while normalizing base type names +// Examples: +// - "character varying(64)" -> "varchar(64)" +// - "integer" -> "int" +// - "numeric(8,2)" -> "numeric(8,2)" (preserved) func normalizeDataType(dataType string) string { - // Convert to lowercase and remove common variations - normalized := strings.ToLower(strings.TrimSpace(dataType)) + dataType = strings.ToLower(strings.TrimSpace(dataType)) - // Handle common type mappings + // Map common type variations to standard forms (preserving any modifiers) + // Check for types with modifiers first (e.g., "character varying(64)") replacements := map[string]string{ "character varying": "varchar", "integer": "int", - "bigint": "int8", + "int4": "int", + "int8": "bigint", + "bigserial": "bigint", + "serial": "int", "smallint": "int2", "boolean": "bool", "timestamp without time zone": "timestamp", @@ -1897,13 +1926,15 @@ func normalizeDataType(dataType string) string { "time with time zone": "timetz", } + // Try to replace the base type name while preserving modifiers for old, new := range replacements { - if strings.Contains(normalized, old) { - normalized = strings.ReplaceAll(normalized, old, new) + if suffix, found := strings.CutPrefix(dataType, old); found { + // Replace the prefix and keep everything after (modifiers, array brackets, etc.) + return new + suffix } } - return normalized + return dataType } // GetDetachedPartitionStats returns statistics about detached partitions for a given table diff --git a/pkg/db/utils.go b/pkg/db/utils.go index 9133bd4b7..04bfe1437 100644 --- a/pkg/db/utils.go +++ b/pkg/db/utils.go @@ -191,21 +191,33 @@ func (dbc *DB) VerifyTablesHaveSameColumns(table1, table2 string, opts ColumnVer return nil } -// GetTableColumns retrieves column information for a table from information_schema +// GetTableColumns retrieves column information for a table from pg_catalog +// Uses format_type() to preserve precise type definitions including: +// - Length modifiers: varchar(64) vs varchar(255) +// - Precision/scale: numeric(8,2) vs numeric(20,10) +// - Enum type names: user_role instead of USER-DEFINED +// - Array types: integer[] vs integer func (dbc *DB) GetTableColumns(tableName string) ([]ColumnInfo, error) { var columns []ColumnInfo + // Use pg_catalog to get precise type information including modifiers + // format_type() preserves varchar(64) vs varchar(255), numeric(8,2) vs numeric(20,10), etc. query := ` SELECT - column_name, - data_type, - is_nullable, - column_default, - ordinal_position - FROM information_schema.columns - WHERE table_schema = 'public' - AND table_name = @table_name - ORDER BY ordinal_position + a.attname AS column_name, + format_type(a.atttypid, a.atttypmod) AS data_type, + CASE WHEN a.attnotnull THEN 'NO' ELSE 'YES' END AS is_nullable, + pg_get_expr(d.adbin, d.adrelid) AS column_default, + a.attnum AS ordinal_position + FROM pg_catalog.pg_attribute a + JOIN pg_catalog.pg_class c ON a.attrelid = c.oid + JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid + LEFT JOIN pg_catalog.pg_attrdef d ON a.attrelid = d.adrelid AND a.attnum = d.adnum + WHERE c.relname = @table_name + AND n.nspname = 'public' + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum ` result := dbc.DB.Raw(query, sql.Named("table_name", tableName)).Scan(&columns) @@ -221,11 +233,17 @@ func (dbc *DB) GetTableColumns(tableName string) ([]ColumnInfo, error) { } // normalizeDataType normalizes PostgreSQL data type names for comparison +// Preserves type modifiers (length, precision, scale) while normalizing base type names +// Examples: +// - "character varying(64)" -> "varchar(64)" +// - "integer" -> "int" +// - "timestamp without time zone" -> "timestamp" func normalizeDataType(dataType string) string { dataType = strings.ToLower(strings.TrimSpace(dataType)) - // Map common type variations to standard forms - typeMap := map[string]string{ + // Map common type variations to standard forms (preserving any modifiers) + // Check for types with modifiers first (e.g., "character varying(64)") + replacements := map[string]string{ "character varying": "varchar", "integer": "int", "int4": "int", @@ -238,13 +256,25 @@ func normalizeDataType(dataType string) string { "boolean": "bool", } - if normalized, exists := typeMap[dataType]; exists { - return normalized + // Try to replace the base type name while preserving modifiers + for old, newType := range replacements { + if suffix, found := strings.CutPrefix(dataType, old); found { + // Replace the prefix and keep everything after (modifiers, array brackets, etc.) + return newType + suffix + } } return dataType } +func quoteIdentifierList(names []string) string { + quoted := make([]string, 0, len(names)) + for _, n := range names { + quoted = append(quoted, pq.QuoteIdentifier(n)) + } + return strings.Join(quoted, ", ") +} + // MigrateTableData migrates all data from sourceTable to targetTable after verifying schemas match // This function performs the following steps: // 1. Verifies that both tables have identical column definitions @@ -335,11 +365,12 @@ func (dbc *DB) MigrateTableData(sourceTable, targetTable string, omitColumns []s // Step 5: Perform the migration using INSERT INTO ... SELECT // This is done in a single statement for efficiency and atomicity + columnList := quoteIdentifierList(columnNames) insertSQL := fmt.Sprintf( "INSERT INTO %s (%s) SELECT %s FROM %s", pq.QuoteIdentifier(targetTable), - pq.QuoteIdentifier(strings.Join(columnNames, ", ")), - pq.QuoteIdentifier(strings.Join(columnNames, ", ")), + columnList, + columnList, pq.QuoteIdentifier(sourceTable), ) @@ -525,15 +556,15 @@ func (dbc *DB) MigrateTableDataRange(sourceTable, targetTable, dateColumn string // Step 6: Perform the migration using INSERT INTO ... SELECT ... WHERE // This is done in a single statement for efficiency and atomicity + columnList := quoteIdentifierList(columnNames) insertSQL := fmt.Sprintf( - "INSERT INTO %s (%s) SELECT %s FROM %s WHERE %s >= @start_date AND %s < @end_date", + "INSERT INTO %s (%s) SELECT %s FROM %s WHERE %s >= `@start_date` AND %s < `@end_date`", pq.QuoteIdentifier(targetTable), - pq.QuoteIdentifier(strings.Join(columnNames, ", ")), - pq.QuoteIdentifier(strings.Join(columnNames, ", ")), + columnList, + columnList, pq.QuoteIdentifier(sourceTable), pq.QuoteIdentifier(dateColumn), - pq.QuoteIdentifier(dateColumn), - ) + pq.QuoteIdentifier(dateColumn)) log.WithFields(log.Fields{ "source": sourceTable, @@ -635,23 +666,30 @@ type partitionDateInfo struct { func (dbc *DB) getPartitionsInDateRange(tableName string, startDate, endDate time.Time) ([]partitionDateInfo, error) { var partitions []partitionDateInfo - // Prepare patterns in Go code since named parameters can't be concatenated in SQL - likePattern := tableName + "_%" - regexPattern := tableName + "_\\d{4}_\\d{2}_\\d{2}$" - + // Query only attached partitions using pg_inherits + // Detached partitions won't appear in pg_inherits query := ` + WITH attached_partitions AS ( + SELECT c.relname AS tablename + FROM pg_inherits i + JOIN pg_class c ON i.inhrelid = c.oid + JOIN pg_class p ON i.inhparent = p.oid + WHERE p.relname = @table_name + ) SELECT tablename AS partition_name, TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date FROM pg_tables WHERE schemaname = 'public' - AND tablename LIKE @like_pattern + AND tablename IN (SELECT tablename FROM attached_partitions) AND tablename ~ @regex_pattern ORDER BY partition_date ` + regexPattern := tableName + "_\\d{4}_\\d{2}_\\d{2}$" + result := dbc.DB.Raw(query, - sql.Named("like_pattern", likePattern), + sql.Named("table_name", tableName), sql.Named("regex_pattern", regexPattern), ).Scan(&partitions) if result.Error != nil { @@ -1470,10 +1508,10 @@ func (dbc *DB) RenameTables(tableRenames []TableRename, renameSequences bool, re // Step 3: Dry run - report what would be renamed if dryRun { log.Info("[DRY RUN] would rename the following tables:") - for source, target := range tableRenames { + for _, rename := range tableRenames { log.WithFields(log.Fields{ - "from": source, - "to": target, + "from": rename.From, + "to": rename.To, }).Info("[DRY RUN] table rename") } From f9b16147d7675015c1505f8042916fc83620a457 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Fri, 13 Mar 2026 09:19:37 -0400 Subject: [PATCH 09/11] TRT-1989: partitions updates --- pkg/dataloader/prowloader/prow.go | 4 +- pkg/db/UTILS_README.md | 24 +-- pkg/db/partitions/README.md | 64 +++--- pkg/db/partitions/partitions.go | 316 +++++++++--------------------- pkg/db/utils.go | 11 +- 5 files changed, 151 insertions(+), 268 deletions(-) diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index 3d1a6f049..8eb3fdb97 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -389,9 +389,9 @@ func (pl *ProwLoader) agePartitions(config PartitionManagementConfig) error { detached, err := partitions.DetachOldPartitions(pl.dbc, config.TableName, config.DetachAfter, false) if err != nil { log.WithError(err).Errorf("error detaching partitions for %s", config.TableName) - } else { - log.Infof("detached %d partitions from %s", detached, config.TableName) + return err } + log.Infof("detached %d partitions from %s", detached, config.TableName) dropped, err := partitions.DropOldDetachedPartitions(pl.dbc, config.TableName, config.DropDetachedAfter, false) if err != nil { log.WithError(err).Errorf("error dropping detached partitions for %s", config.TableName) diff --git a/pkg/db/UTILS_README.md b/pkg/db/UTILS_README.md index f3f1d0d51..633d30d3f 100644 --- a/pkg/db/UTILS_README.md +++ b/pkg/db/UTILS_README.md @@ -477,7 +477,7 @@ To keep partition names consistent with the parent table, use `renamePartitions= - If any rename fails, all are rolled back **How Partition Renaming Works:** -``` +```text Old table: orders Old partitions: orders_2024_01_01, orders_2024_01_02 @@ -560,7 +560,7 @@ To keep constraint names consistent with table names, use `renameConstraints=tru - If any rename fails, all are rolled back **How Constraint Renaming Works:** -``` +```text Old table: orders Old constraints: orders_pkey, orders_email_key, orders_customer_id_fkey @@ -579,7 +579,7 @@ New naming: newtable + suffix - Exclusion constraints (`x`) - e.g., `tablename_excl` **Important Note about Indexes:** -Renaming a constraint does NOT rename the backing index. Indexes are separate objects in PostgreSQL and must be renamed separately if needed. The `RenameTables` function currently only renames constraints, not indexes. +Renaming a constraint does NOT rename the backing index. Indexes are separate objects in PostgreSQL and must be renamed separately. Use `renameIndexes=true` in `RenameTables` to rename indexes alongside constraints. **When to use `renameConstraints=true`:** - ✅ When swapping tables in production (keeps naming consistent) @@ -613,7 +613,7 @@ To keep index names consistent with table names, use `renameIndexes=true`: - If any rename fails, all are rolled back **How Index Renaming Works:** -``` +```text Old table: orders Old indexes: orders_pkey, orders_email_key, orders_customer_id_idx @@ -1130,7 +1130,7 @@ for _, part := range partitions { // If partitions follow naming convention, rename them too if len(partitions) > 0 { renames := []db.TableRename{{From: "orders_old", To: "orders"}} - dbc.RenameTables(renames, true, true, true, false, false) // renamePartitions=true + dbc.RenameTables(renames, true, true, true, true, false) // renamePartitions=true, renameConstraints=true, renameIndexes=true } ``` @@ -1660,13 +1660,13 @@ renames := []db.TableRename{ } // Dry run first -_, err := dbc.RenameTables(renames, true, true, true, false, true) +_, err := dbc.RenameTables(renames, true, true, true, true, true) if err != nil { log.Fatal(err) } -// Execute swap (rename sequences and partitions too) -count, err := dbc.RenameTables(renames, true, true, true, false, false) +// Execute swap (rename sequences, partitions, constraints, and indexes too) +count, err := dbc.RenameTables(renames, true, true, true, true, false) if err != nil { log.Fatal(err) } @@ -1683,7 +1683,7 @@ log.WithFields(log.Fields{ // {From: "orders", To: "orders_partitioned"}, // {From: "orders_old", To: "orders"}, // } -// dbc.RenameTables(rollback, true, true, true, false, false) +// dbc.RenameTables(rollback, true, true, true, true, false) ``` --- @@ -1922,16 +1922,16 @@ renames := []db.TableRename{ {From: "orders_new", To: "orders"}, } -_, err := dbc.RenameTables(renames, true, true, true, false, true) +_, err := dbc.RenameTables(renames, true, true, true, true, true) if err != nil { log.WithError(err).Error("validation failed") return } -count, err := dbc.RenameTables(renames, true, true, true, false, false) +count, err := dbc.RenameTables(renames, true, true, true, true, false) // BAD: Direct rename without validation -count, err := dbc.RenameTables(renames, true, true, true, false, false) +count, err := dbc.RenameTables(renames, true, true, true, true, false) ``` ### Verify Before Swapping Tables diff --git a/pkg/db/partitions/README.md b/pkg/db/partitions/README.md index 4ebf21913..3d0613488 100644 --- a/pkg/db/partitions/README.md +++ b/pkg/db/partitions/README.md @@ -392,20 +392,21 @@ type MyModel struct { } // Dry run - see what changes would be made -sql, err := partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", true) +sql, err := partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", true, false) if err != nil { log.WithError(err).Error("dry run failed") } // Prints all ALTER TABLE statements that would be executed // Actual update -sql, err = partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", false) +sql, err = partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", false, false) ``` **Parameters**: - `model` - GORM model struct with desired schema (must be a pointer, e.g., `&models.MyModel{}`) - `tableName` - Name of the existing partitioned table - `dryRun` - If true, prints SQL without executing +- `dropColumns` - If true, columns present in the database but absent from the model will be dropped **How It Works**: 1. Checks if the table exists @@ -472,11 +473,11 @@ type TestResults struct { } // 2. Dry run to see changes -sql, err := partitions.UpdatePartitionedTable(dbc, &TestResults{}, "test_results", true) +sql, err := partitions.UpdatePartitionedTable(dbc, &TestResults{}, "test_results", true, false) fmt.Println("Would execute:", sql) // 3. Review changes, then apply -sql, err = partitions.UpdatePartitionedTable(dbc, &TestResults{}, "test_results", false) +sql, err = partitions.UpdatePartitionedTable(dbc, &TestResults{}, "test_results", false, false) if err != nil { log.Fatal(err) } @@ -647,17 +648,22 @@ if isAttached { --- -#### ReattachPartition -Reattaches a previously detached partition back to the parent table. +#### AttachPartition +Attaches a partition to the parent table with the appropriate date range. ```go // Dry run -err := partitions.ReattachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", true) +err := partitions.AttachPartition(dbc, "test_analysis_by_job_by_dates", "test_analysis_by_job_by_dates_2024_10_29", true) -// Actual reattach -err := partitions.ReattachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", false) +// Actual attach +err := partitions.AttachPartition(dbc, "test_analysis_by_job_by_dates", "test_analysis_by_job_by_dates_2024_10_29", false) ``` +**Parameters**: +- `tableName` - Name of the parent partitioned table +- `partitionName` - Name of the partition to attach (must follow `tableName_YYYY_MM_DD` convention) +- `dryRun` - If true, only logs what would be done + **Use When**: - You need to restore archived data - You detached a partition by mistake @@ -691,24 +697,26 @@ fmt.Printf("Created %d partitions\n", created) **How It Works**: 1. Lists all existing partitions (attached + detached) -2. Generates list of dates in range that don't have partitions -3. For each missing partition: +2. For existing partitions, verifies they are attached and reattaches any detached ones +3. Generates list of dates in range that don't have partitions +4. All creates and attaches run in a single transaction — if any operation fails, all changes are rolled back +5. For each missing partition: - Creates table with same structure as parent (CREATE TABLE ... LIKE) - - Attaches partition with appropriate date range (FOR VALUES FROM ... TO ...) -4. Skips partitions that already exist -5. Returns count of partitions created + - Attaches partition using `AttachPartition` with appropriate date range +6. Returns count of partitions created **Use When**: - Setting up a new partitioned table with historical dates - Backfilling missing partitions after data gaps - Preparing partitions in advance for future dates -- Recovering from partition management issues +- Recovering from partition management issues (including reattaching detached partitions) **Safety Features**: - Checks for existing partitions before creating -- Dry-run mode to preview what will be created -- Automatically cleans up if attachment fails -- Comprehensive logging for each partition +- Reattaches detached partitions found within the date range +- All operations are transactional — atomic success or full rollback +- Dry-run mode to preview what will be created and reattached +- Errors are returned, not silently logged --- @@ -999,7 +1007,7 @@ func manageDetachedPartitions(dbc *db.DB) error { ### Example 7: Reattach Archived Data ```go -func restoreArchivedPartition(dbc *db.DB, partitionName string) error { +func restoreArchivedPartition(dbc *db.DB, tableName, partitionName string) error { // Check current status isAttached, err := partitions.IsPartitionAttached(dbc, partitionName) if err != nil { @@ -1013,7 +1021,7 @@ func restoreArchivedPartition(dbc *db.DB, partitionName string) error { log.WithField("partition", partitionName).Info("reattaching partition") // Reattach the partition - err = partitions.ReattachPartition(dbc, partitionName, false) + err = partitions.AttachPartition(dbc, tableName, partitionName, false) if err != nil { return fmt.Errorf("reattach failed: %w", err) } @@ -1245,7 +1253,8 @@ func updateTestResultsSchema(dbc *db.DB) error { dbc, &TestResultsV2{}, tableName, - true, // dry-run + true, // dry-run + false, // don't drop columns ) if err != nil { return fmt.Errorf("dry run failed: %w", err) @@ -1271,6 +1280,7 @@ func updateTestResultsSchema(dbc *db.DB) error { &TestResultsV2{}, tableName, false, // execute + false, // don't drop columns ) if err != nil { return fmt.Errorf("schema update failed: %w", err) @@ -1293,7 +1303,8 @@ func automatedSchemaMigration(dbc *db.DB) error { dbc, &TestResultsV2{}, tableName, - true, + true, // dry-run + false, // don't drop columns ) if err != nil { return fmt.Errorf("schema check failed: %w", err) @@ -1312,7 +1323,8 @@ func automatedSchemaMigration(dbc *db.DB) error { dbc, &TestResultsV2{}, tableName, - false, + false, // execute + false, // don't drop columns ) if err != nil { return fmt.Errorf("schema migration failed: %w", err) @@ -1335,7 +1347,7 @@ func evolveSchema(dbc *db.DB) error { } log.Info("Phase 1: Adding nullable columns") - _, err := partitions.UpdatePartitionedTable(dbc, &PhaseOne{}, tableName, false) + _, err := partitions.UpdatePartitionedTable(dbc, &PhaseOne{}, tableName, false, false) if err != nil { return err } @@ -1353,7 +1365,7 @@ func evolveSchema(dbc *db.DB) error { } log.Info("Phase 3: Adding indexes") - _, err = partitions.UpdatePartitionedTable(dbc, &PhaseTwo{}, tableName, false) + _, err = partitions.UpdatePartitionedTable(dbc, &PhaseTwo{}, tableName, false, false) if err != nil { return err } @@ -1623,7 +1635,7 @@ gunzip -c partition_2024_10_29.pgdump.gz | pg_restore -d $SIPPY_DSN 2. **Reattach partition**: ```go -err := partitions.ReattachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", false) +err := partitions.AttachPartition(dbc, "test_analysis_by_job_by_dates", "test_analysis_by_job_by_dates_2024_10_29", false) ``` ### Advantages of Detach vs. DROP diff --git a/pkg/db/partitions/partitions.go b/pkg/db/partitions/partitions.go index 1c2b85f19..fc2bc09ab 100644 --- a/pkg/db/partitions/partitions.go +++ b/pkg/db/partitions/partitions.go @@ -14,6 +14,14 @@ import ( "github.com/openshift/sippy/pkg/db" ) +// escapeForLike escapes characters that have special meaning in SQL LIKE patterns. +func escapeForLike(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `%`, `\%`) + s = strings.ReplaceAll(s, `_`, `\_`) + return s +} + // PartitionInfo holds metadata about a partition type PartitionInfo struct { TableName string `gorm:"column:tablename"` @@ -195,11 +203,11 @@ func ListTablePartitions(dbc *db.DB, tableName string) ([]PartitionInfo, error) LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename AND pg_stat_user_tables.schemaname = pg_tables.schemaname WHERE pg_tables.schemaname = 'public' - AND pg_tables.tablename LIKE @table_pattern + AND pg_tables.tablename LIKE @table_pattern ESCAPE '\' ORDER BY partition_date ASC ` - tablePattern := tableName + "_20%" + tablePattern := escapeForLike(tableName) + `\_20%` result := dbc.DB.Raw(query, sql.Named("table_pattern", tablePattern)).Scan(&partitions) if result.Error != nil { log.WithError(result.Error).WithField("table", tableName).Error("failed to list table partitions") @@ -229,7 +237,7 @@ func GetPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, error) { pg_total_relation_size('public.'||tablename) AS size_bytes FROM pg_tables WHERE schemaname = 'public' - AND tablename LIKE @table_pattern + AND tablename LIKE @table_pattern ESCAPE '\' ) SELECT COUNT(*)::INT AS total_partitions, @@ -242,7 +250,7 @@ func GetPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, error) { FROM partition_info ` - tablePattern := tableName + "_20%" + tablePattern := escapeForLike(tableName) + `\_20%` result := dbc.DB.Raw(query, sql.Named("table_pattern", tablePattern)).Scan(&stats) if result.Error != nil { log.WithError(result.Error).WithField("table", tableName).Error("failed to get partition statistics") @@ -293,7 +301,7 @@ func GetPartitionsForRemoval(dbc *db.DB, tableName string, retentionDays int, at LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename AND pg_stat_user_tables.schemaname = pg_tables.schemaname WHERE pg_tables.schemaname = 'public' - AND pg_tables.tablename LIKE @table_pattern + AND pg_tables.tablename LIKE @table_pattern ESCAPE '\' AND pg_tables.tablename IN (SELECT tablename FROM attached_partitions) AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date ORDER BY partition_date ASC @@ -313,13 +321,13 @@ func GetPartitionsForRemoval(dbc *db.DB, tableName string, retentionDays int, at LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename AND pg_stat_user_tables.schemaname = pg_tables.schemaname WHERE pg_tables.schemaname = 'public' - AND pg_tables.tablename LIKE @table_pattern + AND pg_tables.tablename LIKE @table_pattern ESCAPE '\' AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date ORDER BY partition_date ASC ` } - tablePattern := tableName + "_20%" + tablePattern := escapeForLike(tableName) + `\_20%` result := dbc.DB.Raw(query, sql.Named("table_name", tableName), sql.Named("table_pattern", tablePattern), @@ -373,7 +381,7 @@ func GetRetentionSummary(dbc *db.DB, tableName string, retentionDays int, attach MAX(tablename) AS newest_partition FROM pg_tables WHERE schemaname = 'public' - AND tablename LIKE @table_pattern + AND tablename LIKE @table_pattern ESCAPE '\' AND tablename IN (SELECT tablename FROM attached_partitions) AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date ` @@ -388,12 +396,12 @@ func GetRetentionSummary(dbc *db.DB, tableName string, retentionDays int, attach MAX(tablename) AS newest_partition FROM pg_tables WHERE schemaname = 'public' - AND tablename LIKE @table_pattern + AND tablename LIKE @table_pattern ESCAPE '\' AND TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') < @cutoff_date ` } - tablePattern := tableName + "_20%" + tablePattern := escapeForLike(tableName) + `\_20%` result := dbc.DB.Raw(query, sql.Named("table_name", tableName), sql.Named("table_pattern", tablePattern), @@ -416,94 +424,6 @@ func GetRetentionSummary(dbc *db.DB, tableName string, retentionDays int, attach return &summary, nil } -// GetPartitionsByAgeGroup returns partition counts and sizes grouped by age buckets for a given table -func GetPartitionsByAgeGroup(dbc *db.DB, tableName string) ([]map[string]interface{}, error) { - start := time.Now() - - query := ` - WITH partition_ages AS ( - SELECT - tablename, - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD') AS partition_date, - (CURRENT_DATE - TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD'))::INT AS age_days, - pg_total_relation_size('public.'||tablename) AS size_bytes - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE @table_pattern - ) - SELECT - CASE - WHEN age_days < 0 THEN 'Future' - WHEN age_days < 30 THEN '0-30 days' - WHEN age_days < 90 THEN '30-90 days' - WHEN age_days < 180 THEN '90-180 days' - WHEN age_days < 365 THEN '180-365 days' - ELSE '365+ days' - END AS age_bucket, - COUNT(*)::INT AS partition_count, - SUM(size_bytes)::BIGINT AS total_size_bytes, - pg_size_pretty(SUM(size_bytes)) AS total_size, - ROUND(SUM(size_bytes) * 100.0 / SUM(SUM(size_bytes)) OVER (), 2) AS percentage - FROM partition_ages - GROUP BY age_bucket - ORDER BY MIN(age_days) - ` - - tablePattern := tableName + "_20%" - var results []map[string]interface{} - err := dbc.DB.Raw(query, sql.Named("table_pattern", tablePattern)).Scan(&results).Error - if err != nil { - log.WithError(err).WithField("table", tableName).Error("failed to get partitions by age group") - return nil, err - } - - elapsed := time.Since(start) - log.WithFields(log.Fields{ - "table": tableName, - "groups": len(results), - "elapsed": elapsed, - }).Info("retrieved partitions by age group") - - return results, nil -} - -// GetPartitionsByMonth returns partition counts and sizes grouped by month for a given table -func GetPartitionsByMonth(dbc *db.DB, tableName string) ([]map[string]interface{}, error) { - start := time.Now() - - query := ` - SELECT - DATE_TRUNC('month', TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD')) AS month, - COUNT(*)::INT AS partition_count, - pg_size_pretty(SUM(pg_total_relation_size('public.'||tablename))) AS total_size, - pg_size_pretty(AVG(pg_total_relation_size('public.'||tablename))::BIGINT) AS avg_partition_size, - MIN(tablename) AS first_partition, - MAX(tablename) AS last_partition - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE @table_pattern - GROUP BY DATE_TRUNC('month', TO_DATE(SUBSTRING(tablename FROM '_(\d{4}_\d{2}_\d{2})$'), 'YYYY_MM_DD')) - ORDER BY month DESC - ` - - tablePattern := tableName + "_20%" - var results []map[string]interface{} - err := dbc.DB.Raw(query, sql.Named("table_pattern", tablePattern)).Scan(&results).Error - if err != nil { - log.WithError(err).WithField("table", tableName).Error("failed to get partitions by month") - return nil, err - } - - elapsed := time.Since(start) - log.WithFields(log.Fields{ - "table": tableName, - "months": len(results), - "elapsed": elapsed, - }).Info("retrieved partitions by month") - - return results, nil -} - // ValidateRetentionPolicy checks if a retention policy would be safe to apply for a given table // Returns an error if the policy would delete critical data or too much data // Only considers attached partitions when validating thresholds @@ -643,52 +563,6 @@ func DetachPartition(dbc *db.DB, partitionName string, dryRun bool) error { return nil } -// DropOldPartitions drops all partitions older than the retention period for a given table -// This is a bulk operation wrapper that calls DropPartition for each old partition -func DropOldPartitions(dbc *db.DB, tableName string, retentionDays int, dryRun bool) (int, error) { - start := time.Now() - - // Validate retention policy first - if err := ValidateRetentionPolicy(dbc, tableName, retentionDays); err != nil { - return 0, fmt.Errorf("retention policy validation failed: %w", err) - } - - // Get all partitions for removal (both attached and detached) - partitions, err := GetPartitionsForRemoval(dbc, tableName, retentionDays, true) - if err != nil { - return 0, fmt.Errorf("failed to get partitions for removal: %w", err) - } - - if len(partitions) == 0 { - log.WithField("table", tableName).Info("no partitions to delete") - return 0, nil - } - - droppedCount := 0 - var totalSize int64 - - for _, partition := range partitions { - if err := DropPartition(dbc, partition.TableName, dryRun); err != nil { - log.WithError(err).WithField("partition", partition.TableName).Error("failed to drop partition") - continue - } - droppedCount++ - totalSize += partition.SizeBytes - } - - elapsed := time.Since(start) - log.WithFields(log.Fields{ - "table": tableName, - "retention_days": retentionDays, - "total_dropped": droppedCount, - "storage_reclaimed": fmt.Sprintf("%d bytes", totalSize), - "dry_run": dryRun, - "elapsed": elapsed, - }).Info("completed dropping old partitions") - - return droppedCount, nil -} - // DropOldDetachedPartitions drops detached partitions older than retentionDays (DESTRUCTIVE) // This removes detached partitions that are no longer needed // Use this after archiving detached partitions or when you're sure the data is no longer needed @@ -779,12 +653,12 @@ func ListDetachedPartitions(dbc *db.DB, tableName string) ([]PartitionInfo, erro LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relname = pg_tables.tablename AND pg_stat_user_tables.schemaname = pg_tables.schemaname WHERE pg_tables.schemaname = 'public' - AND pg_tables.tablename LIKE @table_pattern + AND pg_tables.tablename LIKE @table_pattern ESCAPE '\' AND pg_tables.tablename NOT IN (SELECT tablename FROM attached_partitions) ORDER BY partition_date ASC ` - tablePattern := tableName + "_20%" + tablePattern := escapeForLike(tableName) + `\_20%` result := dbc.DB.Raw(query, sql.Named("table_name", tableName), sql.Named("table_pattern", tablePattern)).Scan(&partitions) @@ -939,18 +813,29 @@ func CreateMissingPartitions(dbc *db.DB, tableName string, startDate, endDate ti existingDates[dateStr] = true } - // Generate list of partitions to create + // Generate list of partitions to create and detached partitions to reattach var partitionsToCreate []time.Time + var partitionsToReattach []string currentDate := startDate for !currentDate.After(endDate) { dateStr := currentDate.Format("2006_01_02") if !existingDates[dateStr] { partitionsToCreate = append(partitionsToCreate, currentDate) + } else { + // Partition exists — verify it is attached + partitionName := fmt.Sprintf("%s_%s", tableName, dateStr) + attached, err := IsPartitionAttached(dbc, partitionName) + if err != nil { + return 0, fmt.Errorf("failed to check if partition %s is attached: %w", partitionName, err) + } + if !attached { + partitionsToReattach = append(partitionsToReattach, partitionName) + } } currentDate = currentDate.AddDate(0, 0, 1) // Move to next day } - if len(partitionsToCreate) == 0 { + if len(partitionsToCreate) == 0 && len(partitionsToReattach) == 0 { log.WithFields(log.Fields{ "table": tableName, "start_date": startDate.Format("2006-01-02"), @@ -959,54 +844,52 @@ func CreateMissingPartitions(dbc *db.DB, tableName string, startDate, endDate ti return 0, nil } - createdCount := 0 - for _, partitionDate := range partitionsToCreate { - partitionName := fmt.Sprintf("%s_%s", tableName, partitionDate.Format("2006_01_02")) - rangeStart := partitionDate.Format("2006-01-02") - rangeEnd := partitionDate.AddDate(0, 0, 1).Format("2006-01-02") - - if dryRun { + if dryRun { + for _, partitionDate := range partitionsToCreate { + partitionName := fmt.Sprintf("%s_%s", tableName, partitionDate.Format("2006_01_02")) log.WithFields(log.Fields{ - "partition": partitionName, - "table": tableName, - "range_start": rangeStart, - "range_end": rangeEnd, + "partition": partitionName, + "table": tableName, }).Info("[DRY RUN] would create partition") - createdCount++ - continue } + for _, partitionName := range partitionsToReattach { + log.WithFields(log.Fields{ + "partition": partitionName, + "table": tableName, + }).Info("[DRY RUN] would reattach partition") + } + return len(partitionsToCreate), nil + } - // Create the partition table with same structure as parent - createTableQuery := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (LIKE %s INCLUDING ALL)", pq.QuoteIdentifier(partitionName), pq.QuoteIdentifier(tableName)) - result := dbc.DB.Exec(createTableQuery) - if result.Error != nil { - log.WithError(result.Error).WithField("partition", partitionName).Error("failed to create partition table") - continue + createdCount := 0 + err = dbc.DB.Transaction(func(tx *gorm.DB) error { + txDBC := &db.DB{DB: tx} + + for _, partitionName := range partitionsToReattach { + if err := AttachPartition(txDBC, tableName, partitionName, false); err != nil { + return fmt.Errorf("failed to reattach partition %s: %w", partitionName, err) + } } - // Attach the partition to the parent table - attachQuery := fmt.Sprintf( - "ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM ('%s') TO ('%s')", - pq.QuoteIdentifier(tableName), - pq.QuoteIdentifier(partitionName), - rangeStart, - rangeEnd, - ) - result = dbc.DB.Exec(attachQuery) - if result.Error != nil { - // If attach fails, try to clean up the created table - log.WithError(result.Error).WithField("partition", partitionName).Error("failed to attach partition") - dbc.DB.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", pq.QuoteIdentifier(partitionName))) - continue + for _, partitionDate := range partitionsToCreate { + partitionName := fmt.Sprintf("%s_%s", tableName, partitionDate.Format("2006_01_02")) + + createTableQuery := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (LIKE %s INCLUDING ALL)", pq.QuoteIdentifier(partitionName), pq.QuoteIdentifier(tableName)) + if result := tx.Exec(createTableQuery); result.Error != nil { + return fmt.Errorf("failed to create partition table %s: %w", partitionName, result.Error) + } + + if err := AttachPartition(txDBC, tableName, partitionName, false); err != nil { + return fmt.Errorf("failed to attach partition %s: %w", partitionName, err) + } + + createdCount++ } - log.WithFields(log.Fields{ - "partition": partitionName, - "table": tableName, - "range_start": rangeStart, - "range_end": rangeEnd, - }).Info("created and attached partition") - createdCount++ + return nil + }) + if err != nil { + return 0, err } elapsed := time.Since(start) @@ -1015,7 +898,7 @@ func CreateMissingPartitions(dbc *db.DB, tableName string, startDate, endDate ti "start_date": startDate.Format("2006-01-02"), "end_date": endDate.Format("2006-01-02"), "created": createdCount, - "total_days": len(partitionsToCreate), + "reattached": len(partitionsToReattach), "dry_run": dryRun, "elapsed": elapsed, }).Info("completed creating missing partitions") @@ -1386,15 +1269,16 @@ type indexInfo struct { // - model: GORM model struct (must be a pointer, e.g., &models.MyModel{}) // - tableName: Name of the existing partitioned table // - dryRun: If true, prints SQL without executing +// - dropColumns: If true, columns present in the database but absent from the model will be dropped // // Returns: The SQL statements that were (or would be) executed // // Example: // -// sql, err := partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_table", true) +// sql, err := partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_table", true, false) // // Note: Cannot modify partition keys or add unique constraints without partition keys -func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dryRun bool) (string, error) { +func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dryRun bool, dropColumns bool) (string, error) { start := time.Now() // Check if table exists @@ -1606,10 +1490,12 @@ func UpdatePartitionedTable(dbc *db.DB, model interface{}, tableName string, dry delete(currentColMap, field.DBName) } - // Remaining columns in map should be dropped - for colName := range currentColMap { - alterStatements = append(alterStatements, - fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", pq.QuoteIdentifier(tableName), colName)) + // Remaining columns in map are not in the model + if dropColumns { + for colName := range currentColMap { + alterStatements = append(alterStatements, + fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", pq.QuoteIdentifier(tableName), colName)) + } } // Check indexes @@ -1957,7 +1843,7 @@ func GetDetachedPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, e pg_total_relation_size('public.'||tablename) AS size_bytes FROM pg_tables WHERE schemaname = 'public' - AND tablename LIKE @table_pattern + AND tablename LIKE @table_pattern ESCAPE '\' AND tablename NOT IN (SELECT tablename FROM attached_partitions) ) SELECT @@ -1971,7 +1857,7 @@ func GetDetachedPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, e FROM detached_info ` - tablePattern := tableName + "_20%" + tablePattern := escapeForLike(tableName) + `\_20%` result := dbc.DB.Raw(query, sql.Named("table_name", tableName), sql.Named("table_pattern", tablePattern)).Scan(&stats) @@ -1991,17 +1877,11 @@ func GetDetachedPartitionStats(dbc *db.DB, tableName string) (*PartitionStats, e return &stats, nil } -// ReattachPartition reattaches a previously detached partition back to the parent table -// This is useful if a partition was detached for archival but needs to be restored -func ReattachPartition(dbc *db.DB, partitionName string, dryRun bool) error { +// AttachPartition attaches a partition to the parent table with the appropriate date range +// The partition name must follow the convention tableName_YYYY_MM_DD +func AttachPartition(dbc *db.DB, tableName string, partitionName string, dryRun bool) error { start := time.Now() - // Extract table name from partition name - tableName, err := extractTableNameFromPartition(partitionName) - if err != nil { - return fmt.Errorf("invalid partition name: %w", err) - } - // Validate partition name format for safety if !isValidPartitionName(tableName, partitionName) { return fmt.Errorf("invalid partition name: %s - must match %s_YYYY_MM_DD", partitionName, tableName) @@ -2016,26 +1896,26 @@ func ReattachPartition(dbc *db.DB, partitionName string, dryRun bool) error { } // Calculate date range for the partition - startDate := partitionDate.Format("2006-01-02") - endDate := partitionDate.AddDate(0, 0, 1).Format("2006-01-02") + rangeStart := partitionDate.Format("2006-01-02") + rangeEnd := partitionDate.AddDate(0, 0, 1).Format("2006-01-02") if dryRun { log.WithFields(log.Fields{ - "partition": partitionName, - "table": tableName, - "start_date": startDate, - "end_date": endDate, - }).Info("[DRY RUN] would reattach partition") + "partition": partitionName, + "table": tableName, + "range_start": rangeStart, + "range_end": rangeEnd, + }).Info("[DRY RUN] would attach partition") return nil } - // Reattach the partition with FOR VALUES clause + // Attach the partition with FOR VALUES clause query := fmt.Sprintf( "ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM ('%s') TO ('%s')", pq.QuoteIdentifier(tableName), pq.QuoteIdentifier(partitionName), - startDate, - endDate, + rangeStart, + rangeEnd, ) result := dbc.DB.Exec(query) @@ -2043,7 +1923,7 @@ func ReattachPartition(dbc *db.DB, partitionName string, dryRun bool) error { log.WithError(result.Error).WithFields(log.Fields{ "partition": partitionName, "table": tableName, - }).Error("failed to reattach partition") + }).Error("failed to attach partition") return result.Error } @@ -2052,7 +1932,7 @@ func ReattachPartition(dbc *db.DB, partitionName string, dryRun bool) error { "partition": partitionName, "table": tableName, "elapsed": elapsed, - }).Info("reattached partition") + }).Info("attached partition") return nil } diff --git a/pkg/db/utils.go b/pkg/db/utils.go index 04bfe1437..c40d8ddd4 100644 --- a/pkg/db/utils.go +++ b/pkg/db/utils.go @@ -42,15 +42,6 @@ type ColumnVerificationOptions struct { CheckOrder bool } -// DefaultColumnVerificationOptions returns options with all checks enabled -func DefaultColumnVerificationOptions() ColumnVerificationOptions { - return ColumnVerificationOptions{ - CheckNullable: true, - CheckDefaults: true, - CheckOrder: true, - } -} - // DataMigrationColumnVerificationOptions returns options suitable for data migrations // (only checks column names and types, not constraints or defaults) func DataMigrationColumnVerificationOptions() ColumnVerificationOptions { @@ -558,7 +549,7 @@ func (dbc *DB) MigrateTableDataRange(sourceTable, targetTable, dateColumn string // This is done in a single statement for efficiency and atomicity columnList := quoteIdentifierList(columnNames) insertSQL := fmt.Sprintf( - "INSERT INTO %s (%s) SELECT %s FROM %s WHERE %s >= `@start_date` AND %s < `@end_date`", + "INSERT INTO %s (%s) SELECT %s FROM %s WHERE %s >= @start_date AND %s < @end_date", pq.QuoteIdentifier(targetTable), columnList, columnList, From ec1c1b56a342901aec67252a7df60c00554bb2e8 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Fri, 13 Mar 2026 10:31:09 -0400 Subject: [PATCH 10/11] TRT-1989: remove readmes --- pkg/db/UTILS_README.md | 2083 ----------------------------------- pkg/db/partitions/README.md | 1736 ----------------------------- 2 files changed, 3819 deletions(-) delete mode 100644 pkg/db/UTILS_README.md delete mode 100644 pkg/db/partitions/README.md diff --git a/pkg/db/UTILS_README.md b/pkg/db/UTILS_README.md deleted file mode 100644 index 633d30d3f..000000000 --- a/pkg/db/UTILS_README.md +++ /dev/null @@ -1,2083 +0,0 @@ -# Database Utilities - -This package provides utility functions for database operations including schema verification and data migration. - -## Overview - -The utilities in `utils.go` provide safe, validated operations for working with database tables, particularly useful for: -- Schema migration and validation -- Data migration between tables -- Atomic table renames and swaps -- Sequence management and auditing -- Partition management workflows -- Table consolidation and archival - -## Quick Function Reference - -**Schema Verification:** -- `VerifyTablesHaveSameColumns` - Compare table schemas -- `GetTableColumns` - Get column metadata for a table - -**Data Migration:** -- `MigrateTableData` - Copy all data between tables -- `MigrateTableDataRange` - Copy data for specific date range -- `GetTableRowCount` - Count rows in a table - -**Table Renaming:** -- `RenameTables` - Atomically rename tables, sequences, and partitions - -**Sequence Management:** -- `GetSequenceMetadata` - Get detailed linkage info (SERIAL vs IDENTITY) -- `GetTableSequences` - List sequences for a specific table -- `ListAllTableSequences` - List sequences for all tables -- `SyncIdentityColumn` - Sync IDENTITY sequence after data migration - -**Partition Information:** -- `GetTablePartitions` - List partitions for a specific table -- `GetPartitionStrategy` - Check if table is partitioned (RANGE/LIST/HASH) -- `VerifyPartitionCoverage` - Verify all partitions exist for date range - -**Constraint Information:** -- `GetTableConstraints` - List constraints for a specific table - -**Index Information:** -- `GetTableIndexes` - List indexes for a specific table - -## Functions - -### VerifyTablesHaveSameColumns - -Verifies that two tables have identical column definitions with configurable verification options. - -```go -// Full verification (default) - checks all aspects -err := dbc.VerifyTablesHaveSameColumns("source_table", "target_table", DefaultColumnVerificationOptions()) -if err != nil { - log.WithError(err).Error("tables have different schemas") -} - -// Data migration verification - only checks names and types -err := dbc.VerifyTablesHaveSameColumns("source_table", "target_table", DataMigrationColumnVerificationOptions()) -if err != nil { - log.WithError(err).Error("incompatible schemas for migration") -} -``` - -**Verification Options:** - -| Option | DefaultColumnVerificationOptions | DataMigrationColumnVerificationOptions | -|--------|----------------------------------|---------------------------------------| -| Column names | ✓ | ✓ | -| Data types | ✓ | ✓ | -| NOT NULL constraints | ✓ | ✗ | -| DEFAULT values | ✓ | ✗ | -| Column ordering | ✓ | ✓ | - -**Custom Options:** -```go -opts := ColumnVerificationOptions{ - CheckNullable: true, // Verify NOT NULL constraints match - CheckDefaults: false, // Skip default value comparison - CheckOrder: true, // Verify column order matches -} -err := dbc.VerifyTablesHaveSameColumns("table1", "table2", opts) -``` - -**Always Compared:** -- Column names (always required to match) -- Data types (with normalization, always required to match) - -**Optionally Compared:** -- NOT NULL constraints (controlled by `CheckNullable`) -- DEFAULT values (controlled by `CheckDefaults`) -- Column ordering/position (controlled by `CheckOrder`) - -**Returns:** -- `nil` if tables have compatible schemas -- Error with detailed description of differences - -**Use Cases:** -- **Full verification** (`DefaultColumnVerificationOptions`): - - Verifying partitions match parent table exactly - - Ensuring replicas have identical structures - - Validating table clones or backups - -- **Migration verification** (`DataMigrationColumnVerificationOptions`): - - Pre-migration schema validation - - Verifying data can be copied between tables - - Checking compatibility for INSERT INTO ... SELECT operations - -**Why Skip Nullable/Defaults for Migrations?** - -When migrating data with `INSERT INTO target SELECT * FROM source`, PostgreSQL only requires that: -- Column names exist in both tables -- Data types are compatible - -Nullable constraints and default values don't affect the data copy itself, so checking them is optional for migrations. - ---- - -### MigrateTableData - -Migrates all data from one table to another after verifying schemas match. - -```go -// Dry run first -rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", nil, true) - -// Actual migration -rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", nil, false) -if err != nil { - log.WithError(err).Error("migration failed") -} - -// Migrate with omitting columns (e.g., to use target's auto-increment for id) -rowsMigrated, err := dbc.MigrateTableData("source_table", "target_table", []string{"id"}, false) -if err != nil { - log.WithError(err).Error("migration failed") -} -``` - -**Process:** -1. Verifies schemas match using `VerifyTablesHaveSameColumns` -2. Checks row counts in both tables -3. Performs `INSERT INTO target SELECT * FROM source` -4. Verifies row counts after migration -5. Logs all steps with detailed metrics - -**Parameters:** -- `sourceTable` - Table to copy data from -- `targetTable` - Table to copy data to -- `omitColumns` - List of column names to omit from migration (e.g., `[]string{"id"}` to use target's auto-increment). Pass `nil` to copy all columns. -- `dryRun` - If true, only verifies without copying data - -**Returns:** -- `rowsMigrated` - Number of rows successfully migrated (0 if dry run) -- `error` - Any error encountered during migration - -**Features:** -- Atomic operation (single INSERT statement) -- Dry-run support for safety -- Pre and post verification -- Comprehensive logging -- Handles empty source tables gracefully - -**Safety:** -- DOES NOT truncate target table (appends data) -- DOES NOT drop source table -- Fails fast if schemas don't match -- Warns on row count mismatches - -**Use Cases:** -- Migrating detached partitions to archive tables -- Consolidating multiple tables into one -- Moving data between environments -- Table restructuring workflows - ---- - -### MigrateTableDataRange - -Migrates data within a specific date range from one table to another after verifying schemas match. - -```go -startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) -endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) - -// Dry run first -rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, nil, true) - -// Actual migration -rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, nil, false) -if err != nil { - log.WithError(err).Error("migration failed") -} - -// Migrate with omitting columns (e.g., to use target's auto-increment for id) -rowsMigrated, err := dbc.MigrateTableDataRange("source_table", "target_table", "created_at", startDate, endDate, []string{"id"}, false) -if err != nil { - log.WithError(err).Error("migration failed") -} -``` - -**Process:** -1. Validates date range (endDate must be after startDate) -2. Verifies schemas match using `VerifyTablesHaveSameColumns` -3. Checks if target table is RANGE partitioned and verifies all necessary partitions exist for the date range -4. Counts rows in source table within date range -5. Performs `INSERT INTO target SELECT * FROM source WHERE date_column >= start AND date_column < end` -6. Verifies row counts after migration -7. Logs all steps with detailed metrics - -**Parameters:** -- `sourceTable` - Table to copy data from -- `targetTable` - Table to copy data to -- `dateColumn` - Column name to filter by date (e.g., "created_at") -- `startDate` - Start of date range (inclusive, >=) -- `endDate` - End of date range (exclusive, <) -- `omitColumns` - List of column names to omit from migration (e.g., `[]string{"id"}` to use target's auto-increment). Pass `nil` to copy all columns. -- `dryRun` - If true, only verifies without copying data - -**Returns:** -- `rowsMigrated` - Number of rows successfully migrated (0 if dry run) -- `error` - Any error encountered during migration - -**Features:** -- Atomic operation (single INSERT statement) -- Dry-run support for safety -- Pre and post verification -- Comprehensive logging -- Handles empty date ranges gracefully -- Date range validation -- Automatic partition coverage verification for RANGE partitioned tables -- Prevents migration failures due to missing partitions - -**Safety:** -- DOES NOT truncate target table (appends data) -- DOES NOT drop source table -- Fails fast if schemas don't match -- Warns on row count mismatches -- Validates date range before execution - -**Use Cases:** -- Migrating large tables incrementally (month by month, year by year) -- Testing migrations with a subset of data before full migration -- Moving specific time periods to archive tables -- Backfilling historical data into partitioned tables -- Reducing lock contention by migrating in smaller batches -- Being able to pause and resume large migrations - -**Example - Incremental Monthly Migration:** -```go -// Migrate data month by month for 2024 -for month := 1; month <= 12; month++ { - startDate := time.Date(2024, time.Month(month), 1, 0, 0, 0, 0, time.UTC) - endDate := startDate.AddDate(0, 1, 0) - - rows, err := dbc.MigrateTableDataRange("orders", "orders_new", "order_date", startDate, endDate, false) - if err != nil { - log.WithError(err).WithField("month", month).Error("failed") - continue - } - log.WithField("rows", rows).Info("month migrated") -} -``` - ---- - -### GetTableRowCount - -Returns the number of rows in a table. - -```go -count, err := dbc.GetTableRowCount("table_name") -if err != nil { - log.WithError(err).Error("failed to get row count") -} -log.WithField("count", count).Info("table row count") -``` - -**Use Cases:** -- Pre-migration verification -- Monitoring table growth -- Validating migration success -- Capacity planning - ---- - -### RenameTables - -Renames multiple tables atomically in a single transaction. - -```go -// Order matters - renames are executed in the order provided -renames := []db.TableRename{ - {From: "orders_old", To: "orders_backup"}, - {From: "orders_new", To: "orders"}, -} - -// Dry run first (renameSequences=true, renamePartitions=true, renameConstraints=true, renameIndexes=true) -_, err := dbc.RenameTables(renames, true, true, true, true, true) -if err != nil { - log.WithError(err).Error("validation failed") -} - -// Execute renames (renameSequences=true, renamePartitions=true, renameConstraints=true, renameIndexes=true) -count, err := dbc.RenameTables(renames, true, true, true, true, false) -if err != nil { - log.WithError(err).Error("rename failed") -} -log.WithField("renamed", count).Info("tables, partitions, sequences, constraints, and indexes renamed") -``` - -**How It Works**: -1. Validates that all source tables exist -2. Checks for conflicts (target table already exists, unless it's also being renamed) -3. Executes all `ALTER TABLE ... RENAME TO ...` statements in the order provided -4. Either all renames succeed or all are rolled back in a single transaction - -**Parameters**: -- `tableRenames`: Ordered slice of TableRename structs specifying renames to execute -- `renameSequences`: If true, also renames sequences owned by table columns (SERIAL, BIGSERIAL, IDENTITY) -- `renamePartitions`: If true, also renames child partitions of partitioned tables -- `renameConstraints`: If true, also renames table constraints (primary keys, foreign keys, unique, check) -- `renameIndexes`: If true, also renames table indexes (including those backing constraints) -- `dryRun`: If true, only validates without executing - -**Returns**: -- `renamedCount`: Number of tables successfully renamed (0 if dry run) -- `error`: Any error encountered - -**Note**: Caller is responsible for ordering renames correctly to avoid naming conflicts. For table swaps (A→B, B→C), ensure B→C comes before A→B in the array. - -**Features**: -- **Atomic operation**: All renames happen in one transaction -- **Validation**: Checks source tables exist and no conflicts -- **Dry-run support**: Test before executing -- **Fast**: PostgreSQL only updates metadata, not data -- **Safe**: Views, indexes, and foreign keys are automatically updated - -**Use Cases**: -- Swapping partitioned tables with non-partitioned tables -- Renaming related tables together for consistency -- Atomic schema migrations -- Creating backups before migrations -- Rolling back failed migrations - -**Important Notes**: -- All renames must succeed or all will fail (atomic) -- Table swap scenarios are detected and allowed (when target is also a source) -- Extremely fast - only metadata is updated -- PostgreSQL automatically updates dependent object **references** (views, FKs) but NOT their names -- **Sequences are NOT automatically renamed by PostgreSQL** - use `renameSequences=true` to rename them -- **Partitions are NOT automatically renamed by PostgreSQL** - use `renamePartitions=true` to rename them -- **Constraints are NOT automatically renamed by PostgreSQL** - use `renameConstraints=true` to rename them -- **Indexes are NOT automatically renamed by PostgreSQL** - use `renameIndexes=true` to rename them -- **Rename order matters** - sequences/constraints/indexes are processed in sorted order to avoid naming conflicts during table swaps - -**Understanding SERIAL vs IDENTITY:** - -Both create auto-increment columns, but with different syntax and internal linkage: - -```sql --- Old way: SERIAL (still widely used) -CREATE TABLE orders ( - id SERIAL PRIMARY KEY, - name TEXT -); --- Creates sequence: orders_id_seq --- Linkage: pg_depend (deptype='a') + column DEFAULT nextval('orders_id_seq') - --- Modern way: IDENTITY (SQL standard, recommended) -CREATE TABLE orders ( - id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, - name TEXT -); --- Creates sequence: orders_id_seq --- Linkage: pg_depend (deptype='i') + pg_attribute.attidentity -``` - -**Key Differences:** - -| Aspect | SERIAL | IDENTITY | -|--------|--------|----------| -| SQL Standard | No (PostgreSQL-specific) | Yes (SQL:2003 standard) | -| Dependency Type | `'a'` (auto) | `'i'` (internal) | -| Column Default | `nextval('seq_name')` (name-based) | None (OID-based internally) | -| Rename Safety | Default uses sequence NAME | Fully OID-based, safer | -| PostgreSQL Tables | `pg_depend` + `pg_attrdef` | `pg_depend` + `pg_attribute` | - -**How Sequences Are Linked to Columns:** - -PostgreSQL uses multiple mechanisms to link sequences to columns: - -1. **`pg_depend`** - Dependency tracking (OID-based, survives renames) - - SERIAL: `deptype = 'a'` (auto dependency) - - IDENTITY: `deptype = 'i'` (internal dependency) - -2. **Column Metadata:** - - SERIAL: Column default = `nextval('sequence_name')` (stored as text!) - - IDENTITY: `pg_attribute.attidentity` = `'d'` or `'a'` (uses OID reference) - -3. **Sequence Ownership:** - - Both: `pg_sequence` records which table.column owns the sequence - -**Why Our RenameTables Function Works Safely:** - -When we execute `ALTER SEQUENCE old_seq RENAME TO new_seq`: - -✅ **IDENTITY columns (safe):** -- `pg_depend` uses OID, not name → automatically updated -- `pg_attribute.attidentity` uses OID → no change needed -- Column has NO default expression → nothing to update -- **Result: Fully automatic, zero risk** - -⚠️ **SERIAL columns (mostly safe):** -- `pg_depend` uses OID, not name → automatically updated -- BUT: Column default `nextval('old_seq')` is stored as TEXT -- PostgreSQL does NOT automatically update the default expression -- **However**: `nextval()` resolves the sequence name at runtime, and PostgreSQL's search path finds the renamed sequence -- **Result: Works in practice, but default text is stale** - -**Both are captured by:** -- `GetTableSequences` / `ListAllTableSequences` -- `RenameTables(renameSequences=true)` -- `SyncIdentityColumn` - -**About Sequence Renaming:** - -When you rename a table in PostgreSQL, **sequences are NOT automatically renamed**. This can lead to naming inconsistencies: - -```sql --- Before rename: --- Table: orders --- Sequence: orders_id_seq - -ALTER TABLE orders RENAME TO orders_old; - --- After rename: --- Table: orders_old --- Sequence: orders_id_seq (still has old name!) -``` - -To keep sequence names consistent with table names, use `renameSequences=true`: -- Finds all sequences owned by table columns (SERIAL, BIGSERIAL, IDENTITY) -- Renames them to match new table name: `newtable_columnname_seq` -- All renames (tables + sequences) happen in one atomic transaction -- If any rename fails, all are rolled back - -**When to use `renameSequences=true`:** -- ✅ When swapping production tables (keeps naming consistent) -- ✅ When table names are part of your naming conventions -- ✅ When you want clean, matching names for monitoring/debugging -- ❌ When sequences are shared or manually managed -- ❌ When you don't care about sequence naming consistency - -**About Partition Renaming:** - -When you rename a partitioned table in PostgreSQL, **child partitions are NOT automatically renamed**: - -```sql --- Before rename: --- Parent table: orders --- Partitions: orders_2024_01_01, orders_2024_01_02, etc. - -ALTER TABLE orders RENAME TO orders_old; - --- After rename: --- Parent table: orders_old --- Partitions: orders_2024_01_01, orders_2024_01_02, etc. (still have old prefix!) -``` - -To keep partition names consistent with the parent table, use `renamePartitions=true`: -- Finds all child partitions using PostgreSQL's inheritance system -- Extracts the suffix from each partition name (e.g., `_2024_01_01`) -- Renames to match new parent: `newtable_2024_01_01` -- All renames (tables + partitions + sequences) happen in one atomic transaction -- If any rename fails, all are rolled back - -**How Partition Renaming Works:** -```text -Old table: orders -Old partitions: orders_2024_01_01, orders_2024_01_02 - -New table: orders_old -New partitions: orders_old_2024_01_01, orders_old_2024_01_02 - -Suffix extraction: _2024_01_01, _2024_01_02 -New naming: newtable + suffix -``` - -**When to use `renamePartitions=true`:** -- ✅ When swapping partitioned tables in production -- ✅ When partition naming follows table name prefix convention -- ✅ When you want consistent naming for all related objects -- ✅ When monitoring/debugging relies on naming patterns -- ❌ When partitions use custom naming unrelated to table name -- ❌ When partitions are manually managed with specific names - -**Renaming Partition Sequences, Constraints, and Indexes:** - -When `renamePartitions=true`, the function will **also** rename sequences, constraints, and indexes on those partition tables if the respective flags are enabled: - -- `renamePartitions=true` + `renameSequences=true` → Renames sequences on both parent table AND partition tables -- `renamePartitions=true` + `renameConstraints=true` → Renames constraints on both parent table AND partition tables -- `renamePartitions=true` + `renameIndexes=true` → Renames indexes on both parent table AND partition tables - -Example: -```go -renames := []db.TableRename{ - {From: "orders", To: "orders_v2"}, -} - -// Rename table, partitions, and all their sequences/constraints/indexes -count, err := dbc.RenameTables(renames, true, true, true, true, false) -// ↑ ↑ ↑ ↑ -// sequences ──┘ │ │ │ -// partitions ────────┘ │ │ -// constraints ──────────────┘ │ -// indexes ────────────────────┘ - -// Result: -// Parent table: -// - orders_v2 -// - orders_v2_id_seq -// - orders_v2_pkey -// - orders_v2_pkey (index) -// -// Partitions: -// - orders_v2_2024_01 -// - orders_v2_2024_01_pkey -// - orders_v2_2024_01_pkey (index) -// - orders_v2_2024_02 -// - orders_v2_2024_02_pkey -// - orders_v2_2024_02_pkey (index) -``` - -This ensures complete naming consistency across the entire partitioned table hierarchy. - -**About Constraint Renaming:** - -When you rename a table in PostgreSQL, **constraints are NOT automatically renamed**: - -```sql --- Before rename: --- Table: orders --- Constraints: orders_pkey, orders_email_key, orders_customer_id_fkey - -ALTER TABLE orders RENAME TO orders_old; - --- After rename: --- Table: orders_old --- Constraints: orders_pkey, orders_email_key, orders_customer_id_fkey (still have old names!) -``` - -To keep constraint names consistent with table names, use `renameConstraints=true`: -- Finds all constraints for the table (primary keys, foreign keys, unique, check, exclusion) -- Extracts the suffix from each constraint name (e.g., `_pkey`, `_email_key`) -- Renames to match new table: `newtable_pkey`, `newtable_email_key` -- All renames (tables + partitions + sequences + constraints) happen in one atomic transaction -- If any rename fails, all are rolled back - -**How Constraint Renaming Works:** -```text -Old table: orders -Old constraints: orders_pkey, orders_email_key, orders_customer_id_fkey - -New table: orders_old -New constraints: orders_old_pkey, orders_old_email_key, orders_old_customer_id_fkey - -Suffix extraction: _pkey, _email_key, _customer_id_fkey -New naming: newtable + suffix -``` - -**Constraint Types Renamed:** -- Primary keys (`p`) - e.g., `tablename_pkey` -- Foreign keys (`f`) - e.g., `tablename_column_fkey` -- Unique constraints (`u`) - e.g., `tablename_column_key` -- Check constraints (`c`) - e.g., `tablename_column_check` -- Exclusion constraints (`x`) - e.g., `tablename_excl` - -**Important Note about Indexes:** -Renaming a constraint does NOT rename the backing index. Indexes are separate objects in PostgreSQL and must be renamed separately. Use `renameIndexes=true` in `RenameTables` to rename indexes alongside constraints. - -**When to use `renameConstraints=true`:** -- ✅ When swapping tables in production (keeps naming consistent) -- ✅ When constraint names follow table name prefix convention -- ✅ When you want clean, matching names for schema documentation -- ✅ When monitoring/debugging relies on naming patterns -- ❌ When constraints use custom naming unrelated to table name -- ❌ When constraints are manually managed with specific names - -**About Index Renaming:** - -When you rename a table in PostgreSQL, **indexes are NOT automatically renamed**: - -```sql --- Before rename: --- Table: orders --- Indexes: orders_pkey, orders_email_key, orders_customer_id_idx - -ALTER TABLE orders RENAME TO orders_old; - --- After rename: --- Table: orders_old --- Indexes: orders_pkey, orders_email_key, orders_customer_id_idx (still have old names!) -``` - -To keep index names consistent with table names, use `renameIndexes=true`: -- Finds all indexes for the table (including those backing constraints) -- Extracts the suffix from each index name (e.g., `_pkey`, `_email_key`, `_customer_id_idx`) -- Renames to match new table: `newtable_pkey`, `newtable_email_key`, `newtable_customer_id_idx` -- All renames (tables + partitions + sequences + constraints + indexes) happen in one atomic transaction -- If any rename fails, all are rolled back - -**How Index Renaming Works:** -```text -Old table: orders -Old indexes: orders_pkey, orders_email_key, orders_customer_id_idx - -New table: orders_old -New indexes: orders_old_pkey, orders_old_email_key, orders_old_customer_id_idx - -Suffix extraction: _pkey, _email_key, _customer_id_idx -New naming: newtable + suffix -``` - -**Index Types Renamed:** -- Primary key indexes - e.g., `tablename_pkey` -- Unique indexes - e.g., `tablename_column_key` -- Regular indexes (B-tree, GIN, GiST, etc.) - e.g., `tablename_column_idx` -- Partial indexes - Any index following the naming pattern - -**Important: Indexes vs Constraints** - -Indexes and constraints are separate objects in PostgreSQL: -- Renaming a constraint does NOT rename the backing index -- Renaming an index does NOT rename the constraint -- When you create a primary key, PostgreSQL creates both a constraint AND an index with the same name -- **Recommendation:** Use both `renameConstraints=true` and `renameIndexes=true` together to keep names consistent - -**Performance Note:** -Index renaming is extremely fast - it only updates metadata in PostgreSQL system catalogs, without touching the actual index data structure. However, it does require a brief `ACCESS EXCLUSIVE` lock on the index. - -**When to use `renameIndexes=true`:** -- ✅ When swapping tables in production (keeps naming consistent) -- ✅ When index names follow table name prefix convention -- ✅ When you want clean, matching names for performance analysis -- ✅ When monitoring/debugging relies on naming patterns -- ✅ When renaming constraints (to keep constraint and index names aligned) -- ❌ When indexes use custom naming unrelated to table name -- ❌ When indexes are manually managed with specific names - -**Rename Order Handling:** - -When swapping tables (e.g., `A -> B, C -> A`), the order of operations matters to avoid naming conflicts: - -```go -// Order matters - rename table_base first to free up the name -renames := []db.TableRename{ - {From: "table_base", To: "table_old"}, // Free up "table_base" namespace - {From: "table_new", To: "table_base"}, // Now safe to use "table_base" -} -``` - -Without proper ordering, renames could fail: -```sql --- Wrong order (if table_new renamed first): -ALTER TABLE table_new RENAME TO table_base; -- ERROR! table_base already exists - --- Correct order (as specified in array): -ALTER TABLE table_base RENAME TO table_old; -- Frees up "table_base" -ALTER TABLE table_new RENAME TO table_base; -- Now safe -``` - -**How it works:** -- Tables are renamed in the order specified in the array -- Each rename happens within a single transaction -- Caller is responsible for specifying correct order to avoid conflicts -- All operations are deterministic - renames execute in array order - -**Example - Table Swap**: -```go -// Swap old table with new partitioned table atomically -// Order matters: rename orders first to free up the name -renames := []db.TableRename{ - {From: "orders", To: "orders_old"}, // Save current table - {From: "orders_partitioned", To: "orders"}, // New table becomes production -} - -// Rename sequences, partitions, constraints, and indexes too -count, err := dbc.RenameTables(renames, true, true, true, true, false) -if err != nil { - // If any rename fails, all are rolled back - log.Fatal(err) -} -``` - -**Example - Three-Way Swap**: -```go -// Rotate three tables: production -> backup, new -> production, backup -> archive -// Order matters - must free up names in the right order: -renames := []db.TableRename{ - {From: "orders_backup", To: "orders_archive"}, // Free up "orders_backup" - {From: "orders", To: "orders_backup"}, // Free up "orders" - {From: "orders_new", To: "orders"}, // New becomes production -} - -// Rename sequences, partitions, constraints, and indexes too -count, err := dbc.RenameTables(renames, true, true, true, true, false) -// All renames happen atomically (tables + partitions + sequences + constraints + indexes) -``` - ---- - -### SyncIdentityColumn - -Synchronizes the IDENTITY sequence for a column to match the current maximum value in the table. - -```go -err := dbc.SyncIdentityColumn("table_name", "id") -if err != nil { - log.WithError(err).Error("failed to sync identity column") -} -``` - -**How It Works**: -1. Queries the current maximum value of the column: `SELECT MAX(column) FROM table` -2. Calculates the next value (max + 1, or 1 if table is empty/all NULL) -3. Executes `ALTER TABLE table_name ALTER COLUMN column_name RESTART WITH next_value` -4. Logs the operation with the new sequence value - -**Returns**: Error if the operation fails - -**Use Cases**: -- After migrating data to a partitioned table with IDENTITY columns -- After bulk inserting data with explicit ID values -- When the IDENTITY sequence is out of sync with actual data -- After using `MigrateTableData` to copy data between tables - -**Example Workflow**: -```go -// Migrate data from old table to new partitioned table -rows, err := dbc.MigrateTableData("old_table", "new_partitioned_table", false) -if err != nil { - log.Fatal(err) -} - -// Sync the IDENTITY sequence so new inserts start at the correct value -err = dbc.SyncIdentityColumn("new_partitioned_table", "id") -if err != nil { - log.Fatal(err) -} - -log.Info("Migration complete - sequence synchronized") -``` - -**Important Notes**: -- The column must be an IDENTITY column (created with `GENERATED BY DEFAULT AS IDENTITY`) -- This does NOT work with traditional PostgreSQL sequences created separately -- For traditional sequences, use: `SELECT setval('sequence_name', (SELECT MAX(id) FROM table))` -- Safe to run multiple times - idempotent operation - ---- - -### GetPartitionStrategy - -Checks if a table is partitioned and returns its partition strategy. - -```go -strategy, err := dbc.GetPartitionStrategy("table_name") -if err != nil { - log.WithError(err).Error("failed to check partition strategy") -} - -if strategy == "" { - log.Info("table is not partitioned") -} else if strategy == db.PartitionStrategyRange { - log.Info("table uses RANGE partitioning") -} -``` - -**Returns**: -- Empty string `""` if table is not partitioned -- `PartitionStrategyRange`, `PartitionStrategyList`, `PartitionStrategyHash`, or `"UNKNOWN"` if partitioned - -**Constants**: -```go -db.PartitionStrategyRange // "RANGE" -db.PartitionStrategyList // "LIST" -db.PartitionStrategyHash // "HASH" -``` - -**Use Cases**: -- Before migrations, check if target table is partitioned -- Determine which partition management operations are applicable -- Validate table structure before data operations - -**Example**: -```go -strategy, err := dbc.GetPartitionStrategy("orders") -if err != nil { - log.Fatal(err) -} - -switch strategy { -case db.PartitionStrategyRange: - log.Info("table uses RANGE partitioning") -case db.PartitionStrategyList: - log.Info("table uses LIST partitioning") -case db.PartitionStrategyHash: - log.Info("table uses HASH partitioning") -case "": - log.Info("table is not partitioned") -} -``` - ---- - -### VerifyPartitionCoverage - -Verifies that all necessary partitions exist for a date range. - -```go -startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) -endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) - -err := dbc.VerifyPartitionCoverage("orders", startDate, endDate) -if err != nil { - // Prints: missing partitions for dates: [2024-01-15 2024-01-16] - log.WithError(err).Error("partition coverage check failed") -} -``` - -**How It Works**: -1. Queries all existing partitions for the table -2. Checks that a partition exists for each day in the range [startDate, endDate) -3. Returns error listing all missing partition dates -4. Logs successful verification with partition count - -**Assumptions**: -- Daily partitions with naming convention: `tablename_YYYY_MM_DD` -- Partitions are created for each calendar day -- Date range uses same convention as other functions (startDate inclusive, endDate exclusive) - -**Returns**: Error if any partitions are missing, nil if all exist - -**Use Cases**: -- Before migrating data to partitioned tables -- Verifying partition creation scripts completed successfully -- Pre-flight checks before bulk data operations -- Automated partition management validation - -**Example - Create missing partitions**: -```go -import "github.com/openshift/sippy/pkg/db/partitions" - -// Check if partitions exist -err := dbc.VerifyPartitionCoverage("orders", startDate, endDate) -if err != nil { - log.WithError(err).Warn("missing partitions - creating them") - - // Create missing partitions using partitions package - count, err := partitions.CreateMissingPartitions(dbc, "orders", startDate, endDate, false) - if err != nil { - log.Fatal(err) - } - log.WithField("created", count).Info("created missing partitions") -} - -// Now verify again -if err := dbc.VerifyPartitionCoverage("orders", startDate, endDate); err != nil { - log.Fatal("still missing partitions after creation") -} -``` - ---- - -### GetSequenceMetadata - -Returns detailed metadata about how sequences are linked to columns in a table. - -```go -metadata, err := dbc.GetSequenceMetadata("orders") -if err != nil { - log.WithError(err).Error("failed to get metadata") -} - -for _, m := range metadata { - linkageType := "SERIAL" - if m.IsIdentityColumn { - linkageType = "IDENTITY" - } - - log.WithFields(log.Fields{ - "column": m.ColumnName, - "sequence": m.SequenceName, - "linkage_type": linkageType, - "dep_type": m.DependencyType, - "owner": m.SequenceOwner, - }).Info("sequence linkage") -} -``` - -**Returns**: List of `SequenceMetadata` structs containing: -- `SequenceName`: Name of the sequence -- `TableName`: Name of the table -- `ColumnName`: Name of the column -- `DependencyType`: `'a'` (SERIAL) or `'i'` (IDENTITY) -- `IsIdentityColumn`: `true` if column uses GENERATED AS IDENTITY -- `SequenceOwner`: Owner in format `table.column` - -**Use Cases**: -- Understanding the internal linkage mechanism (OID vs name-based) -- Debugging why a sequence rename might cause issues -- Determining if columns use SERIAL or IDENTITY -- Validating sequence ownership before renames -- Educational/documentation purposes - -**Example - Compare SERIAL vs IDENTITY Linkage**: -```go -metadata, _ := dbc.GetSequenceMetadata("orders") -for _, m := range metadata { - if m.IsIdentityColumn { - fmt.Printf("%s: IDENTITY (OID-based, safe to rename)\n", m.ColumnName) - } else { - fmt.Printf("%s: SERIAL (default uses name, usually safe)\n", m.ColumnName) - } -} -``` - ---- - -### GetTableSequences - -Returns all sequences owned by columns in a specific table (SERIAL, BIGSERIAL, IDENTITY). - -```go -sequences, err := dbc.GetTableSequences("orders") -if err != nil { - log.WithError(err).Error("failed to get sequences") -} - -for _, seq := range sequences { - log.WithFields(log.Fields{ - "sequence": seq.SequenceName, - "table": seq.TableName, - "column": seq.ColumnName, - }).Info("found sequence") -} -``` - -**Returns**: List of `SequenceInfo` structs containing: -- `SequenceName`: Name of the sequence -- `TableName`: Name of the table owning the sequence -- `ColumnName`: Name of the column using the sequence - -**Sequence Types Captured:** -- **SERIAL/BIGSERIAL**: Creates a sequence like `tablename_columnname_seq` -- **IDENTITY**: Creates an internal sequence like `tablename_columnname_seq` (GENERATED BY DEFAULT AS IDENTITY) - -**Use Cases**: -- Checking which sequences will be renamed -- Auditing sequence ownership for a specific table -- Debugging sequence-related issues -- Understanding table dependencies before renames - -**Example - Check Before Rename**: -```go -// Check what sequences exist before renaming -sequences, _ := dbc.GetTableSequences("orders_old") -if len(sequences) > 0 { - log.WithField("count", len(sequences)).Info("found sequences - will rename with table") - - // Use renameSequences=true to keep them consistent - renames := []db.TableRename{{From: "orders_old", To: "orders"}} - dbc.RenameTables(renames, true, false, false, false, false) -} else { - // No sequences to worry about - renames := []db.TableRename{{From: "orders_old", To: "orders"}} - dbc.RenameTables(renames, false, false, false, false, false) -} -``` - ---- - -### ListAllTableSequences - -Returns all sequences owned by table columns across the entire database (public schema). - -```go -allSequences, err := dbc.ListAllTableSequences() -if err != nil { - log.WithError(err).Error("failed to list sequences") -} - -for tableName, sequences := range allSequences { - log.WithFields(log.Fields{ - "table": tableName, - "count": len(sequences), - }).Info("table sequences") - - for _, seq := range sequences { - log.WithFields(log.Fields{ - "sequence": seq.SequenceName, - "column": seq.ColumnName, - }).Debug("sequence detail") - } -} -``` - -**Returns**: Map where: -- **Key**: Table name -- **Value**: List of `SequenceInfo` structs for that table - -**Use Cases**: -- Database-wide sequence auditing -- Understanding auto-increment usage patterns -- Finding all sequences that need syncing after bulk operations -- Generating database documentation -- Preparing for bulk table renames -- Identifying orphaned sequences - -**Example - Audit All Sequences**: -```go -allSequences, err := dbc.ListAllTableSequences() -if err != nil { - log.Fatal(err) -} - -log.WithField("tables", len(allSequences)).Info("tables with sequences") - -// Show summary -totalSequences := 0 -for tableName, sequences := range allSequences { - totalSequences += len(sequences) - fmt.Printf("Table: %s has %d sequence(s)\n", tableName, len(sequences)) - for _, seq := range sequences { - fmt.Printf(" - %s.%s → %s\n", seq.TableName, seq.ColumnName, seq.SequenceName) - } -} - -log.WithField("total_sequences", totalSequences).Info("audit complete") -``` - -**Example - Find Tables Without Sequences**: -```go -// Get all tables -allTables := []string{"orders", "items", "users", "logs"} - -// Get tables with sequences -tablesWithSequences, _ := dbc.ListAllTableSequences() - -// Find tables without sequences -for _, table := range allTables { - if _, hasSequence := tablesWithSequences[table]; !hasSequence { - log.WithField("table", table).Info("table has no sequences - using explicit IDs") - } -} -``` - -**Example - Sync All Identity Sequences**: -```go -// Get all tables with sequences -allSequences, err := dbc.ListAllTableSequences() -if err != nil { - log.Fatal(err) -} - -// Sync identity column for each table with sequences -for tableName, sequences := range allSequences { - for _, seq := range sequences { - // Only sync if column looks like an ID column - if seq.ColumnName == "id" { - err := dbc.SyncIdentityColumn(tableName, seq.ColumnName) - if err != nil { - log.WithError(err).WithField("table", tableName).Error("sync failed") - } else { - log.WithField("table", tableName).Info("synced identity") - } - } - } -} -``` - ---- - -### GetTablePartitions - -Returns all child partitions of a partitioned table. - -```go -partitions, err := dbc.GetTablePartitions("orders") -if err != nil { - log.WithError(err).Error("failed to get partitions") -} - -for _, part := range partitions { - log.WithFields(log.Fields{ - "partition": part.PartitionName, - "parent": part.ParentTable, - }).Info("found partition") -} -``` - -**Returns**: List of `PartitionTableInfo` structs containing: -- `PartitionName`: Name of the partition -- `ParentTable`: Name of the parent partitioned table - -**Use Cases**: -- Checking which partitions will be renamed -- Auditing partition structure -- Understanding table dependencies before renames -- Verifying partition naming conventions - -**Example - Check Partitions Before Rename**: -```go -// Check what partitions exist before renaming -partitions, _ := dbc.GetTablePartitions("orders_old") -log.WithField("count", len(partitions)).Info("found partitions") - -for _, part := range partitions { - // Extract suffix to see naming pattern - suffix := strings.TrimPrefix(part.PartitionName, "orders_old") - log.WithFields(log.Fields{ - "partition": part.PartitionName, - "suffix": suffix, - }).Info("partition details") -} - -// If partitions follow naming convention, rename them too -if len(partitions) > 0 { - renames := []db.TableRename{{From: "orders_old", To: "orders"}} - dbc.RenameTables(renames, true, true, true, true, false) // renamePartitions=true, renameConstraints=true, renameIndexes=true -} -``` - ---- - -### GetTableConstraints - -Returns all constraints for a table (primary keys, foreign keys, unique, check, exclusion). - -```go -constraints, err := dbc.GetTableConstraints("orders") -if err != nil { - log.WithError(err).Error("failed to get constraints") -} - -for _, cons := range constraints { - log.WithFields(log.Fields{ - "constraint": cons.ConstraintName, - "type": cons.ConstraintType, - "definition": cons.Definition, - }).Info("found constraint") -} -``` - -**Returns**: List of `ConstraintInfo` structs containing: -- `ConstraintName`: Name of the constraint (e.g., "orders_pkey") -- `TableName`: Name of the table -- `ConstraintType`: Single character type code: - - `'p'` - Primary key - - `'f'` - Foreign key - - `'u'` - Unique - - `'c'` - Check - - `'x'` - Exclusion -- `Definition`: SQL definition of the constraint (e.g., "PRIMARY KEY (id)") - -**Use Cases**: -- Checking which constraints will be renamed -- Auditing constraint naming conventions -- Understanding table dependencies before renames -- Verifying constraint structure - -**Example - Check Constraints Before Rename**: -```go -// Check what constraints exist before renaming -constraints, _ := dbc.GetTableConstraints("orders_old") -log.WithField("count", len(constraints)).Info("found constraints") - -for _, cons := range constraints { - // Extract suffix to see naming pattern - suffix := strings.TrimPrefix(cons.ConstraintName, "orders_old") - typeNames := map[string]string{ - "p": "PRIMARY KEY", - "f": "FOREIGN KEY", - "u": "UNIQUE", - "c": "CHECK", - "x": "EXCLUSION", - } - - log.WithFields(log.Fields{ - "constraint": cons.ConstraintName, - "suffix": suffix, - "type": typeNames[cons.ConstraintType], - }).Info("constraint details") -} - -// If constraints follow naming convention, rename them too -if len(constraints) > 0 { - renames := []db.TableRename{{From: "orders_old", To: "orders"}} - dbc.RenameTables(renames, true, true, true, true, false) // renameConstraints=true, renameIndexes=true -} -``` - ---- - -### GetTableIndexes - -Returns all indexes for a table (including those backing constraints). - -```go -indexes, err := dbc.GetTableIndexes("orders") -if err != nil { - log.WithError(err).Error("failed to get indexes") -} - -for _, idx := range indexes { - log.WithFields(log.Fields{ - "index": idx.IndexName, - "is_primary": idx.IsPrimary, - "is_unique": idx.IsUnique, - }).Info("found index") -} -``` - -**Returns**: List of `IndexInfo` structs containing: -- `IndexName`: Name of the index (e.g., "orders_pkey") -- `TableName`: Name of the table -- `Definition`: Full CREATE INDEX statement -- `IsPrimary`: true if this is a primary key index -- `IsUnique`: true if this is a unique index - -**Use Cases**: -- Checking which indexes will be renamed -- Auditing index naming conventions -- Understanding table performance characteristics -- Verifying index structure before operations - -**Important Note**: -Indexes and constraints are separate objects. An index backing a primary key or unique constraint has the same name as the constraint, but they are different objects. Renaming one does NOT rename the other. - -**Example - Check Indexes Before Rename**: -```go -// Check what indexes exist before renaming -indexes, _ := dbc.GetTableIndexes("orders_old") -log.WithField("count", len(indexes)).Info("found indexes") - -for _, idx := range indexes { - // Extract suffix to see naming pattern - suffix := strings.TrimPrefix(idx.IndexName, "orders_old") - - indexType := "REGULAR" - if idx.IsPrimary { - indexType = "PRIMARY KEY" - } else if idx.IsUnique { - indexType = "UNIQUE" - } - - log.WithFields(log.Fields{ - "index": idx.IndexName, - "suffix": suffix, - "type": indexType, - }).Info("index details") -} - -// If indexes follow naming convention, rename them too -if len(indexes) > 0 { - renames := []db.TableRename{{From: "orders_old", To: "orders"}} - dbc.RenameTables(renames, true, true, true, true, false) // renameIndexes=true -} -``` - ---- - -## Helper Types - -### ColumnInfo - -Represents metadata about a database column. - -```go -type ColumnInfo struct { - ColumnName string - DataType string - IsNullable string - ColumnDefault sql.NullString - OrdinalPos int -} -``` - ---- - -### SequenceInfo - -Represents basic information about a sequence associated with a table column. - -```go -type SequenceInfo struct { - SequenceName string - TableName string - ColumnName string -} -``` - -**Usage**: -- Returned by `GetTableSequences()` to show sequences owned by table columns -- Used internally by `RenameTables()` when `renameSequences=true` -- Includes sequences from SERIAL, BIGSERIAL, and IDENTITY columns - ---- - -### SequenceMetadata - -Represents detailed metadata about how a sequence is linked to a column. - -```go -type SequenceMetadata struct { - SequenceName string - TableName string - ColumnName string - DependencyType string // 'a' = auto (SERIAL), 'i' = internal (IDENTITY) - IsIdentityColumn bool // true if column uses GENERATED AS IDENTITY - SequenceOwner string // Table.Column that owns this sequence -} -``` - -**Usage**: -- Returned by `GetSequenceMetadata()` to show detailed linkage information -- Helps understand the difference between SERIAL and IDENTITY columns -- Shows PostgreSQL's internal dependency mechanism (OID-based vs name-based) -- Useful for debugging and educational purposes - ---- - -### PartitionTableInfo - -Represents information about a table partition. - -```go -type PartitionTableInfo struct { - PartitionName string - ParentTable string -} -``` - -**Usage**: -- Returned by `GetTablePartitions()` to show child partitions of a table -- Used internally by `RenameTables()` when `renamePartitions=true` -- Works with any partition type (RANGE, LIST, HASH) - ---- - -### PartitionStrategy - -Defines the partitioning strategy type for PostgreSQL partitioned tables. - -```go -type PartitionStrategy string - -const ( - PartitionStrategyRange PartitionStrategy = "RANGE" - PartitionStrategyList PartitionStrategy = "LIST" - PartitionStrategyHash PartitionStrategy = "HASH" -) -``` - -**Usage**: -- Returned by `GetPartitionStrategy()` to indicate table's partitioning type -- Used by the `partitions` package in `PartitionConfig.Strategy` -- Can be compared directly with constants or used in switch statements - -**Example**: -```go -strategy, err := dbc.GetPartitionStrategy("orders") -if err != nil { - return err -} - -switch strategy { -case PartitionStrategyRange: - // Handle RANGE partitioned table -case PartitionStrategyList: - // Handle LIST partitioned table -case PartitionStrategyHash: - // Handle HASH partitioned table -case "": - // Table is not partitioned -} -``` - ---- - -### ColumnVerificationOptions - -Controls which aspects of column definitions to verify when comparing tables. - -```go -type ColumnVerificationOptions struct { - CheckNullable bool // Verify that columns have matching nullable constraints - CheckDefaults bool // Verify that columns have matching default values - CheckOrder bool // Verify that columns are in the same ordinal position -} -``` - -**Predefined Options:** - -```go -// DefaultColumnVerificationOptions - Full verification (all checks enabled) -opts := DefaultColumnVerificationOptions() -// Returns: ColumnVerificationOptions{CheckNullable: true, CheckDefaults: true, CheckOrder: true} - -// DataMigrationColumnVerificationOptions - Minimal verification for migrations -opts := DataMigrationColumnVerificationOptions() -// Returns: ColumnVerificationOptions{CheckNullable: false, CheckDefaults: false, CheckOrder: true} -``` - -**Usage**: -- Used by `VerifyTablesHaveSameColumns()` to control verification behavior -- Column names and data types are **always** verified regardless of options -- Optional checks allow flexibility for different use cases - -**Example - Custom Options**: -```go -// Custom verification: check types and nullability, skip defaults and order -opts := ColumnVerificationOptions{ - CheckNullable: true, - CheckDefaults: false, - CheckOrder: false, -} -err := dbc.VerifyTablesHaveSameColumns("table1", "table2", opts) -``` - -**When to Use Each Option:** - -| Scenario | Recommended Options | -|----------|-------------------| -| Verifying partition matches parent | `DefaultColumnVerificationOptions()` | -| Pre-migration compatibility check | `DataMigrationColumnVerificationOptions()` | -| Validating table replicas | `DefaultColumnVerificationOptions()` | -| Testing table clones | `DefaultColumnVerificationOptions()` | - ---- - -## Data Type Normalization - -The utilities normalize PostgreSQL data type names for accurate comparison: - -| PostgreSQL Type | Normalized | -|----------------|------------| -| `character varying` | `varchar` | -| `integer`, `int4` | `int` | -| `int8`, `bigserial` | `bigint` | -| `serial` | `int` | -| `timestamp without time zone` | `timestamp` | -| `timestamp with time zone` | `timestamptz` | -| `double precision` | `float8` | -| `boolean` | `bool` | - -This ensures that functionally equivalent types are treated as identical during comparison. - ---- - -## Omitting Columns During Migration - -Both `MigrateTableData` and `MigrateTableDataRange` support omitting specific columns during migration. This is useful when: - -- **Auto-increment columns**: The target table has an `id` column with `GENERATED BY DEFAULT AS IDENTITY` and you want new IDs to be generated instead of copying from source -- **Computed columns**: The target table has columns that should be calculated rather than copied -- **Different schemas**: Some columns exist in the source but shouldn't be migrated to the target - -### Example: Omitting ID Column - -```go -// Migrate data but let target table generate new IDs -rows, err := dbc.MigrateTableData( - "old_table", - "new_table", - []string{"id"}, // Omit the id column - false, -) -if err != nil { - log.WithError(err).Error("migration failed") - return -} - -log.WithField("rows", rows).Info("migrated with new IDs generated") -``` - -### Example: Omitting Multiple Columns - -```go -// Omit multiple columns during range migration -rows, err := dbc.MigrateTableDataRange( - "source", - "target", - "created_at", - startDate, - endDate, - []string{"id", "updated_at", "version"}, // Omit these columns - false, -) -``` - -### How It Works - -When you specify `omitColumns`: -1. The function retrieves all columns from the source table -2. Filters out any columns in the `omitColumns` list -3. Generates `INSERT INTO target (col1, col2, ...) SELECT col1, col2, ... FROM source` -4. Only the non-omitted columns are included in both the INSERT and SELECT clauses - -**Important Notes:** -- If you omit a `NOT NULL` column without a default, the migration will fail -- Omitted columns in the target table must either be nullable or have default values -- Pass `nil` or `[]string{}` to copy all columns (default behavior) - ---- - -## Usage Examples - -### Basic Migration - -```go -// Step 1: Verify schemas match -err := dbc.VerifyTablesHaveSameColumns("source_table", "target_table") -if err != nil { - log.Fatal(err) -} - -// Step 2: Dry run -_, err = dbc.MigrateTableData("source_table", "target_table", nil, true) -if err != nil { - log.Fatal(err) -} - -// Step 3: Actual migration -rows, err := dbc.MigrateTableData("source_table", "target_table", nil, false) -log.WithField("rows", rows).Info("migration completed") -``` - ---- - -### Partition to Archive Migration - -```go -// Migrate detached partition to archive table -partition := "test_analysis_by_job_by_dates_2024_01_15" -archive := "test_analysis_archive" - -rows, err := dbc.MigrateTableData(partition, archive, nil, false) -if err != nil { - log.WithError(err).Error("migration failed") - return -} - -log.WithFields(log.Fields{ - "partition": partition, - "rows": rows, -}).Info("partition migrated to archive - safe to drop") -``` - ---- - -### Batch Migration - -```go -partitions := []string{ - "table_2024_01_15", - "table_2024_01_16", - "table_2024_01_17", -} - -var totalRows int64 -for _, partition := range partitions { - rows, err := dbc.MigrateTableData(partition, "archive_table", nil, false) - if err != nil { - log.WithError(err).WithField("partition", partition).Error("failed") - continue - } - totalRows += rows -} - -log.WithField("total_rows", totalRows).Info("batch migration completed") -``` - ---- - -### Migration with Backup - -```go -// Create backup before migration -_, err := dbc.MigrateTableData("target_table", "backup_table", nil, false) -if err != nil { - log.Fatal("backup failed") -} - -// Perform migration -rows, err := dbc.MigrateTableData("source_table", "target_table", nil, false) -if err != nil { - log.Error("migration failed - restore from backup if needed") - return -} - -log.Info("migration successful - backup can be dropped") -``` - ---- - -### Incremental Migration by Date Range - -```go -// Migrate large table incrementally by month to reduce lock contention -for month := 1; month <= 12; month++ { - startDate := time.Date(2024, time.Month(month), 1, 0, 0, 0, 0, time.UTC) - endDate := startDate.AddDate(0, 1, 0) // First day of next month - - log.WithFields(log.Fields{ - "month": time.Month(month).String(), - "start": startDate.Format("2006-01-02"), - "end": endDate.Format("2006-01-02"), - }).Info("migrating month") - - rows, err := dbc.MigrateTableDataRange("large_table", "large_table_new", "created_at", startDate, endDate, nil, false) - if err != nil { - log.WithError(err).WithField("month", month).Error("migration failed") - continue - } - - log.WithFields(log.Fields{ - "month": month, - "rows": rows, - }).Info("month migrated successfully") -} -``` - ---- - -### Swap Partitioned Table with Non-Partitioned Table - -```go -// Complete workflow: Migrate to partitioned table and swap atomically - -oldTable := "orders" -newPartitionedTable := "orders_partitioned" - -// Step 1: Verify data was migrated successfully -oldCount, _ := dbc.GetTableRowCount(oldTable) -newCount, _ := dbc.GetTableRowCount(newPartitionedTable) - -if oldCount != newCount { - log.Fatal("row count mismatch - cannot swap") -} - -// Step 2: Perform atomic table swap -// Order matters: rename orders first to free up the name -renames := []db.TableRename{ - {From: "orders", To: "orders_old"}, // Save current table - {From: "orders_partitioned", To: "orders"}, // New table becomes production -} - -// Dry run first -_, err := dbc.RenameTables(renames, true, true, true, true, true) -if err != nil { - log.Fatal(err) -} - -// Execute swap (rename sequences, partitions, constraints, and indexes too) -count, err := dbc.RenameTables(renames, true, true, true, true, false) -if err != nil { - log.Fatal(err) -} - -log.WithFields(log.Fields{ - "renamed": count, - "old_table": "orders_old", - "new_table": "orders", - "partitioned": true, -}).Info("tables swapped - partitioned table is now active") - -// If something goes wrong, you can easily rollback: -// rollback := []db.TableRename{ -// {From: "orders", To: "orders_partitioned"}, -// {From: "orders_old", To: "orders"}, -// } -// dbc.RenameTables(rollback, true, true, true, true, false) -``` - ---- - -### Three-Way Table Rotation - -```go -// Rotate tables: archive old backup, current becomes backup, new becomes current -// Order matters - must free up names in the right order: -renames := []db.TableRename{ - {From: "orders_backup", To: "orders_archive"}, // Free up "orders_backup" - {From: "orders", To: "orders_backup"}, // Free up "orders" - {From: "orders_new", To: "orders"}, // New becomes production -} - -// All three renames happen atomically in one transaction (rename sequences and partitions too) -count, err := dbc.RenameTables(renames, true, true, false, false, false) -if err != nil { - log.WithError(err).Error("rotation failed - no changes made") - return -} - -log.WithField("renamed", count).Info("three-way rotation completed") - -// Result: -// - orders (was orders_new) - now in production -// - orders_backup (was orders) - current backup -// - orders_archive (was orders_backup) - archived -``` - ---- - -### Migrate with Auto-Generated IDs - -```go -// When migrating to a table with auto-increment ID, omit the id column -// so the target table generates new sequential IDs - -sourceTable := "prow_job_run_tests" -targetTable := "prow_job_run_tests_partitioned" -startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) -endDate := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) - -// Dry run first to verify -_, err := dbc.MigrateTableDataRange( - sourceTable, - targetTable, - "created_at", - startDate, - endDate, - []string{"id"}, // Omit id column - target will auto-generate - true, -) -if err != nil { - log.Fatal(err) -} - -// Actual migration -rows, err := dbc.MigrateTableDataRange( - sourceTable, - targetTable, - "created_at", - startDate, - endDate, - []string{"id"}, // Omit id column - false, -) -if err != nil { - log.Fatal(err) -} - -log.WithFields(log.Fields{ - "rows": rows, - "start_date": startDate.Format("2006-01-02"), - "end_date": endDate.Format("2006-01-02"), -}).Info("data migrated with new IDs generated") - -// Note: No need to sync identity column since we're omitting id -// The target table's auto-increment will continue from its current value -``` - ---- - -### Migrate Specific Date Range to Archive - -```go -// Move Q1 2024 data to archive table -startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) -endDate := time.Date(2024, 4, 1, 0, 0, 0, 0, time.UTC) - -// Dry run first -_, err := dbc.MigrateTableDataRange("orders", "orders_archive", "order_date", startDate, endDate, nil, true) -if err != nil { - log.Fatal(err) -} - -// Actual migration -rows, err := dbc.MigrateTableDataRange("orders", "orders_archive", "order_date", startDate, endDate, nil, false) -log.WithFields(log.Fields{ - "rows": rows, - "start_date": startDate.Format("2006-01-02"), - "end_date": endDate.Format("2006-01-02"), -}).Info("Q1 2024 data archived") -``` - ---- - -### Complete Partitioned Table Migration Workflow - -```go -// End-to-end example: Migrate from non-partitioned to partitioned table - -// Step 1: Create partitioned table (using partitions package) -// import "github.com/openshift/sippy/pkg/db/partitions" -// partitionConfig := partitions.NewRangePartitionConfig("created_at") -// _, err := partitions.CreatePartitionedTableFromExisting(dbc, "orders", "orders_partitioned", partitionConfig, false) - -// Step 2: Create necessary partitions -startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) -endDate := time.Now() -// _, err := partitions.CreateMissingPartitions(dbc, "orders_partitioned", startDate, endDate, false) - -// Step 3: Migrate data (omit id to use auto-increment) -rows, err := dbc.MigrateTableDataRange( - "orders", - "orders_partitioned", - "created_at", - startDate, - endDate, - []string{"id"}, // Omit id - let target generate new IDs - false, -) -if err != nil { - log.Fatal(err) -} -log.WithField("rows", rows).Info("data migrated") - -// Step 4: Verify row counts match -oldCount, _ := dbc.GetTableRowCount("orders") -newCount, _ := dbc.GetTableRowCount("orders_partitioned") -if oldCount != newCount { - log.Fatal("row count mismatch!") -} - -// Step 5: Atomically swap tables (order matters) -renames := []db.TableRename{ - {From: "orders", To: "orders_old"}, - {From: "orders_partitioned", To: "orders"}, -} - -count, err := dbc.RenameTables(renames, true, true, false, false, false) -if err != nil { - log.Fatal(err) -} - -log.WithFields(log.Fields{ - "renamed": count, - "rows": rows, - "partitioned": true, -}).Info("migration completed - partitioned table is now active") - -// Step 6: After verification period, drop old table -// DROP TABLE orders_old; -``` - ---- - -## Best Practices - -### Always Use Dry Run First - -```go -// GOOD: Verify before executing -_, err := dbc.MigrateTableData(source, target, nil, true) -if err != nil { - return err -} -rows, err := dbc.MigrateTableData(source, target, nil, false) - -// BAD: Direct migration without verification -rows, err := dbc.MigrateTableData(source, target, nil, false) -``` - -### Verify Schemas Explicitly - -```go -// GOOD: Explicit verification with clear error handling -if err := dbc.VerifyTablesHaveSameColumns(source, target); err != nil { - log.WithError(err).Error("schema mismatch - cannot proceed") - return err -} - -// Migration happens in MigrateTableData, but explicit check is clearer -``` - -### Check Row Counts - -```go -// GOOD: Verify counts before and after -sourceBefore, _ := dbc.GetTableRowCount(source) -targetBefore, _ := dbc.GetTableRowCount(target) - -rows, err := dbc.MigrateTableData(source, target, nil, false) - -targetAfter, _ := dbc.GetTableRowCount(target) -expected := targetBefore + sourceBefore -if targetAfter != expected { - log.Error("row count mismatch!") -} -``` - -### Use Transactions for Multiple Operations - -When performing multiple related operations, use database transactions: - -```go -tx := dbc.DB.Begin() - -// Perform operations -// ... - -if err != nil { - tx.Rollback() - return err -} - -tx.Commit() -``` - -### Test Table Renames with Dry Run - -```go -// GOOD: Always dry run before renaming -renames := []db.TableRename{ - {From: "orders_old", To: "orders_backup"}, - {From: "orders_new", To: "orders"}, -} - -_, err := dbc.RenameTables(renames, true, true, true, true, true) -if err != nil { - log.WithError(err).Error("validation failed") - return -} - -count, err := dbc.RenameTables(renames, true, true, true, true, false) - -// BAD: Direct rename without validation -count, err := dbc.RenameTables(renames, true, true, true, true, false) -``` - -### Verify Before Swapping Tables - -```go -// GOOD: Verify data integrity before swapping -oldCount, _ := dbc.GetTableRowCount("orders") -newCount, _ := dbc.GetTableRowCount("orders_partitioned") - -if oldCount != newCount { - log.Error("cannot swap - row counts don't match") - return -} - -// Now safe to swap -dbc.RenameTables([]db.TableRename{ - {From: "orders", To: "orders_old"}, - {From: "orders_partitioned", To: "orders"}, -}, true, true, false, false, false) - -// BAD: Swap without verifying data -dbc.RenameTables(renames, true, true, false, false, false) -``` - -### Keep Rollback Plans Ready - -```go -// GOOD: Define rollback before making changes -renames := []db.TableRename{ - {From: "orders", To: "orders_old"}, - {From: "orders_new", To: "orders"}, -} - -// Define rollback upfront (reverse order) -rollback := []db.TableRename{ - {From: "orders", To: "orders_new"}, - {From: "orders_old", To: "orders"}, -} - -// Execute rename -_, err := dbc.RenameTables(renames, true, true, false, false, false) -if err != nil { - log.Error("rename failed - no rollback needed") - return -} - -// If issues found after rename, easy to rollback -// dbc.RenameTables(rollback, true, true, false, false, false) -``` - ---- - -## Error Handling - -All functions return detailed errors: - -```go -err := dbc.VerifyTablesHaveSameColumns("table1", "table2") -if err != nil { - // Error contains specific differences: - // "column name mismatch: columns in table1 but not in table2: [col1, col2]" - // "column definition mismatches: column foo: type mismatch (table1: int vs table2: bigint)" -} -``` - -Common errors: -- **Schema mismatch**: Tables have different columns or types -- **Table not found**: One or both tables don't exist -- **Permission denied**: Insufficient database privileges -- **Row count mismatch**: Data integrity issue after migration - ---- - -## Testing - -Unit tests cover: -- Data type normalization -- ColumnInfo struct -- Parameter validation - -Run tests: -```bash -go test ./pkg/db -v -``` - -Integration tests require a live database and are in separate test suites. - ---- - -## Logging - -All functions use structured logging with relevant fields: - -```go -log.WithFields(log.Fields{ - "source": sourceTable, - "target": targetTable, - "rows": rowsMigrated, -}).Info("migration completed") -``` - -Log levels: -- **Debug**: Column-level comparisons -- **Info**: Operation start/completion, row counts -- **Warn**: Row count mismatches (non-fatal) -- **Error**: Schema mismatches, migration failures - ---- - -## Integration with Partition Management - -These utilities work seamlessly with the partition management APIs in `pkg/db/partitions`: - -```go -import "github.com/openshift/sippy/pkg/db/partitions" - -// Detach old partitions -detached, _ := partitions.DetachOldPartitions(dbc, "parent_table", 180, false) - -// Migrate detached partitions to archive -for _, partition := range detachedPartitions { - dbc.MigrateTableData(partition.TableName, "archive_table", nil, false) -} - -// Drop old partitions -partitions.DropOldDetachedPartitions(dbc, "parent_table", 180, false) -``` - ---- - -## Performance Considerations - -- **Single INSERT statement**: Migration uses `INSERT INTO ... SELECT` for efficiency -- **No row-by-row operations**: Bulk operation handled by PostgreSQL -- **Network efficiency**: Single round-trip for data transfer -- **Index usage**: PostgreSQL optimizer handles query execution - -For very large tables (millions of rows): -- Consider migrating in batches using WHERE clauses -- Monitor transaction log growth -- Use `ANALYZE` after migration -- Consider `VACUUM` on target table - ---- - -## See Also - -- [Partition Management APIs](./partitions/README.md) - For partition-specific operations -- [Database Schema](../../.claude/db-schema-analysis.md) - For schema documentation diff --git a/pkg/db/partitions/README.md b/pkg/db/partitions/README.md deleted file mode 100644 index 3d0613488..000000000 --- a/pkg/db/partitions/README.md +++ /dev/null @@ -1,1736 +0,0 @@ -# Partition Management APIs - -This package provides GORM-based APIs for managing PostgreSQL table partitions, specifically for `test_analysis_by_job_by_dates`. - -## Overview - -The partition management APIs provide read-only analysis and write operations (with dry-run support) for managing the lifecycle of table partitions based on retention policies. - -**Based on**: [partition-retention-management-guide.md](../../../.claude/partition-retention-management-guide.md) - -## Features - -- ✅ List all partitions with metadata -- ✅ Get partition statistics and summaries -- ✅ Identify partitions for removal based on retention policy -- ✅ Analyze partitions by age groups and time periods -- ✅ Validate retention policies (safety checks) -- ✅ Dry-run support for all destructive operations -- ✅ Comprehensive logging -- ✅ SQL injection protection - -## API Reference - -### Read-Only Operations - -#### ListTablePartitions -Returns all partitions for a given table with metadata. - -```go -partitions, err := partitions.ListTablePartitions(dbc, "test_analysis_by_job_by_dates") -if err != nil { - log.WithError(err).Error("failed to list partitions") -} - -for _, p := range partitions { - fmt.Printf("%s: %s, Age: %d days, Size: %s\n", - p.TableName, p.PartitionDate, p.Age, p.SizePretty) -} -``` - -**Parameters**: -- `tableName` - Name of the partitioned parent table - -**Returns**: `[]PartitionInfo` containing: -- `TableName` - Partition table name -- `SchemaName` - Schema (always "public") -- `PartitionDate` - Date the partition represents -- `Age` - Days since partition date -- `SizeBytes` - Storage in bytes -- `SizePretty` - Human-readable size -- `RowEstimate` - Estimated row count - ---- - -#### GetPartitionStats -Returns aggregate statistics about all partitions. - -```go -stats, err := partitions.GetPartitionStats(dbc) -if err != nil { - log.WithError(err).Error("failed to get stats") -} - -fmt.Printf("Total: %d partitions, %s\n", - stats.TotalPartitions, stats.TotalSizePretty) -fmt.Printf("Range: %s to %s\n", - stats.OldestDate.Format("2006-01-02"), - stats.NewestDate.Format("2006-01-02")) -``` - -**Returns**: `*PartitionStats` containing: -- `TotalPartitions` - Total partition count -- `TotalSizeBytes` / `TotalSizePretty` - Total storage -- `OldestDate` / `NewestDate` - Date range -- `AvgSizeBytes` / `AvgSizePretty` - Average partition size - ---- - -#### GetPartitionsForRemoval -Identifies partitions older than the retention period. - -```go -// Get all partitions (attached + detached) older than 180 days -partitions, err := partitions.GetPartitionsForRemoval(dbc, "test_analysis_by_job_by_dates", 180, false) -if err != nil { - log.WithError(err).Error("failed to get partitions for removal") -} - -fmt.Printf("Found %d partitions older than 180 days\n", len(partitions)) - -// Get only attached partitions older than 180 days -attachedPartitions, err := partitions.GetPartitionsForRemoval(dbc, "test_analysis_by_job_by_dates", 180, true) -``` - -**Parameters**: -- `tableName` - Name of the partitioned parent table -- `retentionDays` - Retention period in days -- `attachedOnly` - If true, only returns attached partitions; if false, returns all partitions - -**Returns**: `[]PartitionInfo` for partitions older than retention period - -**Use When**: -- `attachedOnly = true`: Before detaching partitions (can only detach what's attached) -- `attachedOnly = false`: Before dropping partitions (can drop both attached and detached) - ---- - -#### GetRetentionSummary -Provides a summary of what would be affected by a retention policy. - -```go -// Get summary for all partitions (attached + detached) -summary, err := partitions.GetRetentionSummary(dbc, "test_analysis_by_job_by_dates", 180, false) -if err != nil { - log.WithError(err).Error("failed to get summary") -} - -fmt.Printf("Would delete %d partitions, reclaiming %s\n", - summary.PartitionsToRemove, summary.StoragePretty) - -// Get summary for attached partitions only -attachedSummary, err := partitions.GetRetentionSummary(dbc, "test_analysis_by_job_by_dates", 180, true) -``` - -**Parameters**: -- `tableName` - Name of the partitioned parent table -- `retentionDays` - Retention period in days -- `attachedOnly` - If true, only considers attached partitions; if false, considers all partitions - -**Returns**: `*RetentionSummary` containing: -- `RetentionDays` - Policy retention period -- `CutoffDate` - Date cutoff for removal -- `PartitionsToRemove` - Count of partitions to remove -- `StorageToReclaim` / `StoragePretty` - Storage to be freed -- `OldestPartition` / `NewestPartition` - Range of affected partitions - -**Use When**: -- `attachedOnly = true`: Before detaching partitions or when validating against active data only -- `attachedOnly = false`: Before dropping partitions or when showing complete impact - ---- - -#### GetPartitionsByAgeGroup -Returns partition counts and sizes grouped by age buckets. - -```go -groups, err := partitions.GetPartitionsByAgeGroup(dbc) -if err != nil { - log.WithError(err).Error("failed to get age groups") -} - -for _, group := range groups { - fmt.Printf("%s: %d partitions, %s (%.2f%%)\n", - group["age_bucket"], - group["partition_count"], - group["total_size"], - group["percentage"]) -} -``` - -**Age Buckets**: -- Future (dates in the future) -- 0-30 days -- 30-90 days -- 90-180 days -- 180-365 days -- 365+ days - ---- - -#### GetPartitionsByMonth -Returns partition counts and sizes grouped by month. - -```go -months, err := partitions.GetPartitionsByMonth(dbc) -if err != nil { - log.WithError(err).Error("failed to get monthly data") -} -``` - -**Returns**: Monthly aggregates with partition counts and sizes - ---- - -#### ValidateRetentionPolicy -Validates that a retention policy is safe to apply. - -```go -err := partitions.ValidateRetentionPolicy(dbc, "test_analysis_by_job_by_dates", 180) -if err != nil { - log.WithError(err).Error("retention policy is not safe") -} -``` - -**Parameters**: -- `tableName` - Name of the partitioned parent table -- `retentionDays` - Retention period in days - -**Safety Checks**: -- Minimum 90 days retention -- Maximum 75% of attached partitions deleted -- Maximum 80% of attached storage deleted - -**Important**: Only considers **attached partitions** when validating thresholds. Detached partitions are excluded from calculations to ensure the policy is safe for active data. - -**Returns**: Error if policy would be unsafe - ---- - -### Write Operations (Require Write Access) - -⚠️ **Warning**: All write operations require database write access. Read-only users will get permission errors. - -#### CreatePartitionedTable -Creates a new partitioned table from a GORM model struct with a specified partitioning strategy. - -```go -// Define your model (or use an existing one) -type MyModel struct { - ID uint `gorm:"primaryKey"` - CreatedAt time.Time `gorm:"index"` - Name string - Data string -} - -// RANGE partitioning (most common - for dates, timestamps) -config := partitions.NewRangePartitionConfig("created_at") - -// Dry run - see the SQL that would be executed -sql, err := partitions.CreatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", config, true) -if err != nil { - log.WithError(err).Error("dry run failed") -} -// Prints the CREATE TABLE statement with PARTITION BY RANGE clause - -// Actual creation -sql, err = partitions.CreatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", config, false) -``` - -**Parameters**: -- `model` - GORM model struct (must be a pointer, e.g., `&models.MyModel{}`) -- `tableName` - Name for the partitioned table -- `config` - Partition configuration (strategy, columns, etc.) -- `dryRun` - If true, prints SQL without executing - -**Partition Strategies**: - -1. **RANGE Partitioning** (for dates, timestamps, sequential values): -```go -config := partitions.NewRangePartitionConfig("created_at") -// Generates: PARTITION BY RANGE (created_at) -``` - -2. **LIST Partitioning** (for discrete categories): -```go -config := partitions.NewListPartitionConfig("region") -// Generates: PARTITION BY LIST (region) -``` - -3. **HASH Partitioning** (for load distribution): -```go -config := partitions.NewHashPartitionConfig(4, "user_id") -// Generates: PARTITION BY HASH (user_id) -// Modulus = 4 means 4 hash partitions will be needed -``` - -**How It Works**: -1. Validates partition configuration -2. Checks if table already exists (returns without error if it does) -3. Parses the GORM model to extract schema information -4. **Converts GORM/Go types to PostgreSQL types** (see Data Type Mapping below) -5. Generates `CREATE TABLE` statement with columns and data types -6. **Adds PRIMARY KEY constraint** (automatically includes partition columns if not already in primary key) -7. Adds `PARTITION BY [RANGE|LIST|HASH] (columns)` clause -8. Creates indexes (skips unique indexes without all partition keys) -9. In dry-run mode, prints SQL; otherwise executes it - -**Data Type Mapping**: -The function automatically converts Go/GORM types to PostgreSQL types: -- `uint`, `uint32`, `uint64`, `int` → `bigint` -- `uint8`, `int8`, `int16` → `smallint` -- `uint16`, `int32` → `integer` -- `int64` → `bigint` -- `float`, `float64` → `double precision` -- `float32` → `real` -- `string` → `text` -- `bool` → `boolean` -- `time.Time` → `timestamp with time zone` -- `[]byte` → `bytea` - -This ensures your GORM models with Go types like `uint` work correctly with PostgreSQL. - -**Important Notes**: -- **Primary keys**: Automatically generated with `PRIMARY KEY (columns)` constraint - - If your model's primary key doesn't include partition columns, they are automatically added - - For example, if you have `ID` as primary key and partition by `created_at`, the constraint will be `PRIMARY KEY (id, created_at)` - - This is a PostgreSQL requirement for partitioned tables -- **Primary key NOT NULL**: Automatically adds NOT NULL to primary key columns -- **Auto-increment fields**: Fields marked with `gorm:"autoIncrement"` are implemented using `GENERATED BY DEFAULT AS IDENTITY` - - IDENTITY columns are automatically NOT NULL (PostgreSQL requirement) - - Supports `autoIncrementIncrement` for custom increment values (e.g., `gorm:"autoIncrement;autoIncrementIncrement:10"` generates `IDENTITY (INCREMENT BY 10)`) - - Example: `ID uint \`gorm:"primaryKey;autoIncrement"\`` generates `id bigint GENERATED BY DEFAULT AS IDENTITY` -- **Column deduplication**: Automatically deduplicates columns to prevent the same column from appearing multiple times - - GORM can include duplicate fields in `stmt.Schema.Fields` (e.g., from embedded structs like `gorm.Model`) - - First occurrence of each column is used, subsequent duplicates are skipped with debug logging -- **Unique indexes**: Must include ALL partition columns (PostgreSQL requirement) -- **After creation**: Create actual partitions based on strategy -- Table creation is a one-time operation (cannot easily modify schema after) -- **Data types**: Automatically converted from Go types to PostgreSQL types - -**Example Models**: - -```go -// Basic model with auto-increment primary key -type MyModel struct { - ID uint `gorm:"primaryKey;autoIncrement"` - Name string `gorm:"not null"` - CreatedAt time.Time `gorm:"index"` -} -// Generated SQL: -// id bigint GENERATED BY DEFAULT AS IDENTITY -// PRIMARY KEY (id, created_at) -- includes partition column - -// Model with custom increment value -type CustomIncrement struct { - ID uint `gorm:"primaryKey;autoIncrement;autoIncrementIncrement:10"` - Data string - CreatedAt time.Time -} -// Generated SQL: -// id bigint GENERATED BY DEFAULT AS IDENTITY (INCREMENT BY 10) -``` - -**Complete Workflows**: - -**RANGE Partitioning (Date-based)**: -```go -// 1. Create the partitioned table structure -config := partitions.NewRangePartitionConfig("created_at") -_, err := partitions.CreatePartitionedTable(dbc, &models.MyModel{}, "my_table", config, false) - -// 2. Create partitions for date range -startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) -endDate := time.Now() -created, err := partitions.CreateMissingPartitions(dbc, "my_table", startDate, endDate, false) -``` - -**HASH Partitioning (Load Distribution)**: -```go -// 1. Create the partitioned table structure -config := partitions.NewHashPartitionConfig(4, "user_id") -_, err := partitions.CreatePartitionedTable(dbc, &models.MyModel{}, "my_table", config, false) - -// 2. Create hash partitions manually -for i := 0; i < 4; i++ { - partName := fmt.Sprintf("my_table_%d", i) - sql := fmt.Sprintf("CREATE TABLE %s PARTITION OF my_table FOR VALUES WITH (MODULUS 4, REMAINDER %d)", partName, i) - dbc.DB.Exec(sql) -} -``` - -**LIST Partitioning (Category-based)**: -```go -// 1. Create the partitioned table structure -config := partitions.NewListPartitionConfig("region") -_, err := partitions.CreatePartitionedTable(dbc, &models.MyModel{}, "my_table", config, false) - -// 2. Create list partitions manually -regions := []string{"us-east", "us-west", "eu-central"} -for _, region := range regions { - partName := fmt.Sprintf("my_table_%s", region) - sql := fmt.Sprintf("CREATE TABLE %s PARTITION OF my_table FOR VALUES IN ('%s')", partName, region) - dbc.DB.Exec(sql) -} -``` - ---- - - -#### UpdatePartitionedTable -Updates an existing partitioned table schema to match a GORM model. - -```go -// Define your updated model -type MyModel struct { - ID uint `gorm:"primaryKey"` - CreatedAt time.Time `gorm:"index"` - Name string - Data string - NewField string `gorm:"index"` // New field added - // OldField removed -} - -// Dry run - see what changes would be made -sql, err := partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", true, false) -if err != nil { - log.WithError(err).Error("dry run failed") -} -// Prints all ALTER TABLE statements that would be executed - -// Actual update -sql, err = partitions.UpdatePartitionedTable(dbc, &MyModel{}, "my_partitioned_table", false, false) -``` - -**Parameters**: -- `model` - GORM model struct with desired schema (must be a pointer, e.g., `&models.MyModel{}`) -- `tableName` - Name of the existing partitioned table -- `dryRun` - If true, prints SQL without executing -- `dropColumns` - If true, columns present in the database but absent from the model will be dropped - -**How It Works**: -1. Checks if the table exists -2. Parses the GORM model to get desired schema -3. Queries database for current schema (columns, indexes, partition keys) -4. Compares schemas and generates ALTER statements for: - - **New columns**: `ALTER TABLE ADD COLUMN` - - **Modified columns**: `ALTER COLUMN TYPE`, `SET/DROP NOT NULL`, `SET/DROP DEFAULT` - - **Removed columns**: `ALTER TABLE DROP COLUMN` - - **New indexes**: `CREATE INDEX` - - **Modified indexes**: `DROP INDEX` + `CREATE INDEX` - - **Removed indexes**: `DROP INDEX` -5. In dry-run mode, prints SQL; otherwise executes it - -**Important Notes**: -- **Cannot change partition keys**: Partition columns cannot be modified after creation -- **Unique indexes**: Must include ALL partition columns (PostgreSQL requirement) -- **Primary key indexes**: Skipped (named `_pkey` by convention) -- **Primary key NOT NULL**: Automatically adds NOT NULL to primary key columns (PostgreSQL requirement) -- **Data types**: Automatically converted from Go types to PostgreSQL types (same as CreatePartitionedTable) -- **Type changes**: Use caution with data type changes that could cause data loss -- **Column removal**: Destructive operation - ensure data is not needed -- Always run dry-run first to preview changes - -**Schema Changes Detected**: - -1. **Column Changes**: - - New columns added with appropriate data type, NOT NULL, and DEFAULT - - Primary key columns automatically get NOT NULL constraint - - Type changes detected through normalized comparison (uses converted PostgreSQL types) - - NULL constraint changes - - DEFAULT value changes - - Removed columns - -2. **Index Changes**: - - New indexes created - - Modified indexes (column list changes) dropped and recreated - - Removed indexes dropped - - Validates unique indexes include partition keys - -**Use When**: -- Your GORM model schema has evolved -- Adding new fields to track additional data -- Modifying column types or constraints -- Adding or removing indexes -- Schema migrations in production - -**Safety Features**: -- Dry-run mode to preview all changes -- Validates unique indexes include partition keys -- Skips primary key indexes (prevents accidental modification) -- Comprehensive logging for each change -- Returns all SQL executed for audit trail - -**Example Workflow**: -```go -// 1. Update your GORM model -type TestResults struct { - ID uint `gorm:"primaryKey"` - CreatedAt time.Time `gorm:"index"` - TestName string `gorm:"index"` - NewMetric float64 // Added field - // RemovedField deleted -} - -// 2. Dry run to see changes -sql, err := partitions.UpdatePartitionedTable(dbc, &TestResults{}, "test_results", true, false) -fmt.Println("Would execute:", sql) - -// 3. Review changes, then apply -sql, err = partitions.UpdatePartitionedTable(dbc, &TestResults{}, "test_results", false, false) -if err != nil { - log.Fatal(err) -} -``` - -**Limitations**: -- Cannot modify partition strategy (RANGE to LIST, etc.) -- Cannot change partition columns -- Cannot split or merge partitions -- Type conversions must be PostgreSQL-compatible -- For major schema changes, consider creating a new table and migrating data - ---- - -#### DropPartition -Drops a single partition. - -```go -// Dry run (safe) -err := partitions.DropPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", true) - -// Actual drop (DESTRUCTIVE) -err := partitions.DropPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", false) -``` - -**Parameters**: -- `partitionName` - Full partition table name -- `dryRun` - If true, only logs what would happen - -**Safety Features**: -- Validates partition name format -- Prevents SQL injection -- Logs all operations - ---- - -#### DetachPartition -Detaches a partition from the parent table (safer alternative to DROP). - -```go -// Dry run -err := partitions.DetachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", true) - -// Actual detach -err := partitions.DetachPartition(dbc, "test_analysis_by_job_by_dates_2024_10_29", false) -``` - -**Use When**: -- You want to archive data before deletion -- You want a reversible operation (can reattach if needed) - ---- - -#### ListAttachedPartitions -Lists all partitions currently attached to the parent table. - -```go -attached, err := partitions.ListAttachedPartitions(dbc, "test_analysis_by_job_by_dates") -if err != nil { - log.WithError(err).Error("failed to list attached partitions") -} - -for _, p := range attached { - fmt.Printf("%s: %s, Size: %s\n", p.TableName, p.PartitionDate, p.SizePretty) -} -``` - -**Parameters**: -- `tableName` - Name of the partitioned parent table - -**Returns**: `[]PartitionInfo` for attached partitions only - -**How It Works**: -- Queries `pg_inherits` to find partitions in the inheritance hierarchy -- Returns only partitions that are currently attached to the parent table - -**Use When**: -- You need to analyze only active partitions -- You want to distinguish between attached and detached partitions -- You need to check the current state of the partitioned table - ---- - -#### ListDetachedPartitions -Lists all partitions that have been detached from the parent table. - -```go -detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") -if err != nil { - log.WithError(err).Error("failed to list detached partitions") -} - -for _, p := range detached { - fmt.Printf("%s: %s, Size: %s\n", p.TableName, p.PartitionDate, p.SizePretty) -} -``` - -**Parameters**: -- `tableName` - Name of the partitioned parent table - -**Returns**: `[]PartitionInfo` for detached partitions - -**How It Works**: -- Queries `pg_inherits` to find attached partitions -- Returns tables matching the naming pattern but NOT in the inheritance hierarchy - ---- - -#### GetAttachedPartitionStats -Returns statistics about attached partitions only. - -```go -stats, err := partitions.GetAttachedPartitionStats(dbc, "test_analysis_by_job_by_dates") -if err != nil { - log.WithError(err).Error("failed to get attached stats") -} - -fmt.Printf("Attached: %d partitions (%s)\n", - stats.TotalPartitions, stats.TotalSizePretty) -``` - -**Parameters**: -- `tableName` - Name of the partitioned parent table - -**Returns**: `*PartitionStats` with aggregate statistics for attached partitions only - -**Use When**: -- Validating retention policies (should only consider active partitions) -- Analyzing current active storage usage -- Monitoring production partition health - ---- - -#### GetDetachedPartitionStats -Returns statistics about detached partitions. - -```go -stats, err := partitions.GetDetachedPartitionStats(dbc, "test_analysis_by_job_by_dates") -if err != nil { - log.WithError(err).Error("failed to get detached stats") -} - -fmt.Printf("Detached: %d partitions (%s)\n", - stats.TotalPartitions, stats.TotalSizePretty) -``` - -**Returns**: `*PartitionStats` for detached partitions only - ---- - -#### IsPartitionAttached -Checks if a specific partition is currently attached to the parent table. - -```go -isAttached, err := partitions.IsPartitionAttached(dbc, "test_analysis_by_job_by_dates_2024_10_29") -if err != nil { - log.WithError(err).Error("check failed") -} - -if isAttached { - fmt.Println("Partition is part of the parent table") -} else { - fmt.Println("Partition is detached (standalone table)") -} -``` - -**Returns**: `bool` indicating attachment status - ---- - -#### AttachPartition -Attaches a partition to the parent table with the appropriate date range. - -```go -// Dry run -err := partitions.AttachPartition(dbc, "test_analysis_by_job_by_dates", "test_analysis_by_job_by_dates_2024_10_29", true) - -// Actual attach -err := partitions.AttachPartition(dbc, "test_analysis_by_job_by_dates", "test_analysis_by_job_by_dates_2024_10_29", false) -``` - -**Parameters**: -- `tableName` - Name of the parent partitioned table -- `partitionName` - Name of the partition to attach (must follow `tableName_YYYY_MM_DD` convention) -- `dryRun` - If true, only logs what would be done - -**Use When**: -- You need to restore archived data -- You detached a partition by mistake -- Historical analysis requires old data - -**Note**: Automatically calculates the date range from the partition name - ---- - -#### CreateMissingPartitions -Creates missing partitions for a date range. - -```go -startDate := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) -endDate := time.Date(2024, 1, 31, 0, 0, 0, 0, time.UTC) - -// Dry run - see what would be created -created, err := partitions.CreateMissingPartitions(dbc, "test_analysis_by_job_by_dates", startDate, endDate, true) -fmt.Printf("Would create %d partitions\n", created) - -// Actual creation -created, err = partitions.CreateMissingPartitions(dbc, "test_analysis_by_job_by_dates", startDate, endDate, false) -fmt.Printf("Created %d partitions\n", created) -``` - -**Parameters**: -- `tableName` - Name of the partitioned parent table -- `startDate` - Start of date range (inclusive) -- `endDate` - End of date range (inclusive) -- `dryRun` - If true, only simulates the operation - -**How It Works**: -1. Lists all existing partitions (attached + detached) -2. For existing partitions, verifies they are attached and reattaches any detached ones -3. Generates list of dates in range that don't have partitions -4. All creates and attaches run in a single transaction — if any operation fails, all changes are rolled back -5. For each missing partition: - - Creates table with same structure as parent (CREATE TABLE ... LIKE) - - Attaches partition using `AttachPartition` with appropriate date range -6. Returns count of partitions created - -**Use When**: -- Setting up a new partitioned table with historical dates -- Backfilling missing partitions after data gaps -- Preparing partitions in advance for future dates -- Recovering from partition management issues (including reattaching detached partitions) - -**Safety Features**: -- Checks for existing partitions before creating -- Reattaches detached partitions found within the date range -- All operations are transactional — atomic success or full rollback -- Dry-run mode to preview what will be created and reattached -- Errors are returned, not silently logged - ---- - -#### DetachOldPartitions -Bulk operation to detach all partitions older than retention period. - -```go -// Dry run -detached, err := partitions.DetachOldPartitions(dbc, 180, true) -fmt.Printf("Would detach %d partitions\n", detached) - -// Actual detach -detached, err := partitions.DetachOldPartitions(dbc, 180, false) -fmt.Printf("Detached %d partitions\n", detached) -``` - -**Parameters**: -- `retentionDays` - Retention period in days -- `dryRun` - If true, only simulates the operation - -**Features**: -- Validates retention policy before execution -- Processes partitions in order (oldest first) -- Logs each partition detachment -- Returns count of partitions detached - ---- - -#### DropOldPartitions -Bulk operation to drop all partitions older than retention period. - -```go -// Dry run - see what would happen -dropped, err := partitions.DropOldPartitions(dbc, 180, true) -fmt.Printf("Would drop %d partitions\n", dropped) - -// Actual cleanup (DESTRUCTIVE) -dropped, err := partitions.DropOldPartitions(dbc, 180, false) -fmt.Printf("Dropped %d partitions\n", dropped) -``` - -**Parameters**: -- `retentionDays` - Retention period in days -- `dryRun` - If true, only simulates the operation - -**Features**: -- Validates retention policy before execution -- Processes partitions in order (oldest first) -- Logs each partition drop -- Returns count of partitions dropped - ---- - -#### DropOldDetachedPartitions -Bulk operation to drop detached partitions older than retention period. - -```go -// Dry run - see what would happen -dropped, err := partitions.DropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180, true) -fmt.Printf("Would drop %d detached partitions\n", dropped) - -// Actual cleanup (DESTRUCTIVE) -dropped, err := partitions.DropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180, false) -fmt.Printf("Dropped %d detached partitions\n", dropped) -``` - -**Parameters**: -- `tableName` - Name of the parent table -- `retentionDays` - Retention period in days -- `dryRun` - If true, only simulates the operation - -**Use When**: -- You have detached partitions that have been archived -- You want to clean up old detached partitions no longer needed -- You need to reclaim storage from detached partitions - -**Features**: -- Lists all detached partitions first -- Filters by retention period -- Processes partitions in order (oldest first) -- Logs each partition drop -- Returns count of partitions dropped - -**Note**: Unlike `DropOldPartitions`, this only affects detached partitions. Attached partitions remain untouched. - ---- - -## Usage Examples - -### Example 1: Analyze Current State - -```go -import "github.com/openshift/sippy/pkg/db/partitions" - -func analyzePartitions(dbc *db.DB) { - // Get overall statistics - stats, err := partitions.GetPartitionStats(dbc) - if err != nil { - log.Fatal(err) - } - - fmt.Printf("Total: %d partitions (%s)\n", - stats.TotalPartitions, stats.TotalSizePretty) - - // Analyze by age groups - groups, err := partitions.GetPartitionsByAgeGroup(dbc) - if err != nil { - log.Fatal(err) - } - - for _, group := range groups { - fmt.Printf("%s: %s\n", group["age_bucket"], group["total_size"]) - } -} -``` - -### Example 2: Dry Run Cleanup - -```go -func dryRunCleanup(dbc *db.DB, retentionDays int) { - // Validate policy - if err := partitions.ValidateRetentionPolicy(dbc, retentionDays); err != nil { - log.Fatalf("Policy validation failed: %v", err) - } - - // Get summary - summary, err := partitions.GetRetentionSummary(dbc, retentionDays) - if err != nil { - log.Fatal(err) - } - - fmt.Printf("Would delete %d partitions, reclaiming %s\n", - summary.PartitionsToRemove, summary.StoragePretty) - - // Perform dry run - dropped, err := partitions.DropOldPartitions(dbc, retentionDays, true) - if err != nil { - log.Fatal(err) - } - - fmt.Printf("Dry run complete: %d partitions would be dropped\n", dropped) -} -``` - -### Example 3: Execute Cleanup (Production) - -```go -func executeCleanup(dbc *db.DB, retentionDays int) { - // Always validate first - if err := partitions.ValidateRetentionPolicy(dbc, retentionDays); err != nil { - return fmt.Errorf("retention policy failed validation: %w", err) - } - - // Get summary for logging - summary, err := partitions.GetRetentionSummary(dbc, retentionDays) - if err != nil { - return err - } - - log.WithFields(log.Fields{ - "retention_days": retentionDays, - "partitions_to_delete": summary.PartitionsToRemove, - "storage_to_reclaim": summary.StoragePretty, - }).Info("starting partition cleanup") - - // Execute cleanup (NOT a dry run) - dropped, err := partitions.DropOldPartitions(dbc, retentionDays, false) - if err != nil { - return fmt.Errorf("cleanup failed: %w", err) - } - - log.WithField("dropped", dropped).Info("partition cleanup completed") - return nil -} -``` - -### Example 4: Detach Instead of Drop (Safer) - -```go -func detachForArchival(dbc *db.DB, retentionDays int) error { - // Validate policy - if err := partitions.ValidateRetentionPolicy(dbc, retentionDays); err != nil { - return err - } - - // Detach old partitions instead of dropping - detached, err := partitions.DetachOldPartitions(dbc, retentionDays, false) - if err != nil { - return fmt.Errorf("detach failed: %w", err) - } - - log.WithField("detached", detached).Info("partitions detached for archival") - - // Now archive the detached partitions (external process) - // archiveDetachedPartitions(dbc) - - return nil -} -``` - -### Example 5: Compare Attached vs Detached Partitions - -```go -func comparePartitionState(dbc *db.DB, tableName string) error { - // Get all partitions (attached + detached) - allPartitions, err := partitions.ListTablePartitions(dbc, tableName) - if err != nil { - return err - } - - // Get only attached partitions - attached, err := partitions.ListAttachedPartitions(dbc, tableName) - if err != nil { - return err - } - - // Get only detached partitions - detached, err := partitions.ListDetachedPartitions(dbc, tableName) - if err != nil { - return err - } - - // Display summary - fmt.Printf("Partition State for %s:\n", tableName) - fmt.Printf(" Total: %d partitions\n", len(allPartitions)) - fmt.Printf(" Attached: %d partitions\n", len(attached)) - fmt.Printf(" Detached: %d partitions\n", len(detached)) - - // Calculate storage breakdown - var attachedSize, detachedSize int64 - for _, p := range attached { - attachedSize += p.SizeBytes - } - for _, p := range detached { - detachedSize += p.SizeBytes - } - - fmt.Printf("\nStorage Breakdown:\n") - fmt.Printf(" Attached: %d bytes\n", attachedSize) - fmt.Printf(" Detached: %d bytes\n", detachedSize) - fmt.Printf(" Total: %d bytes\n", attachedSize+detachedSize) - - return nil -} -``` - ---- - -### Example 6: Working with Detached Partitions - -```go -func manageDetachedPartitions(dbc *db.DB) error { - // List all detached partitions - detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") - if err != nil { - return err - } - - fmt.Printf("Found %d detached partitions\n", len(detached)) - - // Get statistics - stats, err := partitions.GetDetachedPartitionStats(dbc, "test_analysis_by_job_by_dates") - if err != nil { - return err - } - - fmt.Printf("Detached partitions total: %s\n", stats.TotalSizePretty) - - // Check if specific partition is detached - for _, p := range detached { - isAttached, err := partitions.IsPartitionAttached(dbc, p.TableName) - if err != nil { - continue - } - - if !isAttached { - fmt.Printf("%s is detached and ready for archival\n", p.TableName) - // Archive this partition to S3, compress, etc. - } - } - - return nil -} -``` - ---- - -### Example 7: Reattach Archived Data - -```go -func restoreArchivedPartition(dbc *db.DB, tableName, partitionName string) error { - // Check current status - isAttached, err := partitions.IsPartitionAttached(dbc, partitionName) - if err != nil { - return err - } - - if isAttached { - return fmt.Errorf("partition %s is already attached", partitionName) - } - - log.WithField("partition", partitionName).Info("reattaching partition") - - // Reattach the partition - err = partitions.AttachPartition(dbc, tableName, partitionName, false) - if err != nil { - return fmt.Errorf("reattach failed: %w", err) - } - - log.Info("partition reattached successfully") - return nil -} -``` - ---- - -### Example 8: Create Missing Partitions for Date Range - -```go -func ensurePartitionsExist(dbc *db.DB, tableName string, startDate, endDate time.Time) error { - // Check what partitions would be created - created, err := partitions.CreateMissingPartitions(dbc, tableName, startDate, endDate, true) - if err != nil { - return fmt.Errorf("dry run failed: %w", err) - } - - if created == 0 { - log.Info("all partitions already exist") - return nil - } - - log.WithFields(log.Fields{ - "table": tableName, - "start_date": startDate.Format("2006-01-02"), - "end_date": endDate.Format("2006-01-02"), - "to_create": created, - }).Info("creating missing partitions") - - // Create the missing partitions - created, err = partitions.CreateMissingPartitions(dbc, tableName, startDate, endDate, false) - if err != nil { - return fmt.Errorf("partition creation failed: %w", err) - } - - log.WithField("created", created).Info("partitions created successfully") - return nil -} - -// Example: Prepare partitions for next month -func prepareNextMonthPartitions(dbc *db.DB) error { - now := time.Now() - startOfNextMonth := time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, time.UTC) - endOfNextMonth := startOfNextMonth.AddDate(0, 1, -1) - - return ensurePartitionsExist(dbc, "test_analysis_by_job_by_dates", startOfNextMonth, endOfNextMonth) -} - -// Example: Backfill missing partitions for last 90 days -func backfillRecentPartitions(dbc *db.DB) error { - endDate := time.Now() - startDate := endDate.AddDate(0, 0, -90) - - return ensurePartitionsExist(dbc, "test_analysis_by_job_by_dates", startDate, endDate) -} -``` - ---- - -### Example 9: Create a New Partitioned Table from GORM Model - -```go -package main - -import ( - "time" - "github.com/openshift/sippy/pkg/db" - "github.com/openshift/sippy/pkg/db/partitions" -) - -// Define your model -type TestResults struct { - ID uint `gorm:"primaryKey"` - TestName string `gorm:"index"` - JobName string `gorm:"index"` - Result string - CreatedAt time.Time `gorm:"index"` // This will be the partition column - TestOutput string - Duration int -} - -func setupPartitionedTestResults(dbc *db.DB) error { - tableName := "test_results_partitioned" - - // Configure RANGE partitioning by created_at - config := partitions.NewRangePartitionConfig("created_at") - - // Step 1: Create the partitioned table (dry-run first) - sql, err := partitions.CreatePartitionedTable( - dbc, - &TestResults{}, - tableName, - config, - true, // dry-run - ) - if err != nil { - return fmt.Errorf("dry run failed: %w", err) - } - - log.Info("Would execute SQL:") - log.Info(sql) - - // The generated SQL will look like: - // CREATE TABLE IF NOT EXISTS test_results_partitioned ( - // id bigint NOT NULL, - // test_name text, - // job_name text, - // result text, - // created_at timestamp with time zone NOT NULL, - // test_output text, - // duration bigint, - // PRIMARY KEY (id, created_at) - // ) PARTITION BY RANGE (created_at) - // - // Note: created_at is automatically added to the primary key - // because it's the partition column (PostgreSQL requirement) - - // Step 2: Create the table for real - _, err = partitions.CreatePartitionedTable( - dbc, - &TestResults{}, - tableName, - config, - false, // execute - ) - if err != nil { - return fmt.Errorf("table creation failed: %w", err) - } - - log.WithField("table", tableName).Info("partitioned table created") - - // Step 3: Create partitions for the last 90 days - endDate := time.Now() - startDate := endDate.AddDate(0, 0, -90) - - created, err := partitions.CreateMissingPartitions( - dbc, - tableName, - startDate, - endDate, - false, - ) - if err != nil { - return fmt.Errorf("partition creation failed: %w", err) - } - - log.WithFields(log.Fields{ - "table": tableName, - "partitions": created, - }).Info("created partitions") - - return nil -} - -// You can now use the table normally with GORM -func insertTestResult(dbc *db.DB) error { - result := TestResults{ - TestName: "test-api-health", - JobName: "periodic-ci-test", - Result: "passed", - CreatedAt: time.Now(), - TestOutput: "All checks passed", - Duration: 125, - } - - // GORM will automatically route to the correct partition based on created_at - return dbc.DB.Create(&result).Error -} -``` - -**Key Points**: -- Model must have the partition column (e.g., `created_at`) -- PRIMARY KEY constraint is automatically generated -- Partition columns are automatically added to the primary key (PostgreSQL requirement) -- In the example above, `PRIMARY KEY (id, created_at)` is generated even though only `id` is marked as primaryKey -- Unique indexes must include the partition column -- Data is automatically routed to correct partition by PostgreSQL - ---- - -### Example 10: Update Partitioned Table Schema - -```go -package main - -import ( - "time" - "github.com/openshift/sippy/pkg/db" - "github.com/openshift/sippy/pkg/db/partitions" -) - -// Original model (what was created initially) -type TestResultsV1 struct { - ID uint `gorm:"primaryKey"` - TestName string `gorm:"index"` - JobName string `gorm:"index"` - Result string - CreatedAt time.Time `gorm:"index"` - TestOutput string - Duration int -} - -// Updated model with schema changes -type TestResultsV2 struct { - ID uint `gorm:"primaryKey"` - TestName string `gorm:"index"` - JobName string `gorm:"index"` - Result string - CreatedAt time.Time `gorm:"index"` - TestOutput string - Duration int - // New fields - TestSuite string `gorm:"index"` // Added: track test suite - ErrorCount int // Added: count of errors - // Removed: RemovedField no longer needed -} - -func updateTestResultsSchema(dbc *db.DB) error { - tableName := "test_results_partitioned" - - log.Info("Updating table schema to match new model...") - - // Step 1: Dry run to see what would change - sql, err := partitions.UpdatePartitionedTable( - dbc, - &TestResultsV2{}, - tableName, - true, // dry-run - false, // don't drop columns - ) - if err != nil { - return fmt.Errorf("dry run failed: %w", err) - } - - log.Info("Schema changes that would be applied:") - log.Info(sql) - - // Step 2: Review the changes and confirm - fmt.Println("\nReview the changes above.") - fmt.Print("Apply these changes? (yes/no): ") - var response string - fmt.Scanln(&response) - - if response != "yes" { - log.Info("Schema update cancelled") - return nil - } - - // Step 3: Apply the changes - sql, err = partitions.UpdatePartitionedTable( - dbc, - &TestResultsV2{}, - tableName, - false, // execute - false, // don't drop columns - ) - if err != nil { - return fmt.Errorf("schema update failed: %w", err) - } - - log.WithFields(log.Fields{ - "table": tableName, - "changes": sql, - }).Info("schema updated successfully") - - return nil -} - -// Automated schema migration (for CI/CD) -func automatedSchemaMigration(dbc *db.DB) error { - tableName := "test_results_partitioned" - - // Check what changes would be made - sql, err := partitions.UpdatePartitionedTable( - dbc, - &TestResultsV2{}, - tableName, - true, // dry-run - false, // don't drop columns - ) - if err != nil { - return fmt.Errorf("schema check failed: %w", err) - } - - if sql == "" { - log.Info("Schema is up to date, no changes needed") - return nil - } - - // Log the planned changes - log.WithField("sql", sql).Info("applying schema changes") - - // Apply changes - sql, err = partitions.UpdatePartitionedTable( - dbc, - &TestResultsV2{}, - tableName, - false, // execute - false, // don't drop columns - ) - if err != nil { - return fmt.Errorf("schema migration failed: %w", err) - } - - log.Info("schema migration completed successfully") - return nil -} - -// Example: Gradual schema evolution -func evolveSchema(dbc *db.DB) error { - tableName := "test_results_partitioned" - - // Phase 1: Add nullable columns first (safe) - type PhaseOne struct { - ID uint `gorm:"primaryKey"` - CreatedAt time.Time `gorm:"index"` - TestName string - TestSuite string // New, nullable - } - - log.Info("Phase 1: Adding nullable columns") - _, err := partitions.UpdatePartitionedTable(dbc, &PhaseOne{}, tableName, false, false) - if err != nil { - return err - } - - // Phase 2: Populate new columns with data - log.Info("Phase 2: Populating new columns") - // (Application code populates test_suite from test_name) - - // Phase 3: Add indexes after data is populated - type PhaseTwo struct { - ID uint `gorm:"primaryKey"` - CreatedAt time.Time `gorm:"index"` - TestName string - TestSuite string `gorm:"index"` // Now indexed - } - - log.Info("Phase 3: Adding indexes") - _, err = partitions.UpdatePartitionedTable(dbc, &PhaseTwo{}, tableName, false, false) - if err != nil { - return err - } - - log.Info("Schema evolution completed") - return nil -} -``` - -**Key Scenarios**: - -1. **Adding Columns**: New fields in the model are added to the table -2. **Removing Columns**: Fields removed from model are dropped (use caution) -3. **Changing Types**: Data type changes are detected and applied -4. **Adding Indexes**: New `gorm:"index"` tags create indexes -5. **Modifying Constraints**: NOT NULL and DEFAULT changes - -**Best Practices**: -- Always run dry-run first to preview changes -- Review generated SQL before applying -- Test schema changes in a development environment first -- For production, consider gradual evolution (add nullable, populate, add constraints) -- Back up data before major type conversions -- Monitor query performance after index changes - ---- - -### Example 11: Complete Workflow - -See [examples.go](./examples.go) for a complete workflow demonstration including: -- Current state analysis -- Age distribution -- Retention policy comparison -- Dry run execution - ---- - -## Integration with Automation - -### Option 1: Kubernetes CronJob - -```go -// In your scheduled job -func scheduledCleanup() { - dbc := db.New(...) - - // 180-day retention policy - dropped, err := partitions.DropOldPartitions(dbc, 180, false) - if err != nil { - log.WithError(err).Error("scheduled cleanup failed") - return - } - - log.WithField("dropped", dropped).Info("scheduled cleanup completed") -} -``` - -### Option 2: CLI Command - -```go -func main() { - retentionDays := flag.Int("retention-days", 180, "Retention period in days") - dryRun := flag.Bool("dry-run", true, "Perform dry run only") - flag.Parse() - - dbc := db.New(...) - - dropped, err := partitions.DropOldPartitions(dbc, *retentionDays, *dryRun) - if err != nil { - log.Fatal(err) - } - - if *dryRun { - fmt.Printf("DRY RUN: Would drop %d partitions\n", dropped) - } else { - fmt.Printf("Dropped %d partitions\n", dropped) - } -} -``` - ---- - -## Safety Features - -### Input Validation -- Partition names are validated against expected format -- SQL injection protection through parameterized queries -- Minimum retention period enforcement (90 days) - -### Threshold Checks -- Maximum 75% of partitions can be deleted -- Maximum 80% of storage can be deleted -- Policy must be validated before execution - -### Dry Run Support -- All destructive operations support dry-run mode -- Dry runs log what would happen without making changes -- Always test with dry-run first - -### Comprehensive Logging -- All operations are logged with structured fields -- Errors include context for debugging -- Timing information for performance monitoring - ---- - -## Error Handling - -All functions return errors that should be checked: - -```go -partitions, err := partitions.ListTablePartitions(dbc, "test_analysis_by_job_by_dates") -if err != nil { - log.WithError(err).Error("failed to list partitions") - return err -} -``` - -Common error scenarios: -- Database connection issues -- Permission denied (read-only user attempting writes) -- Invalid retention policy -- Partition name validation failures - ---- - -## Testing - -Run the test suite: - -```bash -go test ./pkg/db/partitions/... -``` - -Test coverage includes: -- Partition name validation -- Struct initialization -- Edge cases and invalid inputs - ---- - -## Detach/Archive Workflow - -### Understanding Detached Partitions - -When a partition is **detached**, it: -1. Becomes a standalone table (no longer part of the partitioned table) -2. Keeps all its data intact -3. Can still be queried directly by table name -4. Can be archived, compressed, or exported -5. Can be reattached if needed -6. Doesn't show up in queries against the parent table - -### How to Find Detached Partitions - -PostgreSQL tracks partition relationships in `pg_inherits`. Detached partitions: -- Still exist as tables in `pg_tables` -- Are NOT in the `pg_inherits` hierarchy -- Match the partition naming pattern - -**Query to find them:** -```go -detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") -// Returns all tables matching naming pattern but not attached -``` - -### Typical Detach/Archive Workflow - -#### Step 1: Detach Old Partitions -```go -// Detach partitions older than 180 days -detached, err := partitions.DetachOldPartitions(dbc, 180, false) -log.Printf("Detached %d partitions\n", detached) -``` - -**Result**: Partitions are now standalone tables - -#### Step 2: List Detached Partitions -```go -// Find all detached partitions -detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") - -for _, p := range detached { - fmt.Printf("Detached: %s (%s)\n", p.TableName, p.SizePretty) -} -``` - -#### Step 3: Archive Detached Partitions -External archival process (examples): - -**Option A: Export to CSV/Parquet** -```bash -# Export to compressed CSV -psql $SIPPY_DSN -c " -COPY test_analysis_by_job_by_dates_2024_10_29 -TO STDOUT CSV HEADER -" | gzip > partition_2024_10_29.csv.gz - -# Upload to S3 -aws s3 cp partition_2024_10_29.csv.gz s3://sippy-archive/ -``` - -**Option B: Use pg_dump** -```bash -pg_dump $SIPPY_DSN \ - -t test_analysis_by_job_by_dates_2024_10_29 \ - --format=custom \ - | gzip > partition_2024_10_29.pgdump.gz -``` - -**Option C: Direct S3 export (requires aws_s3 extension)** -```sql -SELECT aws_s3.query_export_to_s3( - 'SELECT * FROM test_analysis_by_job_by_dates_2024_10_29', - aws_commons.create_s3_uri('sippy-archive', 'partitions/2024_10_29.parquet', 'us-east-1'), - options := 'FORMAT PARQUET' -); -``` - -#### Step 4: Verify Archive -```bash -# Verify archive exists and is readable -aws s3 ls s3://sippy-archive/partition_2024_10_29.csv.gz -# Check file size matches expected -``` - -#### Step 5: Drop Detached Partitions - -**Option A: Bulk drop old detached partitions (recommended)** -```go -// Drop all detached partitions older than 180 days -// (Assumes they have already been archived) - -// Dry run first -dropped, err := partitions.DropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180, true) -fmt.Printf("Would drop %d detached partitions\n", dropped) - -// Actual drop -dropped, err = partitions.DropOldDetachedPartitions(dbc, "test_analysis_by_job_by_dates", 180, false) -fmt.Printf("Dropped %d detached partitions\n", dropped) -``` - -**Option B: Selective drop with archive verification** -```go -// After successful archive, drop detached partitions -detached, err := partitions.ListDetachedPartitions(dbc, "test_analysis_by_job_by_dates") - -for _, p := range detached { - // Verify this partition has been archived - if isArchived(p.TableName) { - err := partitions.DropPartition(dbc, p.TableName, false) - if err != nil { - log.WithError(err).Error("failed to drop detached partition") - } - } -} -``` - -#### Step 6: Restore if Needed -If you need to restore archived data: - -1. **Restore from archive**: -```bash -# Restore table from pg_dump -gunzip -c partition_2024_10_29.pgdump.gz | pg_restore -d $SIPPY_DSN -``` - -2. **Reattach partition**: -```go -err := partitions.AttachPartition(dbc, "test_analysis_by_job_by_dates", "test_analysis_by_job_by_dates_2024_10_29", false) -``` - -### Advantages of Detach vs. DROP - -| Aspect | DETACH | DROP | -|--------|--------|------| -| **Reversible** | ✅ Yes (can reattach) | ❌ No (permanent) | -| **Data preserved** | ✅ Yes (in detached table) | ❌ No (deleted) | -| **Immediate space** | ❌ No (table still exists) | ✅ Yes (storage freed) | -| **Archive time** | ✅ After detach | ⚠️ Before drop | -| **Risk** | 🟢 Low | 🔴 High | -| **Speed** | ⚡ Fast | ⚡ Fast | -| **Query detached data** | ✅ Yes (by table name) | ❌ No (gone) | - -### Complete Automation Example - -```go -func automatedArchiveCleanup(dbc *db.DB, archiver Archiver) error { - retentionDays := 180 - - // 1. Detach old partitions - detached, err := partitions.DetachOldPartitions(dbc, retentionDays, false) - if err != nil { - return err - } - - log.Printf("Detached %d partitions\n", detached) - - // 2. Get list of detached partitions - detachedList, err := partitions.ListDetachedPartitions(dbc) - if err != nil { - return err - } - - // 3. Archive each detached partition - for _, p := range detachedList { - // Archive to S3 - err := archiver.Archive(p.TableName) - if err != nil { - log.WithError(err).WithField("partition", p.TableName).Error("archive failed") - continue - } - - // Verify archive - if !archiver.Verify(p.TableName) { - log.WithField("partition", p.TableName).Error("archive verification failed") - continue - } - - // Drop detached partition - err = partitions.DropPartition(dbc, p.TableName, false) - if err != nil { - log.WithError(err).WithField("partition", p.TableName).Error("drop failed") - continue - } - - log.WithField("partition", p.TableName).Info("archived and dropped successfully") - } - - return nil -} -``` - ---- - -## Related Documentation - -- [Partition Retention Management Guide](../../../.claude/partition-retention-management-guide.md) - Complete guide with SQL examples -- [Database Schema Analysis](../../../.claude/db-schema-analysis.md) - Overall database structure -- [Database Analysis Index](../../../.claude/db-analysis-index.md) - Navigation to all analysis docs - ---- - -## Recommended Retention Policies - -Based on analysis in the retention management guide: - -| Policy | Retention | Storage | Use Case | -|--------|-----------|---------|----------| -| Conservative | 365 days | ~900 GB | Full year of data, Y-o-Y comparisons | -| **Recommended** | **180 days** | **~450 GB** | **6 months, covers release cycles** | -| Aggressive | 90 days | ~225 GB | Recent CI health only, max savings | - -**Current recommendation**: **180-day retention** -- Balances historical data access with storage efficiency -- Covers typical OpenShift release cycles -- Would reclaim ~160 GB immediately -- Stabilizes storage at ~450 GB - ---- - -## Notes - -- All operations require `*db.DB` instance (GORM wrapper) -- Read-only operations are safe with read-only database credentials -- Write operations require admin credentials -- Partition format: `test_analysis_by_job_by_dates_YYYY_MM_DD` -- Only `test_analysis_by_job_by_dates` partitions are supported currently From 4a2d47d8fa6892dc36e8edee2eb4b3f1339ec310 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Fri, 13 Mar 2026 12:41:35 -0400 Subject: [PATCH 11/11] TRT-1989: edge cases --- pkg/dataloader/prowloader/prow.go | 15 ++++- pkg/db/partitions/partitions.go | 107 +++++++++++++++++++----------- pkg/db/utils.go | 38 ++++++++++- 3 files changed, 115 insertions(+), 45 deletions(-) diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index 8eb3fdb97..3e7d2c927 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -416,10 +416,19 @@ func (pl *ProwLoader) preparePartitions(config PartitionManagementConfig) error // When initializing a new table, look back the configured number of days oldestDate := time.Now().Add(-time.Duration(config.InitialLookbackDays) * 24 * time.Hour) if stats.TotalPartitions > 0 { + + var startRange, endRange string + if stats.OldestDate.Valid { + startRange = stats.OldestDate.Time.Format("2006-01-02") + oldestDate = stats.OldestDate.Time + } + if stats.NewestDate.Valid { + endRange = stats.OldestDate.Time.Format("2006-01-02") + } fmt.Printf(" Range: %s to %s\n", - stats.OldestDate.Format("2006-01-02"), - stats.NewestDate.Format("2006-01-02")) - oldestDate = stats.OldestDate + startRange, + endRange) + } futureDate := time.Now().Add(config.FuturePartitionWindow) diff --git a/pkg/db/partitions/partitions.go b/pkg/db/partitions/partitions.go index fc2bc09ab..80b53cbd0 100644 --- a/pkg/db/partitions/partitions.go +++ b/pkg/db/partitions/partitions.go @@ -46,8 +46,8 @@ type PartitionStats struct { TotalPartitions int TotalSizeBytes int64 TotalSizePretty string - OldestDate time.Time - NewestDate time.Time + OldestDate sql.NullTime + NewestDate sql.NullTime AvgSizeBytes int64 AvgSizePretty string } @@ -599,17 +599,32 @@ func DropOldDetachedPartitions(dbc *db.DB, tableName string, retentionDays int, return 0, nil } - // Drop each old detached partition + if dryRun { + for _, partition := range toRemove { + if err := DropPartition(dbc, partition.TableName, true); err != nil { + return 0, fmt.Errorf("failed to dry-run drop partition %s: %w", partition.TableName, err) + } + } + return len(toRemove), nil + } + + // Drop all old detached partitions in a transaction droppedCount := 0 var totalSize int64 - for _, partition := range toRemove { - if err := DropPartition(dbc, partition.TableName, dryRun); err != nil { - log.WithError(err).WithField("partition", partition.TableName).Error("failed to drop detached partition") - continue + err = dbc.DB.Transaction(func(tx *gorm.DB) error { + txDBC := &db.DB{DB: tx} + for _, partition := range toRemove { + if err := DropPartition(txDBC, partition.TableName, false); err != nil { + return fmt.Errorf("failed to drop partition %s: %w", partition.TableName, err) + } + droppedCount++ + totalSize += partition.SizeBytes } - droppedCount++ - totalSize += partition.SizeBytes + return nil + }) + if err != nil { + return 0, err } elapsed := time.Since(start) @@ -618,7 +633,6 @@ func DropOldDetachedPartitions(dbc *db.DB, tableName string, retentionDays int, "retention_days": retentionDays, "total_dropped": droppedCount, "storage_reclaimed": fmt.Sprintf("%d bytes", totalSize), - "dry_run": dryRun, "elapsed": elapsed, }).Info("completed dropping old detached partitions") @@ -1792,31 +1806,31 @@ func getPrimaryKeyColumns(dbc *db.DB, tableName string) ([]string, error) { func normalizeDataType(dataType string) string { dataType = strings.ToLower(strings.TrimSpace(dataType)) - // Map common type variations to standard forms (preserving any modifiers) - // Check for types with modifiers first (e.g., "character varying(64)") - replacements := map[string]string{ - "character varying": "varchar", - "integer": "int", - "int4": "int", - "int8": "bigint", - "bigserial": "bigint", - "serial": "int", - "smallint": "int2", - "boolean": "bool", - "timestamp without time zone": "timestamp", - "timestamp with time zone": "timestamptz", - "double precision": "float8", - "real": "float4", - "character": "char", - "time without time zone": "time", - "time with time zone": "timetz", + // Map common type variations to standard forms (preserving any modifiers). + // Uses a slice for deterministic matching order — longer prefixes must come + // before shorter ones (e.g., "character varying" before "character"). + replacements := []struct{ old, new string }{ + {"character varying", "varchar"}, + {"timestamp without time zone", "timestamp"}, + {"timestamp with time zone", "timestamptz"}, + {"time without time zone", "time"}, + {"time with time zone", "timetz"}, + {"double precision", "float8"}, + {"character", "char"}, + {"integer", "int"}, + {"int4", "int"}, + {"int8", "bigint"}, + {"bigserial", "bigint"}, + {"serial", "int"}, + {"smallint", "int2"}, + {"boolean", "bool"}, + {"real", "float4"}, } // Try to replace the base type name while preserving modifiers - for old, new := range replacements { - if suffix, found := strings.CutPrefix(dataType, old); found { - // Replace the prefix and keep everything after (modifiers, array brackets, etc.) - return new + suffix + for _, r := range replacements { + if suffix, found := strings.CutPrefix(dataType, r.old); found { + return r.new + suffix } } @@ -2007,16 +2021,32 @@ func DetachOldPartitions(dbc *db.DB, tableName string, retentionDays int, dryRun return 0, nil } + if dryRun { + for _, partition := range partitions { + if err := DetachPartition(dbc, partition.TableName, true); err != nil { + return 0, fmt.Errorf("failed to dry-run detach partition %s: %w", partition.TableName, err) + } + } + return len(partitions), nil + } + + // Detach all old partitions in a transaction detachedCount := 0 var totalSize int64 - for _, partition := range partitions { - if err := DetachPartition(dbc, partition.TableName, dryRun); err != nil { - log.WithError(err).WithField("partition", partition.TableName).Error("failed to detach partition") - continue + err = dbc.DB.Transaction(func(tx *gorm.DB) error { + txDBC := &db.DB{DB: tx} + for _, partition := range partitions { + if err := DetachPartition(txDBC, partition.TableName, false); err != nil { + return fmt.Errorf("failed to detach partition %s: %w", partition.TableName, err) + } + detachedCount++ + totalSize += partition.SizeBytes } - detachedCount++ - totalSize += partition.SizeBytes + return nil + }) + if err != nil { + return 0, err } elapsed := time.Since(start) @@ -2025,7 +2055,6 @@ func DetachOldPartitions(dbc *db.DB, tableName string, retentionDays int, dryRun "retention_days": retentionDays, "total_detached": detachedCount, "storage_affected": fmt.Sprintf("%d bytes", totalSize), - "dry_run": dryRun, "elapsed": elapsed, }).Info("completed detaching old partitions") diff --git a/pkg/db/utils.go b/pkg/db/utils.go index c40d8ddd4..0202a4057 100644 --- a/pkg/db/utils.go +++ b/pkg/db/utils.go @@ -40,6 +40,8 @@ type ColumnVerificationOptions struct { CheckDefaults bool // CheckOrder verifies that columns are in the same ordinal position CheckOrder bool + // OmitColumns lists column names to exclude from verification + OmitColumns []string } // DataMigrationColumnVerificationOptions returns options suitable for data migrations @@ -74,6 +76,16 @@ func (dbc *DB) VerifyTablesHaveSameColumns(table1, table2 string, opts ColumnVer return fmt.Errorf("failed to get columns for table %s: %w", table2, err) } + // Filter out omitted columns + if len(opts.OmitColumns) > 0 { + omit := make(map[string]bool, len(opts.OmitColumns)) + for _, col := range opts.OmitColumns { + omit[col] = true + } + cols1 = filterColumns(cols1, omit) + cols2 = filterColumns(cols2, omit) + } + // Check if column counts match if len(cols1) != len(cols2) { return fmt.Errorf("column count mismatch: %s has %d columns, %s has %d columns", @@ -182,6 +194,16 @@ func (dbc *DB) VerifyTablesHaveSameColumns(table1, table2 string, opts ColumnVer return nil } +func filterColumns(cols []ColumnInfo, omit map[string]bool) []ColumnInfo { + filtered := make([]ColumnInfo, 0, len(cols)) + for _, col := range cols { + if !omit[col.ColumnName] { + filtered = append(filtered, col) + } + } + return filtered +} + // GetTableColumns retrieves column information for a table from pg_catalog // Uses format_type() to preserve precise type definitions including: // - Length modifiers: varchar(64) vs varchar(255) @@ -687,10 +709,16 @@ func (dbc *DB) getPartitionsInDateRange(tableName string, startDate, endDate tim return nil, fmt.Errorf("failed to query partitions: %w", result.Error) } + // Normalize bounds to calendar days so that timestamps within a day + // don't exclude the partition for that day (PartitionDate is midnight). + startDay := startDate.Truncate(24 * time.Hour) + endDay := endDate.Truncate(24 * time.Hour) + // Filter to only partitions in the date range var filtered []partitionDateInfo for _, p := range partitions { - if (p.PartitionDate.Equal(startDate) || p.PartitionDate.After(startDate)) && p.PartitionDate.Before(endDate) { + if (p.PartitionDate.Equal(startDay) || p.PartitionDate.After(startDay)) && + (p.PartitionDate.Equal(endDay) || p.PartitionDate.Before(endDay)) { filtered = append(filtered, p) } } @@ -716,10 +744,14 @@ func (dbc *DB) VerifyPartitionCoverage(tableName string, startDate, endDate time existingDates[dateStr] = true } + // Normalize to calendar days so timestamps within a day are handled correctly + startDay := startDate.Truncate(24 * time.Hour) + endDay := endDate.Truncate(24 * time.Hour) + // Check that we have a partition for each day in the range var missingDates []string - currentDate := startDate - for currentDate.Before(endDate) { + currentDate := startDay + for !currentDate.After(endDay) { dateStr := currentDate.Format("2006-01-02") if !existingDates[dateStr] { missingDates = append(missingDates, dateStr)