Skip to content

[PSL-1262] meta worker improvements for the migration #933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
j-rafique wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[PSL-1262] meta worker improvements for the migration #933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions p2p/kademlia/store/sqlite/meta_worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -539,10 +539,10 @@ func (d *MigrationMetaStore) checkAndExecuteMigration(ctx context.Context) {
log.WithContext(ctx).WithError(err).Error("migration worker: check disk space failed")
}

//if !isLow {
// Disk space is sufficient, stop migration
//return
//}
if !isLow {
//Disk space is sufficient, stop migration
return
}

log.WithContext(ctx).WithField("islow", isLow).Info("Starting data migration")
// Step 1: Fetch pending migrations
Expand Down Expand Up @@ -807,8 +807,7 @@ func (d *MigrationMetaStore) InsertMetaMigrationData(ctx context.Context, migrat
now := time.Now()
for _, key := range keys {
if _, err := stmt.Exec(key, migrationID, now, now); err != nil {
tx.Rollback()
return fmt.Errorf("failed to insert meta migration data: %w", err)
continue
}
}

Expand Down
3 changes: 2 additions & 1 deletion supernode/services/metamigrator/migration_data_identifier.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func (task *MetaMigratorTask) IdentifyMigrationData(ctx context.Context) (err er
log.WithContext(ctx).Info("no stale data found to migrate")
return nil
}
log.WithContext(ctx).WithField("total_keys", totalCount).Info("total-data that needs to migrate has been identified")
log.WithContext(ctx).WithField("total_keys", totalCount).Info("total-data that needs to be migrated has been identified")

numOfBatches := getNumOfBatches(totalCount)
log.WithContext(ctx).WithField("no_of_batches", numOfBatches).Info("batches required to store migration-meta has been calculated")
Expand All @@ -65,6 +65,7 @@ func (task *MetaMigratorTask) IdentifyMigrationData(ctx context.Context) (err er
log.WithContext(ctx).WithField("batch", batchNo).Debug("data added to migration-meta for migration")
}

log.WithContext(ctx).Info("all data that needs to be migrated has been added to meta-migration")
return nil
}

Expand Down
2 changes: 1 addition & 1 deletion supernode/services/metamigrator/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (

const (
logPrefix = "MetaMigrator"
defaultMetaMigratorDataIdentifier = 24 * time.Hour
defaultMetaMigratorDataIdentifier = 120 * time.Minute
lowSpaceThresholdGB = 50 // in GB

)
Expand Down