Skip to content

pdpv0: switch from hard delete to soft delete to debug #825

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Kubuxu wants to merge 1 commit into
base: pdpv0
Choose a base branch
from
Open

pdpv0: switch from hard delete to soft delete to debug #825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions pdp/handlers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -484,6 +484,7 @@ func (p *PDPService) handleGetDataSet(w http.ResponseWriter, r *http.Request) {
var pieces []struct {
PieceID uint64 `db:"piece_id"`
PieceCid string `db:"piece"`
Removed bool `db:"removed"`
SubPieceCID string `db:"sub_piece"`
SubPieceOffset int64 `db:"sub_piece_offset"`
SubPieceSize int64 `db:"sub_piece_size"`
Expand All @@ -494,6 +495,7 @@ func (p *PDPService) handleGetDataSet(w http.ResponseWriter, r *http.Request) {
SELECT
dsp.piece_id,
dsp.piece,
dsp.removed,
dsp.sub_piece,
dsp.sub_piece_offset,
dsp.sub_piece_size,
Expand Down Expand Up @@ -549,8 +551,13 @@ func (p *PDPService) handleGetDataSet(w http.ResponseWriter, r *http.Request) {

aggregatePieceCIDs := make(map[uint64]string)

showRemoved := chi.URLParam(r, "XXXshowRemoved") == "true"
// Convert pieces to the desired JSON format
for _, piece := range pieces {
if !showRemoved && piece.Removed {
continue
}

// Calculate aggregate piece CID on first use for this piece_id
pcv2Str, exists := aggregatePieceCIDs[piece.PieceID]
if !exists {
Expand Down Expand Up @@ -948,6 +955,7 @@ func (p *PDPService) handleDeleteDataSetPiece(w http.ResponseWriter, r *http.Req
log.Errorw("Failed to update rm_message_hash in pdp_data_set_pieces", "dataSetId", dataSetId, "pieceID", pieceID, "error", err)
return false, err
}
log.Infow("scheduled user requested deletion", "dataSetId", dataSetId, "pieceID", pieceID, "txHash", txHashLower)

return true, nil
}, harmonydb.OptionRetry())
Expand Down
6 changes: 5 additions & 1 deletion tasks/pdp/task_prove.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -515,12 +515,13 @@ func (p *ProveTask) provePiece(ctx context.Context, dataSetId int64, pieceId int
SubPiece string `db:"sub_piece"`
SubPieceOffset int64 `db:"sub_piece_offset"` // padded offset
SubPieceSize int64 `db:"sub_piece_size"` // padded piece size
Removed bool `db:"removed"`
}

var subPieces []subPieceMeta

err := p.db.Select(context.Background(), &subPieces, `
SELECT piece, sub_piece, sub_piece_offset, sub_piece_size
SELECT piece, sub_piece, sub_piece_offset, sub_piece_size, removed
FROM pdp_data_set_pieces
WHERE data_set = $1 AND piece_id = $2
ORDER BY sub_piece_offset ASC
Expand All @@ -536,6 +537,9 @@ func (p *ProveTask) provePiece(ctx context.Context, dataSetId int64, pieceId int
if !ok {
return contract.IPDPTypesProof{}, xerrors.New("no subpiece found")
}
if challSubPiece.Removed {
log.Errorw("using removed piece", "dataSetId", dataSetId, "pieceId", pieceId)
}

// build subpiece memtree
memtree, err := p.genSubPieceMemtree(ctx, challSubPiece.SubPiece, abi.PaddedPieceSize(challSubPiece.SubPieceSize))
Expand Down
7 changes: 4 additions & 3 deletions tasks/pdp/watch_piece_delete.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ func processPendingPieceDeletes(ctx context.Context, db *harmonydb.DB, ethClient
// Huston! we have a serious problem
return xerrors.Errorf("piece %d is not scheduled for removal", piece.PieceID)
}
log.Infow("noticed scheduled deletion, marking as removed", "dataSetId", piece.DataSetID, "pieceID", piece.PieceID, "txHash", piece.TxHash)

n, err := db.Exec(ctx, `UPDATE pdp_data_set_pieces
SET removed = TRUE
Expand Down Expand Up @@ -143,8 +144,6 @@ func processPendingCleanup(ctx context.Context, db *harmonydb.DB, ethClient *eth
return nil
}

log.Infof("Cleaning up %d pieces", len(pieces))

pdpAddress := contract.ContractAddresses().PDPVerifier

verifier, err := contract.NewPDPVerifier(pdpAddress, ethClient)
Expand All @@ -159,7 +158,9 @@ func processPendingCleanup(ctx context.Context, db *harmonydb.DB, ethClient *eth
}

if !live {
_, err := db.Exec(ctx, `DELETE FROM pdp_data_set_pieces WHERE data_set = $1 AND piece_id = $2`, piece.DataSetID, piece.PieceID)
// XXX(Kubuxu): commented out as this has lead to proving failures
//_, err := db.Exec(ctx, `DELETE FROM pdp_data_set_pieces WHERE data_set = $1 AND piece_id = $2`, piece.DataSetID, piece.PieceID)
err = nil
if err != nil {
return xerrors.Errorf("failed to delete piece %d: %w", piece.PieceID, err)
}
Expand Down