diff --git a/Makefile b/Makefile index 2c4cc78985..5ce75f561c 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ cdc kafka_consumer storage_consumer pulsar_consumer filter_helper \ prepare_test_binaries \ unit_test_in_verify_ci integration_test_build integration_test_build_fast integration_test_mysql integration_test_kafka integration_test_storage integration_test_pulsar \ + integration_test_weekly_rand_ddl_mysql \ generate-next-gen-grafana check-next-gen-grafana @@ -258,6 +259,9 @@ integration_test_storage: check_third_party_binary integration_test_pulsar: check_third_party_binary tests/integration_tests/run.sh pulsar "$(CASE)" "$(START_AT)" +integration_test_weekly_rand_ddl_mysql: check_third_party_binary + tests/integration_tests/run_weekly_rand_ddl_it_in_ci.sh mysql + unit_test: check_failpoint_ctl generate-protobuf mkdir -p "$(TEST_DIR)" $(FAILPOINT_ENABLE) diff --git a/maintainer/barrier.go b/maintainer/barrier.go index 75e5cc2482..0ee93f7de8 100644 --- a/maintainer/barrier.go +++ b/maintainer/barrier.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/maintainer/operator" + "github.com/pingcap/ticdc/maintainer/replica" "github.com/pingcap/ticdc/maintainer/span" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/messaging" @@ -252,7 +253,7 @@ func (b *Barrier) Resend() []*messaging.TargetMessage { eventList := make([]*BarrierEvent, 0) b.blockedEvents.Range(func(key eventKey, barrierEvent *BarrierEvent) bool { // todo: we can limit the number of messages to send in one round here - msgs = append(msgs, barrierEvent.resend(b.mode)...) + msgs = append(msgs, barrierEvent.resendWithSchedule(b.mode, b.tryScheduleEvent)...) eventList = append(eventList, barrierEvent) return true @@ -308,7 +309,7 @@ func (b *Barrier) handleOneStatus(changefeedID *heartbeatpb.ChangefeedID, status Mode: status.Mode, }) if status.State != nil { - span.UpdateBlockState(*status.State) + updateSpanBlockState(span, status.State) } } if status.State.Stage == heartbeatpb.BlockStage_DONE { @@ -317,6 +318,38 @@ func (b *Barrier) handleOneStatus(changefeedID *heartbeatpb.ChangefeedID, status return b.handleBlockState(cfID, dispatcherID, status) } +func updateSpanBlockState(span *replica.SpanReplication, newState *heartbeatpb.State) { + oldState := span.GetBlockState() + if oldState != nil && compareBlockState(oldState, newState) > 0 { + log.Debug("ignore stale block state", + zap.String("dispatcher", span.ID.String()), + zap.Uint64("oldBlockTs", oldState.BlockTs), + zap.Bool("oldIsSyncPoint", oldState.IsSyncPoint), + zap.String("oldStage", oldState.Stage.String()), + zap.Uint64("newBlockTs", newState.BlockTs), + zap.Bool("newIsSyncPoint", newState.IsSyncPoint), + zap.String("newStage", newState.Stage.String())) + return + } + span.UpdateBlockState(*newState) +} + +func compareBlockState(a, b *heartbeatpb.State) int { + if a.BlockTs < b.BlockTs { + return -1 + } + if a.BlockTs > b.BlockTs { + return 1 + } + if a.IsSyncPoint != b.IsSyncPoint { + if !a.IsSyncPoint && b.IsSyncPoint { + return -1 + } + return 1 + } + return int(a.Stage) - int(b.Stage) +} + func (b *Barrier) handleEventDone(changefeedID common.ChangeFeedID, dispatcherID common.DispatcherID, status *heartbeatpb.TableSpanBlockStatus) *BarrierEvent { key := getEventKey(status.State.BlockTs, status.State.IsSyncPoint) event, ok := b.blockedEvents.Get(key) diff --git a/maintainer/barrier_event.go b/maintainer/barrier_event.go index 431aeac039..92031a504a 100644 --- a/maintainer/barrier_event.go +++ b/maintainer/barrier_event.go @@ -224,11 +224,8 @@ func (be *BarrierEvent) onAllDispatcherReportedBlockEvent(dispatcherID common.Di } // Once the event enters selected state, we start a new reporting phase that - // tracks completion after write/pass rather than the initial WAITING - // coverage. Reset both structures so DONE reports are measured from scratch. - be.rangeChecker.Reset() - be.reportedDispatchers = make(map[common.DispatcherID]struct{}) - + // tracks completion after write/pass rather than the initial WAITING coverage. + be.resetProgressAfterSelection() be.selected.Store(true) be.writerDispatcher = dispatcher be.lastResendTime = time.Now() @@ -325,6 +322,152 @@ func (be *BarrierEvent) addDispatchersToRangeChecker() { } } +func (be *BarrierEvent) ensureRangeChecker() { + if be.rangeChecker != nil || be.blockedDispatchers == nil { + return + } + + switch be.blockedDispatchers.InfluenceType { + case heartbeatpb.InfluenceType_Normal: + if be.dynamicSplitEnabled { + be.rangeChecker = range_checker.NewTableSpanRangeChecker(be.spanController.GetkeyspaceID(), be.blockedDispatchers.TableIDs) + } else { + be.rangeChecker = range_checker.NewTableCountChecker(be.blockedDispatchers.TableIDs) + } + case heartbeatpb.InfluenceType_DB: + be.createRangeCheckerForTypeDB() + case heartbeatpb.InfluenceType_All: + be.createRangeCheckerForTypeAll() + } +} + +func (be *BarrierEvent) getTasksByBlockedTableID(tableID int64) []*replica.SpanReplication { + if tableID != common.DDLSpanTableID { + return be.spanController.GetTasksByTableID(tableID) + } + ddlReplication := be.spanController.GetTaskByID(be.spanController.GetDDLDispatcherID()) + if ddlReplication == nil { + return nil + } + return []*replica.SpanReplication{ddlReplication} +} + +func (be *BarrierEvent) relatedReplications() []*replica.SpanReplication { + if be.blockedDispatchers == nil { + return nil + } + + switch be.blockedDispatchers.InfluenceType { + case heartbeatpb.InfluenceType_Normal: + replications := make([]*replica.SpanReplication, 0, len(be.blockedDispatchers.TableIDs)) + for _, tableID := range be.blockedDispatchers.TableIDs { + replications = append(replications, be.getTasksByBlockedTableID(tableID)...) + } + return replications + case heartbeatpb.InfluenceType_DB: + replications := be.spanController.GetTasksBySchemaID(be.blockedDispatchers.SchemaID) + if ddlReplication := be.spanController.GetTaskByID(be.spanController.GetDDLDispatcherID()); ddlReplication != nil { + replications = append(replications, ddlReplication) + } + return replications + case heartbeatpb.InfluenceType_All: + return be.spanController.GetAllTasks() + } + return nil +} + +func (be *BarrierEvent) addAdvancedReplicationsToRangeChecker() { + if be.rangeChecker == nil { + return + } + + for _, replication := range be.relatedReplications() { + if replication == nil || !forwardBarrierEvent(replication, be) { + continue + } + be.reportedDispatchers[replication.ID] = struct{}{} + be.rangeChecker.AddSubRange(replication.Span.TableID, replication.Span.StartKey, replication.Span.EndKey) + } +} + +func (be *BarrierEvent) refreshSelectedProgress() bool { + be.ensureRangeChecker() + be.addAdvancedReplicationsToRangeChecker() + if be.writerDispatcherAdvanced { + return false + } + + writer := be.spanController.GetTaskByID(be.writerDispatcher) + if writer == nil || !forwardBarrierEvent(writer, be) { + return false + } + if be.needSchedule { + return true + } + be.writerDispatcherAdvanced = true + be.lastResendTime = time.Now().Add(-20 * time.Second) + return true +} + +func (be *BarrierEvent) resetProgressAfterSelection() { + be.ensureRangeChecker() + if be.rangeChecker != nil { + be.rangeChecker.Reset() + } + be.reportedDispatchers = make(map[common.DispatcherID]struct{}) + be.addAdvancedReplicationsToRangeChecker() +} + +func (be *BarrierEvent) selectByForwardedDispatcher() { + be.resetProgressAfterSelection() + be.selected.Store(true) + be.writerDispatcherAdvanced = true + be.passActionSent = false +} + +func (be *BarrierEvent) markMissingDroppedTablesDone() bool { + if be.blockedDispatchers == nil || be.blockedDispatchers.InfluenceType != heartbeatpb.InfluenceType_Normal || + be.dropDispatchers == nil || be.dropDispatchers.InfluenceType != heartbeatpb.InfluenceType_Normal { + return false + } + + be.ensureRangeChecker() + if be.rangeChecker == nil { + return false + } + + marked := false + for _, tableID := range be.dropDispatchers.TableIDs { + if tableID == common.DDLSpanTableID || !containsTableID(be.blockedDispatchers.TableIDs, tableID) { + continue + } + if len(be.spanController.GetTasksByTableID(tableID)) != 0 { + continue + } + if be.spanController.GetTaskByID(be.spanController.GetDDLDispatcherID()) == nil { + continue + } + + be.markTableDone(tableID) + marked = true + log.Info("blocked table has no active dispatcher, mark it done", + zap.String("changefeed", be.cfID.Name()), + zap.Uint64("commitTs", be.commitTs), + zap.Int64("tableID", tableID), + zap.Int64("mode", be.mode)) + } + return marked +} + +func containsTableID(tableIDs []int64, target int64) bool { + for _, tableID := range tableIDs { + if tableID == target { + return true + } + } + return false +} + func (be *BarrierEvent) markDispatcherEventDone(dispatcherID common.DispatcherID) { if be.selected.Load() { // After selection, every accepted status means the chosen write/pass path @@ -480,7 +623,7 @@ func (be *BarrierEvent) sendPassAction(mode int64) []*messaging.TargetMessage { } case heartbeatpb.InfluenceType_Normal: for _, tableID := range be.blockedDispatchers.TableIDs { - spans := be.spanController.GetTasksByTableID(tableID) + spans := be.getTasksByBlockedTableID(tableID) if len(spans) == 0 { be.markTableDone(tableID) } else { @@ -521,17 +664,30 @@ func (be *BarrierEvent) sendPassAction(mode int64) []*messaging.TargetMessage { func (be *BarrierEvent) checkBlockedDispatchers() { switch be.blockedDispatchers.InfluenceType { case heartbeatpb.InfluenceType_Normal: - for _, tableId := range be.blockedDispatchers.TableIDs { - replications := be.spanController.GetTasksByTableID(tableId) + if be.markMissingDroppedTablesDone() && be.allDispatcherReported() { + // A normal DDL barrier can be recreated by a late WAITING status after the + // original barrier has already scheduled the drop and removed the table + // dispatcher. The removed table cannot report again, so advance the + // recreated barrier and let sendPassAction notify the remaining DDL span. + be.selectByForwardedDispatcher() + log.Info("all missing dropped blocked tables are removed, advance block event", + zap.String("changefeed", be.cfID.Name()), + zap.Uint64("commitTs", be.commitTs), + zap.Any("blocker", be.blockedDispatchers), + zap.Int64("mode", be.mode)) + return + } + + for _, tableID := range be.blockedDispatchers.TableIDs { + replications := be.getTasksByBlockedTableID(tableID) for _, replication := range replications { if forwardBarrierEvent(replication, be) { // one related table has forward checkpointTs, means the block event can be advanced - be.selected.Store(true) - be.writerDispatcherAdvanced = true + be.selectByForwardedDispatcher() log.Info("one related dispatcher has forward checkpointTs, means the block event can be advanced", zap.String("changefeed", be.cfID.Name()), zap.Uint64("commitTs", be.commitTs), - zap.Int64("tableId", tableId), + zap.Int64("tableID", tableID), zap.Uint64("checkpointTs", replication.GetStatus().CheckpointTs), zap.String("dispatcher", replication.ID.String()), zap.Int64("mode", be.mode), @@ -546,8 +702,7 @@ func (be *BarrierEvent) checkBlockedDispatchers() { for _, replication := range replications { if forwardBarrierEvent(replication, be) { // One related dispatcher has moved past the barrier, so the block event can advance. - be.selected.Store(true) - be.writerDispatcherAdvanced = true + be.selectByForwardedDispatcher() log.Info("one related dispatcher has forward checkpointTs, means the block event can be advanced", zap.String("changefeed", be.cfID.Name()), zap.Uint64("commitTs", be.commitTs), @@ -564,8 +719,7 @@ func (be *BarrierEvent) checkBlockedDispatchers() { for _, replication := range replications { if forwardBarrierEvent(replication, be) { // One related dispatcher has moved past the barrier, so the block event can advance. - be.selected.Store(true) - be.writerDispatcherAdvanced = true + be.selectByForwardedDispatcher() log.Info("one related dispatcher has forward checkpointTs, means the block event can be advanced", zap.String("changefeed", be.cfID.Name()), zap.Uint64("commitTs", be.commitTs), @@ -582,10 +736,10 @@ func (be *BarrierEvent) checkBlockedDispatchers() { // forwardBarrierEvent returns true if `replication` is known to have passed `event`. // // We intentionally avoid `checkpointTs >= commitTs`: a dispatcher may be recreated with -// `startTs == commitTs` and not skip the syncpoint at that ts, so it may report -// `checkpointTs == commitTs` before the syncpoint is actually flushed. We only forward when the -// replication is strictly beyond the barrier, or when ordering guarantees it (replication is in a -// syncpoint barrier at the same ts while `event` is a DDL barrier). +// `startTs == commitTs` and still need to flush the syncpoint at that ts. We only forward when the +// replication is strictly beyond the barrier, when it already reported DONE for this exact barrier, +// or when ordering guarantees it (replication is in a syncpoint barrier at the same ts while `event` +// is a DDL barrier). func forwardBarrierEvent(replication *replica.SpanReplication, event *BarrierEvent) bool { if replication.GetStatus().CheckpointTs > event.commitTs { return true @@ -596,6 +750,9 @@ func forwardBarrierEvent(replication *replica.SpanReplication, event *BarrierEve if blockState.BlockTs > event.commitTs { return true } else if blockState.BlockTs == event.commitTs { + if blockState.Stage == heartbeatpb.BlockStage_DONE && blockState.IsSyncPoint == event.isSyncPoint { + return true + } // If the replication is already blocked by a syncpoint at the same ts, it must have // processed the DDL barrier at that ts already (barrier events are ordered by (commitTs, isSyncPoint)). if blockState.IsSyncPoint && !event.isSyncPoint { @@ -607,6 +764,10 @@ func forwardBarrierEvent(replication *replica.SpanReplication, event *BarrierEve } func (be *BarrierEvent) resend(mode int64) []*messaging.TargetMessage { + return be.resendWithSchedule(mode, nil) +} + +func (be *BarrierEvent) resendWithSchedule(mode int64, trySchedule func(*BarrierEvent) bool) []*messaging.TargetMessage { now := time.Now() if now.Sub(be.lastResendTime) < time.Second { return nil @@ -657,6 +818,12 @@ func (be *BarrierEvent) resend(mode int64) []*messaging.TargetMessage { be.checkBlockedDispatchers() return nil } + writerForwarded := be.refreshSelectedProgress() + if writerForwarded && be.needSchedule && !be.writerDispatcherAdvanced { + if trySchedule == nil || !trySchedule(be) { + return nil + } + } // we select a dispatcher as the writer, still waiting for that dispatcher advance its checkpoint ts if !be.writerDispatcherAdvanced { be.lastResendTime = now @@ -684,7 +851,7 @@ func (be *BarrierEvent) resend(mode int64) []*messaging.TargetMessage { } tableID := be.blockedDispatchers.TableIDs[0] - replications := be.spanController.GetTasksByTableID(tableID) + replications := be.getTasksByBlockedTableID(tableID) if len(replications) == 0 { log.Panic("replications for this block event should not be empty", diff --git a/maintainer/barrier_test.go b/maintainer/barrier_test.go index 5e982762a5..de06d14ce8 100644 --- a/maintainer/barrier_test.go +++ b/maintainer/barrier_test.go @@ -826,6 +826,491 @@ func TestSyncPointBlock(t *testing.T) { require.Len(t, barrier.blockedEvents.m, 0) } +func TestSyncPointBarrierRecreatedCountsAlreadyDoneDispatchers(t *testing.T) { + testutil.SetUpTestServices(t) + tableTriggerEventDispatcherID := common.NewDispatcherID() + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + ddlSpan := replica.NewWorkingSpanReplication(cfID, tableTriggerEventDispatcherID, + common.DDLSpanSchemaID, + common.KeyspaceDDLSpan(common.DefaultKeyspaceID), &heartbeatpb.TableSpanStatus{ + ID: tableTriggerEventDispatcherID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 1, + }, "node1", false) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) + + spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) + spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 2}, 1) + absents := spanController.GetAbsentForTest(10000) + require.Len(t, absents, 2) + for _, stm := range absents { + spanController.BindSpanToNode("", "node1", stm) + spanController.MarkSpanReplicating(stm) + } + + barrier := NewBarrier(spanController, operatorController, false, nil, common.DefaultMode) + waitingState := &heartbeatpb.State{ + IsBlocked: true, + BlockTs: 10, + Stage: heartbeatpb.BlockStage_WAITING, + BlockTables: &heartbeatpb.InfluencedTables{ + InfluenceType: heartbeatpb.InfluenceType_All, + }, + IsSyncPoint: true, + } + + msgs := barrier.HandleStatus("node1", &heartbeatpb.BlockStatusRequest{ + ChangefeedID: cfID.ToPB(), + BlockStatuses: []*heartbeatpb.TableSpanBlockStatus{ + {ID: spanController.GetDDLDispatcherID().ToPB(), State: waitingState}, + {ID: absents[0].ID.ToPB(), State: waitingState}, + {ID: absents[1].ID.ToPB(), State: waitingState}, + }, + }) + require.NotEmpty(t, msgs) + key := getEventKey(10, true) + event := barrier.blockedEvents.m[key] + require.NotNil(t, event) + require.True(t, event.selected.Load()) + + doneState := &heartbeatpb.State{ + IsBlocked: true, + BlockTs: 10, + Stage: heartbeatpb.BlockStage_DONE, + IsSyncPoint: true, + } + _ = barrier.HandleStatus("node1", &heartbeatpb.BlockStatusRequest{ + ChangefeedID: cfID.ToPB(), + BlockStatuses: []*heartbeatpb.TableSpanBlockStatus{ + {ID: spanController.GetDDLDispatcherID().ToPB(), State: doneState}, + {ID: absents[0].ID.ToPB(), State: doneState}, + {ID: absents[1].ID.ToPB(), State: doneState}, + }, + }) + require.Len(t, barrier.blockedEvents.m, 0) + + // A late WAITING report for the same syncpoint can recreate the barrier after + // the first one was removed. The recreated event must still count dispatchers + // whose span state already says DONE for this exact syncpoint. + _ = barrier.HandleStatus("node1", &heartbeatpb.BlockStatusRequest{ + ChangefeedID: cfID.ToPB(), + BlockStatuses: []*heartbeatpb.TableSpanBlockStatus{ + {ID: absents[1].ID.ToPB(), State: waitingState}, + }, + }) + event = barrier.blockedEvents.m[key] + require.NotNil(t, event) + require.Nil(t, event.rangeChecker) + + absents[1].UpdateStatus(&heartbeatpb.TableSpanStatus{ + ID: absents[1].ID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 11, + }) + + resendMsgs := barrier.Resend() + require.Empty(t, resendMsgs) + require.Len(t, barrier.blockedEvents.m, 0) +} + +func TestNormalBarrierRecreatedAfterDroppedTableRemoved(t *testing.T) { + testutil.SetUpTestServices(t) + tableTriggerEventDispatcherID := common.NewDispatcherID() + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + ddlSpan := replica.NewWorkingSpanReplication(cfID, tableTriggerEventDispatcherID, + common.DDLSpanSchemaID, + common.KeyspaceDDLSpan(common.DefaultKeyspaceID), &heartbeatpb.TableSpanStatus{ + ID: tableTriggerEventDispatcherID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 1, + }, "node1", false) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) + + oldTableID := int64(267) + spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: oldTableID}, 1) + oldReplication := spanController.GetTasksByTableID(oldTableID)[0] + spanController.BindSpanToNode("", "node1", oldReplication) + spanController.MarkSpanReplicating(oldReplication) + + barrier := NewBarrier(spanController, operatorController, false, nil, common.DefaultMode) + dropState := &heartbeatpb.State{ + IsBlocked: true, + BlockTs: 10, + Stage: heartbeatpb.BlockStage_WAITING, + BlockTables: &heartbeatpb.InfluencedTables{ + InfluenceType: heartbeatpb.InfluenceType_Normal, + TableIDs: []int64{oldTableID, common.DDLSpanTableID}, + }, + NeedDroppedTables: &heartbeatpb.InfluencedTables{ + InfluenceType: heartbeatpb.InfluenceType_Normal, + TableIDs: []int64{oldTableID}, + }, + } + + // The original barrier has already scheduled the DROP TABLE and removed the + // table dispatcher. A late WAITING report from the DDL dispatcher recreates + // the same barrier and must not wait forever for the removed table dispatcher. + spanController.RemoveByTableIDs(oldTableID) + msgs := barrier.HandleStatus("node1", &heartbeatpb.BlockStatusRequest{ + ChangefeedID: cfID.ToPB(), + BlockStatuses: []*heartbeatpb.TableSpanBlockStatus{ + {ID: spanController.GetDDLDispatcherID().ToPB(), State: dropState}, + }, + }) + require.NotEmpty(t, msgs) + key := getEventKey(10, false) + event := barrier.blockedEvents.m[key] + require.NotNil(t, event) + require.False(t, event.selected.Load()) + + resendMsgs := barrier.Resend() + require.Empty(t, resendMsgs) + event = barrier.blockedEvents.m[key] + require.NotNil(t, event) + require.True(t, event.selected.Load()) + require.True(t, event.writerDispatcherAdvanced) + + resendMsgs = barrier.Resend() + require.Len(t, resendMsgs, 1) + resp := resendMsgs[0].Message[0].(*heartbeatpb.HeartBeatResponse) + require.Len(t, resp.DispatcherStatuses, 1) + require.Equal(t, heartbeatpb.Action_Pass, resp.DispatcherStatuses[0].Action.Action) + require.Equal(t, uint64(10), resp.DispatcherStatuses[0].Action.CommitTs) + require.Len(t, resp.DispatcherStatuses[0].InfluencedDispatchers.DispatcherIDs, 1) + require.Equal(t, spanController.GetDDLDispatcherID().ToPB(), resp.DispatcherStatuses[0].InfluencedDispatchers.DispatcherIDs[0]) + + doneState := &heartbeatpb.State{IsBlocked: true, BlockTs: 10, Stage: heartbeatpb.BlockStage_DONE} + _ = barrier.HandleStatus("node1", &heartbeatpb.BlockStatusRequest{ + ChangefeedID: cfID.ToPB(), + BlockStatuses: []*heartbeatpb.TableSpanBlockStatus{ + {ID: spanController.GetDDLDispatcherID().ToPB(), State: doneState}, + }, + }) + require.Len(t, barrier.blockedEvents.m, 0) +} + +func TestNormalBarrierDoesNotCoverMissingNonDroppedTable(t *testing.T) { + testutil.SetUpTestServices(t) + tableTriggerEventDispatcherID := common.NewDispatcherID() + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + ddlSpan := replica.NewWorkingSpanReplication(cfID, tableTriggerEventDispatcherID, + common.DDLSpanSchemaID, + common.KeyspaceDDLSpan(common.DefaultKeyspaceID), &heartbeatpb.TableSpanStatus{ + ID: tableTriggerEventDispatcherID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 1, + }, "node1", false) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) + + missingTableID := int64(267) + spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: missingTableID}, 1) + missingReplication := spanController.GetTasksByTableID(missingTableID)[0] + spanController.BindSpanToNode("", "node1", missingReplication) + spanController.MarkSpanReplicating(missingReplication) + spanController.RemoveByTableIDs(missingTableID) + + barrier := NewBarrier(spanController, operatorController, false, nil, common.DefaultMode) + alterState := &heartbeatpb.State{ + IsBlocked: true, + BlockTs: 10, + Stage: heartbeatpb.BlockStage_WAITING, + BlockTables: &heartbeatpb.InfluencedTables{ + InfluenceType: heartbeatpb.InfluenceType_Normal, + TableIDs: []int64{missingTableID, common.DDLSpanTableID}, + }, + } + msgs := barrier.HandleStatus("node1", &heartbeatpb.BlockStatusRequest{ + ChangefeedID: cfID.ToPB(), + BlockStatuses: []*heartbeatpb.TableSpanBlockStatus{ + {ID: spanController.GetDDLDispatcherID().ToPB(), State: alterState}, + }, + }) + require.NotEmpty(t, msgs) + key := getEventKey(10, false) + event := barrier.blockedEvents.m[key] + require.NotNil(t, event) + require.False(t, event.selected.Load()) + + resendMsgs := barrier.Resend() + require.Empty(t, resendMsgs) + event = barrier.blockedEvents.m[key] + require.NotNil(t, event) + require.False(t, event.selected.Load()) + require.Contains(t, event.rangeChecker.Detail(), "267") +} + +func TestNormalBarrierUsesDDLDispatcherForDDLSpanTableID(t *testing.T) { + testutil.SetUpTestServices(t) + tableTriggerEventDispatcherID := common.NewDispatcherID() + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + ddlSpan := replica.NewWorkingSpanReplication(cfID, tableTriggerEventDispatcherID, + common.DDLSpanSchemaID, + common.KeyspaceDDLSpan(common.DefaultKeyspaceID), &heartbeatpb.TableSpanStatus{ + ID: tableTriggerEventDispatcherID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 11, + }, "node1", false) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) + + spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) + tableReplication := spanController.GetTasksByTableID(1)[0] + spanController.BindSpanToNode("", "node1", tableReplication) + spanController.MarkSpanReplicating(tableReplication) + + state := &heartbeatpb.State{ + IsBlocked: true, + BlockTs: 10, + Stage: heartbeatpb.BlockStage_WAITING, + BlockTables: &heartbeatpb.InfluencedTables{ + InfluenceType: heartbeatpb.InfluenceType_Normal, + TableIDs: []int64{1, common.DDLSpanTableID}, + }, + } + event := NewBlockEvent(cfID, tableReplication.ID, spanController, operatorController, state, false, common.DefaultMode) + require.False(t, event.selected.Load()) + require.False(t, event.allDispatcherReported()) + + event.checkBlockedDispatchers() + require.True(t, event.selected.Load()) + require.True(t, event.writerDispatcherAdvanced) + + event.lastResendTime = time.Now().Add(-2 * time.Second) + msgs := event.resend(common.DefaultMode) + require.Len(t, msgs, 1) + resp := msgs[0].Message[0].(*heartbeatpb.HeartBeatResponse) + require.Len(t, resp.DispatcherStatuses, 1) + status := resp.DispatcherStatuses[0] + require.Equal(t, heartbeatpb.Action_Pass, status.Action.Action) + require.Equal(t, uint64(10), status.Action.CommitTs) + require.Len(t, status.InfluencedDispatchers.DispatcherIDs, 2) + + gotDispatchers := make(map[common.DispatcherID]struct{}, len(status.InfluencedDispatchers.DispatcherIDs)) + for _, dispatcherID := range status.InfluencedDispatchers.DispatcherIDs { + gotDispatchers[common.NewDispatcherIDFromPB(dispatcherID)] = struct{}{} + } + _, ok := gotDispatchers[tableReplication.ID] + require.True(t, ok) + _, ok = gotDispatchers[spanController.GetDDLDispatcherID()] + require.True(t, ok) +} + +func TestResendSchedulesForwardedNeedScheduleBarrierBeforePass(t *testing.T) { + testutil.SetUpTestServices(t) + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + nodeManager.GetAliveNodes()["node1"] = &node.Info{ID: "node1"} + + tableTriggerEventDispatcherID := common.NewDispatcherID() + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + ddlSpan := replica.NewWorkingSpanReplication(cfID, tableTriggerEventDispatcherID, + common.DDLSpanSchemaID, + common.KeyspaceDDLSpan(common.DefaultKeyspaceID), &heartbeatpb.TableSpanStatus{ + ID: tableTriggerEventDispatcherID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 20, + }, "node1", false) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) + barrier := NewBarrier(spanController, operatorController, false, nil, common.DefaultMode) + + event := NewBlockEvent(cfID, tableTriggerEventDispatcherID, spanController, operatorController, &heartbeatpb.State{ + IsBlocked: true, + BlockTs: 10, + Stage: heartbeatpb.BlockStage_WAITING, + BlockTables: &heartbeatpb.InfluencedTables{ + InfluenceType: heartbeatpb.InfluenceType_Normal, + TableIDs: []int64{common.DDLSpanTableID}, + }, + NeedAddedTables: []*heartbeatpb.Table{{SchemaID: 1, TableID: 2}}, + }, false, common.DefaultMode) + event.selected.Store(true) + event.writerDispatcher = tableTriggerEventDispatcherID + event.lastResendTime = time.Now().Add(-2 * time.Second) + barrier.blockedEvents.Set(getEventKey(10, false), event) + barrier.pendingEvents.add(event) + + msgs := barrier.Resend() + require.NotEmpty(t, msgs) + require.True(t, event.writerDispatcherAdvanced) + require.Equal(t, 0, barrier.pendingEvents.Len()) + require.Equal(t, 1, spanController.GetAbsentSize()) + resp := msgs[0].Message[0].(*heartbeatpb.HeartBeatResponse) + require.Equal(t, heartbeatpb.Action_Pass, resp.DispatcherStatuses[0].Action.Action) +} + +func TestSelectedBarrierRefreshesAdvancedReplications(t *testing.T) { + testutil.SetUpTestServices(t) + tableTriggerEventDispatcherID := common.NewDispatcherID() + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + ddlSpan := replica.NewWorkingSpanReplication(cfID, tableTriggerEventDispatcherID, + common.DDLSpanSchemaID, + common.KeyspaceDDLSpan(common.DefaultKeyspaceID), &heartbeatpb.TableSpanStatus{ + ID: tableTriggerEventDispatcherID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 1, + }, "node1", false) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) + + spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) + spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 2}, 1) + absents := spanController.GetAbsentForTest(10000) + require.Len(t, absents, 2) + for _, stm := range absents { + spanController.BindSpanToNode("", "node1", stm) + spanController.MarkSpanReplicating(stm) + } + + event := NewBlockEvent(cfID, tableTriggerEventDispatcherID, spanController, operatorController, &heartbeatpb.State{ + IsBlocked: true, + BlockTs: 10, + Stage: heartbeatpb.BlockStage_WAITING, + BlockTables: &heartbeatpb.InfluencedTables{ + InfluenceType: heartbeatpb.InfluenceType_All, + }, + IsSyncPoint: true, + }, false, common.DefaultMode) + event.ensureRangeChecker() + event.selected.Store(true) + event.writerDispatcher = tableTriggerEventDispatcherID + event.lastResendTime = time.Now().Add(-2 * time.Second) + + for _, replication := range event.relatedReplications() { + replication.UpdateBlockState(heartbeatpb.State{ + IsBlocked: true, + BlockTs: 20, + Stage: heartbeatpb.BlockStage_WAITING, + }) + } + + _ = event.resend(common.DefaultMode) + require.True(t, event.writerDispatcherAdvanced) + require.True(t, event.allDispatcherReported()) +} + +func TestUpdateSpanBlockStateSkipsStaleState(t *testing.T) { + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + dispatcherID := common.NewDispatcherID() + tableSpan := common.TableIDToComparableSpan(common.DefaultKeyspaceID, 1) + replication := replica.NewWorkingSpanReplication(cfID, dispatcherID, 1, &tableSpan, &heartbeatpb.TableSpanStatus{ + ID: dispatcherID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 1, + }, "node1", false) + + updateSpanBlockState(replication, &heartbeatpb.State{BlockTs: 10, IsSyncPoint: true, Stage: heartbeatpb.BlockStage_DONE}) + updateSpanBlockState(replication, &heartbeatpb.State{BlockTs: 10, IsSyncPoint: true, Stage: heartbeatpb.BlockStage_WAITING}) + state := replication.GetBlockState() + require.Equal(t, uint64(10), state.BlockTs) + require.True(t, state.IsSyncPoint) + require.Equal(t, heartbeatpb.BlockStage_DONE, state.Stage) + + updateSpanBlockState(replication, &heartbeatpb.State{BlockTs: 9, Stage: heartbeatpb.BlockStage_WAITING}) + state = replication.GetBlockState() + require.Equal(t, uint64(10), state.BlockTs) + require.True(t, state.IsSyncPoint) + require.Equal(t, heartbeatpb.BlockStage_DONE, state.Stage) + + updateSpanBlockState(replication, &heartbeatpb.State{BlockTs: 11, Stage: heartbeatpb.BlockStage_WAITING}) + state = replication.GetBlockState() + require.Equal(t, uint64(11), state.BlockTs) + require.False(t, state.IsSyncPoint) + require.Equal(t, heartbeatpb.BlockStage_WAITING, state.Stage) +} + +func TestForwardBarrierEventBoundaries(t *testing.T) { + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + tableSpan := common.TableIDToComparableSpan(common.DefaultKeyspaceID, 1) + newReplication := func(checkpointTs uint64, blockState *heartbeatpb.State) *replica.SpanReplication { + dispatcherID := common.NewDispatcherID() + replication := replica.NewWorkingSpanReplication(cfID, dispatcherID, + 1, &tableSpan, &heartbeatpb.TableSpanStatus{ + ID: dispatcherID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: checkpointTs, + }, "node1", false) + if blockState != nil { + replication.UpdateBlockState(*blockState) + } + return replication + } + + ddlEvent := &BarrierEvent{commitTs: 10, isSyncPoint: false} + syncpointEvent := &BarrierEvent{commitTs: 10, isSyncPoint: true} + tests := []struct { + name string + checkpointTs uint64 + blockState *heartbeatpb.State + event *BarrierEvent + want bool + }{ + { + name: "checkpoint equal commit ts does not forward syncpoint", + checkpointTs: 10, + event: syncpointEvent, + want: false, + }, + { + name: "checkpoint greater than commit ts forwards syncpoint", + checkpointTs: 11, + event: syncpointEvent, + want: true, + }, + { + name: "same ts syncpoint waiting does not forward syncpoint", + checkpointTs: 9, + blockState: &heartbeatpb.State{BlockTs: 10, IsSyncPoint: true, Stage: heartbeatpb.BlockStage_WAITING}, + event: syncpointEvent, + want: false, + }, + { + name: "same ts ddl done does not forward syncpoint", + checkpointTs: 9, + blockState: &heartbeatpb.State{BlockTs: 10, IsSyncPoint: false, Stage: heartbeatpb.BlockStage_DONE}, + event: syncpointEvent, + want: false, + }, + { + name: "same ts syncpoint done forwards syncpoint", + checkpointTs: 9, + blockState: &heartbeatpb.State{BlockTs: 10, IsSyncPoint: true, Stage: heartbeatpb.BlockStage_DONE}, + event: syncpointEvent, + want: true, + }, + { + name: "same ts syncpoint waiting forwards ddl", + checkpointTs: 9, + blockState: &heartbeatpb.State{BlockTs: 10, IsSyncPoint: true, Stage: heartbeatpb.BlockStage_WAITING}, + event: ddlEvent, + want: true, + }, + { + name: "same ts ddl done forwards ddl", + checkpointTs: 9, + blockState: &heartbeatpb.State{BlockTs: 10, IsSyncPoint: false, Stage: heartbeatpb.BlockStage_DONE}, + event: ddlEvent, + want: true, + }, + { + name: "later normal waiting forwards syncpoint", + checkpointTs: 9, + blockState: &heartbeatpb.State{BlockTs: 11, IsSyncPoint: false, Stage: heartbeatpb.BlockStage_WAITING}, + event: syncpointEvent, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + replication := newReplication(tt.checkpointTs, tt.blockState) + require.Equal(t, tt.want, forwardBarrierEvent(replication, tt.event)) + }) + } +} + func TestNonBlocked(t *testing.T) { testutil.SetUpTestServices(t) tableTriggerEventDispatcherID := common.NewDispatcherID() diff --git a/maintainer/operator/operator_move.go b/maintainer/operator/operator_move.go index d45dec35bf..ea7634351f 100644 --- a/maintainer/operator/operator_move.go +++ b/maintainer/operator/operator_move.go @@ -118,6 +118,7 @@ func (m *MoveDispatcherOperator) Check(from node.ID, status *heartbeatpb.TableSp if from == m.origin && status.ComponentStatus != heartbeatpb.ComponentState_Working { log.Info("replica set removed from origin node", zap.String("replicaSet", m.replicaSet.ID.String())) + m.replicaSet.UpdateStatus(status) // reset last send message time m.sendThrottler.reset() diff --git a/maintainer/operator/operator_move_test.go b/maintainer/operator/operator_move_test.go index 60558a9a07..7336606a4e 100644 --- a/maintainer/operator/operator_move_test.go +++ b/maintainer/operator/operator_move_test.go @@ -243,6 +243,26 @@ func TestMoveOperator_OriginNodeRemovedAfterOriginStopped(t *testing.T) { require.True(t, op.IsFinished()) } +func TestMoveOperatorUsesStoppedCheckpointWhenAddingDest(t *testing.T) { + spanController, _, replicaSet, nodeA, nodeB := setupTestEnvironment(t) + op := NewMoveDispatcherOperator(spanController, replicaSet, nodeA, nodeB) + + op.Start() + stoppedStatus := &heartbeatpb.TableSpanStatus{ + ID: replicaSet.ID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Stopped, + CheckpointTs: 1500, + } + op.Check(nodeA, stoppedStatus) + require.Equal(t, moveStateAddDest, op.state) + + msg := op.Schedule() + require.NotNil(t, msg) + req := msg.Message[0].(*heartbeatpb.ScheduleDispatcherRequest) + require.Equal(t, heartbeatpb.ScheduleAction_Create, req.ScheduleAction) + require.Equal(t, uint64(1500), req.Config.StartTs) +} + func TestMoveOperator_BothNodesRemovedBeforeStartDoesNotLeaveSchedulingWithoutNodeID(t *testing.T) { messageCenter, _, _ := messaging.NewMessageCenterForTest(t) appcontext.SetService(appcontext.MessageCenter, messageCenter) diff --git a/maintainer/operator/operator_remove.go b/maintainer/operator/operator_remove.go index 43a95c21c6..6dff5387e6 100644 --- a/maintainer/operator/operator_remove.go +++ b/maintainer/operator/operator_remove.go @@ -97,6 +97,7 @@ func (m *removeDispatcherOperator) Check(from node.ID, status *heartbeatpb.Table (status.ComponentStatus == heartbeatpb.ComponentState_Stopped || status.ComponentStatus == heartbeatpb.ComponentState_Removed) { m.replicaSet.UpdateStatus(status) + m.spanController.RecordRemovedSpanCheckpoint(m.replicaSet, status.CheckpointTs) log.Info("dispatcher report non-working status", zap.String("replicaSet", m.replicaSet.ID.String())) m.finished.Store(true) diff --git a/maintainer/span/span_controller.go b/maintainer/span/span_controller.go index 40fcea4d47..5fca6b7d13 100644 --- a/maintainer/span/span_controller.go +++ b/maintainer/span/span_controller.go @@ -70,6 +70,10 @@ type Controller struct { tableTasks map[int64]map[common.DispatcherID]*replica.SpanReplication // nonReplicatingCheckpointTs tracks absent and scheduling spans so checkpoint calculation does not scan all spans. nonReplicatingCheckpointTs *checkpointTsTracker + // removedTableCheckpointTs records the highest checkpoint reported by removed dispatchers for each table. + // It prevents a stale add-table barrier from recreating a dispatcher below events already flushed by + // the previous dispatcher of the same physical table. + removedTableCheckpointTs map[int64]uint64 // newGroupChecker creates a GroupChecker for validating span groups newGroupChecker func(groupID pkgreplica.GroupID) pkgreplica.GroupChecker[common.DispatcherID, *replica.SpanReplication] @@ -115,6 +119,7 @@ func NewController( tableTasks: make(map[int64]map[common.DispatcherID]*replica.SpanReplication), allTasks: make(map[common.DispatcherID]*replica.SpanReplication), nonReplicatingCheckpointTs: newCheckpointTsTracker(), + removedTableCheckpointTs: make(map[int64]uint64), } c.ReplicationDB = pkgreplica.NewReplicationDB(changefeedID.String(), c.doWithRLock, c.newGroupChecker) c.initializeDDLSpan(ddlSpan) @@ -227,11 +232,47 @@ func (c *Controller) AddNewSpans(schemaID int64, tableSpans []*heartbeatpb.Table for _, span := range tableSpans { dispatcherID := common.NewDispatcherID() span.KeyspaceID = c.GetkeyspaceID() - replicaSet := replica.NewSpanReplication(c.changefeedID, dispatcherID, schemaID, span, startTs, c.mode, enabledSplit) + safeStartTs := c.getSafeStartTsForTable(span.TableID, startTs) + replicaSet := replica.NewSpanReplication(c.changefeedID, dispatcherID, schemaID, span, safeStartTs, c.mode, enabledSplit) c.AddAbsentReplicaSet(replicaSet) } } +// RecordRemovedSpanCheckpoint records the final checkpoint of a dispatcher that has been removed. +func (c *Controller) RecordRemovedSpanCheckpoint(span *replica.SpanReplication, checkpointTs uint64) { + if span == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.advanceRemovedTableCheckpointTsWithoutLock(span.Span.TableID, checkpointTs) +} + +func (c *Controller) getSafeStartTsForTable(tableID int64, startTs uint64) uint64 { + c.mu.RLock() + removedCheckpointTs := c.removedTableCheckpointTs[tableID] + c.mu.RUnlock() + if removedCheckpointTs <= startTs { + return startTs + } + log.Info("clamp new table dispatcher start ts to removed table checkpoint", + zap.Stringer("changefeedID", c.changefeedID), + zap.Int64("tableID", tableID), + zap.Uint64("originalStartTs", startTs), + zap.Uint64("removedCheckpointTs", removedCheckpointTs)) + return removedCheckpointTs +} + +func (c *Controller) advanceRemovedTableCheckpointTsWithoutLock(tableID int64, checkpointTs uint64) { + if checkpointTs == 0 { + return + } + if old := c.removedTableCheckpointTs[tableID]; old >= checkpointTs { + return + } + c.removedTableCheckpointTs[tableID] = checkpointTs +} + func (c *Controller) GetMinCheckpointTsForNonReplicatingSpans(minCheckpointTs uint64) uint64 { c.mu.RLock() defer c.mu.RUnlock() @@ -590,6 +631,9 @@ func (c *Controller) RemoveBySchemaID(schemaID int64) { // removeSpanWithoutLock removes the spans from the db without lock func (c *Controller) removeSpanWithoutLock(spans ...*replica.SpanReplication) { for _, span := range spans { + if status := span.GetStatus(); status != nil { + c.advanceRemovedTableCheckpointTsWithoutLock(span.Span.TableID, status.CheckpointTs) + } c.RemoveReplicaWithoutLock(span) c.untrackNonReplicatingSpan(span) diff --git a/maintainer/span/span_controller_test.go b/maintainer/span/span_controller_test.go index 827f175e12..7357fee943 100644 --- a/maintainer/span/span_controller_test.go +++ b/maintainer/span/span_controller_test.go @@ -610,6 +610,63 @@ func TestMarkSpanAbsent(t *testing.T) { require.Equal(t, "", replicaSpan.GetNodeID().String()) } +func TestControllerAddNewTableClampsToRemovedTableCheckpoint(t *testing.T) { + controller := newControllerWithCheckerForTest(t) + tableID := int64(100) + oldID := common.NewDispatcherID() + oldSpan := replica.NewWorkingSpanReplication( + controller.changefeedID, + oldID, + 1, + testutil.GetTableSpanByID(tableID), + &heartbeatpb.TableSpanStatus{ + ID: oldID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 1000, + }, + "node1", + false, + ) + + controller.AddReplicatingSpan(oldSpan) + controller.RemoveByTableIDs(tableID) + controller.RecordRemovedSpanCheckpoint(oldSpan, 1500) + + controller.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: tableID}, 900) + tasks := controller.GetTasksByTableID(tableID) + require.Len(t, tasks, 1) + require.Equal(t, uint64(1500), tasks[0].GetStatus().CheckpointTs) + + msg := tasks[0].NewAddDispatcherMessage("node1", heartbeatpb.OperatorType_O_Add) + req := msg.Message[0].(*heartbeatpb.ScheduleDispatcherRequest) + require.Equal(t, uint64(1500), req.Config.StartTs) +} + +func TestControllerAddNewTableIgnoresLowerRemovedTableCheckpoint(t *testing.T) { + controller := newControllerWithCheckerForTest(t) + tableID := int64(101) + oldID := common.NewDispatcherID() + oldSpan := replica.NewWorkingSpanReplication( + controller.changefeedID, + oldID, + 1, + testutil.GetTableSpanByID(tableID), + &heartbeatpb.TableSpanStatus{ + ID: oldID.ToPB(), + ComponentStatus: heartbeatpb.ComponentState_Working, + CheckpointTs: 1000, + }, + "node1", + false, + ) + + controller.RecordRemovedSpanCheckpoint(oldSpan, 800) + controller.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: tableID}, 900) + tasks := controller.GetTasksByTableID(tableID) + require.Len(t, tasks, 1) + require.Equal(t, uint64(900), tasks[0].GetStatus().CheckpointTs) +} + func newControllerWithCheckerForTest(t *testing.T) *Controller { testutil.SetUpTestServices(t) cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) diff --git a/tests/integration_tests/run_weekly_rand_ddl_it_in_ci.sh b/tests/integration_tests/run_weekly_rand_ddl_it_in_ci.sh new file mode 100755 index 0000000000..f6bc101457 --- /dev/null +++ b/tests/integration_tests/run_weekly_rand_ddl_it_in_ci.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -eo pipefail + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +sink_type=${1:-mysql} + +# This script is a standalone CI entry for the random DDL+DML suite. +# It intentionally stays outside run_heavy_it_in_ci.sh so the expensive weekly +# workload can be triggered independently from the regular heavy matrix. +case "${sink_type}" in +mysql) + test_names="weekly_rand_single weekly_rand_multi weekly_rand_multi_failover weekly_rand_slow_lossy_ddl" + ;; +kafka | storage | pulsar) + test_names="weekly_rand_single weekly_rand_multi weekly_rand_multi_failover" + ;; +*) + echo "Error: unknown sink type: ${sink_type}" + exit 1 + ;; +esac + +export TICDC_NEWARCH=true +export RUN_PROFILE=${RUN_PROFILE:-weekly} +export RUN_DURATION=${RUN_DURATION:-30m} +export RUN_CONVERGE_TIMEOUT=${RUN_CONVERGE_TIMEOUT:-120m} +export RUN_SEED=${RUN_SEED:-$(date -u +%Y%m%d%H)} + +echo "Sink Type: ${sink_type}" +echo "Run cases: ${test_names}" +echo "RUN_PROFILE=${RUN_PROFILE}" +echo "RUN_DURATION=${RUN_DURATION}" +echo "RUN_CONVERGE_TIMEOUT=${RUN_CONVERGE_TIMEOUT}" +echo "RUN_SEED=${RUN_SEED}" + +"${CUR}"/run.sh "${sink_type}" "${test_names}" diff --git a/tests/integration_tests/weekly_rand_multi/conf/changefeed.toml b/tests/integration_tests/weekly_rand_multi/conf/changefeed.toml new file mode 100644 index 0000000000..16252a3997 --- /dev/null +++ b/tests/integration_tests/weekly_rand_multi/conf/changefeed.toml @@ -0,0 +1,4 @@ +[scheduler] +enable-table-across-nodes=true +region-threshold=10 +region-count-per-span=10 diff --git a/tests/integration_tests/weekly_rand_multi/conf/changefeed_mysql.toml b/tests/integration_tests/weekly_rand_multi/conf/changefeed_mysql.toml new file mode 100644 index 0000000000..b1a3308066 --- /dev/null +++ b/tests/integration_tests/weekly_rand_multi/conf/changefeed_mysql.toml @@ -0,0 +1,7 @@ +enable-sync-point = true +sync-point-interval = "30s" + +[scheduler] +enable-table-across-nodes=true +region-threshold=10 +region-count-per-span=10 diff --git a/tests/integration_tests/weekly_rand_multi/conf/consumer.toml b/tests/integration_tests/weekly_rand_multi/conf/consumer.toml new file mode 100644 index 0000000000..28f76a83b9 --- /dev/null +++ b/tests/integration_tests/weekly_rand_multi/conf/consumer.toml @@ -0,0 +1,2 @@ +[scheduler] +enable-table-across-nodes=true diff --git a/tests/integration_tests/weekly_rand_multi/run.sh b/tests/integration_tests/weekly_rand_multi/run.sh new file mode 100644 index 0000000000..715c10e559 --- /dev/null +++ b/tests/integration_tests/weekly_rand_multi/run.sh @@ -0,0 +1,204 @@ +#!/bin/bash + +# Random DDL+DML weekly smoke test (3 captures, scheduler enabled). +# +# Timeline (high level): +# 1) start_tidb_cluster: start upstream + downstream TiDB. +# 2) random_ddl_test_runner bootstrap: create identical schemas and deterministic seed data on both sides. +# 3) start TiCDC (3 captures) and create changefeed with scheduler settings. +# 4) random_ddl_test_runner workload: concurrent random DML + random DDL on upstream. +# 5) Final diff: sync_diff_inspector compares upstream vs downstream using diff_config.toml generated by the runner. +# 6) Post-check: scan logs for panic/fatal/data race patterns. +# +# Notes: +# - Storage sink: pre-create directories to avoid path creation races under multi-capture runs. +# +# Sequence diagram (simplified): +# run.sh +# |-> start_tidb_cluster +# |-> random_ddl_test_runner --phase bootstrap +# |-> run_cdc_server (capture=3) +# |-> cdc_cli_changefeed create (scheduler enabled) +# |-> consumer (kafka/storage/pulsar) [optional] +# |-> random_ddl_test_runner --phase workload +# |-> check_sync_diff (sync_diff_inspector) + +set -eu + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $CUR/../_utils/test_prepare + +WORK_DIR=$OUT_DIR/$TEST_NAME +CDC_BINARY=cdc.test +SINK_TYPE=$1 +CHANGEFEED_ID="weeklyrand" + +RUN_SEED=${RUN_SEED:-1} +RUN_PROFILE=${RUN_PROFILE:-smoke} +RUN_DURATION=${RUN_DURATION:-3m} +RUN_CONVERGE_TIMEOUT=${RUN_CONVERGE_TIMEOUT:-} +if [ -z "$RUN_CONVERGE_TIMEOUT" ]; then + if [ "$RUN_PROFILE" == "weekly" ]; then + RUN_CONVERGE_TIMEOUT="120m" + else + RUN_CONVERGE_TIMEOUT="30m" + fi +fi + +function build_runner() { + mkdir -p "$WORK_DIR" + go build -o "$WORK_DIR/random_ddl_test_runner" "$CUR/../../utils/random_ddl_test_runner" +} + +function write_runner_config() { + local mysql_sync_enabled="false" + if [ "$SINK_TYPE" == "mysql" ]; then + mysql_sync_enabled="true" + fi + + cat >"$WORK_DIR/runner_config.json" </dev/null 2>&1 || true + cleanup_process cdc_kafka_consumer >/dev/null 2>&1 || true + cleanup_process cdc_storage_consumer >/dev/null 2>&1 || true + cleanup_process cdc_pulsar_consumer >/dev/null 2>&1 || true + stop_test $WORK_DIR +} + +trap 'cleanup' EXIT + +rm -rf $WORK_DIR && mkdir -p $WORK_DIR + +# start_tidb_cluster --workdir $WORK_DIR +cat >"$WORK_DIR/tidb_config.toml" </dev/null 2>&1; then + if rg -n -i "panic|fatal|data race" "$WORK_DIR"/runner.log "$WORK_DIR"/ddl_trace.log "$WORK_DIR"/stdout*.log "$WORK_DIR"/cdc*.log "$WORK_DIR"/cdc_*_consumer*.log "$WORK_DIR"/cdc_*_consumer_stdout*.log 2>/dev/null | head -n 20 | rg -n . >/dev/null 2>&1; then + echo "log scan: panic/fatal/race detected" + rg -n -i "panic|fatal|data race" "$WORK_DIR"/runner.log "$WORK_DIR"/ddl_trace.log "$WORK_DIR"/stdout*.log "$WORK_DIR"/cdc*.log "$WORK_DIR"/cdc_*_consumer*.log "$WORK_DIR"/cdc_*_consumer_stdout*.log 2>/dev/null | head -n 50 || true + exit 1 + fi +fi + +echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" diff --git a/tests/integration_tests/weekly_rand_multi_failover/conf/changefeed.toml b/tests/integration_tests/weekly_rand_multi_failover/conf/changefeed.toml new file mode 100644 index 0000000000..1ba6b2c025 --- /dev/null +++ b/tests/integration_tests/weekly_rand_multi_failover/conf/changefeed.toml @@ -0,0 +1,5 @@ +[scheduler] +enable-table-across-nodes=false + +[sink.csv] +include-commit-ts = true diff --git a/tests/integration_tests/weekly_rand_multi_failover/conf/changefeed_mysql.toml b/tests/integration_tests/weekly_rand_multi_failover/conf/changefeed_mysql.toml new file mode 100644 index 0000000000..b1a3308066 --- /dev/null +++ b/tests/integration_tests/weekly_rand_multi_failover/conf/changefeed_mysql.toml @@ -0,0 +1,7 @@ +enable-sync-point = true +sync-point-interval = "30s" + +[scheduler] +enable-table-across-nodes=true +region-threshold=10 +region-count-per-span=10 diff --git a/tests/integration_tests/weekly_rand_multi_failover/conf/consumer.toml b/tests/integration_tests/weekly_rand_multi_failover/conf/consumer.toml new file mode 100644 index 0000000000..1ba6b2c025 --- /dev/null +++ b/tests/integration_tests/weekly_rand_multi_failover/conf/consumer.toml @@ -0,0 +1,5 @@ +[scheduler] +enable-table-across-nodes=false + +[sink.csv] +include-commit-ts = true diff --git a/tests/integration_tests/weekly_rand_multi_failover/run.sh b/tests/integration_tests/weekly_rand_multi_failover/run.sh new file mode 100644 index 0000000000..94800ba620 --- /dev/null +++ b/tests/integration_tests/weekly_rand_multi_failover/run.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +# Random DDL+DML weekly smoke test (3 captures, random failover). +# +# Timeline (high level): +# 1) start_tidb_cluster: start upstream + downstream TiDB. +# 2) random_ddl_test_runner bootstrap: create identical schemas and deterministic seed data on both sides. +# 3) start TiCDC (3 captures) and create changefeed. +# 4) random_ddl_test_runner workload: +# - concurrent random DML + random DDL on upstream +# - random capture kill + restart (runner failover loop) +# 5) Final diff: sync_diff_inspector compares upstream vs downstream using diff_config.toml generated by the runner. +# 6) Post-check: scan logs for panic/fatal/data race patterns. +# +# Sequence diagram (simplified): +# run.sh +# |-> start_tidb_cluster +# |-> random_ddl_test_runner --phase bootstrap +# |-> run_cdc_server (capture=3) +# |-> cdc_cli_changefeed create +# |-> consumer (kafka/storage/pulsar) [optional] +# |-> random_ddl_test_runner --phase workload (includes failover) +# |-> check_sync_diff (sync_diff_inspector) + +set -eu + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $CUR/../_utils/test_prepare + +WORK_DIR=$OUT_DIR/$TEST_NAME +CDC_BINARY=cdc.test +SINK_TYPE=$1 +CHANGEFEED_ID="weeklyrand" + +RUN_SEED=${RUN_SEED:-1} +RUN_PROFILE=${RUN_PROFILE:-smoke} +RUN_DURATION=${RUN_DURATION:-3m} +RUN_CONVERGE_TIMEOUT=${RUN_CONVERGE_TIMEOUT:-} +if [ -z "$RUN_CONVERGE_TIMEOUT" ]; then + if [ "$RUN_PROFILE" == "weekly" ]; then + RUN_CONVERGE_TIMEOUT="120m" + else + RUN_CONVERGE_TIMEOUT="30m" + fi +fi + +function build_runner() { + mkdir -p "$WORK_DIR" + go build -o "$WORK_DIR/random_ddl_test_runner" "$CUR/../../utils/random_ddl_test_runner" +} + +function write_runner_config() { + local mysql_sync_enabled="false" + if [ "$SINK_TYPE" == "mysql" ]; then + mysql_sync_enabled="true" + fi + + cat >"$WORK_DIR/runner_config.json" </dev/null 2>&1 || true + cleanup_process cdc_kafka_consumer >/dev/null 2>&1 || true + cleanup_process cdc_storage_consumer >/dev/null 2>&1 || true + cleanup_process cdc_pulsar_consumer >/dev/null 2>&1 || true + stop_test $WORK_DIR +} + +trap 'cleanup' EXIT + +rm -rf $WORK_DIR && mkdir -p $WORK_DIR + +# start_tidb_cluster --workdir $WORK_DIR +cat >"$WORK_DIR/tidb_config.toml" </dev/null 2>&1; then + if rg -n -i "panic|fatal|data race" "$WORK_DIR"/runner.log "$WORK_DIR"/ddl_trace.log "$WORK_DIR"/stdout*.log "$WORK_DIR"/cdc*.log "$WORK_DIR"/cdc_*_consumer*.log "$WORK_DIR"/cdc_*_consumer_stdout*.log 2>/dev/null | head -n 20 | rg -n . >/dev/null 2>&1; then + echo "log scan: panic/fatal/race detected" + rg -n -i "panic|fatal|data race" "$WORK_DIR"/runner.log "$WORK_DIR"/ddl_trace.log "$WORK_DIR"/stdout*.log "$WORK_DIR"/cdc*.log "$WORK_DIR"/cdc_*_consumer*.log "$WORK_DIR"/cdc_*_consumer_stdout*.log 2>/dev/null | head -n 50 || true + exit 1 + fi +fi + +echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" diff --git a/tests/integration_tests/weekly_rand_single/conf/changefeed_mysql.toml b/tests/integration_tests/weekly_rand_single/conf/changefeed_mysql.toml new file mode 100644 index 0000000000..d01100dc36 --- /dev/null +++ b/tests/integration_tests/weekly_rand_single/conf/changefeed_mysql.toml @@ -0,0 +1,2 @@ +enable-sync-point = true +sync-point-interval = "30s" diff --git a/tests/integration_tests/weekly_rand_single/conf/consumer.toml b/tests/integration_tests/weekly_rand_single/conf/consumer.toml new file mode 100644 index 0000000000..28f76a83b9 --- /dev/null +++ b/tests/integration_tests/weekly_rand_single/conf/consumer.toml @@ -0,0 +1,2 @@ +[scheduler] +enable-table-across-nodes=true diff --git a/tests/integration_tests/weekly_rand_single/run.sh b/tests/integration_tests/weekly_rand_single/run.sh new file mode 100644 index 0000000000..873fa6077a --- /dev/null +++ b/tests/integration_tests/weekly_rand_single/run.sh @@ -0,0 +1,189 @@ +#!/bin/bash + +# Random DDL+DML weekly smoke test (single capture). +# +# Timeline (high level): +# 1) start_tidb_cluster: start upstream + downstream TiDB. +# 2) random_ddl_test_runner bootstrap: create identical schemas and deterministic seed data on both sides. +# 3) start TiCDC (single capture) and create changefeed (optionally start MQ/storage consumer). +# 4) random_ddl_test_runner workload: run concurrent random DML + random DDL against upstream only. +# 5) Final diff: sync_diff_inspector compares upstream vs downstream using diff_config.toml generated by the runner. +# 6) Post-check: scan logs for panic/fatal/data race patterns. +# +# Sequence diagram (simplified): +# run.sh +# |-> start_tidb_cluster +# |-> random_ddl_test_runner --phase bootstrap +# |-> run_cdc_server (capture=1) +# |-> cdc_cli_changefeed create +# |-> consumer (kafka/storage/pulsar) [optional] +# |-> random_ddl_test_runner --phase workload +# |-> check_sync_diff (sync_diff_inspector) + +set -eu + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $CUR/../_utils/test_prepare + +WORK_DIR=$OUT_DIR/$TEST_NAME +CDC_BINARY=cdc.test +SINK_TYPE=$1 +CHANGEFEED_ID="weeklyrand" + +RUN_SEED=${RUN_SEED:-1} +RUN_PROFILE=${RUN_PROFILE:-smoke} +RUN_DURATION=${RUN_DURATION:-3m} +RUN_CONVERGE_TIMEOUT=${RUN_CONVERGE_TIMEOUT:-} +if [ -z "$RUN_CONVERGE_TIMEOUT" ]; then + if [ "$RUN_PROFILE" == "weekly" ]; then + RUN_CONVERGE_TIMEOUT="120m" + else + RUN_CONVERGE_TIMEOUT="30m" + fi +fi + +function build_runner() { + mkdir -p "$WORK_DIR" + go build -o "$WORK_DIR/random_ddl_test_runner" "$CUR/../../utils/random_ddl_test_runner" +} + +function write_runner_config() { + local mysql_sync_enabled="false" + if [ "$SINK_TYPE" == "mysql" ]; then + mysql_sync_enabled="true" + fi + + cat >"$WORK_DIR/runner_config.json" </dev/null 2>&1 || true + cleanup_process cdc_kafka_consumer >/dev/null 2>&1 || true + cleanup_process cdc_storage_consumer >/dev/null 2>&1 || true + cleanup_process cdc_pulsar_consumer >/dev/null 2>&1 || true + stop_test $WORK_DIR +} + +trap 'cleanup' EXIT + +rm -rf $WORK_DIR && mkdir -p $WORK_DIR + +# start_tidb_cluster --workdir $WORK_DIR +cat >"$WORK_DIR/tidb_config.toml" </dev/null 2>&1; then + if rg -n -i "panic|fatal|data race" "$WORK_DIR"/runner.log "$WORK_DIR"/ddl_trace.log "$WORK_DIR"/stdout*.log "$WORK_DIR"/cdc*.log "$WORK_DIR"/cdc_*_consumer*.log "$WORK_DIR"/cdc_*_consumer_stdout*.log 2>/dev/null | head -n 20 | rg -n . >/dev/null 2>&1; then + echo "log scan: panic/fatal/race detected" + rg -n -i "panic|fatal|data race" "$WORK_DIR"/runner.log "$WORK_DIR"/ddl_trace.log "$WORK_DIR"/stdout*.log "$WORK_DIR"/cdc*.log "$WORK_DIR"/cdc_*_consumer*.log "$WORK_DIR"/cdc_*_consumer_stdout*.log 2>/dev/null | head -n 50 || true + exit 1 + fi +fi + +echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" diff --git a/tests/integration_tests/weekly_rand_slow_lossy_ddl/conf/changefeed_mysql.toml b/tests/integration_tests/weekly_rand_slow_lossy_ddl/conf/changefeed_mysql.toml new file mode 100644 index 0000000000..d01100dc36 --- /dev/null +++ b/tests/integration_tests/weekly_rand_slow_lossy_ddl/conf/changefeed_mysql.toml @@ -0,0 +1,2 @@ +enable-sync-point = true +sync-point-interval = "30s" diff --git a/tests/integration_tests/weekly_rand_slow_lossy_ddl/run.sh b/tests/integration_tests/weekly_rand_slow_lossy_ddl/run.sh new file mode 100644 index 0000000000..522e44cedc --- /dev/null +++ b/tests/integration_tests/weekly_rand_slow_lossy_ddl/run.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Random DDL+DML weekly smoke test: slow downstream apply for lossy DDL (MySQL sink only). +# +# Timeline (high level): +# 1) start_tidb_cluster: start upstream + downstream TiDB. +# 2) random_ddl_test_runner bootstrap: create identical schemas and deterministic seed data on both sides. +# 3) start TiCDC (single capture) with a failpoint that delays downstream DDL execution. +# 4) random_ddl_test_runner workload: run concurrent random DML + random DDL on upstream. +# 5) Final diff: sync_diff_inspector compares upstream vs downstream using diff_config.toml generated by the runner. +# +# Notes: +# - This case requires MySQL sink and sets GO_FAILPOINTS to delay MySQL sink DDL execution. + +set -eu + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $CUR/../_utils/test_prepare + +WORK_DIR=$OUT_DIR/$TEST_NAME +CDC_BINARY=cdc.test +SINK_TYPE=$1 +CHANGEFEED_ID="weeklyrand" + +RUN_SEED=${RUN_SEED:-1} +RUN_PROFILE=${RUN_PROFILE:-smoke} +RUN_DURATION=${RUN_DURATION:-3m} +RUN_CONVERGE_TIMEOUT=${RUN_CONVERGE_TIMEOUT:-} +if [ -z "$RUN_CONVERGE_TIMEOUT" ]; then + if [ "$RUN_PROFILE" == "weekly" ]; then + RUN_CONVERGE_TIMEOUT="120m" + else + RUN_CONVERGE_TIMEOUT="30m" + fi +fi +DDL_DELAY_SEC=${DDL_DELAY_SEC:-3} + +function build_runner() { + mkdir -p "$WORK_DIR" + go build -o "$WORK_DIR/random_ddl_test_runner" "$CUR/../../utils/random_ddl_test_runner" +} + +function write_runner_config() { + cat >"$WORK_DIR/runner_config.json" </dev/null 2>&1 || true + stop_test $WORK_DIR +} + +trap 'cleanup' EXIT + +if [ "$SINK_TYPE" != "mysql" ]; then + echo "Skip test since MySQL sink is required" + exit 0 +fi + +rm -rf $WORK_DIR && mkdir -p $WORK_DIR + +# start_tidb_cluster --workdir $WORK_DIR +cat >"$WORK_DIR/tidb_config.toml" <>>>>>" diff --git a/tests/utils/random_ddl_test_runner/autotune.go b/tests/utils/random_ddl_test_runner/autotune.go new file mode 100644 index 0000000000..8d0909393c --- /dev/null +++ b/tests/utils/random_ddl_test_runner/autotune.go @@ -0,0 +1,57 @@ +package main + +import "time" + +type autoTuneResult struct { + nextDML int32 + nextDDL int32 + fail bool +} + +func autoTuneStep( + sinceAdvance time.Duration, + successRate float64, + activeDML int32, + activeDDL int32, + maxDML int32, + maxDDL int32, + soft time.Duration, + hard time.Duration, +) autoTuneResult { + // autoTuneStep adjusts concurrency to keep replication progressing: + // - If checkpoint is stalled beyond "soft" or DML success rate collapses, scale down first. + // - If checkpoint is healthy, gradually scale up toward configured maxima. + // + // It returns fail=true only when checkpoint is stalled beyond "hard". + if sinceAdvance >= hard { + return autoTuneResult{fail: true} + } + + nextDML := activeDML + nextDDL := activeDDL + + if sinceAdvance >= soft || successRate < 0.10 { + if nextDDL > 1 { + nextDDL-- + return autoTuneResult{nextDML: nextDML, nextDDL: nextDDL} + } + if nextDML > 1 { + nextDML -= 8 + if nextDML < 1 { + nextDML = 1 + } + } + return autoTuneResult{nextDML: nextDML, nextDDL: nextDDL} + } + + if nextDML < maxDML { + nextDML += 8 + if nextDML > maxDML { + nextDML = maxDML + } + } + if nextDDL < maxDDL && sinceAdvance < soft/2 { + nextDDL++ + } + return autoTuneResult{nextDML: nextDML, nextDDL: nextDDL} +} diff --git a/tests/utils/random_ddl_test_runner/autotune_test.go b/tests/utils/random_ddl_test_runner/autotune_test.go new file mode 100644 index 0000000000..67543b2821 --- /dev/null +++ b/tests/utils/random_ddl_test_runner/autotune_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "testing" + "time" +) + +func TestAutoTuneStep_DegradeOnHardStall(t *testing.T) { + res := autoTuneStep(10*time.Minute, 1.0, 64, 4, 128, 8, 2*time.Minute, 5*time.Minute) + if !res.fail { + t.Fatalf("expected fail on hard stall") + } +} + +func TestAutoTuneStep_DegradeDDLFirst(t *testing.T) { + res := autoTuneStep(3*time.Minute, 1.0, 64, 4, 128, 8, 2*time.Minute, 5*time.Minute) + if res.fail { + t.Fatalf("unexpected fail") + } + if res.nextDDL != 3 { + t.Fatalf("expected ddl decrease first, got %d", res.nextDDL) + } + if res.nextDML != 64 { + t.Fatalf("expected dml unchanged, got %d", res.nextDML) + } +} + +func TestAutoTuneStep_DegradeDMLWhenDDLMin(t *testing.T) { + res := autoTuneStep(3*time.Minute, 1.0, 16, 1, 128, 8, 2*time.Minute, 5*time.Minute) + if res.fail { + t.Fatalf("unexpected fail") + } + if res.nextDDL != 1 { + t.Fatalf("expected ddl unchanged at min, got %d", res.nextDDL) + } + if res.nextDML >= 16 { + t.Fatalf("expected dml decreased, got %d", res.nextDML) + } +} + +func TestAutoTuneStep_IncreaseWhenHealthy(t *testing.T) { + res := autoTuneStep(10*time.Second, 0.9, 16, 1, 32, 4, 2*time.Minute, 5*time.Minute) + if res.fail { + t.Fatalf("unexpected fail") + } + if res.nextDML <= 16 { + t.Fatalf("expected dml increased, got %d", res.nextDML) + } + if res.nextDDL != 2 { + t.Fatalf("expected ddl increased, got %d", res.nextDDL) + } +} diff --git a/tests/utils/random_ddl_test_runner/bootstrap.go b/tests/utils/random_ddl_test_runner/bootstrap.go new file mode 100644 index 0000000000..dd20b4128d --- /dev/null +++ b/tests/utils/random_ddl_test_runner/bootstrap.go @@ -0,0 +1,210 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" +) + +func (r *runner) bootstrap() error { + // bootstrap creates an identical starting point on upstream and downstream. + // + // Rationale: + // - The workload phase only writes to upstream. Downstream changes must come from TiCDC replication. + // - A deterministic baseline makes end-to-end diffs and triage reproducible (seeded by cfg.Seed). + ctx := context.Background() + r.logger.Printf("bootstrap start: workdir=%s profile=%s", r.cfg.Workdir, r.cfg.Profile) + + up, err := openMySQL(ctx, r.cfg.Upstream) + if err != nil { + return err + } + defer func() { _ = up.Close() }() + + down, err := openMySQL(ctx, r.cfg.Downstream) + if err != nil { + return err + } + defer func() { _ = down.Close() }() + + model := buildInitialModel(r.cfg) + + for _, dbName := range model.dbs { + if err := execBoth(ctx, up, down, + fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dbName), + fmt.Sprintf("CREATE DATABASE `%s`", dbName), + ); err != nil { + return err + } + } + + for _, tbl := range model.tables { + createSQL := tbl.schema.createTableSQL(tbl.db, tbl.name) + if err := execBoth(ctx, up, down, createSQL); err != nil { + return err + } + } + + // Deny region merge on split candidate tables to keep region pressure stable. + for _, tbl := range model.splitTables { + attrsSQL := fmt.Sprintf("ALTER TABLE %s ATTRIBUTES 'merge_option=deny'", tbl.fqName()) + if err := execBoth(ctx, up, down, attrsSQL); err != nil { + return err + } + } + + baseRows := r.cfg.Bootstrap.BaseRowsPerTable + splitRows := r.cfg.Bootstrap.SplitRowsPerTable + + for _, tbl := range model.tables { + rows := baseRows + if tbl.domain == domainSplit { + rows += splitRows + } + if err := insertInitialRows(ctx, up, down, tbl, rows); err != nil { + return err + } + } + + r.logger.Printf("bootstrap done") + return nil +} + +func execBoth(ctx context.Context, up, down *sql.DB, stmts ...string) error { + for _, s := range stmts { + if _, err := up.ExecContext(ctx, s); err != nil { + return err + } + if _, err := down.ExecContext(ctx, s); err != nil { + return err + } + } + return nil +} + +func insertInitialRows(ctx context.Context, up, down *sql.DB, tbl *table, rows int) error { + tbl.mu.Lock() + schema := tbl.schema.clone() + tbl.mu.Unlock() + + // Use placeholders for values to keep SQL ASCII-only while allowing any binary/JSON payloads. + var cols []column + for _, c := range schema.columns { + if c.generated != "" { + continue + } + cols = append(cols, c) + } + + colNames := make([]string, 0, len(cols)) + for _, c := range cols { + colNames = append(colNames, c.name) + } + + const batchSize = 200 + for start := 1; start <= rows; start += batchSize { + end := start + batchSize - 1 + if end > rows { + end = rows + } + + var args []any + var valuesSQL strings.Builder + for i := start; i <= end; i++ { + if i > start { + valuesSQL.WriteString(",") + } + valuesSQL.WriteString("(") + for j := range cols { + if j > 0 { + valuesSQL.WriteString(",") + } + valuesSQL.WriteString("?") + } + valuesSQL.WriteString(")") + + rowArgs := buildDeterministicRowArgs(tbl, cols, int64(i)) + args = append(args, rowArgs...) + } + + stmt := fmt.Sprintf("INSERT INTO %s (%s) VALUES %s", + tbl.fqName(), + backtickJoin(colNames), + valuesSQL.String(), + ) + if _, err := up.ExecContext(ctx, stmt, args...); err != nil { + return err + } + if _, err := down.ExecContext(ctx, stmt, args...); err != nil { + return err + } + } + return nil +} + +func backtickJoin(cols []string) string { + quoted := make([]string, 0, len(cols)) + for _, c := range cols { + quoted = append(quoted, fmt.Sprintf("`%s`", c)) + } + return strings.Join(quoted, ",") +} + +func buildDeterministicRowArgs(tbl *table, cols []column, rowID int64) []any { + // Deterministic values make bootstrap reproducible. Avoid non-ASCII in the SQL text by + // passing bytes/JSON via placeholders rather than embedding literals into the statement. + args := make([]any, 0, len(cols)) + for _, c := range cols { + switch strings.ToUpper(c.typ.base) { + case "BIGINT": + if c.name == "id" { + args = append(args, rowID) + } else { + args = append(args, deterministicInt64(rowID)) + } + case "INT": + if c.name == "a" { + args = append(args, int32(rowID)) + } else if c.name == "v" { + args = append(args, int32(rowID%1000)) + } else { + args = append(args, int32(deterministicInt64(rowID)%mathMaxInt32())) + } + case "VARCHAR": + if c.name == "pad" { + args = append(args, strings.Repeat("x", 256)) + } else { + args = append(args, asciiStringFromID(fmt.Sprintf("%s_%s", tbl.name, c.name), rowID)) + } + case "DATETIME": + args = append(args, deterministicTime(rowID)) + case "DECIMAL": + args = append(args, deterministicDecimal(rowID)) + case "JSON": + args = append(args, fmt.Sprintf("{\"id\":%d,\"table\":\"%s\"}", rowID, tbl.name)) + case "VARBINARY": + // Keep bytes deterministic; the SQL text remains ASCII-only due to placeholders. + args = append(args, []byte(fmt.Sprintf("%064x", rowID))) + default: + args = append(args, nil) + } + } + return args +} + +func mathMaxInt32() int64 { + return int64(^uint32(0) >> 1) +} + +func sleepWithContext(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} diff --git a/tests/utils/random_ddl_test_runner/config.go b/tests/utils/random_ddl_test_runner/config.go new file mode 100644 index 0000000000..ef061e2935 --- /dev/null +++ b/tests/utils/random_ddl_test_runner/config.go @@ -0,0 +1,329 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "time" +) + +type duration struct { + time.Duration +} + +func (d *duration) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + parsed, err := time.ParseDuration(s) + if err != nil { + return err + } + d.Duration = parsed + return nil +} + +type mysqlConnConfig struct { + Host string `json:"host"` + Port int `json:"port"` + User string `json:"user"` + Password string `json:"password"` +} + +func (c mysqlConnConfig) dsn(database string) string { + // Keep DSN ASCII-only and deterministic. + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=true&multiStatements=true&interpolateParams=true", + c.User, c.Password, c.Host, c.Port, database) +} + +type cdcAPIConfig struct { + Addr string `json:"addr"` + User string `json:"user"` + Password string `json:"password"` + Keyspace string `json:"keyspace"` + // ChangefeedID is required for the workload phase. + ChangefeedID string `json:"changefeed_id"` +} + +type failoverConfig struct { + Enabled bool `json:"enabled"` + CaptureAddrs []string `json:"capture_addrs"` + CdcBinary string `json:"cdc_binary"` + MinInterval duration `json:"min_interval"` + MaxInterval duration `json:"max_interval"` + GatedProbability float64 `json:"gated_probability"` +} + +type dmlConfig struct { + MaxWorkers int `json:"max_workers"` + InitialWorkers int `json:"initial_workers"` + HotspotRatio float64 `json:"hotspot_ratio"` + HotTableRatio float64 `json:"hot_table_ratio"` + BigTxnEnabled bool `json:"big_txn_enabled"` + BigTxnInterval duration `json:"big_txn_interval"` + BigTxnRowsMin int `json:"big_txn_rows_min"` + BigTxnRowsMax int `json:"big_txn_rows_max"` + KeyConflictEnabled bool `json:"key_conflict_enabled"` + KeyConflictKeyspace int `json:"key_conflict_keyspace"` +} + +type ddlConfig struct { + MaxWorkers int `json:"max_workers"` + InitialWorkers int `json:"initial_workers"` +} + +type verifyConfig struct { + HealthInterval duration `json:"health_interval"` + NoAdvanceSoft duration `json:"no_advance_soft"` + NoAdvanceHard duration `json:"no_advance_hard"` + LogScanEnabled bool `json:"log_scan_enabled"` + PanicPatterns []string `json:"panic_patterns"` + FailOnPanicMatch bool `json:"fail_on_panic_match"` + ConvergeWait duration `json:"converge_wait"` + ConvergeTimeout duration `json:"converge_timeout"` +} + +type mysqlSyncpointConfig struct { + Enabled bool `json:"enabled"` + DiffInterval duration `json:"diff_interval"` + MaxDiffChecks int `json:"max_diff_checks"` + UpstreamStatusHost string `json:"upstream_status_host"` + UpstreamStatusPort int `json:"upstream_status_port"` +} + +type config struct { + Workdir string `json:"workdir"` + Profile string `json:"profile"` + Seed int64 `json:"seed"` + Duration duration `json:"duration"` + + Upstream mysqlConnConfig `json:"upstream"` + Downstream mysqlConnConfig `json:"downstream"` + + CDC cdcAPIConfig `json:"cdc"` + SinkType string `json:"sink_type"` + Failover failoverConfig `json:"failover"` + DML dmlConfig `json:"dml"` + DDL ddlConfig `json:"ddl"` + Verify verifyConfig `json:"verify"` + MySQL mysqlSyncpointConfig `json:"mysql"` + + Bootstrap struct { + DBCount int `json:"db_count"` + TablesPerDB int `json:"tables_per_db"` + BaseRowsPerTable int `json:"base_rows_per_table"` + SplitRowsPerTable int `json:"split_rows_per_table"` + FrozenRowsPerTable int `json:"frozen_rows_per_table"` + } `json:"bootstrap"` +} + +func loadConfig(path string) (*config, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var cfg config + if err := json.Unmarshal(b, &cfg); err != nil { + return nil, err + } + if err := cfg.applyDefaultsAndValidate(); err != nil { + return nil, err + } + return &cfg, nil +} + +func (c *config) applyDefaultsAndValidate() error { + // Defaults should keep the smoke profile fast while still exercising key paths. + // "weekly" uses larger concurrency / data volumes to increase coverage. + if c.Workdir == "" { + return fmt.Errorf("workdir is required") + } + if c.Seed == 0 { + // Allow explicit 0 but require deterministic default. + c.Seed = 1 + } + if c.Duration.Duration <= 0 { + c.Duration.Duration = 3 * time.Minute + } + if c.Profile == "" { + c.Profile = "smoke" + } + + if c.Upstream.Host == "" { + c.Upstream.Host = "127.0.0.1" + } + if c.Upstream.Port == 0 { + c.Upstream.Port = 4000 + } + if c.Upstream.User == "" { + c.Upstream.User = "root" + } + + if c.Downstream.Host == "" { + c.Downstream.Host = "127.0.0.1" + } + if c.Downstream.Port == 0 { + c.Downstream.Port = 3306 + } + if c.Downstream.User == "" { + c.Downstream.User = "root" + } + + if c.Bootstrap.DBCount == 0 { + c.Bootstrap.DBCount = 5 + } + if c.Bootstrap.TablesPerDB == 0 { + c.Bootstrap.TablesPerDB = 20 + } + if c.Bootstrap.BaseRowsPerTable == 0 { + if c.Profile == "weekly" { + c.Bootstrap.BaseRowsPerTable = 1000 + } else { + c.Bootstrap.BaseRowsPerTable = 100 + } + } + if c.Bootstrap.SplitRowsPerTable == 0 { + if c.Profile == "weekly" { + c.Bootstrap.SplitRowsPerTable = 5000 + } else { + c.Bootstrap.SplitRowsPerTable = 500 + } + } + if c.Bootstrap.FrozenRowsPerTable == 0 { + c.Bootstrap.FrozenRowsPerTable = 50 + } + + if c.DML.MaxWorkers == 0 { + if c.Profile == "weekly" { + c.DML.MaxWorkers = 128 + } else { + c.DML.MaxWorkers = 32 + } + } + if c.DML.InitialWorkers == 0 { + c.DML.InitialWorkers = c.DML.MaxWorkers / 2 + if c.DML.InitialWorkers < 1 { + c.DML.InitialWorkers = 1 + } + } + if c.DML.HotspotRatio == 0 { + c.DML.HotspotRatio = 0.8 + } + if c.DML.HotTableRatio == 0 { + c.DML.HotTableRatio = 0.1 + } + if c.DML.BigTxnInterval.Duration == 0 { + c.DML.BigTxnInterval.Duration = 20 * time.Second + } + // Keep these enabled by default to ensure the workload includes the key motifs from the design doc. + c.DML.BigTxnEnabled = true + c.DML.KeyConflictEnabled = true + if c.DML.BigTxnRowsMin == 0 { + c.DML.BigTxnRowsMin = 200 + } + if c.DML.BigTxnRowsMax == 0 { + c.DML.BigTxnRowsMax = 400 + } + if c.DML.KeyConflictKeyspace == 0 { + c.DML.KeyConflictKeyspace = 1024 + } + + if c.DDL.MaxWorkers == 0 { + if c.Profile == "weekly" { + c.DDL.MaxWorkers = 8 + } else { + c.DDL.MaxWorkers = 2 + } + } + if c.DDL.InitialWorkers == 0 { + c.DDL.InitialWorkers = c.DDL.MaxWorkers / 2 + if c.DDL.InitialWorkers < 1 { + c.DDL.InitialWorkers = 1 + } + } + + if c.Verify.HealthInterval.Duration == 0 { + c.Verify.HealthInterval.Duration = 10 * time.Second + } + if c.Verify.NoAdvanceSoft.Duration == 0 { + c.Verify.NoAdvanceSoft.Duration = 2 * time.Minute + } + if c.Verify.NoAdvanceHard.Duration == 0 { + if c.Profile == "weekly" { + c.Verify.NoAdvanceHard.Duration = 15 * time.Minute + } else { + c.Verify.NoAdvanceHard.Duration = 5 * time.Minute + } + } + if c.Verify.ConvergeWait.Duration == 0 { + c.Verify.ConvergeWait.Duration = 20 * time.Second + } + if c.Verify.ConvergeTimeout.Duration == 0 { + c.Verify.ConvergeTimeout.Duration = c.Verify.NoAdvanceHard.Duration * 2 + if c.Verify.ConvergeTimeout.Duration < 2*time.Minute { + c.Verify.ConvergeTimeout.Duration = 2 * time.Minute + } + } + if len(c.Verify.PanicPatterns) == 0 { + c.Verify.PanicPatterns = []string{"panic", "fatal", "DATA RACE"} + } + if !c.Verify.LogScanEnabled { + c.Verify.LogScanEnabled = true + } + if !c.Verify.FailOnPanicMatch { + c.Verify.FailOnPanicMatch = true + } + + if c.Failover.MinInterval.Duration == 0 { + c.Failover.MinInterval.Duration = 20 * time.Second + } + if c.Failover.MaxInterval.Duration == 0 { + c.Failover.MaxInterval.Duration = 40 * time.Second + } + if c.Failover.GatedProbability == 0 { + c.Failover.GatedProbability = 0.5 + } + if c.Failover.CdcBinary == "" { + c.Failover.CdcBinary = "cdc.test" + } + if c.CDC.Keyspace == "" { + c.CDC.Keyspace = "default" + } + if c.CDC.User == "" { + c.CDC.User = "ticdc" + } + if c.CDC.Password == "" { + c.CDC.Password = "ticdc_secret" + } + if c.CDC.Addr == "" { + c.CDC.Addr = "127.0.0.1:8300" + } + + if c.MySQL.DiffInterval.Duration == 0 { + c.MySQL.DiffInterval.Duration = 2 * time.Minute + } + if c.MySQL.MaxDiffChecks == 0 { + c.MySQL.MaxDiffChecks = 1 + } + if c.MySQL.UpstreamStatusHost == "" { + c.MySQL.UpstreamStatusHost = "127.0.0.1" + } + if c.MySQL.UpstreamStatusPort == 0 { + c.MySQL.UpstreamStatusPort = 10080 + } + + // Basic validation. + // Keep the schema shape stable so that: + // - bootstrap can create a predictable workload surface, + // - the integration scripts can pre-create storage sink directories deterministically, + // - the model can assume fixed database/table sets. + if c.Bootstrap.DBCount != 5 { + return fmt.Errorf("db_count must be 5 to match the design doc, got %d", c.Bootstrap.DBCount) + } + if c.Bootstrap.TablesPerDB != 20 { + return fmt.Errorf("tables_per_db must be 20 to match the design doc, got %d", c.Bootstrap.TablesPerDB) + } + + return nil +} diff --git a/tests/utils/random_ddl_test_runner/db.go b/tests/utils/random_ddl_test_runner/db.go new file mode 100644 index 0000000000..f89288786b --- /dev/null +++ b/tests/utils/random_ddl_test_runner/db.go @@ -0,0 +1,39 @@ +package main + +import ( + "context" + "database/sql" + "time" + + _ "github.com/go-sql-driver/mysql" +) + +func openMySQL(ctx context.Context, cfg mysqlConnConfig) (*sql.DB, error) { + return openMySQLWithExtraParams(ctx, cfg, "") +} + +func openMySQLWithExtraParams(ctx context.Context, cfg mysqlConnConfig, extraParams string) (*sql.DB, error) { + dsn := cfg.dsn("") + if extraParams != "" { + dsn += "&" + extraParams + } + return openMySQLWithDSN(ctx, dsn) +} + +func openMySQLWithDSN(ctx context.Context, dsn string) (*sql.DB, error) { + db, err := sql.Open("mysql", dsn) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(128) + db.SetMaxIdleConns(128) + db.SetConnMaxLifetime(5 * time.Minute) + + pingCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if err := db.PingContext(pingCtx); err != nil { + _ = db.Close() + return nil, err + } + return db, nil +} diff --git a/tests/utils/random_ddl_test_runner/ddl.go b/tests/utils/random_ddl_test_runner/ddl.go new file mode 100644 index 0000000000..a0347cacde --- /dev/null +++ b/tests/utils/random_ddl_test_runner/ddl.go @@ -0,0 +1,476 @@ +package main + +import ( + "fmt" + "math/rand" + "strings" +) + +type ddlKind struct { + // name is used for logging and selector tracking. + name string + domain domain + baseWeight float64 + // lossy marks DDLs that can drop data or schema information (e.g., DROP/TRUNCATE/DROP COLUMN). + // These are still useful for coverage, but are typically constrained to churn domain. + lossy bool + + // gen returns: + // - sql: the DDL statement to execute on upstream. + // - apply: a callback that mutates the in-memory model when and only when the DDL succeeds. + gen func(rng *rand.Rand, t *table) (sql string, apply func()) +} + +func defaultDDLKinds() []ddlKind { + // DDL kinds are grouped by domain: + // - stable: schema changes that are relatively friendly to snapshot-based diffing. + // - churn: destructive or fragile DDLs that can invalidate snapshot reads and diff configs. + // - split_candidate: a subset used to stress split/region pressure with larger tables. + return []ddlKind{ + { + name: "add_column", + domain: domainStable, + baseWeight: 5, + gen: genAddColumn, + }, + { + name: "add_index", + domain: domainStable, + baseWeight: 4, + gen: genAddIndex, + }, + { + name: "convert_charset", + domain: domainStable, + baseWeight: 1, + gen: genConvertCharset, + }, + { + name: "add_partition", + domain: domainStable, + baseWeight: 1, + gen: genAddPartition, + }, + + // Note: Periodic MySQL syncpoint diffs rely on snapshot reads via sync_diff_inspector. + // Some schema-changing DDLs (e.g., DROP COLUMN / MODIFY COLUMN / DROP INDEX) are + // known to be fragile for snapshot-based diffing. Keep them in churn domain to + // preserve periodic diff stability, while still exercising these DDLs in the workload. + { + name: "drop_column", + domain: domainChurn, + baseWeight: 2, + lossy: true, + gen: genDropColumn, + }, + { + name: "modify_column_type", + domain: domainChurn, + baseWeight: 1, + lossy: true, + gen: genModifyColumnType, + }, + { + name: "drop_index", + domain: domainChurn, + baseWeight: 1, + gen: genDropIndex, + }, + + // Split candidate tables: use the same DDL set as stable, but target split domain. + { + name: "split_add_index", + domain: domainSplit, + baseWeight: 2, + gen: genAddIndex, + }, + { + name: "split_drop_index", + domain: domainSplit, + baseWeight: 1, + gen: genDropIndex, + }, + { + name: "split_add_partition", + domain: domainSplit, + baseWeight: 1, + gen: genAddPartition, + }, + + // Churn domain: destructive operations. + { + name: "truncate_table", + domain: domainChurn, + baseWeight: 2, + lossy: true, + gen: genTruncateTable, + }, + { + name: "drop_table", + domain: domainChurn, + baseWeight: 1, + lossy: true, + gen: genDropTable, + }, + // Do not include RECOVER TABLE in the default CDC random DDL set. + // It restores table data and schema from TiDB's local DDL history / GC + // snapshot state, so executing the same text on the downstream can recover + // a different historical table when names are reused. The recovered rows are + // not emitted as new DML either, making the operation unsafe for this suite. + { + name: "drop_and_recreate_table", + domain: domainChurn, + baseWeight: 1, + lossy: true, + gen: genDropAndRecreateTable, + }, + { + name: "rename_table", + domain: domainChurn, + baseWeight: 1, + gen: genRenameTable, + }, + } +} + +func genAddColumn(rng *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + if len(t.schema.columns) > 32 { + return "", nil + } + newName := fmt.Sprintf("c_%d", rng.Intn(10_000_000)) + for _, c := range t.schema.columns { + if c.name == newName { + return "", nil + } + } + typ := randDDLColType(rng) + def := "0" + if strings.EqualFold(typ.base, "VARCHAR") { + def = "''" + } + sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN `%s` %s NOT NULL DEFAULT %s", + t.fqName(), newName, typ.sql(), def) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + t.schema.columns = append(t.schema.columns, column{ + name: newName, + typ: typ, + nullable: false, + defaultSQL: def, + }) + } + return sql, apply +} + +func genDropColumn(rng *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + var candidates []int + for i, c := range t.schema.columns { + if c.generated != "" { + continue + } + if c.name == "id" { + continue + } + if containsString(t.schema.primaryKey, c.name) { + continue + } + candidates = append(candidates, i) + } + if len(candidates) == 0 { + return "", nil + } + idx := candidates[rng.Intn(len(candidates))] + colName := t.schema.columns[idx].name + sql := fmt.Sprintf("ALTER TABLE %s DROP COLUMN `%s`", t.fqName(), colName) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + if idx >= len(t.schema.columns) || t.schema.columns[idx].name != colName { + // Best-effort: column order may have changed due to concurrent successful DDL. + for i, c := range t.schema.columns { + if c.name == colName { + idx = i + break + } + } + } + if idx >= len(t.schema.columns) || t.schema.columns[idx].name != colName { + return + } + t.schema.columns = append(t.schema.columns[:idx], t.schema.columns[idx+1:]...) + // Drop indexes referencing the column. + var newIdx []index + for _, ix := range t.schema.indexes { + if containsString(ix.columns, colName) { + continue + } + newIdx = append(newIdx, ix) + } + t.schema.indexes = newIdx + } + return sql, apply +} + +func genModifyColumnType(rng *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + var candidates []int + for i, c := range t.schema.columns { + if c.generated != "" { + continue + } + if containsString(t.schema.primaryKey, c.name) { + continue + } + if strings.EqualFold(c.typ.base, "JSON") || strings.EqualFold(c.typ.base, "DATETIME") { + continue + } + candidates = append(candidates, i) + } + if len(candidates) == 0 { + return "", nil + } + idx := candidates[rng.Intn(len(candidates))] + colName := t.schema.columns[idx].name + newTyp := randDDLColType(rng) + sql := fmt.Sprintf("ALTER TABLE %s MODIFY COLUMN `%s` %s NOT NULL", + t.fqName(), colName, newTyp.sql()) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + for i := range t.schema.columns { + if t.schema.columns[i].name == colName { + t.schema.columns[i].typ = newTyp + return + } + } + } + return sql, apply +} + +func genAddIndex(rng *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + var cols []string + for _, c := range t.schema.columns { + if c.generated != "" { + continue + } + if strings.EqualFold(c.typ.base, "JSON") { + continue + } + cols = append(cols, c.name) + } + if len(cols) == 0 { + return "", nil + } + col := cols[rng.Intn(len(cols))] + name := fmt.Sprintf("idx_%s_%d", col, rng.Intn(10_000)) + for _, ix := range t.schema.indexes { + if ix.name == name { + return "", nil + } + } + sql := fmt.Sprintf("CREATE INDEX `%s` ON %s (`%s`)", name, t.fqName(), col) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + t.schema.indexes = append(t.schema.indexes, index{name: name, columns: []string{col}, unique: false}) + } + return sql, apply +} + +func genDropIndex(rng *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + if len(t.schema.indexes) == 0 { + return "", nil + } + ix := t.schema.indexes[rng.Intn(len(t.schema.indexes))] + sql := fmt.Sprintf("DROP INDEX `%s` ON %s", ix.name, t.fqName()) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + var newIdx []index + for _, x := range t.schema.indexes { + if x.name == ix.name { + continue + } + newIdx = append(newIdx, x) + } + t.schema.indexes = newIdx + } + return sql, apply +} + +func genConvertCharset(rng *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + targetCharset := "utf8mb4" + targetCollate := "utf8mb4_bin" + if !strings.EqualFold(t.schema.charset, "gbk") && rng.Intn(2) == 0 { + targetCharset = "gbk" + targetCollate = "gbk_bin" + } + sql := fmt.Sprintf("ALTER TABLE %s CONVERT TO CHARACTER SET %s COLLATE %s", t.fqName(), targetCharset, targetCollate) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + t.schema.charset = targetCharset + t.schema.collation = targetCollate + } + return sql, apply +} + +func genAddPartition(rng *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + if !strings.Contains(strings.ToUpper(t.schema.partitionSQL), "RANGE") { + return "", nil + } + if t.rangePartitionNextID <= 0 || t.rangePartitionNextBound <= 0 { + return "", nil + } + pid := t.rangePartitionNextID + nextBound := t.rangePartitionNextBound + 1_000_000_000_000 + name := fmt.Sprintf("p%d", pid) + sql := fmt.Sprintf("ALTER TABLE %s ADD PARTITION (PARTITION %s VALUES LESS THAN (%d))", + t.fqName(), name, nextBound) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + t.rangePartitionNextID++ + t.rangePartitionNextBound = nextBound + } + return sql, apply +} + +func genTruncateTable(_ *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + sql := fmt.Sprintf("TRUNCATE TABLE %s", t.fqName()) + return sql, func() {} +} + +func genDropTable(_ *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + sql := fmt.Sprintf("DROP TABLE IF EXISTS %s", t.fqName()) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + t.exists = false + } + return sql, apply +} + +func genRecoverTable(_ *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if t.exists { + return "", nil + } + sql := fmt.Sprintf("RECOVER TABLE %s", t.fqName()) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + t.exists = true + } + return sql, apply +} + +func genDropAndRecreateTable(_ *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + sql := fmt.Sprintf("DROP TABLE IF EXISTS %s; %s", + t.fqName(), + t.initialSchema.createTableSQL(t.db, t.name), + ) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + t.exists = true + t.schema = t.initialSchema.clone() + } + return sql, apply +} + +func genRenameTable(rng *rand.Rand, t *table) (string, func()) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.exists { + return "", nil + } + + // Keep the rename logic simple: rename each churn table at most once. + // This avoids creating long chains of renamed tables and reduces the chance of + // duplicate rename DDLs under failover. + if strings.Contains(t.name, "_r_") { + return "", nil + } + + newName := fmt.Sprintf("%s_r_%d", t.name, rng.Intn(10_000_000)) + // The max length of a TiDB table name is 64. Keep a small margin. + if len(newName) > 60 { + return "", nil + } + sql := fmt.Sprintf("RENAME TABLE %s TO `%s`.`%s`", t.fqName(), t.db, newName) + apply := func() { + t.mu.Lock() + defer t.mu.Unlock() + t.name = newName + } + return sql, apply +} + +func randDDLColType(rng *rand.Rand) colType { + switch rng.Intn(3) { + case 0: + return colType{base: "INT"} + case 1: + return colType{base: "BIGINT"} + default: + return colType{base: "VARCHAR", varcharN: 64} + } +} + +func containsString(ss []string, s string) bool { + for _, x := range ss { + if x == s { + return true + } + } + return false +} diff --git a/tests/utils/random_ddl_test_runner/ddl_test.go b/tests/utils/random_ddl_test_runner/ddl_test.go new file mode 100644 index 0000000000..3872d7bd9a --- /dev/null +++ b/tests/utils/random_ddl_test_runner/ddl_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "math/rand" + "strings" + "testing" +) + +func TestGenDropColumn_DoesNotDropPrimaryKey(t *testing.T) { + tbl := &table{ + db: "db1", + name: "t00", + schema: tableSchema{ + columns: []column{ + {name: "id", typ: colType{base: "BIGINT"}, nullable: false}, + {name: "a", typ: colType{base: "INT"}, nullable: false}, + {name: "b", typ: colType{base: "VARCHAR", varcharN: 64}, nullable: false}, + }, + primaryKey: []string{"id"}, + }, + exists: true, + } + rng := rand.New(rand.NewSource(1)) + sqlText, _ := genDropColumn(rng, tbl) + if sqlText == "" { + t.Fatalf("expected a ddl statement") + } + if strings.Contains(sqlText, "`id`") { + t.Fatalf("expected not to drop pk column, sql=%s", sqlText) + } +} + +func TestGenAddPartition_RequiresRangePartition(t *testing.T) { + tbl := &table{ + db: "db1", + name: "t07", + schema: tableSchema{ + columns: []column{{name: "id", typ: colType{base: "BIGINT"}, nullable: false}}, + }, + exists: true, + } + rng := rand.New(rand.NewSource(1)) + sqlText, _ := genAddPartition(rng, tbl) + if sqlText != "" { + t.Fatalf("expected empty ddl for non-partitioned table, got %s", sqlText) + } +} + +func TestDefaultDDLKindsExcludeRecoverTable(t *testing.T) { + for _, kind := range defaultDDLKinds() { + if kind.name == "recover_table" { + t.Fatalf("recover_table should not be enabled by default") + } + } + + sqlText, apply := genRecoverTable(rand.New(rand.NewSource(1)), &table{db: "db1", name: "t1"}) + if sqlText == "" || apply == nil { + t.Fatalf("recover_table generator should remain available for explicit tests") + } +} diff --git a/tests/utils/random_ddl_test_runner/ddl_worker.go b/tests/utils/random_ddl_test_runner/ddl_worker.go new file mode 100644 index 0000000000..24bdcbeb0d --- /dev/null +++ b/tests/utils/random_ddl_test_runner/ddl_worker.go @@ -0,0 +1,165 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "log" + "math/rand" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" +) + +type ddlTrace struct { + mu sync.Mutex + file *os.File + log *log.Logger +} + +func newDDLTrace(workdir string) (*ddlTrace, error) { + if err := os.MkdirAll(workdir, 0o755); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.Join(workdir, "ddl_trace.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return nil, err + } + return &ddlTrace{ + file: f, + log: log.New(f, "", log.LstdFlags|log.Lmicroseconds|log.LUTC), + }, nil +} + +func (t *ddlTrace) close() { + if t == nil || t.file == nil { + return + } + _ = t.file.Close() +} + +func (t *ddlTrace) record(kind string, target string, sql string, err error) { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + status := "ok" + msg := "" + if err != nil { + status = "err" + msg = err.Error() + } + // Avoid printing raw SQL for DML here; DDL is expected ASCII-only. + t.log.Printf("kind=%s target=%s status=%s sql=%q err=%q", kind, target, status, sql, msg) +} + +func ddlWorker( + ctx context.Context, + db *sql.DB, + model *clusterModel, + seed int64, + workerID int, + activeWorkers *int32, + selector *ddlSelector, + trace *ddlTrace, + logger *log.Logger, +) { + // ddlWorker is a best-effort DDL submitter. + // + // Concurrency control: + // - Spawn MaxWorkers goroutines, but only workers with workerID < activeWorkers are "active". + // - healthAndAutotuneLoop adjusts activeWorkers based on checkpoint liveness. + // + // Correctness model: + // - Each DDL kind returns (sql, apply). apply updates the in-memory model and is invoked only + // when the DDL succeeds, so subsequent DML/DDL generation can track schema evolution. + // - DDL failures are expected under concurrency (e.g., conflicts, missing tables) and do not + // stop the worker. + rng := rand.New(rand.NewSource(seed + int64(workerID))) + + for { + select { + case <-ctx.Done(): + return + default: + } + + if int32(workerID) >= atomic.LoadInt32(activeWorkers) { + _ = sleepWithContext(ctx, 500*time.Millisecond) + continue + } + + kind := selector.pick(rng) + var tbl *table + if kind.name == "recover_table" { + tbl = pickMissingTable(rng, model.churnTables) + } else { + tbl = model.pickTableForDomain(rng, kind.domain) + } + if tbl == nil || tbl.isMotif { + _ = sleepWithContext(ctx, 100*time.Millisecond) + continue + } + + sqlText, apply := kind.gen(rng, tbl) + if sqlText == "" || apply == nil { + _ = sleepWithContext(ctx, 100*time.Millisecond) + continue + } + + start := time.Now() + _, err := db.ExecContext(ctx, sqlText) + if err == nil { + apply() + selector.record(kind.name) + } + if logger != nil { + logger.Printf("ddl worker=%d kind=%s target=%s elapsed=%s err=%v", + workerID, kind.name, tbl.fqName(), time.Since(start), err) + } + if trace != nil { + trace.record(kind.name, tbl.fqName(), sqlText, err) + } + + // Keep DDL submission rate bounded. + _ = sleepWithContext(ctx, time.Second) + } +} + +func pickMissingTable(rng *rand.Rand, candidates []*table) *table { + if len(candidates) == 0 { + return nil + } + for i := 0; i < 10; i++ { + t := candidates[rng.Intn(len(candidates))] + t.mu.Lock() + exists := t.exists + t.mu.Unlock() + if !exists { + return t + } + } + return nil +} + +func ensureFileClosed(logger *log.Logger, t *ddlTrace) { + if t == nil { + return + } + t.close() + if logger != nil { + logger.Printf("ddl trace file closed") + } +} + +func writeFileAtomic(path string, data []byte) error { + dir := filepath.Dir(path) + tmp := filepath.Join(dir, fmt.Sprintf(".tmp-%d", time.Now().UnixNano())) + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/tests/utils/random_ddl_test_runner/dml.go b/tests/utils/random_ddl_test_runner/dml.go new file mode 100644 index 0000000000..8454b8c11d --- /dev/null +++ b/tests/utils/random_ddl_test_runner/dml.go @@ -0,0 +1,381 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math/rand" + "strings" + "sync/atomic" + "time" + + "github.com/go-sql-driver/mysql" +) + +type dmlCounters struct { + total uint64 + success uint64 + unknownTable uint64 + unknownColumn uint64 + duplicateEntry uint64 + otherError uint64 +} + +func (c *dmlCounters) record(err error) { + // DML errors are expected under concurrent DDL and are not fatal by themselves. + // Counters are used by the health loop to infer a success rate for auto-tuning. + atomic.AddUint64(&c.total, 1) + if err == nil { + atomic.AddUint64(&c.success, 1) + return + } + var me *mysql.MySQLError + if errors.As(err, &me) { + switch me.Number { + case 1146: + atomic.AddUint64(&c.unknownTable, 1) + return + case 1054: + atomic.AddUint64(&c.unknownColumn, 1) + return + case 1062: + atomic.AddUint64(&c.duplicateEntry, 1) + return + } + } + atomic.AddUint64(&c.otherError, 1) +} + +type dmlSnapshot struct { + Total uint64 `json:"total"` + Success uint64 `json:"success"` + UnknownTable uint64 `json:"unknown_table"` + UnknownColumn uint64 `json:"unknown_column"` + DuplicateEntry uint64 `json:"duplicate_entry"` + OtherError uint64 `json:"other_error"` +} + +func (c *dmlCounters) snapshot() dmlSnapshot { + return dmlSnapshot{ + Total: atomic.LoadUint64(&c.total), + Success: atomic.LoadUint64(&c.success), + UnknownTable: atomic.LoadUint64(&c.unknownTable), + UnknownColumn: atomic.LoadUint64(&c.unknownColumn), + DuplicateEntry: atomic.LoadUint64(&c.duplicateEntry), + OtherError: atomic.LoadUint64(&c.otherError), + } +} + +func dmlWorker( + ctx context.Context, + db *sql.DB, + model *clusterModel, + seed int64, + workerID int, + activeWorkers *int32, + cfg dmlConfig, + counters *dmlCounters, + motifStep *int32, +) { + // dmlWorker generates best-effort DML against upstream. + // + // Concurrency control: + // - Spawn MaxWorkers goroutines, but only workerID < activeWorkers are "active". + // - healthAndAutotuneLoop adjusts activeWorkers based on checkpoint liveness. + // + // Schema handling: + // - DML generation reads the table schema under lock and then executes outside the lock. + // - DDL may race with DML, so unknown table/column errors are tracked and tolerated. + rng := rand.New(rand.NewSource(seed + int64(workerID))) + + for { + select { + case <-ctx.Done(): + return + default: + } + + if int32(workerID) >= atomic.LoadInt32(activeWorkers) { + _ = sleepWithContext(ctx, 200*time.Millisecond) + continue + } + + tbl := model.pickTableForDML(rng, cfg.HotspotRatio) + var ( + stmt string + args []any + err error + ) + + if tbl.isMotif { + stmt, args, err = buildMotifDML(rng, tbl, atomic.LoadInt32(motifStep)) + } else { + stmt, args, err = buildGenericDML(rng, tbl) + } + if err != nil { + // Internal generation error; keep the worker alive. + counters.record(err) + _ = sleepWithContext(ctx, 50*time.Millisecond) + continue + } + if stmt == "" { + _ = sleepWithContext(ctx, 20*time.Millisecond) + continue + } + + _, execErr := db.ExecContext(ctx, stmt, args...) + counters.record(execErr) + } +} + +func buildGenericDML(rng *rand.Rand, tbl *table) (string, []any, error) { + tbl.mu.Lock() + defer tbl.mu.Unlock() + if !tbl.exists { + return "", nil, nil + } + + // Keep the overall mix stable: + // - INSERT is the dominant operation + // - UPDATE/DELETE provide mutation pressure + // + // For tables without a single-column primary key (keyless or composite PK), UPDATE/DELETE + // fall back to bounded operations using LIMIT 1 to avoid requiring key materialization. + switch rng.Intn(10) { + case 0, 1: + stmt, args, err := buildDeleteLocked(rng, tbl) + if stmt != "" || err != nil { + return stmt, args, err + } + case 2, 3: + stmt, args, err := buildUpdateLocked(rng, tbl) + if stmt != "" || err != nil { + return stmt, args, err + } + default: + // Fall through to INSERT. + } + return buildInsertLocked(rng, tbl, nil) +} + +func buildMotifDML(rng *rand.Rand, tbl *table, step int32) (string, []any, error) { + // Motif DML intentionally omits some columns to exercise default value drift + // and schema evolution patterns coordinated by motif.go. + tbl.mu.Lock() + defer tbl.mu.Unlock() + if !tbl.exists { + return "", nil, nil + } + + omit := map[string]struct{}{} + for _, c := range tbl.schema.columns { + if c.name == "site_code" { + // Always omit site_code to exercise default drift. + omit["site_code"] = struct{}{} + break + } + } + + // Before PK is added, focus on inserts to create rows before/after default drift. + if step < 3 { + return buildInsertLocked(rng, tbl, omit) + } + + // After PK is added (a, site_code), only update non-frozen rows inserted after default is unified. + if tbl.motifUnifiedStart > 0 && tbl.nextID > tbl.motifUnifiedStart && rng.Intn(4) == 0 { + return buildMotifUpdateAfterUnifiedLocked(rng, tbl) + } + return buildInsertLocked(rng, tbl, omit) +} + +func buildInsertLocked(rng *rand.Rand, tbl *table, omitCols map[string]struct{}) (string, []any, error) { + // Inserts are generated from a schema snapshot taken under table lock. + schema := tbl.schema.clone() + rowID := tbl.nextID + tbl.nextID++ + + var cols []column + for _, c := range schema.columns { + if c.generated != "" { + continue + } + if omitCols != nil { + if _, ok := omitCols[c.name]; ok { + continue + } + } + cols = append(cols, c) + } + if len(cols) == 0 { + return "", nil, nil + } + + colNames := make([]string, 0, len(cols)) + placeholders := make([]string, 0, len(cols)) + args := make([]any, 0, len(cols)) + for _, c := range cols { + colNames = append(colNames, c.name) + placeholders = append(placeholders, "?") + args = append(args, buildRandomValue(rng, tbl, c, rowID)) + } + + stmt := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", + tbl.fqName(), + backtickJoin(colNames), + strings.Join(placeholders, ","), + ) + return stmt, args, nil +} + +func buildUpdateLocked(rng *rand.Rand, tbl *table) (string, []any, error) { + schema := tbl.schema.clone() + + var candidates []column + for _, c := range schema.columns { + if c.generated != "" { + continue + } + if containsString(schema.primaryKey, c.name) { + continue + } + candidates = append(candidates, c) + } + + if len(schema.primaryKey) == 1 { + // Single-column PK: do targeted updates by PK to keep the operation deterministic. + pk := schema.primaryKey[0] + if tbl.nextID <= 1 { + return "", nil, nil + } + key := int64(rng.Intn(int(tbl.nextID-1)) + 1) + if len(candidates) == 0 { + // As a last resort, update the PK itself to still generate UPDATE traffic. + // Pick a new key different from the old one. + newKey := key + int64(rng.Intn(1024)+1) + stmt := fmt.Sprintf("UPDATE %s SET `%s`=? WHERE `%s`=?", + tbl.fqName(), + pk, + pk, + ) + return stmt, []any{newKey, key}, nil + } + col := candidates[rng.Intn(len(candidates))] + stmt := fmt.Sprintf("UPDATE %s SET `%s`=? WHERE `%s`=?", + tbl.fqName(), + col.name, + pk, + ) + args := []any{buildRandomValue(rng, tbl, col, key), key} + return stmt, args, nil + } + + // Keyless or composite PK tables: do a bounded update without relying on key materialization. + if len(candidates) == 0 { + for _, c := range schema.columns { + if c.generated != "" { + continue + } + candidates = append(candidates, c) + } + } + if len(candidates) == 0 { + return "", nil, nil + } + col := candidates[rng.Intn(len(candidates))] + rowID := tbl.nextID + if rowID <= 0 { + rowID = 1 + } + stmt := fmt.Sprintf("UPDATE %s SET `%s`=? LIMIT 1", + tbl.fqName(), + col.name, + ) + return stmt, []any{buildRandomValue(rng, tbl, col, rowID)}, nil +} + +func buildDeleteLocked(rng *rand.Rand, tbl *table) (string, []any, error) { + schema := tbl.schema.clone() + if len(schema.primaryKey) == 1 { + pk := schema.primaryKey[0] + if tbl.nextID <= 1 { + return "", nil, nil + } + key := int64(rng.Intn(int(tbl.nextID-1)) + 1) + stmt := fmt.Sprintf("DELETE FROM %s WHERE `%s`=?", tbl.fqName(), pk) + return stmt, []any{key}, nil + } + // Keyless or composite PK tables: do a bounded delete without requiring keys. + return fmt.Sprintf("DELETE FROM %s LIMIT 1", tbl.fqName()), nil, nil +} + +func buildMotifUpdateAfterUnifiedLocked(rng *rand.Rand, tbl *table) (string, []any, error) { + // This is only valid after PK evolution: PRIMARY KEY (a, site_code). + schema := tbl.schema.clone() + if len(schema.primaryKey) != 2 { + return "", nil, nil + } + if tbl.nextID <= tbl.motifUnifiedStart { + return "", nil, nil + } + a := int64(rng.Intn(int(tbl.nextID-tbl.motifUnifiedStart)) + int(tbl.motifUnifiedStart)) + + stmt := fmt.Sprintf("UPDATE %s SET `b`=? WHERE `a`=? AND `site_code`=''", + tbl.fqName(), + ) + return stmt, []any{int32(rng.Intn(1_000_000)), a}, nil +} + +func buildRandomValue(rng *rand.Rand, tbl *table, c column, rowID int64) any { + // Keep payloads deterministic enough for triage, and keep SQL text ASCII-only by using placeholders. + switch strings.ToUpper(c.typ.base) { + case "BIGINT": + if c.name == "id" { + return rowID + } + return deterministicInt64(rowID) + case "INT": + if c.name == "a" && tbl.isMotif { + return int32(rowID) + } + return int32(rng.Intn(1_000_000)) + case "VARCHAR": + if c.name == "pad" { + return strings.Repeat("x", 256) + } + return randASCII(rng, min(32, max(8, c.typ.varcharN/2))) + case "DATETIME": + return deterministicTime(rowID) + case "DECIMAL": + return deterministicDecimal(rowID) + case "JSON": + return fmt.Sprintf("{\"id\":%d,\"tbl\":\"%s\"}", rowID, tbl.name) + case "VARBINARY": + return []byte(fmt.Sprintf("%064x", rowID)) + default: + return nil + } +} + +func randASCII(rng *rand.Rand, n int) string { + const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letters[rng.Intn(len(letters))] + } + return string(b) +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/tests/utils/random_ddl_test_runner/dml_test.go b/tests/utils/random_ddl_test_runner/dml_test.go new file mode 100644 index 0000000000..c448cf721b --- /dev/null +++ b/tests/utils/random_ddl_test_runner/dml_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "math/rand" + "strings" + "testing" +) + +func TestRandASCII_IsASCIIAlphaNum(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + s := randASCII(rng, 128) + for i := 0; i < len(s); i++ { + b := s[i] + isNum := b >= '0' && b <= '9' + isLower := b >= 'a' && b <= 'z' + isUpper := b >= 'A' && b <= 'Z' + if !(isNum || isLower || isUpper) { + t.Fatalf("unexpected byte %q in %q", b, s) + } + } +} + +func TestMotifInsert_OmitsSiteCode(t *testing.T) { + tbl := &table{ + db: "db1", + name: "t03", + isMotif: true, + schema: tableSchema{ + columns: []column{ + {name: "a", typ: colType{base: "INT"}, nullable: false}, + {name: "b", typ: colType{base: "INT"}, nullable: false}, + {name: "site_code", typ: colType{base: "VARCHAR", varcharN: 64}, nullable: false, defaultSQL: "'100'"}, + }, + primaryKey: []string{"a", "site_code"}, + }, + exists: true, + nextID: 100, + frozen: map[int64]struct{}{}, + } + rng := rand.New(rand.NewSource(1)) + stmt, _, err := buildMotifDML(rng, tbl, 3) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if strings.Contains(stmt, "site_code") { + t.Fatalf("expected site_code to be omitted, stmt=%s", stmt) + } +} diff --git a/tests/utils/random_ddl_test_runner/extra_workers.go b/tests/utils/random_ddl_test_runner/extra_workers.go new file mode 100644 index 0000000000..d446713dfc --- /dev/null +++ b/tests/utils/random_ddl_test_runner/extra_workers.go @@ -0,0 +1,185 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "math/rand" + "strings" + "sync/atomic" + "time" +) + +func bigTxnWorker( + ctx context.Context, + db *sql.DB, + model *clusterModel, + seed int64, + cfg dmlConfig, + activeWorkers *int32, +) { + // bigTxnWorker periodically runs large insert transactions to stress: + // - large message paths for MQ sinks, + // - large commit and apply paths for MySQL sink. + if !cfg.BigTxnEnabled || cfg.BigTxnInterval.Duration <= 0 { + return + } + rng := rand.New(rand.NewSource(seed)) + ticker := time.NewTicker(cfg.BigTxnInterval.Duration) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + + // When the overall DML pool is heavily degraded, skip big transactions to help recovery. + if atomic.LoadInt32(activeWorkers) <= 4 { + continue + } + + if len(model.splitTables) == 0 { + continue + } + tbl := model.splitTables[rng.Intn(len(model.splitTables))] + rows := cfg.BigTxnRowsMin + if cfg.BigTxnRowsMax > cfg.BigTxnRowsMin { + rows = cfg.BigTxnRowsMin + rng.Intn(cfg.BigTxnRowsMax-cfg.BigTxnRowsMin+1) + } + _ = runBigInsertTxn(ctx, db, tbl, rows) + } +} + +func runBigInsertTxn(ctx context.Context, db *sql.DB, tbl *table, rows int) error { + // Build a single multi-row INSERT inside a transaction to create a "big txn" workload. + tbl.mu.Lock() + if !tbl.exists { + tbl.mu.Unlock() + return nil + } + schema := tbl.schema.clone() + startID := tbl.nextID + tbl.nextID += int64(rows) + tbl.mu.Unlock() + + var cols []column + for _, c := range schema.columns { + if c.generated != "" { + continue + } + cols = append(cols, c) + } + if len(cols) == 0 { + return nil + } + + colNames := make([]string, 0, len(cols)) + for _, c := range cols { + colNames = append(colNames, c.name) + } + + var valuesSQL strings.Builder + var args []any + for i := 0; i < rows; i++ { + if i > 0 { + valuesSQL.WriteString(",") + } + valuesSQL.WriteString("(") + for j := range cols { + if j > 0 { + valuesSQL.WriteString(",") + } + valuesSQL.WriteString("?") + } + valuesSQL.WriteString(")") + rowID := startID + int64(i) + for _, c := range cols { + args = append(args, buildRandomValue(rand.New(rand.NewSource(rowID)), tbl, c, rowID)) + } + } + + stmt := fmt.Sprintf("INSERT INTO %s (%s) VALUES %s", + tbl.fqName(), + backtickJoin(colNames), + valuesSQL.String(), + ) + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + _ = tx.Rollback() + return err + } + return tx.Commit() +} + +func conflictWriter( + ctx context.Context, + db *sql.DB, + model *clusterModel, + seed int64, + cfg dmlConfig, + counters *dmlCounters, +) { + // conflictWriter continuously upserts into a small key space to create write conflicts. + // This targets row-level contention and duplicate key paths. + if !cfg.KeyConflictEnabled || cfg.KeyConflictKeyspace <= 0 { + return + } + rng := rand.New(rand.NewSource(seed)) + targetTables := collectConflictTables(model) + if len(targetTables) == 0 { + return + } + + for { + select { + case <-ctx.Done(): + return + default: + } + + tbl := targetTables[rng.Intn(len(targetTables))] + tbl.mu.Lock() + exists := tbl.exists + tbl.mu.Unlock() + if !exists { + _ = sleepWithContext(ctx, 200*time.Millisecond) + continue + } + + key := rng.Intn(cfg.KeyConflictKeyspace) + 1 + stmt := fmt.Sprintf("INSERT INTO %s (`id`,`a`,`b`,`c`,`d`,`e`,`bin`) VALUES (?,?,?,?,?,?,?) "+ + "ON DUPLICATE KEY UPDATE `a`=VALUES(`a`),`b`=VALUES(`b`),`c`=VALUES(`c`)", + tbl.fqName(), + ) + args := []any{ + int64(key), + int32(rng.Intn(1_000_000)), + randASCII(rng, 16), + deterministicDecimal(int64(key)), + deterministicTime(int64(key)), + fmt.Sprintf("{\"k\":%d}", key), + []byte(fmt.Sprintf("%064x", key)), + } + _, err := db.ExecContext(ctx, stmt, args...) + counters.record(err) + _ = sleepWithContext(ctx, 20*time.Millisecond) + } +} + +func collectConflictTables(model *clusterModel) []*table { + // Pick a stable target table for conflict writes to keep the workload deterministic. + var out []*table + for _, t := range model.churnTables { + // Use a single, deterministic churn family (t10) which is guaranteed to have `id` PK in initial schema. + if t.name == "t10" { + out = append(out, t) + } + } + return out +} diff --git a/tests/utils/random_ddl_test_runner/failover.go b/tests/utils/random_ddl_test_runner/failover.go new file mode 100644 index 0000000000..ddae881feb --- /dev/null +++ b/tests/utils/random_ddl_test_runner/failover.go @@ -0,0 +1,161 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "net/http" + "os" + "os/exec" + "strconv" + "time" +) + +func (r *runner) failoverLoop(ctx context.Context, motifStep *int32, trace *ddlTrace) error { + _ = motifStep + _ = trace + + // failoverLoop randomly kills and restarts captures to simulate process-level failover. + // + // This is only meaningful when the integration harness has started multiple captures + // (and the addresses are provided via config.failover.capture_addrs). + // + // The restart uses the integration helper scripts (run_cdc_server/kill_cdc_pid) which + // are expected to be in PATH when run under tests/integration_tests/*. + minD := r.cfg.Failover.MinInterval.Duration + maxD := r.cfg.Failover.MaxInterval.Duration + if maxD < minD { + maxD, minD = minD, maxD + } + if minD <= 0 { + minD = 20 * time.Second + } + if maxD <= 0 { + maxD = 40 * time.Second + } + if len(r.cfg.Failover.CaptureAddrs) == 0 { + return nil + } + + rng := rand.New(rand.NewSource(r.cfg.Seed + 30_000)) + restartRound := 0 + + for { + sleep := minD + if maxD > minD { + sleep = minD + time.Duration(rng.Int63n(int64(maxD-minD))) + } + if err := sleepWithContext(ctx, sleep); err != nil { + return nil + } + + if rng.Float64() < r.cfg.Failover.GatedProbability { + // Optional gating: avoid failover when checkpoint is not advancing to reduce + // the chance of amplifying an existing stall. + st1, err := r.getChangefeedStatus(ctx) + if err != nil { + return err + } + if err := sleepWithContext(ctx, 3*time.Second); err != nil { + return nil + } + st2, err := r.getChangefeedStatus(ctx) + if err != nil { + return err + } + if st1.Checkpoint != 0 && st1.Checkpoint == st2.Checkpoint { + r.logger.Printf("failover gated: checkpoint did not advance (%d)", st1.Checkpoint) + continue + } + } + + target := r.cfg.Failover.CaptureAddrs[rng.Intn(len(r.cfg.Failover.CaptureAddrs))] + pid, err := getCapturePID(ctx, target, r.cfg.CDC.User, r.cfg.CDC.Password) + if err != nil { + r.logger.Printf("failover: cannot get pid addr=%s err=%v", target, err) + continue + } + if pid == 0 { + r.logger.Printf("failover: empty pid addr=%s", target) + continue + } + + r.logger.Printf("failover: killing capture addr=%s pid=%d", target, pid) + if err := execCommand(ctx, "kill_cdc_pid", strconv.Itoa(pid)); err != nil { + r.logger.Printf("failover: kill failed pid=%d err=%v", pid, err) + continue + } + + restartRound++ + suffix := fmt.Sprintf("failover-%d", restartRound) + r.logger.Printf("failover: restarting capture addr=%s suffix=%s", target, suffix) + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + lastErr = execCommand(ctx, "run_cdc_server", + "--workdir", r.cfg.Workdir, + "--binary", r.cfg.Failover.CdcBinary, + "--logsuffix", suffix, + "--addr", target, + ) + if lastErr == nil { + break + } + r.logger.Printf("failover: restart attempt=%d err=%v", attempt+1, lastErr) + if err := sleepWithContext(ctx, 3*time.Second); err != nil { + return nil + } + } + if lastErr != nil { + return lastErr + } + } +} + +func getCapturePID(ctx context.Context, addr, user, password string) (int, error) { + u := fmt.Sprintf("http://%s/status", addr) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return 0, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusUnauthorized { + req2, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return 0, err + } + req2.SetBasicAuth(user, password) + resp2, err := http.DefaultClient.Do(req2) + if err != nil { + return 0, err + } + defer resp2.Body.Close() + resp = resp2 + } + + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("status http %d", resp.StatusCode) + } + + var raw map[string]any + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + return 0, err + } + p, ok := raw["pid"].(float64) + if !ok { + return 0, nil + } + return int(p), nil +} + +func execCommand(ctx context.Context, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/tests/utils/random_ddl_test_runner/health.go b/tests/utils/random_ddl_test_runner/health.go new file mode 100644 index 0000000000..12287982d5 --- /dev/null +++ b/tests/utils/random_ddl_test_runner/health.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "time" +) + +type changefeedStatus struct { + State string + Checkpoint uint64 +} + +func (r *runner) getChangefeedStatus(ctx context.Context) (changefeedStatus, error) { + // Query TiCDC OpenAPI to obtain changefeed state and checkpoint tso. + // + // The integration tests use a fixed basic auth user/password. This runner keeps the + // request logic small and dependency-free to remain easy to vendor into test envs. + if r.cfg.CDC.ChangefeedID == "" { + return changefeedStatus{}, fmt.Errorf("cdc.changefeed_id is required") + } + + u := url.URL{ + Scheme: "http", + Host: r.cfg.CDC.Addr, + Path: "/api/v2/changefeeds/" + url.PathEscape(r.cfg.CDC.ChangefeedID) + "/status", + } + q := u.Query() + q.Set("keyspace", r.cfg.CDC.Keyspace) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return changefeedStatus{}, err + } + req.SetBasicAuth(r.cfg.CDC.User, r.cfg.CDC.Password) + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return changefeedStatus{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return changefeedStatus{}, fmt.Errorf("cdc status http %d: %s", resp.StatusCode, string(b)) + } + + var raw map[string]any + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&raw); err != nil { + return changefeedStatus{}, err + } + + state, _ := raw["state"].(string) + checkpoint := parseUint64(raw["checkpoint_tso"]) + if checkpoint == 0 { + checkpoint = parseUint64(raw["checkpoint_ts"]) + } + return changefeedStatus{ + State: state, + Checkpoint: checkpoint, + }, nil +} + +func parseUint64(v any) uint64 { + // TiCDC API fields may be encoded as number or string depending on endpoint/version. + switch x := v.(type) { + case float64: + if x < 0 { + return 0 + } + return uint64(x) + case string: + n, err := strconv.ParseUint(x, 10, 64) + if err != nil { + return 0 + } + return n + default: + return 0 + } +} diff --git a/tests/utils/random_ddl_test_runner/logger.go b/tests/utils/random_ddl_test_runner/logger.go new file mode 100644 index 0000000000..00cdb6d0ab --- /dev/null +++ b/tests/utils/random_ddl_test_runner/logger.go @@ -0,0 +1,21 @@ +package main + +import ( + "io" + "log" + "os" + "path/filepath" +) + +func newRunnerLogger(workdir string) (*log.Logger, func(), error) { + if err := os.MkdirAll(workdir, 0o755); err != nil { + return nil, nil, err + } + f, err := os.OpenFile(filepath.Join(workdir, "runner.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return nil, nil, err + } + w := io.MultiWriter(os.Stdout, f) + l := log.New(w, "", log.LstdFlags|log.Lmicroseconds|log.LUTC) + return l, func() { _ = f.Close() }, nil +} diff --git a/tests/utils/random_ddl_test_runner/logscan.go b/tests/utils/random_ddl_test_runner/logscan.go new file mode 100644 index 0000000000..205e305d6c --- /dev/null +++ b/tests/utils/random_ddl_test_runner/logscan.go @@ -0,0 +1,224 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sort" + "strings" +) + +func scanLogsForPatterns(workdir string, patterns []string, failOnMatch bool, logger *log.Logger) error { + // scanLogsForPatterns is a lightweight alternative to external tools (e.g., rg) so the + // runner can be used in minimal environments. + // + // Implementation notes: + // - Convert to lowercase on the fly and use bytes.Contains for substring search. + // - Use bufio.Reader.ReadLine to cap memory and handle very long lines by stitching + // a small suffix ("carry") across fragments. + files, err := collectLogFiles(workdir) + if err != nil { + return err + } + if len(files) == 0 { + if logger != nil { + logger.Printf("log scan: no log files found under %s", workdir) + } + return nil + } + + lowerPatterns := make([]string, 0, len(patterns)) + for _, p := range patterns { + lowerPatterns = append(lowerPatterns, strings.ToLower(p)) + } + + type hit struct { + file string + line int + pat string + } + var hits []hit + + maxPatternLen := 0 + patternBytes := make([][]byte, 0, len(lowerPatterns)) + for _, p := range lowerPatterns { + b := []byte(p) + patternBytes = append(patternBytes, b) + if scanPatternMaxLen(p) > maxPatternLen { + maxPatternLen = scanPatternMaxLen(p) + } + } + + for _, path := range files { + f, err := os.Open(path) + if err != nil { + return err + } + lineNo := 0 + reader := bufio.NewReaderSize(f, 256*1024) + + carry := make([]byte, 0, maxPatternLen) + scratch := make([]byte, 0, 256*1024) + tmp := make([]byte, 0, 256*1024) + lineMatched := false + + for { + part, isPrefix, err := reader.ReadLine() + if err != nil { + if err == io.EOF { + break + } + _ = f.Close() + return err + } + + if !lineMatched { + tmp = append(tmp[:0], part...) + for i := range tmp { + c := tmp[i] + if c >= 'A' && c <= 'Z' { + tmp[i] = c + ('a' - 'A') + } + } + + scratch = append(scratch[:0], carry...) + scratch = append(scratch, tmp...) + + for i, p := range patternBytes { + if logLineMatchesPattern(scratch, p, lowerPatterns[i]) { + hits = append(hits, hit{file: filepath.Base(path), line: lineNo + 1, pat: lowerPatterns[i]}) + lineMatched = true + break + } + } + } + + if !isPrefix { + lineNo++ + carry = carry[:0] + lineMatched = false + continue + } + // Keep a small suffix from the previous fragment to detect patterns spanning boundaries. + if len(scratch) == 0 { + carry = carry[:0] + continue + } + keep := maxPatternLen - 1 + if keep <= 0 { + carry = carry[:0] + continue + } + if keep > len(scratch) { + keep = len(scratch) + } + carry = append(carry[:0], scratch[len(scratch)-keep:]...) + } + + _ = f.Close() + } + + if len(hits) == 0 { + if logger != nil { + logger.Printf("log scan: no matches") + } + return nil + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].file != hits[j].file { + return hits[i].file < hits[j].file + } + return hits[i].line < hits[j].line + }) + + if logger != nil { + logger.Printf("log scan: found %d matches", len(hits)) + for i := 0; i < len(hits) && i < 20; i++ { + logger.Printf("log scan match: file=%s line=%d pattern=%q", hits[i].file, hits[i].line, hits[i].pat) + } + } + + if failOnMatch { + return fmt.Errorf("log scan found %d panic/fatal/race matches", len(hits)) + } + return nil +} + +var ( + logScanFatalLevel = []byte("[fatal]") + logScanFatalKV = []byte("level=fatal") + logScanFatalPrefix = []byte("fatal error:") + logScanPanicLevel = []byte("[panic]") + logScanPanicKV = []byte("level=panic") + logScanPanicPrefix = []byte("panic:") +) + +func scanPatternMaxLen(pattern string) int { + switch pattern { + case "fatal": + return maxInt(len(logScanFatalLevel), len(logScanFatalKV), len(logScanFatalPrefix)) + case "panic": + return maxInt(len(logScanPanicLevel), len(logScanPanicKV), len(logScanPanicPrefix)) + default: + return len(pattern) + } +} + +func logLineMatchesPattern(lowerLine, patternBytes []byte, pattern string) bool { + switch pattern { + case "fatal": + trimmed := bytes.TrimLeft(lowerLine, " \t") + return bytes.Contains(lowerLine, logScanFatalLevel) || + bytes.Contains(lowerLine, logScanFatalKV) || + bytes.HasPrefix(trimmed, logScanFatalPrefix) + case "panic": + trimmed := bytes.TrimLeft(lowerLine, " \t") + return bytes.Contains(lowerLine, logScanPanicLevel) || + bytes.Contains(lowerLine, logScanPanicKV) || + bytes.HasPrefix(trimmed, logScanPanicPrefix) + default: + return bytes.Contains(lowerLine, patternBytes) + } +} + +func maxInt(first int, rest ...int) int { + out := first + for _, v := range rest { + if v > out { + out = v + } + } + return out +} + +func collectLogFiles(workdir string) ([]string, error) { + globs := []string{ + filepath.Join(workdir, "runner.log"), + filepath.Join(workdir, "ddl_trace.log"), + filepath.Join(workdir, "stdout*.log"), + filepath.Join(workdir, "cdc*.log"), + filepath.Join(workdir, "cdc_*_consumer*.log"), + filepath.Join(workdir, "cdc_*_consumer_stdout*.log"), + } + seen := make(map[string]struct{}) + for _, g := range globs { + m, err := filepath.Glob(g) + if err != nil { + return nil, err + } + for _, p := range m { + seen[p] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for p := range seen { + out = append(out, p) + } + sort.Strings(out) + return out, nil +} diff --git a/tests/utils/random_ddl_test_runner/logscan_test.go b/tests/utils/random_ddl_test_runner/logscan_test.go new file mode 100644 index 0000000000..b440c95976 --- /dev/null +++ b/tests/utils/random_ddl_test_runner/logscan_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestScanLogsForPatternsIgnoresPayloadSubstrings(t *testing.T) { + dir := t.TempDir() + content := []byte(` +[2026/06/16 13:31:34.281 +08:00] [DEBUG] [basic_dispatcher.go:600] ["dispatcher receive all event"] [event="Rows: Insert: Row: 1, Bb8bdTFTEIN9i3spwifGjZj3AmFAtalR;"] +[2026/06/16 13:36:34.814 +08:00] [DEBUG] [basic_dispatcher.go:600] ["dispatcher receive all event"] [event="Rows: Insert: Row: 2, 1YCs3x0WFrKYaheC3jpXpAnicxBqG3pe;"] +`) + if err := os.WriteFile(filepath.Join(dir, "cdc.log"), content, 0o644); err != nil { + t.Fatal(err) + } + + if err := scanLogsForPatterns(dir, []string{"panic", "fatal", "DATA RACE"}, true, nil); err != nil { + t.Fatalf("scanLogsForPatterns() unexpected error = %v", err) + } +} + +func TestScanLogsForPatternsDetectsFatalLogLevel(t *testing.T) { + dir := t.TempDir() + content := []byte(`[2026/06/16 13:31:34.281 +08:00] [FATAL] [server.go:1] ["failed"]`) + if err := os.WriteFile(filepath.Join(dir, "cdc.log"), content, 0o644); err != nil { + t.Fatal(err) + } + + if err := scanLogsForPatterns(dir, []string{"fatal"}, true, nil); err == nil { + t.Fatalf("scanLogsForPatterns() expected fatal log level match") + } +} + +func TestScanLogsForPatternsDetectsPanicPrefix(t *testing.T) { + dir := t.TempDir() + content := []byte(`panic: runtime error: invalid memory address`) + if err := os.WriteFile(filepath.Join(dir, "stdout.log"), content, 0o644); err != nil { + t.Fatal(err) + } + + if err := scanLogsForPatterns(dir, []string{"panic"}, true, nil); err == nil { + t.Fatalf("scanLogsForPatterns() expected panic prefix match") + } +} + +func TestScanLogsForPatternsDetectsFatalErrorPrefix(t *testing.T) { + dir := t.TempDir() + content := []byte(`fatal error: concurrent map writes`) + if err := os.WriteFile(filepath.Join(dir, "stdout.log"), content, 0o644); err != nil { + t.Fatal(err) + } + + if err := scanLogsForPatterns(dir, []string{"fatal"}, true, nil); err == nil { + t.Fatalf("scanLogsForPatterns() expected fatal error prefix match") + } +} + +func TestScanLogsForPatternsKeepsCustomSubstringMatch(t *testing.T) { + dir := t.TempDir() + content := []byte(`[INFO] custom marker appeared`) + if err := os.WriteFile(filepath.Join(dir, "cdc.log"), content, 0o644); err != nil { + t.Fatal(err) + } + + if err := scanLogsForPatterns(dir, []string{"custom marker"}, true, nil); err == nil { + t.Fatalf("scanLogsForPatterns() expected custom substring match") + } +} diff --git a/tests/utils/random_ddl_test_runner/main.go b/tests/utils/random_ddl_test_runner/main.go new file mode 100644 index 0000000000..ec05d3da08 --- /dev/null +++ b/tests/utils/random_ddl_test_runner/main.go @@ -0,0 +1,91 @@ +package main + +import ( + "flag" + "fmt" + "os" +) + +// Command random_ddl_test_runner is a deterministic workload generator and verifier +// for TiCDC integration tests. +// +// It has two phases: +// - bootstrap: initialize identical schemas and deterministic seed data on both upstream and downstream. +// - workload: run concurrent random DML/DDL against upstream, while monitoring replication health. +// +// Typical end-to-end flow (driven by tests/integration_tests/*/run.sh): +// +// Components: +// +// H: integration run.sh (harness) +// R: random_ddl_test_runner (this binary) +// U: upstream TiDB +// C: TiCDC capture(s) +// S: sink consumer for MQ/file sinks (optional) +// D: downstream TiDB +// DF: sync_diff_inspector +// +// Sequence (simplified): +// +// H -> U,D: start_tidb_cluster +// H -> R: bootstrap +// R -> U,D: create db/table + insert deterministic rows +// H -> C: start capture(s) +// H -> C: create changefeed (sink uri) +// H -> S: start consumer (optional) +// H -> R: workload +// R -> U: random DML/DDL + motif DDL sequence +// R -> C: poll changefeed status (checkpoint/state) and auto-tune workload +// R -> U: insert finish_mark (replication barrier) +// R -> D: wait finish_mark visible (catch up) +// R -> H: write diff_config.toml +// H -> DF: final upstream vs downstream diff +// +// The runner writes artifacts into , including: +// - runner.log / ddl_trace.log: runner-side logs and executed DDL traces. +// - runner_config.snapshot.json: the effective config used for this run. +// - diff_config.toml: final sync_diff_inspector config for the end-to-end diff. +// - diff_config_syncpoint_.toml: optional syncpoint snapshot diff configs (MySQL sink). +func main() { + var configPath string + var phase string + + flag.StringVar(&configPath, "config", "", "path to runner config json") + flag.StringVar(&phase, "phase", "", "bootstrap|workload") + flag.Parse() + + if configPath == "" || phase == "" { + _, _ = fmt.Fprintln(os.Stderr, "usage: random_ddl_test_runner --config --phase ") + os.Exit(2) + } + + cfg, err := loadConfig(configPath) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + + logger, closeLogger, err := newRunnerLogger(cfg.Workdir) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + defer closeLogger() + + runner := newRunner(cfg, logger) + + switch phase { + case "bootstrap": + err = runner.bootstrap() + case "workload": + err = runner.workload() + default: + err = fmt.Errorf("unknown phase: %s", phase) + } + if err != nil { + logger.Printf("runner failed: %v", err) + _, _ = fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + logger.Printf("runner finished successfully") +} diff --git a/tests/utils/random_ddl_test_runner/model.go b/tests/utils/random_ddl_test_runner/model.go new file mode 100644 index 0000000000..cb48349063 --- /dev/null +++ b/tests/utils/random_ddl_test_runner/model.go @@ -0,0 +1,469 @@ +package main + +import ( + "fmt" + "math" + "math/rand" + "strings" + "sync" + "time" +) + +type domain string + +const ( + domainStable domain = "stable" + domainChurn domain = "churn" + domainSplit domain = "split_candidate" +) + +type colType struct { + base string + varcharN int + decimalP int + decimalS int + varbinaryN int +} + +func (t colType) sql() string { + switch strings.ToUpper(t.base) { + case "VARCHAR": + return fmt.Sprintf("VARCHAR(%d)", t.varcharN) + case "VARBINARY": + return fmt.Sprintf("VARBINARY(%d)", t.varbinaryN) + case "DECIMAL": + return fmt.Sprintf("DECIMAL(%d,%d)", t.decimalP, t.decimalS) + default: + return strings.ToUpper(t.base) + } +} + +type column struct { + name string + typ colType + nullable bool + defaultSQL string + generated string + stored bool +} + +func (c column) sql() string { + var b strings.Builder + b.WriteString("`") + b.WriteString(c.name) + b.WriteString("` ") + b.WriteString(c.typ.sql()) + if c.generated != "" { + kind := "VIRTUAL" + if c.stored { + kind = "STORED" + } + b.WriteString(" GENERATED ALWAYS AS (") + b.WriteString(c.generated) + b.WriteString(") ") + b.WriteString(kind) + } + if !c.nullable { + b.WriteString(" NOT NULL") + } + if c.defaultSQL != "" && c.generated == "" { + b.WriteString(" DEFAULT ") + b.WriteString(c.defaultSQL) + } + return b.String() +} + +type index struct { + name string + columns []string + unique bool +} + +func (idx index) sql() string { + var b strings.Builder + if idx.unique { + b.WriteString("UNIQUE KEY ") + } else { + b.WriteString("KEY ") + } + b.WriteString("`") + b.WriteString(idx.name) + b.WriteString("` (") + for i, c := range idx.columns { + if i > 0 { + b.WriteString(",") + } + b.WriteString("`") + b.WriteString(c) + b.WriteString("`") + } + b.WriteString(")") + return b.String() +} + +type tableSchema struct { + columns []column + primaryKey []string + indexes []index + charset string + collation string + partitionSQL string +} + +func (s tableSchema) clone() tableSchema { + cp := s + cp.columns = append([]column(nil), s.columns...) + cp.primaryKey = append([]string(nil), s.primaryKey...) + cp.indexes = append([]index(nil), s.indexes...) + return cp +} + +func (s tableSchema) createTableSQL(dbName, tableName string) string { + var b strings.Builder + b.WriteString("CREATE TABLE IF NOT EXISTS `") + b.WriteString(dbName) + b.WriteString("`.`") + b.WriteString(tableName) + b.WriteString("` (") + for i, c := range s.columns { + if i > 0 { + b.WriteString(",") + } + b.WriteString("\n ") + b.WriteString(c.sql()) + } + if len(s.primaryKey) > 0 { + b.WriteString(",\n PRIMARY KEY (") + for i, c := range s.primaryKey { + if i > 0 { + b.WriteString(",") + } + b.WriteString("`") + b.WriteString(c) + b.WriteString("`") + } + b.WriteString(")") + } + for _, idx := range s.indexes { + b.WriteString(",\n ") + b.WriteString(idx.sql()) + } + b.WriteString("\n) ") + if s.charset != "" { + b.WriteString("DEFAULT CHARSET=") + b.WriteString(s.charset) + b.WriteString(" ") + } + if s.collation != "" { + b.WriteString("COLLATE=") + b.WriteString(s.collation) + b.WriteString(" ") + } + if s.partitionSQL != "" { + b.WriteString(s.partitionSQL) + b.WriteString(" ") + } + return b.String() +} + +type tableFamily struct { + id int + name string + domain domain + schema tableSchema + isMotif bool +} + +func defaultDatabaseNames() []string { + return []string{"db1", "db2", "db3", "db4", "db5"} +} + +func familyName(i int) string { + return fmt.Sprintf("t%02d", i) +} + +func defaultTableFamilies() []tableFamily { + base := func() tableSchema { + return tableSchema{ + columns: []column{ + {name: "id", typ: colType{base: "BIGINT"}, nullable: false}, + {name: "a", typ: colType{base: "INT"}, nullable: false}, + {name: "b", typ: colType{base: "VARCHAR", varcharN: 64}, nullable: false}, + {name: "c", typ: colType{base: "DECIMAL", decimalP: 10, decimalS: 2}, nullable: false, defaultSQL: "0"}, + {name: "d", typ: colType{base: "DATETIME"}, nullable: false}, + {name: "e", typ: colType{base: "JSON"}, nullable: true}, + {name: "bin", typ: colType{base: "VARBINARY", varbinaryN: 64}, nullable: true}, + }, + primaryKey: []string{"id"}, + } + } + + // Motif table family starts with a not-null unique key and will evolve schema during workload. + motif := tableSchema{ + columns: []column{ + {name: "a", typ: colType{base: "INT"}, nullable: false}, + {name: "b", typ: colType{base: "INT"}, nullable: false}, + }, + indexes: []index{{name: "uk_a", columns: []string{"a"}, unique: true}}, + } + + gbk := base() + gbk.charset = "gbk" + gbk.collation = "gbk_bin" + + // Avoid generated columns in baseline schemas because the current storage sink CSV pipeline + // (cloud storage sink + storage consumer) does not fully support generated columns. + gen := tableSchema{ + columns: []column{ + {name: "id", typ: colType{base: "BIGINT"}, nullable: false}, + {name: "a", typ: colType{base: "INT"}, nullable: false}, + {name: "b", typ: colType{base: "VARBINARY", varbinaryN: 64}, nullable: false}, + {name: "c", typ: colType{base: "VARCHAR", varcharN: 64}, nullable: false}, + }, + primaryKey: []string{"id"}, + } + + keyless := tableSchema{ + columns: []column{ + {name: "id", typ: colType{base: "BIGINT"}, nullable: false}, + {name: "a", typ: colType{base: "INT"}, nullable: false}, + {name: "b", typ: colType{base: "VARCHAR", varcharN: 64}, nullable: false}, + {name: "c", typ: colType{base: "VARCHAR", varcharN: 128}, nullable: false}, + }, + primaryKey: []string{"id"}, + } + + rangePart := base() + rangePart.partitionSQL = "PARTITION BY RANGE (`id`) (PARTITION p0 VALUES LESS THAN (1000000000000), PARTITION p1 VALUES LESS THAN (2000000000000), PARTITION p2 VALUES LESS THAN (3000000000000))" + + hashPart := base() + hashPart.partitionSQL = "PARTITION BY HASH (`id`) PARTITIONS 4" + + split := tableSchema{ + columns: []column{ + {name: "id", typ: colType{base: "BIGINT"}, nullable: false}, + {name: "v", typ: colType{base: "INT"}, nullable: false}, + {name: "pad", typ: colType{base: "VARCHAR", varcharN: 1024}, nullable: false}, + {name: "ts", typ: colType{base: "DATETIME"}, nullable: false}, + }, + primaryKey: []string{"id"}, + partitionSQL: "PARTITION BY RANGE (`id`) (PARTITION p0 VALUES LESS THAN (1000000000000), PARTITION p1 VALUES LESS THAN (2000000000000), PARTITION p2 VALUES LESS THAN (3000000000000))", + } + + families := make([]tableFamily, 0, 20) + for i := 0; i < 20; i++ { + f := tableFamily{ + id: i, + name: familyName(i), + domain: domainStable, + schema: base(), + } + switch { + case i <= 9: + f.domain = domainStable + case i <= 15: + f.domain = domainChurn + default: + f.domain = domainSplit + } + + switch i { + case 0: + f.schema = base() + case 1: + s := base() + s.indexes = append(s.indexes, index{name: "uk_b", columns: []string{"b"}, unique: true}) + f.schema = s + case 2: + s := base() + s.indexes = append(s.indexes, index{name: "idx_a", columns: []string{"a"}, unique: false}) + f.schema = s + case 3: + f.schema = motif + f.isMotif = true + case 4: + s := base() + s.indexes = append(s.indexes, index{name: "uk_a_b", columns: []string{"a", "b"}, unique: true}) + f.schema = s + case 5: + f.schema = keyless + case 6: + f.schema = gen + case 7: + f.schema = rangePart + case 8: + f.schema = hashPart + case 9: + f.schema = gbk + case 16, 17, 18, 19: + f.schema = split + default: + f.schema = base() + } + families = append(families, f) + } + return families +} + +type table struct { + // mu guards mutable state (schema, name, exists, nextID, and motif markers). + mu sync.Mutex + db string + name string + domain domain + family int + isMotif bool + initialSchema tableSchema + schema tableSchema + exists bool + + nextID int64 + frozen map[int64]struct{} + + hot bool + + rangePartitionNextID int + rangePartitionNextBound int64 + + motifUnifiedStart int64 +} + +func (t *table) fqName() string { + return fmt.Sprintf("`%s`.`%s`", t.db, t.name) +} + +type clusterModel struct { + // clusterModel is an in-memory approximation of the workload surface used for + // generating DML and DDL. It is updated only when a DDL succeeds on upstream. + dbs []string + tables []*table + hotTables []*table + coldTables []*table + stableTables []*table + churnTables []*table + splitTables []*table +} + +func buildInitialModel(cfg *config) *clusterModel { + // buildInitialModel constructs the initial schema model used by both bootstrap and workload. + // It must remain deterministic for a given config so tests can be reproduced by seed. + dbs := defaultDatabaseNames() + families := defaultTableFamilies() + + var tables []*table + for _, dbName := range dbs { + for _, fam := range families { + schema := fam.schema.clone() + tbl := &table{ + db: dbName, + name: fam.name, + domain: fam.domain, + family: fam.id, + isMotif: fam.isMotif, + initialSchema: schema.clone(), + schema: schema, + exists: true, + frozen: make(map[int64]struct{}), + } + if strings.Contains(strings.ToUpper(schema.partitionSQL), "RANGE") { + // The initial schemas using RANGE partitioning in this runner define p0/p1/p2. + tbl.rangePartitionNextID = 3 + tbl.rangePartitionNextBound = 3_000_000_000_000 + } + // Use deterministic initial row ranges. + baseRows := int64(cfg.Bootstrap.BaseRowsPerTable) + splitRows := int64(cfg.Bootstrap.SplitRowsPerTable) + switch fam.domain { + case domainSplit: + tbl.nextID = baseRows + splitRows + 1 + default: + tbl.nextID = baseRows + 1 + } + + // Mark per-db hot tables: one stable + one split candidate. + if fam.name == "t00" || fam.name == "t16" { + tbl.hot = true + } + tables = append(tables, tbl) + } + } + + m := &clusterModel{dbs: dbs, tables: tables} + for _, t := range tables { + if t.hot { + m.hotTables = append(m.hotTables, t) + } else { + m.coldTables = append(m.coldTables, t) + } + switch t.domain { + case domainStable: + m.stableTables = append(m.stableTables, t) + case domainChurn: + m.churnTables = append(m.churnTables, t) + case domainSplit: + m.splitTables = append(m.splitTables, t) + } + } + return m +} + +func (m *clusterModel) pickTableForDML(rng *rand.Rand, hotspotRatio float64) *table { + // Prefer "hot" tables with a configurable probability to create hotspot pressure. + if len(m.hotTables) == 0 || len(m.coldTables) == 0 { + return m.tables[rng.Intn(len(m.tables))] + } + if rng.Float64() < hotspotRatio { + return m.hotTables[rng.Intn(len(m.hotTables))] + } + return m.coldTables[rng.Intn(len(m.coldTables))] +} + +func (m *clusterModel) pickTableForDomain(rng *rand.Rand, d domain) *table { + var candidates []*table + switch d { + case domainStable: + candidates = m.stableTables + case domainChurn: + candidates = m.churnTables + case domainSplit: + candidates = m.splitTables + default: + candidates = m.tables + } + if len(candidates) == 0 { + return nil + } + // Retry a few times to avoid frequently selecting dropped churn tables. + for i := 0; i < 10; i++ { + t := candidates[rng.Intn(len(candidates))] + t.mu.Lock() + exists := t.exists + t.mu.Unlock() + if exists { + return t + } + } + return candidates[rng.Intn(len(candidates))] +} + +func deterministicInt64(x int64) int64 { + // A cheap LCG step to mix inputs deterministically. + const a = 6364136223846793005 + const c = 1442695040888963407 + return a*x + c +} + +func asciiStringFromID(prefix string, id int64) string { + // Stable ASCII-only payload. + return fmt.Sprintf("%s_%d", prefix, id) +} + +func deterministicDecimal(id int64) float64 { + return math.Mod(float64(deterministicInt64(id)%100000), 10000) / 100.0 +} + +func deterministicTime(id int64) time.Time { + // Keep within a reasonable range for MySQL/TiDB. + base := int64(1700000000) + return time.Unix(base+(id%86400), 0).UTC() +} diff --git a/tests/utils/random_ddl_test_runner/motif.go b/tests/utils/random_ddl_test_runner/motif.go new file mode 100644 index 0000000000..32163014fc --- /dev/null +++ b/tests/utils/random_ddl_test_runner/motif.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "log" + "sync/atomic" + "time" +) + +// Motif steps: +// 0: initial +// 1: site_code column added with per-db defaults +// 2: site_code default unified to empty string +// 3: primary key evolved to (a, site_code) +func runPrimaryMotif( + ctx context.Context, + db *sql.DB, + model *clusterModel, + motifStep *int32, + trace *ddlTrace, + logger *log.Logger, + profile string, +) { + // The motif is a deterministic DDL sequence on a single table family (t03) intended to + // cover tricky replication cases: + // - adding a new NOT NULL column with different defaults per database + // - unifying defaults over time + // - evolving primary keys after data already exists + // + // DML workers consult motifStep to adjust their write patterns accordingly. + step1At, step2At, step3At := motifSchedule(profile) + + if err := sleepWithContext(ctx, step1At); err != nil { + return + } + if err := motifAddSiteCode(ctx, db, model, trace, logger); err == nil { + atomic.StoreInt32(motifStep, 1) + } + + if err := sleepWithContext(ctx, step2At-step1At); err != nil { + return + } + if err := motifUnifySiteCodeDefault(ctx, db, model, trace, logger); err == nil { + atomic.StoreInt32(motifStep, 2) + } + + if err := sleepWithContext(ctx, step3At-step2At); err != nil { + return + } + if err := motifAddCompositePK(ctx, db, model, trace, logger); err == nil { + atomic.StoreInt32(motifStep, 3) + } +} + +func motifSchedule(profile string) (time.Duration, time.Duration, time.Duration) { + // Use a profile-based schedule so that smoke runs complete all steps quickly, + // while weekly runs keep more steady-state time between transitions. + if profile == "weekly" { + return 2 * time.Minute, 10 * time.Minute, 20 * time.Minute + } + // Smoke mode: ensure all steps execute within a short run. + return 10 * time.Second, 40 * time.Second, 70 * time.Second +} + +func motifAddSiteCode(ctx context.Context, db *sql.DB, model *clusterModel, trace *ddlTrace, logger *log.Logger) error { + for i, dbName := range model.dbs { + defaultVal := fmt.Sprintf("%d", (i+1)*100) + sqlText := fmt.Sprintf("ALTER TABLE `%s`.`t03` ADD COLUMN `site_code` VARCHAR(64) NOT NULL DEFAULT '%s'", + dbName, defaultVal) + _, err := db.ExecContext(ctx, sqlText) + if trace != nil { + trace.record("motif_add_site_code", fmt.Sprintf("`%s`.`t03`", dbName), sqlText, err) + } + if err != nil { + if logger != nil { + logger.Printf("motif step1 failed: db=%s err=%v", dbName, err) + } + return err + } + updateMotifSchemaAfterAdd(dbName, model, defaultVal) + } + return nil +} + +func motifUnifySiteCodeDefault(ctx context.Context, db *sql.DB, model *clusterModel, trace *ddlTrace, logger *log.Logger) error { + for _, dbName := range model.dbs { + sqlText := fmt.Sprintf("ALTER TABLE `%s`.`t03` MODIFY COLUMN `site_code` VARCHAR(64) NOT NULL DEFAULT ''", dbName) + _, err := db.ExecContext(ctx, sqlText) + if trace != nil { + trace.record("motif_unify_site_code_default", fmt.Sprintf("`%s`.`t03`", dbName), sqlText, err) + } + if err != nil { + if logger != nil { + logger.Printf("motif step2 failed: db=%s err=%v", dbName, err) + } + return err + } + updateMotifSchemaAfterUnify(dbName, model) + } + return nil +} + +func motifAddCompositePK(ctx context.Context, db *sql.DB, model *clusterModel, trace *ddlTrace, logger *log.Logger) error { + for _, dbName := range model.dbs { + sqlText := fmt.Sprintf("ALTER TABLE `%s`.`t03` ADD PRIMARY KEY (`a`,`site_code`)", dbName) + _, err := db.ExecContext(ctx, sqlText) + if trace != nil { + trace.record("motif_add_pk", fmt.Sprintf("`%s`.`t03`", dbName), sqlText, err) + } + if err != nil { + if logger != nil { + logger.Printf("motif step3 failed: db=%s err=%v", dbName, err) + } + return err + } + updateMotifSchemaAfterAddPK(dbName, model) + } + return nil +} + +func updateMotifSchemaAfterAdd(dbName string, model *clusterModel, defaultVal string) { + for _, t := range model.tables { + if t.db != dbName || !t.isMotif || t.name != "t03" { + continue + } + t.mu.Lock() + t.schema.columns = append(t.schema.columns, column{ + name: "site_code", + typ: colType{base: "VARCHAR", varcharN: 64}, + nullable: false, + defaultSQL: fmt.Sprintf("'%s'", defaultVal), + }) + t.mu.Unlock() + } +} + +func updateMotifSchemaAfterUnify(dbName string, model *clusterModel) { + for _, t := range model.tables { + if t.db != dbName || !t.isMotif || t.name != "t03" { + continue + } + t.mu.Lock() + for i := range t.schema.columns { + if t.schema.columns[i].name == "site_code" { + t.schema.columns[i].defaultSQL = "''" + break + } + } + // Record the boundary so later updates can avoid touching frozen rows (which have non-empty site_code). + t.motifUnifiedStart = t.nextID + t.mu.Unlock() + } +} + +func updateMotifSchemaAfterAddPK(dbName string, model *clusterModel) { + for _, t := range model.tables { + if t.db != dbName || !t.isMotif || t.name != "t03" { + continue + } + t.mu.Lock() + t.schema.primaryKey = []string{"a", "site_code"} + t.mu.Unlock() + } +} diff --git a/tests/utils/random_ddl_test_runner/runner.go b/tests/utils/random_ddl_test_runner/runner.go new file mode 100644 index 0000000000..75365c6d87 --- /dev/null +++ b/tests/utils/random_ddl_test_runner/runner.go @@ -0,0 +1,18 @@ +package main + +import ( + "log" +) + +type runner struct { + cfg *config + logger *log.Logger +} + +func newRunner(cfg *config, logger *log.Logger) *runner { + // runner is a thin orchestrator. Heavy logic lives in bootstrap.go/workload.go. + return &runner{ + cfg: cfg, + logger: logger, + } +} diff --git a/tests/utils/random_ddl_test_runner/selector.go b/tests/utils/random_ddl_test_runner/selector.go new file mode 100644 index 0000000000..ba4759b2d3 --- /dev/null +++ b/tests/utils/random_ddl_test_runner/selector.go @@ -0,0 +1,71 @@ +package main + +import ( + "math/rand" + "sync" +) + +type ddlSelector struct { + mu sync.Mutex + windowSize int + window []string + counts map[string]int + kinds []ddlKind +} + +func newDDLSelector(kinds []ddlKind, windowSize int) *ddlSelector { + // The selector is stateful: it reduces weight for DDL kinds that were recently successful, + // which helps spread coverage and avoids repeatedly hammering the same schema change. + return &ddlSelector{ + windowSize: windowSize, + counts: make(map[string]int), + kinds: kinds, + } +} + +func (s *ddlSelector) pick(rng *rand.Rand) ddlKind { + // Pick with dynamic weights: + // weight(kind) = baseWeight / (1 + recentSuccessCount(kind)) + // where recentSuccessCount is tracked in a sliding window of the last N successes. + s.mu.Lock() + defer s.mu.Unlock() + + weights := make([]float64, 0, len(s.kinds)) + var sum float64 + for _, k := range s.kinds { + count := s.counts[k.name] + w := k.baseWeight / float64(1+count) + if w < 0.001 { + w = 0.001 + } + weights = append(weights, w) + sum += w + } + x := rng.Float64() * sum + var acc float64 + for i, w := range weights { + acc += w + if x <= acc { + return s.kinds[i] + } + } + return s.kinds[len(s.kinds)-1] +} + +func (s *ddlSelector) record(kindName string) { + // Record a successful DDL kind for weight dampening in a bounded window. + s.mu.Lock() + defer s.mu.Unlock() + + s.window = append(s.window, kindName) + s.counts[kindName]++ + if len(s.window) <= s.windowSize { + return + } + evicted := s.window[0] + s.window = s.window[1:] + s.counts[evicted]-- + if s.counts[evicted] <= 0 { + delete(s.counts, evicted) + } +} diff --git a/tests/utils/random_ddl_test_runner/selector_test.go b/tests/utils/random_ddl_test_runner/selector_test.go new file mode 100644 index 0000000000..42404d874d --- /dev/null +++ b/tests/utils/random_ddl_test_runner/selector_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "math/rand" + "testing" +) + +func TestDDLSelector_CoverageDebtLikeBehavior(t *testing.T) { + kinds := []ddlKind{ + {name: "a", baseWeight: 1}, + {name: "b", baseWeight: 1}, + } + s := newDDLSelector(kinds, 100) + for i := 0; i < 50; i++ { + s.record("a") + } + rng := rand.New(rand.NewSource(1)) + var ca, cb int + for i := 0; i < 1000; i++ { + k := s.pick(rng) + if k.name == "a" { + ca++ + } else if k.name == "b" { + cb++ + } + } + if cb <= ca { + t.Fatalf("expected b to be selected more often than a, got a=%d b=%d", ca, cb) + } +} diff --git a/tests/utils/random_ddl_test_runner/syncpoint_diff.go b/tests/utils/random_ddl_test_runner/syncpoint_diff.go new file mode 100644 index 0000000000..a460b7e8c3 --- /dev/null +++ b/tests/utils/random_ddl_test_runner/syncpoint_diff.go @@ -0,0 +1,471 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync/atomic" + "time" + + "github.com/go-sql-driver/mysql" +) + +type ddlWindow struct { + start uint64 + end uint64 +} + +const mysqlErrNoSuchTable uint16 = 1146 + +func (r *runner) syncpointDiffLoop( + ctx context.Context, + up *sql.DB, + down *sql.DB, + model *clusterModel, + trace *ddlTrace, + successCounter *int32, +) error { + _ = up + _ = trace + + // syncpointDiffLoop periodically runs snapshot diffs based on TiCDC syncpoints. + // + // Motivation: + // - The final diff runs at the end of the test and may not pinpoint when divergence happened. + // - Syncpoints provide pairs of (primary_ts on upstream, secondary_ts on downstream) that + // can be used for snapshot reads. Running diffs at several syncpoints helps localize issues. + // + // Practicality: + // - Snapshot diffing is fragile near DDL windows. We conservatively skip candidates that fall + // into TiDB DDL windows obtained from upstream /ddl/history. + if r.cfg.MySQL.DiffInterval.Duration <= 0 { + return nil + } + ticker := time.NewTicker(r.cfg.MySQL.DiffInterval.Duration) + defer ticker.Stop() + + var ( + lastPrimary uint64 + checked int + ) + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + if checked >= r.cfg.MySQL.MaxDiffChecks { + return nil + } + + n, err := r.runSyncpointDiffChecks(ctx, down, model, 1, &lastPrimary, false) + if err != nil { + return err + } + if n > 0 { + checked += n + if successCounter != nil { + atomic.StoreInt32(successCounter, int32(checked)) + } + } + } +} + +func (r *runner) runSyncpointDiffChecks( + ctx context.Context, + down *sql.DB, + model *clusterModel, + required int, + lastPrimary *uint64, + allowInDDLWindow bool, +) (int, error) { + // Run up to "required" syncpoint diffs and update lastPrimary to advance the cursor. + if required <= 0 { + return 0, nil + } + if lastPrimary == nil { + return 0, fmt.Errorf("lastPrimary must not be nil") + } + + windows, err := fetchDDLWindows(ctx, r.cfg.MySQL.UpstreamStatusHost, r.cfg.MySQL.UpstreamStatusPort) + if err != nil { + return 0, err + } + + checked := 0 + for tries := 0; tries < 50 && checked < required; tries++ { + p, s, got, err := pickNextSyncpointCandidate(ctx, down, *lastPrimary) + if err != nil { + return checked, err + } + if !got { + return checked, nil + } + inWindow := inDDLWindow(p, windows) + if inWindow && !allowInDDLWindow { + r.logger.Printf("syncpoint diff: skip primary_ts=%d (in DDL window)", p) + *lastPrimary = p + continue + } + + confPath := filepath.Join(r.cfg.Workdir, fmt.Sprintf("diff_config_syncpoint_%d.toml", p)) + if err := r.writeSyncpointDiffConfig(confPath, model, p, s); err != nil { + return checked, err + } + + logPath := filepath.Join(r.cfg.Workdir, fmt.Sprintf("sync_diff_inspector_syncpoint_%d.log", p)) + diag, err := r.runSyncDiffInspectorWithSnapshotGuard(ctx, confPath, logPath, 3) + if err != nil { + if ctx.Err() != nil { + return checked, nil + } + if isSkippableSyncDiffFailure(diag) { + r.logger.Printf("syncpoint diff: skip primary_ts=%d (sync diff not applicable, see %s)", p, logPath) + *lastPrimary = p + continue + } + if inWindow { + r.logger.Printf("syncpoint diff: skip primary_ts=%d (diff failed in DDL window, see %s)", p, logPath) + *lastPrimary = p + continue + } + return checked, err + } + + checked++ + *lastPrimary = p + r.logger.Printf("syncpoint diff: success primary_ts=%d secondary_ts=%d", p, s) + } + return checked, nil +} + +func (r *runner) ensureSyncpointDiffAfterWorkload( + ctx context.Context, + down *sql.DB, + model *clusterModel, + required int, +) error { + if required <= 0 { + return nil + } + var lastPrimary uint64 + checked := 0 + for checked < required { + n, err := r.runSyncpointDiffChecks(ctx, down, model, required-checked, &lastPrimary, true) + if err != nil { + return err + } + checked += n + if checked >= required { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("syncpoint diff did not complete: required=%d checked=%d: %w", required, checked, ctx.Err()) + case <-time.After(5 * time.Second): + } + } + return nil +} + +func pickNextSyncpointCandidate(ctx context.Context, down *sql.DB, after uint64) (primary uint64, secondary uint64, ok bool, err error) { + queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + rows, err := down.QueryContext(queryCtx, + "SELECT primary_ts, secondary_ts FROM tidb_cdc.syncpoint_v1 WHERE primary_ts > ? ORDER BY primary_ts ASC LIMIT 200", + after, + ) + if err != nil { + if isNoSuchTableError(err) { + // TiCDC creates tidb_cdc.syncpoint_v1 lazily when the first syncpoint is flushed. + // Treat a missing table as "no candidate yet" so early periodic checks keep waiting. + return 0, 0, false, nil + } + return 0, 0, false, err + } + defer rows.Close() + + for rows.Next() { + var p, s uint64 + if err := rows.Scan(&p, &s); err != nil { + return 0, 0, false, err + } + return p, s, true, nil + } + if err := rows.Err(); err != nil { + return 0, 0, false, err + } + return 0, 0, false, nil +} + +func isNoSuchTableError(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlErrNoSuchTable +} + +func fetchDDLWindows(ctx context.Context, host string, port int) ([]ddlWindow, error) { + // TiDB exposes recent DDL jobs via /ddl/history. We treat the job runtime as a window + // where snapshot reads may be inconsistent across schema versions. + u := fmt.Sprintf("http://%s:%d/ddl/history", host, port) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("ddl history http %d: %s", resp.StatusCode, string(b)) + } + + var v any + if err := json.NewDecoder(resp.Body).Decode(&v); err != nil { + return nil, err + } + var windows []ddlWindow + extractDDLWindows(v, &windows) + + sort.Slice(windows, func(i, j int) bool { return windows[i].start < windows[j].start }) + return windows, nil +} + +func extractDDLWindows(v any, out *[]ddlWindow) { + switch x := v.(type) { + case map[string]any: + start := parseUint64(x["real_start_ts"]) + end := parseUint64(x["FinishedTS"]) + if start != 0 { + // For running DDL jobs, FinishedTS may be 0. Treat it as an open-ended window so + // we can conservatively skip syncpoints that may observe inconsistent snapshots. + if end == 0 { + end = ^uint64(0) + } + *out = append(*out, ddlWindow{start: start, end: end}) + } + for _, vv := range x { + extractDDLWindows(vv, out) + } + case []any: + for _, vv := range x { + extractDDLWindows(vv, out) + } + } +} + +func inDDLWindow(ts uint64, windows []ddlWindow) bool { + for _, w := range windows { + if ts > w.start && ts < w.end { + return true + } + } + return false +} + +func (r *runner) writeSyncpointDiffConfig(path string, model *clusterModel, primary, secondary uint64) error { + // Only diff stable domain tables. + var stable []string + for _, t := range model.stableTables { + stable = append(stable, fmt.Sprintf("%s.%s", t.db, t.name)) + } + sort.Strings(stable) + + var b strings.Builder + b.WriteString("# diff Configuration.\n\n") + b.WriteString("check-thread-count = 4\n\n") + b.WriteString("export-fix-sql = true\n\n") + b.WriteString("check-struct-only = false\n\n") + b.WriteString("[task]\n") + b.WriteString(fmt.Sprintf(" output-dir = %q\n\n", filepath.Join(r.cfg.Workdir, "sync_diff", fmt.Sprintf("syncpoint_%d", primary), "output"))) + b.WriteString(" source-instances = [\"upstream\"]\n\n") + b.WriteString(" target-instance = \"downstream\"\n\n") + b.WriteString(" target-check-tables = [\n") + for i, t := range stable { + sep := "," + if i == len(stable)-1 { + sep = "" + } + b.WriteString(fmt.Sprintf(" %q%s\n", t, sep)) + } + b.WriteString(" ]\n\n") + b.WriteString("[data-sources]\n") + b.WriteString("[data-sources.upstream]\n") + b.WriteString(fmt.Sprintf(" host = %q\n", r.cfg.Upstream.Host)) + b.WriteString(fmt.Sprintf(" port = %d\n", r.cfg.Upstream.Port)) + b.WriteString(fmt.Sprintf(" user = %q\n", r.cfg.Upstream.User)) + b.WriteString(fmt.Sprintf(" password = %q\n", r.cfg.Upstream.Password)) + b.WriteString(fmt.Sprintf(" snapshot = %q\n\n", fmt.Sprintf("%d", primary))) + + b.WriteString("[data-sources.downstream]\n") + b.WriteString(fmt.Sprintf(" host = %q\n", r.cfg.Downstream.Host)) + b.WriteString(fmt.Sprintf(" port = %d\n", r.cfg.Downstream.Port)) + b.WriteString(fmt.Sprintf(" user = %q\n", r.cfg.Downstream.User)) + b.WriteString(fmt.Sprintf(" password = %q\n", r.cfg.Downstream.Password)) + b.WriteString(fmt.Sprintf(" snapshot = %q\n", fmt.Sprintf("%d", secondary))) + + return os.WriteFile(path, []byte(b.String()), 0o644) +} + +type tailBuffer struct { + buf []byte + max int +} + +func newTailBuffer(maxBytes int) *tailBuffer { + return &tailBuffer{max: maxBytes} +} + +func (t *tailBuffer) Write(p []byte) (int, error) { + if t == nil || t.max <= 0 { + return len(p), nil + } + if len(p) >= t.max { + t.buf = append(t.buf[:0], p[len(p)-t.max:]...) + return len(p), nil + } + if len(t.buf)+len(p) <= t.max { + t.buf = append(t.buf, p...) + return len(p), nil + } + overflow := len(t.buf) + len(p) - t.max + t.buf = append(t.buf[overflow:], p...) + return len(p), nil +} + +func (t *tailBuffer) String() string { + if t == nil { + return "" + } + return string(t.buf) +} + +func isSkippableSyncDiffFailure(outputTail string) bool { + // sync_diff_inspector runs snapshot reads and may fail with schema-related errors when a syncpoint + // is observed during (or near) a DDL window. Treat those cases as "invalid syncpoint" and skip. + s := strings.ToLower(outputTail) + switch { + case strings.Contains(s, "unknown column"): + return true + case strings.Contains(s, "no table need to be compared"): + return true + default: + return false + } +} + +const ( + tidbEnableExternalTSReadVar = "tidb_enable_external_ts_read" + externalTSReadOffParam = tidbEnableExternalTSReadVar + "=OFF" +) + +func (r *runner) runSyncDiffInspectorWithSnapshotGuard(ctx context.Context, confPath, logPath string, retries int) (string, error) { + // sync_diff_inspector should compare only the snapshot pair from the config. + // Keep downstream external-ts reads disabled during the diff so any connection + // that misses its configured snapshot cannot fall back to a later syncpoint. + downstream, err := openMySQLWithExtraParams(ctx, r.cfg.Downstream, externalTSReadOffParam) + if err != nil { + return "", err + } + defer func() { + _ = downstream.Close() + }() + + original, err := queryGlobalExternalTSRead(ctx, downstream) + if err != nil { + return "", err + } + if err := setGlobalExternalTSRead(ctx, downstream, "OFF"); err != nil { + return "", err + } + + diag, runErr := runSyncDiffInspector(ctx, confPath, logPath, retries) + if original == "OFF" { + return diag, runErr + } + + restoreCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + restoreErr := setGlobalExternalTSRead(restoreCtx, downstream, original) + cancel() + if restoreErr != nil { + r.logger.Printf("syncpoint diff: failed to restore %s=%s: err=%v", tidbEnableExternalTSReadVar, original, restoreErr) + if runErr == nil { + return diag, restoreErr + } + } + return diag, runErr +} + +func queryGlobalExternalTSRead(ctx context.Context, downstream *sql.DB) (string, error) { + var value string + if err := downstream.QueryRowContext(ctx, "SELECT @@global."+tidbEnableExternalTSReadVar).Scan(&value); err != nil { + return "", err + } + return normalizeExternalTSReadValue(value) +} + +func setGlobalExternalTSRead(ctx context.Context, downstream *sql.DB, value string) error { + normalized, err := normalizeExternalTSReadValue(value) + if err != nil { + return err + } + _, err = downstream.ExecContext(ctx, "SET GLOBAL "+tidbEnableExternalTSReadVar+" = "+normalized) + return err +} + +func normalizeExternalTSReadValue(value string) (string, error) { + switch strings.ToUpper(strings.TrimSpace(value)) { + case "ON", "1", "TRUE": + return "ON", nil + case "OFF", "0", "FALSE": + return "OFF", nil + default: + return "", fmt.Errorf("unexpected %s value: %q", tidbEnableExternalTSReadVar, value) + } +} + +func runSyncDiffInspector(ctx context.Context, confPath, logPath string, retries int) (string, error) { + // sync_diff_inspector output can be large. Keep a tail buffer for diagnostics while + // still appending full logs to a file in the workdir. + if retries < 1 { + retries = 1 + } + + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return "", err + } + defer f.Close() + + var lastTail string + for i := 0; i < retries; i++ { + tail := newTailBuffer(64 * 1024) + w := io.MultiWriter(f, tail) + cmd := exec.CommandContext(ctx, "sync_diff_inspector", "--log-level=debug", "--config="+confPath) + cmd.Stdout = w + cmd.Stderr = w + err = cmd.Run() + if err == nil { + return "", nil + } + lastTail = tail.String() + select { + case <-ctx.Done(): + return lastTail, ctx.Err() + case <-time.After(2 * time.Second): + } + } + return lastTail, err +} diff --git a/tests/utils/random_ddl_test_runner/workload.go b/tests/utils/random_ddl_test_runner/workload.go new file mode 100644 index 0000000000..607276627a --- /dev/null +++ b/tests/utils/random_ddl_test_runner/workload.go @@ -0,0 +1,465 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" +) + +func (r *runner) workload() error { + // workload runs a bounded-duration stress workload against upstream only, while verifying: + // - the changefeed remains in "normal" state, + // - checkpoint continues advancing (with auto-tuning on stalls), + // - optional snapshot diffs at TiCDC syncpoints (MySQL sink), + // - optional capture failover recovery (multi-capture failover case). + // + // After the time budget is consumed, the runner inserts a finish mark row on upstream and + // waits for it to appear on downstream as a "catch-up" barrier, then writes a final + // diff_config.toml for sync_diff_inspector. + ctx, cancel := context.WithTimeout(context.Background(), r.cfg.Duration.Duration) + defer cancel() + + if r.cfg.CDC.ChangefeedID == "" { + return fmt.Errorf("cdc.changefeed_id is required for workload phase") + } + + if err := os.MkdirAll(r.cfg.Workdir, 0o755); err != nil { + return err + } + + cfgSnapshot, _ := json.MarshalIndent(r.cfg, "", " ") + _ = os.WriteFile(filepath.Join(r.cfg.Workdir, "runner_config.snapshot.json"), cfgSnapshot, 0o644) + + r.logger.Printf("workload start: duration=%s seed=%d changefeed=%s sink=%s", + r.cfg.Duration.Duration, r.cfg.Seed, r.cfg.CDC.ChangefeedID, r.cfg.SinkType) + + up, err := openMySQL(ctx, r.cfg.Upstream) + if err != nil { + return err + } + defer func() { _ = up.Close() }() + + down, err := openMySQL(ctx, r.cfg.Downstream) + if err != nil { + return err + } + defer func() { _ = down.Close() }() + + model := buildInitialModel(r.cfg) + + // Initialize frozen rows for the motif family (t03). + for _, t := range model.tables { + if !t.isMotif { + continue + } + t.mu.Lock() + for i := int64(1); i <= int64(r.cfg.Bootstrap.FrozenRowsPerTable); i++ { + t.frozen[i] = struct{}{} + } + t.mu.Unlock() + } + + trace, err := newDDLTrace(r.cfg.Workdir) + if err != nil { + return err + } + defer trace.close() + + var ( + activeDMLWorkers int32 = int32(r.cfg.DML.InitialWorkers) + activeDDLWorkers int32 = int32(r.cfg.DDL.InitialWorkers) + motifStep int32 = 0 + syncpointChecked int32 = 0 + ) + + dmlCounters := &dmlCounters{} + ddlSelector := newDDLSelector(defaultDDLKinds(), 200) + + errCh := make(chan error, 1) + var wg sync.WaitGroup + + // DML workers. + for i := 0; i < r.cfg.DML.MaxWorkers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + dmlWorker(ctx, up, model, r.cfg.Seed+10_000, workerID, &activeDMLWorkers, r.cfg.DML, dmlCounters, &motifStep) + }(i) + } + + // DDL workers. + for i := 0; i < r.cfg.DDL.MaxWorkers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + ddlWorker(ctx, up, model, r.cfg.Seed+20_000, workerID, &activeDDLWorkers, ddlSelector, trace, r.logger) + }(i) + } + + // Primary motif controller. + wg.Add(1) + go func() { + defer wg.Done() + runPrimaryMotif(ctx, up, model, &motifStep, trace, r.logger, r.cfg.Profile) + }() + + // Health monitor + auto-tune. + wg.Add(1) + go func() { + defer wg.Done() + if err := r.healthAndAutotuneLoop(ctx, dmlCounters, &activeDMLWorkers, &activeDDLWorkers); err != nil { + select { + case errCh <- err: + default: + } + cancel() + } + }() + + // Big transactions to stress large commit paths (optional, enabled by default). + if r.cfg.DML.BigTxnEnabled { + wg.Add(1) + go func() { + defer wg.Done() + bigTxnWorker(ctx, up, model, r.cfg.Seed+40_000, r.cfg.DML, &activeDMLWorkers) + }() + } + + // Key conflict writer (optional, enabled by default). + if r.cfg.DML.KeyConflictEnabled { + wg.Add(1) + go func() { + defer wg.Done() + conflictWriter(ctx, up, model, r.cfg.Seed+50_000, r.cfg.DML, dmlCounters) + }() + } + + // MySQL syncpoint diff controller (optional). + if r.cfg.SinkType == "mysql" && r.cfg.MySQL.Enabled { + wg.Add(1) + go func() { + defer wg.Done() + if err := r.syncpointDiffLoop(ctx, up, down, model, trace, &syncpointChecked); err != nil { + select { + case errCh <- err: + default: + } + cancel() + } + }() + } + + // Failover controller (optional). + if r.cfg.Failover.Enabled && len(r.cfg.Failover.CaptureAddrs) > 0 { + wg.Add(1) + go func() { + defer wg.Done() + if err := r.failoverLoop(ctx, &motifStep, trace); err != nil { + select { + case errCh <- err: + default: + } + cancel() + } + }() + } + + <-ctx.Done() + wg.Wait() + + select { + case err := <-errCh: + return err + default: + } + + r.logger.Printf("workload finished, waiting for converge: %s", r.cfg.Verify.ConvergeWait.Duration) + time.Sleep(r.cfg.Verify.ConvergeWait.Duration) + + convergeCtx, convergeCancel := context.WithTimeout(context.Background(), r.cfg.Verify.ConvergeTimeout.Duration) + defer convergeCancel() + + if err := r.createAndWaitFinishMark(convergeCtx, up, down, model); err != nil { + return err + } + + if r.cfg.SinkType == "mysql" && r.cfg.MySQL.Enabled && r.cfg.MySQL.MaxDiffChecks > 0 { + need := r.cfg.MySQL.MaxDiffChecks - int(atomic.LoadInt32(&syncpointChecked)) + if need > 0 { + r.logger.Printf("syncpoint diff: catching up after workload, need=%d", need) + diffCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := r.ensureSyncpointDiffAfterWorkload(diffCtx, down, model, need); err != nil { + return err + } + } + } + + if err := r.writeDMLStats(dmlCounters); err != nil { + return err + } + + // The workload context is already done here. Use a short-lived context for final verification steps. + diffCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := r.writeFinalDiffConfig(diffCtx, up, model); err != nil { + return err + } + + if r.cfg.Verify.LogScanEnabled { + if err := scanLogsForPatterns(r.cfg.Workdir, r.cfg.Verify.PanicPatterns, r.cfg.Verify.FailOnPanicMatch, r.logger); err != nil { + return err + } + } + + return nil +} + +func (r *runner) healthAndAutotuneLoop( + ctx context.Context, + dmlCounters *dmlCounters, + activeDMLWorkers *int32, + activeDDLWorkers *int32, +) error { + // This loop is the guardrail for the stress test: + // - It continuously checks changefeed state and checkpoint progress. + // - It degrades worker concurrency on stalls or low DML success rate to help recovery. + // - It fails the run if checkpoint does not advance for NoAdvanceHard. + ticker := time.NewTicker(r.cfg.Verify.HealthInterval.Duration) + defer ticker.Stop() + + var ( + lastCheckpoint uint64 + lastAdvance = time.Now() + prevSnap = dmlCounters.snapshot() + ) + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + + st, err := r.getChangefeedStatus(ctx) + if err != nil { + return err + } + if st.State != "normal" { + return fmt.Errorf("changefeed state is not normal: %s", st.State) + } + now := time.Now() + if lastCheckpoint == 0 { + lastCheckpoint = st.Checkpoint + lastAdvance = now + } else if st.Checkpoint != 0 && st.Checkpoint != lastCheckpoint { + lastCheckpoint = st.Checkpoint + lastAdvance = now + } + + sinceAdvance := now.Sub(lastAdvance) + snap := dmlCounters.snapshot() + intervalTotal := snap.Total - prevSnap.Total + intervalSuccess := snap.Success - prevSnap.Success + prevSnap = snap + + successRate := 1.0 + if intervalTotal > 0 { + successRate = float64(intervalSuccess) / float64(intervalTotal) + } + + r.logger.Printf("health: state=%s checkpoint=%d since_advance=%s dml_total=%d dml_success=%d success_rate=%.3f active_dml=%d active_ddl=%d", + st.State, st.Checkpoint, sinceAdvance, intervalTotal, intervalSuccess, successRate, + atomic.LoadInt32(activeDMLWorkers), atomic.LoadInt32(activeDDLWorkers)) + + res := autoTuneStep( + sinceAdvance, + successRate, + atomic.LoadInt32(activeDMLWorkers), + atomic.LoadInt32(activeDDLWorkers), + int32(r.cfg.DML.MaxWorkers), + int32(r.cfg.DDL.MaxWorkers), + r.cfg.Verify.NoAdvanceSoft.Duration, + r.cfg.Verify.NoAdvanceHard.Duration, + ) + if res.fail { + return fmt.Errorf("checkpoint did not advance for %s (hard=%s)", sinceAdvance, r.cfg.Verify.NoAdvanceHard.Duration) + } + atomic.StoreInt32(activeDMLWorkers, res.nextDML) + atomic.StoreInt32(activeDDLWorkers, res.nextDDL) + } +} + +func (r *runner) writeDMLStats(counters *dmlCounters) error { + snap := counters.snapshot() + b, err := json.MarshalIndent(snap, "", " ") + if err != nil { + return err + } + return writeFileAtomic(filepath.Join(r.cfg.Workdir, "dml_stats.json"), b) +} + +func (r *runner) writeFinalDiffConfig(ctx context.Context, up *sql.DB, model *clusterModel) error { + // Write a TOML-like config by template to avoid adding new dependencies. + // + // Use actual upstream table existence to build the final diff table list. + // This avoids sync_diff_inspector init failures when churn-domain tables are dropped, + // recovered to a different name, or partially modified due to concurrent DDL. + tables, err := listExistingBaseTables(ctx, up, model.dbs) + if err != nil { + // Fall back to model state when upstream introspection fails. + // This should be rare and keeps the runner resilient to transient DB issues. + r.logger.Printf("final diff: failed to list upstream tables, falling back to model state: err=%v", err) + for _, t := range model.tables { + t.mu.Lock() + exists := t.exists + dbName := t.db + tableName := t.name + t.mu.Unlock() + if !exists { + continue + } + tables = append(tables, fmt.Sprintf("%s.%s", dbName, tableName)) + } + sort.Strings(tables) + } + + var b strings.Builder + b.WriteString("# diff Configuration.\n\n") + b.WriteString("check-thread-count = 4\n\n") + b.WriteString("export-fix-sql = true\n\n") + b.WriteString("check-struct-only = false\n\n") + b.WriteString("[task]\n") + b.WriteString(fmt.Sprintf(" output-dir = %q\n\n", filepath.Join(r.cfg.Workdir, "sync_diff", "output"))) + b.WriteString(" source-instances = [\"upstream\"]\n\n") + b.WriteString(" target-instance = \"downstream\"\n\n") + b.WriteString(" target-check-tables = [\n") + for i, t := range tables { + sep := "," + if i == len(tables)-1 { + sep = "" + } + b.WriteString(fmt.Sprintf(" %q%s\n", t, sep)) + } + b.WriteString(" ]\n\n") + b.WriteString("[data-sources]\n") + b.WriteString("[data-sources.upstream]\n") + b.WriteString(fmt.Sprintf(" host = %q\n", r.cfg.Upstream.Host)) + b.WriteString(fmt.Sprintf(" port = %d\n", r.cfg.Upstream.Port)) + b.WriteString(fmt.Sprintf(" user = %q\n", r.cfg.Upstream.User)) + b.WriteString(fmt.Sprintf(" password = %q\n\n", r.cfg.Upstream.Password)) + + b.WriteString("[data-sources.downstream]\n") + b.WriteString(fmt.Sprintf(" host = %q\n", r.cfg.Downstream.Host)) + b.WriteString(fmt.Sprintf(" port = %d\n", r.cfg.Downstream.Port)) + b.WriteString(fmt.Sprintf(" user = %q\n", r.cfg.Downstream.User)) + b.WriteString(fmt.Sprintf(" password = %q\n", r.cfg.Downstream.Password)) + + return os.WriteFile(filepath.Join(r.cfg.Workdir, "diff_config.toml"), []byte(b.String()), 0o644) +} + +func listExistingBaseTables(ctx context.Context, db *sql.DB, dbs []string) ([]string, error) { + var tables []string + for _, dbName := range dbs { + // dbName is generated by the runner and should be safe to embed in a quoted identifier. + q := fmt.Sprintf("SHOW FULL TABLES IN `%s` WHERE Table_Type = 'BASE TABLE';", dbName) + rows, err := db.QueryContext(ctx, q) + if err != nil { + return nil, err + } + for rows.Next() { + var tblName string + var tblType string + if err := rows.Scan(&tblName, &tblType); err != nil { + _ = rows.Close() + return nil, err + } + tables = append(tables, fmt.Sprintf("%s.%s", dbName, tblName)) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, err + } + _ = rows.Close() + } + sort.Strings(tables) + return tables, nil +} + +func (r *runner) createAndWaitFinishMark(ctx context.Context, up, down *sql.DB, model *clusterModel) error { + // The finish mark is a replication barrier: the workload is already stopped, but the + // sink / consumer may still be draining. Waiting for the finish mark to appear on + // downstream provides a deterministic "catch up" point before running the final diff. + if len(model.dbs) == 0 { + return fmt.Errorf("no databases in model") + } + + markerDB := model.dbs[0] + const markerTable = "finish_mark" + const markerID int64 = 1 + markerValue := r.cfg.Seed + + r.logger.Printf("converge: creating finish mark table: db=%s table=%s id=%d value=%d", + markerDB, markerTable, markerID, markerValue) + + createSQL := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s`.`%s` (`id` BIGINT PRIMARY KEY, `v` BIGINT NOT NULL)", + markerDB, markerTable) + if _, err := up.ExecContext(ctx, createSQL); err != nil { + return err + } + insertSQL := fmt.Sprintf("REPLACE INTO `%s`.`%s` (`id`, `v`) VALUES (?, ?)", markerDB, markerTable) + if _, err := up.ExecContext(ctx, insertSQL, markerID, markerValue); err != nil { + return err + } + + r.logger.Printf("converge: waiting for finish mark to appear in downstream") + + pollTicker := time.NewTicker(2 * time.Second) + defer pollTicker.Stop() + healthTicker := time.NewTicker(r.cfg.Verify.HealthInterval.Duration) + defer healthTicker.Stop() + + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + case <-healthTicker.C: + st, err := r.getChangefeedStatus(ctx) + if err != nil { + return err + } + if st.State != "normal" { + return fmt.Errorf("changefeed state is not normal: %s", st.State) + } + r.logger.Printf("converge: waiting for finish mark, checkpoint=%d", st.Checkpoint) + case <-pollTicker.C: + var got int64 + q := fmt.Sprintf("SELECT `v` FROM `%s`.`%s` WHERE `id` = ?", markerDB, markerTable) + err := down.QueryRowContext(ctx, q, markerID).Scan(&got) + if err == nil { + if got == markerValue { + r.logger.Printf("converge done: finish mark applied downstream") + return nil + } + r.logger.Printf("converge: finish mark row value mismatch: got=%d want=%d", got, markerValue) + continue + } + if err == sql.ErrNoRows { + continue + } + // Table may not exist yet on MQ sinks. + if strings.Contains(err.Error(), "doesn't exist") { + continue + } + return err + } + } +}