From 20a625f1ff4a458ca9cfed934696a4266b478d80 Mon Sep 17 00:00:00 2001 From: dongmen <20351731+asddongmen@users.noreply.github.com> Date: Tue, 3 Mar 2026 11:16:05 +0800 Subject: [PATCH 1/4] *:improve memory control (#4030) close pingcap/ticdc#4172 --- .../eventcollector/event_collector.go | 73 +++- metrics/grafana/ticdc_new_arch.json | 244 ++++++++++- .../ticdc_new_arch_next_gen.json | 244 ++++++++++- .../ticdc_new_arch_with_keyspace_name.json | 218 +++++++++- pkg/common/event/congestion_control.go | 275 +++++++++++- pkg/common/event/congestion_control_test.go | 32 +- pkg/eventservice/dispatcher_stat.go | 24 +- pkg/eventservice/event_broker.go | 69 ++- pkg/eventservice/event_broker_test.go | 111 +++++ pkg/eventservice/event_scanner_test.go | 2 +- pkg/eventservice/event_service_test.go | 4 +- pkg/eventservice/scan_window.go | 401 ++++++++++++++++++ pkg/eventservice/scan_window_test.go | 243 +++++++++++ pkg/metrics/event_service.go | 16 + pkg/sink/mysql/mysql_writer.go | 1 + tools/workload/Makefile | 4 +- tools/workload/app.go | 12 +- tools/workload/config.go | 24 +- tools/workload/ddl_app.go | 41 ++ tools/workload/ddl_config.go | 170 ++++++++ tools/workload/ddl_config_test.go | 136 ++++++ tools/workload/ddl_executor.go | 282 ++++++++++++ tools/workload/ddl_runner.go | 285 +++++++++++++ tools/workload/ddl_types.go | 41 ++ tools/workload/go.mod | 1 + tools/workload/go.sum | 2 + tools/workload/readme.md | 49 ++- tools/workload/statistics.go | 24 +- utils/dynstream/memory_control.go | 4 +- utils/dynstream/memory_control_test.go | 65 ++- 30 files changed, 2992 insertions(+), 105 deletions(-) create mode 100644 pkg/eventservice/scan_window.go create mode 100644 pkg/eventservice/scan_window_test.go create mode 100644 tools/workload/ddl_app.go create mode 100644 tools/workload/ddl_config.go create mode 100644 tools/workload/ddl_config_test.go create mode 100644 tools/workload/ddl_executor.go create mode 100644 tools/workload/ddl_runner.go create mode 100644 tools/workload/ddl_types.go diff --git a/downstreamadapter/eventcollector/event_collector.go b/downstreamadapter/eventcollector/event_collector.go index c1bb13ed89..17d8e77583 100644 --- a/downstreamadapter/eventcollector/event_collector.go +++ b/downstreamadapter/eventcollector/event_collector.go @@ -84,6 +84,7 @@ type changefeedStat struct { metricMemoryUsageMaxRedo prometheus.Gauge metricMemoryUsageUsedRedo prometheus.Gauge dispatcherCount atomic.Int32 + memoryReleaseCount atomic.Uint32 } func newChangefeedStat(changefeedID common.ChangeFeedID) *changefeedStat { @@ -440,11 +441,17 @@ func (c *EventCollector) processDSFeedback(ctx context.Context) error { return context.Cause(ctx) case feedback := <-c.ds.Feedback(): if feedback.FeedbackType == dynstream.ReleasePath { + if v, ok := c.changefeedMap.Load(feedback.Area); ok { + v.(*changefeedStat).memoryReleaseCount.Add(1) + } log.Info("release dispatcher memory in DS", zap.Any("dispatcherID", feedback.Path)) c.ds.Release(feedback.Path) } case feedback := <-c.redoDs.Feedback(): if feedback.FeedbackType == dynstream.ReleasePath { + if v, ok := c.changefeedMap.Load(feedback.Area); ok { + v.(*changefeedStat).memoryReleaseCount.Add(1) + } log.Info("release dispatcher memory in redo DS", zap.Any("dispatcherID", feedback.Path)) c.redoDs.Release(feedback.Path) } @@ -617,9 +624,24 @@ func (c *EventCollector) controlCongestion(ctx context.Context) error { } func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.CongestionControl { + changefeedMemoryReleaseCount := make(map[common.ChangeFeedID]uint32) + getAndResetMemoryReleaseCount := func(changefeedID common.ChangeFeedID) uint32 { + if count, ok := changefeedMemoryReleaseCount[changefeedID]; ok { + return count + } + v, ok := c.changefeedMap.Load(changefeedID.ID()) + if !ok { + return 0 + } + count := v.(*changefeedStat).memoryReleaseCount.Swap(0) + changefeedMemoryReleaseCount[changefeedID] = count + return count + } + // collect path-level available memory and total available memory for each changefeed changefeedPathMemory := make(map[common.ChangeFeedID]map[common.DispatcherID]uint64) changefeedTotalMemory := make(map[common.ChangeFeedID]uint64) + changefeedUsageRatio := make(map[common.ChangeFeedID]float64) // collect from main dynamic stream for _, quota := range c.ds.GetMetrics().MemoryControl.AreaMemoryMetrics { @@ -637,6 +659,7 @@ func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.Conge } // store total available memory from AreaMemoryMetric changefeedTotalMemory[cfID] = uint64(quota.AvailableMemory()) + changefeedUsageRatio[cfID] = calcUsageRatio(quota.MemoryUsage(), quota.MaxMemory()) } // collect from redo dynamic stream and take minimum @@ -658,11 +681,9 @@ func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.Conge } } // take minimum total available memory between main and redo streams - if existing, exists := changefeedTotalMemory[cfID]; exists { - changefeedTotalMemory[cfID] = min(existing, uint64(quota.AvailableMemory())) - } else { - changefeedTotalMemory[cfID] = uint64(quota.AvailableMemory()) - } + updateMinUint64MapValue(changefeedTotalMemory, cfID, uint64(quota.AvailableMemory())) + // take maximum usage ratio between main and redo streams + changefeedUsageRatio[cfID] = max(changefeedUsageRatio[cfID], calcUsageRatio(quota.MemoryUsage(), quota.MaxMemory())) } if len(changefeedPathMemory) == 0 { @@ -699,7 +720,7 @@ func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.Conge // build congestion control messages for each node result := make(map[node.ID]*event.CongestionControl) for nodeID, changefeedDispatchers := range nodeDispatcherMemory { - congestionControl := event.NewCongestionControl() + congestionControl := event.NewCongestionControlWithVersion(event.CongestionControlVersion2) for changefeedID, dispatcherMemory := range changefeedDispatchers { if len(dispatcherMemory) == 0 { @@ -707,11 +728,16 @@ func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.Conge } // get total available memory directly from AreaMemoryMetric - totalAvailable := uint64(changefeedTotalMemory[changefeedID]) - congestionControl.AddAvailableMemoryWithDispatchers( + totalAvailable, ok := changefeedTotalMemory[changefeedID] + if !ok { + continue + } + congestionControl.AddAvailableMemoryWithDispatchersAndUsageAndReleaseCount( changefeedID.ID(), totalAvailable, + changefeedUsageRatio[changefeedID], dispatcherMemory, + getAndResetMemoryReleaseCount(changefeedID), ) } @@ -719,10 +745,39 @@ func (c *EventCollector) newCongestionControlMessages() map[node.ID]*event.Conge result[nodeID] = congestionControl } } - return result } +func updateMinUint64MapValue(m map[common.ChangeFeedID]uint64, key common.ChangeFeedID, value uint64) { + if existing, exists := m[key]; exists { + m[key] = min(existing, value) + } else { + m[key] = value + } +} + +func updateMaxUint64MapValue(m map[common.ChangeFeedID]uint64, key common.ChangeFeedID, value uint64) { + if existing, exists := m[key]; exists { + m[key] = max(existing, value) + } else { + m[key] = value + } +} + +func calcUsageRatio(usedMemory int64, maxMemory int64) float64 { + if maxMemory <= 0 { + return 0 + } + ratio := float64(usedMemory) / float64(maxMemory) + if ratio < 0 { + return 0 + } + if ratio > 1 { + return 1 + } + return ratio +} + func (c *EventCollector) updateMetrics(ctx context.Context) error { ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 3a7d24963e..97458a8ad7 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -12173,6 +12173,216 @@ }, "id": 20007, "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The lag between changefeed checkpoint ts and the lac1 ts of upstream TiDB.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 60029, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(ticdc_event_service_scan_window_interval{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace, changefeed,instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}{{namespace}}-{{changefeed}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scan window interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:495", + "format": "s", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:496", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 60030, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/approximate current time.*/", + "bars": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(ticdc_event_service_scan_window_base_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed,instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}{{namespace}}-{{changefeed}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scan window base ts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "max": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:520", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:521", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "cards": { "cardPadding": 0, @@ -12197,7 +12407,7 @@ "h": 6, "w": 12, "x": 0, - "y": 12 + "y": 18 }, "heatmap": {}, "hideZeroBuckets": true, @@ -12274,7 +12484,7 @@ "h": 6, "w": 12, "x": 12, - "y": 12 + "y": 18 }, "hiddenSeries": false, "id": 22472, @@ -12385,7 +12595,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 24 }, "hiddenSeries": false, "id": 22262, @@ -12476,7 +12686,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 24 }, "hiddenSeries": false, "id": 22263, @@ -12567,7 +12777,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 32 }, "hiddenSeries": false, "id": 22264, @@ -12666,7 +12876,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 32 }, "hiddenSeries": false, "id": 22265, @@ -12757,7 +12967,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 40 }, "hiddenSeries": false, "id": 20018, @@ -12851,7 +13061,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 40 }, "hiddenSeries": false, "id": 20015, @@ -12946,7 +13156,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 48 }, "hiddenSeries": false, "id": 20025, @@ -13042,7 +13252,7 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 48 }, "hiddenSeries": false, "id": 22258, @@ -13138,7 +13348,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 56 }, "hiddenSeries": false, "id": 22266, @@ -13228,7 +13438,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 56 }, "hiddenSeries": false, "id": 22260, @@ -13328,7 +13538,7 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 64 }, "hiddenSeries": false, "id": 22259, @@ -13426,7 +13636,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 64 }, "hiddenSeries": false, "id": 22469, @@ -13525,7 +13735,7 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 72 }, "hiddenSeries": false, "id": 60005, @@ -13623,7 +13833,7 @@ "h": 8, "w": 12, "x": 12, - "y": 66 + "y": 72 }, "hiddenSeries": false, "id": 22474, @@ -13721,7 +13931,7 @@ "h": 11, "w": 12, "x": 0, - "y": 74 + "y": 80 }, "hiddenSeries": false, "id": 22475, diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index cc18801f23..71d5ad374a 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -12173,6 +12173,216 @@ }, "id": 20007, "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The lag between changefeed checkpoint ts and the lac1 ts of upstream TiDB.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 60029, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(ticdc_event_service_scan_window_interval{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name, changefeed,instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}{{keyspace_name}}-{{changefeed}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scan window interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:495", + "format": "s", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:496", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 60030, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/approximate current time.*/", + "bars": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(ticdc_event_service_scan_window_base_ts{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed,instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}{{keyspace_name}}-{{changefeed}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scan window base ts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "max": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:520", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:521", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "cards": { "cardPadding": 0, @@ -12197,7 +12407,7 @@ "h": 6, "w": 12, "x": 0, - "y": 12 + "y": 18 }, "heatmap": {}, "hideZeroBuckets": true, @@ -12274,7 +12484,7 @@ "h": 6, "w": 12, "x": 12, - "y": 12 + "y": 18 }, "hiddenSeries": false, "id": 22472, @@ -12385,7 +12595,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 24 }, "hiddenSeries": false, "id": 22262, @@ -12476,7 +12686,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 24 }, "hiddenSeries": false, "id": 22263, @@ -12567,7 +12777,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 32 }, "hiddenSeries": false, "id": 22264, @@ -12666,7 +12876,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 32 }, "hiddenSeries": false, "id": 22265, @@ -12757,7 +12967,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 40 }, "hiddenSeries": false, "id": 20018, @@ -12851,7 +13061,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 40 }, "hiddenSeries": false, "id": 20015, @@ -12946,7 +13156,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 48 }, "hiddenSeries": false, "id": 20025, @@ -13042,7 +13252,7 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 48 }, "hiddenSeries": false, "id": 22258, @@ -13138,7 +13348,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 56 }, "hiddenSeries": false, "id": 22266, @@ -13228,7 +13438,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 56 }, "hiddenSeries": false, "id": 22260, @@ -13328,7 +13538,7 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 64 }, "hiddenSeries": false, "id": 22259, @@ -13426,7 +13636,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 64 }, "hiddenSeries": false, "id": 22469, @@ -13525,7 +13735,7 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 72 }, "hiddenSeries": false, "id": 60005, @@ -13623,7 +13833,7 @@ "h": 8, "w": 12, "x": 12, - "y": 66 + "y": 72 }, "hiddenSeries": false, "id": 22474, @@ -13721,7 +13931,7 @@ "h": 11, "w": 12, "x": 0, - "y": 74 + "y": 80 }, "hiddenSeries": false, "id": 22475, diff --git a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json index c79e3593d9..9191ad00ff 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json +++ b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json @@ -3860,6 +3860,216 @@ }, "id": 20007, "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The lag between changefeed checkpoint ts and the lac1 ts of upstream TiDB.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 60029, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(ticdc_event_service_scan_window_interval{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name, changefeed,instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}{{keyspace_name}}-{{changefeed}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scan window interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:495", + "format": "s", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:496", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 60030, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/approximate current time.*/", + "bars": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(ticdc_event_service_scan_window_base_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed,instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}{{keyspace_name}}-{{changefeed}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scan window base ts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "max": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:520", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:521", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -3876,7 +4086,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 48 }, "hiddenSeries": false, "id": 20025, @@ -3972,7 +4182,7 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 48 }, "hiddenSeries": false, "id": 22258, @@ -4068,7 +4278,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 56 }, "hiddenSeries": false, "id": 22266, @@ -4158,7 +4368,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 56 }, "hiddenSeries": false, "id": 22260, diff --git a/pkg/common/event/congestion_control.go b/pkg/common/event/congestion_control.go index 5f50b80e25..ce414ce821 100644 --- a/pkg/common/event/congestion_control.go +++ b/pkg/common/event/congestion_control.go @@ -17,18 +17,24 @@ import ( "bytes" "encoding/binary" "fmt" + "math" "github.com/pingcap/ticdc/pkg/common" ) -const CongestionControlVersion1 = 1 +const ( + CongestionControlVersion1 = 1 + CongestionControlVersion2 = 2 +) type AvailableMemory struct { Version byte // 1 byte, it should be the same as CongestionControlVersion Gid common.GID // GID is the internal representation of ChangeFeedID - Available uint64 // in bytes, used to report the Available memory + Available uint64 // in bytes, used to report the available memory + UsageRatio float64 // [0,1], used to report memory usage ratio of the changefeed DispatcherCount uint32 // used to report the number of dispatchers DispatcherAvailable map[common.DispatcherID]uint64 // in bytes, used to report the memory usage of each dispatcher + MemoryReleaseCount uint32 // used to report the number of memory release events } func NewAvailableMemory(gid common.GID, available uint64) AvailableMemory { @@ -41,30 +47,114 @@ func NewAvailableMemory(gid common.GID, available uint64) AvailableMemory { } func (m AvailableMemory) Marshal() []byte { + return m.marshalV1() +} + +func (m *AvailableMemory) Unmarshal(buf *bytes.Buffer) { + m.unmarshalV1(buf) +} + +func (m AvailableMemory) GetSize() int { + return m.sizeV1() +} + +func (m AvailableMemory) marshalV1() []byte { buf := bytes.NewBuffer(make([]byte, 0)) buf.Write(m.Gid.Marshal()) - binary.Write(buf, binary.BigEndian, m.Available) - binary.Write(buf, binary.BigEndian, m.DispatcherCount) + _ = binary.Write(buf, binary.BigEndian, m.Available) + _ = binary.Write(buf, binary.BigEndian, m.DispatcherCount) for dispatcherID, available := range m.DispatcherAvailable { buf.Write(dispatcherID.Marshal()) - binary.Write(buf, binary.BigEndian, available) + _ = binary.Write(buf, binary.BigEndian, available) } return buf.Bytes() } -func (m *AvailableMemory) Unmarshal(buf *bytes.Buffer) { - m.Gid.Unmarshal(buf.Next(m.Gid.GetSize())) +func (m AvailableMemory) marshalV2() []byte { + buf := bytes.NewBuffer(make([]byte, 0)) + buf.Write(m.Gid.Marshal()) + _ = binary.Write(buf, binary.BigEndian, m.Available) + _ = binary.Write(buf, binary.BigEndian, math.Float64bits(m.UsageRatio)) + _ = binary.Write(buf, binary.BigEndian, m.DispatcherCount) + for dispatcherID, available := range m.DispatcherAvailable { + buf.Write(dispatcherID.Marshal()) + _ = binary.Write(buf, binary.BigEndian, available) + } + return buf.Bytes() +} + +func (m *AvailableMemory) unmarshalV1(buf *bytes.Buffer) error { + gidSize := m.Gid.GetSize() + if buf.Len() < gidSize { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for gid, need %d, got %d", gidSize, buf.Len()) + } + m.Gid.Unmarshal(buf.Next(gidSize)) + + if buf.Len() < 8 { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for available, need %d, got %d", 8, buf.Len()) + } m.Available = binary.BigEndian.Uint64(buf.Next(8)) + + if buf.Len() < 4 { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for dispatcher count, need %d, got %d", 4, buf.Len()) + } m.DispatcherCount = binary.BigEndian.Uint32(buf.Next(4)) m.DispatcherAvailable = make(map[common.DispatcherID]uint64) for range m.DispatcherCount { dispatcherID := common.DispatcherID{} - dispatcherID.Unmarshal(buf.Next(dispatcherID.GetSize())) + dispatcherIDSize := dispatcherID.GetSize() + if buf.Len() < dispatcherIDSize { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for dispatcher id, need %d, got %d", dispatcherIDSize, buf.Len()) + } + dispatcherID.Unmarshal(buf.Next(dispatcherIDSize)) + + if buf.Len() < 8 { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for dispatcher available, need %d, got %d", 8, buf.Len()) + } m.DispatcherAvailable[dispatcherID] = binary.BigEndian.Uint64(buf.Next(8)) } + return nil } -func (m AvailableMemory) GetSize() int { +func (m *AvailableMemory) unmarshalV2(buf *bytes.Buffer) error { + gidSize := m.Gid.GetSize() + if buf.Len() < gidSize { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for gid, need %d, got %d", gidSize, buf.Len()) + } + m.Gid.Unmarshal(buf.Next(gidSize)) + + if buf.Len() < 8 { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for available, need %d, got %d", 8, buf.Len()) + } + m.Available = binary.BigEndian.Uint64(buf.Next(8)) + + if buf.Len() < 8 { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for usage ratio, need %d, got %d", 8, buf.Len()) + } + m.UsageRatio = math.Float64frombits(binary.BigEndian.Uint64(buf.Next(8))) + + if buf.Len() < 4 { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for dispatcher count, need %d, got %d", 4, buf.Len()) + } + m.DispatcherCount = binary.BigEndian.Uint32(buf.Next(4)) + m.DispatcherAvailable = make(map[common.DispatcherID]uint64) + for range m.DispatcherCount { + dispatcherID := common.DispatcherID{} + dispatcherIDSize := dispatcherID.GetSize() + if buf.Len() < dispatcherIDSize { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for dispatcher id, need %d, got %d", dispatcherIDSize, buf.Len()) + } + dispatcherID.Unmarshal(buf.Next(dispatcherIDSize)) + + if buf.Len() < 8 { + return fmt.Errorf("invalid AvailableMemory payload: insufficient bytes for dispatcher available, need %d, got %d", 8, buf.Len()) + } + m.DispatcherAvailable[dispatcherID] = binary.BigEndian.Uint64(buf.Next(8)) + } + return nil +} + +func (m AvailableMemory) sizeV1() int { // changefeedID size + changefeed available size size := m.Gid.GetSize() + 8 size += 4 // dispatcher count @@ -76,6 +166,18 @@ func (m AvailableMemory) GetSize() int { return size } +func (m AvailableMemory) sizeV2() int { + // changefeedID size + changefeed available size + usage ratio size + size := m.Gid.GetSize() + 8 + 8 + size += 4 // dispatcher count + for range m.DispatcherCount { + dispatcherID := &common.DispatcherID{} + // dispatcherID size + dispatcher available size + size += dispatcherID.GetSize() + 8 + } + return size +} + type CongestionControl struct { version int clusterID uint64 @@ -90,11 +192,34 @@ func NewCongestionControl() *CongestionControl { } } +func NewCongestionControlWithVersion(version int) *CongestionControl { + return &CongestionControl{ + version: version, + } +} + func (c *CongestionControl) GetSize() int { size := 8 // clusterID size += 4 // changefeed count for _, mem := range c.availables { - size += mem.GetSize() + switch c.version { + case CongestionControlVersion2: + size += mem.sizeV2() + default: + size += mem.sizeV1() + } + } + if c.version == CongestionControlVersion2 { + gidSize := (&common.GID{}).GetSize() + releaseEntryCount := 0 + for _, mem := range c.availables { + if mem.MemoryReleaseCount > 0 { + releaseEntryCount++ + } + } + if releaseEntryCount > 0 { + size += 4 + releaseEntryCount*(gidSize+4) + } } return size } @@ -109,6 +234,11 @@ func (c *CongestionControl) Marshal() ([]byte, error) { if err != nil { return nil, err } + case CongestionControlVersion2: + payload, err = c.encodeV2() + if err != nil { + return nil, err + } default: return nil, fmt.Errorf("unsupported CongestionControl version: %d", c.version) } @@ -123,9 +253,38 @@ func (c *CongestionControl) encodeV1() ([]byte, error) { _ = binary.Write(buf, binary.BigEndian, c.changefeedCount) for _, item := range c.availables { - data := item.Marshal() + data := item.marshalV1() + buf.Write(data) + } + return buf.Bytes(), nil +} + +func (c *CongestionControl) encodeV2() ([]byte, error) { + buf := bytes.NewBuffer(make([]byte, 0)) + _ = binary.Write(buf, binary.BigEndian, c.clusterID) + _ = binary.Write(buf, binary.BigEndian, c.changefeedCount) + + for _, item := range c.availables { + data := item.marshalV2() buf.Write(data) } + + releaseEntryCount := uint32(0) + for _, item := range c.availables { + if item.MemoryReleaseCount > 0 { + releaseEntryCount++ + } + } + if releaseEntryCount > 0 { + _ = binary.Write(buf, binary.BigEndian, releaseEntryCount) + for _, item := range c.availables { + if item.MemoryReleaseCount == 0 { + continue + } + buf.Write(item.Gid.Marshal()) + _ = binary.Write(buf, binary.BigEndian, item.MemoryReleaseCount) + } + } return buf.Bytes(), nil } @@ -143,6 +302,8 @@ func (c *CongestionControl) Unmarshal(data []byte) error { switch version { case CongestionControlVersion1: return c.decodeV1(payload) + case CongestionControlVersion2: + return c.decodeV2(payload) default: return fmt.Errorf("unsupported CongestionControl version: %d", version) } @@ -155,9 +316,59 @@ func (c *CongestionControl) decodeV1(data []byte) error { c.availables = make([]AvailableMemory, 0, c.changefeedCount) for i := uint32(0); i < c.changefeedCount; i++ { var item AvailableMemory - item.Unmarshal(buf) + if err := item.unmarshalV1(buf); err != nil { + return err + } + c.availables = append(c.availables, item) + } + return nil +} + +func (c *CongestionControl) decodeV2(data []byte) error { + buf := bytes.NewBuffer(data) + c.clusterID = binary.BigEndian.Uint64(buf.Next(8)) + c.changefeedCount = binary.BigEndian.Uint32(buf.Next(4)) + c.availables = make([]AvailableMemory, 0, c.changefeedCount) + for i := uint32(0); i < c.changefeedCount; i++ { + var item AvailableMemory + if err := item.unmarshalV2(buf); err != nil { + return err + } c.availables = append(c.availables, item) } + + if buf.Len() == 0 { + return nil + } + + if buf.Len() < 4 { + return fmt.Errorf("invalid CongestionControl payload: insufficient bytes for memory release section count, need %d, got %d", 4, buf.Len()) + } + + releaseEntryCount := binary.BigEndian.Uint32(buf.Next(4)) + if releaseEntryCount == 0 { + if buf.Len() != 0 { + return fmt.Errorf("invalid CongestionControl payload: unexpected bytes after empty memory release section, got %d", buf.Len()) + } + return nil + } + + gidSize := (&common.GID{}).GetSize() + expectedBytes := int(releaseEntryCount) * (gidSize + 4) + if buf.Len() != expectedBytes { + return fmt.Errorf("invalid CongestionControl payload: inconsistent memory release section length, expected %d, got %d", expectedBytes, buf.Len()) + } + + releaseByGid := make(map[common.GID]uint32, releaseEntryCount) + for i := uint32(0); i < releaseEntryCount; i++ { + gid := common.GID{} + gid.Unmarshal(buf.Next(gidSize)) + releaseByGid[gid] = binary.BigEndian.Uint32(buf.Next(4)) + } + + for i := range c.availables { + c.availables[i].MemoryReleaseCount = releaseByGid[c.availables[i].Gid] + } return nil } @@ -174,6 +385,38 @@ func (c *CongestionControl) AddAvailableMemoryWithDispatchers(gid common.GID, av c.availables = append(c.availables, availMem) } +func (c *CongestionControl) AddAvailableMemoryWithDispatchersAndUsage( + gid common.GID, + available uint64, + usageRatio float64, + dispatcherAvailable map[common.DispatcherID]uint64, +) { + c.changefeedCount++ + availMem := NewAvailableMemory(gid, available) + availMem.Version = CongestionControlVersion2 + availMem.UsageRatio = usageRatio + availMem.DispatcherAvailable = dispatcherAvailable + availMem.DispatcherCount = uint32(len(dispatcherAvailable)) + c.availables = append(c.availables, availMem) +} + +func (c *CongestionControl) AddAvailableMemoryWithDispatchersAndUsageAndReleaseCount( + gid common.GID, + available uint64, + usageRatio float64, + dispatcherAvailable map[common.DispatcherID]uint64, + memoryReleaseCount uint32, +) { + c.changefeedCount++ + availMem := NewAvailableMemory(gid, available) + availMem.Version = CongestionControlVersion2 + availMem.UsageRatio = usageRatio + availMem.DispatcherAvailable = dispatcherAvailable + availMem.DispatcherCount = uint32(len(dispatcherAvailable)) + availMem.MemoryReleaseCount = memoryReleaseCount + c.availables = append(c.availables, availMem) +} + func (c *CongestionControl) GetAvailables() []AvailableMemory { return c.availables } @@ -181,3 +424,11 @@ func (c *CongestionControl) GetAvailables() []AvailableMemory { func (c *CongestionControl) GetClusterID() uint64 { return c.clusterID } + +func (c *CongestionControl) GetVersion() int { + return c.version +} + +func (c *CongestionControl) HasUsageRatio() bool { + return c.version >= CongestionControlVersion2 +} diff --git a/pkg/common/event/congestion_control_test.go b/pkg/common/event/congestion_control_test.go index 159a309894..d4d9daf1e0 100644 --- a/pkg/common/event/congestion_control_test.go +++ b/pkg/common/event/congestion_control_test.go @@ -142,8 +142,6 @@ func TestCongestionControlMarshalUnmarshal(t *testing.T) { require.Equal(t, uint64(4096), availables[1].Available) // Test case 4: CongestionControl with AvailableMemoryWithDispatchers - // Note: DispatcherAvailable field is not properly serialized/deserialized in current implementation - // So we only test the basic GID and Available fields control4 := NewCongestionControl() control4.clusterID = 22222 gid4 := common.NewGID() @@ -170,6 +168,36 @@ func TestCongestionControlMarshalUnmarshal(t *testing.T) { require.Len(t, availableMem.DispatcherAvailable, 2) } +func TestCongestionControlV2(t *testing.T) { + t.Parallel() + + control := NewCongestionControlWithVersion(CongestionControlVersion2) + control.clusterID = 33333 + gid := common.NewGID() + dispatcherAvailable := map[common.DispatcherID]uint64{ + common.NewDispatcherID(): 100, + common.NewDispatcherID(): 200, + } + control.AddAvailableMemoryWithDispatchersAndUsageAndReleaseCount(gid, 3000, 0.3, dispatcherAvailable, 7) + + data, err := control.Marshal() + require.NoError(t, err) + + // Verify header version + require.Equal(t, uint16(CongestionControlVersion2), binary.BigEndian.Uint16(data[6:8]), "version") + + var decoded CongestionControl + err = decoded.Unmarshal(data) + require.NoError(t, err) + require.Equal(t, CongestionControlVersion2, decoded.version) + require.Equal(t, control.clusterID, decoded.clusterID) + require.Len(t, decoded.availables, 1) + require.Equal(t, uint64(3000), decoded.availables[0].Available) + require.InDelta(t, 0.3, decoded.availables[0].UsageRatio, 1e-9) + require.Len(t, decoded.availables[0].DispatcherAvailable, 2) + require.Equal(t, uint32(7), decoded.availables[0].MemoryReleaseCount) +} + func TestCongestionControlMarshalUnmarshalEdgeCases(t *testing.T) { t.Parallel() diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 79edc2a688..552a684ea2 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -428,12 +428,26 @@ type changefeedStatus struct { dispatchers sync.Map // common.DispatcherID -> *atomic.Pointer[dispatcherStat] availableMemoryQuota sync.Map // nodeID -> atomic.Uint64 (memory quota in bytes) + minSentTs atomic.Uint64 + scanInterval atomic.Int64 + + lastAdjustTime atomic.Time + lastTrendAdjustTime atomic.Time + usageWindow *memoryUsageWindow + syncPointInterval time.Duration } -func newChangefeedStatus(changefeedID common.ChangeFeedID) *changefeedStatus { - return &changefeedStatus{ - changefeedID: changefeedID, +func newChangefeedStatus(changefeedID common.ChangeFeedID, syncPointInterval time.Duration) *changefeedStatus { + status := &changefeedStatus{ + changefeedID: changefeedID, + usageWindow: newMemoryUsageWindow(memoryUsageWindowDuration), + syncPointInterval: syncPointInterval, } + status.scanInterval.Store(int64(defaultScanInterval)) + status.lastAdjustTime.Store(time.Now()) + status.lastTrendAdjustTime.Store(time.Now()) + + return status } func (c *changefeedStatus) addDispatcher(id common.DispatcherID, dispatcher *atomic.Pointer[dispatcherStat]) { @@ -452,3 +466,7 @@ func (c *changefeedStatus) isEmpty() bool { }) return empty } + +func (c *changefeedStatus) isSyncpointEnabled() bool { + return c.syncPointInterval > 0 +} diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index d41df332b5..b3bc597d8e 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -51,7 +51,8 @@ const ( maxReadyEventIntervalSeconds = 10 // defaultSendResolvedTsInterval use to control whether to send a resolvedTs event to the dispatcher when its scan is skipped. - defaultSendResolvedTsInterval = time.Second * 2 + defaultSendResolvedTsInterval = time.Second * 2 + defaultRefreshMinSentResolvedTsInterval = time.Second * 1 ) // eventBroker get event from the eventStore, and send the event to the dispatchers. @@ -183,6 +184,10 @@ func newEventBroker( return c.metricsCollector.Run(ctx) }) + g.Go(func() error { + return c.refreshMinSentResolvedTs(ctx) + }) + log.Info("new event broker created", zap.Uint64("id", id), zap.Uint64("scanLimitInBytes", c.scanLimitInBytes)) return c } @@ -257,6 +262,23 @@ func (c *eventBroker) sendDDL(ctx context.Context, remoteID node.ID, e *event.DD zap.Uint64("seq", e.Seq), zap.Int64("mode", d.info.GetMode())) } +func (c *eventBroker) refreshMinSentResolvedTs(ctx context.Context) error { + ticker := time.NewTicker(defaultRefreshMinSentResolvedTsInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + case <-ticker.C: + c.changefeedMap.Range(func(key, value interface{}) bool { + status := value.(*changefeedStatus) + status.refreshMinSentResolvedTs() + return true + }) + } + } +} + func (c *eventBroker) sendSignalResolvedTs(d *dispatcherStat) { // Can't send resolvedTs if there was a interrupted scan task happened before. // d.lastScannedStartTs.Load() != 0 indicates that there was a interrupted scan task happened before. @@ -407,9 +429,29 @@ func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRang return false, common.DataRange{} } dataRange.CommitTsEnd = min(dataRange.CommitTsEnd, ddlState.ResolvedTs) + commitTsEndBeforeWindow := dataRange.CommitTsEnd + scanMaxTs := task.changefeedStat.getScanMaxTs() + if scanMaxTs > 0 { + dataRange.CommitTsEnd = min(dataRange.CommitTsEnd, scanMaxTs) + if dataRange.CommitTsEnd < commitTsEndBeforeWindow { + log.Debug("scan window capped", + zap.Stringer("changefeedID", task.changefeedStat.changefeedID), + zap.Stringer("dispatcherID", task.id), + zap.Uint64("baseTs", task.changefeedStat.minSentTs.Load()), + zap.Uint64("scanMaxTs", scanMaxTs), + zap.Uint64("beforeEndTs", commitTsEndBeforeWindow), + zap.Uint64("afterEndTs", dataRange.CommitTsEnd), + zap.Duration("scanInterval", time.Duration(task.changefeedStat.scanInterval.Load())), + ) + } + } if dataRange.CommitTsEnd <= dataRange.CommitTsStart { updateMetricEventServiceSkipResolvedTsCount(task.info.GetMode()) + // Scan range can become empty after applying capping (for example, scan window). + // Send a signal resolved-ts event (rate limited) to keep downstream responsive, + // but do not advance the watermark here. + c.sendSignalResolvedTs(task) return false, common.DataRange{} } @@ -559,6 +601,7 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { if task.isRemoved.Load() { return } + // If the target is not ready to send, we don't need to scan the event store. // To avoid the useless scan task. if !c.msgSender.IsReadyToSend(remoteID) { @@ -1073,6 +1116,8 @@ func (c *eventBroker) removeChangefeedStatus(status *changefeedStatus) { filter.GetSharedFilterStorage().RemoveFilter(changefeedID) metrics.EventServiceAvailableMemoryQuotaGaugeVec.DeleteLabelValues(changefeedID.String()) + metrics.EventServiceScanWindowBaseTsGaugeVec.DeleteLabelValues(changefeedID.String()) + metrics.EventServiceScanWindowIntervalGaugeVec.DeleteLabelValues(changefeedID.String()) } func (c *eventBroker) resetDispatcher(dispatcherInfo DispatcherInfo) error { @@ -1184,13 +1229,15 @@ func (c *eventBroker) getOrSetChangefeedStatus(info DispatcherInfo) *changefeedS zap.Error(err)) } - status := newChangefeedStatus(changefeedID) + status := newChangefeedStatus(changefeedID, info.GetSyncPointInterval()) status.filter = changefeedFilter actual, loaded := c.changefeedMap.LoadOrStore(changefeedID, status) if loaded { return actual.(*changefeedStatus) } log.Info("new changefeed status", zap.Stringer("changefeedID", changefeedID)) + metrics.EventServiceScanWindowBaseTsGaugeVec.WithLabelValues(changefeedID.String()).Set(0) + metrics.EventServiceScanWindowIntervalGaugeVec.WithLabelValues(changefeedID.String()).Set(defaultScanInterval.Seconds()) return status } @@ -1254,21 +1301,33 @@ func (c *eventBroker) handleCongestionControl(from node.ID, m *event.CongestionC } holder := make(map[common.GID]uint64, len(availables)) + usage := make(map[common.GID]float64, len(availables)) + memoryRelease := make(map[common.GID]uint32, len(availables)) dispatcherAvailable := make(map[common.DispatcherID]uint64, len(availables)) for _, item := range availables { holder[item.Gid] = item.Available + if m.HasUsageRatio() { + usage[item.Gid] = item.UsageRatio + } + memoryRelease[item.Gid] = item.MemoryReleaseCount for dispatcherID, available := range item.DispatcherAvailable { dispatcherAvailable[dispatcherID] = available } } + now := time.Now() c.changefeedMap.Range(func(k, v interface{}) bool { changefeedID := k.(common.ChangeFeedID) changefeed := v.(*changefeedStatus) - available, ok := holder[changefeedID.ID()] + availableInMsg, ok := holder[changefeedID.ID()] if ok { - changefeed.availableMemoryQuota.Store(from, atomic.NewUint64(available)) - metrics.EventServiceAvailableMemoryQuotaGaugeVec.WithLabelValues(changefeedID.String()).Set(float64(available)) + changefeed.availableMemoryQuota.Store(from, atomic.NewUint64(availableInMsg)) + metrics.EventServiceAvailableMemoryQuotaGaugeVec.WithLabelValues(changefeedID.String()).Set(float64(availableInMsg)) + } + if m.HasUsageRatio() { + if ratio, okUsage := usage[changefeedID.ID()]; okUsage && ok { + changefeed.updateMemoryUsage(now, ratio, memoryRelease[changefeedID.ID()]) + } } return true }) diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index 5f470bdea3..89e074267d 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/util" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/oracle" "go.uber.org/atomic" "go.uber.org/zap" ) @@ -201,6 +202,116 @@ func TestAddDispatcherUnregisterOnSchemaStoreError(t *testing.T) { require.Equal(t, uint64(1), es.unregisterCount.Load()) } +func TestScanRangeCappedByScanWindow(t *testing.T) { + broker, _, _, _ := newEventBrokerForTest() + // Close the broker, so we can catch all message in the test. + broker.close() + + info := newMockDispatcherInfoForTest(t) + info.epoch = 1 + changefeedStatus := broker.getOrSetChangefeedStatus(info) + + disp := newDispatcherStat(info, 1, 1, nil, changefeedStatus) + disp.seq.Store(1) + + dispPtr := &atomic.Pointer[dispatcherStat]{} + dispPtr.Store(disp) + changefeedStatus.addDispatcher(disp.id, dispPtr) + + baseTime := time.Now() + baseTs := oracle.GoTimeToTS(baseTime) + disp.sentResolvedTs.Store(baseTs) + disp.receivedResolvedTs.Store(oracle.GoTimeToTS(baseTime.Add(20 * time.Second))) + disp.eventStoreCommitTs.Store(oracle.GoTimeToTS(baseTime.Add(15 * time.Second))) + changefeedStatus.refreshMinSentResolvedTs() + + needScan, dataRange := broker.getScanTaskDataRange(disp) + require.True(t, needScan) + require.Equal(t, oracle.GoTimeToTS(baseTime.Add(defaultScanInterval)), dataRange.CommitTsEnd) +} + +func TestGetScanTaskDataRangeEmptyAfterCappingDoesNotResetScanRange(t *testing.T) { + broker, _, _, _ := newEventBrokerForTest() + // Close the broker, so we can catch all message in the test. + broker.close() + + info := newMockDispatcherInfoForTest(t) + info.epoch = 1 + changefeedStatus := broker.getOrSetChangefeedStatus(info) + + disp := newDispatcherStat(info, 1, 1, nil, changefeedStatus) + disp.seq.Store(1) + + baseTime := time.Now() + baseTs := oracle.GoTimeToTS(baseTime) + commitStart := oracle.GoTimeToTS(baseTime.Add(20 * time.Second)) + lastStartTs := commitStart - 1 + + disp.sentResolvedTs.Store(baseTs) + disp.receivedResolvedTs.Store(oracle.GoTimeToTS(baseTime.Add(40 * time.Second))) + disp.eventStoreCommitTs.Store(commitStart) + disp.lastScannedCommitTs.Store(commitStart) + disp.lastScannedStartTs.Store(lastStartTs) + + changefeedStatus.minSentTs.Store(baseTs) + changefeedStatus.scanInterval.Store(int64(defaultScanInterval)) + + needScan, _ := broker.getScanTaskDataRange(disp) + require.False(t, needScan) + require.Equal(t, commitStart, disp.lastScannedCommitTs.Load()) + require.Equal(t, lastStartTs, disp.lastScannedStartTs.Load()) +} + +func TestHandleCongestionControlV2AdjustsScanInterval(t *testing.T) { + broker, _, _, _ := newEventBrokerForTest() + defer broker.close() + + changefeedID := common.NewChangefeedID4Test("default", "test") + status := addChangefeedStatusToBrokerForTest(t, broker, changefeedID, time.Second*10) + + status.scanInterval.Store(int64(40 * time.Second)) + status.lastAdjustTime.Store(time.Now()) + + control := event.NewCongestionControlWithVersion(event.CongestionControlVersion2) + control.AddAvailableMemoryWithDispatchersAndUsage(changefeedID.ID(), 0, 1, nil) + broker.handleCongestionControl(node.ID("event-collector-1"), control) + + require.Equal(t, int64(10*time.Second), status.scanInterval.Load()) +} + +func TestHandleCongestionControlV2ResetsScanIntervalOnMemoryRelease(t *testing.T) { + broker, _, _, _ := newEventBrokerForTest() + defer broker.close() + + changefeedID := common.NewChangefeedID4Test("default", "test") + status := addChangefeedStatusToBrokerForTest(t, broker, changefeedID, time.Second*10) + + status.scanInterval.Store(int64(40 * time.Second)) + + control := event.NewCongestionControlWithVersion(event.CongestionControlVersion2) + control.AddAvailableMemoryWithDispatchersAndUsageAndReleaseCount(changefeedID.ID(), 0, 0.5, nil, 1) + broker.handleCongestionControl(node.ID("event-collector-1"), control) + + require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) +} + +func TestHandleCongestionControlV1DoesNotAdjustScanInterval(t *testing.T) { + broker, _, _, _ := newEventBrokerForTest() + defer broker.close() + + changefeedID := common.NewChangefeedID4Test("default", "test") + status := addChangefeedStatusToBrokerForTest(t, broker, changefeedID, time.Second*10) + + status.scanInterval.Store(int64(40 * time.Second)) + status.lastAdjustTime.Store(time.Now()) + + control := event.NewCongestionControl() + control.AddAvailableMemoryWithDispatchers(changefeedID.ID(), 0, nil) + broker.handleCongestionControl(node.ID("event-collector-1"), control) + + require.Equal(t, int64(40*time.Second), status.scanInterval.Load()) +} + func TestDoScanSkipWhenChangefeedStatusNotFound(t *testing.T) { broker, _, _, _ := newEventBrokerForTest() broker.close() diff --git a/pkg/eventservice/event_scanner_test.go b/pkg/eventservice/event_scanner_test.go index 050569ff09..15c37209ad 100644 --- a/pkg/eventservice/event_scanner_test.go +++ b/pkg/eventservice/event_scanner_test.go @@ -60,7 +60,7 @@ func (m *mockMounter) DecodeToChunk(rawKV *common.RawKVEntry, tableInfo *common. func TestEventScannerReturnsIteratorErrors(t *testing.T) { disInfo := newMockDispatcherInfoForTest(t) - changefeedStatus := newChangefeedStatus(disInfo.GetChangefeedID()) + changefeedStatus := newChangefeedStatusForTest(t, disInfo) disp := newDispatcherStat(disInfo, 1, 1, nil, changefeedStatus) makeDispatcherReady(disp) diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index c890a22865..e58508d10a 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -505,7 +505,7 @@ func (m *mockDispatcherInfo) GetTxnAtomicity() config.AtomicityLevel { func newChangefeedStatusForTest(t testing.TB, info DispatcherInfo) *changefeedStatus { t.Helper() - status := newChangefeedStatus(info.GetChangefeedID()) + status := newChangefeedStatus(info.GetChangefeedID(), info.GetSyncPointInterval()) status.filter = newChangefeedFilterForTest(t, info, time.UTC.String()) return status } @@ -518,7 +518,7 @@ func addChangefeedStatusToBrokerForTest( ) *changefeedStatus { t.Helper() - status := newChangefeedStatus(changefeedID) + status := newChangefeedStatus(changefeedID, syncPointInterval) broker.changefeedMap.Store(changefeedID, status) return status } diff --git a/pkg/eventservice/scan_window.go b/pkg/eventservice/scan_window.go new file mode 100644 index 0000000000..46cb9cffb3 --- /dev/null +++ b/pkg/eventservice/scan_window.go @@ -0,0 +1,401 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventservice + +import ( + "sync" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/tikv/client-go/v2/oracle" + "go.uber.org/atomic" + "go.uber.org/zap" +) + +const ( + // defaultScanInterval is the initial scan interval used when starting up + // or when the current interval is invalid. + defaultScanInterval = 5 * time.Second + + // minScanInterval is the minimum allowed scan interval. Even under critical + // memory pressure, the interval will never go below this value. + minScanInterval = 1 * time.Second + + // maxScanInterval is the maximum allowed scan interval. Even under very low + // memory pressure, the interval will never exceed this value. + maxScanInterval = 30 * time.Minute + + // scanIntervalAdjustCooldown is the minimum time that must pass between + // scan interval increases. This prevents oscillation by enforcing a waiting + // period before allowing another increase. Decreases are not affected by + // this cooldown and are applied immediately. + scanIntervalAdjustCooldown = 30 * time.Second + + // scanTrendAdjustCooldown is the minimum time between trend-based interval + // adjustments. This is shorter than the general cooldown because trend + // adjustments need to be more responsive to rising memory pressure. + scanTrendAdjustCooldown = 5 * time.Second + + // memoryUsageWindowDuration is the duration of the sliding window for + // collecting memory usage samples. Samples older than this duration are + // pruned from the window. + memoryUsageWindowDuration = 30 * time.Second + + // memoryUsageHighThreshold (70%) triggers a moderate reduction of the scan + // interval to 1/2 of its current value when memory usage exceeds this level. + memoryUsageHighThreshold = 0.7 + + // memoryUsageCriticalThreshold (90%) triggers an aggressive reduction of + // the scan interval to 1/4 of its current value when memory usage exceeds + // this level. + memoryUsageCriticalThreshold = 0.9 + + // memoryUsageLowThreshold (20%) allows the scan interval to be increased + // by 25% when both max and average memory usage are below this level. + memoryUsageLowThreshold = 0.2 + + // memoryUsageVeryLowThreshold (10%) allows the scan interval to be increased + // by 50% when both max and average memory usage are below this level. This + // increase may exceed the normal sync point interval cap. + memoryUsageVeryLowThreshold = 0.1 + + // scanWindowStaleDispatcherHeartbeatThreshold is the duration after which a + // dispatcher is treated as stale for scan window base ts calculation if it + // hasn't sent heartbeat updates. This prevents stale dispatchers (for example, + // after frequent table truncate) from blocking scan window advancement for the + // whole changefeed. + // + // Note: This is intentionally much smaller than heartbeatTimeout, which is + // used for actual dispatcher removal. + scanWindowStaleDispatcherHeartbeatThreshold = 1 * time.Minute +) + +type memoryUsageSample struct { + ts time.Time + ratio float64 +} + +type memoryUsageWindow struct { + window time.Duration + mu sync.Mutex + samples []memoryUsageSample +} + +type memoryUsageStats struct { + avg float64 + max float64 + first float64 + last float64 + span time.Duration + cnt int +} + +func newMemoryUsageWindow(window time.Duration) *memoryUsageWindow { + return &memoryUsageWindow{ + window: window, + } +} + +func (w *memoryUsageWindow) addSample(now time.Time, ratio float64) { + if ratio < 0 { + ratio = 0 + } + w.mu.Lock() + defer w.mu.Unlock() + + w.samples = append(w.samples, memoryUsageSample{ts: now, ratio: ratio}) + w.pruneLocked(now) +} + +func (w *memoryUsageWindow) reset() { + w.mu.Lock() + defer w.mu.Unlock() + w.samples = nil +} + +func (w *memoryUsageWindow) stats(now time.Time) memoryUsageStats { + w.mu.Lock() + defer w.mu.Unlock() + + w.pruneLocked(now) + if len(w.samples) == 0 { + return memoryUsageStats{} + } + + sum := 0.0 + firstRatio := w.samples[0].ratio + maxRatio := firstRatio + for _, sample := range w.samples { + sum += sample.ratio + if sample.ratio > maxRatio { + maxRatio = sample.ratio + } + } + + return memoryUsageStats{ + avg: sum / float64(len(w.samples)), + max: maxRatio, + first: firstRatio, + last: w.samples[len(w.samples)-1].ratio, + span: now.Sub(w.samples[0].ts), + cnt: len(w.samples), + } +} + +func (w *memoryUsageWindow) pruneLocked(now time.Time) { + cutoff := now.Add(-w.window) + idx := 0 + for idx < len(w.samples) && w.samples[idx].ts.Before(cutoff) { + idx++ + } + if idx > 0 { + w.samples = w.samples[idx:] + } +} + +func (c *changefeedStatus) updateMemoryUsage(now time.Time, usageRatio float64, memoryReleaseCount uint32) { + if c.usageWindow == nil { + return + } + + if usageRatio != usageRatio || usageRatio < 0 { + usageRatio = 0 + } + if usageRatio > 1 { + usageRatio = 1 + } + + if memoryReleaseCount > 0 { + c.resetScanIntervalToDefault(now) + c.usageWindow.reset() + c.usageWindow.addSample(now, usageRatio) + return + } + + c.usageWindow.addSample(now, usageRatio) + stats := c.usageWindow.stats(now) + c.adjustScanInterval(now, stats) +} + +func (c *changefeedStatus) resetScanIntervalToDefault(now time.Time) { + current := time.Duration(c.scanInterval.Load()) + if current != defaultScanInterval { + c.scanInterval.Store(int64(defaultScanInterval)) + metrics.EventServiceScanWindowIntervalGaugeVec.WithLabelValues(c.changefeedID.String()).Set(defaultScanInterval.Seconds()) + + log.Info("scan interval reset to default", + zap.Stringer("changefeedID", c.changefeedID), + zap.Duration("oldInterval", current), + zap.Duration("newInterval", defaultScanInterval)) + } + + c.lastAdjustTime.Store(now) + c.lastTrendAdjustTime.Store(now) +} + +// Constants for trend detection and increase eligibility. +const ( + minTrendSamples = 4 // Minimum samples needed to detect a valid trend + increasingTrendEpsilon = 0.02 // Minimum delta to consider as "increasing" + increasingTrendStartRatio = 0.3 // Threshold (30%) above which trend damping kicks in + + minIncreaseSamples = 10 // Minimum samples needed before allowing increase + minIncreaseSpanNumerator = 4 // Observation span must be at least 4/5 of window + minIncreaseSpanDenominator = 5 +) + +// adjustScanInterval dynamically adjusts the scan interval based on memory pressure. +// +// Algorithm overview: +// - "Fast brake, slow accelerate": Decreases are applied immediately when memory +// pressure is high, while increases require cooldown periods and stable conditions. +// - Tiered response: Different thresholds trigger different adjustment magnitudes. +// - Trend prediction: Detects rising memory pressure early and proactively reduces +// the interval before hitting critical thresholds. +// +// Thresholds and actions: +// - Critical (>90%): Reduce interval to 1/4 (aggressive) +// - High (>70%): Reduce interval to 1/2 +// - Trend damping (>30% AND rising): Reduce interval by 10% +// - Low (<30% max AND avg): Increase interval by 25% +// - Very low (<10% max AND avg): Increase interval by 50%, may exceed normal cap +func (c *changefeedStatus) adjustScanInterval(now time.Time, usage memoryUsageStats) { + current := time.Duration(c.scanInterval.Load()) + if current <= 0 { + current = defaultScanInterval + } + maxInterval := c.maxScanInterval() + if maxInterval < minScanInterval { + maxInterval = minScanInterval + } + // Trend detection: check if memory usage is rising over the observation window. + // This enables proactive intervention before hitting high thresholds. + trendDelta := usage.last - usage.first + isIncreasing := usage.cnt >= minTrendSamples && trendDelta > increasingTrendEpsilon + isAboveTrendStart := usage.last > increasingTrendStartRatio + canAdjustOnTrend := now.Sub(c.lastTrendAdjustTime.Load()) >= scanTrendAdjustCooldown + shouldDampOnTrend := isAboveTrendStart && isIncreasing && canAdjustOnTrend + + // Increase eligibility: conservative conditions to prevent oscillation. + // Requires: cooldown passed, enough samples, sufficient observation span, + // and NOT in an increasing trend situation (to avoid fighting against pressure). + minIncreaseSpan := memoryUsageWindowDuration * minIncreaseSpanNumerator / minIncreaseSpanDenominator + allowedToIncrease := now.Sub(c.lastAdjustTime.Load()) >= scanIntervalAdjustCooldown && + usage.cnt >= minIncreaseSamples && + usage.span >= minIncreaseSpan && + !(isAboveTrendStart && isIncreasing) + + // Determine the new interval based on memory pressure levels. + // Priority order: critical > high > trend damping > very low > low + adjustedOnTrend := false + newInterval := current + switch { + case usage.last > memoryUsageCriticalThreshold || usage.max > memoryUsageCriticalThreshold: + // Critical pressure: aggressive reduction to 1/4 + newInterval = max(current/4, minScanInterval) + case usage.last > memoryUsageHighThreshold || usage.max > memoryUsageHighThreshold: + // High pressure: reduce to 1/2 + newInterval = max(current/2, minScanInterval) + case shouldDampOnTrend: + // Trend damping: pressure is moderate (>30%) but rising. Reduce by 10% to + // preemptively slow down before downstream gets overwhelmed. + newInterval = max(scaleDuration(current, 9, 10), minScanInterval) + adjustedOnTrend = true + case allowedToIncrease && usage.max < memoryUsageVeryLowThreshold && usage.avg < memoryUsageVeryLowThreshold: + // Very low pressure (<20%): increase by 50%, allowed to exceed sync point cap. + maxInterval = maxScanInterval + newInterval = min(scaleDuration(current, 3, 2), maxInterval) + case allowedToIncrease && usage.max < memoryUsageLowThreshold && usage.avg < memoryUsageLowThreshold: + // Low pressure (<40%): increase by 25%, capped by sync point interval. + newInterval = min(scaleDuration(current, 5, 4), maxInterval) + } + + // Anti-oscillation guard: decreases are always applied immediately, + // but increases are blocked if cooldown conditions aren't met. + if newInterval > current && !allowedToIncrease { + return + } + + if newInterval != current { + c.scanInterval.Store(int64(newInterval)) + metrics.EventServiceScanWindowIntervalGaugeVec.WithLabelValues(c.changefeedID.String()).Set(newInterval.Seconds()) + c.lastAdjustTime.Store(now) + if adjustedOnTrend { + c.lastTrendAdjustTime.Store(now) + } + + log.Info("scan interval adjusted", + zap.Stringer("changefeedID", c.changefeedID), + zap.Duration("oldInterval", current), + zap.Duration("newInterval", newInterval), + zap.Duration("maxInterval", maxInterval), + zap.Float64("avgUsage", usage.avg), + zap.Float64("maxUsage", usage.max), + zap.Float64("firstUsage", usage.first), + zap.Float64("lastUsage", usage.last), + zap.Float64("trendDelta", trendDelta), + zap.Int("usageSamples", usage.cnt), + zap.Bool("syncPointEnabled", c.isSyncpointEnabled()), + zap.Duration("syncPointInterval", c.syncPointInterval)) + } +} + +func (c *changefeedStatus) maxScanInterval() time.Duration { + if !c.isSyncpointEnabled() { + return maxScanInterval + } + + interval := c.syncPointInterval + if interval <= 0 { + return maxScanInterval + } + + if interval < maxScanInterval { + return interval + } + return maxScanInterval +} + +func (c *changefeedStatus) refreshMinSentResolvedTs() { + now := time.Now() + minSentResolvedTs := ^uint64(0) + minSentResolvedTsWithStale := ^uint64(0) + hasEligible := false + hasNonStale := false + c.dispatchers.Range(func(_ any, value any) bool { + dispatcher := value.(*atomic.Pointer[dispatcherStat]).Load() + if dispatcher == nil || dispatcher.isRemoved.Load() || dispatcher.seq.Load() == 0 { + return true + } + + hasEligible = true + sentResolvedTs := dispatcher.sentResolvedTs.Load() + if sentResolvedTs < minSentResolvedTsWithStale { + minSentResolvedTsWithStale = sentResolvedTs + } + + lastHeartbeatTime := dispatcher.lastReceivedHeartbeatTime.Load() + if lastHeartbeatTime > 0 && + now.Sub(time.Unix(lastHeartbeatTime, 0)) > scanWindowStaleDispatcherHeartbeatThreshold { + log.Info("dispatcher is stale, skip it's sent resolved ts", zap.Stringer("changefeedID", c.changefeedID), zap.Stringer("dispatcherID", dispatcher.id)) + return true + } + + hasNonStale = true + if sentResolvedTs < minSentResolvedTs { + minSentResolvedTs = sentResolvedTs + } + return true + }) + + if !hasEligible { + c.storeMinSentTs(0) + return + } + if !hasNonStale { + c.storeMinSentTs(minSentResolvedTsWithStale) + return + } + c.storeMinSentTs(minSentResolvedTs) +} + +func (c *changefeedStatus) getScanMaxTs() uint64 { + baseTs := c.minSentTs.Load() + if baseTs == 0 { + return 0 + } + interval := time.Duration(c.scanInterval.Load()) + if interval <= 0 { + interval = defaultScanInterval + } + + return oracle.GoTimeToTS(oracle.GetTimeFromTS(baseTs).Add(interval)) +} + +func (c *changefeedStatus) storeMinSentTs(value uint64) { + prev := c.minSentTs.Load() + if prev == value { + return + } + c.minSentTs.Store(value) + metrics.EventServiceScanWindowBaseTsGaugeVec.WithLabelValues(c.changefeedID.String()).Set(float64(value)) +} + +func scaleDuration(d time.Duration, numerator int64, denominator int64) time.Duration { + if numerator <= 0 || denominator <= 0 { + return d + } + return time.Duration(int64(d) * numerator / denominator) +} diff --git a/pkg/eventservice/scan_window_test.go b/pkg/eventservice/scan_window_test.go new file mode 100644 index 0000000000..01ee458e70 --- /dev/null +++ b/pkg/eventservice/scan_window_test.go @@ -0,0 +1,243 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventservice + +import ( + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/oracle" + "go.uber.org/atomic" +) + +func TestAdjustScanIntervalVeryLowBypassesSyncPointCap(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + + now := time.Now() + status.lastAdjustTime.Store(now.Add(-scanIntervalAdjustCooldown - time.Second)) + + // Start from the sync point capped max interval, then allow it to grow slowly. + status.scanInterval.Store(int64(1 * time.Minute)) + + // Maintain a very low pressure for a full window to allow bypassing the sync point cap. + for i := 0; i <= int(memoryUsageWindowDuration/time.Second); i++ { + status.updateMemoryUsage(now.Add(time.Duration(i)*time.Second), 0, 0) + } + require.Equal(t, int64(90*time.Second), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalLowRespectsSyncPointCap(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + + now := time.Now() + status.lastAdjustTime.Store(now.Add(-scanIntervalAdjustCooldown - time.Second)) + + status.scanInterval.Store(int64(40 * time.Second)) + + for i := 0; i <= int(memoryUsageWindowDuration/time.Second); i++ { + status.updateMemoryUsage(now.Add(time.Duration(i)*time.Second), 0.15, 0) + } + require.Equal(t, int64(50*time.Second), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalDecreaseIgnoresCooldown(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + now := time.Now() + status.lastAdjustTime.Store(now) + + status.scanInterval.Store(int64(40 * time.Second)) + status.updateMemoryUsage(now.Add(memoryUsageWindowDuration), 0.8, 0) + require.Equal(t, int64(20*time.Second), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalCriticalPressure(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + now := time.Now() + status.lastAdjustTime.Store(now) + + status.scanInterval.Store(int64(40 * time.Second)) + + status.updateMemoryUsage(now.Add(memoryUsageWindowDuration), 1, 0) + require.Equal(t, int64(10*time.Second), status.scanInterval.Load()) +} + +func TestUpdateMemoryUsageResetsScanIntervalOnMemoryRelease(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + now := time.Now() + status.scanInterval.Store(int64(40 * time.Second)) + + status.updateMemoryUsage(now, 0.5, 1) + require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalIncreaseWithJitteredSamples(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + + start := time.Now() + status.lastAdjustTime.Store(start.Add(-scanIntervalAdjustCooldown - time.Second)) + + status.scanInterval.Store(int64(40 * time.Second)) + + // Use a >1s interval to simulate heartbeat jitter, so the window span will be + // slightly less than memoryUsageWindowDuration. + step := 1100 * time.Millisecond + for i := 0; i < 28; i++ { + status.updateMemoryUsage(start.Add(time.Duration(i)*step), 0.15, 0) + } + require.Equal(t, int64(50*time.Second), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalDecreasesWhenUsageIncreasing(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + now := time.Now() + status.lastAdjustTime.Store(now) + + status.scanInterval.Store(int64(40 * time.Second)) + + status.updateMemoryUsage(now, 0.10, 0) + status.updateMemoryUsage(now.Add(1*time.Second), 0.11, 0) + status.updateMemoryUsage(now.Add(2*time.Second), 0.12, 0) + status.updateMemoryUsage(now.Add(3*time.Second), 0.13, 0) + require.Equal(t, int64(40*time.Second), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalDecreasesWhenUsageIncreasingAboveThirtyPercent(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + now := time.Now() + status.lastAdjustTime.Store(now) + status.lastTrendAdjustTime.Store(now.Add(-scanTrendAdjustCooldown - time.Second)) + + status.scanInterval.Store(int64(40 * time.Second)) + + status.updateMemoryUsage(now, 0.31, 0) + status.updateMemoryUsage(now.Add(1*time.Second), 0.32, 0) + status.updateMemoryUsage(now.Add(2*time.Second), 0.33, 0) + status.updateMemoryUsage(now.Add(3*time.Second), 0.34, 0) + require.Equal(t, int64(36*time.Second), status.scanInterval.Load()) +} + +func TestRefreshMinSentResolvedTsMinAndSkipRules(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + + stale := &dispatcherStat{} + stale.seq.Store(1) + stale.sentResolvedTs.Store(10) + stale.lastReceivedHeartbeatTime.Store(time.Now().Add(-scanWindowStaleDispatcherHeartbeatThreshold - time.Second).Unix()) + + removed := &dispatcherStat{} + removed.seq.Store(1) + removed.sentResolvedTs.Store(150) + removed.isRemoved.Store(true) + + uninitialized := &dispatcherStat{} + uninitialized.seq.Store(0) + uninitialized.sentResolvedTs.Store(10) + + first := &dispatcherStat{} + first.seq.Store(1) + first.sentResolvedTs.Store(200) + + second := &dispatcherStat{} + second.seq.Store(1) + second.sentResolvedTs.Store(50) + + stalePtr := &atomic.Pointer[dispatcherStat]{} + stalePtr.Store(stale) + status.addDispatcher(common.NewDispatcherID(), stalePtr) + + removedPtr := &atomic.Pointer[dispatcherStat]{} + removedPtr.Store(removed) + status.addDispatcher(common.NewDispatcherID(), removedPtr) + + uninitializedPtr := &atomic.Pointer[dispatcherStat]{} + uninitializedPtr.Store(uninitialized) + status.addDispatcher(common.NewDispatcherID(), uninitializedPtr) + + firstPtr := &atomic.Pointer[dispatcherStat]{} + firstPtr.Store(first) + status.addDispatcher(common.NewDispatcherID(), firstPtr) + + secondPtr := &atomic.Pointer[dispatcherStat]{} + secondPtr.Store(second) + status.addDispatcher(common.NewDispatcherID(), secondPtr) + + status.refreshMinSentResolvedTs() + require.Equal(t, uint64(50), status.minSentTs.Load()) + + second.isRemoved.Store(true) + status.refreshMinSentResolvedTs() + require.Equal(t, uint64(200), status.minSentTs.Load()) + + stale.isRemoved.Store(true) + first.seq.Store(0) + status.refreshMinSentResolvedTs() + require.Equal(t, uint64(0), status.minSentTs.Load()) +} + +func TestRefreshMinSentResolvedTsStaleFallback(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + + stale := &dispatcherStat{} + stale.seq.Store(1) + stale.sentResolvedTs.Store(123) + stale.lastReceivedHeartbeatTime.Store(time.Now().Add(-scanWindowStaleDispatcherHeartbeatThreshold - time.Second).Unix()) + + stalePtr := &atomic.Pointer[dispatcherStat]{} + stalePtr.Store(stale) + status.addDispatcher(common.NewDispatcherID(), stalePtr) + + status.refreshMinSentResolvedTs() + require.Equal(t, uint64(123), status.minSentTs.Load()) +} + +func TestGetScanMaxTsFallbackInterval(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + + baseTime := time.Unix(1234, 0) + baseTs := oracle.GoTimeToTS(baseTime) + status.minSentTs.Store(baseTs) + + status.scanInterval.Store(0) + require.Equal(t, oracle.GoTimeToTS(baseTime.Add(defaultScanInterval)), status.getScanMaxTs()) + + status.scanInterval.Store(int64(10 * time.Second)) + require.Equal(t, oracle.GoTimeToTS(baseTime.Add(10*time.Second)), status.getScanMaxTs()) + + status.minSentTs.Store(0) + require.Equal(t, uint64(0), status.getScanMaxTs()) +} diff --git a/pkg/metrics/event_service.go b/pkg/metrics/event_service.go index 4c08ada207..e6ed10cc90 100644 --- a/pkg/metrics/event_service.go +++ b/pkg/metrics/event_service.go @@ -55,6 +55,20 @@ var ( Name: "resolved_ts_lag", Help: "resolved ts lag of eventService in seconds", }, []string{"type"}) + EventServiceScanWindowBaseTsGaugeVec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "scan_window_base_ts", + Help: "The base ts of the scan window for each changefeed", + }, []string{"changefeed"}) + EventServiceScanWindowIntervalGaugeVec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "scan_window_interval", + Help: "The scan window interval in seconds for each changefeed", + }, []string{"changefeed"}) EventServiceScanDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "ticdc", @@ -185,6 +199,8 @@ func initEventServiceMetrics(registry *prometheus.Registry) { registry.MustRegister(EventServiceSendEventDuration) registry.MustRegister(EventServiceResolvedTsGauge) registry.MustRegister(EventServiceResolvedTsLagGauge) + registry.MustRegister(EventServiceScanWindowBaseTsGaugeVec) + registry.MustRegister(EventServiceScanWindowIntervalGaugeVec) registry.MustRegister(EventServiceScanDuration) registry.MustRegister(EventServiceScannedCount) registry.MustRegister(EventServiceDispatcherGauge) diff --git a/pkg/sink/mysql/mysql_writer.go b/pkg/sink/mysql/mysql_writer.go index a7ecb632d6..b8f98c3749 100644 --- a/pkg/sink/mysql/mysql_writer.go +++ b/pkg/sink/mysql/mysql_writer.go @@ -143,6 +143,7 @@ func (w *Writer) FlushDDLEvent(event *commonEvent.DDLEvent) error { func (w *Writer) FlushSyncPointEvent(event *commonEvent.SyncPointEvent) error { if w.cfg.DryRun { + log.Info("dry-run mode, skip send syncpoint event", zap.Stringer("changefeedID", w.ChangefeedID), zap.Uint64("commitTs", event.GetCommitTs())) return nil } diff --git a/tools/workload/Makefile b/tools/workload/Makefile index 498c32262e..9b8f819c87 100644 --- a/tools/workload/Makefile +++ b/tools/workload/Makefile @@ -7,10 +7,10 @@ all: build build: @echo "Building $(BINARY_NAME)..." @mkdir -p $(OUTPUT_DIR) - @go build -o $(OUTPUT_DIR)/$(BINARY_NAME) . + @go build -mod=vendor -o $(OUTPUT_DIR)/$(BINARY_NAME) . clean: @echo "Cleaning up..." @rm -rf $(OUTPUT_DIR) -.PHONY: all build clean \ No newline at end of file +.PHONY: all build clean diff --git a/tools/workload/app.go b/tools/workload/app.go index 47f9f0629b..3c37b7f57d 100644 --- a/tools/workload/app.go +++ b/tools/workload/app.go @@ -56,6 +56,10 @@ type WorkloadStats struct { QueryCount atomic.Uint64 ErrorCount atomic.Uint64 CreatedTableNum atomic.Int32 + DDLExecuted atomic.Uint64 + DDLSucceeded atomic.Uint64 + DDLSkipped atomic.Uint64 + DDLFailed atomic.Uint64 } // WorkloadApp is the main structure of the application @@ -161,12 +165,12 @@ func (app *WorkloadApp) executeWorkload(wg *sync.WaitGroup) error { zap.Int("dbCount", len(app.DBManager.GetDBs())), zap.Int("tableCount", app.Config.TableCount)) - if !app.Config.SkipCreateTable && app.Config.Action == "prepare" { - app.handlePrepareAction(insertConcurrency, wg) - return nil + if app.Config.Action == "ddl" || app.Config.OnlyDDL { + return app.handleDDLExecution(wg) } - if app.Config.OnlyDDL { + if !app.Config.SkipCreateTable && app.Config.Action == "prepare" { + app.handlePrepareAction(insertConcurrency, wg) return nil } diff --git a/tools/workload/config.go b/tools/workload/config.go index e79bc63ea2..7fca1636f6 100644 --- a/tools/workload/config.go +++ b/tools/workload/config.go @@ -17,6 +17,7 @@ import ( "flag" "fmt" "sync" + "time" ) // WorkloadConfig saves all the configurations for the workload @@ -44,6 +45,9 @@ type WorkloadConfig struct { Action string SkipCreateTable bool OnlyDDL bool + DDLConfigPath string + DDLWorker int + DDLTimeout time.Duration // Special workload config RowSize int @@ -85,6 +89,9 @@ func NewWorkloadConfig() *WorkloadConfig { Action: "prepare", SkipCreateTable: false, OnlyDDL: false, + DDLConfigPath: "", + DDLWorker: 1, + DDLTimeout: 2 * time.Minute, // For large row workload RowSize: 10240, @@ -115,7 +122,7 @@ func (c *WorkloadConfig) ParseFlags() error { flag.Float64Var(&c.PercentageForUpdate, "percentage-for-update", c.PercentageForUpdate, "percentage for update: [0, 1.0]") flag.Float64Var(&c.PercentageForDelete, "percentage-for-delete", c.PercentageForDelete, "percentage for delete: [0, 1.0]") flag.BoolVar(&c.SkipCreateTable, "skip-create-table", c.SkipCreateTable, "do not create tables") - flag.StringVar(&c.Action, "action", c.Action, "action of the workload: [prepare, insert, update, delete, write, cleanup]") + flag.StringVar(&c.Action, "action", c.Action, "action of the workload: [prepare, insert, update, delete, write, ddl, cleanup]") flag.StringVar(&c.WorkloadType, "workload-type", c.WorkloadType, "workload type: [bank, sysbench, large_row, shop_item, uuu, bank2, bank_update, crawler, dc]") flag.StringVar(&c.DBHost, "database-host", c.DBHost, "database host") flag.StringVar(&c.DBUser, "database-user", c.DBUser, "database user") @@ -123,6 +130,9 @@ func (c *WorkloadConfig) ParseFlags() error { flag.StringVar(&c.DBName, "database-db-name", c.DBName, "database db name") flag.IntVar(&c.DBPort, "database-port", c.DBPort, "database port") flag.BoolVar(&c.OnlyDDL, "only-ddl", c.OnlyDDL, "only generate ddl") + flag.StringVar(&c.DDLConfigPath, "ddl-config", c.DDLConfigPath, "ddl config file path, must be .toml") + flag.IntVar(&c.DDLWorker, "ddl-worker", c.DDLWorker, "ddl worker concurrency") + flag.DurationVar(&c.DDLTimeout, "ddl-timeout", c.DDLTimeout, "timeout for each ddl statement") flag.StringVar(&c.LogFile, "log-file", c.LogFile, "log file path") flag.StringVar(&c.LogLevel, "log-level", c.LogLevel, "log file path") // For large row workload @@ -146,6 +156,18 @@ func (c *WorkloadConfig) ParseFlags() error { c.PercentageForUpdate, c.PercentageForDelete) } + if c.Action == "ddl" || c.OnlyDDL { + if c.DDLConfigPath == "" { + return fmt.Errorf("ddl requires -ddl-config") + } + if c.DDLWorker <= 0 { + return fmt.Errorf("ddl-worker must be > 0") + } + if c.DDLTimeout <= 0 { + return fmt.Errorf("ddl-timeout must be > 0") + } + } + return nil } diff --git a/tools/workload/ddl_app.go b/tools/workload/ddl_app.go new file mode 100644 index 0000000000..f4578b9fea --- /dev/null +++ b/tools/workload/ddl_app.go @@ -0,0 +1,41 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "sync" + + "github.com/pingcap/errors" + plog "github.com/pingcap/log" + "go.uber.org/zap" +) + +func (app *WorkloadApp) handleDDLExecution(wg *sync.WaitGroup) error { + cfg, err := LoadDDLConfig(app.Config.DDLConfigPath) + if err != nil { + return err + } + + runner, err := NewDDLRunner(app, cfg) + if err != nil { + return errors.Trace(err) + } + + plog.Info("start ddl workload", + zap.String("mode", cfg.Mode), + zap.Int("ddlWorker", app.Config.DDLWorker), + zap.String("ddlTimeout", app.Config.DDLTimeout.String())) + runner.Start(wg) + return nil +} diff --git a/tools/workload/ddl_config.go b/tools/workload/ddl_config.go new file mode 100644 index 0000000000..cf9f0f9cc7 --- /dev/null +++ b/tools/workload/ddl_config.go @@ -0,0 +1,170 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/BurntSushi/toml" + "github.com/pingcap/errors" +) + +const ( + ddlModeFixed = "fixed" + ddlModeRandom = "random" +) + +type DDLConfig struct { + Mode string `toml:"mode"` + RatePerMinute DDLRatePerMinute `toml:"rate_per_minute"` + Tables []string `toml:"tables"` +} + +type DDLRatePerMinute struct { + AddColumn int `toml:"add_column"` + DropColumn int `toml:"drop_column"` + AddIndex int `toml:"add_index"` + DropIndex int `toml:"drop_index"` + TruncateTable int `toml:"truncate_table"` +} + +func LoadDDLConfig(path string) (*DDLConfig, error) { + if strings.TrimSpace(path) == "" { + return nil, errors.New("ddl config path is empty") + } + if filepath.Ext(path) != ".toml" { + return nil, errors.Errorf("ddl config must be a .toml file: %s", path) + } + + var cfg DDLConfig + meta, err := toml.DecodeFile(path, &cfg) + if err != nil { + return nil, errors.Annotate(err, "decode ddl config failed") + } + if undecoded := meta.Undecoded(); len(undecoded) > 0 { + return nil, errors.Errorf("unknown keys in ddl config: %v", undecoded) + } + + cfg.normalize() + if err := cfg.validate(); err != nil { + return nil, err + } + return &cfg, nil +} + +func (c *DDLConfig) normalize() { + c.Mode = strings.ToLower(strings.TrimSpace(c.Mode)) + if c.Mode == "" { + if len(c.Tables) > 0 { + c.Mode = ddlModeFixed + } else { + c.Mode = ddlModeRandom + } + } + + // Trim and drop empty entries. + tables := make([]string, 0, len(c.Tables)) + for _, t := range c.Tables { + t = strings.TrimSpace(t) + if t != "" { + tables = append(tables, t) + } + } + c.Tables = tables +} + +func (c *DDLConfig) validate() error { + if c.Mode != ddlModeFixed && c.Mode != ddlModeRandom { + return errors.Errorf("unsupported ddl mode: %s", c.Mode) + } + if c.Mode == ddlModeFixed && len(c.Tables) == 0 { + return errors.New("ddl mode fixed requires tables") + } + + if err := validateRate("add_column", c.RatePerMinute.AddColumn); err != nil { + return err + } + if err := validateRate("drop_column", c.RatePerMinute.DropColumn); err != nil { + return err + } + if err := validateRate("add_index", c.RatePerMinute.AddIndex); err != nil { + return err + } + if err := validateRate("drop_index", c.RatePerMinute.DropIndex); err != nil { + return err + } + if err := validateRate("truncate_table", c.RatePerMinute.TruncateTable); err != nil { + return err + } + + if c.totalRate() == 0 { + return errors.New("ddl config has no enabled ddl types") + } + return nil +} + +func validateRate(name string, v int) error { + if v < 0 { + return errors.Errorf("ddl rate must be >= 0: %s=%d", name, v) + } + return nil +} + +func (c *DDLConfig) totalRate() int { + return c.RatePerMinute.AddColumn + + c.RatePerMinute.DropColumn + + c.RatePerMinute.AddIndex + + c.RatePerMinute.DropIndex + + c.RatePerMinute.TruncateTable +} + +type TableName struct { + Schema string + Name string +} + +func ParseTableName(raw string, defaultSchema string) (TableName, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return TableName{}, errors.New("table name is empty") + } + + parts := strings.Split(raw, ".") + switch len(parts) { + case 1: + name := strings.TrimSpace(parts[0]) + if name == "" { + return TableName{}, errors.Errorf("invalid table name: %s", raw) + } + if strings.TrimSpace(defaultSchema) == "" { + return TableName{}, errors.Errorf("table %s missing schema", raw) + } + return TableName{Schema: strings.TrimSpace(defaultSchema), Name: name}, nil + case 2: + schema := strings.TrimSpace(parts[0]) + name := strings.TrimSpace(parts[1]) + if schema == "" || name == "" { + return TableName{}, errors.Errorf("invalid table name: %s", raw) + } + return TableName{Schema: schema, Name: name}, nil + default: + return TableName{}, errors.Errorf("invalid table name: %s", raw) + } +} + +func (t TableName) String() string { + return fmt.Sprintf("%s.%s", t.Schema, t.Name) +} diff --git a/tools/workload/ddl_config_test.go b/tools/workload/ddl_config_test.go new file mode 100644 index 0000000000..61db9d1c54 --- /dev/null +++ b/tools/workload/ddl_config_test.go @@ -0,0 +1,136 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseTableName(t *testing.T) { + t.Parallel() + + t.Run("schema qualified", func(t *testing.T) { + table, err := ParseTableName("test.sbtest1", "ignored") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if table.Schema != "test" || table.Name != "sbtest1" { + t.Fatalf("unexpected table: %+v", table) + } + }) + + t.Run("default schema", func(t *testing.T) { + table, err := ParseTableName("sbtest1", "test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if table.Schema != "test" || table.Name != "sbtest1" { + t.Fatalf("unexpected table: %+v", table) + } + }) + + t.Run("missing schema", func(t *testing.T) { + _, err := ParseTableName("sbtest1", "") + if err == nil { + t.Fatalf("expected error") + } + }) + + t.Run("invalid", func(t *testing.T) { + _, err := ParseTableName("a.b.c", "test") + if err == nil { + t.Fatalf("expected error") + } + }) +} + +func TestLoadDDLConfig(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + t.Run("extension check", func(t *testing.T) { + path := filepath.Join(dir, "ddl.txt") + if err := os.WriteFile(path, []byte("mode = \"fixed\""), 0o644); err != nil { + t.Fatalf("write file failed: %v", err) + } + _, err := LoadDDLConfig(path) + if err == nil { + t.Fatalf("expected error") + } + }) + + t.Run("default mode fixed", func(t *testing.T) { + path := filepath.Join(dir, "ddl_fixed.toml") + content := ` +tables = ["test.sbtest1"] + +[rate_per_minute] +add_column = 1 +` + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write file failed: %v", err) + } + cfg, err := LoadDDLConfig(path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.Mode != ddlModeFixed { + t.Fatalf("expected mode fixed, got %s", cfg.Mode) + } + }) + + t.Run("default mode random", func(t *testing.T) { + path := filepath.Join(dir, "ddl_random.toml") + content := ` +[rate_per_minute] +add_column = 1 +` + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write file failed: %v", err) + } + cfg, err := LoadDDLConfig(path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.Mode != ddlModeRandom { + t.Fatalf("expected mode random, got %s", cfg.Mode) + } + }) + + t.Run("no enabled ddl types", func(t *testing.T) { + path := filepath.Join(dir, "ddl_empty.toml") + content := ` +mode = "fixed" + +tables = ["test.sbtest1"] + +[rate_per_minute] +add_column = 0 +drop_column = 0 +add_index = 0 +drop_index = 0 +truncate_table = 0 +` + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write file failed: %v", err) + } + _, err := LoadDDLConfig(path) + if err == nil { + t.Fatalf("expected error") + } + }) +} diff --git a/tools/workload/ddl_executor.go b/tools/workload/ddl_executor.go new file mode 100644 index 0000000000..6b4c7c768c --- /dev/null +++ b/tools/workload/ddl_executor.go @@ -0,0 +1,282 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "database/sql" + "fmt" + "math/rand" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/pingcap/errors" + plog "github.com/pingcap/log" + "go.uber.org/zap" +) + +const ( + ddlColumnPrefix = "ddl_col_" + ddlIndexPrefix = "ddl_idx_" +) + +var ddlNameSeq atomic.Uint64 + +func (r *DDLRunner) startWorkers(wg *sync.WaitGroup) { + workerCount := r.app.Config.DDLWorker + if workerCount <= 0 { + workerCount = 1 + } + + wg.Add(workerCount) + for workerID := range workerCount { + db := r.app.DBManager.GetDB() + go func(workerID int, db *DBWrapper) { + defer func() { + plog.Info("ddl worker exited", zap.Int("worker", workerID)) + wg.Done() + }() + + conn, err := getConnWithTimeout(db.DB, 10*time.Second) + if err != nil { + plog.Info("get connection failed for ddl worker", zap.Error(err)) + time.Sleep(5 * time.Second) + return + } + defer func() { + if conn != nil { + conn.Close() + } + }() + + plog.Info("start ddl worker", zap.Int("worker", workerID), zap.String("db", db.Name)) + + for { + task := <-r.taskCh + if err := r.executeTask(conn, task); err != nil { + if r.app.isConnectionError(err) { + conn.Close() + time.Sleep(2 * time.Second) + newConn, err := getConnWithTimeout(db.DB, 10*time.Second) + if err != nil { + plog.Info("reconnect failed for ddl worker", zap.Error(err)) + time.Sleep(5 * time.Second) + continue + } + conn = newConn + } + } + } + }(workerID, db) + } +} + +func getConnWithTimeout(db *sql.DB, timeout time.Duration) (*sql.Conn, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + conn, err := db.Conn(ctx) + if err != nil { + return nil, errors.Trace(err) + } + return conn, nil +} + +func (r *DDLRunner) executeTask(conn *sql.Conn, task DDLTask) error { + ctx, cancel := context.WithTimeout(context.Background(), r.app.Config.DDLTimeout) + defer cancel() + + sqlStr, skipped, reason, err := r.buildDDL(ctx, conn, task) + if err != nil { + r.app.Stats.DDLFailed.Add(1) + r.app.Stats.ErrorCount.Add(1) + plog.Info("build ddl failed", + zap.String("ddlType", task.Type.String()), + zap.String("table", task.Table.String()), + zap.Error(err)) + return err + } + if skipped { + r.app.Stats.DDLSkipped.Add(1) + if reason != "" { + plog.Debug("ddl task skipped", + zap.String("ddlType", task.Type.String()), + zap.String("table", task.Table.String()), + zap.String("reason", reason)) + } + return nil + } + + r.app.Stats.DDLExecuted.Add(1) + r.app.Stats.QueryCount.Add(1) + start := time.Now() + if _, err := conn.ExecContext(ctx, sqlStr); err != nil { + r.app.Stats.DDLFailed.Add(1) + r.app.Stats.ErrorCount.Add(1) + plog.Info("execute ddl failed", + zap.String("ddlType", task.Type.String()), + zap.String("table", task.Table.String()), + zap.Duration("cost", time.Since(start)), + zap.String("sql", getSQLPreview(sqlStr)), + zap.Error(err)) + return err + } + + r.app.Stats.DDLSucceeded.Add(1) + plog.Debug("ddl executed", + zap.String("ddlType", task.Type.String()), + zap.String("table", task.Table.String()), + zap.Duration("cost", time.Since(start))) + return nil +} + +func (r *DDLRunner) buildDDL(ctx context.Context, conn *sql.Conn, task DDLTask) (sqlStr string, skipped bool, reason string, err error) { + switch task.Type { + case ddlAddColumn: + return r.buildAddColumnDDL(task.Table), false, "", nil + case ddlDropColumn: + return r.buildDropColumnDDL(ctx, conn, task.Table) + case ddlAddIndex: + return r.buildAddIndexDDL(ctx, conn, task.Table) + case ddlDropIndex: + return r.buildDropIndexDDL(ctx, conn, task.Table) + case ddlTruncateTable: + return r.buildTruncateTableDDL(task.Table), false, "", nil + default: + return "", false, "", errors.Errorf("unknown ddl type: %d", task.Type) + } +} + +func (r *DDLRunner) buildAddColumnDDL(table TableName) string { + colName := ddlColumnPrefix + ddlNameSuffix() + return fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s BIGINT NOT NULL DEFAULT 0", + quoteTable(table), + quoteIdent(colName)) +} + +func (r *DDLRunner) buildDropColumnDDL(ctx context.Context, conn *sql.Conn, table TableName) (string, bool, string, error) { + col, ok, err := selectOne(ctx, conn, ` +SELECT column_name +FROM information_schema.columns +WHERE table_schema = ? + AND table_name = ? + AND column_name LIKE ? +`, table.Schema, table.Name, ddlColumnPrefix+"%") + if err != nil { + return "", false, "", err + } + if !ok { + return "", true, "no ddl columns", nil + } + return fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", + quoteTable(table), + quoteIdent(col)), false, "", nil +} + +func (r *DDLRunner) buildAddIndexDDL(ctx context.Context, conn *sql.Conn, table TableName) (string, bool, string, error) { + col, ok, err := selectOne(ctx, conn, ` +SELECT column_name +FROM information_schema.columns +WHERE table_schema = ? + AND table_name = ? + AND column_name LIKE ? +`, table.Schema, table.Name, ddlColumnPrefix+"%") + if err != nil { + return "", false, "", err + } + + if !ok { + col, ok, err = selectOne(ctx, conn, ` +SELECT column_name +FROM information_schema.columns +WHERE table_schema = ? + AND table_name = ? + AND data_type NOT IN ('json','tinyblob','blob','mediumblob','longblob','tinytext','text','mediumtext','longtext') +`, table.Schema, table.Name) + if err != nil { + return "", false, "", err + } + if !ok { + return "", true, "no suitable columns", nil + } + } + + idxName := ddlIndexPrefix + ddlNameSuffix() + return fmt.Sprintf("ALTER TABLE %s ADD INDEX %s (%s)", + quoteTable(table), + quoteIdent(idxName), + quoteIdent(col)), false, "", nil +} + +func (r *DDLRunner) buildDropIndexDDL(ctx context.Context, conn *sql.Conn, table TableName) (string, bool, string, error) { + idx, ok, err := selectOne(ctx, conn, ` +SELECT DISTINCT index_name +FROM information_schema.statistics +WHERE table_schema = ? + AND table_name = ? + AND index_name LIKE ? +`, table.Schema, table.Name, ddlIndexPrefix+"%") + if err != nil { + return "", false, "", err + } + if !ok { + return "", true, "no ddl indexes", nil + } + return fmt.Sprintf("ALTER TABLE %s DROP INDEX %s", + quoteTable(table), + quoteIdent(idx)), false, "", nil +} + +func (r *DDLRunner) buildTruncateTableDDL(table TableName) string { + return fmt.Sprintf("TRUNCATE TABLE %s", quoteTable(table)) +} + +func ddlNameSuffix() string { + seq := ddlNameSeq.Add(1) + return fmt.Sprintf("%d_%d", time.Now().UnixNano(), seq) +} + +func quoteIdent(name string) string { + escaped := strings.ReplaceAll(name, "`", "``") + return "`" + escaped + "`" +} + +func quoteTable(table TableName) string { + return quoteIdent(table.Schema) + "." + quoteIdent(table.Name) +} + +func selectOne(ctx context.Context, conn *sql.Conn, query string, args ...interface{}) (value string, ok bool, err error) { + rows, err := conn.QueryContext(ctx, query, args...) + if err != nil { + return "", false, errors.Trace(err) + } + defer rows.Close() + + var values []string + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + return "", false, errors.Trace(err) + } + values = append(values, v) + } + if err := rows.Err(); err != nil { + return "", false, errors.Trace(err) + } + if len(values) == 0 { + return "", false, nil + } + return values[rand.Intn(len(values))], true, nil +} diff --git a/tools/workload/ddl_runner.go b/tools/workload/ddl_runner.go new file mode 100644 index 0000000000..5cdcf99d2b --- /dev/null +++ b/tools/workload/ddl_runner.go @@ -0,0 +1,285 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "database/sql" + "math/rand" + "sync" + "sync/atomic" + "time" + + "github.com/pingcap/errors" + plog "github.com/pingcap/log" + "go.uber.org/zap" +) + +const ( + randomModeTableSampleSize = 10 + ddlTaskChanMinBufSize = 1024 +) + +type DDLTask struct { + Type DDLType + Table TableName +} + +type DDLRunner struct { + app *WorkloadApp + cfg *DDLConfig + + taskCh chan DDLTask + selector ddlTableSelector + + randomSchema string +} + +func NewDDLRunner(app *WorkloadApp, cfg *DDLConfig) (*DDLRunner, error) { + if app == nil || app.Config == nil || app.DBManager == nil { + return nil, errors.New("ddl runner requires initialized app") + } + + r := &DDLRunner{ + app: app, + cfg: cfg, + } + + bufSize := cfg.totalRate() * 2 + if bufSize < ddlTaskChanMinBufSize { + bufSize = ddlTaskChanMinBufSize + } + r.taskCh = make(chan DDLTask, bufSize) + + switch cfg.Mode { + case ddlModeFixed: + defaultSchema := app.Config.DBName + if app.Config.DBPrefix != "" && app.Config.DBNum > 0 { + defaultSchema = "" + } + tables, err := parseTableList(cfg.Tables, defaultSchema) + if err != nil { + return nil, err + } + r.selector = newFixedTableSelector(tables) + case ddlModeRandom: + if app.Config.DBPrefix != "" || app.Config.DBNum != 1 { + return nil, errors.New("ddl random mode only supports single database connection") + } + r.randomSchema = app.Config.DBName + randomSelector := newRandomTableSelector() + if err := r.refreshRandomTables(randomSelector); err != nil { + return nil, err + } + r.selector = randomSelector + default: + return nil, errors.Errorf("unsupported ddl mode: %s", cfg.Mode) + } + + return r, nil +} + +func (r *DDLRunner) Start(wg *sync.WaitGroup) { + if r.cfg.Mode == ddlModeRandom { + r.startRandomTableRefresh() + } + + r.startTaskSchedulers() + r.startWorkers(wg) +} + +func (r *DDLRunner) startTaskSchedulers() { + r.startTypeScheduler(ddlAddColumn, r.cfg.RatePerMinute.AddColumn) + r.startTypeScheduler(ddlDropColumn, r.cfg.RatePerMinute.DropColumn) + r.startTypeScheduler(ddlAddIndex, r.cfg.RatePerMinute.AddIndex) + r.startTypeScheduler(ddlDropIndex, r.cfg.RatePerMinute.DropIndex) + r.startTypeScheduler(ddlTruncateTable, r.cfg.RatePerMinute.TruncateTable) +} + +func (r *DDLRunner) startTypeScheduler(ddlType DDLType, perMinute int) { + if perMinute <= 0 { + return + } + + go func() { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + + for { + for i := 0; i < perMinute; i++ { + table, ok := r.selector.Next() + if !ok { + r.app.Stats.DDLSkipped.Add(1) + continue + } + r.taskCh <- DDLTask{Type: ddlType, Table: table} + } + <-ticker.C + } + }() +} + +func (r *DDLRunner) startRandomTableRefresh() { + go func() { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + + randomSelector := r.selector.(*randomTableSelector) + for range ticker.C { + if err := r.refreshRandomTables(randomSelector); err != nil { + plog.Info("refresh random tables failed", zap.Error(err)) + } + } + }() +} + +func (r *DDLRunner) refreshRandomTables(selector *randomTableSelector) error { + dbs := r.app.DBManager.GetDBs() + if len(dbs) == 0 { + return errors.New("no database connections available") + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + tableNames, err := fetchBaseTables(ctx, dbs[0].DB, r.randomSchema) + if err != nil { + return err + } + + if len(tableNames) == 0 { + selector.Update(nil) + plog.Info("no tables found for ddl random mode", zap.String("schema", r.randomSchema)) + return nil + } + + sampled := sampleStrings(tableNames, randomModeTableSampleSize) + tables := make([]TableName, 0, len(sampled)) + for _, name := range sampled { + tables = append(tables, TableName{Schema: r.randomSchema, Name: name}) + } + selector.Update(tables) + + plog.Info("random tables refreshed", + zap.String("schema", r.randomSchema), + zap.Int("tableCount", len(tables))) + return nil +} + +func fetchBaseTables(ctx context.Context, db *sql.DB, schema string) ([]string, error) { + const query = ` +SELECT table_name +FROM information_schema.tables +WHERE table_schema = ? + AND table_type = 'BASE TABLE'` + rows, err := db.QueryContext(ctx, query, schema) + if err != nil { + return nil, errors.Annotate(err, "query tables from information_schema failed") + } + defer rows.Close() + + var tables []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, errors.Annotate(err, "scan table name failed") + } + tables = append(tables, name) + } + if err := rows.Err(); err != nil { + return nil, errors.Annotate(err, "iterate table names failed") + } + return tables, nil +} + +func sampleStrings(in []string, n int) []string { + if n <= 0 || len(in) == 0 { + return nil + } + if len(in) <= n { + out := make([]string, len(in)) + copy(out, in) + return out + } + + indices := rand.Perm(len(in))[:n] + out := make([]string, 0, n) + for _, idx := range indices { + out = append(out, in[idx]) + } + return out +} + +type ddlTableSelector interface { + Next() (TableName, bool) +} + +type fixedTableSelector struct { + tables []TableName + next atomic.Uint64 +} + +func newFixedTableSelector(tables []TableName) *fixedTableSelector { + return &fixedTableSelector{tables: tables} +} + +func (s *fixedTableSelector) Next() (TableName, bool) { + if len(s.tables) == 0 { + return TableName{}, false + } + i := int(s.next.Add(1)-1) % len(s.tables) + return s.tables[i], true +} + +type randomTableSelector struct { + mu sync.RWMutex + tables []TableName +} + +func newRandomTableSelector() *randomTableSelector { + return &randomTableSelector{} +} + +func (s *randomTableSelector) Update(tables []TableName) { + s.mu.Lock() + s.tables = tables + s.mu.Unlock() +} + +func (s *randomTableSelector) Next() (TableName, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + if len(s.tables) == 0 { + return TableName{}, false + } + return s.tables[rand.Intn(len(s.tables))], true +} + +func parseTableList(rawTables []string, defaultSchema string) ([]TableName, error) { + seen := make(map[string]struct{}, len(rawTables)) + out := make([]TableName, 0, len(rawTables)) + for _, raw := range rawTables { + table, err := ParseTableName(raw, defaultSchema) + if err != nil { + return nil, err + } + key := table.String() + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, table) + } + return out, nil +} diff --git a/tools/workload/ddl_types.go b/tools/workload/ddl_types.go new file mode 100644 index 0000000000..f03e2a258a --- /dev/null +++ b/tools/workload/ddl_types.go @@ -0,0 +1,41 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +type DDLType int + +const ( + ddlAddColumn DDLType = iota + ddlDropColumn + ddlAddIndex + ddlDropIndex + ddlTruncateTable +) + +func (t DDLType) String() string { + switch t { + case ddlAddColumn: + return "add_column" + case ddlDropColumn: + return "drop_column" + case ddlAddIndex: + return "add_index" + case ddlDropIndex: + return "drop_index" + case ddlTruncateTable: + return "truncate_table" + default: + return "unknown" + } +} diff --git a/tools/workload/go.mod b/tools/workload/go.mod index 839aeddc7f..b8704fb2fd 100644 --- a/tools/workload/go.mod +++ b/tools/workload/go.mod @@ -3,6 +3,7 @@ module workload go 1.25.10 require ( + github.com/BurntSushi/toml v1.5.0 github.com/go-sql-driver/mysql v1.9.3 github.com/google/uuid v1.6.0 github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f diff --git a/tools/workload/go.sum b/tools/workload/go.sum index 7a230faced..3b60c86951 100644 --- a/tools/workload/go.sum +++ b/tools/workload/go.sum @@ -1,5 +1,7 @@ filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/tools/workload/readme.md b/tools/workload/readme.md index 5b27407923..188f68f8d4 100644 --- a/tools/workload/readme.md +++ b/tools/workload/readme.md @@ -17,12 +17,57 @@ make ## Common Usage Scenarios +### 0. DDL Workload + +Run DDL workload based on a TOML config file: + +```bash +./bin/workload -action ddl \ + -database-host 127.0.0.1 \ + -database-port 4000 \ + -database-db-name test \ + -ddl-config ./ddl.toml \ + -ddl-worker 1 \ + -ddl-timeout 2m +``` + +`ddl.toml` example (fixed mode): + +```toml +mode = "fixed" + +tables = [ + "test.sbtest1", + "test.sbtest2", +] + +[rate_per_minute] +add_column = 10 +drop_column = 10 +add_index = 5 +drop_index = 5 +truncate_table = 1 +``` + +`ddl.toml` example (random mode, omit `tables`): + +```toml +mode = "random" + +[rate_per_minute] +add_column = 10 +drop_column = 10 +add_index = 5 +drop_index = 5 +truncate_table = 0 +``` + ### 1. Sysbench-style Data Insertion Insert test data using sysbench-compatible schema: ```bash -./workload -action insert \ +./bin/workload -action insert \ -database-host 127.0.0.1 \ -database-port 4000 \ -database-db-name db1 \ @@ -38,7 +83,7 @@ Insert test data using sysbench-compatible schema: Update existing data with large row operations: ```bash -./workload -action update \ +./bin/workload -action update \ -database-host 127.0.0.1 \ -database-port 4000 \ -database-db-name large \ diff --git a/tools/workload/statistics.go b/tools/workload/statistics.go index 2835af3949..815727ed08 100644 --- a/tools/workload/statistics.go +++ b/tools/workload/statistics.go @@ -27,12 +27,18 @@ type statistics struct { queryCount uint64 flushedRowCount uint64 errCount uint64 + ddlExecuted uint64 + ddlSucceeded uint64 + ddlSkipped uint64 + ddlFailed uint64 // QPS qps int // row/s rps int // error/s eps int + // ddl/s + ddls int } // calculateStats calculates the statistics @@ -40,33 +46,45 @@ func (app *WorkloadApp) calculateStats( lastQueryCount, lastFlushed, lastErrors uint64, + lastDDLExecuted uint64, reportInterval time.Duration, ) statistics { currentFlushed := app.Stats.FlushedRowCount.Load() currentErrors := app.Stats.ErrorCount.Load() currentQueryCount := app.Stats.QueryCount.Load() + currentDDLExecuted := app.Stats.DDLExecuted.Load() return statistics{ queryCount: currentQueryCount, flushedRowCount: currentFlushed, errCount: currentErrors, + ddlExecuted: currentDDLExecuted, + ddlSucceeded: app.Stats.DDLSucceeded.Load(), + ddlSkipped: app.Stats.DDLSkipped.Load(), + ddlFailed: app.Stats.DDLFailed.Load(), qps: int(currentQueryCount-lastQueryCount) / int(reportInterval.Seconds()), rps: int(currentFlushed-lastFlushed) / int(reportInterval.Seconds()), eps: int(currentErrors-lastErrors) / int(reportInterval.Seconds()), + ddls: int(currentDDLExecuted-lastDDLExecuted) / int(reportInterval.Seconds()), } } // printStats prints the statistics func (app *WorkloadApp) printStats(stats statistics) { status := fmt.Sprintf( - "Total Write Rows: %d, Total Queries: %d, Total Created Tables: %d, Total Errors: %d, QPS: %d, Row/s: %d, Error/s: %d", + "Total Write Rows: %d, Total Queries: %d, Total Created Tables: %d, Total Errors: %d, Total DDL Executed: %d, Total DDL Succeeded: %d, Total DDL Skipped: %d, Total DDL Failed: %d, QPS: %d, Row/s: %d, Error/s: %d, DDL/s: %d", stats.flushedRowCount, stats.queryCount, app.Stats.CreatedTableNum.Load(), stats.errCount, + stats.ddlExecuted, + stats.ddlSucceeded, + stats.ddlSkipped, + stats.ddlFailed, stats.qps, stats.rps, stats.eps, + stats.ddls, ) plog.Info(status) } @@ -85,14 +103,16 @@ func (app *WorkloadApp) reportMetrics() { lastQueryCount uint64 lastFlushed uint64 lastErrorCount uint64 + lastDDLCount uint64 ) for range ticker.C { - stats := app.calculateStats(lastQueryCount, lastFlushed, lastErrorCount, reportInterval) + stats := app.calculateStats(lastQueryCount, lastFlushed, lastErrorCount, lastDDLCount, reportInterval) // Update last values for next iteration lastQueryCount = stats.queryCount lastFlushed = stats.flushedRowCount lastErrorCount = stats.errCount + lastDDLCount = stats.ddlExecuted // Print statistics app.printStats(stats) diff --git a/utils/dynstream/memory_control.go b/utils/dynstream/memory_control.go index 8a2e7c1d80..170ec91ce3 100644 --- a/utils/dynstream/memory_control.go +++ b/utils/dynstream/memory_control.go @@ -33,7 +33,7 @@ const ( // For now, we only use it in event collector. MemoryControlForEventCollector = 1 - defaultReleaseMemoryRatio = 0.4 + defaultReleaseMemoryRatio = 0.6 defaultDeadlockDuration = 5 * time.Second defaultReleaseMemoryThreshold = 256 ) @@ -171,7 +171,7 @@ func (as *areaMemStat[A, P, T, D, H]) checkDeadlock() bool { hasEventComeButNotOut := time.Since(as.lastAppendEventTime.Load().(time.Time)) < defaultDeadlockDuration && time.Since(as.lastSizeDecreaseTime.Load().(time.Time)) > defaultDeadlockDuration - memoryHighWaterMark := as.memoryUsageRatio() > (1 - defaultReleaseMemoryRatio) + memoryHighWaterMark := as.memoryUsageRatio() > defaultReleaseMemoryRatio return hasEventComeButNotOut && memoryHighWaterMark } diff --git a/utils/dynstream/memory_control_test.go b/utils/dynstream/memory_control_test.go index 508f1c405e..15657565b6 100644 --- a/utils/dynstream/memory_control_test.go +++ b/utils/dynstream/memory_control_test.go @@ -320,6 +320,25 @@ func TestReleaseMemory(t *testing.T) { } feedbackChan := make(chan Feedback[int, string, any], 10) + calcExpectedReleasedPaths := func( + as *areaMemStat[int, string, *mockEvent, any, *mockHandler], + paths ...*pathInfo[int, string, *mockEvent, any, *mockHandler], + ) []string { + sizeToRelease := int64(float64(as.totalPendingSize.Load()) * defaultReleaseMemoryRatio) + releasedSize := int64(0) + res := make([]string, 0) + for _, path := range paths { + if releasedSize >= sizeToRelease || + path.pendingSize.Load() < int64(defaultReleaseMemoryThreshold) || + !path.blocking.Load() { + continue + } + releasedSize += path.pendingSize.Load() + res = append(res, path.path) + } + return res + } + // Create 3 paths with different last handle event timestamps path1 := &pathInfo[int, string, *mockEvent, any, *mockHandler]{ area: area, @@ -358,7 +377,7 @@ func TestReleaseMemory(t *testing.T) { path2.lastHandleEventTs.Store(200) path3.lastHandleEventTs.Store(100) - // Case 1: release path1 + // Case 1: release most recent paths // Add events to each path // Each event has size 100 for i := 0; i < 4; i++ { @@ -403,20 +422,27 @@ func TestReleaseMemory(t *testing.T) { path1.areaMemStat.lastReleaseMemoryTime.Store(time.Now().Add(-2 * time.Second)) path1.areaMemStat.releaseMemory() + expectedPaths := calcExpectedReleasedPaths(path1.areaMemStat, path1, path2, path3) feedbacks := make([]Feedback[int, string, any], 0) - for i := 0; i < 1; i++ { + for i := 0; i < len(expectedPaths); i++ { select { case fb := <-feedbackChan: feedbacks = append(feedbacks, fb) case <-time.After(100 * time.Millisecond): - require.Fail(t, "should receive 1 feedbacks") + require.Fail(t, "should receive feedbacks") } } - require.Equal(t, 1, len(feedbacks)) - require.Equal(t, ReleasePath, feedbacks[0].FeedbackType) - require.Equal(t, area, feedbacks[0].Area) - require.Equal(t, path1.path, feedbacks[0].Path) + require.Len(t, feedbacks, len(expectedPaths)) + gotPaths := make(map[string]bool, len(feedbacks)) + for _, fb := range feedbacks { + require.Equal(t, ReleasePath, fb.FeedbackType) + require.Equal(t, area, fb.Area) + gotPaths[fb.Path] = true + } + for _, path := range expectedPaths { + require.True(t, gotPaths[path]) + } // Case 2: release path1 and path2 // Reset the paths @@ -455,40 +481,39 @@ func TestReleaseMemory(t *testing.T) { path1.areaMemStat.totalPendingSize.Store(900) // Call releaseMemory - // sizeToRelease = 1000 * 0.4 = 360 - // path1 (ts=300): release 300 bytes, sizeToRelease = 360 - 300 = 60 - // path2 (ts=200): release 300 bytes, sizeToRelease = 60 - 300 = -240 + // sizeToRelease = totalPendingSize * defaultReleaseMemoryRatio path1.areaMemStat.lastReleaseMemoryTime.Store(time.Now().Add(-2 * time.Second)) path1.areaMemStat.releaseMemory() // Verify feedback messages - // Should receive 2 ResetPath feedbacks + // Should receive feedbacks to release memory. + expectedPaths = calcExpectedReleasedPaths(path1.areaMemStat, path1, path2, path3) feedbacks = make([]Feedback[int, string, any], 0) timer := time.After(100 * time.Millisecond) - for i := 0; i < 2; i++ { + for i := 0; i < len(expectedPaths); i++ { select { case fb := <-feedbackChan: feedbacks = append(feedbacks, fb) case <-timer: - require.Fail(t, "should receive 2 feedbacks") + require.Fail(t, "should receive feedbacks") } } - require.Equal(t, 2, len(feedbacks)) - // Both should be ResetPath type + require.Len(t, feedbacks, len(expectedPaths)) + // Both should be ReleasePath type for _, fb := range feedbacks { require.Equal(t, ReleasePath, fb.FeedbackType) require.Equal(t, area, fb.Area) } - // Check that we got feedbacks for path1 and path2 - paths := make(map[string]bool) + // Check that we got expected feedbacks. + paths := make(map[string]bool, len(feedbacks)) for _, fb := range feedbacks { paths[fb.Path] = true } - require.True(t, paths["path-1"]) - require.True(t, paths["path-2"]) - require.False(t, paths["path-3"]) + for _, path := range expectedPaths { + require.True(t, paths[path]) + } // Verify no more feedbacks select { From ba4807188c3c5480efa5f49b3b2ef05635a3917c Mon Sep 17 00:00:00 2001 From: dongmen <20351731+asddongmen@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:55:59 +0800 Subject: [PATCH 2/4] eventservice: fix changefeed getting stuck (#4460) close pingcap/ticdc#4365 --- .../dispatcher/basic_dispatcher.go | 4 +- pkg/eventservice/event_broker.go | 27 +++++ pkg/eventservice/event_broker_test.go | 100 ++++++++++++++++++ 3 files changed, 129 insertions(+), 2 deletions(-) diff --git a/downstreamadapter/dispatcher/basic_dispatcher.go b/downstreamadapter/dispatcher/basic_dispatcher.go index d70efaed03..0044f9ae13 100644 --- a/downstreamadapter/dispatcher/basic_dispatcher.go +++ b/downstreamadapter/dispatcher/basic_dispatcher.go @@ -526,8 +526,8 @@ func (d *BasicDispatcher) handleEvents(dispatcherEvents []DispatcherEvent, wakeC log.Info("dispatcher receive ddl event", zap.Stringer("dispatcher", d.id), zap.String("query", ddl.Query), - zap.Any("tableSpan", d.GetTableSpan()), - zap.Int64("table", ddl.GetTableID()), + zap.Int64("oldTableID", d.tableSpan.GetTableID()), + zap.Int64("currentTableID", ddl.GetTableID()), zap.Uint64("commitTs", event.GetCommitTs()), zap.Uint64("seq", event.GetSeq())) now := time.Now() diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index b3bc597d8e..f85162a76c 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -430,6 +430,10 @@ func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRang } dataRange.CommitTsEnd = min(dataRange.CommitTsEnd, ddlState.ResolvedTs) commitTsEndBeforeWindow := dataRange.CommitTsEnd + // If the latest ddl commit ts is in current resolved range and larger than current scan start, + // this dispatcher still has pending ddl to catch up. + hasPendingDDLEventInCurrentRange := dataRange.CommitTsStart < ddlState.MaxEventCommitTs && + ddlState.MaxEventCommitTs <= commitTsEndBeforeWindow scanMaxTs := task.changefeedStat.getScanMaxTs() if scanMaxTs > 0 { dataRange.CommitTsEnd = min(dataRange.CommitTsEnd, scanMaxTs) @@ -446,6 +450,28 @@ func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRang } } + if dataRange.CommitTsEnd <= dataRange.CommitTsStart && hasPendingDDLEventInCurrentRange { + // Global scan window base can be pinned by other lagging dispatchers. + // For a table with pending ddl in current range, use a local bounded step to keep + // this dispatcher making forward progress, so barrier coverage can eventually complete. + interval := time.Duration(task.changefeedStat.scanInterval.Load()) + if interval <= 0 { + interval = defaultScanInterval + } + localScanMaxTs := oracle.GoTimeToTS(oracle.GetTimeFromTS(dataRange.CommitTsStart).Add(interval)) + dataRange.CommitTsEnd = min(commitTsEndBeforeWindow, localScanMaxTs) + if dataRange.CommitTsEnd > dataRange.CommitTsStart { + log.Info("scan window local advance due to pending ddl", + zap.Stringer("changefeedID", task.changefeedStat.changefeedID), + zap.Stringer("dispatcherID", task.id), + zap.Uint64("startTs", dataRange.CommitTsStart), + zap.Uint64("globalScanMaxTs", scanMaxTs), + zap.Uint64("localScanMaxTs", localScanMaxTs), + zap.Uint64("ddlCommitTs", ddlState.MaxEventCommitTs), + zap.Uint64("newEndTs", dataRange.CommitTsEnd)) + } + } + if dataRange.CommitTsEnd <= dataRange.CommitTsStart { updateMetricEventServiceSkipResolvedTsCount(task.info.GetMode()) // Scan range can become empty after applying capping (for example, scan window). @@ -686,6 +712,7 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { } if err != nil { + log.Error("scan events failed", zap.Stringer("changefeedID", task.changefeedStat.changefeedID), zap.Stringer("dispatcherID", task.id), zap.Int64("tableID", task.info.GetTableSpan().GetTableID()), diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index 89e074267d..b6233f40a5 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -262,6 +262,106 @@ func TestGetScanTaskDataRangeEmptyAfterCappingDoesNotResetScanRange(t *testing.T require.Equal(t, lastStartTs, disp.lastScannedStartTs.Load()) } +func TestGetScanTaskDataRangeEmptyAfterCappingWithPendingDDLEventUsesLocalWindow(t *testing.T) { + broker, _, ss, _ := newEventBrokerForTest() + // Close the broker, so we can catch all message in the test. + broker.close() + + info := newMockDispatcherInfoForTest(t) + info.epoch = 1 + changefeedStatus := broker.getOrSetChangefeedStatus(info) + + disp := newDispatcherStat(info, 1, 1, nil, changefeedStatus) + disp.seq.Store(1) + + baseTime := time.Now() + baseTs := oracle.GoTimeToTS(baseTime) + commitStart := oracle.GoTimeToTS(baseTime.Add(20 * time.Second)) + ddlCommitTs := oracle.GoTimeToTS(baseTime.Add(23 * time.Second)) + resolvedTs := oracle.GoTimeToTS(baseTime.Add(40 * time.Second)) + + disp.sentResolvedTs.Store(baseTs) + disp.receivedResolvedTs.Store(resolvedTs) + disp.eventStoreCommitTs.Store(commitStart) + disp.lastScannedCommitTs.Store(commitStart) + disp.lastScannedStartTs.Store(commitStart - 1) + + changefeedStatus.minSentTs.Store(baseTs) + changefeedStatus.scanInterval.Store(int64(defaultScanInterval)) + + ss.resolvedTs = resolvedTs + ss.maxDDLCommitTs = ddlCommitTs + + needScan, dataRange := broker.getScanTaskDataRange(disp) + require.True(t, needScan) + require.Equal(t, commitStart, dataRange.CommitTsStart) + require.Equal(t, oracle.GoTimeToTS(oracle.GetTimeFromTS(commitStart).Add(defaultScanInterval)), dataRange.CommitTsEnd) +} + +func TestGetScanTaskDataRangeRingWaitWithThreeDispatchersCanAdvancePendingDDL(t *testing.T) { + broker, _, ss, _ := newEventBrokerForTest() + // Close the broker, so we can catch all message in the test. + broker.close() + + changefeedID := common.NewChangefeedID4Test("default", "test") + changefeedStatus := addChangefeedStatusToBrokerForTest(t, broker, changefeedID, 0) + changefeedStatus.scanInterval.Store(int64(1 * time.Second)) + + baseTime := time.Now() + ts100 := oracle.GoTimeToTS(baseTime) + ts101 := oracle.GoTimeToTS(baseTime.Add(1 * time.Second)) + ts102 := oracle.GoTimeToTS(baseTime.Add(2 * time.Second)) + ts103 := oracle.GoTimeToTS(baseTime.Add(3 * time.Second)) + ts110 := oracle.GoTimeToTS(baseTime.Add(10 * time.Second)) + + newDispatcher := func(tableID int64, sentTs uint64) *dispatcherStat { + info := newMockDispatcherInfo(t, ts100, common.NewDispatcherID(), tableID, eventpb.ActionType_ACTION_TYPE_REGISTER) + info.epoch = 1 + disp := newDispatcherStat(info, 1, 1, nil, changefeedStatus) + disp.seq.Store(1) + disp.sentResolvedTs.Store(sentTs) + disp.lastReceivedHeartbeatTime.Store(time.Now().Unix()) + + dispPtr := &atomic.Pointer[dispatcherStat]{} + dispPtr.Store(disp) + changefeedStatus.addDispatcher(disp.id, dispPtr) + return disp + } + + // D0(table trigger) and D2(other table) form the same changefeed. + // D0 lags at ts100, so global scan window base is pinned at ts100. + _ = newDispatcher(common.DDLSpanTableID, ts100) + // D1 is the blocked table waiting to cross a truncate ddl barrier at ts103. + d1 := newDispatcher(1313112, ts101) + _ = newDispatcher(1313999, ts110) + + changefeedStatus.refreshMinSentResolvedTs() + require.Equal(t, ts100, changefeedStatus.minSentTs.Load()) + + d1.receivedResolvedTs.Store(ts110) + d1.eventStoreCommitTs.Store(ts103) + d1.lastScannedCommitTs.Store(ts101) + d1.lastScannedStartTs.Store(ts101 - 1) + + ss.resolvedTs = ts110 + ss.maxDDLCommitTs = ts103 + + // Round 1: global cap makes range empty (end=ts101), fallback should locally move it to ts102. + needScan, dataRange := broker.getScanTaskDataRange(d1) + require.True(t, needScan) + require.Equal(t, ts101, dataRange.CommitTsStart) + require.Equal(t, ts102, dataRange.CommitTsEnd) + + // Round 2: still globally capped by ts100, but fallback should continue moving to ts103, + // which allows this dispatcher to eventually reach the pending truncate ddl barrier. + d1.lastScannedCommitTs.Store(ts102) + d1.lastScannedStartTs.Store(0) + needScan, dataRange = broker.getScanTaskDataRange(d1) + require.True(t, needScan) + require.Equal(t, ts102, dataRange.CommitTsStart) + require.Equal(t, ts103, dataRange.CommitTsEnd) +} + func TestHandleCongestionControlV2AdjustsScanInterval(t *testing.T) { broker, _, _, _ := newEventBrokerForTest() defer broker.close() From da7abf286ddc8de02af4ae7f7ca8f4155f0cb883 Mon Sep 17 00:00:00 2001 From: dongmen <20351731+asddongmen@users.noreply.github.com> Date: Mon, 18 May 2026 00:23:14 +0800 Subject: [PATCH 3/4] eventservice: optimize scanwindow (#4950) close pingcap/ticdc#5041 --- pkg/eventservice/dispatcher_stat.go | 17 +- pkg/eventservice/event_broker.go | 7 +- pkg/eventservice/event_broker_test.go | 22 +- pkg/eventservice/scan_window.go | 714 +++++++++++++++++++++----- pkg/eventservice/scan_window_test.go | 303 +++++++++-- pkg/metrics/event_service.go | 56 ++ 6 files changed, 930 insertions(+), 189 deletions(-) diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 552a684ea2..4fbf45f9a6 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -430,22 +430,21 @@ type changefeedStatus struct { availableMemoryQuota sync.Map // nodeID -> atomic.Uint64 (memory quota in bytes) minSentTs atomic.Uint64 scanInterval atomic.Int64 + reportBandState atomic.Int32 + fastBandState atomic.Int32 + slowBandState atomic.Int32 - lastAdjustTime atomic.Time - lastTrendAdjustTime atomic.Time - usageWindow *memoryUsageWindow - syncPointInterval time.Duration + scanWindowController *adaptiveScanWindowController + syncPointInterval time.Duration } func newChangefeedStatus(changefeedID common.ChangeFeedID, syncPointInterval time.Duration) *changefeedStatus { status := &changefeedStatus{ - changefeedID: changefeedID, - usageWindow: newMemoryUsageWindow(memoryUsageWindowDuration), - syncPointInterval: syncPointInterval, + changefeedID: changefeedID, + scanWindowController: newAdaptiveScanWindowController(time.Now()), + syncPointInterval: syncPointInterval, } status.scanInterval.Store(int64(defaultScanInterval)) - status.lastAdjustTime.Store(time.Now()) - status.lastTrendAdjustTime.Store(time.Now()) return status } diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index f85162a76c..dff6ff6085 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -1142,9 +1142,7 @@ func (c *eventBroker) removeChangefeedStatus(status *changefeedStatus) { } filter.GetSharedFilterStorage().RemoveFilter(changefeedID) - metrics.EventServiceAvailableMemoryQuotaGaugeVec.DeleteLabelValues(changefeedID.String()) - metrics.EventServiceScanWindowBaseTsGaugeVec.DeleteLabelValues(changefeedID.String()) - metrics.EventServiceScanWindowIntervalGaugeVec.DeleteLabelValues(changefeedID.String()) + deleteScanWindowMetrics(changefeedID.String()) } func (c *eventBroker) resetDispatcher(dispatcherInfo DispatcherInfo) error { @@ -1263,8 +1261,7 @@ func (c *eventBroker) getOrSetChangefeedStatus(info DispatcherInfo) *changefeedS return actual.(*changefeedStatus) } log.Info("new changefeed status", zap.Stringer("changefeedID", changefeedID)) - metrics.EventServiceScanWindowBaseTsGaugeVec.WithLabelValues(changefeedID.String()).Set(0) - metrics.EventServiceScanWindowIntervalGaugeVec.WithLabelValues(changefeedID.String()).Set(defaultScanInterval.Seconds()) + initializeScanWindowMetrics(changefeedID.String()) return status } diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index b6233f40a5..bc934398ea 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -362,24 +362,7 @@ func TestGetScanTaskDataRangeRingWaitWithThreeDispatchersCanAdvancePendingDDL(t require.Equal(t, ts103, dataRange.CommitTsEnd) } -func TestHandleCongestionControlV2AdjustsScanInterval(t *testing.T) { - broker, _, _, _ := newEventBrokerForTest() - defer broker.close() - - changefeedID := common.NewChangefeedID4Test("default", "test") - status := addChangefeedStatusToBrokerForTest(t, broker, changefeedID, time.Second*10) - - status.scanInterval.Store(int64(40 * time.Second)) - status.lastAdjustTime.Store(time.Now()) - - control := event.NewCongestionControlWithVersion(event.CongestionControlVersion2) - control.AddAvailableMemoryWithDispatchersAndUsage(changefeedID.ID(), 0, 1, nil) - broker.handleCongestionControl(node.ID("event-collector-1"), control) - - require.Equal(t, int64(10*time.Second), status.scanInterval.Load()) -} - -func TestHandleCongestionControlV2ResetsScanIntervalOnMemoryRelease(t *testing.T) { +func TestHandleCongestionControlV2DoesNotResetScanIntervalOnMemoryRelease(t *testing.T) { broker, _, _, _ := newEventBrokerForTest() defer broker.close() @@ -392,7 +375,7 @@ func TestHandleCongestionControlV2ResetsScanIntervalOnMemoryRelease(t *testing.T control.AddAvailableMemoryWithDispatchersAndUsageAndReleaseCount(changefeedID.ID(), 0, 0.5, nil, 1) broker.handleCongestionControl(node.ID("event-collector-1"), control) - require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) + require.Equal(t, int64(40*time.Second), status.scanInterval.Load()) } func TestHandleCongestionControlV1DoesNotAdjustScanInterval(t *testing.T) { @@ -403,7 +386,6 @@ func TestHandleCongestionControlV1DoesNotAdjustScanInterval(t *testing.T) { status := addChangefeedStatusToBrokerForTest(t, broker, changefeedID, time.Second*10) status.scanInterval.Store(int64(40 * time.Second)) - status.lastAdjustTime.Store(time.Now()) control := event.NewCongestionControl() control.AddAvailableMemoryWithDispatchers(changefeedID.ID(), 0, nil) diff --git a/pkg/eventservice/scan_window.go b/pkg/eventservice/scan_window.go index 46cb9cffb3..5bcb8e16c9 100644 --- a/pkg/eventservice/scan_window.go +++ b/pkg/eventservice/scan_window.go @@ -43,11 +43,6 @@ const ( // this cooldown and are applied immediately. scanIntervalAdjustCooldown = 30 * time.Second - // scanTrendAdjustCooldown is the minimum time between trend-based interval - // adjustments. This is shorter than the general cooldown because trend - // adjustments need to be more responsive to rising memory pressure. - scanTrendAdjustCooldown = 5 * time.Second - // memoryUsageWindowDuration is the duration of the sliding window for // collecting memory usage samples. Samples older than this duration are // pruned from the window. @@ -58,19 +53,94 @@ const ( memoryUsageHighThreshold = 0.7 // memoryUsageCriticalThreshold (90%) triggers an aggressive reduction of - // the scan interval to 1/4 of its current value when memory usage exceeds - // this level. + // the scan interval once memory usage exceeds this level. memoryUsageCriticalThreshold = 0.9 + // memoryUsageEmergencyThreshold (98%) triggers the strongest emergency brake. + memoryUsageEmergencyThreshold = 0.98 + // memoryUsageLowThreshold (20%) allows the scan interval to be increased // by 25% when both max and average memory usage are below this level. memoryUsageLowThreshold = 0.2 + // scanWindowLowPressureFastEMAOffset widens the fast EMA threshold slightly + // for low-pressure recovery decisions. + scanWindowLowPressureFastEMAOffset = 0.03 + + // scanWindowLowPressureSlowEMAOffset widens the slow EMA threshold slightly + // for low-pressure recovery decisions. + scanWindowLowPressureSlowEMAOffset = 0.02 + // memoryUsageVeryLowThreshold (10%) allows the scan interval to be increased // by 50% when both max and average memory usage are below this level. This // increase may exceed the normal sync point interval cap. memoryUsageVeryLowThreshold = 0.1 + // scanWindowModeratePressureThreshold is the smoothed usage threshold that + // starts accumulating pressure score for gradual interval reductions. + scanWindowModeratePressureThreshold = 0.55 + + // scanWindowHighPressureThreshold triggers a stronger but still bounded + // interval reduction when sustained high pressure is observed. + scanWindowHighPressureThreshold = 0.75 + + // scanWindowPressureAdjustCooldown is the minimum time between non-critical + // downward adjustments. It prevents the controller from overreacting before + // previous interval changes have time to take effect. + scanWindowPressureAdjustCooldown = 10 * time.Second + + // scanWindowCriticalBrakeCooldown deduplicates repeated critical brakes + // caused by the same short burst. Without this cooldown, one peak retained + // in the usage window can repeatedly trigger critical_brake on every report. + scanWindowCriticalBrakeCooldown = 10 * time.Second + + // scanWindowReleaseRecoveryCooldown is the minimum time after a downward + // adjustment before the controller is allowed to recover upward again. + scanWindowReleaseRecoveryCooldown = 15 * time.Second + + // scanWindowVeryLowRecoveryCooldown is the minimum time after a recent + // instability event before the controller can re-enter the aggressive + // very_low_recovery path. + scanWindowVeryLowRecoveryCooldown = 90 * time.Second + + // scanWindowFloorRecoveryCooldown allows the controller to escape from the + // default floor faster once the observed pressure has clearly fallen. This + // specifically mitigates feedback-lag cases where a late critical report + // pushes the interval to the floor after the real pressure has already eased. + scanWindowFloorRecoveryCooldown = 5 * time.Second + + // scanWindowEmergencyBrakePlateauInterval keeps the emergency brake + // continuous when transitioning from the small-window moderate brake path to + // the large-window strong brake path. + scanWindowEmergencyBrakePlateauInterval = 3 * defaultScanInterval + + // scanWindowEmergencyMinIntervalUnlockSamples is the minimum number of + // observed samples before emergency pressure is allowed to drive the scan + // window below the default floor toward the minimum interval. + scanWindowEmergencyMinIntervalUnlockSamples = 3 + + // scanWindowFastUsageAlpha controls the responsiveness of the short-term EMA. + scanWindowFastUsageAlpha = 0.4 + + // scanWindowSlowUsageAlpha controls the responsiveness of the long-term EMA. + scanWindowSlowUsageAlpha = 0.2 + + // scanWindowPressureTriggerScore is the score required to trigger a gradual + // downward adjustment under sustained but non-critical pressure. + scanWindowPressureTriggerScore = 3.0 + + // scanWindowPressureScoreCeiling bounds the pressure accumulator. + scanWindowPressureScoreCeiling = 8.0 + + // scanWindowPressureReliefPerRelease is the amount of accumulated pressure + // cleared by one downstream release pulse. + scanWindowPressureReliefPerRelease = 2.0 + + // scanWindowTargetBandLower and scanWindowTargetBandUpper define the desired + // operating region for observed pressure related signals. + scanWindowTargetBandLower = 0.30 + scanWindowTargetBandUpper = 0.50 + // scanWindowStaleDispatcherHeartbeatThreshold is the duration after which a // dispatcher is treated as stale for scan window base ts calculation if it // hasn't sent heartbeat updates. This prevents stale dispatchers (for example, @@ -102,12 +172,72 @@ type memoryUsageStats struct { cnt int } +type scanWindowReport struct { + usageRatio float64 + memoryReleaseCount uint32 +} + +type scanWindowDecisionReason string + +const ( + scanWindowDecisionNone scanWindowDecisionReason = "none" + scanWindowDecisionCriticalBrake scanWindowDecisionReason = "critical_brake" + scanWindowDecisionHighPressure scanWindowDecisionReason = "high_pressure" + scanWindowDecisionSustainedPressure scanWindowDecisionReason = "sustained_pressure" + scanWindowDecisionLowRecovery scanWindowDecisionReason = "low_recovery" + scanWindowDecisionVeryLowRecovery scanWindowDecisionReason = "very_low_recovery" +) + +type scanWindowDecision struct { + newInterval time.Duration + maxInterval time.Duration + reason scanWindowDecisionReason + usage memoryUsageStats + fastUsageEMA float64 + slowUsageEMA float64 + pressureScore float64 +} + +type scanWindowBandState int32 + +const ( + scanWindowBandUnknown scanWindowBandState = iota + scanWindowBandBelow + scanWindowBandIn + scanWindowBandAbove +) + +type adaptiveScanWindowController struct { + mu sync.Mutex + + usageWindow *memoryUsageWindow + + lastAdjustTime time.Time + lastDownAdjustTime time.Time + lastCriticalTime time.Time + lastInstabilityTime time.Time + + fastUsageEMA float64 + slowUsageEMA float64 + emaInitialized bool + + pressureScore float64 +} + func newMemoryUsageWindow(window time.Duration) *memoryUsageWindow { return &memoryUsageWindow{ window: window, } } +func newAdaptiveScanWindowController(now time.Time) *adaptiveScanWindowController { + return &adaptiveScanWindowController{ + usageWindow: newMemoryUsageWindow(memoryUsageWindowDuration), + lastAdjustTime: now, + lastDownAdjustTime: now, + } +} + func (w *memoryUsageWindow) addSample(now time.Time, ratio float64) { if ratio < 0 { ratio = 0 @@ -166,150 +296,502 @@ func (w *memoryUsageWindow) pruneLocked(now time.Time) { } func (c *changefeedStatus) updateMemoryUsage(now time.Time, usageRatio float64, memoryReleaseCount uint32) { - if c.usageWindow == nil { + if c.scanWindowController == nil { return } - if usageRatio != usageRatio || usageRatio < 0 { - usageRatio = 0 + normalizedUsageRatio := normalizeUsageRatio(usageRatio) + current := time.Duration(c.scanInterval.Load()) + decision := c.scanWindowController.OnCongestionReport(now, current, c.maxScanInterval(), scanWindowReport{ + usageRatio: normalizedUsageRatio, + memoryReleaseCount: memoryReleaseCount, + }) + c.observeScanWindowControllerMetrics(normalizedUsageRatio, memoryReleaseCount, current, decision) + if decision.newInterval == current { + return } - if usageRatio > 1 { - usageRatio = 1 + + c.scanInterval.Store(int64(decision.newInterval)) + metrics.EventServiceScanWindowIntervalGaugeVec.WithLabelValues(c.changefeedID.String()).Set(decision.newInterval.Seconds()) + + log.Info("scan interval adjusted", + zap.Stringer("changefeedID", c.changefeedID), + zap.String("reason", string(decision.reason)), + zap.Duration("oldInterval", current), + zap.Duration("newInterval", decision.newInterval), + zap.Duration("maxInterval", decision.maxInterval), + zap.Float64("avgUsage", decision.usage.avg), + zap.Float64("maxUsage", decision.usage.max), + zap.Float64("firstUsage", decision.usage.first), + zap.Float64("lastUsage", decision.usage.last), + zap.Float64("fastUsageEMA", decision.fastUsageEMA), + zap.Float64("slowUsageEMA", decision.slowUsageEMA), + zap.Float64("pressureScore", decision.pressureScore), + zap.Uint32("memoryReleaseCount", memoryReleaseCount), + zap.Bool("syncPointEnabled", c.isSyncpointEnabled()), + zap.Duration("syncPointInterval", c.syncPointInterval)) +} + +func initializeScanWindowMetrics(changefeed string) { + metrics.EventServiceScanWindowBaseTsGaugeVec.WithLabelValues(changefeed).Set(0) + metrics.EventServiceScanWindowIntervalGaugeVec.WithLabelValues(changefeed).Set(defaultScanInterval.Seconds()) + metrics.EventServiceScanWindowUsageRatioGaugeVec.WithLabelValues(changefeed, "report").Set(0) + metrics.EventServiceScanWindowUsageRatioGaugeVec.WithLabelValues(changefeed, "avg").Set(0) + metrics.EventServiceScanWindowUsageRatioGaugeVec.WithLabelValues(changefeed, "max").Set(0) + metrics.EventServiceScanWindowUsageEMAGaugeVec.WithLabelValues(changefeed, "fast").Set(0) + metrics.EventServiceScanWindowUsageEMAGaugeVec.WithLabelValues(changefeed, "slow").Set(0) + metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, "report").Set(0) + metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, "fast").Set(0) + metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, "slow").Set(0) + metrics.EventServiceScanWindowPressureScoreGaugeVec.WithLabelValues(changefeed).Set(0) +} + +func deleteScanWindowMetrics(changefeed string) { + // Available memory quota is a changefeed-lifecycle metric that is cleaned + // together with scan window metrics when the changefeed status is removed. + metrics.EventServiceAvailableMemoryQuotaGaugeVec.DeleteLabelValues(changefeed) + metrics.EventServiceScanWindowBaseTsGaugeVec.DeleteLabelValues(changefeed) + metrics.EventServiceScanWindowIntervalGaugeVec.DeleteLabelValues(changefeed) + metrics.EventServiceScanWindowUsageRatioGaugeVec.DeleteLabelValues(changefeed, "report") + metrics.EventServiceScanWindowUsageRatioGaugeVec.DeleteLabelValues(changefeed, "avg") + metrics.EventServiceScanWindowUsageRatioGaugeVec.DeleteLabelValues(changefeed, "max") + metrics.EventServiceScanWindowUsageEMAGaugeVec.DeleteLabelValues(changefeed, "fast") + metrics.EventServiceScanWindowUsageEMAGaugeVec.DeleteLabelValues(changefeed, "slow") + metrics.EventServiceScanWindowTargetBandGaugeVec.DeleteLabelValues(changefeed, "report") + metrics.EventServiceScanWindowTargetBandGaugeVec.DeleteLabelValues(changefeed, "fast") + metrics.EventServiceScanWindowTargetBandGaugeVec.DeleteLabelValues(changefeed, "slow") + metrics.EventServiceScanWindowTargetBandCrossCount.DeleteLabelValues(changefeed, "report") + metrics.EventServiceScanWindowTargetBandCrossCount.DeleteLabelValues(changefeed, "fast") + metrics.EventServiceScanWindowTargetBandCrossCount.DeleteLabelValues(changefeed, "slow") + metrics.EventServiceScanWindowPressureScoreGaugeVec.DeleteLabelValues(changefeed) + metrics.EventServiceScanWindowMemoryReleaseCount.DeleteLabelValues(changefeed) + for _, reason := range []scanWindowDecisionReason{ + scanWindowDecisionNone, + scanWindowDecisionCriticalBrake, + scanWindowDecisionHighPressure, + scanWindowDecisionSustainedPressure, + scanWindowDecisionLowRecovery, + scanWindowDecisionVeryLowRecovery, + } { + metrics.EventServiceScanWindowAdjustCount.DeleteLabelValues(changefeed, string(reason)) } +} +func (c *changefeedStatus) observeScanWindowControllerMetrics( + usageRatio float64, + memoryReleaseCount uint32, + current time.Duration, + decision scanWindowDecision, +) { + changefeed := c.changefeedID.String() + metrics.EventServiceScanWindowUsageRatioGaugeVec.WithLabelValues(changefeed, "report").Set(usageRatio) + metrics.EventServiceScanWindowUsageRatioGaugeVec.WithLabelValues(changefeed, "avg").Set(decision.usage.avg) + metrics.EventServiceScanWindowUsageRatioGaugeVec.WithLabelValues(changefeed, "max").Set(decision.usage.max) + metrics.EventServiceScanWindowUsageEMAGaugeVec.WithLabelValues(changefeed, "fast").Set(decision.fastUsageEMA) + metrics.EventServiceScanWindowUsageEMAGaugeVec.WithLabelValues(changefeed, "slow").Set(decision.slowUsageEMA) + c.observeScanWindowTargetBandMetrics(changefeed, "report", usageRatio, &c.reportBandState) + c.observeScanWindowTargetBandMetrics(changefeed, "fast", decision.fastUsageEMA, &c.fastBandState) + c.observeScanWindowTargetBandMetrics(changefeed, "slow", decision.slowUsageEMA, &c.slowBandState) + metrics.EventServiceScanWindowPressureScoreGaugeVec.WithLabelValues(changefeed).Set(decision.pressureScore) if memoryReleaseCount > 0 { - c.resetScanIntervalToDefault(now) - c.usageWindow.reset() - c.usageWindow.addSample(now, usageRatio) - return + metrics.EventServiceScanWindowMemoryReleaseCount.WithLabelValues(changefeed).Add(float64(memoryReleaseCount)) + } + if decision.newInterval != current { + metrics.EventServiceScanWindowAdjustCount.WithLabelValues(changefeed, string(decision.reason)).Inc() } - - c.usageWindow.addSample(now, usageRatio) - stats := c.usageWindow.stats(now) - c.adjustScanInterval(now, stats) } -func (c *changefeedStatus) resetScanIntervalToDefault(now time.Time) { - current := time.Duration(c.scanInterval.Load()) - if current != defaultScanInterval { - c.scanInterval.Store(int64(defaultScanInterval)) - metrics.EventServiceScanWindowIntervalGaugeVec.WithLabelValues(c.changefeedID.String()).Set(defaultScanInterval.Seconds()) +func (c *changefeedStatus) observeScanWindowTargetBandMetrics( + changefeed string, + metricType string, + value float64, + state *atomic.Int32, +) { + currentState := classifyScanWindowBandState(value) + if currentState == scanWindowBandIn { + metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, metricType).Set(1) + } else { + metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, metricType).Set(0) + } - log.Info("scan interval reset to default", - zap.Stringer("changefeedID", c.changefeedID), - zap.Duration("oldInterval", current), - zap.Duration("newInterval", defaultScanInterval)) + previousState := scanWindowBandState(state.Swap(int32(currentState))) + if previousState != scanWindowBandUnknown && previousState != currentState { + metrics.EventServiceScanWindowTargetBandCrossCount.WithLabelValues(changefeed, metricType).Inc() } +} - c.lastAdjustTime.Store(now) - c.lastTrendAdjustTime.Store(now) +func classifyScanWindowBandState(value float64) scanWindowBandState { + switch { + case value < scanWindowTargetBandLower: + return scanWindowBandBelow + case value > scanWindowTargetBandUpper: + return scanWindowBandAbove + default: + return scanWindowBandIn + } } -// Constants for trend detection and increase eligibility. const ( - minTrendSamples = 4 // Minimum samples needed to detect a valid trend - increasingTrendEpsilon = 0.02 // Minimum delta to consider as "increasing" - increasingTrendStartRatio = 0.3 // Threshold (30%) above which trend damping kicks in - minIncreaseSamples = 10 // Minimum samples needed before allowing increase minIncreaseSpanNumerator = 4 // Observation span must be at least 4/5 of window minIncreaseSpanDenominator = 5 ) -// adjustScanInterval dynamically adjusts the scan interval based on memory pressure. -// -// Algorithm overview: -// - "Fast brake, slow accelerate": Decreases are applied immediately when memory -// pressure is high, while increases require cooldown periods and stable conditions. -// - Tiered response: Different thresholds trigger different adjustment magnitudes. -// - Trend prediction: Detects rising memory pressure early and proactively reduces -// the interval before hitting critical thresholds. -// -// Thresholds and actions: -// - Critical (>90%): Reduce interval to 1/4 (aggressive) -// - High (>70%): Reduce interval to 1/2 -// - Trend damping (>30% AND rising): Reduce interval by 10% -// - Low (<30% max AND avg): Increase interval by 25% -// - Very low (<10% max AND avg): Increase interval by 50%, may exceed normal cap -func (c *changefeedStatus) adjustScanInterval(now time.Time, usage memoryUsageStats) { - current := time.Duration(c.scanInterval.Load()) +func (c *adaptiveScanWindowController) OnCongestionReport(now time.Time, current time.Duration, maxInterval time.Duration, report scanWindowReport) scanWindowDecision { + c.mu.Lock() + defer c.mu.Unlock() + if current <= 0 { current = defaultScanInterval } - maxInterval := c.maxScanInterval() if maxInterval < minScanInterval { maxInterval = minScanInterval } - // Trend detection: check if memory usage is rising over the observation window. - // This enables proactive intervention before hitting high thresholds. - trendDelta := usage.last - usage.first - isIncreasing := usage.cnt >= minTrendSamples && trendDelta > increasingTrendEpsilon - isAboveTrendStart := usage.last > increasingTrendStartRatio - canAdjustOnTrend := now.Sub(c.lastTrendAdjustTime.Load()) >= scanTrendAdjustCooldown - shouldDampOnTrend := isAboveTrendStart && isIncreasing && canAdjustOnTrend - - // Increase eligibility: conservative conditions to prevent oscillation. - // Requires: cooldown passed, enough samples, sufficient observation span, - // and NOT in an increasing trend situation (to avoid fighting against pressure). + + c.usageWindow.addSample(now, report.usageRatio) + usage := c.usageWindow.stats(now) + c.updateUsageEMALocked(report.usageRatio) + + if decision, ok := c.tryCriticalBrakeLocked(now, current, maxInterval, usage); ok { + return decision + } + + c.updatePressureScoreLocked(usage) + if report.memoryReleaseCount > 0 { + c.relievePressureLocked(report.memoryReleaseCount) + } + + if c.shouldReduceForHighPressureLocked(now, usage) { + newInterval := scanWindowPressureInterval(current, max(scaleDuration(current, 3, 4), defaultScanInterval)) + c.noteAdjustmentLocked(now, true) + return scanWindowDecision{ + newInterval: newInterval, + maxInterval: maxInterval, + reason: scanWindowDecisionHighPressure, + usage: usage, + fastUsageEMA: c.fastUsageEMA, + slowUsageEMA: c.slowUsageEMA, + pressureScore: c.pressureScore, + } + } + + if c.shouldReduceForSustainedPressureLocked(now, usage) { + newInterval := scanWindowPressureInterval(current, max(scaleDuration(current, 9, 10), defaultScanInterval)) + c.noteAdjustmentLocked(now, true) + return scanWindowDecision{ + newInterval: newInterval, + maxInterval: maxInterval, + reason: scanWindowDecisionSustainedPressure, + usage: usage, + fastUsageEMA: c.fastUsageEMA, + slowUsageEMA: c.slowUsageEMA, + pressureScore: c.pressureScore, + } + } + + if c.shouldRecoverFromFloorLocked(now, current, usage) { + newInterval := min(scaleDuration(current, 5, 4), maxInterval) + if newInterval > current { + c.noteAdjustmentLocked(now, false) + return scanWindowDecision{ + newInterval: newInterval, + maxInterval: maxInterval, + reason: scanWindowDecisionLowRecovery, + usage: usage, + fastUsageEMA: c.fastUsageEMA, + slowUsageEMA: c.slowUsageEMA, + pressureScore: c.pressureScore, + } + } + } + + if !c.allowedToIncreaseLocked(now, usage) { + return scanWindowDecision{ + newInterval: current, + maxInterval: maxInterval, + reason: scanWindowDecisionNone, + usage: usage, + fastUsageEMA: c.fastUsageEMA, + slowUsageEMA: c.slowUsageEMA, + pressureScore: c.pressureScore, + } + } + + if c.isVeryLowPressureLocked(usage) && c.allowedToVeryLowRecoverLocked(now) { + effectiveMaxInterval := maxScanInterval + numerator, denominator := scanWindowVeryLowRecoveryScale(current) + newInterval := min(scaleDuration(current, numerator, denominator), effectiveMaxInterval) + if newInterval > current { + c.noteAdjustmentLocked(now, false) + return scanWindowDecision{ + newInterval: newInterval, + maxInterval: effectiveMaxInterval, + reason: scanWindowDecisionVeryLowRecovery, + usage: usage, + fastUsageEMA: c.fastUsageEMA, + slowUsageEMA: c.slowUsageEMA, + pressureScore: c.pressureScore, + } + } + } + + if current < maxInterval && c.isLowPressureLocked(usage) { + numerator, denominator := scanWindowLowRecoveryScale(current) + newInterval := min(scaleDuration(current, numerator, denominator), maxInterval) + if newInterval > current { + c.noteAdjustmentLocked(now, false) + return scanWindowDecision{ + newInterval: newInterval, + maxInterval: maxInterval, + reason: scanWindowDecisionLowRecovery, + usage: usage, + fastUsageEMA: c.fastUsageEMA, + slowUsageEMA: c.slowUsageEMA, + pressureScore: c.pressureScore, + } + } + } + + return scanWindowDecision{ + newInterval: current, + maxInterval: maxInterval, + reason: scanWindowDecisionNone, + usage: usage, + fastUsageEMA: c.fastUsageEMA, + slowUsageEMA: c.slowUsageEMA, + pressureScore: c.pressureScore, + } +} + +func scanWindowVeryLowRecoveryScale(current time.Duration) (numerator int64, denominator int64) { + switch { + case current >= 120*time.Second: + return 11, 10 + case current >= 60*time.Second: + return 6, 5 + default: + return 3, 2 + } +} + +func scanWindowEmergencyBrakeInterval(current time.Duration, allowMinInterval bool) time.Duration { + if current <= defaultScanInterval && allowMinInterval { + return max(current/2, minScanInterval) + } + if current <= 6*defaultScanInterval { + return scanWindowPressureInterval(current, max(current/2, defaultScanInterval)) + } + return max(current/4, scanWindowEmergencyBrakePlateauInterval) +} + +func scanWindowPressureInterval(current time.Duration, next time.Duration) time.Duration { + return min(next, current) +} + +func scanWindowLowRecoveryScale(current time.Duration) (numerator int64, denominator int64) { + switch { + case current >= 120*time.Second: + return 21, 20 + case current >= 60*time.Second: + return 11, 10 + default: + return 5, 4 + } +} + +func (c *adaptiveScanWindowController) tryCriticalBrakeLocked( + now time.Time, + current time.Duration, + maxInterval time.Duration, + usage memoryUsageStats, +) (scanWindowDecision, bool) { + if now.Sub(c.lastCriticalTime) < scanWindowCriticalBrakeCooldown { + return scanWindowDecision{}, false + } + + switch { + case usage.last > memoryUsageEmergencyThreshold: + newInterval := scanWindowEmergencyBrakeInterval(current, c.shouldAllowEmergencyMinIntervalLocked(current, usage)) + c.lastCriticalTime = now + c.noteAdjustmentLocked(now, true) + return scanWindowDecision{ + newInterval: newInterval, + maxInterval: maxInterval, + reason: scanWindowDecisionCriticalBrake, + usage: usage, + fastUsageEMA: c.fastUsageEMA, + slowUsageEMA: c.slowUsageEMA, + pressureScore: c.pressureScore, + }, true + case usage.last > memoryUsageCriticalThreshold: + newInterval := scanWindowPressureInterval(current, max(current/2, defaultScanInterval)) + c.lastCriticalTime = now + c.noteAdjustmentLocked(now, true) + return scanWindowDecision{ + newInterval: newInterval, + maxInterval: maxInterval, + reason: scanWindowDecisionCriticalBrake, + usage: usage, + fastUsageEMA: c.fastUsageEMA, + slowUsageEMA: c.slowUsageEMA, + pressureScore: c.pressureScore, + }, true + default: + return scanWindowDecision{}, false + } +} + +func (c *adaptiveScanWindowController) shouldAllowEmergencyMinIntervalLocked( + current time.Duration, + usage memoryUsageStats, +) bool { + return current <= defaultScanInterval && + usage.cnt >= scanWindowEmergencyMinIntervalUnlockSamples && + c.fastUsageEMA >= memoryUsageCriticalThreshold +} + +func (c *adaptiveScanWindowController) shouldRecoverFromFloorLocked( + now time.Time, + current time.Duration, + usage memoryUsageStats, +) bool { + if current > defaultScanInterval { + return false + } + if now.Sub(c.lastAdjustTime) < scanWindowFloorRecoveryCooldown { + return false + } + if now.Sub(c.lastDownAdjustTime) < scanWindowFloorRecoveryCooldown { + return false + } + if usage.cnt < 3 { + return false + } + + return usage.last < 0.35 && + usage.avg < scanWindowModeratePressureThreshold && + c.fastUsageEMA < 0.45 && + c.slowUsageEMA < 0.40 && + c.pressureScore < 1.5 +} + +func (c *adaptiveScanWindowController) updateUsageEMALocked(value float64) { + if !c.emaInitialized { + c.fastUsageEMA = value + c.slowUsageEMA = value + c.emaInitialized = true + return + } + c.fastUsageEMA = ema(c.fastUsageEMA, value, scanWindowFastUsageAlpha) + c.slowUsageEMA = ema(c.slowUsageEMA, value, scanWindowSlowUsageAlpha) +} + +func (c *adaptiveScanWindowController) updatePressureScoreLocked(usage memoryUsageStats) { + switch { + case c.fastUsageEMA >= scanWindowHighPressureThreshold || + c.slowUsageEMA >= scanWindowHighPressureThreshold || + usage.last >= memoryUsageHighThreshold: + c.pressureScore = min(c.pressureScore+2, scanWindowPressureScoreCeiling) + case c.fastUsageEMA >= scanWindowModeratePressureThreshold || + c.slowUsageEMA >= scanWindowModeratePressureThreshold || + usage.avg >= scanWindowModeratePressureThreshold: + c.pressureScore = min(c.pressureScore+1, scanWindowPressureScoreCeiling) + case c.fastUsageEMA < 0.30 && c.slowUsageEMA < 0.25 && usage.last < 0.30: + c.pressureScore = max(0.0, c.pressureScore-1.5) + default: + c.pressureScore = max(0.0, c.pressureScore-0.5) + } +} + +func (c *adaptiveScanWindowController) relievePressureLocked(memoryReleaseCount uint32) { + relief := min(float64(memoryReleaseCount)*scanWindowPressureReliefPerRelease, scanWindowPressureScoreCeiling) + c.pressureScore = max(0.0, c.pressureScore-relief) +} + +func (c *adaptiveScanWindowController) shouldReduceForHighPressureLocked(now time.Time, usage memoryUsageStats) bool { + if now.Sub(c.lastDownAdjustTime) < scanWindowPressureAdjustCooldown { + return false + } + + return c.fastUsageEMA >= scanWindowHighPressureThreshold || + c.slowUsageEMA >= scanWindowHighPressureThreshold || + usage.last >= memoryUsageHighThreshold +} + +func (c *adaptiveScanWindowController) shouldReduceForSustainedPressureLocked(now time.Time, usage memoryUsageStats) bool { + if now.Sub(c.lastDownAdjustTime) < scanWindowPressureAdjustCooldown { + return false + } + if c.pressureScore < scanWindowPressureTriggerScore { + return false + } + return c.fastUsageEMA >= scanWindowModeratePressureThreshold || + c.slowUsageEMA >= scanWindowModeratePressureThreshold || + usage.avg >= scanWindowModeratePressureThreshold +} + +func (c *adaptiveScanWindowController) allowedToIncreaseLocked(now time.Time, usage memoryUsageStats) bool { minIncreaseSpan := memoryUsageWindowDuration * minIncreaseSpanNumerator / minIncreaseSpanDenominator - allowedToIncrease := now.Sub(c.lastAdjustTime.Load()) >= scanIntervalAdjustCooldown && + return now.Sub(c.lastAdjustTime) >= scanIntervalAdjustCooldown && + now.Sub(c.lastDownAdjustTime) >= scanWindowReleaseRecoveryCooldown && usage.cnt >= minIncreaseSamples && usage.span >= minIncreaseSpan && - !(isAboveTrendStart && isIncreasing) + c.pressureScore < 1 +} - // Determine the new interval based on memory pressure levels. - // Priority order: critical > high > trend damping > very low > low - adjustedOnTrend := false - newInterval := current - switch { - case usage.last > memoryUsageCriticalThreshold || usage.max > memoryUsageCriticalThreshold: - // Critical pressure: aggressive reduction to 1/4 - newInterval = max(current/4, minScanInterval) - case usage.last > memoryUsageHighThreshold || usage.max > memoryUsageHighThreshold: - // High pressure: reduce to 1/2 - newInterval = max(current/2, minScanInterval) - case shouldDampOnTrend: - // Trend damping: pressure is moderate (>30%) but rising. Reduce by 10% to - // preemptively slow down before downstream gets overwhelmed. - newInterval = max(scaleDuration(current, 9, 10), minScanInterval) - adjustedOnTrend = true - case allowedToIncrease && usage.max < memoryUsageVeryLowThreshold && usage.avg < memoryUsageVeryLowThreshold: - // Very low pressure (<20%): increase by 50%, allowed to exceed sync point cap. - maxInterval = maxScanInterval - newInterval = min(scaleDuration(current, 3, 2), maxInterval) - case allowedToIncrease && usage.max < memoryUsageLowThreshold && usage.avg < memoryUsageLowThreshold: - // Low pressure (<40%): increase by 25%, capped by sync point interval. - newInterval = min(scaleDuration(current, 5, 4), maxInterval) - } - - // Anti-oscillation guard: decreases are always applied immediately, - // but increases are blocked if cooldown conditions aren't met. - if newInterval > current && !allowedToIncrease { - return +func (c *adaptiveScanWindowController) allowedToVeryLowRecoverLocked(now time.Time) bool { + if c.lastInstabilityTime.IsZero() { + return true } + return now.Sub(c.lastInstabilityTime) >= scanWindowVeryLowRecoveryCooldown +} - if newInterval != current { - c.scanInterval.Store(int64(newInterval)) - metrics.EventServiceScanWindowIntervalGaugeVec.WithLabelValues(c.changefeedID.String()).Set(newInterval.Seconds()) - c.lastAdjustTime.Store(now) - if adjustedOnTrend { - c.lastTrendAdjustTime.Store(now) - } +func (c *adaptiveScanWindowController) isVeryLowPressureLocked(usage memoryUsageStats) bool { + return usage.max < memoryUsageVeryLowThreshold && + usage.avg < memoryUsageVeryLowThreshold && + c.fastUsageEMA < memoryUsageVeryLowThreshold && + c.slowUsageEMA < memoryUsageVeryLowThreshold +} + +func (c *adaptiveScanWindowController) isLowPressureLocked(usage memoryUsageStats) bool { + return usage.max < memoryUsageLowThreshold && + usage.avg < memoryUsageLowThreshold && + c.fastUsageEMA < memoryUsageLowThreshold+scanWindowLowPressureFastEMAOffset && + c.slowUsageEMA < memoryUsageLowThreshold+scanWindowLowPressureSlowEMAOffset +} + +func (c *adaptiveScanWindowController) noteAdjustmentLocked(now time.Time, downward bool) { + c.lastAdjustTime = now + if downward { + c.lastDownAdjustTime = now + c.lastInstabilityTime = now + } +} - log.Info("scan interval adjusted", - zap.Stringer("changefeedID", c.changefeedID), - zap.Duration("oldInterval", current), - zap.Duration("newInterval", newInterval), - zap.Duration("maxInterval", maxInterval), - zap.Float64("avgUsage", usage.avg), - zap.Float64("maxUsage", usage.max), - zap.Float64("firstUsage", usage.first), - zap.Float64("lastUsage", usage.last), - zap.Float64("trendDelta", trendDelta), - zap.Int("usageSamples", usage.cnt), - zap.Bool("syncPointEnabled", c.isSyncpointEnabled()), - zap.Duration("syncPointInterval", c.syncPointInterval)) +func (c *adaptiveScanWindowController) setLastAdjustTimeForTest(now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.lastAdjustTime = now +} + +func (c *adaptiveScanWindowController) setLastDownAdjustTimeForTest(now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.lastDownAdjustTime = now +} + +func normalizeUsageRatio(usageRatio float64) float64 { + if usageRatio != usageRatio || usageRatio < 0 { + return 0 + } + if usageRatio > 1 { + return 1 } + return usageRatio +} + +func ema(previous float64, value float64, alpha float64) float64 { + return previous + alpha*(value-previous) } func (c *changefeedStatus) maxScanInterval() time.Duration { diff --git a/pkg/eventservice/scan_window_test.go b/pkg/eventservice/scan_window_test.go index 01ee458e70..712fe9d789 100644 --- a/pkg/eventservice/scan_window_test.go +++ b/pkg/eventservice/scan_window_test.go @@ -18,71 +18,225 @@ import ( "time" "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" "go.uber.org/atomic" ) -func TestAdjustScanIntervalVeryLowBypassesSyncPointCap(t *testing.T) { +func markScanWindowReadyForIncrease(status *changefeedStatus, now time.Time) { + status.scanWindowController.setLastAdjustTimeForTest(now.Add(-scanIntervalAdjustCooldown - time.Second)) + status.scanWindowController.setLastDownAdjustTimeForTest(now.Add(-scanWindowReleaseRecoveryCooldown - time.Second)) +} + +func markScanWindowReadyForDecrease(status *changefeedStatus, now time.Time) { + status.scanWindowController.setLastDownAdjustTimeForTest(now.Add(-scanWindowPressureAdjustCooldown - time.Second)) +} + +func TestAdjustScanIntervalLowPressureSlowsRecoveryForLargeWindow(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) now := time.Now() - status.lastAdjustTime.Store(now.Add(-scanIntervalAdjustCooldown - time.Second)) + markScanWindowReadyForIncrease(status, now) + status.scanInterval.Store(int64(80 * time.Second)) + + for i := 0; i <= int(memoryUsageWindowDuration/time.Second); i++ { + status.updateMemoryUsage(now.Add(time.Duration(i)*time.Second), 0.15, 0) + } + require.Equal(t, int64(88*time.Second), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalVeryLowPressureSlowsRecoveryForVeryLargeWindow(t *testing.T) { + t.Parallel() - // Start from the sync point capped max interval, then allow it to grow slowly. - status.scanInterval.Store(int64(1 * time.Minute)) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + + now := time.Now() + markScanWindowReadyForIncrease(status, now) + status.scanInterval.Store(int64(150 * time.Second)) - // Maintain a very low pressure for a full window to allow bypassing the sync point cap. for i := 0; i <= int(memoryUsageWindowDuration/time.Second); i++ { status.updateMemoryUsage(now.Add(time.Duration(i)*time.Second), 0, 0) } - require.Equal(t, int64(90*time.Second), status.scanInterval.Load()) + require.Equal(t, int64(165*time.Second), status.scanInterval.Load()) } -func TestAdjustScanIntervalLowRespectsSyncPointCap(t *testing.T) { +func TestAdjustScanIntervalHighPressureUsesBoundedReduction(t *testing.T) { t.Parallel() status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) - now := time.Now() - status.lastAdjustTime.Store(now.Add(-scanIntervalAdjustCooldown - time.Second)) + markScanWindowReadyForDecrease(status, now) status.scanInterval.Store(int64(40 * time.Second)) + status.updateMemoryUsage(now.Add(memoryUsageWindowDuration), 0.8, 0) + require.Equal(t, int64(30*time.Second), status.scanInterval.Load()) +} - for i := 0; i <= int(memoryUsageWindowDuration/time.Second); i++ { - status.updateMemoryUsage(now.Add(time.Duration(i)*time.Second), 0.15, 0) +func TestAdjustScanIntervalDoesNotKeepReducingAfterTransientHighPressure(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute) + changefeed := status.changefeedID.String() + t.Cleanup(func() { + deleteScanWindowMetrics(changefeed) + }) + + now := time.Now() + markScanWindowReadyForDecrease(status, now) + status.scanInterval.Store(int64(40 * time.Second)) + + status.updateMemoryUsage(now, 0.8, 0) + require.Equal(t, int64(30*time.Second), status.scanInterval.Load()) + + for i := 1; i <= int(scanWindowPressureAdjustCooldown/time.Second)+1; i++ { + status.updateMemoryUsage(now.Add(time.Duration(i)*time.Second), 0.1, 0) } - require.Equal(t, int64(50*time.Second), status.scanInterval.Load()) + require.Equal(t, int64(30*time.Second), status.scanInterval.Load()) } -func TestAdjustScanIntervalDecreaseIgnoresCooldown(t *testing.T) { +func TestAdjustScanIntervalCriticalPressure(t *testing.T) { t.Parallel() status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) - now := time.Now() - status.lastAdjustTime.Store(now) + status.scanInterval.Store(int64(40 * time.Second)) + status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) + require.Equal(t, int64(scanWindowEmergencyBrakePlateauInterval), status.scanInterval.Load()) +} +func TestAdjustScanIntervalCriticalPressureIgnoresLowPressureHistory(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 10*time.Minute) + changefeed := status.changefeedID.String() + t.Cleanup(func() { + deleteScanWindowMetrics(changefeed) + }) + + now := time.Now() status.scanInterval.Store(int64(40 * time.Second)) - status.updateMemoryUsage(now.Add(memoryUsageWindowDuration), 0.8, 0) + for i := 0; i < 5; i++ { + status.updateMemoryUsage(now.Add(time.Duration(i)*time.Second), 0.05, 0) + } + require.Equal(t, int64(40*time.Second), status.scanInterval.Load()) + + status.updateMemoryUsage(now.Add(5*time.Second), 0.95, 0) require.Equal(t, int64(20*time.Second), status.scanInterval.Load()) } -func TestAdjustScanIntervalCriticalPressure(t *testing.T) { +func TestAdjustScanIntervalCriticalPressureUsesDefaultFloor(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status.scanInterval.Store(int64(8 * time.Second)) + status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 0.95, 0) + require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalHighPressureDoesNotIncreaseBelowDefaultFloor(t *testing.T) { t.Parallel() status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) now := time.Now() - status.lastAdjustTime.Store(now) + markScanWindowReadyForDecrease(status, now) - status.scanInterval.Store(int64(40 * time.Second)) + status.scanInterval.Store(int64(2 * time.Second)) + status.updateMemoryUsage(now.Add(memoryUsageWindowDuration), 0.8, 0) + require.Equal(t, int64(2*time.Second), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalCriticalPressureDoesNotIncreaseBelowDefaultFloor(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status.scanInterval.Store(int64(2 * time.Second)) + status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 0.95, 0) + require.Equal(t, int64(2*time.Second), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalEmergencyPressureUsesModerateBrakeForSmallWindow(t *testing.T) { + t.Parallel() - status.updateMemoryUsage(now.Add(memoryUsageWindowDuration), 1, 0) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status.scanInterval.Store(int64(20 * time.Second)) + status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) require.Equal(t, int64(10*time.Second), status.scanInterval.Load()) } -func TestUpdateMemoryUsageResetsScanIntervalOnMemoryRelease(t *testing.T) { +func TestScanWindowEmergencyBrakeIntervalIsContinuousAtThirtySeconds(t *testing.T) { + t.Parallel() + + require.Equal(t, 15*time.Second, scanWindowEmergencyBrakeInterval(30*time.Second, false)) + require.Equal(t, 15*time.Second, scanWindowEmergencyBrakeInterval(31*time.Second, false)) + require.Equal(t, 15*time.Second, scanWindowEmergencyBrakeInterval(60*time.Second, false)) +} + +func TestScanWindowEmergencyBrakeIntervalUsesStrongBrakeForLargeWindow(t *testing.T) { + t.Parallel() + + require.Equal(t, 20*time.Second, scanWindowEmergencyBrakeInterval(80*time.Second, false)) +} + +func TestAdjustScanIntervalEmergencyPressureUsesDefaultFloorForVerySmallWindow(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status.scanInterval.Store(int64(8 * time.Second)) + status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) + require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalEmergencyPressureDoesNotImmediatelyDropBelowDefaultFloor(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status.scanInterval.Store(int64(defaultScanInterval)) + status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) + require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalEmergencyPressureDoesNotIncreaseBelowDefaultFloor(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status.scanInterval.Store(int64(2 * time.Second)) + status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) + require.Equal(t, int64(2*time.Second), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalEmergencyPressureCanReachMinFloorWhenSustained(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status.scanInterval.Store(int64(defaultScanInterval)) + start := time.Now() + + for i := 0; i <= int(memoryUsageWindowDuration/time.Second); i++ { + status.updateMemoryUsage(start.Add(time.Duration(i)*time.Second), 1, 0) + } + + require.Equal(t, int64(minScanInterval), status.scanInterval.Load()) +} + +func TestAdjustScanIntervalRecoversFromFloorBeforeNormalIncreaseCooldown(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + now := time.Now() + status.scanInterval.Store(int64(defaultScanInterval)) + status.scanWindowController.setLastAdjustTimeForTest(now.Add(-scanWindowFloorRecoveryCooldown - time.Second)) + status.scanWindowController.setLastDownAdjustTimeForTest(now.Add(-scanWindowFloorRecoveryCooldown - time.Second)) + + for i, usage := range []float64{0.30, 0.25, 0.20, 0.18, 0.15} { + status.updateMemoryUsage(now.Add(time.Duration(i)*time.Second), usage, 0) + } + require.Greater(t, status.scanInterval.Load(), int64(defaultScanInterval)) +} + +func TestUpdateMemoryUsageDoesNotResetScanIntervalOnMemoryRelease(t *testing.T) { t.Parallel() status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) @@ -90,7 +244,68 @@ func TestUpdateMemoryUsageResetsScanIntervalOnMemoryRelease(t *testing.T) { status.scanInterval.Store(int64(40 * time.Second)) status.updateMemoryUsage(now, 0.5, 1) - require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) + require.Equal(t, int64(40*time.Second), status.scanInterval.Load()) +} + +func TestUpdateMemoryUsageRecordsScanWindowObservationMetrics(t *testing.T) { + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute) + changefeed := status.changefeedID.String() + t.Cleanup(func() { + deleteScanWindowMetrics(changefeed) + }) + + now := time.Now() + status.scanInterval.Store(int64(40 * time.Second)) + + status.updateMemoryUsage(now, 0.6, 1) + + require.InDelta(t, 0.6, testutil.ToFloat64(metrics.EventServiceScanWindowUsageRatioGaugeVec.WithLabelValues(changefeed, "report")), 1e-9) + require.InDelta(t, 0.6, testutil.ToFloat64(metrics.EventServiceScanWindowUsageRatioGaugeVec.WithLabelValues(changefeed, "avg")), 1e-9) + require.InDelta(t, 0.6, testutil.ToFloat64(metrics.EventServiceScanWindowUsageRatioGaugeVec.WithLabelValues(changefeed, "max")), 1e-9) + require.InDelta(t, 0.6, testutil.ToFloat64(metrics.EventServiceScanWindowUsageEMAGaugeVec.WithLabelValues(changefeed, "fast")), 1e-9) + require.InDelta(t, 0.6, testutil.ToFloat64(metrics.EventServiceScanWindowUsageEMAGaugeVec.WithLabelValues(changefeed, "slow")), 1e-9) + require.InDelta(t, 0, testutil.ToFloat64(metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, "report")), 1e-9) + require.InDelta(t, 0, testutil.ToFloat64(metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, "fast")), 1e-9) + require.InDelta(t, 0, testutil.ToFloat64(metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, "slow")), 1e-9) + require.InDelta(t, 0, testutil.ToFloat64(metrics.EventServiceScanWindowPressureScoreGaugeVec.WithLabelValues(changefeed)), 1e-9) + require.InDelta(t, 1, testutil.ToFloat64(metrics.EventServiceScanWindowMemoryReleaseCount.WithLabelValues(changefeed)), 1e-9) +} + +func TestUpdateMemoryUsageRecordsScanWindowAdjustCount(t *testing.T) { + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute) + changefeed := status.changefeedID.String() + t.Cleanup(func() { + deleteScanWindowMetrics(changefeed) + }) + + now := time.Now() + markScanWindowReadyForDecrease(status, now) + status.scanInterval.Store(int64(40 * time.Second)) + + status.updateMemoryUsage(now.Add(memoryUsageWindowDuration), 0.8, 0) + + require.Equal(t, int64(30*time.Second), status.scanInterval.Load()) + require.InDelta(t, 1, testutil.ToFloat64(metrics.EventServiceScanWindowAdjustCount.WithLabelValues(changefeed, string(scanWindowDecisionHighPressure))), 1e-9) +} + +func TestUpdateMemoryUsageRecordsScanWindowTargetBandMetrics(t *testing.T) { + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 10*time.Minute) + changefeed := status.changefeedID.String() + t.Cleanup(func() { + deleteScanWindowMetrics(changefeed) + }) + + start := time.Now() + status.updateMemoryUsage(start, 0.20, 0) + status.updateMemoryUsage(start.Add(time.Second), 0.40, 0) + status.updateMemoryUsage(start.Add(2*time.Second), 0.60, 0) + + require.InDelta(t, 0, testutil.ToFloat64(metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, "report")), 1e-9) + require.InDelta(t, 1, testutil.ToFloat64(metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, "fast")), 1e-9) + require.InDelta(t, 1, testutil.ToFloat64(metrics.EventServiceScanWindowTargetBandGaugeVec.WithLabelValues(changefeed, "slow")), 1e-9) + require.InDelta(t, 2, testutil.ToFloat64(metrics.EventServiceScanWindowTargetBandCrossCount.WithLabelValues(changefeed, "report")), 1e-9) + require.InDelta(t, 1, testutil.ToFloat64(metrics.EventServiceScanWindowTargetBandCrossCount.WithLabelValues(changefeed, "fast")), 1e-9) + require.InDelta(t, 1, testutil.ToFloat64(metrics.EventServiceScanWindowTargetBandCrossCount.WithLabelValues(changefeed, "slow")), 1e-9) } func TestAdjustScanIntervalIncreaseWithJitteredSamples(t *testing.T) { @@ -99,7 +314,7 @@ func TestAdjustScanIntervalIncreaseWithJitteredSamples(t *testing.T) { status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) start := time.Now() - status.lastAdjustTime.Store(start.Add(-scanIntervalAdjustCooldown - time.Second)) + markScanWindowReadyForIncrease(status, start) status.scanInterval.Store(int64(40 * time.Second)) @@ -112,37 +327,47 @@ func TestAdjustScanIntervalIncreaseWithJitteredSamples(t *testing.T) { require.Equal(t, int64(50*time.Second), status.scanInterval.Load()) } -func TestAdjustScanIntervalDecreasesWhenUsageIncreasing(t *testing.T) { +func TestAdjustScanIntervalReducesOnSustainedPressure(t *testing.T) { t.Parallel() status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) now := time.Now() - status.lastAdjustTime.Store(now) + markScanWindowReadyForDecrease(status, now) status.scanInterval.Store(int64(40 * time.Second)) - status.updateMemoryUsage(now, 0.10, 0) - status.updateMemoryUsage(now.Add(1*time.Second), 0.11, 0) - status.updateMemoryUsage(now.Add(2*time.Second), 0.12, 0) - status.updateMemoryUsage(now.Add(3*time.Second), 0.13, 0) - require.Equal(t, int64(40*time.Second), status.scanInterval.Load()) + status.updateMemoryUsage(now, 0.60, 0) + status.updateMemoryUsage(now.Add(1*time.Second), 0.60, 0) + status.updateMemoryUsage(now.Add(2*time.Second), 0.60, 0) + require.Equal(t, int64(36*time.Second), status.scanInterval.Load()) } -func TestAdjustScanIntervalDecreasesWhenUsageIncreasingAboveThirtyPercent(t *testing.T) { +func TestAdjustScanIntervalSustainedPressureDoesNotIncreaseBelowDefaultFloor(t *testing.T) { t.Parallel() status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) now := time.Now() - status.lastAdjustTime.Store(now) - status.lastTrendAdjustTime.Store(now.Add(-scanTrendAdjustCooldown - time.Second)) + markScanWindowReadyForDecrease(status, now) + + status.scanInterval.Store(int64(2 * time.Second)) + + status.updateMemoryUsage(now, 0.60, 0) + status.updateMemoryUsage(now.Add(1*time.Second), 0.60, 0) + status.updateMemoryUsage(now.Add(2*time.Second), 0.60, 0) + require.Equal(t, int64(2*time.Second), status.scanInterval.Load()) +} +func TestAdjustScanIntervalDoesNotIncreaseBeforeCooldown(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + now := time.Now() status.scanInterval.Store(int64(40 * time.Second)) - status.updateMemoryUsage(now, 0.31, 0) - status.updateMemoryUsage(now.Add(1*time.Second), 0.32, 0) - status.updateMemoryUsage(now.Add(2*time.Second), 0.33, 0) - status.updateMemoryUsage(now.Add(3*time.Second), 0.34, 0) - require.Equal(t, int64(36*time.Second), status.scanInterval.Load()) + for i := 0; i < 10; i++ { + status.updateMemoryUsage(now.Add(time.Duration(i)*time.Second), 0.05, 0) + } + require.Equal(t, int64(40*time.Second), status.scanInterval.Load()) } func TestRefreshMinSentResolvedTsMinAndSkipRules(t *testing.T) { diff --git a/pkg/metrics/event_service.go b/pkg/metrics/event_service.go index e6ed10cc90..a8d96a2bbd 100644 --- a/pkg/metrics/event_service.go +++ b/pkg/metrics/event_service.go @@ -69,6 +69,55 @@ var ( Name: "scan_window_interval", Help: "The scan window interval in seconds for each changefeed", }, []string{"changefeed"}) + EventServiceScanWindowUsageRatioGaugeVec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "scan_window_usage_ratio", + Help: "The usage ratio observed by the scan window controller for each changefeed", + }, []string{"changefeed", "type"}) + EventServiceScanWindowUsageEMAGaugeVec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "scan_window_usage_ema", + Help: "The usage EMA values used by the scan window controller for each changefeed", + }, []string{"changefeed", "type"}) + EventServiceScanWindowTargetBandGaugeVec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "scan_window_target_band", + Help: "Whether the observed scan window value is currently inside the target band for each changefeed", + }, []string{"changefeed", "type"}) + EventServiceScanWindowTargetBandCrossCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "scan_window_target_band_cross_count", + Help: "The number of target band state changes observed by the scan window controller for each changefeed", + }, []string{"changefeed", "type"}) + EventServiceScanWindowPressureScoreGaugeVec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "scan_window_pressure_score", + Help: "The pressure score maintained by the scan window controller for each changefeed", + }, []string{"changefeed"}) + EventServiceScanWindowMemoryReleaseCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "scan_window_memory_release_count", + Help: "The number of memory release events reported to the scan window controller for each changefeed", + }, []string{"changefeed"}) + EventServiceScanWindowAdjustCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "scan_window_adjust_count", + Help: "The number of scan window adjustments made by the controller for each changefeed", + }, []string{"changefeed", "reason"}) EventServiceScanDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "ticdc", @@ -201,6 +250,13 @@ func initEventServiceMetrics(registry *prometheus.Registry) { registry.MustRegister(EventServiceResolvedTsLagGauge) registry.MustRegister(EventServiceScanWindowBaseTsGaugeVec) registry.MustRegister(EventServiceScanWindowIntervalGaugeVec) + registry.MustRegister(EventServiceScanWindowUsageRatioGaugeVec) + registry.MustRegister(EventServiceScanWindowUsageEMAGaugeVec) + registry.MustRegister(EventServiceScanWindowTargetBandGaugeVec) + registry.MustRegister(EventServiceScanWindowTargetBandCrossCount) + registry.MustRegister(EventServiceScanWindowPressureScoreGaugeVec) + registry.MustRegister(EventServiceScanWindowMemoryReleaseCount) + registry.MustRegister(EventServiceScanWindowAdjustCount) registry.MustRegister(EventServiceScanDuration) registry.MustRegister(EventServiceScannedCount) registry.MustRegister(EventServiceDispatcherGauge) From 81e85006c1f308be549431be35dcddb7256b0588 Mon Sep 17 00:00:00 2001 From: "shenhaibo@pingcap.cn" Date: Sun, 14 Jun 2026 01:00:12 +0800 Subject: [PATCH 4/4] *: add enable-scan-window switch to gate adaptive scan window Add a per-changefeed `enable-scan-window` replica config (default false) and plumb it through the changefeed config, dispatcher info and event service. When the switch is off the adaptive scan-window feature is fully inert and the changefeed behaves as if it was never introduced: - event service: memory control, adaptive scan interval, base-ts capping, empty-range signal, pending-DDL local advance and scan-window metrics are all gated. - dynstream: the memory release ratio follows the switch (0.4 off / 0.6 on); the deadlock high-water-mark stays 0.6 in both modes. --- .../dispatcher/basic_dispatcher.go | 1 + .../dispatcher/basic_dispatcher_info.go | 10 + .../dispatcher/event_dispatcher_test.go | 7 +- .../dispatcher/redo_dispatcher_test.go | 1 + .../dispatchermanager/dispatcher_manager.go | 1 + .../dispatcher_manager_test.go | 1 + .../eventcollector/dispatcher_session.go | 2 + .../eventcollector/dispatcher_stat_test.go | 4 + .../eventcollector/event_collector.go | 11 + .../eventcollector/event_collector_test.go | 4 + eventpb/event.pb.go | 195 +++++++++++------- eventpb/event.proto | 4 + pkg/config/changefeed.go | 4 + pkg/config/replica_config.go | 6 + pkg/eventservice/dispatcher_stat.go | 8 +- pkg/eventservice/event_broker.go | 23 ++- pkg/eventservice/event_broker_test.go | 57 +++++ pkg/eventservice/event_service.go | 4 + pkg/eventservice/event_service_test.go | 11 +- pkg/eventservice/scan_window.go | 17 ++ pkg/eventservice/scan_window_test.go | 80 ++++--- utils/dynstream/interfaces.go | 11 + utils/dynstream/memory_control.go | 27 ++- 23 files changed, 374 insertions(+), 115 deletions(-) diff --git a/downstreamadapter/dispatcher/basic_dispatcher.go b/downstreamadapter/dispatcher/basic_dispatcher.go index 0044f9ae13..ddbe54bb3b 100644 --- a/downstreamadapter/dispatcher/basic_dispatcher.go +++ b/downstreamadapter/dispatcher/basic_dispatcher.go @@ -49,6 +49,7 @@ type DispatcherService interface { GetFilterConfig() *eventpb.FilterConfig EnableSyncPoint() bool GetSyncPointInterval() time.Duration + GetEnableScanWindow() bool GetSkipSyncpointAtStartTs() bool GetTxnAtomicity() config.AtomicityLevel GetResolvedTs() uint64 diff --git a/downstreamadapter/dispatcher/basic_dispatcher_info.go b/downstreamadapter/dispatcher/basic_dispatcher_info.go index b4e591ffd0..d09e3c7f5e 100644 --- a/downstreamadapter/dispatcher/basic_dispatcher_info.go +++ b/downstreamadapter/dispatcher/basic_dispatcher_info.go @@ -55,6 +55,10 @@ type SharedInfo struct { // will break the splittability of this table. enableSplittableCheck bool + // enableScanWindow controls whether the event service applies the adaptive scan + // window (memory control + adaptive scan interval) for this changefeed. + enableScanWindow bool + // router is used to route source schema/table names to target schema/table names. // It is used to apply routing to TableInfo before storing it. router routing.Router @@ -90,6 +94,7 @@ func NewSharedInfo( syncPointConfig *syncpoint.SyncPointConfig, txnAtomicity *config.AtomicityLevel, enableSplittableCheck bool, + enableScanWindow bool, router routing.Router, statusesChan chan TableSpanStatusWithSeq, blockStatusBufferSize int, @@ -104,6 +109,7 @@ func NewSharedInfo( filterConfig: filterConfig, syncPointConfig: syncPointConfig, enableSplittableCheck: enableSplittableCheck, + enableScanWindow: enableScanWindow, router: router, statusesChan: statusesChan, blockStatusBuffer: NewBlockStatusBuffer(blockStatusBufferSize), @@ -215,6 +221,10 @@ func (d *BasicDispatcher) GetSyncPointInterval() time.Duration { return time.Duration(0) } +func (d *BasicDispatcher) GetEnableScanWindow() bool { + return d.sharedInfo.enableScanWindow +} + func (d *BasicDispatcher) GetTableSpan() *heartbeatpb.TableSpan { return d.tableSpan } diff --git a/downstreamadapter/dispatcher/event_dispatcher_test.go b/downstreamadapter/dispatcher/event_dispatcher_test.go index fce198502b..f75b846158 100644 --- a/downstreamadapter/dispatcher/event_dispatcher_test.go +++ b/downstreamadapter/dispatcher/event_dispatcher_test.go @@ -74,6 +74,7 @@ func newTestSharedInfo( syncPointConfig, &defaultAtomicity, enableSplittableCheck, + false, // enableScanWindow routing.Router{}, make(chan TableSpanStatusWithSeq, 128), 128, @@ -128,6 +129,7 @@ func newDispatcherForTest(sink sink.Sink, tableSpan *heartbeatpb.TableSpan) *Eve }, // syncPointConfig &defaultAtomicity, false, // enableSplittableCheck + false, // enableScanWindow routing.Router{}, make(chan TableSpanStatusWithSeq, 128), 128, @@ -1090,7 +1092,8 @@ func TestDispatcherSplittableCheck(t *testing.T) { SyncPointRetention: time.Duration(10 * time.Minute), }, &defaultAtomicity, - true, // enableSplittableCheck = true + true, // enableSplittableCheck = true + false, // enableScanWindow routing.Router{}, make(chan TableSpanStatusWithSeq, 128), 128, @@ -1201,6 +1204,7 @@ func TestDispatcher_SkipDMLAsStartTs_FilterCorrectly(t *testing.T) { }, &defaultAtomicity, false, + false, // enableScanWindow routing.Router{}, make(chan TableSpanStatusWithSeq, 128), 128, @@ -1281,6 +1285,7 @@ func TestDispatcher_SkipDMLAsStartTs_Disabled(t *testing.T) { }, &defaultAtomicity, false, + false, // enableScanWindow routing.Router{}, make(chan TableSpanStatusWithSeq, 128), 128, diff --git a/downstreamadapter/dispatcher/redo_dispatcher_test.go b/downstreamadapter/dispatcher/redo_dispatcher_test.go index f4dbe45cf9..22b617cb98 100644 --- a/downstreamadapter/dispatcher/redo_dispatcher_test.go +++ b/downstreamadapter/dispatcher/redo_dispatcher_test.go @@ -47,6 +47,7 @@ func newRedoDispatcherForTest(sink sink.Sink, tableSpan *heartbeatpb.TableSpan) nil, // redo dispatcher doesn't need syncPointConfig &defaultAtomicity, false, // enableSplittableCheck + false, // enableScanWindow routing.Router{}, make(chan TableSpanStatusWithSeq, 128), 128, diff --git a/downstreamadapter/dispatchermanager/dispatcher_manager.go b/downstreamadapter/dispatchermanager/dispatcher_manager.go index b200e2c936..b1000b950d 100644 --- a/downstreamadapter/dispatchermanager/dispatcher_manager.go +++ b/downstreamadapter/dispatchermanager/dispatcher_manager.go @@ -284,6 +284,7 @@ func NewDispatcherManager( syncPointConfig, manager.config.SinkConfig.TxnAtomicity, manager.config.EnableSplittableCheck, + manager.config.EnableScanWindow, router, make(chan dispatcher.TableSpanStatusWithSeq, 8192), blockStatusBufferSize, diff --git a/downstreamadapter/dispatchermanager/dispatcher_manager_test.go b/downstreamadapter/dispatchermanager/dispatcher_manager_test.go index d725bbbcbf..2951b44383 100644 --- a/downstreamadapter/dispatchermanager/dispatcher_manager_test.go +++ b/downstreamadapter/dispatchermanager/dispatcher_manager_test.go @@ -130,6 +130,7 @@ func createTestManager(t *testing.T) *DispatcherManager { nil, // syncPointConfig &defaultAtomicity, false, + false, // enableScanWindow routing.Router{}, make(chan dispatcher.TableSpanStatusWithSeq, 8192), blockStatusBufferSize, diff --git a/downstreamadapter/eventcollector/dispatcher_session.go b/downstreamadapter/eventcollector/dispatcher_session.go index e3ec7de114..8d32753c98 100644 --- a/downstreamadapter/eventcollector/dispatcher_session.go +++ b/downstreamadapter/eventcollector/dispatcher_session.go @@ -533,6 +533,7 @@ func (s *dispatcherSession) newDispatcherRegisterRequest(serverID string, onlyRe Integrity: s.target.GetIntegrityConfig(), OutputRawChangeEvent: s.target.IsOutputRawChangeEvent(), TxnAtomicity: string(s.target.GetTxnAtomicity()), + EnableScanWindow: s.target.GetEnableScanWindow(), }, } } @@ -568,6 +569,7 @@ func (s *dispatcherSession) newDispatcherResetRequest(serverID string, resetTs u Integrity: s.target.GetIntegrityConfig(), OutputRawChangeEvent: s.target.IsOutputRawChangeEvent(), TxnAtomicity: string(s.target.GetTxnAtomicity()), + EnableScanWindow: s.target.GetEnableScanWindow(), }, } } diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index d2317ecbda..092ca19115 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -116,6 +116,10 @@ func (m *mockDispatcher) GetSyncPointInterval() time.Duration { return time.Second * 10 } +func (m *mockDispatcher) GetEnableScanWindow() bool { + return false +} + func (m *mockDispatcher) GetSkipSyncpointAtStartTs() bool { return m.skipSyncpointAtStartTs } diff --git a/downstreamadapter/eventcollector/event_collector.go b/downstreamadapter/eventcollector/event_collector.go index 17d8e77583..0b3c7b85e5 100644 --- a/downstreamadapter/eventcollector/event_collector.go +++ b/downstreamadapter/eventcollector/event_collector.go @@ -40,6 +40,11 @@ const ( receiveChanSize = 1024 * 8 commonMsgRetryQuota = 3 // The number of retries for most droppable dispatcher requests. eventServiceHeartbeatInterval = 10 * time.Second + + // scanWindowReleaseMemoryRatio is the fraction of pending memory the event collector + // releases per cycle for changefeeds with the adaptive scan window enabled. It is + // more aggressive than the dynstream default to keep up with the larger scan windows. + scanWindowReleaseMemoryRatio = 0.6 ) // DispatcherMessage is the message send to EventService. @@ -274,6 +279,12 @@ func (c *EventCollector) PrepareAddDispatcher( ds := c.getDynamicStream(target.GetMode()) areaSetting := dynstream.NewAreaSettingsWithMaxPendingSize(memoryQuota, dynstream.MemoryControlForEventCollector, "eventCollector") + // The adaptive scan window releases memory more aggressively to keep up with the + // larger scan windows. Only apply it when the changefeed enables the scan window, + // so a changefeed with the feature off keeps the pre-scan-window release ratio. + if target.GetEnableScanWindow() { + areaSetting.SetReleaseMemoryRatio(scanWindowReleaseMemoryRatio) + } err := ds.AddPath(target.GetId(), stat, areaSetting) if err != nil { log.Warn("add dispatcher to dynamic stream failed", zap.Error(err)) diff --git a/downstreamadapter/eventcollector/event_collector_test.go b/downstreamadapter/eventcollector/event_collector_test.go index b171a08d1b..a6830741cb 100644 --- a/downstreamadapter/eventcollector/event_collector_test.go +++ b/downstreamadapter/eventcollector/event_collector_test.go @@ -87,6 +87,10 @@ func (m *mockEventDispatcher) GetSyncPointInterval() time.Duration { return time.Second } +func (m *mockEventDispatcher) GetEnableScanWindow() bool { + return false +} + func (m *mockEventDispatcher) GetSkipSyncpointAtStartTs() bool { return false } diff --git a/eventpb/event.pb.go b/eventpb/event.pb.go index 485ed22007..d0986d6313 100644 --- a/eventpb/event.pb.go +++ b/eventpb/event.pb.go @@ -646,6 +646,10 @@ type DispatcherRequest struct { OutputRawChangeEvent bool `protobuf:"varint,17,opt,name=output_raw_change_event,json=outputRawChangeEvent,proto3" json:"output_raw_change_event,omitempty"` Mode int64 `protobuf:"varint,18,opt,name=mode,proto3" json:"mode,omitempty"` TxnAtomicity string `protobuf:"bytes,19,opt,name=txn_atomicity,json=txnAtomicity,proto3" json:"txn_atomicity,omitempty"` + // enable_scan_window controls whether the event service applies the adaptive + // scan window (memory control + scan interval) for this changefeed. + // It defaults to false so the feature behaves as if it was never introduced. + EnableScanWindow bool `protobuf:"varint,20,opt,name=enable_scan_window,json=enableScanWindow,proto3" json:"enable_scan_window,omitempty"` } func (m *DispatcherRequest) Reset() { *m = DispatcherRequest{} } @@ -814,6 +818,13 @@ func (m *DispatcherRequest) GetTxnAtomicity() string { return "" } +func (m *DispatcherRequest) GetEnableScanWindow() bool { + if m != nil { + return m.EnableScanWindow + } + return false +} + func init() { proto.RegisterEnum("eventpb.OpType", OpType_name, OpType_value) proto.RegisterEnum("eventpb.ActionType", ActionType_name, ActionType_value) @@ -832,80 +843,81 @@ func init() { func init() { proto.RegisterFile("eventpb/event.proto", fileDescriptor_d7fb2554dfcf7f7d) } var fileDescriptor_d7fb2554dfcf7f7d = []byte{ - // 1157 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0x4f, 0x73, 0xdb, 0x44, - 0x14, 0x8f, 0x9c, 0x3f, 0xb6, 0x9e, 0xed, 0xc4, 0xde, 0x34, 0xad, 0xda, 0x42, 0x08, 0x86, 0xe9, - 0x98, 0xce, 0xe0, 0x94, 0xd0, 0xc2, 0x4c, 0x87, 0xe9, 0x4c, 0x49, 0x5c, 0xd0, 0x0c, 0x6d, 0x32, - 0x6b, 0xb5, 0x33, 0x70, 0xd1, 0xc8, 0xd2, 0x4b, 0x22, 0x2a, 0xef, 0xaa, 0xab, 0x95, 0x13, 0xf3, - 0x29, 0x38, 0x71, 0xe2, 0xce, 0x57, 0xe1, 0xd8, 0x23, 0x37, 0x98, 0xf6, 0xc0, 0xd7, 0x60, 0x76, - 0x57, 0x96, 0xad, 0xb6, 0x70, 0xe1, 0xe4, 0xdd, 0xf7, 0xfb, 0xbd, 0xdd, 0xb7, 0xbf, 0xf7, 0x47, - 0x86, 0x6d, 0x9c, 0x22, 0x93, 0xe9, 0x78, 0x5f, 0xff, 0x0e, 0x52, 0xc1, 0x25, 0x27, 0xf5, 0xc2, - 0x78, 0xe3, 0xe6, 0x39, 0x06, 0x42, 0x8e, 0x31, 0x50, 0x8c, 0x72, 0x6d, 0x58, 0xbd, 0x3f, 0x6b, - 0xb0, 0x35, 0x54, 0xc4, 0x47, 0x71, 0x22, 0x51, 0xd0, 0x3c, 0x41, 0xe2, 0x40, 0x7d, 0x12, 0xc8, - 0xf0, 0x1c, 0x85, 0x63, 0xed, 0xad, 0xf6, 0x6d, 0x3a, 0xdf, 0x92, 0x0f, 0xa1, 0x15, 0x9f, 0x31, - 0x2e, 0xd0, 0xd7, 0x87, 0x3b, 0x35, 0x0d, 0x37, 0x8d, 0x4d, 0x1f, 0x43, 0xde, 0x07, 0x28, 0x28, - 0xd9, 0x8b, 0xc4, 0x59, 0xd5, 0x04, 0xdb, 0x58, 0x46, 0x2f, 0x12, 0xf2, 0x25, 0x38, 0x05, 0x1c, - 0xb3, 0x0c, 0x85, 0xf4, 0xa7, 0x41, 0x92, 0xa3, 0x8f, 0x97, 0xa9, 0x70, 0xd6, 0xf6, 0xac, 0xbe, - 0x4d, 0x77, 0x0c, 0xee, 0x6a, 0xf8, 0x99, 0x42, 0x87, 0x97, 0xa9, 0x20, 0x0f, 0xe0, 0xbd, 0xc2, - 0x31, 0x4f, 0xa3, 0x40, 0xa2, 0xcf, 0xf0, 0x62, 0xd9, 0x79, 0x5d, 0x3b, 0x17, 0x87, 0x3f, 0xd5, - 0x94, 0x27, 0x78, 0xf1, 0x1f, 0xfe, 0x3c, 0x89, 0x96, 0xfd, 0x37, 0xde, 0xf6, 0x3f, 0x4e, 0xa2, - 0x85, 0xff, 0x22, 0xf0, 0x08, 0x13, 0x94, 0xb8, 0xec, 0x5b, 0x5f, 0x0e, 0xfc, 0x48, 0xc3, 0xa5, - 0x63, 0xef, 0x17, 0x0b, 0xba, 0x2e, 0x63, 0x28, 0x8c, 0xc2, 0x87, 0x9c, 0x9d, 0xc6, 0x67, 0xe4, - 0x0a, 0xac, 0x8b, 0x3c, 0xc1, 0xac, 0x50, 0xd8, 0x6c, 0xc8, 0xa7, 0xb0, 0x5d, 0x5c, 0x22, 0x2f, - 0x99, 0x9f, 0xc9, 0x40, 0x48, 0x5f, 0x66, 0x5a, 0xe6, 0x35, 0xda, 0x31, 0x90, 0x77, 0xc9, 0x46, - 0x0a, 0xf0, 0x32, 0xf2, 0x15, 0xb4, 0x96, 0x72, 0x97, 0x69, 0xb5, 0x9b, 0x07, 0xce, 0xa0, 0xc8, - 0xfc, 0xe0, 0x8d, 0xc4, 0xd2, 0x0a, 0xbb, 0xf7, 0xab, 0x05, 0xad, 0x4a, 0x4c, 0x1f, 0x43, 0x3b, - 0x0c, 0x32, 0x1c, 0x21, 0xcb, 0x62, 0x19, 0x4f, 0xd1, 0xb1, 0xf6, 0xac, 0x7e, 0x83, 0x56, 0x8d, - 0xe4, 0x16, 0x6c, 0x9e, 0x72, 0x11, 0x22, 0xc5, 0x34, 0x89, 0xc3, 0x40, 0xa2, 0x53, 0xd3, 0xb4, - 0x37, 0xac, 0xe4, 0x01, 0xb4, 0x4e, 0x97, 0x4e, 0x77, 0x56, 0xf7, 0xac, 0x7e, 0xf3, 0xe0, 0x46, - 0x19, 0xdc, 0x5b, 0x9a, 0xd0, 0x0a, 0xbf, 0xd7, 0x02, 0xa0, 0x98, 0xf1, 0x64, 0x8a, 0x91, 0x97, - 0xf5, 0x72, 0x58, 0x37, 0xf5, 0xd5, 0x81, 0xd5, 0xe7, 0x38, 0xd3, 0xa1, 0xb5, 0xa8, 0x5a, 0x2a, - 0x29, 0x75, 0x2e, 0x74, 0x1c, 0x2d, 0x6a, 0x36, 0xe4, 0x06, 0x34, 0xe6, 0xf9, 0xd3, 0x57, 0xb7, - 0x68, 0xb9, 0x27, 0x7d, 0xa8, 0xf3, 0xd4, 0x97, 0xb3, 0x14, 0x75, 0xcd, 0x6d, 0x1e, 0x6c, 0x95, - 0x51, 0x1d, 0xa7, 0xde, 0x2c, 0x45, 0xba, 0xc1, 0xf5, 0x6f, 0xef, 0x47, 0x68, 0x78, 0x97, 0xcc, - 0xdc, 0x7c, 0x0b, 0x36, 0x34, 0xcb, 0xe4, 0xac, 0x79, 0xb0, 0x59, 0xd5, 0x99, 0x16, 0x28, 0xb9, - 0x09, 0x76, 0xc8, 0x27, 0x93, 0xb8, 0x48, 0x9d, 0xd5, 0x5f, 0xa3, 0x0d, 0x63, 0xf0, 0x32, 0x72, - 0x1d, 0x1a, 0x65, 0x5a, 0x57, 0x35, 0x56, 0xcf, 0x4c, 0x36, 0x7b, 0x4d, 0xb0, 0xbd, 0x60, 0x9c, - 0xa0, 0xcb, 0x4e, 0x79, 0xef, 0x6f, 0x0b, 0x6c, 0x93, 0x2d, 0xc4, 0x88, 0xdc, 0x01, 0x50, 0x05, - 0x51, 0xb9, 0xbe, 0x5b, 0x5e, 0x3f, 0x8f, 0x90, 0xda, 0xb2, 0x58, 0x65, 0xe4, 0x03, 0x68, 0x8a, - 0x42, 0xbd, 0x45, 0x18, 0x20, 0x4a, 0x41, 0xc9, 0x03, 0x68, 0x47, 0x71, 0x96, 0x9a, 0xc6, 0xf6, - 0xe3, 0xa8, 0xc8, 0xcf, 0xf5, 0xc1, 0xd2, 0xb4, 0x18, 0x1c, 0x95, 0x0c, 0xf7, 0x88, 0xb6, 0x16, - 0x7c, 0x37, 0xd2, 0x05, 0x1c, 0xc8, 0x98, 0x6b, 0x05, 0x6b, 0xd4, 0x6c, 0xc8, 0x67, 0x00, 0x52, - 0xbd, 0xc1, 0x8f, 0xd9, 0x29, 0xd7, 0x3d, 0xd9, 0x3c, 0x20, 0x8b, 0x40, 0xe7, 0xcf, 0xa3, 0xb6, - 0x2c, 0x5f, 0x3a, 0x83, 0x2d, 0x97, 0x49, 0x3c, 0x13, 0xb1, 0x9c, 0x15, 0x85, 0x78, 0x07, 0xb6, - 0x17, 0xa6, 0x73, 0x0c, 0x9f, 0x7f, 0x87, 0x53, 0x4c, 0x74, 0xce, 0x6d, 0xfa, 0x2e, 0x88, 0xdc, - 0x85, 0x9d, 0x43, 0x2e, 0x44, 0x9e, 0xca, 0x98, 0xb3, 0x6f, 0x03, 0x16, 0x25, 0x68, 0x7c, 0x6a, - 0xa6, 0x35, 0xdf, 0x09, 0xf6, 0x7e, 0xdb, 0x80, 0xee, 0xe2, 0x89, 0x14, 0x5f, 0xe4, 0x98, 0xe9, - 0x09, 0x16, 0x26, 0x79, 0x26, 0x8d, 0x2c, 0x96, 0x56, 0xce, 0x2e, 0x2c, 0x6e, 0xa4, 0x84, 0x0b, - 0xcf, 0x03, 0x76, 0x86, 0xa7, 0x88, 0x91, 0x62, 0xd4, 0xde, 0x21, 0xdc, 0x61, 0xc9, 0x50, 0xc2, - 0x2d, 0xf8, 0xc6, 0xff, 0x7f, 0x09, 0x7f, 0x6f, 0x2e, 0x71, 0x96, 0x06, 0x4c, 0xab, 0xdf, 0x3c, - 0xb8, 0x5a, 0x71, 0xd6, 0x32, 0x8f, 0xd2, 0x80, 0x15, 0x32, 0xab, 0x65, 0xa5, 0xf0, 0xd6, 0x2b, - 0x85, 0xa7, 0x0a, 0x36, 0x43, 0x31, 0x35, 0xd1, 0x98, 0x39, 0xd8, 0x30, 0x06, 0x37, 0x22, 0x77, - 0xa1, 0x19, 0x84, 0x4a, 0x38, 0xd3, 0x2f, 0x75, 0xdd, 0x2f, 0xdb, 0x65, 0x4a, 0x1f, 0x6a, 0x4c, - 0xf7, 0x0c, 0x04, 0xe5, 0x9a, 0xdc, 0x87, 0xb6, 0x69, 0x66, 0x3f, 0x34, 0xdd, 0xdf, 0xd0, 0x71, - 0xee, 0x94, 0x7e, 0xff, 0xde, 0xf8, 0xe4, 0x36, 0x74, 0x91, 0x99, 0x17, 0xce, 0x58, 0xe8, 0xa7, - 0x3c, 0x66, 0xd2, 0xb1, 0xf5, 0x8c, 0xd9, 0x32, 0xc0, 0x68, 0xc6, 0xc2, 0x13, 0x65, 0x26, 0x3d, - 0x68, 0x2f, 0x48, 0xea, 0x69, 0xa0, 0x9f, 0xd6, 0xcc, 0xe6, 0x0c, 0x2f, 0x23, 0x03, 0xd8, 0x5e, - 0xe2, 0xc4, 0x4c, 0xa2, 0x98, 0x06, 0x89, 0xd3, 0xd4, 0xcc, 0x6e, 0xc9, 0x74, 0x0b, 0x40, 0xe5, - 0x9f, 0xb3, 0x64, 0xe6, 0x0b, 0xcc, 0x33, 0x74, 0x5a, 0xfa, 0x62, 0x5b, 0x59, 0xa8, 0x32, 0x28, - 0x21, 0xc7, 0x91, 0xf0, 0x27, 0x3c, 0x42, 0xa7, 0xad, 0xc1, 0xfa, 0x38, 0x12, 0x8f, 0x79, 0x84, - 0xe4, 0x0b, 0xb0, 0xe3, 0x79, 0x71, 0x3a, 0x9b, 0xfa, 0xc5, 0xce, 0xd2, 0xbc, 0xab, 0x14, 0x39, - 0x5d, 0x50, 0xd5, 0xac, 0x92, 0xf1, 0x04, 0x7f, 0xe2, 0x0c, 0x9d, 0x2d, 0xa3, 0xff, 0x7c, 0xaf, - 0xfa, 0x0c, 0x53, 0x1e, 0x9e, 0x3b, 0x1d, 0x1d, 0xaf, 0xd9, 0x90, 0x7b, 0x70, 0x8d, 0xe7, 0x32, - 0xcd, 0xa5, 0x2f, 0x82, 0x0b, 0xdf, 0xd4, 0x57, 0xf1, 0x4d, 0xee, 0xea, 0x98, 0xae, 0x18, 0x98, - 0x06, 0x17, 0xa6, 0x14, 0xcd, 0x08, 0x23, 0xb0, 0xa6, 0xe3, 0x26, 0x7b, 0x56, 0x7f, 0x95, 0xea, - 0x35, 0xf9, 0x08, 0xda, 0x6a, 0xb6, 0x04, 0x92, 0x4f, 0xe2, 0x50, 0x05, 0xbe, 0xad, 0x23, 0x68, - 0xc9, 0x4b, 0xf6, 0x70, 0x6e, 0xbb, 0xfd, 0x09, 0x6c, 0x98, 0xc9, 0x48, 0xda, 0x60, 0x9b, 0xd5, - 0x49, 0x2e, 0x3b, 0x2b, 0xa4, 0x03, 0x2d, 0xb3, 0x35, 0x9f, 0xbd, 0x8e, 0x75, 0x5b, 0x00, 0x2c, - 0x8a, 0x82, 0xdc, 0x84, 0x6b, 0x0f, 0x0f, 0x3d, 0xf7, 0xf8, 0x89, 0xef, 0x7d, 0x7f, 0x32, 0xf4, - 0x9f, 0x3e, 0x19, 0x9d, 0x0c, 0x0f, 0xdd, 0x47, 0xee, 0xf0, 0xa8, 0xb3, 0x42, 0x1c, 0xb8, 0xb2, - 0x0c, 0xd2, 0xe1, 0x37, 0xee, 0xc8, 0x1b, 0xd2, 0x8e, 0x45, 0xae, 0x02, 0xa9, 0x22, 0x8f, 0x8f, - 0x9f, 0x0d, 0x3b, 0x35, 0xb2, 0x03, 0xdd, 0xaa, 0x7d, 0x34, 0xf4, 0x3a, 0xeb, 0x5f, 0xdf, 0xff, - 0xfd, 0xd5, 0xae, 0xf5, 0xf2, 0xd5, 0xae, 0xf5, 0xd7, 0xab, 0x5d, 0xeb, 0xe7, 0xd7, 0xbb, 0x2b, - 0x2f, 0x5f, 0xef, 0xae, 0xfc, 0xf1, 0x7a, 0x77, 0xe5, 0x87, 0xbd, 0xb3, 0x58, 0x9e, 0xe7, 0xe3, - 0x41, 0xc8, 0x27, 0xfb, 0x69, 0xcc, 0xce, 0xc2, 0x20, 0xdd, 0x97, 0x71, 0x18, 0x85, 0xfb, 0x45, - 0x5e, 0xc6, 0x1b, 0xfa, 0x8f, 0xd0, 0xe7, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0xce, 0xc1, 0x69, - 0x79, 0x45, 0x09, 0x00, 0x00, + // 1181 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0xcf, 0x3a, 0x7f, 0xec, 0x7d, 0xb6, 0x13, 0x7b, 0x92, 0xb4, 0xdb, 0x16, 0x42, 0x30, 0xa8, + 0x32, 0x15, 0x38, 0x25, 0xb4, 0x20, 0x55, 0xa8, 0x52, 0x49, 0x5c, 0x58, 0x89, 0x36, 0xd1, 0xd8, + 0x2d, 0x82, 0xcb, 0x6a, 0xbd, 0xfb, 0x92, 0x2c, 0x5d, 0xcf, 0x6c, 0x67, 0x67, 0x1d, 0x9b, 0x4f, + 0xc1, 0x89, 0x13, 0x1f, 0x88, 0x63, 0x2f, 0x48, 0xdc, 0x40, 0xed, 0x81, 0xaf, 0x81, 0x66, 0x66, + 0xbd, 0xf6, 0xb6, 0x85, 0x0b, 0x27, 0xcf, 0xbc, 0xdf, 0xef, 0xcd, 0xbc, 0xf9, 0xbd, 0x3f, 0x6b, + 0xd8, 0xc6, 0x09, 0x32, 0x99, 0x8c, 0x0e, 0xf4, 0x6f, 0x2f, 0x11, 0x5c, 0x72, 0x52, 0xcd, 0x8d, + 0xd7, 0x6f, 0x5c, 0xa0, 0x2f, 0xe4, 0x08, 0x7d, 0xc5, 0x28, 0xd6, 0x86, 0xd5, 0xf9, 0xb3, 0x02, + 0x5b, 0x7d, 0x45, 0x7c, 0x18, 0xc5, 0x12, 0x05, 0xcd, 0x62, 0x24, 0x0e, 0x54, 0xc7, 0xbe, 0x0c, + 0x2e, 0x50, 0x38, 0xd6, 0xfe, 0x6a, 0xd7, 0xa6, 0xf3, 0x2d, 0x79, 0x1f, 0x1a, 0xd1, 0x39, 0xe3, + 0x02, 0x3d, 0x7d, 0xb8, 0x53, 0xd1, 0x70, 0xdd, 0xd8, 0xf4, 0x31, 0xe4, 0x5d, 0x80, 0x9c, 0x92, + 0x3e, 0x8f, 0x9d, 0x55, 0x4d, 0xb0, 0x8d, 0x65, 0xf0, 0x3c, 0x26, 0x5f, 0x80, 0x93, 0xc3, 0x11, + 0x4b, 0x51, 0x48, 0x6f, 0xe2, 0xc7, 0x19, 0x7a, 0x38, 0x4d, 0x84, 0xb3, 0xb6, 0x6f, 0x75, 0x6d, + 0xba, 0x6b, 0x70, 0x57, 0xc3, 0x4f, 0x15, 0xda, 0x9f, 0x26, 0x82, 0xdc, 0x87, 0x77, 0x72, 0xc7, + 0x2c, 0x09, 0x7d, 0x89, 0x1e, 0xc3, 0xcb, 0x65, 0xe7, 0x75, 0xed, 0x9c, 0x1f, 0xfe, 0x44, 0x53, + 0x1e, 0xe3, 0xe5, 0x7f, 0xf8, 0xf3, 0x38, 0x5c, 0xf6, 0xdf, 0x78, 0xd3, 0xff, 0x24, 0x0e, 0x17, + 0xfe, 0x8b, 0xc0, 0x43, 0x8c, 0x51, 0xe2, 0xb2, 0x6f, 0x75, 0x39, 0xf0, 0x63, 0x0d, 0x17, 0x8e, + 0x9d, 0x5f, 0x2c, 0x68, 0xbb, 0x8c, 0xa1, 0x30, 0x0a, 0x1f, 0x71, 0x76, 0x16, 0x9d, 0x93, 0x1d, + 0x58, 0x17, 0x59, 0x8c, 0x69, 0xae, 0xb0, 0xd9, 0x90, 0x4f, 0x60, 0x3b, 0xbf, 0x44, 0x4e, 0x99, + 0x97, 0x4a, 0x5f, 0x48, 0x4f, 0xa6, 0x5a, 0xe6, 0x35, 0xda, 0x32, 0xd0, 0x70, 0xca, 0x06, 0x0a, + 0x18, 0xa6, 0xe4, 0x4b, 0x68, 0x2c, 0xe5, 0x2e, 0xd5, 0x6a, 0xd7, 0x0f, 0x9d, 0x5e, 0x9e, 0xf9, + 0xde, 0x6b, 0x89, 0xa5, 0x25, 0x76, 0xe7, 0x57, 0x0b, 0x1a, 0xa5, 0x98, 0x3e, 0x84, 0x66, 0xe0, + 0xa7, 0x38, 0x40, 0x96, 0x46, 0x32, 0x9a, 0xa0, 0x63, 0xed, 0x5b, 0xdd, 0x1a, 0x2d, 0x1b, 0xc9, + 0x4d, 0xd8, 0x3c, 0xe3, 0x22, 0x40, 0x8a, 0x49, 0x1c, 0x05, 0xbe, 0x44, 0xa7, 0xa2, 0x69, 0xaf, + 0x59, 0xc9, 0x7d, 0x68, 0x9c, 0x2d, 0x9d, 0xee, 0xac, 0xee, 0x5b, 0xdd, 0xfa, 0xe1, 0xf5, 0x22, + 0xb8, 0x37, 0x34, 0xa1, 0x25, 0x7e, 0xa7, 0x01, 0x40, 0x31, 0xe5, 0xf1, 0x04, 0xc3, 0x61, 0xda, + 0xc9, 0x60, 0xdd, 0xd4, 0x57, 0x0b, 0x56, 0x9f, 0xe1, 0x4c, 0x87, 0xd6, 0xa0, 0x6a, 0xa9, 0xa4, + 0xd4, 0xb9, 0xd0, 0x71, 0x34, 0xa8, 0xd9, 0x90, 0xeb, 0x50, 0x9b, 0xe7, 0x4f, 0x5f, 0xdd, 0xa0, + 0xc5, 0x9e, 0x74, 0xa1, 0xca, 0x13, 0x4f, 0xce, 0x12, 0xd4, 0x35, 0xb7, 0x79, 0xb8, 0x55, 0x44, + 0x75, 0x92, 0x0c, 0x67, 0x09, 0xd2, 0x0d, 0xae, 0x7f, 0x3b, 0x3f, 0x42, 0x6d, 0x38, 0x65, 0xe6, + 0xe6, 0x9b, 0xb0, 0xa1, 0x59, 0x26, 0x67, 0xf5, 0xc3, 0xcd, 0xb2, 0xce, 0x34, 0x47, 0xc9, 0x0d, + 0xb0, 0x03, 0x3e, 0x1e, 0x47, 0x79, 0xea, 0xac, 0xee, 0x1a, 0xad, 0x19, 0xc3, 0x30, 0x25, 0xd7, + 0xa0, 0x56, 0xa4, 0x75, 0x55, 0x63, 0xd5, 0xd4, 0x64, 0xb3, 0x53, 0x07, 0x7b, 0xe8, 0x8f, 0x62, + 0x74, 0xd9, 0x19, 0xef, 0xfc, 0x6d, 0x81, 0x6d, 0xb2, 0x85, 0x18, 0x92, 0xdb, 0x00, 0xaa, 0x20, + 0x4a, 0xd7, 0xb7, 0x8b, 0xeb, 0xe7, 0x11, 0x52, 0x5b, 0xe6, 0xab, 0x94, 0xbc, 0x07, 0x75, 0x91, + 0xab, 0xb7, 0x08, 0x03, 0x44, 0x21, 0x28, 0xb9, 0x0f, 0xcd, 0x30, 0x4a, 0x13, 0xd3, 0xd8, 0x5e, + 0x14, 0xe6, 0xf9, 0xb9, 0xd6, 0x5b, 0x9a, 0x16, 0xbd, 0xe3, 0x82, 0xe1, 0x1e, 0xd3, 0xc6, 0x82, + 0xef, 0x86, 0xba, 0x80, 0x7d, 0x19, 0x71, 0xad, 0x60, 0x85, 0x9a, 0x0d, 0xf9, 0x14, 0x40, 0xaa, + 0x37, 0x78, 0x11, 0x3b, 0xe3, 0xba, 0x27, 0xeb, 0x87, 0x64, 0x11, 0xe8, 0xfc, 0x79, 0xd4, 0x96, + 0xc5, 0x4b, 0x67, 0xb0, 0xe5, 0x32, 0x89, 0xe7, 0x22, 0x92, 0xb3, 0xbc, 0x10, 0x6f, 0xc3, 0xf6, + 0xc2, 0x74, 0x81, 0xc1, 0xb3, 0x6f, 0x71, 0x82, 0xb1, 0xce, 0xb9, 0x4d, 0xdf, 0x06, 0x91, 0x3b, + 0xb0, 0x7b, 0xc4, 0x85, 0xc8, 0x12, 0x19, 0x71, 0xf6, 0x8d, 0xcf, 0xc2, 0x18, 0x8d, 0x4f, 0xc5, + 0xb4, 0xe6, 0x5b, 0xc1, 0xce, 0xef, 0x1b, 0xd0, 0x5e, 0x3c, 0x91, 0xe2, 0xf3, 0x0c, 0x53, 0x3d, + 0xc1, 0x82, 0x38, 0x4b, 0xa5, 0x91, 0xc5, 0xd2, 0xca, 0xd9, 0xb9, 0xc5, 0x0d, 0x95, 0x70, 0xc1, + 0x85, 0xcf, 0xce, 0xf1, 0x0c, 0x31, 0x54, 0x8c, 0xca, 0x5b, 0x84, 0x3b, 0x2a, 0x18, 0x4a, 0xb8, + 0x05, 0xdf, 0xf8, 0xff, 0x2f, 0xe1, 0xef, 0xce, 0x25, 0x4e, 0x13, 0x9f, 0x69, 0xf5, 0xeb, 0x87, + 0x57, 0x4a, 0xce, 0x5a, 0xe6, 0x41, 0xe2, 0xb3, 0x5c, 0x66, 0xb5, 0x2c, 0x15, 0xde, 0x7a, 0xa9, + 0xf0, 0x54, 0xc1, 0xa6, 0x28, 0x26, 0x26, 0x1a, 0x33, 0x07, 0x6b, 0xc6, 0xe0, 0x86, 0xe4, 0x0e, + 0xd4, 0xfd, 0x40, 0x09, 0x67, 0xfa, 0xa5, 0xaa, 0xfb, 0x65, 0xbb, 0x48, 0xe9, 0x03, 0x8d, 0xe9, + 0x9e, 0x01, 0xbf, 0x58, 0x93, 0x7b, 0xd0, 0x34, 0xcd, 0xec, 0x05, 0xa6, 0xfb, 0x6b, 0x3a, 0xce, + 0xdd, 0xc2, 0xef, 0xdf, 0x1b, 0x9f, 0xdc, 0x82, 0x36, 0x32, 0xf3, 0xc2, 0x19, 0x0b, 0xbc, 0x84, + 0x47, 0x4c, 0x3a, 0xb6, 0x9e, 0x31, 0x5b, 0x06, 0x18, 0xcc, 0x58, 0x70, 0xaa, 0xcc, 0xa4, 0x03, + 0xcd, 0x05, 0x49, 0x3d, 0x0d, 0xf4, 0xd3, 0xea, 0xe9, 0x9c, 0x31, 0x4c, 0x49, 0x0f, 0xb6, 0x97, + 0x38, 0x11, 0x93, 0x28, 0x26, 0x7e, 0xec, 0xd4, 0x35, 0xb3, 0x5d, 0x30, 0xdd, 0x1c, 0x50, 0xf9, + 0xe7, 0x2c, 0x9e, 0x79, 0x02, 0xb3, 0x14, 0x9d, 0x86, 0xbe, 0xd8, 0x56, 0x16, 0xaa, 0x0c, 0x4a, + 0xc8, 0x51, 0x28, 0xbc, 0x31, 0x0f, 0xd1, 0x69, 0x6a, 0xb0, 0x3a, 0x0a, 0xc5, 0x23, 0x1e, 0x22, + 0xf9, 0x1c, 0xec, 0x68, 0x5e, 0x9c, 0xce, 0xa6, 0x7e, 0xb1, 0xb3, 0x34, 0xef, 0x4a, 0x45, 0x4e, + 0x17, 0x54, 0x35, 0xab, 0x64, 0x34, 0xc6, 0x9f, 0x38, 0x43, 0x67, 0xcb, 0xe8, 0x3f, 0xdf, 0xab, + 0x3e, 0xc3, 0x84, 0x07, 0x17, 0x4e, 0x4b, 0xc7, 0x6b, 0x36, 0xe4, 0x2e, 0x5c, 0xe5, 0x99, 0x4c, + 0x32, 0xe9, 0x09, 0xff, 0xd2, 0x33, 0xf5, 0x95, 0x7f, 0x93, 0xdb, 0x3a, 0xa6, 0x1d, 0x03, 0x53, + 0xff, 0xd2, 0x94, 0xa2, 0x19, 0x61, 0x04, 0xd6, 0x74, 0xdc, 0x64, 0xdf, 0xea, 0xae, 0x52, 0xbd, + 0x26, 0x1f, 0x40, 0x53, 0xcd, 0x16, 0x5f, 0xf2, 0x71, 0x14, 0xa8, 0xc0, 0xb7, 0x75, 0x04, 0x0d, + 0x39, 0x65, 0x0f, 0xe6, 0x36, 0xf2, 0x31, 0x90, 0x79, 0x4e, 0x02, 0x9f, 0x79, 0x97, 0x11, 0x0b, + 0xf9, 0xa5, 0xb3, 0xa3, 0xaf, 0x6a, 0xe5, 0x49, 0x09, 0x7c, 0xf6, 0x9d, 0xb6, 0xdf, 0xfa, 0x08, + 0x36, 0xcc, 0x1c, 0x25, 0x4d, 0xb0, 0xcd, 0xea, 0x34, 0x93, 0xad, 0x15, 0xd2, 0x82, 0x86, 0xd9, + 0x9a, 0x8f, 0x64, 0xcb, 0xba, 0x25, 0x00, 0x16, 0x25, 0x44, 0x6e, 0xc0, 0xd5, 0x07, 0x47, 0x43, + 0xf7, 0xe4, 0xb1, 0x37, 0xfc, 0xfe, 0xb4, 0xef, 0x3d, 0x79, 0x3c, 0x38, 0xed, 0x1f, 0xb9, 0x0f, + 0xdd, 0xfe, 0x71, 0x6b, 0x85, 0x38, 0xb0, 0xb3, 0x0c, 0xd2, 0xfe, 0xd7, 0xee, 0x60, 0xd8, 0xa7, + 0x2d, 0x8b, 0x5c, 0x01, 0x52, 0x46, 0x1e, 0x9d, 0x3c, 0xed, 0xb7, 0x2a, 0x64, 0x17, 0xda, 0x65, + 0xfb, 0xa0, 0x3f, 0x6c, 0xad, 0x7f, 0x75, 0xef, 0xb7, 0x97, 0x7b, 0xd6, 0x8b, 0x97, 0x7b, 0xd6, + 0x5f, 0x2f, 0xf7, 0xac, 0x9f, 0x5f, 0xed, 0xad, 0xbc, 0x78, 0xb5, 0xb7, 0xf2, 0xc7, 0xab, 0xbd, + 0x95, 0x1f, 0xf6, 0xcf, 0x23, 0x79, 0x91, 0x8d, 0x7a, 0x01, 0x1f, 0x1f, 0x24, 0x11, 0x3b, 0x0f, + 0xfc, 0xe4, 0x40, 0x46, 0x41, 0x18, 0x1c, 0xe4, 0x59, 0x1c, 0x6d, 0xe8, 0xbf, 0x4d, 0x9f, 0xfd, + 0x13, 0x00, 0x00, 0xff, 0xff, 0x5a, 0xb3, 0xd4, 0x66, 0x73, 0x09, 0x00, 0x00, } func (m *EventFilterRule) Marshal() (dAtA []byte, err error) { @@ -1376,6 +1388,18 @@ func (m *DispatcherRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.EnableScanWindow { + i-- + if m.EnableScanWindow { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa0 + } if len(m.TxnAtomicity) > 0 { i -= len(m.TxnAtomicity) copy(dAtA[i:], m.TxnAtomicity) @@ -1824,6 +1848,9 @@ func (m *DispatcherRequest) Size() (n int) { if l > 0 { n += 2 + l + sovEvent(uint64(l)) } + if m.EnableScanWindow { + n += 3 + } return n } @@ -3636,6 +3663,26 @@ func (m *DispatcherRequest) Unmarshal(dAtA []byte) error { } m.TxnAtomicity = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 20: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EnableScanWindow", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.EnableScanWindow = bool(v != 0) default: iNdEx = preIndex skippy, err := skipEvent(dAtA[iNdEx:]) diff --git a/eventpb/event.proto b/eventpb/event.proto index ff9b155aef..6e836992d1 100644 --- a/eventpb/event.proto +++ b/eventpb/event.proto @@ -100,4 +100,8 @@ message DispatcherRequest { bool output_raw_change_event = 17; int64 mode = 18; string txn_atomicity = 19; + // enable_scan_window controls whether the event service applies the adaptive + // scan window (memory control + scan interval) for this changefeed. + // It defaults to false so the feature behaves as if it was never introduced. + bool enable_scan_window = 20; } diff --git a/pkg/config/changefeed.go b/pkg/config/changefeed.go index 00ba876dd1..aba37c2a2d 100644 --- a/pkg/config/changefeed.go +++ b/pkg/config/changefeed.go @@ -204,6 +204,9 @@ type ChangefeedConfig struct { // redo releated Consistent *ConsistentConfig `toml:"consistent" json:"consistent,omitempty"` EnableTableAcrossNodes bool `toml:"enable-table-across-nodes" json:"enable-table-across-nodes,omitempty"` + // EnableScanWindow controls whether the event service applies the adaptive scan + // window (memory control + adaptive scan interval) for this changefeed. + EnableScanWindow bool `json:"enable_scan_window" default:"false"` } // String implements fmt.Stringer interface, but hide some sensitive information @@ -280,6 +283,7 @@ func (info *ChangeFeedInfo) ToChangefeedConfig() *ChangefeedConfig { TimeZone: GetGlobalServerConfig().TZ, Consistent: info.Config.Consistent, EnableTableAcrossNodes: util.GetOrZero(info.Config.Scheduler.EnableTableAcrossNodes), + EnableScanWindow: util.GetOrZero(info.Config.EnableScanWindow), // other fields are not necessary for dispatcherManager } } diff --git a/pkg/config/replica_config.go b/pkg/config/replica_config.go index 9aa58ee7b7..9386178219 100644 --- a/pkg/config/replica_config.go +++ b/pkg/config/replica_config.go @@ -51,6 +51,7 @@ var defaultReplicaConfig = &ReplicaConfig{ SyncPointInterval: util.AddressOf(10 * time.Minute), SyncPointRetention: util.AddressOf(24 * time.Hour), BDRMode: util.AddressOf(false), + EnableScanWindow: util.AddressOf(false), Filter: NewDefaultFilterConfig(), Mounter: &MounterConfig{ WorkerNum: 16, @@ -172,6 +173,11 @@ type replicaConfig struct { ChangefeedErrorStuckDuration *time.Duration `toml:"changefeed-error-stuck-duration" json:"changefeed-error-stuck-duration,omitempty"` SyncedStatus *SyncedStatusConfig `toml:"synced-status" json:"synced-status,omitempty"` + // EnableScanWindow controls whether the event service applies the adaptive scan + // window (memory control + adaptive scan interval) for this changefeed. + // It defaults to false so the feature behaves as if it was never introduced. + EnableScanWindow *bool `toml:"enable-scan-window" json:"enable-scan-window,omitempty"` + // Deprecated: we don't use this field since v8.0.0. SQLMode string `toml:"sql-mode" json:"sql-mode"` } diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 4fbf45f9a6..8379eca259 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -436,13 +436,19 @@ type changefeedStatus struct { scanWindowController *adaptiveScanWindowController syncPointInterval time.Duration + + // enableScanWindow controls whether the adaptive scan window (memory control + + // adaptive scan interval) takes effect for this changefeed. When false, the + // feature behaves as if it was never introduced. + enableScanWindow bool } -func newChangefeedStatus(changefeedID common.ChangeFeedID, syncPointInterval time.Duration) *changefeedStatus { +func newChangefeedStatus(changefeedID common.ChangeFeedID, syncPointInterval time.Duration, enableScanWindow bool) *changefeedStatus { status := &changefeedStatus{ changefeedID: changefeedID, scanWindowController: newAdaptiveScanWindowController(time.Now()), syncPointInterval: syncPointInterval, + enableScanWindow: enableScanWindow, } status.scanInterval.Store(int64(defaultScanInterval)) diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index dff6ff6085..005349a396 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -450,7 +450,7 @@ func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRang } } - if dataRange.CommitTsEnd <= dataRange.CommitTsStart && hasPendingDDLEventInCurrentRange { + if task.changefeedStat.enableScanWindow && dataRange.CommitTsEnd <= dataRange.CommitTsStart && hasPendingDDLEventInCurrentRange { // Global scan window base can be pinned by other lagging dispatchers. // For a table with pending ddl in current range, use a local bounded step to keep // this dispatcher making forward progress, so barrier coverage can eventually complete. @@ -474,10 +474,15 @@ func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRang if dataRange.CommitTsEnd <= dataRange.CommitTsStart { updateMetricEventServiceSkipResolvedTsCount(task.info.GetMode()) - // Scan range can become empty after applying capping (for example, scan window). - // Send a signal resolved-ts event (rate limited) to keep downstream responsive, - // but do not advance the watermark here. - c.sendSignalResolvedTs(task) + // Scan range can become empty after applying scan-window capping. In that case send a + // signal resolved-ts event (rate limited) to keep downstream responsive, but do not + // advance the watermark here. + // When the scan window is disabled, an empty range can only come from the pre-existing + // DDL-state capping, where the baseline behavior is to just return without sending a + // signal. Gate the signal on enableScanWindow to keep that parity. + if task.changefeedStat.enableScanWindow { + c.sendSignalResolvedTs(task) + } return false, common.DataRange{} } @@ -1254,14 +1259,18 @@ func (c *eventBroker) getOrSetChangefeedStatus(info DispatcherInfo) *changefeedS zap.Error(err)) } - status := newChangefeedStatus(changefeedID, info.GetSyncPointInterval()) + status := newChangefeedStatus(changefeedID, info.GetSyncPointInterval(), info.GetEnableScanWindow()) status.filter = changefeedFilter actual, loaded := c.changefeedMap.LoadOrStore(changefeedID, status) if loaded { return actual.(*changefeedStatus) } log.Info("new changefeed status", zap.Stringer("changefeedID", changefeedID)) - initializeScanWindowMetrics(changefeedID.String()) + // Only emit scan-window metrics when the feature is enabled, so a changefeed + // running with the feature off stays silent on the scan-window dashboards. + if status.enableScanWindow { + initializeScanWindowMetrics(changefeedID.String()) + } return status } diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index bc934398ea..01ac91d3d4 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -948,3 +948,60 @@ func TestAddDispatcherFailure(t *testing.T) { _, ok := broker.changefeedMap.Load(dispInfo.GetChangefeedID()) require.False(t, ok, "changefeedStatus should be removed after failed registration") } + +// TestGetScanTaskDataRangeEmptySignalGatedByScanWindow verifies gate 6 of the +// enable-scan-window switch (see ai_context/switch_4030_4460_4950.md): when the scan +// range becomes empty due to the pre-existing DDL-state capping, the rate-limited +// signal resolved-ts is emitted only when the scan window is enabled. With the +// feature off the behavior matches the baseline (no signal is sent). +func TestGetScanTaskDataRangeEmptySignalGatedByScanWindow(t *testing.T) { + cases := []struct { + name string + enableScanWindow bool + wantSignal bool + }{ + {"enabled sends signal", true, true}, + {"disabled stays silent", false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + broker, _, ss, _ := newEventBrokerForTest() + // Close the broker so the send-message workers stop; emitted messages then + // stay buffered in messageCh and can be observed by length. + broker.close() + + info := newMockDispatcherInfoForTest(t) + info.epoch = 1 + // syncPointInterval=0 keeps emitSyncPointEventIfNeeded a no-op. + status := newChangefeedStatus(info.GetChangefeedID(), 0, tc.enableScanWindow) + disp := newDispatcherStat(info, 1, 1, nil, status) + disp.seq.Store(1) + + baseTime := time.Now() + commitStart := oracle.GoTimeToTS(baseTime.Add(20 * time.Second)) + disp.sentResolvedTs.Store(oracle.GoTimeToTS(baseTime)) + disp.receivedResolvedTs.Store(oracle.GoTimeToTS(baseTime.Add(40 * time.Second))) + disp.eventStoreCommitTs.Store(commitStart) + disp.lastScannedCommitTs.Store(commitStart) // CommitTsStart + disp.lastScannedStartTs.Store(0) // allow the signal resolved-ts + // Make sure the rate limiter does not suppress the signal. + disp.lastSentResolvedTsTime.Store(baseTime.Add(-time.Hour)) + + // DDL-state ResolvedTs caps CommitTsEnd down to CommitTsStart, producing an empty + // range regardless of the scan window. maxDDLCommitTs <= CommitTsStart avoids the + // pending-ddl local advance branch. + ss.resolvedTs = commitStart + ss.maxDDLCommitTs = commitStart + + needScan, _ := broker.getScanTaskDataRange(disp) + require.False(t, needScan) + + got := len(broker.messageCh[disp.messageWorkerIndex]) + if tc.wantSignal { + require.Equal(t, 1, got, "expected a signal resolved-ts message when scan window is enabled") + } else { + require.Equal(t, 0, got, "expected no message when scan window is disabled") + } + }) + } +} diff --git a/pkg/eventservice/event_service.go b/pkg/eventservice/event_service.go index efea08ea24..9532ad598d 100644 --- a/pkg/eventservice/event_service.go +++ b/pkg/eventservice/event_service.go @@ -53,6 +53,10 @@ type DispatcherInfo interface { GetSyncPointTs() uint64 GetSyncPointInterval() time.Duration + // GetEnableScanWindow reports whether the adaptive scan window feature is + // enabled for this changefeed. + GetEnableScanWindow() bool + IsOnlyReuse() bool GetBdrMode() bool GetIntegrity() *integrity.Config diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index e58508d10a..3d66d9e22f 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -400,6 +400,7 @@ type mockDispatcherInfo struct { enableSyncPoint bool nextSyncPoint uint64 syncPointInterval time.Duration + enableScanWindow bool } func newMockDispatcherInfo(t *testing.T, startTs uint64, dispatcherID common.DispatcherID, tableID int64, actionType eventpb.ActionType) *mockDispatcherInfo { @@ -423,6 +424,8 @@ func newMockDispatcherInfo(t *testing.T, startTs uint64, dispatcherID common.Dis }, bdrMode: false, integrity: config.GetDefaultReplicaConfig().Integrity, + // Tests exercise the scan window behavior, so enable it by default here. + enableScanWindow: true, } } @@ -474,6 +477,10 @@ func (m *mockDispatcherInfo) GetSyncPointInterval() time.Duration { return m.syncPointInterval } +func (m *mockDispatcherInfo) GetEnableScanWindow() bool { + return m.enableScanWindow +} + func (m *mockDispatcherInfo) IsOnlyReuse() bool { return false } @@ -505,7 +512,7 @@ func (m *mockDispatcherInfo) GetTxnAtomicity() config.AtomicityLevel { func newChangefeedStatusForTest(t testing.TB, info DispatcherInfo) *changefeedStatus { t.Helper() - status := newChangefeedStatus(info.GetChangefeedID(), info.GetSyncPointInterval()) + status := newChangefeedStatus(info.GetChangefeedID(), info.GetSyncPointInterval(), info.GetEnableScanWindow()) status.filter = newChangefeedFilterForTest(t, info, time.UTC.String()) return status } @@ -518,7 +525,7 @@ func addChangefeedStatusToBrokerForTest( ) *changefeedStatus { t.Helper() - status := newChangefeedStatus(changefeedID, syncPointInterval) + status := newChangefeedStatus(changefeedID, syncPointInterval, true) broker.changefeedMap.Store(changefeedID, status) return status } diff --git a/pkg/eventservice/scan_window.go b/pkg/eventservice/scan_window.go index 5bcb8e16c9..8c629475a5 100644 --- a/pkg/eventservice/scan_window.go +++ b/pkg/eventservice/scan_window.go @@ -296,6 +296,11 @@ func (w *memoryUsageWindow) pruneLocked(now time.Time) { } func (c *changefeedStatus) updateMemoryUsage(now time.Time, usageRatio float64, memoryReleaseCount uint32) { + // When the scan window feature is disabled, keep the scan interval untouched so + // the changefeed behaves as if the adaptive scan window was never introduced. + if !c.enableScanWindow { + return + } if c.scanWindowController == nil { return } @@ -811,6 +816,13 @@ func (c *changefeedStatus) maxScanInterval() time.Duration { } func (c *changefeedStatus) refreshMinSentResolvedTs() { + // When the scan window feature is disabled, minSentTs is never consumed + // (getScanMaxTs returns 0 unconditionally), so skip the per-changefeed scan and + // the base-ts gauge update entirely. This keeps the feature fully inert and + // avoids emitting scan-window metrics when it is off. + if !c.enableScanWindow { + return + } now := time.Now() minSentResolvedTs := ^uint64(0) minSentResolvedTsWithStale := ^uint64(0) @@ -854,6 +866,11 @@ func (c *changefeedStatus) refreshMinSentResolvedTs() { } func (c *changefeedStatus) getScanMaxTs() uint64 { + // When the scan window feature is disabled, return 0 so that the scan range is + // never capped, behaving as if the scan window was never introduced. + if !c.enableScanWindow { + return 0 + } baseTs := c.minSentTs.Load() if baseTs == 0 { return 0 diff --git a/pkg/eventservice/scan_window_test.go b/pkg/eventservice/scan_window_test.go index 712fe9d789..56f050088b 100644 --- a/pkg/eventservice/scan_window_test.go +++ b/pkg/eventservice/scan_window_test.go @@ -37,7 +37,7 @@ func markScanWindowReadyForDecrease(status *changefeedStatus, now time.Time) { func TestAdjustScanIntervalLowPressureSlowsRecoveryForLargeWindow(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) now := time.Now() markScanWindowReadyForIncrease(status, now) @@ -52,7 +52,7 @@ func TestAdjustScanIntervalLowPressureSlowsRecoveryForLargeWindow(t *testing.T) func TestAdjustScanIntervalVeryLowPressureSlowsRecoveryForVeryLargeWindow(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) now := time.Now() markScanWindowReadyForIncrease(status, now) @@ -67,7 +67,7 @@ func TestAdjustScanIntervalVeryLowPressureSlowsRecoveryForVeryLargeWindow(t *tes func TestAdjustScanIntervalHighPressureUsesBoundedReduction(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) now := time.Now() markScanWindowReadyForDecrease(status, now) @@ -79,7 +79,7 @@ func TestAdjustScanIntervalHighPressureUsesBoundedReduction(t *testing.T) { func TestAdjustScanIntervalDoesNotKeepReducingAfterTransientHighPressure(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute, true) changefeed := status.changefeedID.String() t.Cleanup(func() { deleteScanWindowMetrics(changefeed) @@ -101,7 +101,7 @@ func TestAdjustScanIntervalDoesNotKeepReducingAfterTransientHighPressure(t *test func TestAdjustScanIntervalCriticalPressure(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) status.scanInterval.Store(int64(40 * time.Second)) status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) require.Equal(t, int64(scanWindowEmergencyBrakePlateauInterval), status.scanInterval.Load()) @@ -110,7 +110,7 @@ func TestAdjustScanIntervalCriticalPressure(t *testing.T) { func TestAdjustScanIntervalCriticalPressureIgnoresLowPressureHistory(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 10*time.Minute, true) changefeed := status.changefeedID.String() t.Cleanup(func() { deleteScanWindowMetrics(changefeed) @@ -130,7 +130,7 @@ func TestAdjustScanIntervalCriticalPressureIgnoresLowPressureHistory(t *testing. func TestAdjustScanIntervalCriticalPressureUsesDefaultFloor(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) status.scanInterval.Store(int64(8 * time.Second)) status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 0.95, 0) require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) @@ -139,7 +139,7 @@ func TestAdjustScanIntervalCriticalPressureUsesDefaultFloor(t *testing.T) { func TestAdjustScanIntervalHighPressureDoesNotIncreaseBelowDefaultFloor(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) now := time.Now() markScanWindowReadyForDecrease(status, now) @@ -151,7 +151,7 @@ func TestAdjustScanIntervalHighPressureDoesNotIncreaseBelowDefaultFloor(t *testi func TestAdjustScanIntervalCriticalPressureDoesNotIncreaseBelowDefaultFloor(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) status.scanInterval.Store(int64(2 * time.Second)) status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 0.95, 0) require.Equal(t, int64(2*time.Second), status.scanInterval.Load()) @@ -160,7 +160,7 @@ func TestAdjustScanIntervalCriticalPressureDoesNotIncreaseBelowDefaultFloor(t *t func TestAdjustScanIntervalEmergencyPressureUsesModerateBrakeForSmallWindow(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) status.scanInterval.Store(int64(20 * time.Second)) status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) require.Equal(t, int64(10*time.Second), status.scanInterval.Load()) @@ -183,7 +183,7 @@ func TestScanWindowEmergencyBrakeIntervalUsesStrongBrakeForLargeWindow(t *testin func TestAdjustScanIntervalEmergencyPressureUsesDefaultFloorForVerySmallWindow(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) status.scanInterval.Store(int64(8 * time.Second)) status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) @@ -192,7 +192,7 @@ func TestAdjustScanIntervalEmergencyPressureUsesDefaultFloorForVerySmallWindow(t func TestAdjustScanIntervalEmergencyPressureDoesNotImmediatelyDropBelowDefaultFloor(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) status.scanInterval.Store(int64(defaultScanInterval)) status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) require.Equal(t, int64(defaultScanInterval), status.scanInterval.Load()) @@ -201,7 +201,7 @@ func TestAdjustScanIntervalEmergencyPressureDoesNotImmediatelyDropBelowDefaultFl func TestAdjustScanIntervalEmergencyPressureDoesNotIncreaseBelowDefaultFloor(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) status.scanInterval.Store(int64(2 * time.Second)) status.updateMemoryUsage(time.Now().Add(memoryUsageWindowDuration), 1, 0) require.Equal(t, int64(2*time.Second), status.scanInterval.Load()) @@ -210,7 +210,7 @@ func TestAdjustScanIntervalEmergencyPressureDoesNotIncreaseBelowDefaultFloor(t * func TestAdjustScanIntervalEmergencyPressureCanReachMinFloorWhenSustained(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) status.scanInterval.Store(int64(defaultScanInterval)) start := time.Now() @@ -224,7 +224,7 @@ func TestAdjustScanIntervalEmergencyPressureCanReachMinFloorWhenSustained(t *tes func TestAdjustScanIntervalRecoversFromFloorBeforeNormalIncreaseCooldown(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 10*time.Minute, true) now := time.Now() status.scanInterval.Store(int64(defaultScanInterval)) status.scanWindowController.setLastAdjustTimeForTest(now.Add(-scanWindowFloorRecoveryCooldown - time.Second)) @@ -239,7 +239,7 @@ func TestAdjustScanIntervalRecoversFromFloorBeforeNormalIncreaseCooldown(t *test func TestUpdateMemoryUsageDoesNotResetScanIntervalOnMemoryRelease(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) now := time.Now() status.scanInterval.Store(int64(40 * time.Second)) @@ -248,7 +248,7 @@ func TestUpdateMemoryUsageDoesNotResetScanIntervalOnMemoryRelease(t *testing.T) } func TestUpdateMemoryUsageRecordsScanWindowObservationMetrics(t *testing.T) { - status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute, true) changefeed := status.changefeedID.String() t.Cleanup(func() { deleteScanWindowMetrics(changefeed) @@ -272,7 +272,7 @@ func TestUpdateMemoryUsageRecordsScanWindowObservationMetrics(t *testing.T) { } func TestUpdateMemoryUsageRecordsScanWindowAdjustCount(t *testing.T) { - status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute, true) changefeed := status.changefeedID.String() t.Cleanup(func() { deleteScanWindowMetrics(changefeed) @@ -289,7 +289,7 @@ func TestUpdateMemoryUsageRecordsScanWindowAdjustCount(t *testing.T) { } func TestUpdateMemoryUsageRecordsScanWindowTargetBandMetrics(t *testing.T) { - status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 10*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 10*time.Minute, true) changefeed := status.changefeedID.String() t.Cleanup(func() { deleteScanWindowMetrics(changefeed) @@ -311,7 +311,7 @@ func TestUpdateMemoryUsageRecordsScanWindowTargetBandMetrics(t *testing.T) { func TestAdjustScanIntervalIncreaseWithJitteredSamples(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) start := time.Now() markScanWindowReadyForIncrease(status, start) @@ -330,7 +330,7 @@ func TestAdjustScanIntervalIncreaseWithJitteredSamples(t *testing.T) { func TestAdjustScanIntervalReducesOnSustainedPressure(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) now := time.Now() markScanWindowReadyForDecrease(status, now) @@ -345,7 +345,7 @@ func TestAdjustScanIntervalReducesOnSustainedPressure(t *testing.T) { func TestAdjustScanIntervalSustainedPressureDoesNotIncreaseBelowDefaultFloor(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) now := time.Now() markScanWindowReadyForDecrease(status, now) @@ -360,7 +360,7 @@ func TestAdjustScanIntervalSustainedPressureDoesNotIncreaseBelowDefaultFloor(t * func TestAdjustScanIntervalDoesNotIncreaseBeforeCooldown(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) now := time.Now() status.scanInterval.Store(int64(40 * time.Second)) @@ -373,7 +373,7 @@ func TestAdjustScanIntervalDoesNotIncreaseBeforeCooldown(t *testing.T) { func TestRefreshMinSentResolvedTsMinAndSkipRules(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) stale := &dispatcherStat{} stale.seq.Store(1) @@ -433,7 +433,7 @@ func TestRefreshMinSentResolvedTsMinAndSkipRules(t *testing.T) { func TestRefreshMinSentResolvedTsStaleFallback(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) stale := &dispatcherStat{} stale.seq.Store(1) @@ -451,7 +451,7 @@ func TestRefreshMinSentResolvedTsStaleFallback(t *testing.T) { func TestGetScanMaxTsFallbackInterval(t *testing.T) { t.Parallel() - status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute) + status := newChangefeedStatus(common.NewChangefeedID4Test("default", "test"), 1*time.Minute, true) baseTime := time.Unix(1234, 0) baseTs := oracle.GoTimeToTS(baseTime) @@ -466,3 +466,31 @@ func TestGetScanMaxTsFallbackInterval(t *testing.T) { status.minSentTs.Store(0) require.Equal(t, uint64(0), status.getScanMaxTs()) } + +// TestScanWindowDisabledIsNoOp verifies that when the scan window feature is +// disabled (the default), getScanMaxTs never caps the range and updateMemoryUsage +// keeps the scan interval untouched, so the changefeed behaves as if the feature +// was never introduced. +func TestScanWindowDisabledIsNoOp(t *testing.T) { + t.Parallel() + + status := newChangefeedStatus(common.NewChangefeedID4Test("default", t.Name()), 1*time.Minute, false) + now := time.Now() + + // Even with a non-zero base ts, the scan range must not be capped. + status.minSentTs.Store(oracle.GoTimeToTS(now)) + require.Equal(t, uint64(0), status.getScanMaxTs()) + + // Memory usage reports must not adjust the scan interval. + status.scanInterval.Store(int64(40 * time.Second)) + markScanWindowReadyForDecrease(status, now) + status.updateMemoryUsage(now.Add(memoryUsageWindowDuration), 1.0, 0) + require.Equal(t, int64(40*time.Second), status.scanInterval.Load()) + + // refreshMinSentResolvedTs must be a no-op: it leaves minSentTs untouched + // (an enabled status would recompute it to 0 here since there is no dispatcher). + sentinel := oracle.GoTimeToTS(now) + status.minSentTs.Store(sentinel) + status.refreshMinSentResolvedTs() + require.Equal(t, sentinel, status.minSentTs.Load()) +} diff --git a/utils/dynstream/interfaces.go b/utils/dynstream/interfaces.go index 9be424e1e6..cc5bd41129 100644 --- a/utils/dynstream/interfaces.go +++ b/utils/dynstream/interfaces.go @@ -250,6 +250,17 @@ type AreaSettings struct { feedbackInterval time.Duration // The interval of the feedback. By default 1000ms. // Remove it when we determine the v2 is working well. algorithm int // The algorithm of the memory control. + + // releaseMemoryRatio overrides the fraction of pending memory released on each + // release cycle. A value <= 0 falls back to defaultReleaseMemoryRatio, keeping the + // pre-scan-window behavior. + releaseMemoryRatio float64 +} + +// SetReleaseMemoryRatio overrides the fraction of pending memory the area releases on +// each release cycle. A value <= 0 keeps the default (defaultReleaseMemoryRatio). +func (s *AreaSettings) SetReleaseMemoryRatio(ratio float64) { + s.releaseMemoryRatio = ratio } func (s *AreaSettings) fix() { diff --git a/utils/dynstream/memory_control.go b/utils/dynstream/memory_control.go index 170ec91ce3..d474aaf85b 100644 --- a/utils/dynstream/memory_control.go +++ b/utils/dynstream/memory_control.go @@ -33,7 +33,14 @@ const ( // For now, we only use it in event collector. MemoryControlForEventCollector = 1 - defaultReleaseMemoryRatio = 0.6 + // defaultReleaseMemoryRatio is the fraction of an area's pending memory released on + // each release cycle when the area does not override it (i.e. the adaptive scan + // window is disabled). It matches the pre-scan-window behavior. + defaultReleaseMemoryRatio = 0.4 + // deadlockMemoryHighWaterMark is the memory usage ratio above which a stalled area + // is treated as deadlocked. It is independent of the release ratio and stays the + // same regardless of whether the scan window is enabled. + deadlockMemoryHighWaterMark = 0.6 defaultDeadlockDuration = 5 * time.Second defaultReleaseMemoryThreshold = 256 ) @@ -171,7 +178,7 @@ func (as *areaMemStat[A, P, T, D, H]) checkDeadlock() bool { hasEventComeButNotOut := time.Since(as.lastAppendEventTime.Load().(time.Time)) < defaultDeadlockDuration && time.Since(as.lastSizeDecreaseTime.Load().(time.Time)) > defaultDeadlockDuration - memoryHighWaterMark := as.memoryUsageRatio() > defaultReleaseMemoryRatio + memoryHighWaterMark := as.memoryUsageRatio() > deadlockMemoryHighWaterMark return hasEventComeButNotOut && memoryHighWaterMark } @@ -193,11 +200,12 @@ func (as *areaMemStat[A, P, T, D, H]) releaseMemory() { return paths[i].lastHandleEventTs.Load() > paths[j].lastHandleEventTs.Load() }) - sizeToRelease := int64(float64(as.totalPendingSize.Load()) * defaultReleaseMemoryRatio) + releaseRatio := as.releaseMemoryRatio() + sizeToRelease := int64(float64(as.totalPendingSize.Load()) * releaseRatio) releasedSize := int64(0) releasedPaths := make([]*pathInfo[A, P, T, D, H], 0) - log.Info("release memory", zap.Any("area", as.area), zap.Int64("sizeToRelease", sizeToRelease), zap.Int64("totalPendingSize", as.totalPendingSize.Load()), zap.Float64("releaseMemoryRatio", defaultReleaseMemoryRatio)) + log.Info("release memory", zap.Any("area", as.area), zap.Int64("sizeToRelease", sizeToRelease), zap.Int64("totalPendingSize", as.totalPendingSize.Load()), zap.Float64("releaseMemoryRatio", releaseRatio)) for _, path := range paths { // Only release path that is blocking and has pending size larger than the threshold. @@ -227,6 +235,17 @@ func (as *areaMemStat[A, P, T, D, H]) memoryUsageRatio() float64 { return float64(as.totalPendingSize.Load()) / float64(as.settings.Load().maxPendingSize) } +// releaseMemoryRatio returns the fraction of pending memory to release on each cycle. +// It honors the per-area override (set when the scan window is enabled) and falls back +// to defaultReleaseMemoryRatio when unset, so a non-overriding area keeps the +// pre-scan-window behavior. +func (as *areaMemStat[A, P, T, D, H]) releaseMemoryRatio() float64 { + if ratio := as.settings.Load().releaseMemoryRatio; ratio > 0 { + return ratio + } + return defaultReleaseMemoryRatio +} + func (as *areaMemStat[A, P, T, D, H]) updateAreaPauseState(path *pathInfo[A, P, T, D, H]) { pause, resume, memoryUsageRatio := as.algorithm.ShouldPauseArea( as.paused.Load(),