From 6d2adef45c121b59d77514bb6f53b7b55c7a18f3 Mon Sep 17 00:00:00 2001 From: wlwilliamx <53336371+wlwilliamx@users.noreply.github.com> Date: Mon, 8 Jun 2026 13:11:56 +0800 Subject: [PATCH 1/2] This is an automated cherry-pick of #4278 Signed-off-by: ti-chi-bot --- pkg/sink/sqlmodel/row_change_test.go | 6 +- .../_utils/start_tidb_cluster_nextgen | 51 ++- .../start_tls_tidb_cluster_impl_nextgen | 20 +- .../_utils/stop_tidb_cluster | 4 + tools/workload/schema/bank3/bank.go | 366 ++++++++++++++++++ 5 files changed, 413 insertions(+), 34 deletions(-) create mode 100644 tools/workload/schema/bank3/bank.go diff --git a/pkg/sink/sqlmodel/row_change_test.go b/pkg/sink/sqlmodel/row_change_test.go index f6d67c5e27..ee9fe2d5a5 100644 --- a/pkg/sink/sqlmodel/row_change_test.go +++ b/pkg/sink/sqlmodel/row_change_test.go @@ -275,9 +275,9 @@ func (s *dpanicSuite) TestGenUpdate() { func (s *dpanicSuite) TestExpressionIndex() { source := &common.TableName{Schema: "db", Table: "tb1"} sql := `CREATE TABLE tb1 ( - id INT PRIMARY KEY, - j JSON, - UNIQUE KEY j_index ((cast(json_extract(j,'$[*]') as signed array)), id) + id INT PRIMARY KEY, + j JSON, + UNIQUE KEY j_index ((cast(json_extract(j,'$[*]') as signed array)), id) )` ti := mockTableInfo(s.T(), sql) change := NewRowChange(source, nil, nil, []interface{}{1, `[1,2,3]`, nil}, ti, nil, nil) diff --git a/tests/integration_tests/_utils/start_tidb_cluster_nextgen b/tests/integration_tests/_utils/start_tidb_cluster_nextgen index 2f91b3af27..189f1d71a6 100755 --- a/tests/integration_tests/_utils/start_tidb_cluster_nextgen +++ b/tests/integration_tests/_utils/start_tidb_cluster_nextgen @@ -13,8 +13,10 @@ TEST_DATA_DIR="$CASE_DIR/$TEST_OUT_DATA_DIR" echo "TEST_DATA_DIR: $TEST_DATA_DIR" -# Random generate the sockets config. -# Make sure we dont use the same sock. +# Random generate the `socket` config for TiDB. +# TiDB parses `socket = ...` as a root-level key. Our generated config ends with +# `[instance]`, so appending `socket = ...` at the end would become `instance.socket`, +# which is rejected by newer TiDB versions (and makes TiDB fail to start). randomGenSocketsConf() { config_file="$1" random_str=$(date '+%s%N') @@ -22,7 +24,13 @@ randomGenSocketsConf() { random_str=$(cat /dev/random | LC_ALL=C tr -dc "a-zA-Z0-9" | head -c 10) fi - echo "socket = \"/tmp/tidb-$random_str.sock\"" >>"$config_file" + # Prepend the root-level `socket` and drop any existing `socket = ...` lines + # (old scripts appended it under `[instance]`, which becomes `instance.socket`). + { + echo "socket = \"/tmp/tidb-$random_str.sock\"" + sed '/^[[:space:]]*socket[[:space:]]*=/d' "$config_file" + } >"$config_file.tmp" + mv "$config_file.tmp" "$config_file" } check_bin() { @@ -350,6 +358,13 @@ check_port_available "$DOWN_TIKV_WORKER_HOST" "$DOWN_TIKV_WORKER_PORT" "Downstre echo "Waiting for next-gen keyspace pre-allocation to settle..." sleep 30 +<<<<<<< HEAD +======= +# NOTE: TiDB config keys are section-sensitive. Keep `socket` and +# `max-server-connections` at root level, and keep tuning knobs like +# `run-auto-analyze` / `server-memory-quota` under `[performance]`. Otherwise +# TiDB will report "invalid configuration options" and silently ignore them. +>>>>>>> bef9fba03 (tests: fix next gen integration TiDB startup flakiness (#4278)) touch "$OUT_DIR/upstream-tidb-system.toml" if [ -f "$tidb_config" ]; then cat "$tidb_config" >>"$OUT_DIR/upstream-tidb-system.toml" @@ -359,9 +374,7 @@ cat >>"$OUT_DIR/upstream-tidb-system.toml" <>"$OUT_DIR/upstream-tidb-$KEYSPACE_NAME.toml" <>"$OUT_DIR/downstream-tidb-system.toml" <>"$OUT_DIR/downstream-tidb-$KEYSPACE_NAME.toml" <"$OUT_DIR/tidb-system-config-tls.toml" < 0 { + buf.WriteString(",") + } + + n := rand.Int63() + col1Value := "A01" + col2Value := "B2" + col3Value := fmt.Sprintf("acct-%d", n) + col4Value := randomBankDatetime() + + col5Value := fmt.Sprintf("%02d", rand.Intn(100)) + col6Value := fmt.Sprintf("desc-%d", n%1000000) + col7Value := fmt.Sprintf("%02d", rand.Intn(100)) + col8Value := fmt.Sprintf("branch-%d", n%1000000) + col9Value := fmt.Sprintf("%014d", n%100000000000000) + col10Value := rand.Intn(100) + col11Value := rand.Intn(10000) + col12Value := fmt.Sprintf("note-%d", n%1000000) + col13Value := rand.Intn(100) + col14Value := "acct" + col15Value := fmt.Sprintf("i%013d", n%10000000000000) + col16Value := fmt.Sprintf("type-%d", n%100000) + col17Value := fmt.Sprintf("memo-%d", n) + col18Value := fmt.Sprintf("%d", rand.Intn(2)) + col19Value := fmt.Sprintf("%d", rand.Intn(2)) + col20Value := fmt.Sprintf("%d", rand.Intn(2)) + col21Value := fmt.Sprintf("customer-%d", n%1000000) + col22Value := "A001" + col23Value := n % 1000000000000000 + col24Value := "B0001" + col25Value := fmt.Sprintf("ref-%d", n%1000000000000000000) + col26Value := "E1" + col27Value := randomBankDatetime() + col28Value := rand.Intn(1000) + col29Value := rand.Intn(1000) + col30Value := rand.Intn(1000) + + buf.WriteString(fmt.Sprintf( + "('%s','%s','%s','%s','%s','%s','%s','%s','%s',%d,%d,'%s',%d,'%s','%s','%s','%s','%s','%s','%s','%s','%s',%d,'%s','%s','%s','%s',%d,%d,%d)", + col1Value, + col2Value, + col3Value, + col4Value, + col5Value, + col6Value, + col7Value, + col8Value, + col9Value, + col10Value, + col11Value, + col12Value, + col13Value, + col14Value, + col15Value, + col16Value, + col17Value, + col18Value, + col19Value, + col20Value, + col21Value, + col22Value, + col23Value, + col24Value, + col25Value, + col26Value, + col27Value, + col28Value, + col29Value, + col30Value, + )) + } + return buf.String() +} + +func (c *BankWorkload) BuildUpdateSql(opts schema.UpdateOption) string { + if opts.Batch <= 0 { + return "" + } + + tableName := getBankTableName(opts.TableIndex) + startTime, endTime := randomBankMonthRange() + newCol30 := rand.Intn(1000) + newCol15 := fmt.Sprintf("u%013d", rand.Int63n(10000000000000)) + newCol27 := randomBankDatetime() + + return fmt.Sprintf( + `UPDATE %[1]s +SET col30 = %[2]d, col15 = '%[3]s', col27 = '%[4]s' +WHERE auto_id IN ( + SELECT auto_id FROM ( + SELECT auto_id FROM %[1]s + WHERE col4 >= '%[5]s' AND col4 < '%[6]s' + ORDER BY auto_id DESC LIMIT %[7]d + ) t +)`, + tableName, + newCol30, + newCol15, + newCol27, + startTime, + endTime, + opts.Batch, + ) +} + +func (c *BankWorkload) BuildDeleteSql(opts schema.DeleteOption) string { + if opts.Batch <= 0 { + return "" + } + + deleteType := rand.Intn(3) + tableName := getBankTableName(opts.TableIndex) + + switch deleteType { + case 0: + // Strategy 1: Delete the newest rows by auto_id + return fmt.Sprintf( + `DELETE FROM %[1]s +WHERE auto_id IN ( + SELECT auto_id FROM ( + SELECT auto_id FROM %[1]s ORDER BY auto_id DESC LIMIT %[2]d + ) t +)`, + tableName, opts.Batch, + ) + + case 1: + // Strategy 2: Range delete by partition key (col4) + startTime, endTime := randomBankMonthRange() + return fmt.Sprintf( + "DELETE FROM %s WHERE col4 >= '%s' AND col4 < '%s' LIMIT %d", + tableName, startTime, endTime, opts.Batch, + ) + + case 2: + // Strategy 3: Conditional delete by indexed columns + return fmt.Sprintf( + "DELETE FROM %s WHERE col14 = 'acct' AND col22 = 'A001' LIMIT %d", + tableName, opts.Batch, + ) + + default: + return "" + } +} + +func (c *BankWorkload) BuildDDLSql(opts schema.DDLOption) string { + tableName := getBankTableName(opts.TableIndex) + return fmt.Sprintf("truncate table %s;", tableName) +} + +func getBankTableName(n int) string { + return fmt.Sprintf("bank_%d", n) +} + +func randomBankDatetime() string { + // bank partitions cover [< 2027-04-01), so keep generated timestamps within [2021-07, 2027-03]. + const ( + startYear = 2021 + startMonth = 7 + endYear = 2027 + endMonth = 3 + ) + startTotal := startYear*12 + (startMonth - 1) + endTotal := endYear*12 + (endMonth - 1) + + total := startTotal + rand.Intn(endTotal-startTotal+1) + year := total / 12 + month := total%12 + 1 + + day := rand.Intn(28) + 1 + hour := rand.Intn(24) + minute := rand.Intn(60) + second := rand.Intn(60) + return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second) +} + +func randomBankMonthRange() (start string, end string) { + // Align with the partition boundary to increase the chance of partition pruning. + const ( + startYear = 2021 + startMonth = 7 + endYear = 2027 + endMonth = 3 + ) + startTotal := startYear*12 + (startMonth - 1) + endTotal := endYear*12 + (endMonth - 1) + + total := startTotal + rand.Intn(endTotal-startTotal+1) + year := total / 12 + month := total%12 + 1 + start = fmt.Sprintf("%04d-%02d-01 00:00:00", year, month) + + next := total + 1 + endYearValue := next / 12 + endMonthValue := next%12 + 1 + end = fmt.Sprintf("%04d-%02d-01 00:00:00", endYearValue, endMonthValue) + return start, end +} From 140bce04707eecee4396a5e3b2735223180f79e7 Mon Sep 17 00:00:00 2001 From: wlwilliamx Date: Thu, 11 Jun 2026 11:35:58 +0800 Subject: [PATCH 2/2] tests: resolve nextgen startup cherry-pick conflict Signed-off-by: wlwilliamx --- tests/integration_tests/_utils/start_tidb_cluster_nextgen | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration_tests/_utils/start_tidb_cluster_nextgen b/tests/integration_tests/_utils/start_tidb_cluster_nextgen index 189f1d71a6..fee2547fcd 100755 --- a/tests/integration_tests/_utils/start_tidb_cluster_nextgen +++ b/tests/integration_tests/_utils/start_tidb_cluster_nextgen @@ -358,13 +358,10 @@ check_port_available "$DOWN_TIKV_WORKER_HOST" "$DOWN_TIKV_WORKER_PORT" "Downstre echo "Waiting for next-gen keyspace pre-allocation to settle..." sleep 30 -<<<<<<< HEAD -======= # NOTE: TiDB config keys are section-sensitive. Keep `socket` and # `max-server-connections` at root level, and keep tuning knobs like # `run-auto-analyze` / `server-memory-quota` under `[performance]`. Otherwise # TiDB will report "invalid configuration options" and silently ignore them. ->>>>>>> bef9fba03 (tests: fix next gen integration TiDB startup flakiness (#4278)) touch "$OUT_DIR/upstream-tidb-system.toml" if [ -f "$tidb_config" ]; then cat "$tidb_config" >>"$OUT_DIR/upstream-tidb-system.toml"