Skip to content
Open
3 changes: 3 additions & 0 deletions pkg/core/constant/kind.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ func (k KeyType) String() string {
}

// StringToKeyType creates a key type with string.
// It return Table if the input string is empty, since Table is the default key type in most cases.
func StringToKeyType(input string) KeyType {
switch input {
case Table.String():
Expand All @@ -151,6 +152,8 @@ func StringToKeyType(input string) KeyType {
return Raw
case Txn.String():
return Txn
case "":
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need it?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the unit test.

return Table
default:
panic("invalid key type: " + input)
}
Expand Down
53 changes: 37 additions & 16 deletions pkg/keyspace/keyspace.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ import (
"github.com/tikv/pd/pkg/storage/kv"
"github.com/tikv/pd/pkg/utils/etcdutil"
"github.com/tikv/pd/pkg/utils/keypath"
"github.com/tikv/pd/pkg/utils/logutil"
"github.com/tikv/pd/pkg/utils/syncutil"
"github.com/tikv/pd/pkg/versioninfo/kerneltype"
)
Expand Down Expand Up @@ -506,7 +505,8 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er
})

start := time.Now()
keyspaceRule := MakeLabelRule(id)
boundType := manager.getRegionBoundType()
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to prevent dynamically changing the keytype?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, later we should persist the key type when we create the keyspace.

keyspaceRule := buildLabelRule(id, boundType)
cl, ok := manager.cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler })
if !ok {
return errors.New("cluster does not support region label")
Expand All @@ -527,11 +527,18 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er
zap.Error(err),
)
}
return
}
log.Info("added region label for keyspace",
zap.Uint32("keyspace-id", id),
zap.Any("label-rule", keyspaceRule),
zap.Duration("takes", time.Since(start)),
zap.Stringer("key-type", boundType),
)
}()

if waitRegionSplit {
err = manager.waitKeyspaceRegionSplit(id)
err = manager.waitKeyspaceRegionSplit(id, boundType)
if err != nil {
log.Warn("[keyspace] wait region split meets error",
zap.Uint32("keyspace-id", id),
Expand All @@ -540,16 +547,10 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er
}
return err
}

log.Info("[keyspace] added region label for keyspace",
zap.Uint32("keyspace-id", id),
logutil.ZapRedactString("label-rule", keyspaceRule.String()),
zap.Duration("takes", time.Since(start)),
)
return
return nil
}

func (manager *Manager) waitKeyspaceRegionSplit(id uint32) error {
func (manager *Manager) waitKeyspaceRegionSplit(id uint32, boundType regionBoundType) error {
ticker := time.NewTicker(manager.config.GetCheckRegionSplitInterval())
timer := time.NewTimer(manager.config.GetWaitRegionSplitTimeout())
defer func() {
Expand All @@ -561,7 +562,7 @@ func (manager *Manager) waitKeyspaceRegionSplit(id uint32) error {
case <-manager.ctx.Done():
return errors.New("[keyspace] wait region split canceled")
case <-ticker.C:
if manager.CheckKeyspaceRegionBound(id) {
if manager.hasKeyspaceRegionBound(id, boundType) {
log.Info("[keyspace] wait region split successfully", zap.Uint32("keyspace-id", id))
return nil
}
Expand All @@ -576,11 +577,31 @@ func (manager *Manager) waitKeyspaceRegionSplit(id uint32) error {

// CheckKeyspaceRegionBound checks whether the keyspace region has been split.
func (manager *Manager) CheckKeyspaceRegionBound(id uint32) bool {
Comment thread
bufferflies marked this conversation as resolved.
return manager.hasKeyspaceRegionBound(id, manager.getRegionBoundType())
}

func (manager *Manager) hasKeyspaceRegionBound(id uint32, boundType regionBoundType) bool {
regionBound := MakeRegionBound(id)
return manager.checkBound(regionBound.RawLeftBound) &&
manager.checkBound(regionBound.RawRightBound) &&
manager.checkBound(regionBound.TxnLeftBound) &&
manager.checkBound(regionBound.TxnRightBound)
switch boundType {
case txnRegionBound:
return manager.checkBound(regionBound.TxnLeftBound) &&
manager.checkBound(regionBound.TxnRightBound)
case rawRegionBound:
return manager.checkBound(regionBound.RawLeftBound) &&
manager.checkBound(regionBound.RawRightBound)
default:
return manager.checkBound(regionBound.RawLeftBound) &&
manager.checkBound(regionBound.RawRightBound) &&
manager.checkBound(regionBound.TxnLeftBound) &&
manager.checkBound(regionBound.TxnRightBound)
}
}

func (manager *Manager) getRegionBoundType() regionBoundType {
if manager.cluster == nil || manager.cluster.GetSharedConfig() == nil {
Comment thread
bufferflies marked this conversation as resolved.
return allRegionBound
}
return keyTypeToRegionBoundType(manager.cluster.GetSharedConfig().GetKeyType())
}

func (manager *Manager) checkBound(key []byte) bool {
Expand Down
58 changes: 56 additions & 2 deletions pkg/keyspace/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/pingcap/kvproto/pkg/keyspacepb"

"github.com/tikv/pd/pkg/codec"
coreconstant "github.com/tikv/pd/pkg/core/constant"
"github.com/tikv/pd/pkg/errs"
"github.com/tikv/pd/pkg/keyspace/constant"
"github.com/tikv/pd/pkg/schedule/labeler"
Expand Down Expand Up @@ -112,6 +113,35 @@ type RegionBound struct {
TxnRightBound []byte
}

type regionBoundType int

const (
allRegionBound regionBoundType = iota
rawRegionBound
txnRegionBound
)

// String returns the string representation of the regionBoundType.
func (t regionBoundType) String() string {
switch t {
case allRegionBound:
return "all"
case rawRegionBound:
return "raw"
case txnRegionBound:
return "txn"
default:
return "unknown"
}
}

func keyTypeToRegionBoundType(keyType coreconstant.KeyType) regionBoundType {
if keyType == coreconstant.Table {
return txnRegionBound
}
return rawRegionBound
}

// MakeRegionBound constructs the correct region boundaries of the given keyspace.
func MakeRegionBound(id uint32) *RegionBound {
keyspaceIDBytes := make([]byte, 4)
Expand All @@ -126,9 +156,29 @@ func MakeRegionBound(id uint32) *RegionBound {
}
}

// MakeKeyRanges encodes keyspace ID to correct LabelRule data.
// MakeKeyRanges encodes keyspace ID to the correct LabelRule data.
func MakeKeyRanges(id uint32) []any {
return buildKeyRanges(id, allRegionBound)
}

func buildKeyRanges(id uint32, boundType regionBoundType) []any {
regionBound := MakeRegionBound(id)
switch boundType {
case txnRegionBound:
return []any{
map[string]any{
"start_key": hex.EncodeToString(regionBound.TxnLeftBound),
"end_key": hex.EncodeToString(regionBound.TxnRightBound),
},
}
case rawRegionBound:
return []any{
map[string]any{
"start_key": hex.EncodeToString(regionBound.RawLeftBound),
"end_key": hex.EncodeToString(regionBound.RawRightBound),
},
}
}
return []any{
map[string]any{
"start_key": hex.EncodeToString(regionBound.RawLeftBound),
Expand All @@ -148,6 +198,10 @@ func getRegionLabelID(id uint32) string {

// MakeLabelRule makes the label rule for the given keyspace id.
func MakeLabelRule(id uint32) *labeler.LabelRule {
return buildLabelRule(id, allRegionBound)
}

func buildLabelRule(id uint32, boundType regionBoundType) *labeler.LabelRule {
return &labeler.LabelRule{
ID: getRegionLabelID(id),
Index: 0,
Expand All @@ -158,7 +212,7 @@ func MakeLabelRule(id uint32) *labeler.LabelRule {
},
},
RuleType: labeler.KeyRange,
Data: MakeKeyRanges(id),
Data: buildKeyRanges(id, boundType),
}
}

Expand Down
80 changes: 77 additions & 3 deletions pkg/keyspace/util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/pingcap/failpoint"

"github.com/tikv/pd/pkg/codec"
coreconstant "github.com/tikv/pd/pkg/core/constant"
"github.com/tikv/pd/pkg/keyspace/constant"
"github.com/tikv/pd/pkg/schedule/labeler"
"github.com/tikv/pd/pkg/versioninfo/kerneltype"
Expand Down Expand Up @@ -144,10 +145,12 @@ func TestMakeLabelRule(t *testing.T) {
re := require.New(t)
testCases := []struct {
id uint32
boundType regionBoundType
expectedLabelRule *labeler.LabelRule
}{
{
id: 0,
id: 0,
boundType: allRegionBound,
expectedLabelRule: &labeler.LabelRule{
ID: getRegionLabelID(0),
Index: 0,
Expand All @@ -171,7 +174,8 @@ func TestMakeLabelRule(t *testing.T) {
},
},
{
id: 4242,
id: 4242,
boundType: allRegionBound,
expectedLabelRule: &labeler.LabelRule{
ID: getRegionLabelID(4242),
Index: 0,
Expand All @@ -194,12 +198,82 @@ func TestMakeLabelRule(t *testing.T) {
},
},
},
{
id: 0,
boundType: txnRegionBound,
expectedLabelRule: &labeler.LabelRule{
ID: "keyspaces/0",
Index: 0,
Labels: []labeler.RegionLabel{
{
Key: "id",
Value: "0",
},
},
RuleType: "key-range",
Data: []any{
map[string]any{
"start_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0, 0})),
"end_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0, 1})),
},
},
},
},
{
id: 4242,
boundType: txnRegionBound,
expectedLabelRule: &labeler.LabelRule{
ID: "keyspaces/4242",
Index: 0,
Labels: []labeler.RegionLabel{
{
Key: "id",
Value: "4242",
},
},
RuleType: "key-range",
Data: []any{
map[string]any{
"start_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0x10, 0x92})),
"end_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0x10, 0x93})),
},
},
},
},
{
id: 4242,
boundType: rawRegionBound,
expectedLabelRule: &labeler.LabelRule{
ID: "keyspaces/4242",
Index: 0,
Labels: []labeler.RegionLabel{
{
Key: "id",
Value: "4242",
},
},
RuleType: "key-range",
Data: []any{
map[string]any{
"start_key": hex.EncodeToString(codec.EncodeBytes([]byte{'r', 0, 0x10, 0x92})),
"end_key": hex.EncodeToString(codec.EncodeBytes([]byte{'r', 0, 0x10, 0x93})),
},
},
},
},
}
for _, testCase := range testCases {
re.Equal(testCase.expectedLabelRule, MakeLabelRule(testCase.id))
re.Equal(testCase.expectedLabelRule, buildLabelRule(testCase.id, testCase.boundType))
}
}

func TestKeyTypeToRegionBoundType(t *testing.T) {
re := require.New(t)
re.Equal(txnRegionBound, keyTypeToRegionBoundType(coreconstant.Table))
re.Equal(rawRegionBound, keyTypeToRegionBoundType(coreconstant.Raw))
re.Equal(rawRegionBound, keyTypeToRegionBoundType(coreconstant.Txn))
}

func TestParseKeyspaceIDFromLabelRule(t *testing.T) {
re := require.New(t)
testCases := []struct {
Expand Down
17 changes: 17 additions & 0 deletions server/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,23 @@ disable-make-up-replica = false
re.Error(err)
}

func TestLegacyDisableRawKVRegionSplitConfigDoesNotPanic(t *testing.T) {
re := require.New(t)

re.NotPanics(func() {
cfg := NewConfig()
err := json.Unmarshal([]byte(`{"keyspace":{"disable-raw-kv-region-split":true}}`), cfg)
re.NoError(err)
})

re.NotPanics(func() {
cfg := NewConfig()
meta, err := toml.Decode("[keyspace]\ndisable-raw-kv-region-split = true\n", cfg)
re.NoError(err)
re.NoError(cfg.Adjust(&meta, false))
})
}

func TestPDServerConfig(t *testing.T) {
re := require.New(t)
tests := []struct {
Expand Down
9 changes: 3 additions & 6 deletions tests/server/keyspace/keyspace_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,14 +109,11 @@ func checkLabelRule(re *require.Assertions, id uint32, regionLabeler *labeler.Re

rangeRule, ok := loadedLabel.Data.([]*labeler.KeyRangeRule)
re.True(ok)
re.Len(rangeRule, 2)
re.Len(rangeRule, 1)

bound := keyspace.MakeRegionBound(id)

re.Equal(hex.EncodeToString(bound.RawLeftBound), rangeRule[0].StartKeyHex)
re.Equal(hex.EncodeToString(bound.RawRightBound), rangeRule[0].EndKeyHex)
re.Equal(hex.EncodeToString(bound.TxnLeftBound), rangeRule[1].StartKeyHex)
re.Equal(hex.EncodeToString(bound.TxnRightBound), rangeRule[1].EndKeyHex)
re.Equal(hex.EncodeToString(bound.TxnLeftBound), rangeRule[0].StartKeyHex)
re.Equal(hex.EncodeToString(bound.TxnRightBound), rangeRule[0].EndKeyHex)
}

func (suite *keyspaceTestSuite) TestPreAlloc() {
Expand Down
Loading