From 913f27d09d1a54b5ceb52ff02e532eb544368e82 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 7 Jun 2026 01:26:56 +0800 Subject: [PATCH 01/76] Add remote checkpoint tooling and logical table view --- cmd/mo-object-tool/README.md | 200 ++++++++++++++++++ cmd/mo-object-tool/ckp/checkpoint.go | 42 +++- cmd/mo-object-tool/info.go | 27 ++- cmd/mo-object-tool/object.go | 30 ++- cmd/mo-object-tool/view.go | 8 +- pkg/tools/checkpointtool/checkpoint_reader.go | 21 ++ .../checkpointtool/interactive/bubbletea.go | 7 +- .../checkpointtool/interactive/providers.go | 41 ++++ pkg/tools/checkpointtool/interactive/state.go | 23 ++ .../interactive/unified_model.go | 35 ++- .../interactive/unified_model_test.go | 10 + pkg/tools/checkpointtool/logical_table.go | 198 +++++++++++++++++ .../checkpointtool/logical_table_test.go | 35 +++ pkg/tools/checkpointtool/types.go | 9 + pkg/tools/objecttool/interactive/repl.go | 11 +- .../objecttool/interactive/unified_model.go | 27 ++- pkg/tools/objecttool/object_reader.go | 10 +- pkg/tools/toolfs/storage.go | 145 +++++++++++++ pkg/tools/toolfs/storage_test.go | 83 ++++++++ 19 files changed, 930 insertions(+), 32 deletions(-) create mode 100644 cmd/mo-object-tool/README.md create mode 100644 pkg/tools/checkpointtool/logical_table.go create mode 100644 pkg/tools/checkpointtool/logical_table_test.go create mode 100644 pkg/tools/toolfs/storage.go create mode 100644 pkg/tools/toolfs/storage_test.go diff --git a/cmd/mo-object-tool/README.md b/cmd/mo-object-tool/README.md new file mode 100644 index 0000000000000..695f05caa33a7 --- /dev/null +++ b/cmd/mo-object-tool/README.md @@ -0,0 +1,200 @@ +# mo-tool Object And Checkpoint Usage + +`mo-tool` provides offline tools for inspecting MatrixOne checkpoint and object +files. It can read from a local data directory, or from a remote MatrixOne +fileservice such as S3 or MinIO. + +## Build + +```bash +go build -o mo-tool ./cmd/mo-tool +``` + +You can also run it directly during development: + +```bash +go run ./cmd/mo-tool --help +``` + +## Local Checkpoint Usage + +Read checkpoint files from a local MatrixOne data directory: + +```bash +./mo-tool ckp info ./mo-data +./mo-tool ckp view ./mo-data +``` + +If no directory is provided, the current working directory is used: + +```bash +./mo-tool ckp info +./mo-tool ckp view +``` + +The checkpoint viewer lists checkpoint entries and lets you drill down into +tables, object ranges, and object data. + +Inside table detail view: + +- Press `Enter` on a row to open the physical object view for that object range. +- Press `L` to open a logical table view with tombstones applied to table rows. + +## Local Object Usage + +Read a single local object file: + +```bash +./mo-tool object info ./mo-data/ +./mo-tool object view ./mo-data/ +``` + +Replace `` with the object path shown by the checkpoint viewer or +stored in checkpoint metadata. + +## Remote Fileservice From MatrixOne Config + +The recommended remote workflow is to reuse the MatrixOne service TOML file. +This avoids manually copying bucket, endpoint, key prefix, and credential +settings. + +For example, with a MinIO launch config: + +```bash +./mo-tool ckp info \ + --fs-config etc/launch-minio-local/tn.toml + +./mo-tool ckp view \ + --fs-config etc/launch-minio-local/tn.toml +``` + +By default, `mo-tool` uses the fileservice configured in +`[tn.Txn.Storage].fileservice`. If that field is absent, it uses `SHARED`. + +To select a specific fileservice: + +```bash +./mo-tool ckp view \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED +``` + +The same fileservice options work for object inspection: + +```bash +./mo-tool object info \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED + +./mo-tool object view \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED +``` + +When checkpoint data is opened from a remote fileservice, object drill-down from +the checkpoint viewer reuses the same fileservice. This means checkpoint meta, +checkpoint data objects, and table data objects are all read from the same S3 or +MinIO backend. + +The logical table view also uses the same fileservice and applies tombstones at +the selected checkpoint snapshot timestamp. + +## Remote S3 Or MinIO With Inline Arguments + +You can also provide object storage settings directly on the command line. + +MinIO example: + +```bash +./mo-tool ckp view \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 +``` + +AWS S3 example: + +```bash +./mo-tool ckp info \ + --backend S3 \ + --s3 bucket=my-mo-bucket,region=us-east-1,key-prefix=prod/mo-data +``` + +Object viewer with inline S3 arguments: + +```bash +./mo-tool object view \ + --backend S3 \ + --s3 bucket=my-mo-bucket,region=us-east-1,key-prefix=prod/mo-data +``` + +Supported `--s3` keys include: + +- `bucket` +- `endpoint` +- `region` +- `key-prefix` +- `key-id` +- `key-secret` +- `session-token` +- `role-arn` +- `external-id` +- `shared-config-profile` +- `no-default-credentials` +- `no-bucket-validation` +- `is-minio` +- `cert-files` + +Credentials can also come from the default AWS credential chain when supported +by the selected backend and when `no-default-credentials` is not set. + +## Important Path Rules + +For remote checkpoint inspection, `key-prefix` should point to the MatrixOne +data root, not to the checkpoint directory itself. + +Correct: + +```text +s3://my-bucket/prod/mo-data/ + ckp/ + +``` + +Use: + +```bash +--s3 bucket=my-bucket,key-prefix=prod/mo-data,... +``` + +Do not use: + +```bash +--s3 bucket=my-bucket,key-prefix=prod/mo-data/ckp,... +``` + +Checkpoint metadata stores object names relative to the fileservice root. If the +fileservice root is pointed directly at `ckp/`, checkpoint metadata may be found +but table data object drill-down will fail. + +## Command Summary + +Checkpoint commands: + +```bash +./mo-tool ckp info [directory] [--fs-config FILE] [--fs-name NAME] +./mo-tool ckp view [directory] [--fs-config FILE] [--fs-name NAME] +./mo-tool ckp info --backend S3 --s3 key=value,... +./mo-tool ckp view --backend MINIO --s3 key=value,... +``` + +Object commands: + +```bash +./mo-tool object info [--fs-config FILE] [--fs-name NAME] +./mo-tool object view [--fs-config FILE] [--fs-name NAME] +./mo-tool object info --backend S3 --s3 key=value,... +./mo-tool object view --backend MINIO --s3 key=value,... +``` + +Local mode treats paths as local filesystem paths. Remote mode treats object +names as paths relative to the selected fileservice root. diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 59d4889416169..47fa724d5b90b 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -22,10 +22,12 @@ import ( "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool/interactive" + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/spf13/cobra" ) func PrepareCommand() *cobra.Command { + var storage toolfs.StorageOptions cmd := &cobra.Command{ Use: "ckp [directory]", Short: "Checkpoint viewer tool", @@ -36,16 +38,24 @@ func PrepareCommand() *cobra.Command { if len(args) == 1 { dir = args[0] } - return runViewer(dir) + return runViewer(dir, storage) }, } + addStorageFlags(cmd, &storage) - cmd.AddCommand(infoCommand()) - cmd.AddCommand(viewCommand()) + cmd.AddCommand(infoCommand(&storage)) + cmd.AddCommand(viewCommand(&storage)) return cmd } +func addStorageFlags(cmd *cobra.Command, storage *toolfs.StorageOptions) { + cmd.PersistentFlags().StringVar(&storage.FSConfig, "fs-config", "", "MO config TOML containing fileservice settings") + cmd.PersistentFlags().StringVar(&storage.FSName, "fs-name", "SHARED", "fileservice name to use from --fs-config") + cmd.PersistentFlags().StringVar(&storage.S3, "s3", "", "S3 arguments, for example bucket=...,endpoint=...,region=...,key-prefix=...,key-id=...,key-secret=...") + cmd.PersistentFlags().StringVar(&storage.Backend, "backend", "", "remote backend for --s3: S3 or MINIO") +} + func setupLogFile() (*os.File, error) { homeDir, err := os.UserHomeDir() if err != nil { @@ -76,7 +86,7 @@ func setupLogFile() (*os.File, error) { return logFile, nil } -func runViewer(dir string) error { +func runViewer(dir string, storage toolfs.StorageOptions) error { logFile, err := setupLogFile() if err != nil { return fmt.Errorf("setup log file: %w", err) @@ -84,7 +94,7 @@ func runViewer(dir string) error { defer logFile.Close() ctx := context.Background() - reader, err := checkpointtool.Open(ctx, dir) + reader, err := openReader(ctx, dir, storage) if err != nil { return fmt.Errorf("open checkpoint dir: %w", err) } @@ -93,7 +103,7 @@ func runViewer(dir string) error { return interactive.Run(reader) } -func infoCommand() *cobra.Command { +func infoCommand(storage *toolfs.StorageOptions) *cobra.Command { return &cobra.Command{ Use: "info [directory]", Short: "Show checkpoint summary", @@ -111,7 +121,7 @@ func infoCommand() *cobra.Command { defer logFile.Close() ctx := context.Background() - reader, err := checkpointtool.Open(ctx, dir) + reader, err := openReader(ctx, dir, *storage) if err != nil { return fmt.Errorf("open checkpoint dir: %w", err) } @@ -134,7 +144,7 @@ func infoCommand() *cobra.Command { } } -func viewCommand() *cobra.Command { +func viewCommand(storage *toolfs.StorageOptions) *cobra.Command { return &cobra.Command{ Use: "view [directory]", Short: "Interactive checkpoint viewer", @@ -144,7 +154,21 @@ func viewCommand() *cobra.Command { if len(args) == 1 { dir = args[0] } - return runViewer(dir) + return runViewer(dir, *storage) }, } } + +func openReader(ctx context.Context, dir string, storage toolfs.StorageOptions) (*checkpointtool.CheckpointReader, error) { + if !storage.IsRemote() { + return checkpointtool.Open(ctx, dir) + } + fs, display, err := toolfs.Open(ctx, storage) + if err != nil { + return nil, err + } + if display == "" { + display = dir + } + return checkpointtool.OpenWithFS(ctx, fs, display, checkpointtool.WithCloseFS()) +} diff --git a/cmd/mo-object-tool/info.go b/cmd/mo-object-tool/info.go index b1906de20f007..3d65b48aa5fb8 100644 --- a/cmd/mo-object-tool/info.go +++ b/cmd/mo-object-tool/info.go @@ -21,10 +21,11 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/tools/objecttool" + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/spf13/cobra" ) -func infoCommand() *cobra.Command { +func infoCommand(storage *toolfs.StorageOptions) *cobra.Command { cmd := &cobra.Command{ Use: "info ", Short: "Show object file information", @@ -32,19 +33,33 @@ func infoCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { path := args[0] - return showInfo(path) + return showInfo(path, *storage) }, } return cmd } -func showInfo(path string) error { +func showInfo(path string, storage toolfs.StorageOptions) error { ctx := context.Background() - reader, err := objecttool.Open(ctx, path) - if err != nil { - return moerr.NewInternalErrorf(ctx, "failed to open object: %v", err) + var reader *objecttool.ObjectReader + if storage.IsRemote() { + fs, _, err := toolfs.Open(ctx, storage) + if err != nil { + return err + } + defer fs.Close(ctx) + reader, err = objecttool.OpenWithFS(ctx, fs, path, path) + if err != nil { + return moerr.NewInternalErrorf(ctx, "failed to open object: %v", err) + } + } else { + var err error + reader, err = objecttool.Open(ctx, path) + if err != nil { + return moerr.NewInternalErrorf(ctx, "failed to open object: %v", err) + } } defer reader.Close() diff --git a/cmd/mo-object-tool/object.go b/cmd/mo-object-tool/object.go index 0223fee8451a0..d2c5442d5faf0 100644 --- a/cmd/mo-object-tool/object.go +++ b/cmd/mo-object-tool/object.go @@ -15,11 +15,15 @@ package object import ( + "context" + "github.com/matrixorigin/matrixone/pkg/tools/objecttool/interactive" + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/spf13/cobra" ) func PrepareCommand() *cobra.Command { + var storage toolfs.StorageOptions cmd := &cobra.Command{ Use: "object [file]", Short: "Object file tools", @@ -28,15 +32,35 @@ func PrepareCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { // If file path is provided, enter view mode directly if len(args) == 1 { - return interactive.Run(args[0]) + return runObjectView(context.Background(), args[0], storage) } // Otherwise show help return cmd.Help() }, } + addStorageFlags(cmd, &storage) - cmd.AddCommand(viewCommand()) - cmd.AddCommand(infoCommand()) + cmd.AddCommand(viewCommand(&storage)) + cmd.AddCommand(infoCommand(&storage)) return cmd } + +func addStorageFlags(cmd *cobra.Command, storage *toolfs.StorageOptions) { + cmd.PersistentFlags().StringVar(&storage.FSConfig, "fs-config", "", "MO config TOML containing fileservice settings") + cmd.PersistentFlags().StringVar(&storage.FSName, "fs-name", "SHARED", "fileservice name to use from --fs-config") + cmd.PersistentFlags().StringVar(&storage.S3, "s3", "", "S3 arguments, for example bucket=...,endpoint=...,region=...,key-prefix=...,key-id=...,key-secret=...") + cmd.PersistentFlags().StringVar(&storage.Backend, "backend", "", "remote backend for --s3: S3 or MINIO") +} + +func runObjectView(ctx context.Context, path string, storage toolfs.StorageOptions) error { + if !storage.IsRemote() { + return interactive.Run(path) + } + fs, _, err := toolfs.Open(ctx, storage) + if err != nil { + return err + } + defer fs.Close(ctx) + return interactive.RunWithFS(ctx, fs, path) +} diff --git a/cmd/mo-object-tool/view.go b/cmd/mo-object-tool/view.go index aebe0927b7dab..12910c8efb26a 100644 --- a/cmd/mo-object-tool/view.go +++ b/cmd/mo-object-tool/view.go @@ -15,11 +15,13 @@ package object import ( - "github.com/matrixorigin/matrixone/pkg/tools/objecttool/interactive" + "context" + + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/spf13/cobra" ) -func viewCommand() *cobra.Command { +func viewCommand(storage *toolfs.StorageOptions) *cobra.Command { cmd := &cobra.Command{ Use: "view ", Short: "Interactive object file viewer", @@ -27,7 +29,7 @@ func viewCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { path := args[0] - return interactive.Run(path) + return runObjectView(context.Background(), path, *storage) }, } diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index 628933f9b3e4a..0cb64634ab563 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -37,6 +37,7 @@ type CheckpointReader struct { dir string entries []*checkpoint.CheckpointEntry mp *mpool.MPool + closeFS bool } // Option configures CheckpointReader @@ -49,6 +50,13 @@ func WithMPool(mp *mpool.MPool) Option { } } +// WithCloseFS closes the file service when the reader is closed. +func WithCloseFS() Option { + return func(r *CheckpointReader) { + r.closeFS = true + } +} + // Open opens checkpoint data from a directory func Open(ctx context.Context, dir string, opts ...Option) (*CheckpointReader, error) { fs, err := fileservice.NewLocalFS(ctx, "local", dir, fileservice.DisabledCacheConfig, nil) @@ -56,6 +64,11 @@ func Open(ctx context.Context, dir string, opts ...Option) (*CheckpointReader, e return nil, moerr.NewInternalErrorf(ctx, "create file service: %v", err) } + return OpenWithFS(ctx, fs, dir, append(opts, WithCloseFS())...) +} + +// OpenWithFS opens checkpoint data from an existing file service. +func OpenWithFS(ctx context.Context, fs fileservice.FileService, dir string, opts ...Option) (*CheckpointReader, error) { r := &CheckpointReader{ ctx: ctx, fs: fs, @@ -74,6 +87,11 @@ func Open(ctx context.Context, dir string, opts ...Option) (*CheckpointReader, e return r, nil } +// FS returns the file service used by this reader. +func (r *CheckpointReader) FS() fileservice.FileService { + return r.fs +} + func (r *CheckpointReader) loadEntries() error { names, err := ckputil.ListCKPMetaNames(r.ctx, r.fs) if err != nil { @@ -375,6 +393,9 @@ func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { // Close releases resources func (r *CheckpointReader) Close() error { r.entries = nil + if r.closeFS && r.fs != nil { + r.fs.Close(r.ctx) + } return nil } diff --git a/pkg/tools/checkpointtool/interactive/bubbletea.go b/pkg/tools/checkpointtool/interactive/bubbletea.go index 2744b35abd2c8..dd53b2c4add18 100644 --- a/pkg/tools/checkpointtool/interactive/bubbletea.go +++ b/pkg/tools/checkpointtool/interactive/bubbletea.go @@ -74,11 +74,12 @@ func Run(reader *checkpointtool.CheckpointReader) error { }, ExpandFunc: expandObjectStats, }, - ObjectNameCol: 4, // object_name column after expansion - BaseDir: m.state.reader.Dir(), // Base directory for nested objects + ObjectNameCol: 4, // object_name column after expansion CustomOverview: ckpDataOverview, } - if err := objectinteractive.RunUnified(context.Background(), um.GetObjectToOpen(), opts); err != nil { + if err := objectinteractive.RunUnifiedWithFS( + context.Background(), m.state.reader.FS(), um.GetObjectToOpen(), opts, + ); err != nil { return err } // Clear the object to open flag and continue with current state diff --git a/pkg/tools/checkpointtool/interactive/providers.go b/pkg/tools/checkpointtool/interactive/providers.go index 232a9476a23cb..5c2d523e11742 100644 --- a/pkg/tools/checkpointtool/interactive/providers.go +++ b/pkg/tools/checkpointtool/interactive/providers.go @@ -242,6 +242,32 @@ func (p *TableDetailProvider) GetOverview() string { formatSize(totalOriginSize), formatSize(totalCompSize)) } +type LogicalTableProvider struct { + state *State +} + +func (p *LogicalTableProvider) GetRows() [][]string { + view := p.state.LogicalView() + if view == nil { + return nil + } + return view.Rows +} + +func (p *LogicalTableProvider) GetRowNums() []string { + return nil +} + +func (p *LogicalTableProvider) GetOverview() string { + view := p.state.LogicalView() + tbl := p.state.GetSelectedTable() + if view == nil || tbl == nil { + return "" + } + return fmt.Sprintf("Table %d logical view │ visible: %d │ deleted: %d │ physical: %d", + tbl.TableID, view.VisibleRows, view.DeletedRows, view.PhysicalRows) +} + // === Table Detail Handler === type tableDetailHandler struct { @@ -270,6 +296,13 @@ func (h *tableDetailHandler) OnBack() tea.Cmd { return func() tea.Msg { return goBackMsg{} } } +func (h *tableDetailHandler) OnCustomKey(key string) tea.Cmd { + if key == "L" { + return func() tea.Msg { return openLogicalTableMsg{} } + } + return nil +} + // FilterRow filters by type (Data/Tomb) func (h *tableDetailHandler) FilterRow(row []string, filter string) bool { if len(row) > 0 { @@ -278,6 +311,14 @@ func (h *tableDetailHandler) FilterRow(row []string, filter string) bool { return false } +type logicalTableHandler struct { + interactive.DefaultHandler +} + +func (h *logicalTableHandler) OnBack() tea.Cmd { + return func() tea.Msg { return goBackMsg{} } +} + // === Account List Provider === type AccountListProvider struct { diff --git a/pkg/tools/checkpointtool/interactive/state.go b/pkg/tools/checkpointtool/interactive/state.go index 3c7ac334cc7f8..fa60113cb54ee 100644 --- a/pkg/tools/checkpointtool/interactive/state.go +++ b/pkg/tools/checkpointtool/interactive/state.go @@ -15,6 +15,8 @@ package interactive import ( + "context" + "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" ) @@ -47,6 +49,7 @@ type State struct { info *checkpointtool.CheckpointInfo dataEntries []*checkpointtool.ObjectEntryInfo tombEntries []*checkpointtool.ObjectEntryInfo + logicalView *checkpointtool.LogicalTableView } // NewState creates a new state @@ -66,6 +69,7 @@ func (s *State) Mode() ViewMode { return s.mode func (s *State) Entries() []*checkpoint.CheckpointEntry { return s.entries } func (s *State) DataEntries() []*checkpointtool.ObjectEntryInfo { return s.dataEntries } func (s *State) TombEntries() []*checkpointtool.ObjectEntryInfo { return s.tombEntries } +func (s *State) LogicalView() *checkpointtool.LogicalTableView { return s.logicalView } func (s *State) HasAccountFilter() bool { return s.filterAccountID >= 0 } func (s *State) GetAccountFilter() int64 { return s.filterAccountID } @@ -129,6 +133,7 @@ func (s *State) SwitchToTables() error { return err } s.tables = tables + s.logicalView = nil s.mode = ViewModeTable return nil } @@ -142,6 +147,24 @@ func (s *State) SelectTable(tableID uint64) error { } s.dataEntries = dataEntries s.tombEntries = tombEntries + s.logicalView = nil + return nil +} + +// LoadLogicalView materializes the tombstone-applied logical rows for the selected table. +func (s *State) LoadLogicalView() error { + if s.logicalView != nil { + return nil + } + if s.selectedEntry < 0 || s.selectedEntry >= len(s.entries) { + return nil + } + snapshotTS := s.entries[s.selectedEntry].GetEnd() + view, err := s.reader.BuildLogicalTableView(context.Background(), snapshotTS, s.dataEntries, s.tombEntries) + if err != nil { + return err + } + s.logicalView = view return nil } diff --git a/pkg/tools/checkpointtool/interactive/unified_model.go b/pkg/tools/checkpointtool/interactive/unified_model.go index b85511152dc58..acb2cb4199deb 100644 --- a/pkg/tools/checkpointtool/interactive/unified_model.go +++ b/pkg/tools/checkpointtool/interactive/unified_model.go @@ -15,8 +15,6 @@ package interactive import ( - "path/filepath" - tea "github.com/charmbracelet/bubbletea" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/interactive" @@ -27,6 +25,7 @@ import ( type selectCheckpointMsg struct{ idx int } type selectTableMsg struct{ tableID uint64 } type openObjectMsg struct{ path string } +type openLogicalTableMsg struct{} type goBackMsg struct{} // UnifiedModel uses GenericPage for all views @@ -95,12 +94,33 @@ func (m *UnifiedModel) createTableDetailPage() *interactive.GenericPage { EnableHScroll: true, EnableBack: true, MaxColWidth: 50, // Limit column width, use h/l to scroll + CustomHints: "[j/k] Navigate [Enter] Open object [L] Logical table [h/l] Scroll [/] Search [b/ESC] Back [q] Quit", } provider := &TableDetailProvider{state: m.state} handler := &tableDetailHandler{state: m.state} return interactive.NewGenericPage(config, provider, handler) } +func (m *UnifiedModel) createLogicalTablePage() *interactive.GenericPage { + headers := []string{"object", "block", "row"} + if view := m.state.LogicalView(); view != nil && len(view.Headers) > 0 { + headers = view.Headers + } + config := interactive.PageConfig{ + Title: "═══ Logical Table View ═══", + Headers: headers, + ShowRowNumber: true, + EnableCursor: false, + EnableSearch: true, + EnableHScroll: true, + EnableBack: true, + MaxColWidth: 40, + } + provider := &LogicalTableProvider{state: m.state} + handler := &logicalTableHandler{} + return interactive.NewGenericPage(config, provider, handler) +} + func (m *UnifiedModel) Init() tea.Cmd { return m.currentPage.Init() } @@ -131,7 +151,7 @@ func (m *UnifiedModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil case openObjectMsg: - m.objectToOpen = filepath.Join(m.state.reader.Dir(), msg.path) + m.objectToOpen = msg.path idx := m.currentPage.GetCursor() dataEntries := m.state.DataEntries() tombEntries := m.state.TombEntries() @@ -143,6 +163,15 @@ func (m *UnifiedModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.quitting = true return m, tea.Quit + case openLogicalTableMsg: + if err := m.state.LoadLogicalView(); err != nil { + return m, nil + } + m.pageStack = append(m.pageStack, m.currentPage) + m.currentPage = m.createLogicalTablePage() + m.currentPage.Refresh() + return m, nil + case goBackMsg: if len(m.pageStack) > 0 { m.currentPage = m.pageStack[len(m.pageStack)-1] diff --git a/pkg/tools/checkpointtool/interactive/unified_model_test.go b/pkg/tools/checkpointtool/interactive/unified_model_test.go index e212b65432dd6..33df08d71bbc9 100644 --- a/pkg/tools/checkpointtool/interactive/unified_model_test.go +++ b/pkg/tools/checkpointtool/interactive/unified_model_test.go @@ -221,6 +221,16 @@ func TestTableDetailHandler(t *testing.T) { assert.True(t, handler.MatchRow(row, "123")) assert.False(t, handler.MatchRow(row, "notfound")) }) + + t.Run("custom_key_logical_view", func(t *testing.T) { + handler := &tableDetailHandler{} + cmd := handler.OnCustomKey("L") + if assert.NotNil(t, cmd) { + msg := cmd() + _, ok := msg.(openLogicalTableMsg) + assert.True(t, ok) + } + }) } // TestTSFormatting tests timestamp formatting diff --git a/pkg/tools/checkpointtool/logical_table.go b/pkg/tools/checkpointtool/logical_table.go new file mode 100644 index 0000000000000..8d129589ded42 --- /dev/null +++ b/pkg/tools/checkpointtool/logical_table.go @@ -0,0 +1,198 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "context" + "fmt" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" + objectioutil "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" + "github.com/matrixorigin/matrixone/pkg/tools/objecttool" +) + +// BuildLogicalTableView materializes a tombstone-applied logical table view. +func (r *CheckpointReader) BuildLogicalTableView( + ctx context.Context, + snapshotTS types.TS, + dataEntries []*ObjectEntryInfo, + tombEntries []*ObjectEntryInfo, +) (*LogicalTableView, error) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row"}, + Rows: make([][]string, 0), + } + if len(dataEntries) == 0 { + return view, nil + } + + tombstoneStats := dedupeObjectStats(tombEntries) + for _, entry := range dataEntries { + objName := entry.Range.ObjectStats.ObjectName().String() + reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) + if err != nil { + return nil, err + } + + if len(view.Headers) == 3 { + cols := reader.Columns() + for _, col := range cols { + view.Headers = append(view.Headers, fmt.Sprintf("col_%d", col.Idx)) + } + } + + relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.Range.ObjectStats.ObjectName().ObjectId(), tombstoneStats) + if err != nil { + _ = reader.Close() + return nil, err + } + + startBlock := int(entry.Range.Start.GetBlockOffset()) + endBlock := int(entry.Range.End.GetBlockOffset()) + for blockIdx := startBlock; blockIdx <= endBlock; blockIdx++ { + bat, release, err := reader.ReadBlock(ctx, uint32(blockIdx)) + if err != nil { + _ = reader.Close() + return nil, err + } + + startRow := 0 + endRow := bat.RowCount() - 1 + if blockIdx == startBlock { + startRow = int(entry.Range.Start.GetRowOffset()) + } + if blockIdx == endBlock { + endRow = int(entry.Range.End.GetRowOffset()) + } + if startRow < 0 { + startRow = 0 + } + if endRow >= bat.RowCount() { + endRow = bat.RowCount() - 1 + } + if startRow > endRow || startRow >= bat.RowCount() { + release() + continue + } + + view.PhysicalRows += endRow - startRow + 1 + + deleteMask, err := r.buildDeleteMaskForBlock(ctx, &snapshotTS, entry.Range.ObjectStats, uint16(blockIdx), relevantTombstones) + if err != nil { + release() + _ = reader.Close() + return nil, err + } + + for rowIdx := startRow; rowIdx <= endRow; rowIdx++ { + if deleteMask.IsValid() && deleteMask.Contains(uint64(rowIdx)) { + view.DeletedRows++ + continue + } + row := make([]string, 0, len(bat.Vecs)+3) + row = append(row, + entry.Range.ObjectStats.ObjectName().Short().ShortString(), + fmt.Sprintf("%d", blockIdx), + fmt.Sprintf("%d", rowIdx), + ) + for _, vec := range bat.Vecs { + row = append(row, vecValueToString(vec, rowIdx)) + } + view.Rows = append(view.Rows, row) + } + + if deleteMask.IsValid() { + deleteMask.Release() + } + release() + } + + if err := reader.Close(); err != nil { + return nil, err + } + } + + view.VisibleRows = len(view.Rows) + return view, nil +} + +func dedupeObjectStats(entries []*ObjectEntryInfo) []objectio.ObjectStats { + seen := make(map[string]struct{}) + stats := make([]objectio.ObjectStats, 0, len(entries)) + for _, entry := range entries { + name := entry.Range.ObjectStats.ObjectName().String() + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + stats = append(stats, entry.Range.ObjectStats) + } + return stats +} + +func (r *CheckpointReader) filterTombstonesForObject( + ctx context.Context, + objectID *objectio.ObjectId, + tombstoneStats []objectio.ObjectStats, +) ([]objectio.ObjectStats, error) { + if len(tombstoneStats) == 0 { + return nil, nil + } + selection, err := objectioutil.FindTombstonesOfObject(ctx, objectID, tombstoneStats, r.fs) + if err != nil { + return nil, err + } + relevant := make([]objectio.ObjectStats, 0, selection.Count()) + iter := selection.Iterator() + for iter.HasNext() { + idx := iter.Next() + if idx >= uint64(len(tombstoneStats)) { + return nil, moerr.NewInternalErrorf(ctx, "tombstone selection index out of range: %d", idx) + } + relevant = append(relevant, tombstoneStats[int(idx)]) + } + return relevant, nil +} + +func (r *CheckpointReader) buildDeleteMaskForBlock( + ctx context.Context, + snapshotTS *types.TS, + stats objectio.ObjectStats, + blockIdx uint16, + tombstoneStats []objectio.ObjectStats, +) (objectio.Bitmap, error) { + if len(tombstoneStats) == 0 { + return objectio.NullBitmap, nil + } + + mask := objectio.GetReusableBitmap() + blockID := stats.ConstructBlockId(blockIdx) + current := 0 + getTombstone := func() (*objectio.ObjectStats, error) { + if current >= len(tombstoneStats) { + return nil, nil + } + stat := &tombstoneStats[current] + current++ + return stat, nil + } + if err := objectioutil.GetTombstonesByBlockId(ctx, snapshotTS, &blockID, getTombstone, &mask, r.fs); err != nil { + mask.Release() + return objectio.NullBitmap, err + } + return mask, nil +} diff --git a/pkg/tools/checkpointtool/logical_table_test.go b/pkg/tools/checkpointtool/logical_table_test.go new file mode 100644 index 0000000000000..c3c81fb6dc79f --- /dev/null +++ b/pkg/tools/checkpointtool/logical_table_test.go @@ -0,0 +1,35 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "context" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildLogicalTableViewEmpty(t *testing.T) { + reader := &CheckpointReader{} + view, err := reader.BuildLogicalTableView(context.Background(), types.TS{}, nil, nil) + require.NoError(t, err) + assert.Equal(t, []string{"object", "block", "row"}, view.Headers) + assert.Empty(t, view.Rows) + assert.Equal(t, 0, view.PhysicalRows) + assert.Equal(t, 0, view.DeletedRows) + assert.Equal(t, 0, view.VisibleRows) +} diff --git a/pkg/tools/checkpointtool/types.go b/pkg/tools/checkpointtool/types.go index cc3ee2f09de88..3b6f350fc4154 100644 --- a/pkg/tools/checkpointtool/types.go +++ b/pkg/tools/checkpointtool/types.go @@ -80,3 +80,12 @@ type ComposedView struct { Incrementals []*EntryInfo Tables map[uint64]*TableInfo } + +// LogicalTableView contains a tombstone-applied table view for a checkpoint table. +type LogicalTableView struct { + Headers []string + Rows [][]string + PhysicalRows int + DeletedRows int + VisibleRows int +} diff --git a/pkg/tools/objecttool/interactive/repl.go b/pkg/tools/objecttool/interactive/repl.go index 14354a8447bba..8b68bf3903b76 100644 --- a/pkg/tools/objecttool/interactive/repl.go +++ b/pkg/tools/objecttool/interactive/repl.go @@ -14,9 +14,18 @@ package interactive -import "context" +import ( + "context" + + "github.com/matrixorigin/matrixone/pkg/fileservice" +) // Run runs the interactive interface func Run(path string) error { return RunUnified(context.Background(), path, nil) } + +// RunWithFS runs the interactive interface using an existing file service. +func RunWithFS(ctx context.Context, fs fileservice.FileService, path string) error { + return RunUnifiedWithFS(ctx, fs, path, nil) +} diff --git a/pkg/tools/objecttool/interactive/unified_model.go b/pkg/tools/objecttool/interactive/unified_model.go index 2f1f0a6697b48..e5aa06bc84ac9 100644 --- a/pkg/tools/objecttool/interactive/unified_model.go +++ b/pkg/tools/objecttool/interactive/unified_model.go @@ -18,6 +18,7 @@ import ( "context" tea "github.com/charmbracelet/bubbletea" + "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/tools/interactive" "github.com/matrixorigin/matrixone/pkg/tools/objecttool" ) @@ -199,6 +200,30 @@ func RunUnified(ctx context.Context, path string, opts *ViewOptions) error { } defer reader.Close() + return runUnifiedWithReader(ctx, reader, opts, func(nestedPath string) error { + return RunUnified(ctx, nestedPath, nil) + }) +} + +// RunUnifiedWithFS runs the object viewer using an existing file service. +func RunUnifiedWithFS(ctx context.Context, fs fileservice.FileService, path string, opts *ViewOptions) error { + reader, err := objecttool.OpenWithFS(ctx, fs, path, path) + if err != nil { + return err + } + defer reader.Close() + + return runUnifiedWithReader(ctx, reader, opts, func(nestedPath string) error { + return RunUnifiedWithFS(ctx, fs, nestedPath, nil) + }) +} + +func runUnifiedWithReader( + ctx context.Context, + reader *objecttool.ObjectReader, + opts *ViewOptions, + openNested func(string) error, +) error { m := NewObjectUnifiedModel(ctx, reader, opts) for { @@ -218,7 +243,7 @@ func RunUnified(ctx context.Context, path string, opts *ViewOptions) error { nestedPath := um.GetObjectToOpen() um.ClearObjectToOpen() // Open nested object with no special options - if err := RunUnified(ctx, nestedPath, nil); err != nil { + if err := openNested(nestedPath); err != nil { return err } // Continue with current viewer after returning diff --git a/pkg/tools/objecttool/object_reader.go b/pkg/tools/objecttool/object_reader.go index 1f5897f395936..9c45f28d89057 100644 --- a/pkg/tools/objecttool/object_reader.go +++ b/pkg/tools/objecttool/object_reader.go @@ -81,8 +81,12 @@ func Open(ctx context.Context, path string) (*ObjectReader, error) { return nil, moerr.NewInternalErrorf(ctx, "create file service: %v", err) } - // 3. Create reader - objReader, err := objectio.NewObjectReaderWithStr(filename, fs, + return OpenWithFS(ctx, fs, filename, path) +} + +// OpenWithFS opens an object file from an existing file service. +func OpenWithFS(ctx context.Context, fs fileservice.FileService, fileName string, displayPath string) (*ObjectReader, error) { + objReader, err := objectio.NewObjectReaderWithStr(fileName, fs, objectio.WithMetaCachePolicyOption(fileservice.SkipMemoryCache|fileservice.SkipFullFilePreloads)) if err != nil { return nil, moerr.NewInternalErrorf(ctx, "create object reader: %v", err) @@ -99,7 +103,7 @@ func Open(ctx context.Context, path string) (*ObjectReader, error) { dataMeta := meta.MustDataMeta() // 5. Build info - info := buildObjectInfo(path, dataMeta) + info := buildObjectInfo(displayPath, dataMeta) cols := buildColInfo(dataMeta) return &ObjectReader{ diff --git a/pkg/tools/toolfs/storage.go b/pkg/tools/toolfs/storage.go new file mode 100644 index 0000000000000..b83a9f25df5ca --- /dev/null +++ b/pkg/tools/toolfs/storage.go @@ -0,0 +1,145 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toolfs + +import ( + "context" + "fmt" + "strings" + + "github.com/BurntSushi/toml" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/fileservice" +) + +const defaultFSName = "SHARED" + +// StorageOptions describes a fileservice backend requested by a tool command. +type StorageOptions struct { + FSConfig string + FSName string + S3 string + Backend string +} + +// IsRemote reports whether the command should build an explicit fileservice. +func (o StorageOptions) IsRemote() bool { + return o.FSConfig != "" || o.S3 != "" || o.Backend != "" +} + +// Open creates a FileService from tool options. +func Open(ctx context.Context, opts StorageOptions) (fileservice.FileService, string, error) { + if opts.FSName == "" { + opts.FSName = defaultFSName + } + + if opts.FSConfig != "" { + return openFromConfig(ctx, opts.FSConfig, opts.FSName) + } + + backend := strings.ToUpper(opts.Backend) + if backend == "" { + backend = "S3" + } + if backend != "S3" && backend != "MINIO" { + return nil, "", moerr.NewInvalidInputNoCtxf("unsupported backend %q, use S3 or MINIO", opts.Backend) + } + if opts.S3 == "" { + return nil, "", moerr.NewInvalidInputNoCtx("missing --s3 arguments") + } + + args := fileservice.ObjectStorageArguments{Name: opts.FSName} + if err := args.SetFromString(splitArgs(opts.S3)); err != nil { + return nil, "", err + } + cfg := fileservice.Config{ + Name: opts.FSName, + Backend: backend, + S3: args, + Cache: fileservice.DisabledCacheConfig, + } + fs, err := fileservice.NewFileService(ctx, cfg, nil) + if err != nil { + return nil, "", err + } + return fs, fmt.Sprintf("%s:%s", strings.ToLower(backend), args.KeyPrefix), nil +} + +type launchConfig struct { + DataDir string `toml:"data-dir"` + FileService []fileservice.Config `toml:"fileservice"` + TN struct { + Txn struct { + Storage struct { + FileService string `toml:"fileservice"` + } `toml:"Storage"` + } `toml:"Txn"` + } `toml:"tn"` +} + +func openFromConfig(ctx context.Context, path string, fsName string) (fileservice.FileService, string, error) { + var cfg launchConfig + if _, err := toml.DecodeFile(path, &cfg); err != nil { + return nil, "", err + } + + if cfg.TN.Txn.Storage.FileService != "" && fsName == defaultFSName { + fsName = cfg.TN.Txn.Storage.FileService + } + for _, fsCfg := range cfg.FileService { + if strings.EqualFold(fsCfg.Name, fsName) { + fs, err := fileservice.NewFileService(ctx, fsCfg, nil) + if err != nil { + return nil, "", err + } + return fs, fmt.Sprintf("%s:%s", path, fsCfg.Name), nil + } + } + + if len(cfg.FileService) == 0 && cfg.DataDir != "" { + fsCfg := fileservice.Config{ + Name: fsName, + Backend: "DISK", + DataDir: cfg.DataDir, + Cache: fileservice.DisabledCacheConfig, + } + fs, err := fileservice.NewFileService(ctx, fsCfg, nil) + if err != nil { + return nil, "", err + } + return fs, fmt.Sprintf("%s:%s", path, cfg.DataDir), nil + } + + var names []string + for _, fsCfg := range cfg.FileService { + names = append(names, fsCfg.Name) + } + return nil, "", moerr.NewInvalidInputNoCtxf( + "fileservice %q not found in %s; available: %s", + fsName, path, strings.Join(names, ","), + ) +} + +func splitArgs(s string) []string { + parts := strings.Split(s, ",") + ret := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + ret = append(ret, part) + } + } + return ret +} diff --git a/pkg/tools/toolfs/storage_test.go b/pkg/tools/toolfs/storage_test.go new file mode 100644 index 0000000000000..99419129ae62c --- /dev/null +++ b/pkg/tools/toolfs/storage_test.go @@ -0,0 +1,83 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toolfs + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOpenFromConfigSelectsRequestedFileService(t *testing.T) { + dir := t.TempDir() + cfgPath := writeConfig(t, ` +[[fileservice]] +backend = "DISK" +data-dir = "`+dir+`" +name = "LOCAL" +`) + + fs, display, err := Open(context.Background(), StorageOptions{ + FSConfig: cfgPath, + FSName: "LOCAL", + }) + require.NoError(t, err) + defer fs.Close(context.Background()) + + assert.Contains(t, display, "LOCAL") + assert.Equal(t, "LOCAL", fs.Name()) +} + +func TestOpenFromConfigUsesTNStorageFileServiceByDefault(t *testing.T) { + dir := t.TempDir() + cfgPath := writeConfig(t, ` +[[fileservice]] +backend = "DISK" +data-dir = "`+dir+`" +name = "SHARED" + +[tn.Txn.Storage] +fileservice = "SHARED" +`) + + fs, _, err := Open(context.Background(), StorageOptions{FSConfig: cfgPath}) + require.NoError(t, err) + defer fs.Close(context.Background()) + + assert.Equal(t, "SHARED", fs.Name()) +} + +func TestOpenFromConfigFallsBackToDataDir(t *testing.T) { + dir := t.TempDir() + cfgPath := writeConfig(t, `data-dir = "`+dir+`"`) + + fs, display, err := Open(context.Background(), StorageOptions{FSConfig: cfgPath}) + require.NoError(t, err) + defer fs.Close(context.Background()) + + assert.Equal(t, "SHARED", fs.Name()) + assert.Contains(t, display, dir) +} + +func writeConfig(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "tn.toml") + require.NoError(t, os.WriteFile(path, []byte(content), 0644)) + return path +} From eaa1bfc110c4e6f119a9d09d6f94e626128bbca8 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 7 Jun 2026 04:04:01 +0800 Subject: [PATCH 02/76] Fix checkpoint schema replay and CSV column mapping --- cmd/mo-object-tool/ckp/checkpoint.go | 195 +++++ pkg/tools/checkpointtool/checkpoint_reader.go | 153 ++-- .../checkpointtool/interactive/bubbletea.go | 154 +--- .../checkpointtool/interactive/helpers.go | 108 +++ .../checkpointtool/interactive/providers.go | 44 +- .../interactive/unified_model.go | 18 +- pkg/tools/checkpointtool/logical_table.go | 86 ++- .../checkpointtool/logical_table_test.go | 60 ++ pkg/tools/checkpointtool/table_dump.go | 697 ++++++++++++++++++ pkg/tools/checkpointtool/table_dump_test.go | 505 +++++++++++++ pkg/tools/checkpointtool/types.go | 8 +- 11 files changed, 1716 insertions(+), 312 deletions(-) create mode 100644 pkg/tools/checkpointtool/table_dump.go create mode 100644 pkg/tools/checkpointtool/table_dump_test.go diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 47fa724d5b90b..c271f2144485d 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -18,7 +18,10 @@ import ( "context" "fmt" "os" + "strconv" + "strings" + "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool/interactive" @@ -45,6 +48,8 @@ func PrepareCommand() *cobra.Command { cmd.AddCommand(infoCommand(&storage)) cmd.AddCommand(viewCommand(&storage)) + cmd.AddCommand(dumpCommand(&storage)) + cmd.AddCommand(showCreateTableCommand(&storage)) return cmd } @@ -172,3 +177,193 @@ func openReader(ctx context.Context, dir string, storage toolfs.StorageOptions) } return checkpointtool.OpenWithFS(ctx, fs, display, checkpointtool.WithCloseFS()) } + +// dumpCommand implements the "ckp dump" subcommand for offline CSV export. +// +// Usage: +// +// mo-tool ckp dump --table-id=12345 [--ts=...] [--output=table.csv] [directory] +func dumpCommand(storage *toolfs.StorageOptions) *cobra.Command { + var ( + tableID uint64 + tsStr string + output string + ) + + cmd := &cobra.Command{ + Use: "dump [directory]", + Short: "Dump a table from checkpoint to CSV (offline)", + Long: `Dump a table from checkpoint to CSV with tombstone filtering applied. + +The schema (column names and visible columns) is resolved from mo_tables +and mo_columns system tables in the checkpoint. If that catalog metadata is +unavailable or incomplete, the command fails instead of exporting hidden +physical columns. + +Examples: + mo-tool ckp dump --table-id=12345 /path/to/ckp # dump to stdout + mo-tool ckp dump --table-id=12345 -o users.csv . # dump to file + mo-tool ckp dump --table-id=12345 --ts=1749001234567890:1 . # dump at specific TS`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := "." + if len(args) == 1 { + dir = args[0] + } + + if tableID == 0 { + return fmt.Errorf("--table-id is required") + } + + logFile, err := setupLogFile() + if err != nil { + return fmt.Errorf("setup log file: %w", err) + } + defer logFile.Close() + + ctx := context.Background() + reader, err := openReader(ctx, dir, *storage) + if err != nil { + return fmt.Errorf("open checkpoint dir: %w", err) + } + defer reader.Close() + + // Determine snapshot TS + snapshotTS := types.TS{} + if tsStr != "" { + snapshotTS, err = parseTS(tsStr) + if err != nil { + return fmt.Errorf("parse --ts: %w", err) + } + } else { + // Use the latest available TS + info := reader.Info() + if !info.LatestTS.IsEmpty() { + snapshotTS = info.LatestTS + } + } + + // Open output writer + var w = cmd.OutOrStdout() + var outFile *os.File + if output != "" { + outFile, err = os.Create(output) + if err != nil { + return fmt.Errorf("create output file: %w", err) + } + defer outFile.Close() + w = outFile + } + + if err := reader.DumpTableCSVComposed(ctx, w, tableID, snapshotTS); err != nil { + return fmt.Errorf("dump table %d: %w", tableID, err) + } + + if output != "" { + fmt.Fprintf(cmd.OutOrStdout(), "Table %d dumped to %s\n", tableID, output) + } + return nil + }, + } + + cmd.Flags().Uint64Var(&tableID, "table-id", 0, "Table ID to dump (required)") + cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp in 'physical:logical' format (default: latest)") + cmd.Flags().StringVarP(&output, "output", "o", "", "Output CSV file path (default: stdout)") + + return cmd +} + +// parseTS parses a timestamp string in "physical:logical" format. +func parseTS(s string) (types.TS, error) { + parts := strings.SplitN(s, ":", 2) + if len(parts) == 2 { + physical, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return types.TS{}, fmt.Errorf("invalid physical in timestamp: %w", err) + } + logical, err := strconv.ParseUint(parts[1], 10, 32) + if err != nil { + return types.TS{}, fmt.Errorf("invalid logical in timestamp: %w", err) + } + return types.BuildTS(physical, uint32(logical)), nil + } + return types.TS{}, fmt.Errorf("timestamp must be in 'physical:logical' format, got %q", s) +} + +// showCreateTableCommand implements the "ckp show-create-table" subcommand +// that prints a CREATE TABLE DDL for a table at a given checkpoint snapshot. +// +// Usage: +// +// mo-tool ckp show-create-table --table-id=12345 [--ts=...] [directory] +func showCreateTableCommand(storage *toolfs.StorageOptions) *cobra.Command { + var ( + tableID uint64 + tsStr string + ) + + cmd := &cobra.Command{ + Use: "show-create-table [directory]", + Short: "Show CREATE TABLE DDL for a table from checkpoint", + Long: `Display the CREATE TABLE SQL for a given table by reading the checkpoint's +mo_tables and mo_columns system tables (GCKP + following ICKPs). + +The DDL is resolved from mo_tables.rel_createsql if available, otherwise +reconstructed from mo_columns visible column definitions, with hardcoded +fallbacks for core built-in system tables. + +Examples: + mo-tool ckp show-create-table --table-id=12345 /path/to/ckp + mo-tool ckp show-create-table --table-id=2 --ts=1749001234567890:1 .`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := "." + if len(args) == 1 { + dir = args[0] + } + + if tableID == 0 { + return fmt.Errorf("--table-id is required") + } + + logFile, err := setupLogFile() + if err != nil { + return fmt.Errorf("setup log file: %w", err) + } + defer logFile.Close() + + ctx := context.Background() + reader, err := openReader(ctx, dir, *storage) + if err != nil { + return fmt.Errorf("open checkpoint dir: %w", err) + } + defer reader.Close() + + snapshotTS := types.TS{} + if tsStr != "" { + snapshotTS, err = parseTS(tsStr) + if err != nil { + return fmt.Errorf("parse --ts: %w", err) + } + } else { + info := reader.Info() + if !info.LatestTS.IsEmpty() { + snapshotTS = info.LatestTS + } + } + + ddl, err := reader.ShowCreateTable(ctx, tableID, snapshotTS) + if err != nil { + return fmt.Errorf("show create table %d: %w", tableID, err) + } + + fmt.Fprintln(cmd.OutOrStdout(), ddl) + return nil + }, + } + + cmd.Flags().Uint64Var(&tableID, "table-id", 0, "Table ID to show CREATE TABLE for (required)") + cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp in 'physical:logical' format (default: latest)") + + return cmd +} diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index 0cb64634ab563..422a0aab2e221 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -17,6 +17,7 @@ package checkpointtool import ( "context" "fmt" + "os" "sort" "github.com/matrixorigin/matrixone/pkg/common/moerr" @@ -25,9 +26,11 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/logtail" ) // CheckpointReader reads checkpoint data from a directory @@ -203,32 +206,19 @@ func (r *CheckpointReader) GetTableRanges(entry *checkpoint.CheckpointEntry) ([] return nil, nil } - // Check if file exists before trying to read - fileName := loc.Name().String() - _, err := r.fs.StatFile(r.ctx, fileName) + _, err := r.fs.StatFile(r.ctx, loc.Name().String()) if err != nil { - if moerr.IsMoErrCode(err, moerr.ErrFileNotFound) { - return nil, moerr.NewInternalErrorf(r.ctx, "checkpoint data file not found (may have been GC'd): %s", fileName) + if moerr.IsMoErrCode(err, moerr.ErrFileNotFound) || os.IsNotExist(err) { + return nil, moerr.NewFileNotFoundErrorf(r.ctx, "checkpoint data file not found (may have been GC'd): %s", loc.Name().String()) } return nil, err } - reader, err := ioutil.NewFileReader(r.fs, fileName) - if err != nil { - return nil, err - } - - bats, release, err := reader.LoadAllColumns(r.ctx, nil, r.mp) - if err != nil { + reader := logtail.NewCKPReader(entry.GetVersion(), loc, r.mp, r.fs) + if err := reader.ReadMeta(r.ctx); err != nil { return nil, err } - defer release() - - var ranges []ckputil.TableRange - for _, bat := range bats { - ranges = append(ranges, ckputil.ExportToTableRanges(bat)...) - } - return ranges, nil + return reader.GetTableRanges(r.ctx) } // GetAccounts extracts unique accounts from table ranges @@ -341,28 +331,31 @@ func (r *CheckpointReader) ReadTableData(ctx context.Context, rng ckputil.TableR func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { view := &ComposedView{Timestamp: ts, Tables: make(map[uint64]*TableInfo)} - // Find latest GCKP <= ts + // Find latest GCKP <= ts that still has data on disk. + // GC may have cleaned up older GCKP data files; skip those and use the next one. var baseEntry *checkpoint.CheckpointEntry + var baseEntryIdx int for i := len(r.entries) - 1; i >= 0; i-- { e := r.entries[i] end := e.GetEnd() if e.IsGlobal() && end.LE(&ts) { + tables, err := r.GetTables(e) + if err != nil { + if isDataFileNotFound(err) { + continue // GC'd entry; try the next older GCKP + } + return nil, err + } baseEntry = e + baseEntryIdx = i + for _, t := range tables { + view.Tables[t.TableID] = t + } + view.BaseEntry = r.EntryInfo(baseEntryIdx, baseEntry) break } } - if baseEntry != nil { - view.BaseEntry = r.EntryInfo(0, baseEntry) - tables, err := r.GetTables(baseEntry) - if err != nil { - return nil, err - } - for _, t := range tables { - view.Tables[t.TableID] = t - } - } - // Add ICKPs after GCKP.end and <= ts baseEnd := types.TS{} if baseEntry != nil { @@ -372,11 +365,14 @@ func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { start := e.GetStart() end := e.GetEnd() if e.IsIncremental() && start.GT(&baseEnd) && end.LE(&ts) { - view.Incrementals = append(view.Incrementals, r.EntryInfo(i, e)) tables, err := r.GetTables(e) if err != nil { + if isDataFileNotFound(err) { + continue // GC'd entry; skip + } return nil, err } + view.Incrementals = append(view.Incrementals, r.EntryInfo(i, e)) for _, t := range tables { if existing, ok := view.Tables[t.TableID]; ok { existing.DataRanges = append(existing.DataRanges, t.DataRanges...) @@ -390,6 +386,14 @@ func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { return view, nil } +// isDataFileNotFound checks if err indicates a checkpoint data file was GC'd/missing. +func isDataFileNotFound(err error) bool { + if err == nil { + return false + } + return moerr.IsMoErrCode(err, moerr.ErrFileNotFound) || os.IsNotExist(err) +} + // Close releases resources func (r *CheckpointReader) Close() error { r.entries = nil @@ -401,80 +405,47 @@ func (r *CheckpointReader) Close() error { // GetObjectEntries reads detailed object entries with timestamps for a table func (r *CheckpointReader) GetObjectEntries(entry *checkpoint.CheckpointEntry, tableID uint64) ([]*ObjectEntryInfo, []*ObjectEntryInfo, error) { - // First get all table ranges - allRanges, err := r.GetTableRanges(entry) - if err != nil { - return nil, nil, err - } - - // Now read the checkpoint file again to get timestamps loc := entry.GetLocation() if loc.IsEmpty() { return nil, nil, nil } - fileName := loc.Name().String() - reader, err := ioutil.NewFileReader(r.fs, fileName) - if err != nil { - return nil, nil, err - } - - bats, release, err := reader.LoadAllColumns(r.ctx, nil, r.mp) + _, err := r.fs.StatFile(r.ctx, loc.Name().String()) if err != nil { + if isDataFileNotFound(err) { + return nil, nil, moerr.NewFileNotFoundErrorf(r.ctx, "checkpoint data file not found (may have been GC'd): %s", loc.Name().String()) + } return nil, nil, err } - defer release() - // Build a map of ranges with their timestamps var dataEntries, tombEntries []*ObjectEntryInfo - rangeIdx := 0 - for _, bat := range bats { - if bat.RowCount() == 0 { - continue - } + reader := logtail.NewCKPReaderWithTableID_V2(entry.GetVersion(), loc, tableID, r.mp, r.fs) + if err := reader.ReadMeta(r.ctx); err != nil { + return nil, nil, err + } - // Check if timestamp columns exist - var createTSVec, deleteTSVec []types.TS - if len(bat.Vecs) > ckputil.TableObjectsAttr_CreateTS_Idx { - createTSVec = vector.MustFixedColWithTypeCheck[types.TS](bat.Vecs[ckputil.TableObjectsAttr_CreateTS_Idx]) - } - if len(bat.Vecs) > ckputil.TableObjectsAttr_DeleteTS_Idx { - deleteTSVec = vector.MustFixedColWithTypeCheck[types.TS](bat.Vecs[ckputil.TableObjectsAttr_DeleteTS_Idx]) + err = reader.ConsumeCheckpointWithTableID(r.ctx, func( + _ context.Context, + _ fileservice.FileService, + obj objectio.ObjectEntry, + isTombstone bool, + ) error { + info := &ObjectEntryInfo{ + ObjectStats: obj.ObjectStats, + CreateTime: obj.CreateTime, + DeleteTime: obj.DeleteTime, } - - for i := 0; i < bat.RowCount(); i++ { - if rangeIdx >= len(allRanges) { - break - } - - rng := allRanges[rangeIdx] - rangeIdx++ - - if rng.TableID != tableID { - continue - } - - entry := &ObjectEntryInfo{ - Range: rng, - } - - // Set timestamps if available - if createTSVec != nil && i < len(createTSVec) { - entry.CreateTime = createTSVec[i] - } - if deleteTSVec != nil && i < len(deleteTSVec) { - entry.DeleteTime = deleteTSVec[i] - } - - if rng.ObjectType == ckputil.ObjectType_Data { - dataEntries = append(dataEntries, entry) - } else if rng.ObjectType == ckputil.ObjectType_Tombstone { - tombEntries = append(tombEntries, entry) - } + if isTombstone { + tombEntries = append(tombEntries, info) + } else { + dataEntries = append(dataEntries, info) } + return nil + }) + if err != nil { + return nil, nil, err } - return dataEntries, tombEntries, nil } diff --git a/pkg/tools/checkpointtool/interactive/bubbletea.go b/pkg/tools/checkpointtool/interactive/bubbletea.go index dd53b2c4add18..cd11804d56f01 100644 --- a/pkg/tools/checkpointtool/interactive/bubbletea.go +++ b/pkg/tools/checkpointtool/interactive/bubbletea.go @@ -16,14 +16,10 @@ package interactive import ( "context" - "fmt" tea "github.com/charmbracelet/bubbletea" - "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" objectinteractive "github.com/matrixorigin/matrixone/pkg/tools/objecttool/interactive" - "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" ) // Run starts the interactive checkpoint viewer @@ -43,42 +39,9 @@ func Run(reader *checkpointtool.CheckpointReader) error { } // Check if we need to open an object file - if um.GetObjectToOpen() != "" && um.GetRangeToOpen() != nil { - rng := um.GetRangeToOpen() - opts := &objectinteractive.ViewOptions{ - StartRow: int64(rng.Start.GetRowOffset()), - EndRow: int64(rng.End.GetRowOffset()), - ColumnNames: map[uint16]string{ - 0: ckputil.TableObjectsAttr_Accout, - 1: ckputil.TableObjectsAttr_DB, - 2: ckputil.TableObjectsAttr_Table, - 3: ckputil.TableObjectsAttr_ObjectType, - 4: "object_name", - 5: "flags", - 6: "rows", - 7: "osize", - 8: "csize", - 9: ckputil.TableObjectsAttr_CreateTS, - 10: ckputil.TableObjectsAttr_DeleteTS, - 11: ckputil.TableObjectsAttr_Cluster, - }, - ColumnExpander: &objectinteractive.ColumnExpander{ - SourceCol: 4, // id column - NewCols: []string{"object_name", "flags", "rows", "osize", "csize"}, - NewTypes: []types.Type{ - types.T_varchar.ToType(), - types.T_varchar.ToType(), - types.T_uint32.ToType(), - types.T_varchar.ToType(), - types.T_varchar.ToType(), - }, - ExpandFunc: expandObjectStats, - }, - ObjectNameCol: 4, // object_name column after expansion - CustomOverview: ckpDataOverview, - } + if um.GetObjectToOpen() != "" { if err := objectinteractive.RunUnifiedWithFS( - context.Background(), m.state.reader.FS(), um.GetObjectToOpen(), opts, + context.Background(), m.state.reader.FS(), um.GetObjectToOpen(), nil, ); err != nil { return err } @@ -90,116 +53,3 @@ func Run(reader *checkpointtool.CheckpointReader) error { return nil } } - -// ckpDataOverview creates custom overview for checkpoint data viewer -// Columns after expansion: account_id, db_id, table_id, object_type, object_name, flags, rows, osize, csize, create_ts, delete_ts, cluster -// Indices: 0 1 2 3 4 5 6 7 8 9 10 11 -func ckpDataOverview(rows [][]string) string { - if len(rows) == 0 { - return "No data" - } - - type objStats struct { - rows uint32 - osize uint32 - csize uint32 - } - dataObjs := make(map[string]*objStats) - tombObjs := make(map[string]*objStats) - - for _, row := range rows { - if len(row) < 9 { - continue - } - objType := row[3] // object_type: "1" = data, "2" = tombstone - objName := row[4] // object_name - rowsStr := row[6] // rows - osize := parseSize(row[7]) - csize := parseSize(row[8]) - - var rowCount uint32 - fmt.Sscanf(rowsStr, "%d", &rowCount) - - if objType == "1" { // Data - if _, exists := dataObjs[objName]; !exists { - dataObjs[objName] = &objStats{rows: rowCount, osize: osize, csize: csize} - } - } else if objType == "2" { // Tombstone - if _, exists := tombObjs[objName]; !exists { - tombObjs[objName] = &objStats{rows: rowCount, osize: osize, csize: csize} - } - } - } - - var totalRows, totalDeletes, totalOsize, totalCsize uint32 - for _, s := range dataObjs { - totalRows += s.rows - totalOsize += s.osize - totalCsize += s.csize - } - for _, s := range tombObjs { - totalDeletes += s.rows - } - - ratio := float64(0) - if totalOsize > 0 { - ratio = float64(totalCsize) / float64(totalOsize) * 100 - } - - return fmt.Sprintf("%d ranges │ %d data objs │ %d tomb objs │ %d rows │ %d deletes │ osize: %s │ csize: %s │ ratio: %.1f%%", - len(rows), len(dataObjs), len(tombObjs), totalRows, totalDeletes, - formatSize(totalOsize), formatSize(totalCsize), ratio) -} - -// parseSize parses size string like "279.6KB" back to bytes -func parseSize(s string) uint32 { - var val float64 - var unit string - fmt.Sscanf(s, "%f%s", &val, &unit) - switch unit { - case "KB": - return uint32(val * 1024) - case "MB": - return uint32(val * 1024 * 1024) - case "GB": - return uint32(val * 1024 * 1024 * 1024) - case "B": - return uint32(val) - default: - return uint32(val) - } -} - -// expandObjectStats expands ObjectStats bytes into multiple values -func expandObjectStats(value any) []any { - data, ok := value.([]byte) - if !ok || len(data) != objectio.ObjectStatsLen { - return []any{"", "", "", "", ""} - } - - var stats objectio.ObjectStats - stats.UnMarshal(data) - - // Format flags: A=Appendable, S=Sorted, C=CNCreated - flags := "" - if stats.GetAppendable() { - flags += "A" - } - if stats.GetSorted() { - flags += "S" - } - if stats.GetCNCreated() { - flags += "C" - } - if flags == "" { - flags = "-" - } - - return []any{ - stats.ObjectName().String(), - flags, - stats.Rows(), - formatSize(stats.OriginSize()), - formatSize(stats.Size()), - } -} diff --git a/pkg/tools/checkpointtool/interactive/helpers.go b/pkg/tools/checkpointtool/interactive/helpers.go index ee90f49b59d39..144594c9fa349 100644 --- a/pkg/tools/checkpointtool/interactive/helpers.go +++ b/pkg/tools/checkpointtool/interactive/helpers.go @@ -19,6 +19,7 @@ import ( "time" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" ) @@ -66,3 +67,110 @@ func formatSize(bytes uint32) string { } return fmt.Sprintf("%.1f%cB", float64(bytes)/float64(div), "KMGTPE"[exp]) } + +func parseSize(s string) uint32 { + var val float64 + var unit string + fmt.Sscanf(s, "%f%s", &val, &unit) + switch unit { + case "KB": + return uint32(val * 1024) + case "MB": + return uint32(val * 1024 * 1024) + case "GB": + return uint32(val * 1024 * 1024 * 1024) + case "B": + return uint32(val) + default: + return uint32(val) + } +} + +func expandObjectStats(value any) []any { + data, ok := value.([]byte) + if !ok || len(data) != objectio.ObjectStatsLen { + return []any{"", "", "", "", ""} + } + + var stats objectio.ObjectStats + stats.UnMarshal(data) + + flags := "" + if stats.GetAppendable() { + flags += "A" + } + if stats.GetSorted() { + flags += "S" + } + if stats.GetCNCreated() { + flags += "C" + } + if flags == "" { + flags = "-" + } + + return []any{ + stats.ObjectName().String(), + flags, + stats.Rows(), + formatSize(stats.OriginSize()), + formatSize(stats.Size()), + } +} + +func ckpDataOverview(rows [][]string) string { + if len(rows) == 0 { + return "No data" + } + + type objStats struct { + rows uint32 + osize uint32 + csize uint32 + } + dataObjs := make(map[string]*objStats) + tombObjs := make(map[string]*objStats) + + for _, row := range rows { + if len(row) < 9 { + continue + } + objType := row[3] + objName := row[4] + rowsStr := row[6] + osize := parseSize(row[7]) + csize := parseSize(row[8]) + + var rowCount uint32 + fmt.Sscanf(rowsStr, "%d", &rowCount) + + if objType == "1" { + if _, exists := dataObjs[objName]; !exists { + dataObjs[objName] = &objStats{rows: rowCount, osize: osize, csize: csize} + } + } else if objType == "2" { + if _, exists := tombObjs[objName]; !exists { + tombObjs[objName] = &objStats{rows: rowCount, osize: osize, csize: csize} + } + } + } + + var totalRows, totalDeletes, totalOsize, totalCsize uint32 + for _, s := range dataObjs { + totalRows += s.rows + totalOsize += s.osize + totalCsize += s.csize + } + for _, s := range tombObjs { + totalDeletes += s.rows + } + + ratio := float64(0) + if totalOsize > 0 { + ratio = float64(totalCsize) / float64(totalOsize) * 100 + } + + return fmt.Sprintf("%d ranges │ %d data objs │ %d tomb objs │ %d rows │ %d deletes │ osize: %s │ csize: %s │ ratio: %.1f%%", + len(rows), len(dataObjs), len(tombObjs), totalRows, totalDeletes, + formatSize(totalOsize), formatSize(totalCsize), ratio) +} diff --git a/pkg/tools/checkpointtool/interactive/providers.go b/pkg/tools/checkpointtool/interactive/providers.go index 5c2d523e11742..b8af4dbc09d32 100644 --- a/pkg/tools/checkpointtool/interactive/providers.go +++ b/pkg/tools/checkpointtool/interactive/providers.go @@ -161,35 +161,37 @@ func (p *TableDetailProvider) GetRows() [][]string { rows := make([][]string, 0, len(dataEntries)+len(tombEntries)) for _, entry := range dataEntries { - objName := entry.Range.ObjectStats.ObjectName().String() - rangeStr := fmt.Sprintf("%d-%d~%d-%d", - entry.Range.Start.GetBlockOffset(), entry.Range.Start.GetRowOffset(), - entry.Range.End.GetBlockOffset(), entry.Range.End.GetRowOffset()) - rangeRows := entry.Range.End.GetRowOffset() - entry.Range.Start.GetRowOffset() + 1 + objName := entry.ObjectStats.ObjectName().String() + blockEnd := int(entry.ObjectStats.BlkCnt()) - 1 + if blockEnd < 0 { + blockEnd = 0 + } + blockSpan := fmt.Sprintf("0..%d", blockEnd) rows = append(rows, []string{ "Data", objName, - rangeStr, - fmt.Sprintf("%d", rangeRows), - formatSize(entry.Range.ObjectStats.Size()), + blockSpan, + fmt.Sprintf("%d", entry.ObjectStats.Rows()), + formatSize(entry.ObjectStats.Size()), formatTSShort(entry.CreateTime), }) } for _, entry := range tombEntries { - objName := entry.Range.ObjectStats.ObjectName().String() - rangeStr := fmt.Sprintf("%d-%d~%d-%d", - entry.Range.Start.GetBlockOffset(), entry.Range.Start.GetRowOffset(), - entry.Range.End.GetBlockOffset(), entry.Range.End.GetRowOffset()) - rangeRows := entry.Range.End.GetRowOffset() - entry.Range.Start.GetRowOffset() + 1 + objName := entry.ObjectStats.ObjectName().String() + blockEnd := int(entry.ObjectStats.BlkCnt()) - 1 + if blockEnd < 0 { + blockEnd = 0 + } + blockSpan := fmt.Sprintf("0..%d", blockEnd) rows = append(rows, []string{ "Tomb", objName, - rangeStr, - fmt.Sprintf("%d", rangeRows), - formatSize(entry.Range.ObjectStats.Size()), + blockSpan, + fmt.Sprintf("%d", entry.ObjectStats.Rows()), + formatSize(entry.ObjectStats.Size()), formatTSShort(entry.CreateTime), }) } @@ -217,16 +219,16 @@ func (p *TableDetailProvider) GetOverview() string { tombObjects := make(map[string]struct{}) for _, e := range dataEntries { - objName := e.Range.ObjectStats.ObjectName().String() + objName := e.ObjectStats.ObjectName().String() if _, exists := dataObjects[objName]; !exists { dataObjects[objName] = &objStats{ - originSize: e.Range.ObjectStats.OriginSize(), - compSize: e.Range.ObjectStats.Size(), + originSize: e.ObjectStats.OriginSize(), + compSize: e.ObjectStats.Size(), } } } for _, e := range tombEntries { - objName := e.Range.ObjectStats.ObjectName().String() + objName := e.ObjectStats.ObjectName().String() tombObjects[objName] = struct{}{} } @@ -287,7 +289,7 @@ func (h *tableDetailHandler) OnSelect(rowIdx int) tea.Cmd { } if entry != nil { - return func() tea.Msg { return openObjectMsg{path: entry.Range.ObjectStats.ObjectName().String()} } + return func() tea.Msg { return openObjectMsg{path: entry.ObjectStats.ObjectName().String()} } } return nil } diff --git a/pkg/tools/checkpointtool/interactive/unified_model.go b/pkg/tools/checkpointtool/interactive/unified_model.go index acb2cb4199deb..b7951327f8b85 100644 --- a/pkg/tools/checkpointtool/interactive/unified_model.go +++ b/pkg/tools/checkpointtool/interactive/unified_model.go @@ -18,7 +18,6 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/interactive" - "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" ) // Messages for page navigation @@ -36,7 +35,6 @@ type UnifiedModel struct { // For opening objects objectToOpen string - rangeToOpen *ckputil.TableRange quitting bool } @@ -87,7 +85,7 @@ func (m *UnifiedModel) createTablesListPage() *interactive.GenericPage { func (m *UnifiedModel) createTableDetailPage() *interactive.GenericPage { config := interactive.PageConfig{ Title: "═══ Table Detail ═══", - Headers: []string{"Type", "Object", "Range", "Rows", "Size", "Created"}, + Headers: []string{"Type", "Object", "Blocks", "Rows", "Size", "Created"}, ShowRowNumber: true, EnableCursor: true, EnableSearch: true, @@ -152,14 +150,6 @@ func (m *UnifiedModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case openObjectMsg: m.objectToOpen = msg.path - idx := m.currentPage.GetCursor() - dataEntries := m.state.DataEntries() - tombEntries := m.state.TombEntries() - if idx < len(dataEntries) { - m.rangeToOpen = &dataEntries[idx].Range - } else if idx < len(dataEntries)+len(tombEntries) { - m.rangeToOpen = &tombEntries[idx-len(dataEntries)].Range - } m.quitting = true return m, tea.Quit @@ -205,13 +195,7 @@ func (m *UnifiedModel) GetObjectToOpen() string { return m.objectToOpen } -// GetRangeToOpen returns the range to open (if any) -func (m *UnifiedModel) GetRangeToOpen() *ckputil.TableRange { - return m.rangeToOpen -} - // ClearObjectToOpen clears the object to open flag func (m *UnifiedModel) ClearObjectToOpen() { m.objectToOpen = "" - m.rangeToOpen = nil } diff --git a/pkg/tools/checkpointtool/logical_table.go b/pkg/tools/checkpointtool/logical_table.go index 8d129589ded42..2c2f760b7baee 100644 --- a/pkg/tools/checkpointtool/logical_table.go +++ b/pkg/tools/checkpointtool/logical_table.go @@ -17,6 +17,7 @@ package checkpointtool import ( "context" "fmt" + "sort" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" @@ -40,11 +41,16 @@ func (r *CheckpointReader) BuildLogicalTableView( return view, nil } - tombstoneStats := dedupeObjectStats(tombEntries) - for _, entry := range dataEntries { - objName := entry.Range.ObjectStats.ObjectName().String() + visibleDataEntries := visibleObjectEntries(dataEntries, snapshotTS) + visibleTombEntries := visibleObjectEntries(tombEntries, snapshotTS) + tombstoneStats := dedupeObjectStats(visibleTombEntries) + for _, entry := range visibleDataEntries { + objName := entry.ObjectStats.ObjectName().String() reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) if err != nil { + if isDataFileNotFound(err) { + continue // GC'd data object; skip + } return nil, err } @@ -52,60 +58,45 @@ func (r *CheckpointReader) BuildLogicalTableView( cols := reader.Columns() for _, col := range cols { view.Headers = append(view.Headers, fmt.Sprintf("col_%d", col.Idx)) + view.ColTypes = append(view.ColTypes, col.Type) } } - relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.Range.ObjectStats.ObjectName().ObjectId(), tombstoneStats) + relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.ObjectStats.ObjectName().ObjectId(), tombstoneStats) if err != nil { _ = reader.Close() return nil, err } - startBlock := int(entry.Range.Start.GetBlockOffset()) - endBlock := int(entry.Range.End.GetBlockOffset()) - for blockIdx := startBlock; blockIdx <= endBlock; blockIdx++ { + for blockIdx := 0; blockIdx < int(entry.ObjectStats.BlkCnt()); blockIdx++ { bat, release, err := reader.ReadBlock(ctx, uint32(blockIdx)) if err != nil { _ = reader.Close() return nil, err } - startRow := 0 - endRow := bat.RowCount() - 1 - if blockIdx == startBlock { - startRow = int(entry.Range.Start.GetRowOffset()) - } - if blockIdx == endBlock { - endRow = int(entry.Range.End.GetRowOffset()) - } - if startRow < 0 { - startRow = 0 - } - if endRow >= bat.RowCount() { - endRow = bat.RowCount() - 1 - } - if startRow > endRow || startRow >= bat.RowCount() { + if bat.RowCount() == 0 { release() continue } - view.PhysicalRows += endRow - startRow + 1 + view.PhysicalRows += bat.RowCount() - deleteMask, err := r.buildDeleteMaskForBlock(ctx, &snapshotTS, entry.Range.ObjectStats, uint16(blockIdx), relevantTombstones) + deleteMask, err := r.buildDeleteMaskForBlock(ctx, &snapshotTS, entry.ObjectStats, uint16(blockIdx), relevantTombstones) if err != nil { release() _ = reader.Close() return nil, err } - for rowIdx := startRow; rowIdx <= endRow; rowIdx++ { + for rowIdx := 0; rowIdx < bat.RowCount(); rowIdx++ { if deleteMask.IsValid() && deleteMask.Contains(uint64(rowIdx)) { view.DeletedRows++ continue } row := make([]string, 0, len(bat.Vecs)+3) row = append(row, - entry.Range.ObjectStats.ObjectName().Short().ShortString(), + entry.ObjectStats.ObjectName().Short().ShortString(), fmt.Sprintf("%d", blockIdx), fmt.Sprintf("%d", rowIdx), ) @@ -134,16 +125,55 @@ func dedupeObjectStats(entries []*ObjectEntryInfo) []objectio.ObjectStats { seen := make(map[string]struct{}) stats := make([]objectio.ObjectStats, 0, len(entries)) for _, entry := range entries { - name := entry.Range.ObjectStats.ObjectName().String() + name := entry.ObjectStats.ObjectName().String() if _, ok := seen[name]; ok { continue } seen[name] = struct{}{} - stats = append(stats, entry.Range.ObjectStats) + stats = append(stats, entry.ObjectStats) } return stats } +func visibleObjectEntries(entries []*ObjectEntryInfo, snapshotTS types.TS) []*ObjectEntryInfo { + if len(entries) == 0 { + return nil + } + + merged := make(map[string]*ObjectEntryInfo, len(entries)) + for _, entry := range entries { + name := entry.ObjectStats.ObjectName().String() + existing, ok := merged[name] + if !ok { + copyEntry := *entry + merged[name] = ©Entry + continue + } + if existing.CreateTime.IsEmpty() || (!entry.CreateTime.IsEmpty() && entry.CreateTime.LT(&existing.CreateTime)) { + existing.CreateTime = entry.CreateTime + } + if existing.DeleteTime.IsEmpty() || (!entry.DeleteTime.IsEmpty() && existing.DeleteTime.LT(&entry.DeleteTime)) { + existing.DeleteTime = entry.DeleteTime + } + } + + visible := make([]*ObjectEntryInfo, 0, len(merged)) + for _, entry := range merged { + obj := objectio.ObjectEntry{ + ObjectStats: entry.ObjectStats, + CreateTime: entry.CreateTime, + DeleteTime: entry.DeleteTime, + } + if obj.Visible(snapshotTS) { + visible = append(visible, entry) + } + } + sort.Slice(visible, func(i, j int) bool { + return visible[i].ObjectStats.ObjectName().String() < visible[j].ObjectStats.ObjectName().String() + }) + return visible +} + func (r *CheckpointReader) filterTombstonesForObject( ctx context.Context, objectID *objectio.ObjectId, diff --git a/pkg/tools/checkpointtool/logical_table_test.go b/pkg/tools/checkpointtool/logical_table_test.go index c3c81fb6dc79f..a0a844353112d 100644 --- a/pkg/tools/checkpointtool/logical_table_test.go +++ b/pkg/tools/checkpointtool/logical_table_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,3 +34,62 @@ func TestBuildLogicalTableViewEmpty(t *testing.T) { assert.Equal(t, 0, view.DeletedRows) assert.Equal(t, 0, view.VisibleRows) } + +func TestVisibleObjectEntriesFiltersAndDedupesBySnapshot(t *testing.T) { + snapshot := types.BuildTS(15, 0) + entries := []*ObjectEntryInfo{ + newTestObjectEntryInfo(1, 10, 0), + newTestObjectEntryInfo(1, 12, 20), + newTestObjectEntryInfo(2, 16, 0), // created after snapshot + newTestObjectEntryInfo(3, 1, 14), // deleted before snapshot + } + + visible := visibleObjectEntries(entries, snapshot) + require.Len(t, visible, 1) + assert.Equal(t, uint32(1), visible[0].ObjectStats.Rows()) + assert.Equal(t, types.BuildTS(10, 0), visible[0].CreateTime) + assert.Equal(t, types.BuildTS(20, 0), visible[0].DeleteTime) +} + +func TestVisibleObjectEntriesDeletedAfterSnapshotBecomesInvisibleLater(t *testing.T) { + entries := []*ObjectEntryInfo{ + newTestObjectEntryInfo(7, 5, 0), + newTestObjectEntryInfo(7, 8, 30), + } + + visibleAt20 := visibleObjectEntries(entries, types.BuildTS(20, 0)) + require.Len(t, visibleAt20, 1) + + visibleAt30 := visibleObjectEntries(entries, types.BuildTS(30, 0)) + assert.Empty(t, visibleAt30) +} + +func TestDedupeObjectStats(t *testing.T) { + entries := []*ObjectEntryInfo{ + newTestObjectEntryInfo(1, 1, 0), + newTestObjectEntryInfo(1, 2, 0), + newTestObjectEntryInfo(2, 3, 0), + } + + stats := dedupeObjectStats(entries) + require.Len(t, stats, 2) + assert.NotEqual(t, stats[0].ObjectName().String(), stats[1].ObjectName().String()) +} + +func newTestObjectEntryInfo(idByte byte, createPhysical int64, deletePhysical int64) *ObjectEntryInfo { + var objectID objectio.ObjectId + objectID[0] = idByte + stats := objectio.NewObjectStats() + _ = objectio.SetObjectStatsObjectName(stats, objectio.BuildObjectNameWithObjectID(&objectID)) + _ = objectio.SetObjectStatsRowCnt(stats, uint32(idByte)) + _ = objectio.SetObjectStatsBlkCnt(stats, 1) + + entry := &ObjectEntryInfo{ + ObjectStats: *stats, + CreateTime: types.BuildTS(createPhysical, 0), + } + if deletePhysical > 0 { + entry.DeleteTime = types.BuildTS(deletePhysical, 0) + } + return entry +} diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go new file mode 100644 index 0000000000000..cd496b343cc35 --- /dev/null +++ b/pkg/tools/checkpointtool/table_dump.go @@ -0,0 +1,697 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "sort" + "strconv" + "strings" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +const ( + // Meta columns in LogicalTableView (object, block, row) + logicalViewMetaCols = 3 + + // System table IDs + moTablesID = uint64(catalog.MO_TABLES_ID) + moColumnsID = uint64(catalog.MO_COLUMNS_ID) +) + +// TableColumn describes one column in a user table schema. +type TableColumn struct { + Name string // SQL column name + SQLType string // SQL type string (e.g. "BIGINT", "VARCHAR(100)") + Position int // SQL ordinal position + PhysicalPosition int // physical/object column position +} + +// TableSchema holds the decoded schema for one user table. +type TableSchema struct { + TableName string + DatabaseName string + Columns []TableColumn // sorted by Position + CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql +} + +// ReadTableSchema reads the schema for a user table by reading mo_tables and mo_columns +// from the checkpoint. +// +// It composes the logical view at snapshotTS, reads system table data for mo_tables (table 2) +// and mo_columns (table 3), and extracts the schema for the given user tableID. +// +// IMPORTANT: LogicalTableView includes hidden columns (e.g. __mo_fake_pk_col) in its data +// rows, so we cannot use catalog index constants directly. Instead, column positions are +// resolved by name from the view headers. +// +// If the schema cannot be resolved from checkpoint catalog metadata, the returned schema +// will not contain visible columns. Callers that need an exact user-facing schema must +// treat that as an error instead of falling back to raw physical object columns. +func (r *CheckpointReader) ReadTableSchema( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, + dataView *LogicalTableView, +) *TableSchema { + _ = dataView + schema := &TableSchema{TableName: fmt.Sprintf("%d", tableID)} + + // Try to read mo_tables from the checkpoint (table 2) + moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) + if err != nil { + // Don't log - expected for system tables or when mo_tables not in checkpoint + } else if moTablesView != nil { + // Resolve column positions by name (handles hidden column offset) + relIDCol := moTablesView.columnDataIndex("rel_id") + if relIDCol < 0 { + relIDCol = moTablesPhysRelID + } + for _, row := range moTablesView.Rows { + dataRow := row[logicalViewMetaCols:] + if relIDCol < 0 || relIDCol >= len(dataRow) { + continue + } + if dataRow[relIDCol] == fmt.Sprintf("%d", tableID) { + schema = buildSchemaFromMoTablesRow(moTablesView, row) + break + } + } + } + + // Try to read mo_columns from the checkpoint (table 3) + if schema.TableName == fmt.Sprintf("%d", tableID) { + // Couldn't get table name from mo_tables; still try for columns + } + moColumnsView, err := r.getTableLogicalView(ctx, moColumnsID, snapshotTS) + if err == nil && moColumnsView != nil { + cols := buildColumnsFromMoColumnsRows(moColumnsView, tableID) + if len(cols) > 0 { + schema.Columns = cols + } + } + + return schema +} + +// getTableLogicalView composes the checkpoint view at snapshotTS and builds a logical +// table view for the given tableID by aggregating data/tomb entries across all +// relevant checkpoint entries (GCKP + ICKPs). +func (r *CheckpointReader) getTableLogicalView( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) (*LogicalTableView, error) { + composed, err := r.ComposeAt(snapshotTS) + if err != nil { + return nil, err + } + + tbl, ok := composed.Tables[tableID] + if !ok || (len(tbl.DataRanges) == 0 && len(tbl.TombRanges) == 0) { + return nil, moerr.NewInternalErrorf(ctx, "table %d not found in checkpoint at ts %s", tableID, snapshotTS.ToString()) + } + + // Collect all entries that contain this table (base + incrementals) + type entryRef struct { + entry *EntryInfo + } + var entryRefs []entryRef + if composed.BaseEntry != nil { + entryRefs = append(entryRefs, entryRef{entry: composed.BaseEntry}) + } + for _, incr := range composed.Incrementals { + entryRefs = append(entryRefs, entryRef{entry: incr}) + } + + // Aggregate data/tomb entries across all checkpoint entries + var allData, allTomb []*ObjectEntryInfo + for _, ref := range entryRefs { + e := r.entries[ref.entry.Index] + dataEntries, tombEntries, err := r.GetObjectEntries(e, tableID) + if err != nil { + continue // skip entries that don't have this table + } + allData = append(allData, dataEntries...) + allTomb = append(allTomb, tombEntries...) + } + + if len(allData) == 0 { + return nil, moerr.NewInternalErrorf(ctx, "no data entries for table %d", tableID) + } + + return r.BuildLogicalTableView(ctx, snapshotTS, allData, allTomb) +} + +// DumpTableCSV reads a user table's logical view and writes it as CSV to w. +// +// It first resolves the table schema from mo_tables/mo_columns, then writes the +// CSV with proper column headers and data rows. +func (r *CheckpointReader) DumpTableCSV( + ctx context.Context, + w io.Writer, + tableID uint64, + snapshotTS types.TS, + dataEntries, tombEntries []*ObjectEntryInfo, +) error { + view, err := r.BuildLogicalTableView(ctx, snapshotTS, dataEntries, tombEntries) + if err != nil { + return err + } + + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, view) + if len(schema.Columns) == 0 { + return moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) + } + return WriteCSV(w, schema, view) +} + +// DumpTableCSVComposed dumps a table to CSV by composing the full checkpoint view +// at snapshotTS (aggregating across GCKP + all ICKPs). This is equivalent to calling +// DumpTableCSV with entries from ComposeAt. +func (r *CheckpointReader) DumpTableCSVComposed( + ctx context.Context, + w io.Writer, + tableID uint64, + snapshotTS types.TS, +) error { + view, err := r.getTableLogicalView(ctx, tableID, snapshotTS) + if err != nil { + return err + } + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, view) + if len(schema.Columns) == 0 { + return moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) + } + return WriteCSV(w, schema, view) +} + +// WriteCSV writes a LogicalTableView with the given schema as CSV to w. +func WriteCSV(w io.Writer, schema *TableSchema, view *LogicalTableView) error { + cw := csv.NewWriter(w) + defer cw.Flush() + + // Write header comments (DDL + metadata) + if schema.CreateSQL != "" { + if _, err := fmt.Fprintf(w, "-- %s\n", schema.CreateSQL); err != nil { + return err + } + } + if schema.DatabaseName != "" { + if _, err := fmt.Fprintf(w, "-- Database: %s\n", schema.DatabaseName); err != nil { + return err + } + } + if schema.TableName != "" { + if _, err := fmt.Fprintf(w, "-- Table: %s\n", schema.TableName); err != nil { + return err + } + } + if _, err := fmt.Fprintf(w, "-- Visible rows: %d (deleted: %d, physical: %d)\n", + view.VisibleRows, view.DeletedRows, view.PhysicalRows); err != nil { + return err + } + + // Merge schema column names into headers + merged := MergeLogicalViewWithSchema(view, schema) + if err := cw.Write(merged.Headers); err != nil { + return err + } + + // Write data rows (skip the 3 meta columns) + for _, row := range merged.Rows { + if err := cw.Write(row); err != nil { + return err + } + } + + cw.Flush() + return cw.Error() +} + +// MergeLogicalViewWithSchema replaces col_N headers with real column names from the schema +// and filters data rows to only include visible (non-hidden) columns by their physical position. +// Returns a new LogicalTableView with merged headers and filtered data rows. +func MergeLogicalViewWithSchema(view *LogicalTableView, schema *TableSchema) *LogicalTableView { + dataWidth := len(view.Headers) - logicalViewMetaCols + if dataWidth < 0 { + dataWidth = 0 + } + + // Without schema columns we cannot safely distinguish visible columns from hidden ones. + if len(schema.Columns) == 0 { + return &LogicalTableView{ + Headers: nil, + Rows: nil, + VisibleRows: view.VisibleRows, + DeletedRows: view.DeletedRows, + PhysicalRows: view.PhysicalRows, + } + } + + // Build visible column headers and their physical data positions + newHeaders := make([]string, 0, len(schema.Columns)) + colMap := make([]int, 0, len(schema.Columns)) // visibleIdx → physical data column index + + for _, col := range schema.Columns { + newHeaders = append(newHeaders, col.Name) + physicalPos := col.PhysicalPosition + if physicalPos < 0 { + physicalPos = col.Position + } + colMap = append(colMap, physicalPos) + } + + // Extract data rows: pick only visible columns by their physical position + newRows := make([][]string, len(view.Rows)) + for i, row := range view.Rows { + newRow := make([]string, len(colMap)) + for j, pos := range colMap { + dataIdx := logicalViewMetaCols + pos + if dataIdx < len(row) { + newRow[j] = row[dataIdx] + } + } + newRows[i] = newRow + } + + return &LogicalTableView{ + Headers: newHeaders, + Rows: newRows, + VisibleRows: view.VisibleRows, + DeletedRows: view.DeletedRows, + PhysicalRows: view.PhysicalRows, + } +} + +// buildSchemaFromMoTablesRow extracts a TableSchema from a mo_tables data row. +// Uses column name lookup from view headers to handle hidden column offsets. +// mo_tables columns: rel_id, relname, reldatabase, reldatabase_id, ..., rel_createsql +func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *TableSchema { + schema := &TableSchema{} + dataRow := fullRow[logicalViewMetaCols:] + + relNameIdx := view.columnDataIndex("relname") + if relNameIdx < 0 { + relNameIdx = moTablesPhysRelName + } + if relNameIdx < len(dataRow) { + schema.TableName = dataRow[relNameIdx] + } + + relDBIdx := view.columnDataIndex("reldatabase") + if relDBIdx < 0 { + relDBIdx = moTablesPhysRelDatabase + } + if relDBIdx < len(dataRow) { + schema.DatabaseName = dataRow[relDBIdx] + } + + createSQLIdx := view.columnDataIndex("rel_createsql") + if createSQLIdx < 0 { + createSQLIdx = moTablesPhysRelCreateSQL + } + if createSQLIdx < len(dataRow) { + schema.CreateSQL = dataRow[createSQLIdx] + } + return schema +} + +// buildColumnsFromMoColumnsRows builds a sorted column list from mo_columns data rows +// filtered for a specific tableID and excluding hidden columns. +// Uses column name lookup from view headers to handle hidden column offsets. +// mo_columns columns: att_relname_id, att_relname, attname, atttyp, attnum, ..., att_is_hidden +func buildColumnsFromMoColumnsRows(view *LogicalTableView, tableID uint64) []TableColumn { + tableIDStr := fmt.Sprintf("%d", tableID) + + // Resolve column positions by name with hardcoded fallback for col_N headers + relnameIDCol := view.columnDataIndex("att_relname_id") + if relnameIDCol < 0 { + relnameIDCol = moColsPhysRelNameID + } + nameCol := view.columnDataIndex("attname") + if nameCol < 0 { + nameCol = moColsPhysAttName + } + typCol := view.columnDataIndex("atttyp") + if typCol < 0 { + typCol = moColsPhysAttTyp + } + numCol := view.columnDataIndex("attnum") + if numCol < 0 { + numCol = moColsPhysAttNum + } + hiddenCol := view.columnDataIndex("att_is_hidden") + if hiddenCol < 0 { + hiddenCol = moColsPhysIsHidden + } + seqNumCol := view.columnDataIndex("att_seqnum") + if seqNumCol < 0 { + seqNumCol = moColsPhysSeqNum + } + + var cols []TableColumn + for _, fullRow := range view.Rows { + row := fullRow[logicalViewMetaCols:] + + // Skip hidden columns + if hiddenCol >= 0 && hiddenCol < len(row) { + if row[hiddenCol] == "1" || row[hiddenCol] == "true" { + continue + } + } + + // Filter by table ID + if relnameIDCol < 0 || relnameIDCol >= len(row) { + continue + } + if row[relnameIDCol] != tableIDStr { + continue + } + + pos := len(cols) + if numCol >= 0 && numCol < len(row) { + if n, err := strconv.Atoi(row[numCol]); err == nil { + pos = n + } + } + + physicalPos := pos - 1 + if seqNumCol >= 0 && seqNumCol < len(row) { + if n, err := strconv.Atoi(row[seqNumCol]); err == nil { + physicalPos = n + } + } + + col := TableColumn{ + Position: pos, + PhysicalPosition: physicalPos, + } + if nameCol >= 0 && nameCol < len(row) { + col.Name = row[nameCol] + } + if typCol >= 0 && typCol < len(row) { + col.SQLType = row[typCol] + } + + cols = append(cols, col) + } + + // Sort by position + sort.Slice(cols, func(i, j int) bool { + return cols[i].Position < cols[j].Position + }) + + return cols +} + +// getDataColumnsFromView extracts only the data columns (skipping object/block/row meta columns). +// ShowCreateTable returns the CREATE TABLE DDL for a given tableID by reading +// the checkpoint's mo_tables and mo_columns system tables (GCKP + following ICKPs). +// +// Priority: +// 1. mo_tables.rel_createsql — the original CREATE TABLE SQL (most accurate) +// 2. Reconstructed from mo_columns (attname, atttyp, attnum, att_is_hidden) +// 3. Hardcoded built-in table schemas (for core system tables like mo_tables, mo_columns, etc.) +// +// NOTE: +// We intentionally do not synthesize a CREATE TABLE from raw physical object column +// types when mo_tables/mo_columns metadata is unavailable. Physical object columns may +// include hidden system columns, and without mo_columns we cannot safely distinguish +// visible columns from hidden ones. +// +// hardcoded physical column positions for mo_tables (after stripping meta cols). +// Layout: [rel_id(0), relname(1), reldatabase(2), reldatabase_id(3), ...] +// These are used as fallback when the LogicalTableView has col_N headers. +const ( + moTablesPhysRelID = 0 + moTablesPhysRelName = 1 + moTablesPhysRelDatabase = 2 + moTablesPhysRelCreateSQL = 7 + + // mo_columns hardcoded physical positions (after meta cols). + moColsPhysRelNameID = 4 + moColsPhysAttName = 6 + moColsPhysAttTyp = 7 + moColsPhysAttNum = 8 + moColsPhysIsHidden = 18 + moColsPhysSeqNum = 22 +) + +func (r *CheckpointReader) ShowCreateTable( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) (string, error) { + // 1. Try mo_tables.rel_createsql + moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) + if err == nil && moTablesView != nil { + relIDCol := moTablesView.columnDataIndex("rel_id") + if relIDCol < 0 { + relIDCol = moTablesPhysRelID + } + createSQLCol := moTablesView.columnDataIndex("rel_createsql") + if createSQLCol < 0 { + createSQLCol = moTablesPhysRelCreateSQL + } + for _, fullRow := range moTablesView.Rows { + row := fullRow[logicalViewMetaCols:] + if relIDCol < len(row) && row[relIDCol] == fmt.Sprintf("%d", tableID) { + if createSQLCol < len(row) && row[createSQLCol] != "" { + return row[createSQLCol], nil + } + break + } + } + } + + // 2. Reconstruct from mo_columns + moColumnsView, err := r.getTableLogicalView(ctx, moColumnsID, snapshotTS) + if err == nil && moColumnsView != nil { + if ddl := buildCreateTableFromMoColumns(moColumnsView, tableID); ddl != "" { + return ddl, nil + } + } + + // 3. Hardcoded built-in table schemas + if ddl := hardcodedCreateTable(tableID); ddl != "" { + return ddl, nil + } + + return "", moerr.NewInternalErrorf( + ctx, + "cannot resolve exact schema for table %d from checkpoint metadata; mo_tables/mo_columns data is unavailable or incomplete", + tableID, + ) +} + +// getTableName tries to get the table name for a tableID from the LogicalTableView's +// mo_tables data if available, falling back to the table ID as string. +func getTableName(view *LogicalTableView, tableID uint64) string { + // view here is the table's own data view, not mo_tables. + // We can't get the name from it, so use table ID + return fmt.Sprintf("%d", tableID) +} + +// buildCreateTableFromMoColumns reconstructs a CREATE TABLE DDL from mo_columns data. +func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) string { + tableIDStr := fmt.Sprintf("%d", tableID) + relnameIDCol := view.columnDataIndex("att_relname_id") + if relnameIDCol < 0 { + relnameIDCol = moColsPhysRelNameID + } + nameCol := view.columnDataIndex("attname") + if nameCol < 0 { + nameCol = moColsPhysAttName + } + typCol := view.columnDataIndex("atttyp") + if typCol < 0 { + typCol = moColsPhysAttTyp + } + numCol := view.columnDataIndex("attnum") + if numCol < 0 { + numCol = moColsPhysAttNum + } + hiddenCol := view.columnDataIndex("att_is_hidden") + if hiddenCol < 0 { + hiddenCol = moColsPhysIsHidden + } + + type colInfo struct { + name string + sqlType string + position int + } + var cols []colInfo + + for _, fullRow := range view.Rows { + row := fullRow[logicalViewMetaCols:] + if hiddenCol >= 0 && hiddenCol < len(row) { + if row[hiddenCol] == "1" || row[hiddenCol] == "true" { + continue + } + } + if relnameIDCol < 0 || relnameIDCol >= len(row) || row[relnameIDCol] != tableIDStr { + continue + } + pos := len(cols) + if numCol >= 0 && numCol < len(row) { + if n, err := strconv.Atoi(row[numCol]); err == nil { + pos = n + } + } + c := colInfo{position: pos} + if nameCol >= 0 && nameCol < len(row) { + c.name = row[nameCol] + } + if typCol >= 0 && typCol < len(row) { + c.sqlType = row[typCol] + } + cols = append(cols, c) + } + + if len(cols) == 0 { + return "" + } + + sort.Slice(cols, func(i, j int) bool { return cols[i].position < cols[j].position }) + + var sb strings.Builder + sb.WriteString("CREATE TABLE ") + sb.WriteString(fmt.Sprintf("`%d`", tableID)) + sb.WriteString(" (\n") + for i, c := range cols { + sb.WriteString(" `") + if c.name != "" { + sb.WriteString(c.name) + } else { + sb.WriteString(fmt.Sprintf("col_%d", i)) + } + sb.WriteString("`") + if c.sqlType != "" { + sb.WriteString(" ") + sb.WriteString(c.sqlType) + } + if i < len(cols)-1 { + sb.WriteString(",\n") + } else { + sb.WriteString("\n") + } + } + sb.WriteString(");") + return sb.String() +} + +// hardcodedCreateTable returns the CREATE TABLE DDL for core built-in system tables. +// These tables' schemas are known at compile time and may not appear in the checkpoint's +// mo_tables/mo_columns (due to minimal deployments). +func hardcodedCreateTable(tableID uint64) string { + switch tableID { + case catalog.MO_DATABASE_ID: + return "CREATE TABLE `mo_database` (\n" + + " `dat_id` BIGINT,\n" + + " `datname` VARCHAR(5000),\n" + + " `dat_catalog_name` VARCHAR(5000),\n" + + " `dat_createsql` VARCHAR(5000),\n" + + " `owner` INT UNSIGNED,\n" + + " `creator` INT UNSIGNED,\n" + + " `created_time` TIMESTAMP,\n" + + " `account_id` INT UNSIGNED,\n" + + " `dat_type` VARCHAR(32),\n" + + " `__mo_cpkey_dat` VARCHAR(65535)\n" + + ");" + case catalog.MO_TABLES_ID: + return "CREATE TABLE `mo_tables` (\n" + + " `rel_id` BIGINT,\n" + + " `relname` VARCHAR(5000),\n" + + " `reldatabase` VARCHAR(5000),\n" + + " `reldatabase_id` BIGINT,\n" + + " `relpersistence` VARCHAR(5000),\n" + + " `relkind` VARCHAR(5000),\n" + + " `rel_comment` VARCHAR(5000),\n" + + " `rel_createsql` TEXT,\n" + + " `created_time` TIMESTAMP,\n" + + " `creator` INT UNSIGNED,\n" + + " `owner` INT UNSIGNED,\n" + + " `account_id` INT UNSIGNED,\n" + + " `partitioned` TINYINT,\n" + + " `partition_info` BLOB,\n" + + " `viewdef` VARCHAR(5000),\n" + + " `constraint` VARCHAR(5000),\n" + + " `schema_version` INT UNSIGNED,\n" + + " `schema_catalog_version` INT UNSIGNED,\n" + + " `extra_info` VARCHAR,\n" + + " `__mo_cpkey_rel` VARCHAR(65535),\n" + + " `rel_logical_id` BIGINT\n" + + ");" + case catalog.MO_COLUMNS_ID: + return "CREATE TABLE `mo_columns` (\n" + + " `att_uniq_name` VARCHAR(256),\n" + + " `account_id` INT UNSIGNED,\n" + + " `att_database_id` BIGINT,\n" + + " `att_database` VARCHAR(256),\n" + + " `att_relname_id` BIGINT,\n" + + " `att_relname` VARCHAR(256),\n" + + " `attname` VARCHAR(256),\n" + + " `atttyp` VARCHAR(256),\n" + + " `attnum` INT,\n" + + " `att_length` INT,\n" + + " `attnotnull` TINYINT,\n" + + " `atthasdef` TINYINT,\n" + + " `att_default` VARCHAR(2048),\n" + + " `attisdropped` TINYINT,\n" + + " `att_constraint_type` CHAR(1),\n" + + " `att_is_unsigned` TINYINT,\n" + + " `att_is_auto_increment` TINYINT,\n" + + " `att_comment` VARCHAR(2048),\n" + + " `att_is_hidden` TINYINT,\n" + + " `att_has_update` TINYINT,\n" + + " `att_update` VARCHAR(2048),\n" + + " `att_is_clusterby` TINYINT,\n" + + " `att_seqnum` SMALLINT UNSIGNED,\n" + + " `att_enum` VARCHAR,\n" + + " `__mo_cpkey_col` VARCHAR(65535),\n" + + " `attr_has_generated` TINYINT,\n" + + " `attr_generated` VARCHAR(2048)\n" + + ");" + default: + return "" + } +} + +// columnDataIndex returns the data-column index (0-based, after stripping meta columns) +// for the named column, or -1 if not found. +func (v *LogicalTableView) columnDataIndex(colName string) int { + for i := logicalViewMetaCols; i < len(v.Headers); i++ { + if v.Headers[i] == colName { + return i - logicalViewMetaCols + } + } + return -1 +} diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go new file mode 100644 index 0000000000000..9ba0b015a6fb6 --- /dev/null +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -0,0 +1,505 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestWriteCSV_empty tests writing an empty logical view to CSV. +func TestWriteCSV_empty(t *testing.T) { + schema := &TableSchema{ + TableName: "test_table", + DatabaseName: "test_db", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + {Name: "name", SQLType: "VARCHAR(100)", Position: 2, PhysicalPosition: 1}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{}, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + lines := strings.Split(strings.TrimSpace(output), "\n") + // Skip comment lines to find the CSV header + csvStart := 0 + for i, line := range lines { + if !strings.HasPrefix(line, "--") { + csvStart = i + break + } + } + require.GreaterOrEqual(t, len(lines), csvStart+1, "expected at least a header line") + assert.Equal(t, "id,name", lines[csvStart]) +} + +// TestWriteCSV_withData tests writing data rows to CSV. +func TestWriteCSV_withData(t *testing.T) { + schema := &TableSchema{ + TableName: "users", + DatabaseName: "mydb", + CreateSQL: "CREATE TABLE users (id BIGINT, name VARCHAR(100), age INT)", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + {Name: "name", SQLType: "VARCHAR(100)", Position: 2, PhysicalPosition: 1}, + {Name: "age", SQLType: "INT", Position: 3, PhysicalPosition: 2}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2"}, + Rows: [][]string{ + {"obj1", "0", "0", "1", "Alice", "30"}, + {"obj1", "0", "1", "2", "Bob", "25"}, + {"obj2", "1", "0", "3", "Charlie, Jr.", "35"}, + {"obj2", "1", "1", "4", "NULL", "NULL"}, + }, + VisibleRows: 4, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + lines := strings.Split(strings.TrimSpace(output), "\n") + + // Skip comment lines + var dataLines []string + for _, line := range lines { + if !strings.HasPrefix(line, "--") { + dataLines = append(dataLines, line) + } + } + + assert.Equal(t, 5, len(dataLines)) // header + 4 rows + assert.Equal(t, "id,name,age", dataLines[0]) + assert.Equal(t, "1,Alice,30", dataLines[1]) + assert.Equal(t, "2,Bob,25", dataLines[2]) + assert.Equal(t, `3,"Charlie, Jr.",35`, dataLines[3]) + assert.Equal(t, "4,NULL,NULL", dataLines[4]) +} + +// TestWriteCSV_withCreateSQLHeader tests the header comment from CreateSQL. +func TestWriteCSV_withCreateSQLHeader(t *testing.T) { + schema := &TableSchema{ + TableName: "t1", + DatabaseName: "db1", + CreateSQL: "CREATE TABLE t1 (id BIGINT)", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0"}, + Rows: [][]string{ + {"obj1", "0", "0", "42"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + assert.Contains(t, output, "-- CREATE TABLE t1 (id BIGINT)") + assert.Contains(t, output, "-- Database: db1") + assert.Contains(t, output, "-- Table: t1") +} + +// TestWriteCSV_mismatchedColumns verifies that position-based column mapping +// only includes columns matching the schema positions. +func TestWriteCSV_mismatchedColumns(t *testing.T) { + schema := &TableSchema{ + TableName: "t1", + Columns: []TableColumn{ + {Name: "a", SQLType: "INT", Position: 1, PhysicalPosition: 0}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{ + {"obj1", "0", "0", "1", "extra"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + // Skip comment lines + var dataLines []string + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + if !strings.HasPrefix(line, "--") { + dataLines = append(dataLines, line) + } + } + // Position-based: only column at position 0 (a) is included, extra columns dropped + assert.Equal(t, "a", dataLines[0]) + assert.Equal(t, "1", dataLines[1]) +} + +// TestBuildSchemaFromMoTablesRow tests extracting a TableSchema from mo_tables row data. +func TestBuildSchemaFromMoTablesRow(t *testing.T) { + // mo_tables physical layout: rel_id, relname, reldatabase, ... + // After meta cols stripped, data cols = [rel_id, relname, reldatabase, reldatabase_id, ..., rel_createsql] + view := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "col_4", "col_5", "col_6", "rel_createsql", + }, + } + + // Full row with meta + data: rel_id="12345", relname="my_table", reldatabase="my_db", rel_createsql="CREATE TABLE my_table (x INT)" + fullRow := []string{ + "obj1", "0", "0", // meta cols + "12345", "my_table", "my_db", "100", "x", "x", "x", "CREATE TABLE my_table (x INT)", + } + + schema := buildSchemaFromMoTablesRow(view, fullRow) + assert.Equal(t, "my_table", schema.TableName) + assert.Equal(t, "my_db", schema.DatabaseName) + assert.Equal(t, "CREATE TABLE my_table (x INT)", schema.CreateSQL) +} + +// TestBuildColumnsFromMoColumnsRows tests building column list from mo_columns rows. +func TestBuildColumnsFromMoColumnsRows(t *testing.T) { + // mo_columns physical layout: ..., att_relname_id, att_relname, attname, atttyp, attnum, ..., att_is_hidden + view := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "col_0", "col_1", "col_2", "col_3", + "att_relname_id", "att_relname", "attname", "atttyp", "attnum", + "col_9", "col_10", "col_11", "col_12", "col_13", "col_14", "col_15", "col_16", "col_17", + "att_is_hidden", + "col_19", "col_20", "col_21", "att_seqnum", + }, + Rows: [][]string{ + {"obj1", "0", "0", "", "", "", "", "12345", "my_table", "id", "BIGINT", "1", "", "", "", "", "", "", "", "", "", "0", "", "", "", "0"}, + {"obj1", "0", "1", "", "", "", "", "12345", "my_table", "name", "VARCHAR(100)", "2", "", "", "", "", "", "", "", "", "", "0", "", "", "", "1"}, + {"obj1", "0", "2", "", "", "", "", "12345", "my_table", "_hidden_col", "INT", "3", "", "", "", "", "", "", "", "", "", "1", "", "", "", "2"}, + }, + } + + cols := buildColumnsFromMoColumnsRows(view, 12345) + require.Len(t, cols, 2) // hidden column filtered + + assert.Equal(t, "id", cols[0].Name) + assert.Equal(t, "BIGINT", cols[0].SQLType) + assert.Equal(t, 1, cols[0].Position) + assert.Equal(t, 0, cols[0].PhysicalPosition) + + assert.Equal(t, "name", cols[1].Name) + assert.Equal(t, "VARCHAR(100)", cols[1].SQLType) + assert.Equal(t, 2, cols[1].Position) + assert.Equal(t, 1, cols[1].PhysicalPosition) +} + +func TestBuildColumnsFromMoColumnsRows_FallbackToAttnumWhenSeqnumMissing(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "col_0", "col_1", "col_2", "col_3", + "att_relname_id", "att_relname", "attname", "atttyp", "attnum", + "col_9", "col_10", "col_11", "col_12", "col_13", "col_14", "col_15", "col_16", "col_17", + "att_is_hidden", + }, + Rows: [][]string{ + {"obj1", "0", "0", "", "", "", "", "12345", "my_table", "id", "BIGINT", "1", "", "", "", "", "", "", "", "", "", "0"}, + {"obj1", "0", "1", "", "", "", "", "12345", "my_table", "name", "VARCHAR(100)", "2", "", "", "", "", "", "", "", "", "", "0"}, + }, + } + + cols := buildColumnsFromMoColumnsRows(view, 12345) + require.Len(t, cols, 2) + assert.Equal(t, 0, cols[0].PhysicalPosition) + assert.Equal(t, 1, cols[1].PhysicalPosition) +} + +// TestLogicalTableViewWithSchema_mergeHeaders merges schema column names using position-based mapping. +func TestLogicalTableViewWithSchema_mergeHeaders(t *testing.T) { + schema := &TableSchema{ + TableName: "t1", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + {Name: "val", SQLType: "INT", Position: 2, PhysicalPosition: 1}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{ + {"obj1", "0", "0", "42", "100"}, + }, + } + + merged := MergeLogicalViewWithSchema(view, schema) + assert.Equal(t, []string{"id", "val"}, merged.Headers) + // Data rows only include columns matching schema positions + require.Len(t, merged.Rows, 1) + assert.Equal(t, []string{"42", "100"}, merged.Rows[0]) +} + +func TestMergeLogicalViewWithSchema_requiresResolvedSchema(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{ + {"obj1", "0", "0", "42", "100"}, + }, + VisibleRows: 1, + DeletedRows: 0, + PhysicalRows: 1, + } + + merged := MergeLogicalViewWithSchema(view, &TableSchema{}) + assert.Empty(t, merged.Headers) + assert.Empty(t, merged.Rows) + assert.Equal(t, 1, merged.VisibleRows) + assert.Equal(t, 1, merged.PhysicalRows) +} + +func TestMergeLogicalViewWithSchema_UsesPhysicalPosition(t *testing.T) { + schema := &TableSchema{ + TableName: "shifted", + Columns: []TableColumn{ + {Name: "id", SQLType: "VARCHAR(36)", Position: 1, PhysicalPosition: 4}, + {Name: "task_id", SQLType: "VARCHAR(36)", Position: 2, PhysicalPosition: 5}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3", "col_4", "col_5"}, + Rows: [][]string{ + {"obj1", "0", "0", "ignored0", "ignored1", "ignored2", "ignored3", "case-001", "task-001"}, + }, + } + + merged := MergeLogicalViewWithSchema(view, schema) + require.Len(t, merged.Rows, 1) + assert.Equal(t, []string{"case-001", "task-001"}, merged.Rows[0]) +} + +// TestMergeLogicalViewWithSchema_hiddenColumns verifies that hidden columns (e.g. __mo_fake_pk_col, +// __mo_cpkey_col) are excluded from the merged view using position-based column mapping. +func TestMergeLogicalViewWithSchema_hiddenColumns(t *testing.T) { + // Simulate: physical columns = [__mo_fake_pk_col(hidden,pos=0), id(pos=1), name(pos=2), __mo_cpkey_col(hidden,pos=3)] + // Schema only has visible columns: id(pos=1), name(pos=2) + schema := &TableSchema{ + TableName: "users", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 1}, + {Name: "name", SQLType: "VARCHAR(100)", Position: 2, PhysicalPosition: 2}, + }, + } + + // The LogicalTableView has all 4 physical columns + 3 meta columns = 7 total + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3"}, + Rows: [][]string{ + {"obj1", "0", "0", "fake_pk_123", "1", "Alice", "cpkey_456"}, + {"obj1", "0", "1", "fake_pk_789", "2", "Bob", "cpkey_999"}, + }, + } + + merged := MergeLogicalViewWithSchema(view, schema) + + // Headers should only contain visible columns + assert.Equal(t, []string{"id", "name"}, merged.Headers) + + // Data rows: only columns at positions 1 and 2 (skip pos 0 and pos 3) + require.Len(t, merged.Rows, 2) + assert.Equal(t, []string{"1", "Alice"}, merged.Rows[0]) + assert.Equal(t, []string{"2", "Bob"}, merged.Rows[1]) + + // Full CSV round-trip + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + assert.Contains(t, output, "id,name") + assert.Contains(t, output, "1,Alice") + assert.Contains(t, output, "2,Bob") + assert.NotContains(t, output, "fake_pk") // hidden columns excluded + assert.NotContains(t, output, "cpkey") // hidden columns excluded +} + +func TestMergeLogicalViewWithSchema_LargeSparsePhysicalPositions(t *testing.T) { + const physicalCols = 128 + headers := []string{"object", "block", "row"} + row := []string{"obj1", "0", "0"} + for i := 0; i < physicalCols; i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + row = append(row, fmt.Sprintf("v%d", i)) + } + + schema := &TableSchema{ + TableName: "wide_table", + Columns: []TableColumn{ + {Name: "first", Position: 1, PhysicalPosition: 0}, + {Name: "middle", Position: 2, PhysicalPosition: 63}, + {Name: "last", Position: 3, PhysicalPosition: 127}, + }, + } + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{row}, + } + + merged := MergeLogicalViewWithSchema(view, schema) + require.Len(t, merged.Rows, 1) + assert.Equal(t, []string{"v0", "v63", "v127"}, merged.Rows[0]) +} + +// makeColumnRow creates a mock mo_columns row with specific values at given indexes. +func makeColumnRow(indexValuePairs ...any) []string { + // 27 columns in mo_columns + row := make([]string, 27) + for i := range row { + row[i] = "" + } + for i := 0; i < len(indexValuePairs); i += 2 { + idx := indexValuePairs[i].(int) + val := indexValuePairs[i+1].(string) + row[idx] = val + } + // set a default att_relname_id if not provided + if row[4] == "" { + row[4] = "12345" + } + return row +} + +// TestColumnSchemaRoundTrip tests the full pipeline: rows → columns → schema → CSV header. +func TestColumnSchemaRoundTrip(t *testing.T) { + moColumnsView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "col_0", "col_1", "col_2", "col_3", + "att_relname_id", "att_relname", "attname", "atttyp", "attnum", + "col_9", "col_10", "col_11", "col_12", "col_13", "col_14", "col_15", "col_16", "col_17", + "att_is_hidden", + "col_19", "col_20", "col_21", "att_seqnum", + }, + Rows: [][]string{ + {"obj1", "0", "0", "", "", "", "", "12345", "my_table", "a", "INT", "1", "", "", "", "", "", "", "", "", "", "0", "", "", "", "0"}, + {"obj1", "0", "1", "", "", "", "", "12345", "my_table", "b", "TEXT", "2", "", "", "", "", "", "", "", "", "", "0", "", "", "", "1"}, + {"obj1", "0", "2", "", "", "", "", "12345", "my_table", "c", "FLOAT", "3", "", "", "", "", "", "", "", "", "", "0", "", "", "", "2"}, + }, + } + + cols := buildColumnsFromMoColumnsRows(moColumnsView, 12345) + require.Len(t, cols, 3) + + schema := &TableSchema{ + TableName: "roundtrip", + DatabaseName: "test", + Columns: cols, + CreateSQL: "CREATE TABLE roundtrip (a INT, b TEXT, c FLOAT)", + } + + dataView := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2"}, + Rows: [][]string{ + {"obj1", "0", "0", "1", "hello", "1.500000"}, + {"obj1", "0", "1", "2", "world", "2.000000"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, dataView) + require.NoError(t, err) + + output := buf.String() + assert.Contains(t, output, "a,b,c") + assert.Contains(t, output, "1,hello,1.500000") + + // Verify comments are present + assert.Contains(t, output, "-- CREATE TABLE roundtrip") + assert.Contains(t, output, "-- Database: test") + fmt.Println(output) +} + +func TestBuildCreateTableFromMoColumns(t *testing.T) { + // Simulate mo_columns data for a user table (id=100) with 3 columns + view := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden", + }, + Rows: [][]string{ + // id BIGINT, pos=0 + {"", "0", "0", "100", "id", "BIGINT", "1", "0"}, + // name VARCHAR(100), pos=1 + {"", "0", "1", "100", "name", "VARCHAR(100)", "2", "0"}, + // __mo_fake_pk_col, pos=2, hidden -> skipped + {"", "0", "2", "100", "__mo_fake_pk_col", "VARCHAR(65535)", "3", "1"}, + }, + } + + ddl := buildCreateTableFromMoColumns(view, 100) + assert.NotEmpty(t, ddl) + assert.Contains(t, ddl, "BIGINT") + assert.Contains(t, ddl, "VARCHAR(100)") + assert.NotContains(t, ddl, "__mo_fake_pk_col") + assert.Contains(t, ddl, "CREATE TABLE") +} + +func TestBuildCreateTableFromMoColumns_empty(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden"}, + Rows: [][]string{ + {"", "0", "0", "200", "x", "INT", "1", "0"}, + }, + } + ddl := buildCreateTableFromMoColumns(view, 999) // different table id + assert.Empty(t, ddl) +} + +func TestHardcodedCreateTable(t *testing.T) { + tests := []struct { + tableID uint64 + wantName string + }{ + {catalog.MO_TABLES_ID, "mo_tables"}, + {catalog.MO_COLUMNS_ID, "mo_columns"}, + {catalog.MO_DATABASE_ID, "mo_database"}, + {999999, ""}, // unknown table should return empty + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("table-%d", tt.tableID), func(t *testing.T) { + ddl := hardcodedCreateTable(tt.tableID) + if tt.wantName == "" { + assert.Empty(t, ddl) + } else { + assert.Contains(t, ddl, tt.wantName) + assert.Contains(t, ddl, "CREATE TABLE") + } + }) + } +} diff --git a/pkg/tools/checkpointtool/types.go b/pkg/tools/checkpointtool/types.go index 3b6f350fc4154..1cf030edeee99 100644 --- a/pkg/tools/checkpointtool/types.go +++ b/pkg/tools/checkpointtool/types.go @@ -16,6 +16,7 @@ package checkpointtool import ( "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" ) @@ -68,9 +69,9 @@ type TableInfo struct { // ObjectEntryInfo contains detailed object entry information with timestamps type ObjectEntryInfo struct { - Range ckputil.TableRange - CreateTime types.TS - DeleteTime types.TS + ObjectStats objectio.ObjectStats + CreateTime types.TS + DeleteTime types.TS } // ComposedView represents logical checkpoint view at a timestamp @@ -85,6 +86,7 @@ type ComposedView struct { type LogicalTableView struct { Headers []string Rows [][]string + ColTypes []types.Type // column types for data columns (after meta cols) PhysicalRows int DeletedRows int VisibleRows int From 953b6e76b28330c4fbc647d8a6856808d92bc021 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 7 Jun 2026 04:12:00 +0800 Subject: [PATCH 03/76] Optimize checkpoint CSV dump and add layout compatibility --- pkg/tools/checkpointtool/logical_table.go | 99 +++-- pkg/tools/checkpointtool/table_dump.go | 379 +++++++++++++------- pkg/tools/checkpointtool/table_dump_test.go | 55 ++- 3 files changed, 382 insertions(+), 151 deletions(-) diff --git a/pkg/tools/checkpointtool/logical_table.go b/pkg/tools/checkpointtool/logical_table.go index 2c2f760b7baee..f887516047d1e 100644 --- a/pkg/tools/checkpointtool/logical_table.go +++ b/pkg/tools/checkpointtool/logical_table.go @@ -26,6 +26,12 @@ import ( "github.com/matrixorigin/matrixone/pkg/tools/objecttool" ) +type logicalTableStats struct { + PhysicalRows int + DeletedRows int + VisibleRows int +} + // BuildLogicalTableView materializes a tombstone-applied logical table view. func (r *CheckpointReader) BuildLogicalTableView( ctx context.Context, @@ -37,42 +43,81 @@ func (r *CheckpointReader) BuildLogicalTableView( Headers: []string{"object", "block", "row"}, Rows: make([][]string, 0), } + stats, err := r.scanLogicalTable(ctx, snapshotTS, dataEntries, tombEntries, + func(cols []objecttool.ColInfo) error { + if len(view.Headers) != 3 { + return nil + } + for _, col := range cols { + view.Headers = append(view.Headers, fmt.Sprintf("col_%d", col.Idx)) + view.ColTypes = append(view.ColTypes, col.Type) + } + return nil + }, + func(objShort string, blockIdx int, rowIdx int, values []string) error { + row := make([]string, 0, len(values)+3) + row = append(row, objShort, fmt.Sprintf("%d", blockIdx), fmt.Sprintf("%d", rowIdx)) + row = append(row, values...) + view.Rows = append(view.Rows, row) + return nil + }, + ) + if err != nil { + return nil, err + } + view.PhysicalRows = stats.PhysicalRows + view.DeletedRows = stats.DeletedRows + view.VisibleRows = stats.VisibleRows + return view, nil +} + +func (r *CheckpointReader) scanLogicalTable( + ctx context.Context, + snapshotTS types.TS, + dataEntries []*ObjectEntryInfo, + tombEntries []*ObjectEntryInfo, + onColumns func([]objecttool.ColInfo) error, + onRow func(objShort string, blockIdx int, rowIdx int, values []string) error, +) (logicalTableStats, error) { + stats := logicalTableStats{} if len(dataEntries) == 0 { - return view, nil + return stats, nil } visibleDataEntries := visibleObjectEntries(dataEntries, snapshotTS) visibleTombEntries := visibleObjectEntries(tombEntries, snapshotTS) tombstoneStats := dedupeObjectStats(visibleTombEntries) + columnsSent := false + for _, entry := range visibleDataEntries { objName := entry.ObjectStats.ObjectName().String() reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) if err != nil { if isDataFileNotFound(err) { - continue // GC'd data object; skip + continue } - return nil, err + return stats, err } - if len(view.Headers) == 3 { - cols := reader.Columns() - for _, col := range cols { - view.Headers = append(view.Headers, fmt.Sprintf("col_%d", col.Idx)) - view.ColTypes = append(view.ColTypes, col.Type) + if !columnsSent && onColumns != nil { + if err := onColumns(reader.Columns()); err != nil { + _ = reader.Close() + return stats, err } + columnsSent = true } relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.ObjectStats.ObjectName().ObjectId(), tombstoneStats) if err != nil { _ = reader.Close() - return nil, err + return stats, err } for blockIdx := 0; blockIdx < int(entry.ObjectStats.BlkCnt()); blockIdx++ { bat, release, err := reader.ReadBlock(ctx, uint32(blockIdx)) if err != nil { _ = reader.Close() - return nil, err + return stats, err } if bat.RowCount() == 0 { @@ -80,30 +125,35 @@ func (r *CheckpointReader) BuildLogicalTableView( continue } - view.PhysicalRows += bat.RowCount() + stats.PhysicalRows += bat.RowCount() deleteMask, err := r.buildDeleteMaskForBlock(ctx, &snapshotTS, entry.ObjectStats, uint16(blockIdx), relevantTombstones) if err != nil { release() _ = reader.Close() - return nil, err + return stats, err } for rowIdx := 0; rowIdx < bat.RowCount(); rowIdx++ { if deleteMask.IsValid() && deleteMask.Contains(uint64(rowIdx)) { - view.DeletedRows++ + stats.DeletedRows++ continue } - row := make([]string, 0, len(bat.Vecs)+3) - row = append(row, - entry.ObjectStats.ObjectName().Short().ShortString(), - fmt.Sprintf("%d", blockIdx), - fmt.Sprintf("%d", rowIdx), - ) - for _, vec := range bat.Vecs { - row = append(row, vecValueToString(vec, rowIdx)) + if onRow != nil { + values := make([]string, len(bat.Vecs)) + for i, vec := range bat.Vecs { + values[i] = vecValueToString(vec, rowIdx) + } + if err := onRow(entry.ObjectStats.ObjectName().Short().ShortString(), blockIdx, rowIdx, values); err != nil { + if deleteMask.IsValid() { + deleteMask.Release() + } + release() + _ = reader.Close() + return stats, err + } } - view.Rows = append(view.Rows, row) + stats.VisibleRows++ } if deleteMask.IsValid() { @@ -113,12 +163,11 @@ func (r *CheckpointReader) BuildLogicalTableView( } if err := reader.Close(); err != nil { - return nil, err + return stats, err } } - view.VisibleRows = len(view.Rows) - return view, nil + return stats, nil } func dedupeObjectStats(entries []*ObjectEntryInfo) []objectio.ObjectStats { diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index cd496b343cc35..135a0fa9db21d 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -19,6 +19,7 @@ import ( "encoding/csv" "fmt" "io" + "os" "sort" "strconv" "strings" @@ -37,6 +38,25 @@ const ( moColumnsID = uint64(catalog.MO_COLUMNS_ID) ) +type catalogLayout struct { + name string + moTablesSchema []string + moColumnsSchema []string +} + +var ( + currentCatalogLayout = catalogLayout{ + name: "current", + moTablesSchema: append([]string(nil), catalog.MoTablesSchema...), + moColumnsSchema: append([]string(nil), catalog.MoColumnsSchema...), + } + legacy3CatalogLayout = catalogLayout{ + name: "3.0-dev", + moTablesSchema: append([]string(nil), catalog.MoTablesSchema[:len(catalog.MoTablesSchema)-1]...), + moColumnsSchema: append([]string(nil), catalog.MoColumnsSchema[:len(catalog.MoColumnsSchema)-2]...), + } +) + // TableColumn describes one column in a user table schema. type TableColumn struct { Name string // SQL column name @@ -53,6 +73,50 @@ type TableSchema struct { CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql } +func knownCatalogLayouts() []catalogLayout { + return []catalogLayout{currentCatalogLayout, legacy3CatalogLayout} +} + +func schemaForLayout(layout catalogLayout, tableID uint64) []string { + switch tableID { + case moTablesID: + return layout.moTablesSchema + case moColumnsID: + return layout.moColumnsSchema + default: + return nil + } +} + +func inferCatalogLayout(dataWidth int, tableID uint64) (catalogLayout, int) { + for _, layout := range knownCatalogLayouts() { + schema := schemaForLayout(layout, tableID) + if len(schema) == 0 { + continue + } + switch dataWidth { + case len(schema): + return layout, 0 + case len(schema) + 1: + return layout, 1 + } + } + return currentCatalogLayout, 0 +} + +func fallbackCatalogColIndex(view *LogicalTableView, tableID uint64, colName string) int { + if idx := view.columnDataIndex(colName); idx >= 0 { + return idx + } + layout, offset := inferCatalogLayout(len(view.Headers)-logicalViewMetaCols, tableID) + for i, name := range schemaForLayout(layout, tableID) { + if name == colName { + return i + offset + } + } + return -1 +} + // ReadTableSchema reads the schema for a user table by reading mo_tables and mo_columns // from the checkpoint. // @@ -81,10 +145,7 @@ func (r *CheckpointReader) ReadTableSchema( // Don't log - expected for system tables or when mo_tables not in checkpoint } else if moTablesView != nil { // Resolve column positions by name (handles hidden column offset) - relIDCol := moTablesView.columnDataIndex("rel_id") - if relIDCol < 0 { - relIDCol = moTablesPhysRelID - } + relIDCol := fallbackCatalogColIndex(moTablesView, moTablesID, "rel_id") for _, row := range moTablesView.Rows { dataRow := row[logicalViewMetaCols:] if relIDCol < 0 || relIDCol >= len(dataRow) { @@ -120,17 +181,28 @@ func (r *CheckpointReader) getTableLogicalView( tableID uint64, snapshotTS types.TS, ) (*LogicalTableView, error) { - composed, err := r.ComposeAt(snapshotTS) + allData, allTomb, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) if err != nil { return nil, err } + return r.BuildLogicalTableView(ctx, snapshotTS, allData, allTomb) +} + +func (r *CheckpointReader) getTableEntriesAt( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) ([]*ObjectEntryInfo, []*ObjectEntryInfo, error) { + composed, err := r.ComposeAt(snapshotTS) + if err != nil { + return nil, nil, err + } tbl, ok := composed.Tables[tableID] if !ok || (len(tbl.DataRanges) == 0 && len(tbl.TombRanges) == 0) { - return nil, moerr.NewInternalErrorf(ctx, "table %d not found in checkpoint at ts %s", tableID, snapshotTS.ToString()) + return nil, nil, moerr.NewInternalErrorf(ctx, "table %d not found in checkpoint at ts %s", tableID, snapshotTS.ToString()) } - // Collect all entries that contain this table (base + incrementals) type entryRef struct { entry *EntryInfo } @@ -142,23 +214,21 @@ func (r *CheckpointReader) getTableLogicalView( entryRefs = append(entryRefs, entryRef{entry: incr}) } - // Aggregate data/tomb entries across all checkpoint entries var allData, allTomb []*ObjectEntryInfo for _, ref := range entryRefs { e := r.entries[ref.entry.Index] dataEntries, tombEntries, err := r.GetObjectEntries(e, tableID) if err != nil { - continue // skip entries that don't have this table + continue } allData = append(allData, dataEntries...) allTomb = append(allTomb, tombEntries...) } if len(allData) == 0 { - return nil, moerr.NewInternalErrorf(ctx, "no data entries for table %d", tableID) + return nil, nil, moerr.NewInternalErrorf(ctx, "no data entries for table %d", tableID) } - - return r.BuildLogicalTableView(ctx, snapshotTS, allData, allTomb) + return allData, allTomb, nil } // DumpTableCSV reads a user table's logical view and writes it as CSV to w. @@ -172,12 +242,7 @@ func (r *CheckpointReader) DumpTableCSV( snapshotTS types.TS, dataEntries, tombEntries []*ObjectEntryInfo, ) error { - view, err := r.BuildLogicalTableView(ctx, snapshotTS, dataEntries, tombEntries) - if err != nil { - return err - } - - schema := r.ReadTableSchema(ctx, tableID, snapshotTS, view) + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) if len(schema.Columns) == 0 { return moerr.NewInternalErrorf( ctx, @@ -185,7 +250,7 @@ func (r *CheckpointReader) DumpTableCSV( tableID, ) } - return WriteCSV(w, schema, view) + return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries) } // DumpTableCSVComposed dumps a table to CSV by composing the full checkpoint view @@ -197,11 +262,11 @@ func (r *CheckpointReader) DumpTableCSVComposed( tableID uint64, snapshotTS types.TS, ) error { - view, err := r.getTableLogicalView(ctx, tableID, snapshotTS) + dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) if err != nil { return err } - schema := r.ReadTableSchema(ctx, tableID, snapshotTS, view) + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) if len(schema.Columns) == 0 { return moerr.NewInternalErrorf( ctx, @@ -209,7 +274,66 @@ func (r *CheckpointReader) DumpTableCSVComposed( tableID, ) } - return WriteCSV(w, schema, view) + return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries) +} + +func (r *CheckpointReader) streamTableCSV( + ctx context.Context, + w io.Writer, + schema *TableSchema, + snapshotTS types.TS, + dataEntries, tombEntries []*ObjectEntryInfo, +) error { + tmpFile, err := os.CreateTemp("", "mo-tool-table-dump-*.csv") + if err != nil { + return err + } + tmpName := tmpFile.Name() + defer os.Remove(tmpName) + defer tmpFile.Close() + + cw := csv.NewWriter(tmpFile) + header := make([]string, 0, len(schema.Columns)) + physicalPositions := make([]int, 0, len(schema.Columns)) + for _, col := range schema.Columns { + header = append(header, col.Name) + physicalPos := col.PhysicalPosition + if physicalPos < 0 { + physicalPos = col.Position + } + physicalPositions = append(physicalPositions, physicalPos) + } + if err := cw.Write(header); err != nil { + return err + } + + stats, err := r.scanLogicalTable(ctx, snapshotTS, dataEntries, tombEntries, nil, + func(_ string, _ int, _ int, values []string) error { + row := make([]string, len(physicalPositions)) + for i, pos := range physicalPositions { + if pos >= 0 && pos < len(values) { + row[i] = values[pos] + } + } + return cw.Write(row) + }, + ) + if err != nil { + return err + } + cw.Flush() + if err := cw.Error(); err != nil { + return err + } + + if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { + return err + } + if err := writeCSVMetadata(w, schema, stats); err != nil { + return err + } + _, err = io.Copy(w, tmpFile) + return err } // WriteCSV writes a LogicalTableView with the given schema as CSV to w. @@ -218,23 +342,11 @@ func WriteCSV(w io.Writer, schema *TableSchema, view *LogicalTableView) error { defer cw.Flush() // Write header comments (DDL + metadata) - if schema.CreateSQL != "" { - if _, err := fmt.Fprintf(w, "-- %s\n", schema.CreateSQL); err != nil { - return err - } - } - if schema.DatabaseName != "" { - if _, err := fmt.Fprintf(w, "-- Database: %s\n", schema.DatabaseName); err != nil { - return err - } - } - if schema.TableName != "" { - if _, err := fmt.Fprintf(w, "-- Table: %s\n", schema.TableName); err != nil { - return err - } - } - if _, err := fmt.Fprintf(w, "-- Visible rows: %d (deleted: %d, physical: %d)\n", - view.VisibleRows, view.DeletedRows, view.PhysicalRows); err != nil { + if err := writeCSVMetadata(w, schema, logicalTableStats{ + VisibleRows: view.VisibleRows, + DeletedRows: view.DeletedRows, + PhysicalRows: view.PhysicalRows, + }); err != nil { return err } @@ -255,6 +367,27 @@ func WriteCSV(w io.Writer, schema *TableSchema, view *LogicalTableView) error { return cw.Error() } +func writeCSVMetadata(w io.Writer, schema *TableSchema, stats logicalTableStats) error { + if schema.CreateSQL != "" { + if _, err := fmt.Fprintf(w, "-- %s\n", schema.CreateSQL); err != nil { + return err + } + } + if schema.DatabaseName != "" { + if _, err := fmt.Fprintf(w, "-- Database: %s\n", schema.DatabaseName); err != nil { + return err + } + } + if schema.TableName != "" { + if _, err := fmt.Fprintf(w, "-- Table: %s\n", schema.TableName); err != nil { + return err + } + } + _, err := fmt.Fprintf(w, "-- Visible rows: %d (deleted: %d, physical: %d)\n", + stats.VisibleRows, stats.DeletedRows, stats.PhysicalRows) + return err +} + // MergeLogicalViewWithSchema replaces col_N headers with real column names from the schema // and filters data rows to only include visible (non-hidden) columns by their physical position. // Returns a new LogicalTableView with merged headers and filtered data rows. @@ -317,26 +450,17 @@ func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *Table schema := &TableSchema{} dataRow := fullRow[logicalViewMetaCols:] - relNameIdx := view.columnDataIndex("relname") - if relNameIdx < 0 { - relNameIdx = moTablesPhysRelName - } + relNameIdx := fallbackCatalogColIndex(view, moTablesID, "relname") if relNameIdx < len(dataRow) { schema.TableName = dataRow[relNameIdx] } - relDBIdx := view.columnDataIndex("reldatabase") - if relDBIdx < 0 { - relDBIdx = moTablesPhysRelDatabase - } + relDBIdx := fallbackCatalogColIndex(view, moTablesID, "reldatabase") if relDBIdx < len(dataRow) { schema.DatabaseName = dataRow[relDBIdx] } - createSQLIdx := view.columnDataIndex("rel_createsql") - if createSQLIdx < 0 { - createSQLIdx = moTablesPhysRelCreateSQL - } + createSQLIdx := fallbackCatalogColIndex(view, moTablesID, "rel_createsql") if createSQLIdx < len(dataRow) { schema.CreateSQL = dataRow[createSQLIdx] } @@ -350,31 +474,12 @@ func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *Table func buildColumnsFromMoColumnsRows(view *LogicalTableView, tableID uint64) []TableColumn { tableIDStr := fmt.Sprintf("%d", tableID) - // Resolve column positions by name with hardcoded fallback for col_N headers - relnameIDCol := view.columnDataIndex("att_relname_id") - if relnameIDCol < 0 { - relnameIDCol = moColsPhysRelNameID - } - nameCol := view.columnDataIndex("attname") - if nameCol < 0 { - nameCol = moColsPhysAttName - } - typCol := view.columnDataIndex("atttyp") - if typCol < 0 { - typCol = moColsPhysAttTyp - } - numCol := view.columnDataIndex("attnum") - if numCol < 0 { - numCol = moColsPhysAttNum - } - hiddenCol := view.columnDataIndex("att_is_hidden") - if hiddenCol < 0 { - hiddenCol = moColsPhysIsHidden - } - seqNumCol := view.columnDataIndex("att_seqnum") - if seqNumCol < 0 { - seqNumCol = moColsPhysSeqNum - } + relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") + nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") + typCol := fallbackCatalogColIndex(view, moColumnsID, "atttyp") + numCol := fallbackCatalogColIndex(view, moColumnsID, "attnum") + hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") + seqNumCol := fallbackCatalogColIndex(view, moColumnsID, "att_seqnum") var cols []TableColumn for _, fullRow := range view.Rows { @@ -445,25 +550,6 @@ func buildColumnsFromMoColumnsRows(view *LogicalTableView, tableID uint64) []Tab // types when mo_tables/mo_columns metadata is unavailable. Physical object columns may // include hidden system columns, and without mo_columns we cannot safely distinguish // visible columns from hidden ones. -// -// hardcoded physical column positions for mo_tables (after stripping meta cols). -// Layout: [rel_id(0), relname(1), reldatabase(2), reldatabase_id(3), ...] -// These are used as fallback when the LogicalTableView has col_N headers. -const ( - moTablesPhysRelID = 0 - moTablesPhysRelName = 1 - moTablesPhysRelDatabase = 2 - moTablesPhysRelCreateSQL = 7 - - // mo_columns hardcoded physical positions (after meta cols). - moColsPhysRelNameID = 4 - moColsPhysAttName = 6 - moColsPhysAttTyp = 7 - moColsPhysAttNum = 8 - moColsPhysIsHidden = 18 - moColsPhysSeqNum = 22 -) - func (r *CheckpointReader) ShowCreateTable( ctx context.Context, tableID uint64, @@ -472,14 +558,8 @@ func (r *CheckpointReader) ShowCreateTable( // 1. Try mo_tables.rel_createsql moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) if err == nil && moTablesView != nil { - relIDCol := moTablesView.columnDataIndex("rel_id") - if relIDCol < 0 { - relIDCol = moTablesPhysRelID - } - createSQLCol := moTablesView.columnDataIndex("rel_createsql") - if createSQLCol < 0 { - createSQLCol = moTablesPhysRelCreateSQL - } + relIDCol := fallbackCatalogColIndex(moTablesView, moTablesID, "rel_id") + createSQLCol := fallbackCatalogColIndex(moTablesView, moTablesID, "rel_createsql") for _, fullRow := range moTablesView.Rows { row := fullRow[logicalViewMetaCols:] if relIDCol < len(row) && row[relIDCol] == fmt.Sprintf("%d", tableID) { @@ -500,7 +580,18 @@ func (r *CheckpointReader) ShowCreateTable( } // 3. Hardcoded built-in table schemas - if ddl := hardcodedCreateTable(tableID); ddl != "" { + layout := currentCatalogLayout + switch tableID { + case moTablesID: + if moTablesView != nil { + layout, _ = inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID) + } + case moColumnsID: + if moColumnsView != nil { + layout, _ = inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID) + } + } + if ddl := hardcodedCreateTableForLayout(tableID, layout); ddl != "" { return ddl, nil } @@ -522,26 +613,11 @@ func getTableName(view *LogicalTableView, tableID uint64) string { // buildCreateTableFromMoColumns reconstructs a CREATE TABLE DDL from mo_columns data. func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) string { tableIDStr := fmt.Sprintf("%d", tableID) - relnameIDCol := view.columnDataIndex("att_relname_id") - if relnameIDCol < 0 { - relnameIDCol = moColsPhysRelNameID - } - nameCol := view.columnDataIndex("attname") - if nameCol < 0 { - nameCol = moColsPhysAttName - } - typCol := view.columnDataIndex("atttyp") - if typCol < 0 { - typCol = moColsPhysAttTyp - } - numCol := view.columnDataIndex("attnum") - if numCol < 0 { - numCol = moColsPhysAttNum - } - hiddenCol := view.columnDataIndex("att_is_hidden") - if hiddenCol < 0 { - hiddenCol = moColsPhysIsHidden - } + relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") + nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") + typCol := fallbackCatalogColIndex(view, moColumnsID, "atttyp") + numCol := fallbackCatalogColIndex(view, moColumnsID, "attnum") + hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") type colInfo struct { name string @@ -611,7 +687,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) strin // hardcodedCreateTable returns the CREATE TABLE DDL for core built-in system tables. // These tables' schemas are known at compile time and may not appear in the checkpoint's // mo_tables/mo_columns (due to minimal deployments). -func hardcodedCreateTable(tableID uint64) string { +func hardcodedCreateTableForLayout(tableID uint64, layout catalogLayout) string { switch tableID { case catalog.MO_DATABASE_ID: return "CREATE TABLE `mo_database` (\n" + @@ -627,6 +703,30 @@ func hardcodedCreateTable(tableID uint64) string { " `__mo_cpkey_dat` VARCHAR(65535)\n" + ");" case catalog.MO_TABLES_ID: + if layout.name == legacy3CatalogLayout.name { + return "CREATE TABLE `mo_tables` (\n" + + " `rel_id` BIGINT,\n" + + " `relname` VARCHAR(5000),\n" + + " `reldatabase` VARCHAR(5000),\n" + + " `reldatabase_id` BIGINT,\n" + + " `relpersistence` VARCHAR(5000),\n" + + " `relkind` VARCHAR(5000),\n" + + " `rel_comment` VARCHAR(5000),\n" + + " `rel_createsql` TEXT,\n" + + " `created_time` TIMESTAMP,\n" + + " `creator` INT UNSIGNED,\n" + + " `owner` INT UNSIGNED,\n" + + " `account_id` INT UNSIGNED,\n" + + " `partitioned` TINYINT,\n" + + " `partition_info` BLOB,\n" + + " `viewdef` VARCHAR(5000),\n" + + " `constraint` VARCHAR(5000),\n" + + " `schema_version` INT UNSIGNED,\n" + + " `schema_catalog_version` INT UNSIGNED,\n" + + " `extra_info` VARCHAR,\n" + + " `__mo_cpkey_rel` VARCHAR(65535)\n" + + ");" + } return "CREATE TABLE `mo_tables` (\n" + " `rel_id` BIGINT,\n" + " `relname` VARCHAR(5000),\n" + @@ -651,6 +751,35 @@ func hardcodedCreateTable(tableID uint64) string { " `rel_logical_id` BIGINT\n" + ");" case catalog.MO_COLUMNS_ID: + if layout.name == legacy3CatalogLayout.name { + return "CREATE TABLE `mo_columns` (\n" + + " `att_uniq_name` VARCHAR(256),\n" + + " `account_id` INT UNSIGNED,\n" + + " `att_database_id` BIGINT,\n" + + " `att_database` VARCHAR(256),\n" + + " `att_relname_id` BIGINT,\n" + + " `att_relname` VARCHAR(256),\n" + + " `attname` VARCHAR(256),\n" + + " `atttyp` VARCHAR(256),\n" + + " `attnum` INT,\n" + + " `att_length` INT,\n" + + " `attnotnull` TINYINT,\n" + + " `atthasdef` TINYINT,\n" + + " `att_default` VARCHAR(2048),\n" + + " `attisdropped` TINYINT,\n" + + " `att_constraint_type` CHAR(1),\n" + + " `att_is_unsigned` TINYINT,\n" + + " `att_is_auto_increment` TINYINT,\n" + + " `att_comment` VARCHAR(2048),\n" + + " `att_is_hidden` TINYINT,\n" + + " `att_has_update` TINYINT,\n" + + " `att_update` VARCHAR(2048),\n" + + " `att_is_clusterby` TINYINT,\n" + + " `att_seqnum` SMALLINT UNSIGNED,\n" + + " `att_enum` VARCHAR,\n" + + " `__mo_cpkey_col` VARCHAR(65535)\n" + + ");" + } return "CREATE TABLE `mo_columns` (\n" + " `att_uniq_name` VARCHAR(256),\n" + " `account_id` INT UNSIGNED,\n" + diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 9ba0b015a6fb6..bcd182d0022fc 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -242,6 +242,29 @@ func TestBuildColumnsFromMoColumnsRows_FallbackToAttnumWhenSeqnumMissing(t *test assert.Equal(t, 1, cols[1].PhysicalPosition) } +func TestInferCatalogLayout(t *testing.T) { + tests := []struct { + name string + dataWidth int + tableID uint64 + layout string + offset int + }{ + {name: "current_tables", dataWidth: len(catalog.MoTablesSchema), tableID: moTablesID, layout: currentCatalogLayout.name, offset: 0}, + {name: "current_tables_fakepk", dataWidth: len(catalog.MoTablesSchema) + 1, tableID: moTablesID, layout: currentCatalogLayout.name, offset: 1}, + {name: "legacy3_tables", dataWidth: len(legacy3CatalogLayout.moTablesSchema), tableID: moTablesID, layout: legacy3CatalogLayout.name, offset: 0}, + {name: "legacy3_columns", dataWidth: len(legacy3CatalogLayout.moColumnsSchema), tableID: moColumnsID, layout: legacy3CatalogLayout.name, offset: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + layout, offset := inferCatalogLayout(tt.dataWidth, tt.tableID) + assert.Equal(t, tt.layout, layout.name) + assert.Equal(t, tt.offset, offset) + }) + } +} + // TestLogicalTableViewWithSchema_mergeHeaders merges schema column names using position-based mapping. func TestLogicalTableViewWithSchema_mergeHeaders(t *testing.T) { schema := &TableSchema{ @@ -469,6 +492,17 @@ func TestBuildCreateTableFromMoColumns(t *testing.T) { assert.Contains(t, ddl, "CREATE TABLE") } +func TestHardcodedCreateTableForLegacy3Dev(t *testing.T) { + ddl := hardcodedCreateTableForLayout(catalog.MO_TABLES_ID, legacy3CatalogLayout) + assert.Contains(t, ddl, "CREATE TABLE `mo_tables`") + assert.NotContains(t, ddl, "rel_logical_id") + + ddl = hardcodedCreateTableForLayout(catalog.MO_COLUMNS_ID, legacy3CatalogLayout) + assert.Contains(t, ddl, "CREATE TABLE `mo_columns`") + assert.NotContains(t, ddl, "attr_generated") + assert.NotContains(t, ddl, "attr_has_generated") +} + func TestBuildCreateTableFromMoColumns_empty(t *testing.T) { view := &LogicalTableView{ Headers: []string{"object", "block", "row", "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden"}, @@ -493,7 +527,7 @@ func TestHardcodedCreateTable(t *testing.T) { for _, tt := range tests { t.Run(fmt.Sprintf("table-%d", tt.tableID), func(t *testing.T) { - ddl := hardcodedCreateTable(tt.tableID) + ddl := hardcodedCreateTableForLayout(tt.tableID, currentCatalogLayout) if tt.wantName == "" { assert.Empty(t, ddl) } else { @@ -503,3 +537,22 @@ func TestHardcodedCreateTable(t *testing.T) { }) } } + +func TestWriteCSVMetadata(t *testing.T) { + var buf bytes.Buffer + err := writeCSVMetadata(&buf, &TableSchema{ + TableName: "t1", + DatabaseName: "db1", + CreateSQL: "CREATE TABLE t1 (id INT)", + }, logicalTableStats{ + VisibleRows: 10, + DeletedRows: 3, + PhysicalRows: 13, + }) + require.NoError(t, err) + out := buf.String() + assert.Contains(t, out, "-- CREATE TABLE t1 (id INT)") + assert.Contains(t, out, "-- Database: db1") + assert.Contains(t, out, "-- Table: t1") + assert.Contains(t, out, "-- Visible rows: 10 (deleted: 3, physical: 13)") +} From 9a0e208a0fcee8caf867bfca4b3fe4af10a62ac2 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 7 Jun 2026 10:08:56 +0800 Subject: [PATCH 04/76] Fix checkpoint system table dumps --- cmd/mo-object-tool/README.md | 95 +++++- pkg/common/moerr/error.go | 4 + pkg/tools/checkpointtool/table_dump.go | 329 ++++++++++++-------- pkg/tools/checkpointtool/table_dump_test.go | 62 ++++ 4 files changed, 367 insertions(+), 123 deletions(-) diff --git a/cmd/mo-object-tool/README.md b/cmd/mo-object-tool/README.md index 695f05caa33a7..d65bac8f2d96e 100644 --- a/cmd/mo-object-tool/README.md +++ b/cmd/mo-object-tool/README.md @@ -37,9 +37,46 @@ tables, object ranges, and object data. Inside table detail view: -- Press `Enter` on a row to open the physical object view for that object range. +- Press `Enter` on a row to open the physical object view for that object. - Press `L` to open a logical table view with tombstones applied to table rows. +## Offline Table Schema And CSV Export + +`mo-tool ckp` can reconstruct table schema and export logical table rows directly +from checkpoint data. + +Show the `CREATE TABLE` statement for a table at the latest checkpoint: + +```bash +./mo-tool ckp show-create-table --table-id=272387 ./mo-data +``` + +Show the schema at a specific snapshot timestamp: + +```bash +./mo-tool ckp show-create-table \ + --table-id=272387 \ + --ts=1780574341899433000:0 \ + ./mo-data +``` + +Dump a logical table view to CSV: + +```bash +./mo-tool ckp dump --table-id=323820 ./mo-data +./mo-tool ckp dump --table-id=323820 -o workflow_cases.csv ./mo-data +``` + +The CSV dump: + +- applies tombstones at the selected checkpoint snapshot +- excludes hidden system columns +- uses catalog metadata from `mo_tables` and `mo_columns` +- streams row output instead of materializing the full CSV in memory first + +If catalog metadata cannot be resolved exactly, the command fails instead of +guessing from raw physical object columns. + ## Local Object Usage Read a single local object file: @@ -99,6 +136,18 @@ MinIO backend. The logical table view also uses the same fileservice and applies tombstones at the selected checkpoint snapshot timestamp. +The same remote options work for schema lookup and CSV export: + +```bash +./mo-tool ckp show-create-table \ + --table-id=272387 \ + --fs-config etc/launch-minio-local/tn.toml + +./mo-tool ckp dump \ + --table-id=323820 \ + --fs-config etc/launch-minio-local/tn.toml +``` + ## Remote S3 Or MinIO With Inline Arguments You can also provide object storage settings directly on the command line. @@ -147,6 +196,46 @@ Supported `--s3` keys include: Credentials can also come from the default AWS credential chain when supported by the selected backend and when `no-default-credentials` is not set. +## Catalog Layout Compatibility + +Checkpoint catalog objects are not completely stable across MatrixOne branches. +In particular, older `3.0-dev` generated data may differ from newer builds in +the layout of `mo_tables` and `mo_columns`. + +Current tool behavior: + +- it first uses actual column names from checkpoint catalog rows when present +- if the checkpoint only exposes generic `col_N` headers, it infers the system + table layout from the observed column count +- it supports both: + - current layout, which includes `mo_tables.rel_logical_id` and + `mo_columns.attr_has_generated` / `mo_columns.attr_generated` + - older `3.0-dev` layout, which does not include those columns + +For `mo_columns`, CSV extraction uses: + +- `attnum` for SQL column order +- `att_seqnum` for physical object column position + +This matters for tables with hidden columns, index-related internal columns, or +wide schemas. Without `att_seqnum`, visible values can shift to the wrong CSV +column. + +In practice, you do not need a separate flag for `3.0-dev` generated data. The +tool auto-detects the known layout variants and falls back to the matching +built-in schema for system tables when needed. + +Example with older checkpoint data: + +```bash +./mo-tool ckp show-create-table --table-id=2 ./mo-data +./mo-tool ckp dump --table-id=323820 ./mo-data +``` + +If a checkpoint comes from an unknown future layout and neither named columns +nor known-width inference matches, schema reconstruction can still fail closed. +That is intentional. + ## Important Path Rules For remote checkpoint inspection, `key-prefix` should point to the MatrixOne @@ -183,8 +272,12 @@ Checkpoint commands: ```bash ./mo-tool ckp info [directory] [--fs-config FILE] [--fs-name NAME] ./mo-tool ckp view [directory] [--fs-config FILE] [--fs-name NAME] +./mo-tool ckp dump --table-id ID [directory] [--ts PHYSICAL:LOGICAL] [-o FILE] +./mo-tool ckp show-create-table --table-id ID [directory] [--ts PHYSICAL:LOGICAL] ./mo-tool ckp info --backend S3 --s3 key=value,... ./mo-tool ckp view --backend MINIO --s3 key=value,... +./mo-tool ckp dump --table-id ID --backend S3 --s3 key=value,... +./mo-tool ckp show-create-table --table-id ID --backend S3 --s3 key=value,... ``` Object commands: diff --git a/pkg/common/moerr/error.go b/pkg/common/moerr/error.go index 444e2cbfeaefb..05c67f2271b82 100644 --- a/pkg/common/moerr/error.go +++ b/pkg/common/moerr/error.go @@ -991,6 +991,10 @@ func NewFileNotFound(ctx context.Context, f string) *Error { return newError(ctx, ErrFileNotFound, f) } +func NewFileNotFoundErrorf(ctx context.Context, format string, args ...any) *Error { + return newError(ctx, ErrFileNotFound, fmt.Sprintf(format, args...)) +} + func NewResultFileNotFound(ctx context.Context, f string) *Error { return newError(ctx, ErrResultFileNotFound, f) } diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 135a0fa9db21d..2663c9ba13189 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -73,6 +73,13 @@ type TableSchema struct { CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql } +type builtinColumnDef struct { + Name string + SQLType string + Position int + Hidden bool +} + func knownCatalogLayouts() []catalogLayout { return []catalogLayout{currentCatalogLayout, legacy3CatalogLayout} } @@ -88,6 +95,198 @@ func schemaForLayout(layout catalogLayout, tableID uint64) []string { } } +func builtinColumnsForLayout(layout catalogLayout, tableID uint64) []builtinColumnDef { + switch tableID { + case catalog.MO_DATABASE_ID: + return []builtinColumnDef{ + {Name: "dat_id", SQLType: "BIGINT", Position: 0}, + {Name: "datname", SQLType: "VARCHAR(5000)", Position: 1}, + {Name: "dat_catalog_name", SQLType: "VARCHAR(5000)", Position: 2}, + {Name: "dat_createsql", SQLType: "VARCHAR(5000)", Position: 3}, + {Name: "owner", SQLType: "INT UNSIGNED", Position: 4}, + {Name: "creator", SQLType: "INT UNSIGNED", Position: 5}, + {Name: "created_time", SQLType: "TIMESTAMP", Position: 6}, + {Name: "account_id", SQLType: "INT UNSIGNED", Position: 7}, + {Name: "dat_type", SQLType: "VARCHAR(32)", Position: 8}, + {Name: catalog.SystemDBAttr_CPKey, SQLType: "VARCHAR(65535)", Position: 9, Hidden: true}, + } + case catalog.MO_TABLES_ID: + cols := []builtinColumnDef{ + {Name: "rel_id", SQLType: "BIGINT", Position: 0}, + {Name: "relname", SQLType: "VARCHAR(5000)", Position: 1}, + {Name: "reldatabase", SQLType: "VARCHAR(5000)", Position: 2}, + {Name: "reldatabase_id", SQLType: "BIGINT", Position: 3}, + {Name: "relpersistence", SQLType: "VARCHAR(5000)", Position: 4}, + {Name: "relkind", SQLType: "VARCHAR(5000)", Position: 5}, + {Name: "rel_comment", SQLType: "VARCHAR(5000)", Position: 6}, + {Name: "rel_createsql", SQLType: "TEXT", Position: 7}, + {Name: "created_time", SQLType: "TIMESTAMP", Position: 8}, + {Name: "creator", SQLType: "INT UNSIGNED", Position: 9}, + {Name: "owner", SQLType: "INT UNSIGNED", Position: 10}, + {Name: "account_id", SQLType: "INT UNSIGNED", Position: 11}, + {Name: "partitioned", SQLType: "TINYINT", Position: 12}, + {Name: "partition_info", SQLType: "BLOB", Position: 13}, + {Name: "viewdef", SQLType: "VARCHAR(5000)", Position: 14}, + {Name: "constraint", SQLType: "VARCHAR(5000)", Position: 15}, + {Name: "schema_version", SQLType: "INT UNSIGNED", Position: 16}, + {Name: "schema_catalog_version", SQLType: "INT UNSIGNED", Position: 17}, + {Name: "extra_info", SQLType: "VARCHAR", Position: 18, Hidden: true}, + {Name: catalog.SystemRelAttr_CPKey, SQLType: "VARCHAR(65535)", Position: 19, Hidden: true}, + } + if layout.name != legacy3CatalogLayout.name { + cols = append(cols, builtinColumnDef{Name: "rel_logical_id", SQLType: "BIGINT", Position: 20}) + } + return cols + case catalog.MO_COLUMNS_ID: + cols := []builtinColumnDef{ + {Name: "att_uniq_name", SQLType: "VARCHAR(256)", Position: 0}, + {Name: "account_id", SQLType: "INT UNSIGNED", Position: 1}, + {Name: "att_database_id", SQLType: "BIGINT", Position: 2}, + {Name: "att_database", SQLType: "VARCHAR(256)", Position: 3}, + {Name: "att_relname_id", SQLType: "BIGINT", Position: 4}, + {Name: "att_relname", SQLType: "VARCHAR(256)", Position: 5}, + {Name: "attname", SQLType: "VARCHAR(256)", Position: 6}, + {Name: "atttyp", SQLType: "VARCHAR(256)", Position: 7}, + {Name: "attnum", SQLType: "INT", Position: 8}, + {Name: "att_length", SQLType: "INT", Position: 9}, + {Name: "attnotnull", SQLType: "TINYINT", Position: 10}, + {Name: "atthasdef", SQLType: "TINYINT", Position: 11}, + {Name: "att_default", SQLType: "VARCHAR(2048)", Position: 12}, + {Name: "attisdropped", SQLType: "TINYINT", Position: 13}, + {Name: "att_constraint_type", SQLType: "CHAR(1)", Position: 14}, + {Name: "att_is_unsigned", SQLType: "TINYINT", Position: 15}, + {Name: "att_is_auto_increment", SQLType: "TINYINT", Position: 16}, + {Name: "att_comment", SQLType: "VARCHAR(2048)", Position: 17}, + {Name: "att_is_hidden", SQLType: "TINYINT", Position: 18}, + {Name: "att_has_update", SQLType: "TINYINT", Position: 19}, + {Name: "att_update", SQLType: "VARCHAR(2048)", Position: 20}, + {Name: "att_is_clusterby", SQLType: "TINYINT", Position: 21}, + {Name: "att_seqnum", SQLType: "SMALLINT UNSIGNED", Position: 22}, + {Name: "att_enum", SQLType: "VARCHAR", Position: 23}, + {Name: catalog.SystemColAttr_CPKey, SQLType: "VARCHAR(65535)", Position: 24, Hidden: true}, + } + if layout.name != legacy3CatalogLayout.name { + cols = append(cols, + builtinColumnDef{Name: "attr_has_generated", SQLType: "TINYINT", Position: 25}, + builtinColumnDef{Name: "attr_generated", SQLType: "VARCHAR(2048)", Position: 26}, + ) + } + return cols + default: + return nil + } +} + +func builtinTableSchemaForLayout(layout catalogLayout, tableID uint64) *TableSchema { + cols := builtinColumnsForLayout(layout, tableID) + if len(cols) == 0 { + return nil + } + + schema := &TableSchema{ + DatabaseName: "mo_catalog", + } + switch tableID { + case catalog.MO_DATABASE_ID: + schema.TableName = "mo_database" + case catalog.MO_TABLES_ID: + schema.TableName = "mo_tables" + case catalog.MO_COLUMNS_ID: + schema.TableName = "mo_columns" + } + for _, col := range cols { + if col.Hidden { + continue + } + schema.Columns = append(schema.Columns, TableColumn{ + Name: col.Name, + SQLType: col.SQLType, + Position: col.Position, + PhysicalPosition: col.Position, + }) + } + schema.CreateSQL = renderCreateTableDDL(schema.TableName, schema.Columns) + return schema +} + +func renderCreateTableDDL(tableName string, cols []TableColumn) string { + if tableName == "" || len(cols) == 0 { + return "" + } + + var sb strings.Builder + sb.WriteString("CREATE TABLE `") + sb.WriteString(tableName) + sb.WriteString("` (\n") + for i, col := range cols { + sb.WriteString(" `") + sb.WriteString(col.Name) + sb.WriteString("`") + if col.SQLType != "" { + sb.WriteString(" ") + sb.WriteString(col.SQLType) + } + if i < len(cols)-1 { + sb.WriteString(",\n") + } else { + sb.WriteString("\n") + } + } + sb.WriteString(");") + return sb.String() +} + +func inferBuiltinCatalogLayout( + tableID uint64, + moTablesView *LogicalTableView, + moColumnsView *LogicalTableView, +) catalogLayout { + switch tableID { + case moTablesID: + if moTablesView != nil { + layout, _ := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID) + return layout + } + case moColumnsID: + if moColumnsView != nil { + layout, _ := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID) + return layout + } + case catalog.MO_DATABASE_ID: + if moTablesView != nil { + layout, _ := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID) + return layout + } + if moColumnsView != nil { + layout, _ := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID) + return layout + } + } + return currentCatalogLayout +} + +func mergeBuiltinSchemaFallback(schema *TableSchema, builtin *TableSchema, tableID uint64) *TableSchema { + if builtin == nil { + return schema + } + if schema == nil { + schema = &TableSchema{TableName: fmt.Sprintf("%d", tableID)} + } + if schema.TableName == "" || schema.TableName == fmt.Sprintf("%d", tableID) { + schema.TableName = builtin.TableName + } + if schema.DatabaseName == "" { + schema.DatabaseName = builtin.DatabaseName + } + if schema.CreateSQL == "" { + schema.CreateSQL = builtin.CreateSQL + } + if len(schema.Columns) == 0 { + schema.Columns = builtin.Columns + } + return schema +} + func inferCatalogLayout(dataWidth int, tableID uint64) (catalogLayout, int) { for _, layout := range knownCatalogLayouts() { schema := schemaForLayout(layout, tableID) @@ -170,6 +369,11 @@ func (r *CheckpointReader) ReadTableSchema( } } + if len(schema.Columns) == 0 { + layout := inferBuiltinCatalogLayout(tableID, moTablesView, moColumnsView) + schema = mergeBuiltinSchemaFallback(schema, builtinTableSchemaForLayout(layout, tableID), tableID) + } + return schema } @@ -688,130 +892,11 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) strin // These tables' schemas are known at compile time and may not appear in the checkpoint's // mo_tables/mo_columns (due to minimal deployments). func hardcodedCreateTableForLayout(tableID uint64, layout catalogLayout) string { - switch tableID { - case catalog.MO_DATABASE_ID: - return "CREATE TABLE `mo_database` (\n" + - " `dat_id` BIGINT,\n" + - " `datname` VARCHAR(5000),\n" + - " `dat_catalog_name` VARCHAR(5000),\n" + - " `dat_createsql` VARCHAR(5000),\n" + - " `owner` INT UNSIGNED,\n" + - " `creator` INT UNSIGNED,\n" + - " `created_time` TIMESTAMP,\n" + - " `account_id` INT UNSIGNED,\n" + - " `dat_type` VARCHAR(32),\n" + - " `__mo_cpkey_dat` VARCHAR(65535)\n" + - ");" - case catalog.MO_TABLES_ID: - if layout.name == legacy3CatalogLayout.name { - return "CREATE TABLE `mo_tables` (\n" + - " `rel_id` BIGINT,\n" + - " `relname` VARCHAR(5000),\n" + - " `reldatabase` VARCHAR(5000),\n" + - " `reldatabase_id` BIGINT,\n" + - " `relpersistence` VARCHAR(5000),\n" + - " `relkind` VARCHAR(5000),\n" + - " `rel_comment` VARCHAR(5000),\n" + - " `rel_createsql` TEXT,\n" + - " `created_time` TIMESTAMP,\n" + - " `creator` INT UNSIGNED,\n" + - " `owner` INT UNSIGNED,\n" + - " `account_id` INT UNSIGNED,\n" + - " `partitioned` TINYINT,\n" + - " `partition_info` BLOB,\n" + - " `viewdef` VARCHAR(5000),\n" + - " `constraint` VARCHAR(5000),\n" + - " `schema_version` INT UNSIGNED,\n" + - " `schema_catalog_version` INT UNSIGNED,\n" + - " `extra_info` VARCHAR,\n" + - " `__mo_cpkey_rel` VARCHAR(65535)\n" + - ");" - } - return "CREATE TABLE `mo_tables` (\n" + - " `rel_id` BIGINT,\n" + - " `relname` VARCHAR(5000),\n" + - " `reldatabase` VARCHAR(5000),\n" + - " `reldatabase_id` BIGINT,\n" + - " `relpersistence` VARCHAR(5000),\n" + - " `relkind` VARCHAR(5000),\n" + - " `rel_comment` VARCHAR(5000),\n" + - " `rel_createsql` TEXT,\n" + - " `created_time` TIMESTAMP,\n" + - " `creator` INT UNSIGNED,\n" + - " `owner` INT UNSIGNED,\n" + - " `account_id` INT UNSIGNED,\n" + - " `partitioned` TINYINT,\n" + - " `partition_info` BLOB,\n" + - " `viewdef` VARCHAR(5000),\n" + - " `constraint` VARCHAR(5000),\n" + - " `schema_version` INT UNSIGNED,\n" + - " `schema_catalog_version` INT UNSIGNED,\n" + - " `extra_info` VARCHAR,\n" + - " `__mo_cpkey_rel` VARCHAR(65535),\n" + - " `rel_logical_id` BIGINT\n" + - ");" - case catalog.MO_COLUMNS_ID: - if layout.name == legacy3CatalogLayout.name { - return "CREATE TABLE `mo_columns` (\n" + - " `att_uniq_name` VARCHAR(256),\n" + - " `account_id` INT UNSIGNED,\n" + - " `att_database_id` BIGINT,\n" + - " `att_database` VARCHAR(256),\n" + - " `att_relname_id` BIGINT,\n" + - " `att_relname` VARCHAR(256),\n" + - " `attname` VARCHAR(256),\n" + - " `atttyp` VARCHAR(256),\n" + - " `attnum` INT,\n" + - " `att_length` INT,\n" + - " `attnotnull` TINYINT,\n" + - " `atthasdef` TINYINT,\n" + - " `att_default` VARCHAR(2048),\n" + - " `attisdropped` TINYINT,\n" + - " `att_constraint_type` CHAR(1),\n" + - " `att_is_unsigned` TINYINT,\n" + - " `att_is_auto_increment` TINYINT,\n" + - " `att_comment` VARCHAR(2048),\n" + - " `att_is_hidden` TINYINT,\n" + - " `att_has_update` TINYINT,\n" + - " `att_update` VARCHAR(2048),\n" + - " `att_is_clusterby` TINYINT,\n" + - " `att_seqnum` SMALLINT UNSIGNED,\n" + - " `att_enum` VARCHAR,\n" + - " `__mo_cpkey_col` VARCHAR(65535)\n" + - ");" - } - return "CREATE TABLE `mo_columns` (\n" + - " `att_uniq_name` VARCHAR(256),\n" + - " `account_id` INT UNSIGNED,\n" + - " `att_database_id` BIGINT,\n" + - " `att_database` VARCHAR(256),\n" + - " `att_relname_id` BIGINT,\n" + - " `att_relname` VARCHAR(256),\n" + - " `attname` VARCHAR(256),\n" + - " `atttyp` VARCHAR(256),\n" + - " `attnum` INT,\n" + - " `att_length` INT,\n" + - " `attnotnull` TINYINT,\n" + - " `atthasdef` TINYINT,\n" + - " `att_default` VARCHAR(2048),\n" + - " `attisdropped` TINYINT,\n" + - " `att_constraint_type` CHAR(1),\n" + - " `att_is_unsigned` TINYINT,\n" + - " `att_is_auto_increment` TINYINT,\n" + - " `att_comment` VARCHAR(2048),\n" + - " `att_is_hidden` TINYINT,\n" + - " `att_has_update` TINYINT,\n" + - " `att_update` VARCHAR(2048),\n" + - " `att_is_clusterby` TINYINT,\n" + - " `att_seqnum` SMALLINT UNSIGNED,\n" + - " `att_enum` VARCHAR,\n" + - " `__mo_cpkey_col` VARCHAR(65535),\n" + - " `attr_has_generated` TINYINT,\n" + - " `attr_generated` VARCHAR(2048)\n" + - ");" - default: + schema := builtinTableSchemaForLayout(layout, tableID) + if schema == nil { return "" } + return renderCreateTableDDL(schema.TableName, schema.Columns) } // columnDataIndex returns the data-column index (0-based, after stripping meta columns) diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index bcd182d0022fc..ddb2ff3139322 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -16,11 +16,13 @@ package checkpointtool import ( "bytes" + "context" "fmt" "strings" "testing" "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -496,11 +498,14 @@ func TestHardcodedCreateTableForLegacy3Dev(t *testing.T) { ddl := hardcodedCreateTableForLayout(catalog.MO_TABLES_ID, legacy3CatalogLayout) assert.Contains(t, ddl, "CREATE TABLE `mo_tables`") assert.NotContains(t, ddl, "rel_logical_id") + assert.NotContains(t, ddl, "extra_info") + assert.NotContains(t, ddl, catalog.SystemRelAttr_CPKey) ddl = hardcodedCreateTableForLayout(catalog.MO_COLUMNS_ID, legacy3CatalogLayout) assert.Contains(t, ddl, "CREATE TABLE `mo_columns`") assert.NotContains(t, ddl, "attr_generated") assert.NotContains(t, ddl, "attr_has_generated") + assert.NotContains(t, ddl, catalog.SystemColAttr_CPKey) } func TestBuildCreateTableFromMoColumns_empty(t *testing.T) { @@ -538,6 +543,63 @@ func TestHardcodedCreateTable(t *testing.T) { } } +func TestBuiltinTableSchemaForLayout_CurrentVisibleColumnsOnly(t *testing.T) { + schema := builtinTableSchemaForLayout(currentCatalogLayout, catalog.MO_TABLES_ID) + require.NotNil(t, schema) + assert.Equal(t, "mo_tables", schema.TableName) + assert.Equal(t, "mo_catalog", schema.DatabaseName) + assert.NotEmpty(t, schema.CreateSQL) + + var names []string + for _, col := range schema.Columns { + names = append(names, col.Name) + assert.Equal(t, col.Position, col.PhysicalPosition) + } + assert.Contains(t, names, "rel_logical_id") + assert.NotContains(t, names, "extra_info") + assert.NotContains(t, names, catalog.SystemRelAttr_CPKey) +} + +func TestBuiltinTableSchemaForLayout_Legacy3Compatibility(t *testing.T) { + tablesSchema := builtinTableSchemaForLayout(legacy3CatalogLayout, catalog.MO_TABLES_ID) + require.NotNil(t, tablesSchema) + var tableNames []string + for _, col := range tablesSchema.Columns { + tableNames = append(tableNames, col.Name) + } + assert.NotContains(t, tableNames, "rel_logical_id") + assert.NotContains(t, tableNames, "extra_info") + assert.NotContains(t, tableNames, catalog.SystemRelAttr_CPKey) + + columnsSchema := builtinTableSchemaForLayout(legacy3CatalogLayout, catalog.MO_COLUMNS_ID) + require.NotNil(t, columnsSchema) + var columnNames []string + for _, col := range columnsSchema.Columns { + columnNames = append(columnNames, col.Name) + } + assert.NotContains(t, columnNames, "attr_has_generated") + assert.NotContains(t, columnNames, "attr_generated") + assert.NotContains(t, columnNames, catalog.SystemColAttr_CPKey) +} + +func TestReadTableSchema_BuiltinFallbackForSystemTable(t *testing.T) { + reader := &CheckpointReader{} + schema := reader.ReadTableSchema(context.Background(), catalog.MO_TABLES_ID, types.TS{}, nil) + + require.NotNil(t, schema) + assert.Equal(t, "mo_tables", schema.TableName) + assert.Equal(t, "mo_catalog", schema.DatabaseName) + require.NotEmpty(t, schema.Columns) + + var names []string + for _, col := range schema.Columns { + names = append(names, col.Name) + } + assert.Contains(t, names, "relname") + assert.NotContains(t, names, "extra_info") + assert.NotContains(t, names, catalog.SystemRelAttr_CPKey) +} + func TestWriteCSVMetadata(t *testing.T) { var buf bytes.Buffer err := writeCSVMetadata(&buf, &TableSchema{ From 721cab6e510f9753c5690a0122c551cbbd6afe81 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 7 Jun 2026 11:01:54 +0800 Subject: [PATCH 05/76] Make checkpoint CSV dumps load-data compatible --- cmd/mo-object-tool/README.md | 35 +- cmd/mo-object-tool/ckp/checkpoint.go | 30 +- pkg/tools/checkpointtool/checkpoint_reader.go | 70 +--- .../checkpointtool/checkpoint_reader_test.go | 76 +++++ pkg/tools/checkpointtool/logical_table.go | 8 +- pkg/tools/checkpointtool/table_dump.go | 322 ++++++++++++++++-- pkg/tools/checkpointtool/table_dump_test.go | 117 ++++++- pkg/tools/toolfs/lazy_cache_fs.go | 183 ++++++++++ pkg/tools/toolfs/storage.go | 13 + 9 files changed, 744 insertions(+), 110 deletions(-) create mode 100644 pkg/tools/checkpointtool/checkpoint_reader_test.go create mode 100644 pkg/tools/toolfs/lazy_cache_fs.go diff --git a/cmd/mo-object-tool/README.md b/cmd/mo-object-tool/README.md index d65bac8f2d96e..a5aa23cc2d912 100644 --- a/cmd/mo-object-tool/README.md +++ b/cmd/mo-object-tool/README.md @@ -65,6 +65,8 @@ Dump a logical table view to CSV: ```bash ./mo-tool ckp dump --table-id=323820 ./mo-data ./mo-tool ckp dump --table-id=323820 -o workflow_cases.csv ./mo-data +./mo-tool ckp dump --table-id=323820 --header --meta-comments ./mo-data +./mo-tool ckp dump --table-id=323820 --row-order=lexical ./mo-data ``` The CSV dump: @@ -72,11 +74,34 @@ The CSV dump: - applies tombstones at the selected checkpoint snapshot - excludes hidden system columns - uses catalog metadata from `mo_tables` and `mo_columns` -- streams row output instead of materializing the full CSV in memory first +- uses MatrixOne SQL export style field formatting so the output can be loaded + back with `LOAD DATA` +- uses `\N` for nulls +- streams row output instead of materializing the full CSV in memory first when + `--row-order=storage` (the default) If catalog metadata cannot be resolved exactly, the command fails instead of guessing from raw physical object columns. +By default, `ckp dump` is geared toward reloadability: + +- no metadata comment lines +- no header row + +Use these flags when you want a more human-readable export: + +- `--meta-comments` to prepend `-- CREATE TABLE ...` and row count comment lines +- `--header` to include a header row with visible column names +- use `--row-order=lexical` if you want deterministic CSV ordering by visible + column values for diffing or reproducible snapshots + +Row order modes: + +- `storage`: object/block/row scan order after tombstone application; this is + the default and remains large-table friendly +- `lexical`: sorts the final visible CSV rows lexicographically by exported + column values; this buffers rows in memory and is intended for smaller exports + ## Local Object Usage Read a single local object file: @@ -236,6 +261,10 @@ If a checkpoint comes from an unknown future layout and neither named columns nor known-width inference matches, schema reconstruction can still fail closed. That is intentional. +This means `3.0-dev` checkpoint data is supported for the known catalog layout +differences above, but compatibility is not an open-ended promise for arbitrary +older or future internal layouts. + ## Important Path Rules For remote checkpoint inspection, `key-prefix` should point to the MatrixOne @@ -272,11 +301,11 @@ Checkpoint commands: ```bash ./mo-tool ckp info [directory] [--fs-config FILE] [--fs-name NAME] ./mo-tool ckp view [directory] [--fs-config FILE] [--fs-name NAME] -./mo-tool ckp dump --table-id ID [directory] [--ts PHYSICAL:LOGICAL] [-o FILE] +./mo-tool ckp dump --table-id ID [directory] [--ts PHYSICAL:LOGICAL] [-o FILE] [--header] [--meta-comments] [--row-order storage|lexical] ./mo-tool ckp show-create-table --table-id ID [directory] [--ts PHYSICAL:LOGICAL] ./mo-tool ckp info --backend S3 --s3 key=value,... ./mo-tool ckp view --backend MINIO --s3 key=value,... -./mo-tool ckp dump --table-id ID --backend S3 --s3 key=value,... +./mo-tool ckp dump --table-id ID --backend S3 --s3 key=value,... [--header] [--meta-comments] [--row-order storage|lexical] ./mo-tool ckp show-create-table --table-id ID --backend S3 --s3 key=value,... ``` diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index c271f2144485d..3347373c54995 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -185,9 +185,12 @@ func openReader(ctx context.Context, dir string, storage toolfs.StorageOptions) // mo-tool ckp dump --table-id=12345 [--ts=...] [--output=table.csv] [directory] func dumpCommand(storage *toolfs.StorageOptions) *cobra.Command { var ( - tableID uint64 - tsStr string - output string + tableID uint64 + tsStr string + output string + metaComments bool + header bool + rowOrder string ) cmd := &cobra.Command{ @@ -203,7 +206,8 @@ physical columns. Examples: mo-tool ckp dump --table-id=12345 /path/to/ckp # dump to stdout mo-tool ckp dump --table-id=12345 -o users.csv . # dump to file - mo-tool ckp dump --table-id=12345 --ts=1749001234567890:1 . # dump at specific TS`, + mo-tool ckp dump --table-id=12345 --ts=1749001234567890:1 . # dump at specific TS + mo-tool ckp dump --table-id=12345 --header --meta-comments .`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { dir := "." @@ -255,7 +259,20 @@ Examples: w = outFile } - if err := reader.DumpTableCSVComposed(ctx, w, tableID, snapshotTS); err != nil { + parsedRowOrder, err := checkpointtool.ParseCSVRowOrder(rowOrder) + if err != nil { + return err + } + + if err := reader.DumpTableCSVComposed( + ctx, + w, + tableID, + snapshotTS, + checkpointtool.WithCSVMetaComments(metaComments), + checkpointtool.WithCSVHeader(header), + checkpointtool.WithCSVRowOrder(parsedRowOrder), + ); err != nil { return fmt.Errorf("dump table %d: %w", tableID, err) } @@ -269,6 +286,9 @@ Examples: cmd.Flags().Uint64Var(&tableID, "table-id", 0, "Table ID to dump (required)") cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp in 'physical:logical' format (default: latest)") cmd.Flags().StringVarP(&output, "output", "o", "", "Output CSV file path (default: stdout)") + cmd.Flags().BoolVar(&metaComments, "meta-comments", false, "Prepend DDL and row-count comment lines (disabled by default so output can be loaded directly)") + cmd.Flags().BoolVar(&header, "header", false, "Include a CSV header row with column names") + cmd.Flags().StringVar(&rowOrder, "row-order", string(checkpointtool.CSVRowOrderStorage), "CSV row order: storage (streaming, large-table friendly) or lexical (sort by visible CSV values in memory)") return cmd } diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index 422a0aab2e221..b141200a26cb3 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -509,63 +509,19 @@ func vecValueToString(vec *vector.Vector, idx int) string { return "NULL" } - switch vec.GetType().Oid { - case types.T_bool: - v := vector.MustFixedColWithTypeCheck[bool](vec) - return fmt.Sprintf("%v", v[idx]) - case types.T_int8: - v := vector.MustFixedColWithTypeCheck[int8](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_int16: - v := vector.MustFixedColWithTypeCheck[int16](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_int32: - v := vector.MustFixedColWithTypeCheck[int32](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_int64: - v := vector.MustFixedColWithTypeCheck[int64](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_uint8: - v := vector.MustFixedColWithTypeCheck[uint8](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_uint16: - v := vector.MustFixedColWithTypeCheck[uint16](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_uint32: - v := vector.MustFixedColWithTypeCheck[uint32](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_uint64: - v := vector.MustFixedColWithTypeCheck[uint64](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_float32: - v := vector.MustFixedColWithTypeCheck[float32](vec) - return fmt.Sprintf("%.4f", v[idx]) - case types.T_float64: - v := vector.MustFixedColWithTypeCheck[float64](vec) - return fmt.Sprintf("%.4f", v[idx]) - case types.T_char, types.T_varchar, types.T_text: - return vec.GetStringAt(idx) - case types.T_TS: - v := vector.MustFixedColWithTypeCheck[types.TS](vec) - return v[idx].ToString() - case types.T_Rowid: - v := vector.MustFixedColWithTypeCheck[types.Rowid](vec) - return fmt.Sprintf("%d-%d", v[idx].GetBlockOffset(), v[idx].GetRowOffset()) - case types.T_Blockid: - v := vector.MustFixedColWithTypeCheck[types.Blockid](vec) - return v[idx].String() - case types.T_Objectid: - v := vector.MustFixedColWithTypeCheck[types.Objectid](vec) - return v[idx].ShortStringEx() - default: - // For binary/blob types, show hex or truncated - if vec.GetType().IsVarlen() { - bs := vec.GetBytesAt(idx) - if len(bs) > 16 { - return fmt.Sprintf("%x...", bs[:16]) - } - return fmt.Sprintf("%x", bs) + if vec.GetType().Oid == types.T_time { + rowIdx := idx + if vec.IsConst() { + rowIdx = 0 + } + values := vector.MustFixedColWithTypeCheck[types.Time](vec) + if rowIdx >= 0 && rowIdx < len(values) { + return values[rowIdx].String2(vec.GetType().Scale) } - return fmt.Sprintf("<%s>", vec.GetType().String()) } + value := vec.RowToString(idx) + if value == "null" { + return "NULL" + } + return value } diff --git a/pkg/tools/checkpointtool/checkpoint_reader_test.go b/pkg/tools/checkpointtool/checkpoint_reader_test.go new file mode 100644 index 0000000000000..25e5efc9cd003 --- /dev/null +++ b/pkg/tools/checkpointtool/checkpoint_reader_test.go @@ -0,0 +1,76 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/stretchr/testify/require" +) + +func TestVecValueToString_SQLLikeFormatting(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + timeVec := vector.NewVec(types.New(types.T_time, 0, 6)) + defer timeVec.Free(mp) + timeValue, err := types.ParseTime("12:34:56.123456", 6) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(timeVec, timeValue, false, mp)) + require.Equal(t, "12:34:56.123456", vecValueToString(timeVec, 0)) + + datetimeVec := vector.NewVec(types.New(types.T_datetime, 0, 6)) + defer datetimeVec.Free(mp) + datetimeValue, err := types.ParseDatetime("2024-01-02 03:04:05.123456", 6) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(datetimeVec, datetimeValue, false, mp)) + require.Equal(t, "2024-01-02 03:04:05.123456", vecValueToString(datetimeVec, 0)) + + timestampVec := vector.NewVec(types.New(types.T_timestamp, 0, 6)) + defer timestampVec.Free(mp) + timestampValue, err := types.ParseTimestamp(time.Local, "2024-01-02 03:04:05.123456", 6) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(timestampVec, timestampValue, false, mp)) + require.Equal(t, timestampValue.String2(time.Local, 6), vecValueToString(timestampVec, 0)) + + decimalVec := vector.NewVec(types.New(types.T_decimal64, 18, 2)) + defer decimalVec.Free(mp) + decimalValue := types.Decimal64(12345) + require.NoError(t, vector.AppendFixed(decimalVec, decimalValue, false, mp)) + require.Equal(t, "123.45", vecValueToString(decimalVec, 0)) + + uuidVec := vector.NewVec(types.T_uuid.ToType()) + defer uuidVec.Free(mp) + uuidValue, err := types.ParseUuid("123e4567-e89b-12d3-a456-426614174000") + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(uuidVec, uuidValue, false, mp)) + require.Equal(t, "123e4567-e89b-12d3-a456-426614174000", vecValueToString(uuidVec, 0)) + + jsonVec := vector.NewVec(types.T_json.ToType()) + defer jsonVec.Free(mp) + require.NoError(t, vector.AppendBytes(jsonVec, []byte(`{"a":1}`), false, mp)) + require.Equal(t, `{"a":1}`, vecValueToString(jsonVec, 0)) + + nullVec := vector.NewVec(types.T_int64.ToType()) + defer nullVec.Free(mp) + require.NoError(t, vector.AppendFixed(nullVec, int64(0), true, mp)) + require.Equal(t, "NULL", vecValueToString(nullVec, 0)) +} diff --git a/pkg/tools/checkpointtool/logical_table.go b/pkg/tools/checkpointtool/logical_table.go index f887516047d1e..b30646fc8c8b9 100644 --- a/pkg/tools/checkpointtool/logical_table.go +++ b/pkg/tools/checkpointtool/logical_table.go @@ -54,7 +54,7 @@ func (r *CheckpointReader) BuildLogicalTableView( } return nil }, - func(objShort string, blockIdx int, rowIdx int, values []string) error { + func(objShort string, blockIdx int, rowIdx int, values []string, _ []bool) error { row := make([]string, 0, len(values)+3) row = append(row, objShort, fmt.Sprintf("%d", blockIdx), fmt.Sprintf("%d", rowIdx)) row = append(row, values...) @@ -77,7 +77,7 @@ func (r *CheckpointReader) scanLogicalTable( dataEntries []*ObjectEntryInfo, tombEntries []*ObjectEntryInfo, onColumns func([]objecttool.ColInfo) error, - onRow func(objShort string, blockIdx int, rowIdx int, values []string) error, + onRow func(objShort string, blockIdx int, rowIdx int, values []string, nulls []bool) error, ) (logicalTableStats, error) { stats := logicalTableStats{} if len(dataEntries) == 0 { @@ -141,10 +141,12 @@ func (r *CheckpointReader) scanLogicalTable( } if onRow != nil { values := make([]string, len(bat.Vecs)) + nulls := make([]bool, len(bat.Vecs)) for i, vec := range bat.Vecs { + nulls[i] = vec.IsNull(uint64(rowIdx)) values[i] = vecValueToString(vec, rowIdx) } - if err := onRow(entry.ObjectStats.ObjectName().Short().ShortString(), blockIdx, rowIdx, values); err != nil { + if err := onRow(entry.ObjectStats.ObjectName().Short().ShortString(), blockIdx, rowIdx, values, nulls); err != nil { if deleteMask.IsValid() { deleteMask.Release() } diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 2663c9ba13189..8dc7962a0f3a1 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -16,7 +16,6 @@ package checkpointtool import ( "context" - "encoding/csv" "fmt" "io" "os" @@ -27,6 +26,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/tools/objecttool" ) const ( @@ -73,6 +73,73 @@ type TableSchema struct { CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql } +type CSVRowOrder string + +const ( + CSVRowOrderStorage CSVRowOrder = "storage" + CSVRowOrderLexical CSVRowOrder = "lexical" +) + +type CSVExportOptions struct { + IncludeMetadata bool + IncludeHeader bool + RowOrder CSVRowOrder +} + +type exportedCSVRow struct { + values []string + nulls []bool +} + +type CSVExportOption func(*CSVExportOptions) + +func defaultCSVExportOptions() CSVExportOptions { + return CSVExportOptions{ + IncludeMetadata: true, + IncludeHeader: true, + RowOrder: CSVRowOrderStorage, + } +} + +func WithCSVMetaComments(include bool) CSVExportOption { + return func(opts *CSVExportOptions) { + opts.IncludeMetadata = include + } +} + +func WithCSVHeader(include bool) CSVExportOption { + return func(opts *CSVExportOptions) { + opts.IncludeHeader = include + } +} + +func WithCSVRowOrder(order CSVRowOrder) CSVExportOption { + return func(opts *CSVExportOptions) { + opts.RowOrder = order + } +} + +func ParseCSVRowOrder(s string) (CSVRowOrder, error) { + switch CSVRowOrder(strings.ToLower(strings.TrimSpace(s))) { + case "", CSVRowOrderStorage: + return CSVRowOrderStorage, nil + case CSVRowOrderLexical: + return CSVRowOrderLexical, nil + default: + return "", fmt.Errorf("unsupported row order %q (supported: %s, %s)", s, CSVRowOrderStorage, CSVRowOrderLexical) + } +} + +func resolveCSVExportOptions(opts []CSVExportOption) CSVExportOptions { + resolved := defaultCSVExportOptions() + for _, opt := range opts { + if opt != nil { + opt(&resolved) + } + } + return resolved +} + type builtinColumnDef struct { Name string SQLType string @@ -445,6 +512,7 @@ func (r *CheckpointReader) DumpTableCSV( tableID uint64, snapshotTS types.TS, dataEntries, tombEntries []*ObjectEntryInfo, + opts ...CSVExportOption, ) error { schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) if len(schema.Columns) == 0 { @@ -454,7 +522,7 @@ func (r *CheckpointReader) DumpTableCSV( tableID, ) } - return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries) + return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries, resolveCSVExportOptions(opts)) } // DumpTableCSVComposed dumps a table to CSV by composing the full checkpoint view @@ -465,6 +533,7 @@ func (r *CheckpointReader) DumpTableCSVComposed( w io.Writer, tableID uint64, snapshotTS types.TS, + opts ...CSVExportOption, ) error { dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) if err != nil { @@ -478,7 +547,7 @@ func (r *CheckpointReader) DumpTableCSVComposed( tableID, ) } - return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries) + return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries, resolveCSVExportOptions(opts)) } func (r *CheckpointReader) streamTableCSV( @@ -487,6 +556,7 @@ func (r *CheckpointReader) streamTableCSV( schema *TableSchema, snapshotTS types.TS, dataEntries, tombEntries []*ObjectEntryInfo, + options CSVExportOptions, ) error { tmpFile, err := os.CreateTemp("", "mo-tool-table-dump-*.csv") if err != nil { @@ -496,7 +566,6 @@ func (r *CheckpointReader) streamTableCSV( defer os.Remove(tmpName) defer tmpFile.Close() - cw := csv.NewWriter(tmpFile) header := make([]string, 0, len(schema.Columns)) physicalPositions := make([]int, 0, len(schema.Columns)) for _, col := range schema.Columns { @@ -507,68 +576,249 @@ func (r *CheckpointReader) streamTableCSV( } physicalPositions = append(physicalPositions, physicalPos) } - if err := cw.Write(header); err != nil { - return err + var projectedTypes []types.Type + if options.IncludeHeader { + // header is emitted after we know output mode but before rows + if err := writeSQLLoadCSVRow(tmpFile, nil, header, nil); err != nil { + return err + } } - stats, err := r.scanLogicalTable(ctx, snapshotTS, dataEntries, tombEntries, nil, - func(_ string, _ int, _ int, values []string) error { - row := make([]string, len(physicalPositions)) - for i, pos := range physicalPositions { - if pos >= 0 && pos < len(values) { - row[i] = values[pos] - } - } - return cw.Write(row) + var lexicalRows []exportedCSVRow + onRow := func(_ string, _ int, _ int, values []string, nulls []bool) error { + row := projectCSVRow(values, physicalPositions) + rowNulls := projectCSVNulls(nulls, physicalPositions) + if options.RowOrder == CSVRowOrderLexical { + lexicalRows = append(lexicalRows, exportedCSVRow{values: row, nulls: rowNulls}) + return nil + } + return writeSQLLoadCSVRow(tmpFile, projectedTypes, row, rowNulls) + } + + stats, err := r.scanLogicalTable(ctx, snapshotTS, dataEntries, tombEntries, + func(cols []objecttool.ColInfo) error { + projectedTypes = buildProjectedTypes(cols, physicalPositions) + return nil }, + onRow, ) if err != nil { return err } - cw.Flush() - if err := cw.Error(); err != nil { - return err + if options.RowOrder == CSVRowOrderLexical { + sortCSVRowsLexical(lexicalRows) + for _, row := range lexicalRows { + if err := writeSQLLoadCSVRow(tmpFile, projectedTypes, row.values, row.nulls); err != nil { + return err + } + } } if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { return err } - if err := writeCSVMetadata(w, schema, stats); err != nil { - return err + if options.IncludeMetadata { + if err := writeCSVMetadata(w, schema, stats); err != nil { + return err + } } _, err = io.Copy(w, tmpFile) return err } // WriteCSV writes a LogicalTableView with the given schema as CSV to w. -func WriteCSV(w io.Writer, schema *TableSchema, view *LogicalTableView) error { - cw := csv.NewWriter(w) - defer cw.Flush() +func WriteCSV(w io.Writer, schema *TableSchema, view *LogicalTableView, opts ...CSVExportOption) error { + options := resolveCSVExportOptions(opts) - // Write header comments (DDL + metadata) - if err := writeCSVMetadata(w, schema, logicalTableStats{ - VisibleRows: view.VisibleRows, - DeletedRows: view.DeletedRows, - PhysicalRows: view.PhysicalRows, - }); err != nil { - return err + if options.IncludeMetadata { + if err := writeCSVMetadata(w, schema, logicalTableStats{ + VisibleRows: view.VisibleRows, + DeletedRows: view.DeletedRows, + PhysicalRows: view.PhysicalRows, + }); err != nil { + return err + } } // Merge schema column names into headers merged := MergeLogicalViewWithSchema(view, schema) - if err := cw.Write(merged.Headers); err != nil { + projectedTypes := make([]types.Type, len(schema.Columns)) + for i, col := range schema.Columns { + projectedTypes[i] = sqlTypeStringToType(col.SQLType) + } + if options.IncludeHeader { + if err := writeSQLLoadCSVRow(w, nil, merged.Headers, nil); err != nil { + return err + } + } + + rows := merged.Rows + if options.RowOrder == CSVRowOrderLexical { + rows = make([][]string, len(merged.Rows)) + for i, row := range merged.Rows { + rows[i] = append([]string(nil), row...) + } + sort.SliceStable(rows, func(i, j int) bool { + return compareCSVRowsLexical(rows[i], rows[j]) < 0 + }) + } + for _, row := range rows { + if err := writeSQLLoadCSVRow(w, projectedTypes, row, nil); err != nil { + return err + } + } + return nil +} + +func projectCSVRow(values []string, physicalPositions []int) []string { + row := make([]string, len(physicalPositions)) + for i, pos := range physicalPositions { + if pos >= 0 && pos < len(values) { + row[i] = values[pos] + } + } + return row +} + +func projectCSVNulls(values []bool, physicalPositions []int) []bool { + row := make([]bool, len(physicalPositions)) + for i, pos := range physicalPositions { + if pos >= 0 && pos < len(values) { + row[i] = values[pos] + } + } + return row +} + +func sortCSVRowsLexical(rows []exportedCSVRow) { + sort.SliceStable(rows, func(i, j int) bool { + return compareCSVRowsLexical(rows[i].values, rows[j].values) < 0 + }) +} + +func buildProjectedTypes(cols []objecttool.ColInfo, physicalPositions []int) []types.Type { + projected := make([]types.Type, len(physicalPositions)) + for i, pos := range physicalPositions { + if pos >= 0 && pos < len(cols) { + projected[i] = cols[pos].Type + } + } + return projected +} + +func writeSQLLoadCSVRow(w io.Writer, colTypes []types.Type, fields []string, nulls []bool) error { + for i, field := range fields { + if i > 0 { + if _, err := io.WriteString(w, ","); err != nil { + return err + } + } + typ := types.Type{} + if i < len(colTypes) { + typ = colTypes[i] + } + isNull := i < len(nulls) && nulls[i] + if err := writeSQLLoadCSVField(w, typ, field, isNull); err != nil { + return err + } + } + _, err := io.WriteString(w, "\n") + return err +} + +func writeSQLLoadCSVField(w io.Writer, typ types.Type, field string, isNull bool) error { + if isNull { + _, err := io.WriteString(w, `\N`) return err } - // Write data rows (skip the 3 meta columns) - for _, row := range merged.Rows { - if err := cw.Write(row); err != nil { + if shouldQuoteSQLLoadType(typ) { + if _, err := io.WriteString(w, `"`); err != nil { + return err + } + escaped := escapeSQLLoadString(field, '"') + if _, err := io.WriteString(w, escaped); err != nil { return err } + _, err := io.WriteString(w, `"`) + return err } - cw.Flush() - return cw.Error() + _, err := io.WriteString(w, field) + return err +} + +func escapeSQLLoadString(s string, enclosed byte) string { + s = strings.ReplaceAll(s, `\`, `\\`) + if enclosed != 0 && enclosed != '\\' { + s = strings.ReplaceAll(s, string(enclosed), string([]byte{enclosed, enclosed})) + } + return s +} + +func shouldQuoteSQLLoadType(typ types.Type) bool { + switch typ.Oid { + case types.T_char, types.T_varchar, types.T_blob, types.T_text, + types.T_binary, types.T_varbinary, types.T_datalink, types.T_json, + types.T_geometry, types.T_array_float32, types.T_array_float64: + return true + default: + return false + } +} + +func sqlTypeStringToType(sqlType string) types.Type { + s := strings.ToUpper(strings.TrimSpace(sqlType)) + switch { + case strings.Contains(s, "CHAR"), strings.Contains(s, "TEXT"), strings.Contains(s, "BLOB"), strings.Contains(s, "DATALINK"): + return types.T_varchar.ToType() + case strings.Contains(s, "JSON"): + return types.T_json.ToType() + case strings.Contains(s, "GEOMETRY"): + return types.T_geometry.ToType() + case strings.Contains(s, "DECIMAL"): + return types.New(types.T_decimal128, 0, 0) + case strings.Contains(s, "TIMESTAMP"): + return types.T_timestamp.ToType() + case strings.Contains(s, "DATETIME"): + return types.T_datetime.ToType() + case strings.HasPrefix(s, "TIME"): + return types.T_time.ToType() + case strings.Contains(s, "DATE"): + return types.T_date.ToType() + case strings.Contains(s, "DOUBLE"): + return types.T_float64.ToType() + case strings.Contains(s, "FLOAT"): + return types.T_float32.ToType() + case strings.Contains(s, "BOOL"): + return types.T_bool.ToType() + case strings.Contains(s, "BIGINT"): + return types.T_int64.ToType() + case strings.Contains(s, "INT"): + return types.T_int32.ToType() + default: + return types.Type{} + } +} + +func compareCSVRowsLexical(left, right []string) int { + limit := len(left) + if len(right) < limit { + limit = len(right) + } + for i := 0; i < limit; i++ { + if cmp := strings.Compare(left[i], right[i]); cmp != 0 { + return cmp + } + } + switch { + case len(left) < len(right): + return -1 + case len(left) > len(right): + return 1 + default: + return 0 + } } func writeCSVMetadata(w io.Writer, schema *TableSchema, stats logicalTableStats) error { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index ddb2ff3139322..a399a33f59adc 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -23,6 +23,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/sql/util/csvparser" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -100,10 +101,10 @@ func TestWriteCSV_withData(t *testing.T) { assert.Equal(t, 5, len(dataLines)) // header + 4 rows assert.Equal(t, "id,name,age", dataLines[0]) - assert.Equal(t, "1,Alice,30", dataLines[1]) - assert.Equal(t, "2,Bob,25", dataLines[2]) + assert.Equal(t, `1,"Alice",30`, dataLines[1]) + assert.Equal(t, `2,"Bob",25`, dataLines[2]) assert.Equal(t, `3,"Charlie, Jr.",35`, dataLines[3]) - assert.Equal(t, "4,NULL,NULL", dataLines[4]) + assert.Equal(t, `4,"NULL",NULL`, dataLines[4]) } // TestWriteCSV_withCreateSQLHeader tests the header comment from CreateSQL. @@ -133,6 +134,31 @@ func TestWriteCSV_withCreateSQLHeader(t *testing.T) { assert.Contains(t, output, "-- Table: t1") } +func TestWriteCSV_WithoutMetadataComments(t *testing.T) { + schema := &TableSchema{ + TableName: "t1", + DatabaseName: "db1", + CreateSQL: "CREATE TABLE t1 (id BIGINT)", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0"}, + Rows: [][]string{ + {"obj1", "0", "0", "42"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view, WithCSVMetaComments(false)) + require.NoError(t, err) + + output := strings.TrimSpace(buf.String()) + assert.Equal(t, "id\n42", output) + assert.NotContains(t, output, "-- CREATE TABLE") +} + // TestWriteCSV_mismatchedColumns verifies that position-based column mapping // only includes columns matching the schema positions. func TestWriteCSV_mismatchedColumns(t *testing.T) { @@ -367,8 +393,8 @@ func TestMergeLogicalViewWithSchema_hiddenColumns(t *testing.T) { output := buf.String() assert.Contains(t, output, "id,name") - assert.Contains(t, output, "1,Alice") - assert.Contains(t, output, "2,Bob") + assert.Contains(t, output, `1,"Alice"`) + assert.Contains(t, output, `2,"Bob"`) assert.NotContains(t, output, "fake_pk") // hidden columns excluded assert.NotContains(t, output, "cpkey") // hidden columns excluded } @@ -400,6 +426,36 @@ func TestMergeLogicalViewWithSchema_LargeSparsePhysicalPositions(t *testing.T) { assert.Equal(t, []string{"v0", "v63", "v127"}, merged.Rows[0]) } +func TestWriteCSV_LexicalRowOrder(t *testing.T) { + schema := &TableSchema{ + TableName: "sorted", + Columns: []TableColumn{ + {Name: "id", Position: 1, PhysicalPosition: 0}, + {Name: "name", Position: 2, PhysicalPosition: 1}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{ + {"obj2", "0", "1", "2", "zeta"}, + {"obj1", "0", "0", "1", "beta"}, + {"obj3", "0", "0", "1", "alpha"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view, WithCSVMetaComments(false), WithCSVRowOrder(CSVRowOrderLexical)) + require.NoError(t, err) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Equal(t, []string{ + "id,name", + "1,alpha", + "1,beta", + "2,zeta", + }, lines) +} + // makeColumnRow creates a mock mo_columns row with specific values at given indexes. func makeColumnRow(indexValuePairs ...any) []string { // 27 columns in mo_columns @@ -461,7 +517,7 @@ func TestColumnSchemaRoundTrip(t *testing.T) { output := buf.String() assert.Contains(t, output, "a,b,c") - assert.Contains(t, output, "1,hello,1.500000") + assert.Contains(t, output, `1,"hello",1.500000`) // Verify comments are present assert.Contains(t, output, "-- CREATE TABLE roundtrip") @@ -618,3 +674,52 @@ func TestWriteCSVMetadata(t *testing.T) { assert.Contains(t, out, "-- Table: t1") assert.Contains(t, out, "-- Visible rows: 10 (deleted: 3, physical: 13)") } + +func TestWriteSQLLoadCSVRow_NullAndEscaping(t *testing.T) { + var buf bytes.Buffer + types := []types.Type{types.T_varchar.ToType(), types.T_int64.ToType(), types.T_json.ToType()} + err := writeSQLLoadCSVRow(&buf, types, []string{`a"b\c`, "42", `{"x":"y"}`}, []bool{false, true, false}) + require.NoError(t, err) + assert.Equal(t, "\"a\"\"b\\\\c\",\\N,\"{\"\"x\"\":\"\"y\"\"}\"\n", buf.String()) +} + +func TestWriteSQLLoadCSVRow_RoundTripsThroughCSVParser(t *testing.T) { + var buf bytes.Buffer + colTypes := []types.Type{types.T_varchar.ToType(), types.T_int64.ToType(), types.T_json.ToType()} + err := writeSQLLoadCSVRow(&buf, colTypes, []string{`a"b\c`, "42", `{"x":"y"}`}, []bool{false, true, false}) + require.NoError(t, err) + + parser, err := csvparser.NewCSVParser(&csvparser.CSVConfig{ + FieldsTerminatedBy: ",", + FieldsEnclosedBy: `"`, + FieldsEscapedBy: `\`, + LinesTerminatedBy: "\n", + Null: []string{`\N`}, + UnescapedQuote: true, + }, strings.NewReader(buf.String()), csvparser.ReadBlockSize, false) + require.NoError(t, err) + + row, err := parser.Read(nil) + require.NoError(t, err) + require.Len(t, row, 3) + assert.Equal(t, `a"b\c`, row[0].Val) + assert.False(t, row[0].IsNull) + assert.True(t, row[0].HasStringQuote) + assert.True(t, row[1].IsNull) + assert.Equal(t, `{"x":"y"}`, row[2].Val) + assert.False(t, row[2].IsNull) + assert.True(t, row[2].HasStringQuote) +} + +func TestParseCSVRowOrder(t *testing.T) { + order, err := ParseCSVRowOrder("storage") + require.NoError(t, err) + assert.Equal(t, CSVRowOrderStorage, order) + + order, err = ParseCSVRowOrder("lexical") + require.NoError(t, err) + assert.Equal(t, CSVRowOrderLexical, order) + + _, err = ParseCSVRowOrder("unknown") + require.Error(t, err) +} diff --git a/pkg/tools/toolfs/lazy_cache_fs.go b/pkg/tools/toolfs/lazy_cache_fs.go new file mode 100644 index 0000000000000..bdc14f40604b7 --- /dev/null +++ b/pkg/tools/toolfs/lazy_cache_fs.go @@ -0,0 +1,183 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toolfs + +import ( + "context" + "iter" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/matrixorigin/matrixone/pkg/fileservice" +) + +type lazyCacheFS struct { + name string + remote fileservice.FileService + local fileservice.FileService + root string + + mu sync.Mutex + caching map[string]*sync.Once +} + +func newLazyCacheFS(ctx context.Context, remote fileservice.FileService) (fileservice.FileService, string, error) { + root, err := os.MkdirTemp("", "mo-tool-remote-cache-*") + if err != nil { + return nil, "", err + } + local, err := fileservice.NewLocalFS(ctx, remote.Name(), root, fileservice.DisabledCacheConfig, nil) + if err != nil { + _ = os.RemoveAll(root) + return nil, "", err + } + return &lazyCacheFS{ + name: remote.Name(), + remote: remote, + local: local, + root: root, + caching: make(map[string]*sync.Once), + }, root, nil +} + +func (l *lazyCacheFS) Name() string { return l.name } + +func (l *lazyCacheFS) Write(ctx context.Context, vector fileservice.IOVector) error { + if err := l.remote.Write(ctx, vector); err != nil { + return err + } + return l.local.Write(ctx, vector) +} + +func (l *lazyCacheFS) Read(ctx context.Context, vector *fileservice.IOVector) error { + if err := l.ensureCached(ctx, vector.FilePath); err != nil { + return err + } + return l.local.Read(ctx, vector) +} + +func (l *lazyCacheFS) ReadCache(ctx context.Context, vector *fileservice.IOVector) error { + if err := l.ensureCached(ctx, vector.FilePath); err != nil { + return err + } + return l.local.ReadCache(ctx, vector) +} + +func (l *lazyCacheFS) List(ctx context.Context, dirPath string) iter.Seq2[*fileservice.DirEntry, error] { + return l.remote.List(ctx, dirPath) +} + +func (l *lazyCacheFS) Delete(ctx context.Context, filePaths ...string) error { + _ = l.local.Delete(ctx, filePaths...) + return l.remote.Delete(ctx, filePaths...) +} + +func (l *lazyCacheFS) StatFile(ctx context.Context, filePath string) (*fileservice.DirEntry, error) { + return l.remote.StatFile(ctx, filePath) +} + +func (l *lazyCacheFS) PrefetchFile(ctx context.Context, filePath string) error { + return l.ensureCached(ctx, filePath) +} + +func (l *lazyCacheFS) Cost() *fileservice.CostAttr { + return l.remote.Cost() +} + +func (l *lazyCacheFS) Close(ctx context.Context) { + l.local.Close(ctx) + l.remote.Close(ctx) + _ = os.RemoveAll(l.root) +} + +func (l *lazyCacheFS) ensureCached(ctx context.Context, filePath string) error { + normalized := stripServiceName(filePath, l.name) + localPath := filepath.Join(l.root, filepath.FromSlash(normalized)) + if _, err := os.Stat(localPath); err == nil { + return nil + } + + once := l.getOnce(normalized) + var cacheErr error + once.Do(func() { + cacheErr = l.cacheFile(ctx, normalized, localPath) + if cacheErr != nil { + l.mu.Lock() + delete(l.caching, normalized) + l.mu.Unlock() + } + }) + return cacheErr +} + +func (l *lazyCacheFS) getOnce(path string) *sync.Once { + l.mu.Lock() + defer l.mu.Unlock() + if once, ok := l.caching[path]; ok { + return once + } + once := &sync.Once{} + l.caching[path] = once + return once +} + +func (l *lazyCacheFS) cacheFile(ctx context.Context, normalized string, localPath string) error { + if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(localPath), ".cache-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer func() { + _ = tmp.Close() + _ = os.Remove(tmpName) + }() + + vec := &fileservice.IOVector{ + FilePath: normalized, + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: -1, + WriterForRead: tmp, + }}, + } + if err := l.remote.Read(ctx, vec); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmpName, localPath); err != nil { + return err + } + return nil +} + +func stripServiceName(path string, service string) string { + prefix := strings.ToUpper(service) + ":" + if strings.HasPrefix(strings.ToUpper(path), prefix) { + return path[len(service)+1:] + } + if idx := strings.Index(path, ":"); idx >= 0 { + return path[idx+1:] + } + return path +} + +var _ fileservice.FileService = (*lazyCacheFS)(nil) diff --git a/pkg/tools/toolfs/storage.go b/pkg/tools/toolfs/storage.go index b83a9f25df5ca..732d68803bc4a 100644 --- a/pkg/tools/toolfs/storage.go +++ b/pkg/tools/toolfs/storage.go @@ -74,6 +74,12 @@ func Open(ctx context.Context, opts StorageOptions) (fileservice.FileService, st if err != nil { return nil, "", err } + if backend == "S3" || backend == "MINIO" { + fs, _, err = newLazyCacheFS(ctx, fs) + if err != nil { + return nil, "", err + } + } return fs, fmt.Sprintf("%s:%s", strings.ToLower(backend), args.KeyPrefix), nil } @@ -104,6 +110,13 @@ func openFromConfig(ctx context.Context, path string, fsName string) (fileservic if err != nil { return nil, "", err } + backend := strings.ToUpper(fsCfg.Backend) + if backend == "S3" || backend == "MINIO" { + fs, _, err = newLazyCacheFS(ctx, fs) + if err != nil { + return nil, "", err + } + } return fs, fmt.Sprintf("%s:%s", path, fsCfg.Name), nil } } From 2c7c75e182ad57c5a7a2ba6b34077edf62ca9740 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 11 Jun 2026 14:06:55 +0800 Subject: [PATCH 06/76] fix checkpoint dump timestamp handling --- VIEW_CKP_STATUS.md | 484 ++++++++++++ cmd/mo-object-tool/ckp/checkpoint.go | 655 +++++++++++++++- .../checkpoint_tool_feature_analysis.md | 714 ++++++++++++++++++ docs/design/checkpoint_tool_fragility.md | 263 +++++++ etc/launch-minio-local/log.toml | 1 - etc/launch-minio-local/tn.toml | 10 +- pkg/tools/checkpointtool/checkpoint_reader.go | 48 +- pkg/tools/checkpointtool/logical_table.go | 35 +- pkg/tools/checkpointtool/table_dump.go | 416 +++++++++- pkg/tools/checkpointtool/table_dump_test.go | 137 +++- pkg/tools/checkpointtool/types.go | 44 ++ pkg/tools/objecttool/interactive/state.go | 10 +- pkg/tools/objecttool/object_reader.go | 51 +- pkg/tools/toolfs/lazy_cache_fs.go | 61 +- pkg/tools/toolfs/lazy_cache_fs_test.go | 72 ++ 15 files changed, 2876 insertions(+), 125 deletions(-) create mode 100644 VIEW_CKP_STATUS.md create mode 100644 docs/design/checkpoint_tool_feature_analysis.md create mode 100644 docs/design/checkpoint_tool_fragility.md create mode 100644 pkg/tools/toolfs/lazy_cache_fs_test.go diff --git a/VIEW_CKP_STATUS.md b/VIEW_CKP_STATUS.md new file mode 100644 index 0000000000000..78b3853af0e83 --- /dev/null +++ b/VIEW_CKP_STATUS.md @@ -0,0 +1,484 @@ +# view_ckp 分支状态总结 + +## 分支概述 + +`view_ckp` 分支主题是 **checkpoint 离线工具链**：checkpoint 浏览、表 schema 解析、CSV 数据 dump、LOAD DATA 回灌兼容，以及远程 S3/MinIO fileservice 读取。 + +| 提交 | 内容 | +|------|------| +| `913f27d09` | 添加远程 checkpoint 工具和逻辑表视图 | +| `eaa1bfc11` | 修复 checkpoint schema 回放和 CSV 列映射 | +| `953b6e76b` | 优化 checkpoint CSV dump 并添加 layout 兼容性 | +| `9a0e208a0` | 修复 checkpoint 系统表 dump | +| `721cab6e5` | 使 CSV dump 兼容 LOAD DATA | + +## 当前环境 + +- **mo-service**：本次验证结束后已停止；测试时 MySQL 端口为 `6001`，root 密码 `111` +- **存储后端**：当前 `etc/launch-minio-local/*.toml` 使用 MinIO 作为 `SHARED`/`ETL`，`LOCAL` 使用 DISK +- **checkpoint 目录**：`etc/launch-minio-local/mo-data` +- **当前 checkpoint**：`Total Entries: 44`，`Global: 11`，`Incremental: 33`，`Latest TS: 1781084792256166694-1` +- **构建命令**：`make mo-tool` + +注意：在当前 Codex/exec 环境里，`nohup ... &` 启动的后台进程可能会随命令会话结束被清理；本次验证使用前台 exec session 启动 `mo-service`。普通 shell 环境可继续按后台方式启动。 + +## 已修复问题 + +### 1. 远程 S3/MinIO checksum 不匹配 + +现象： + +```text +Error: open checkpoint dir: internal error: checksum not match +``` + +根因： + +- MO 写入 S3/MinIO 的 object 文件以 `pkg/objectio/object.go` 中的 `Magic = 0xFFFFFFFF` 开头。 +- `mo-tool` 的 `lazyCacheFS` 把远程文件下载到本地临时目录后，再通过 `LocalFS.Read` 读取。 +- `LocalFS.Read` 会按本地 fileservice checksum 格式解析，期望文件块以 4 字节 CRC32 开头。 +- 远程 object 文件的 Magic 前缀不是 LocalFS checksum，因此必然报 checksum mismatch。 + +修复： + +- `pkg/tools/toolfs/lazy_cache_fs.go` 中，缓存文件仍落本地，但读取缓存时直接按 OS file range 读取，不再通过 `LocalFS.Read` 的 checksum 包装路径。 +- `Write` 后清理本地缓存，避免读到旧缓存。 + +验证： + +```bash +go test ./pkg/tools/toolfs +``` + +结果：通过。 + +### 2. 用户表 CSV dump 无法解析 mo_columns + +原始报错： + +```text +cannot resolve visible columns for table 272535 from checkpoint metadata; +mo_columns data is unavailable or incomplete +``` + +根因不是单纯的固定列宽问题。实测当前 checkpoint raw logical view： + +- `mo_tables` 可见导出为 19 列，但 raw view 可能带尾部额外列。 +- `mo_columns` raw view 为 `data=28`，目标行字段从 `att_uniq_name` 开始，额外列不是稳定前缀。 +- 当前源码里的 `catalog.MoTablesSchema`/`catalog.MoColumnsSchema` 与 checkpoint 中的 catalog 布局不完全一致，尤其涉及 `CPKey`、`ExtraInfo`、生成列等字段。 +- 只按 `len(schema)+offset` 做一次 layout 推断，会在 `pre-cpk/current/legacy` 布局之间误判。 + +修复： + +- 新增 `preCPKLayout`，覆盖无 `CPKey`、无 `ExtraInfo` 的历史 catalog 布局。 +- `mo_columns` 和 `mo_tables` 解析改为候选布局试读：优先使用实际 header 名称；header 只有 `col_N` 时，按已知 catalog 布局和偏移逐个尝试，只有能读出目标 table_id 的列/DDL 时才采纳。 +- `mo_columns` 字段名改为当前 catalog 常量，例如 `attr_seqnum`、`attr_update`，避免旧的 `att_seqnum` 写法导致索引失配。 +- `show-create-table` 优先通过候选布局读取 `mo_tables.rel_createsql`，避免退化到 `mo_columns.atttyp` 的内部二进制类型编码。 + +相关文件： + +- `pkg/tools/checkpointtool/table_dump.go` +- `pkg/tools/checkpointtool/table_dump_test.go` + +验证： + +```bash +CGO_CFLAGS="-I$(pwd)/thirdparties/install/include " \ +CGO_LDFLAGS="-L$(pwd)/thirdparties/install/lib" \ +LD_LIBRARY_PATH="$(pwd)/thirdparties/install/lib:$LD_LIBRARY_PATH" \ +go test ./pkg/tools/checkpointtool + +make mo-tool +``` + +结果：通过。 + +## 实测结果 + +### 普通表导出与 LOAD DATA + +测试表： + +```sql +CREATE TABLE test_ckp.employees ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); +``` + +table_id： + +```text +272535 +``` + +导出： + +```bash +./mo-tool ckp dump --table-id=272535 --header \ + -o /tmp/employees_dump.csv \ + etc/launch-minio-local/mo-data +``` + +结果： + +```text +104 /tmp/employees_dump.csv +id,name,age,salary,department,hire_date,is_active +1,"Alice",30,75000.00,"Engineering",2020-01-15,true +2,"Bob",\N,\N,\N,\N,\N +3,"Charlie",\N,65000.00,\N,2021-03-10,false +``` + +`show-create-table`： + +```bash +./mo-tool ckp show-create-table --table-id=272535 etc/launch-minio-local/mo-data +``` + +结果正确返回原始 DDL： + +```sql +CREATE TABLE employees ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +) +``` + +LOAD DATA 校验： + +```text +original_rows loaded_rows +103 103 + +diff_rows +0 +``` + +### MinIO 远程 checkpoint 导出与 LOAD DATA + +本次追加验证时间：`2026-06-11`。 + +测试环境： + +- 使用本地 `/usr/local/bin/minio` 启动 MinIO，API 为 `http://127.0.0.1:9000` +- bucket：`mo-test` +- `SHARED` key-prefix：`server/data` +- 访问密钥：`minio` / `minio123` +- Docker compose 路径未使用：`docker pull minio/minio:RELEASE.2023-11-01T18-37-25Z` 通过当前 Docker registry 代理返回 500，因此改用本地 MinIO 二进制 + +测试表： + +```sql +CREATE TABLE test_ckp_minio.employees_minio ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); +``` + +table_id： + +```text +272535 +``` + +确认 MO 实际使用 MinIO： + +```text +new object storage {"sdk": "minio", "arguments": {"Name":"SHARED","Bucket":"mo-test","Endpoint":"127.0.0.1:9000","KeyPrefix":"server/data","IsMinio":true}} +new object storage {"sdk": "minio", "arguments": {"Name":"ETL","Bucket":"mo-test","Endpoint":"127.0.0.1:9000","KeyPrefix":"server/etl","IsMinio":true}} +``` + +远程 checkpoint ready： + +```bash +./mo-tool ckp dump --table-id=2 \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED \ + -o /tmp/minio_mo_tables_probe.csv +``` + +结果： + +```text +2008 /tmp/minio_mo_tables_probe.csv +``` + +`ckp info` 验证了两条远程打开路径： + +```bash +./mo-tool ckp info \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED + +./mo-tool ckp info \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 +``` + +结果均为： + +```text +Total Entries: 2 + Global: 0 + Incremental: 2 +Latest TS: 1781142243226306710-0 +``` + +用户表 dump： + +```bash +./mo-tool ckp dump --table-id=272535 --header \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED \ + -o /tmp/minio_employees_dump.csv +``` + +结果： + +```text +104 /tmp/minio_employees_dump.csv +id,name,age,salary,department,hire_date,is_active +``` + +`--s3` 直连路径也通过，且与 `--fs-config` 输出一致： + +```bash +./mo-tool ckp dump --table-id=272535 --header \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 \ + -o /tmp/minio_employees_dump_s3.csv + +cmp -s /tmp/minio_employees_dump.csv /tmp/minio_employees_dump_s3.csv +``` + +结果： + +```text +104 /tmp/minio_employees_dump_s3.csv +cmp_exit=0 +``` + +`show-create-table`： + +```bash +./mo-tool ckp show-create-table --table-id=272535 \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED +``` + +结果正确返回原始 DDL： + +```sql +CREATE TABLE employees_minio ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +) +``` + +LOAD DATA 校验： + +```text +original_rows loaded_rows +103 103 + +diff_rows +0 +``` + +结论： + +- `mo-tool ckp info/dump/show-create-table` 通过 MinIO 远程 fileservice 可用。 +- `--fs-config ... --fs-name SHARED` 和 `--backend MINIO --s3 ...` 两条路径均可读取 checkpoint。 +- 未再出现 `checksum not match`。 +- 从 MinIO checkpoint 导出的 CSV 可直接 `LOAD DATA` 回灌，行数和差异校验通过。 + +### ALTER TABLE 后新旧数据兼容 + +测试过程： + +1. 创建旧布局表并插入 3 行： + +```sql +CREATE TABLE test_ckp.alter_compat ( + id INT PRIMARY KEY, + name VARCHAR(50) +); +INSERT INTO alter_compat VALUES (1, 'old_a'), (2, 'old_b'), (3, 'old_c'); +``` + +2. 等待旧 table_id `272537` 能从 checkpoint 导出： + +```text +id,name +1,"old_a" +2,"old_b" +3,"old_c" +``` + +3. 执行 ALTER 并插入新布局数据： + +```sql +ALTER TABLE alter_compat ADD COLUMN note VARCHAR(50) DEFAULT 'legacy'; +INSERT INTO alter_compat VALUES (4, 'new_d', 'fresh'), (5, 'new_e', NULL); +``` + +4. ALTER 后 MatrixOne 将当前 catalog table_id 切到 `272538`： + +```text +rel_id relname +272538 alter_compat +``` + +用旧 table_id `272537` 导出会失败，因为当前 catalog metadata 已经不再暴露旧 table_id 的 `mo_columns` 可见列；这是预期边界。应使用 ALTER 后的当前 table_id `272538` 导出。 + +导出 `272538` 结果： + +```text +id,name,note +1,"old_a","legacy" +2,"old_b","legacy" +3,"old_c","legacy" +4,"new_d","fresh" +5,"new_e",\N +``` + +LOAD DATA 校验： + +```text +original_rows loaded_rows +5 5 + +diff_rows +0 +``` + +兼容性结论： + +- ALTER 前旧对象和 ALTER 后新对象在当前 table_id `272538` 的 checkpoint 视图下可一起导出。 +- 新增列默认值已在导出结果中物化，旧行 `note='legacy'`。 +- 使用 ALTER 前旧 table_id 导出当前表不是支持目标；旧 ID 在 catalog 中已被新 ID 替代，schema 解析会失败。 + +## catalog/checkpoint 版本兼容性结论 + +当前实现支持以下 catalog layout： + +| Layout | 适用场景 | 处理方式 | +|--------|----------|----------| +| `current` | 当前源码中的 `catalog.MoTablesSchema` / `catalog.MoColumnsSchema` | 直接按当前 schema 或 header 名解析 | +| `pre-cpk` | checkpoint 中 catalog 尚未包含 `CPKey`/`ExtraInfo` 的布局 | 移除这些字段后按候选布局试读 | +| `3.0-dev` | 已知 3.0 开发期尾部字段较少的布局 | 保留原兼容路径 | + +重要规则： + +- 如果 checkpoint logical view 提供真实列名，优先按列名解析。 +- 如果只有 `col_N`，不再相信单一宽度推断；按候选 layout 和偏移逐个试读。 +- dataWidth 可能包含尾部额外物理列，不能只按 `len(schema)+offset` 精确匹配。 +- 未知未来 catalog layout 若无法通过候选试读解析出目标 table_id，会明确失败，不回退到物理列导出，避免把隐藏列或错列导出成用户数据。 + +## 推荐测试步骤 + +### 1. 构建 + +```bash +make mo-tool +``` + +### 2. 启动 mo-service + +普通 shell 可用： + +```bash +kill -9 $(pgrep mo-service) 2>/dev/null || true +rm -rf etc/launch-minio-local/mo-data +mkdir -p etc/launch-minio-local/mo-data +./mo-service -launch etc/launch-minio-local/launch.toml > /tmp/mo-service.log 2>&1 & +until mysqladmin -h 127.0.0.1 -P 6001 -u root -p111 ping >/dev/null 2>&1; do + sleep 1 +done +``` + +在 Codex exec 环境中建议前台运行： + +```bash +./mo-service -launch etc/launch-minio-local/launch.toml +``` + +### 3. 等 checkpoint 可用于 dump + +不要只等 `Incremental: [1-9]`。第一次 incremental 可能还没有目标表或系统表范围。更稳妥的方式是直接等系统表 dump 成功： + +```bash +until ./mo-tool ckp dump --table-id=2 -o /tmp/mo_tables_probe.csv \ + etc/launch-minio-local/mo-data >/dev/null 2>&1; do + sleep 5 +done +``` + +### 4. 导出并回灌 + +```bash +TABLE_ID=272535 + +./mo-tool ckp dump --table-id=${TABLE_ID} --header \ + -o /tmp/employees_dump.csv \ + etc/launch-minio-local/mo-data + +wc -l /tmp/employees_dump.csv +head -5 /tmp/employees_dump.csv +``` + +```sql +CREATE TABLE test_ckp.employees_load ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); + +LOAD DATA INFILE '/tmp/employees_dump.csv' +INTO TABLE test_ckp.employees_load +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; + +SELECT (SELECT COUNT(*) FROM test_ckp.employees) AS original_rows, + (SELECT COUNT(*) FROM test_ckp.employees_load) AS loaded_rows; + +SELECT COUNT(*) FROM + (SELECT * FROM test_ckp.employees EXCEPT SELECT * FROM test_ckp.employees_load) AS diff; +``` + +## 常用命令 + +```bash +./mo-tool ckp info etc/launch-minio-local/mo-data +./mo-tool ckp show-create-table --table-id= etc/launch-minio-local/mo-data +./mo-tool ckp dump --table-id= --header -o /tmp/table.csv etc/launch-minio-local/mo-data +./mo-tool ckp dump --table-id=2 -o /tmp/mo_tables.csv etc/launch-minio-local/mo-data +./mo-tool ckp dump --table-id=3 -o /tmp/mo_columns.csv etc/launch-minio-local/mo-data +``` diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 3347373c54995..5fa3c42eae5ee 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -17,9 +17,15 @@ package ckp import ( "context" "fmt" + "io" "os" + "path/filepath" + "sort" "strconv" "strings" + "sync" + "text/tabwriter" + "time" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -48,6 +54,7 @@ func PrepareCommand() *cobra.Command { cmd.AddCommand(infoCommand(&storage)) cmd.AddCommand(viewCommand(&storage)) + cmd.AddCommand(listCommand(&storage)) cmd.AddCommand(dumpCommand(&storage)) cmd.AddCommand(showCreateTableCommand(&storage)) @@ -164,6 +171,132 @@ func viewCommand(storage *toolfs.StorageOptions) *cobra.Command { } } +func listCommand(storage *toolfs.StorageOptions) *cobra.Command { + var ( + accountID uint32 + databaseID uint64 + tsStr string + includeViews bool + listType string + ) + + cmd := &cobra.Command{ + Use: "list [directory]", + Short: "List checkpoint catalog tables", + Long: `List table metadata from the checkpoint catalog. + +By default this lists ordinary tables. Use --include-views to include views, +and use --database-id or --account-id to narrow the result.`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := "." + if len(args) == 1 { + dir = args[0] + } + + logFile, err := setupLogFile() + if err != nil { + return fmt.Errorf("setup log file: %w", err) + } + defer logFile.Close() + + ctx := context.Background() + reader, err := openReader(ctx, dir, *storage) + if err != nil { + return fmt.Errorf("open checkpoint dir: %w", err) + } + defer reader.Close() + + snapshotTS, err := resolveSnapshotTS(ctx, reader, tsStr) + if err != nil { + return fmt.Errorf("resolve --ts: %w", err) + } + + var accountFilter *uint32 + if cmd.Flags().Changed("account-id") { + accountFilter = &accountID + } + var dbFilter *uint64 + if cmd.Flags().Changed("database-id") { + dbFilter = &databaseID + } + tables, err := reader.ListCatalogTables(ctx, snapshotTS, checkpointtool.TableListOptions{ + AccountID: accountFilter, + DatabaseID: dbFilter, + IncludeViews: includeViews, + }) + if err != nil { + return fmt.Errorf("list checkpoint catalog tables: %w", err) + } + if err := printCatalogList(cmd.OutOrStdout(), tables, listType); err != nil { + return err + } + return nil + }, + } + cmd.Flags().Uint32Var(&accountID, "account-id", 0, "Account ID to list") + cmd.Flags().Uint64Var(&databaseID, "database-id", 0, "Database ID to list") + cmd.Flags().BoolVar(&includeViews, "include-views", false, "Include views and non-table relations") + cmd.Flags().StringVar(&listType, "type", "tables", "List type: tables, databases, or accounts") + cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp: physical:logical, physical-logical, RFC3339, or local time (default: latest)") + return cmd +} + +func printCatalogList(w io.Writer, tables []checkpointtool.TableCatalogEntry, listType string) error { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + switch listType { + case "", "tables": + fmt.Fprintln(tw, "ACCOUNT_ID\tDATABASE\tTABLE\tTABLE_ID\tREL_KIND") + for _, table := range tables { + fmt.Fprintf(tw, "%d\t%s\t%s\t%d\t%s\n", table.AccountID, table.DatabaseName, table.TableName, table.TableID, table.RelKind) + } + case "databases", "dbs": + type dbKey struct { + accountID uint32 + name string + id uint64 + } + seen := make(map[dbKey]struct{}) + var dbs []dbKey + for _, table := range tables { + key := dbKey{accountID: table.AccountID, name: table.DatabaseName, id: table.DatabaseID} + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + dbs = append(dbs, key) + } + sort.Slice(dbs, func(i, j int) bool { + if dbs[i].accountID != dbs[j].accountID { + return dbs[i].accountID < dbs[j].accountID + } + return dbs[i].name < dbs[j].name + }) + fmt.Fprintln(tw, "ACCOUNT_ID\tDATABASE\tDATABASE_ID") + for _, db := range dbs { + fmt.Fprintf(tw, "%d\t%s\t%d\n", db.accountID, db.name, db.id) + } + case "accounts", "tenants": + seen := make(map[uint32]struct{}) + var accounts []uint32 + for _, table := range tables { + if _, ok := seen[table.AccountID]; ok { + continue + } + seen[table.AccountID] = struct{}{} + accounts = append(accounts, table.AccountID) + } + sort.Slice(accounts, func(i, j int) bool { return accounts[i] < accounts[j] }) + fmt.Fprintln(tw, "ACCOUNT_ID") + for _, accountID := range accounts { + fmt.Fprintf(tw, "%d\n", accountID) + } + default: + return fmt.Errorf("unknown --type %q; expected tables, databases, or accounts", listType) + } + return tw.Flush() +} + func openReader(ctx context.Context, dir string, storage toolfs.StorageOptions) (*checkpointtool.CheckpointReader, error) { if !storage.IsRemote() { return checkpointtool.Open(ctx, dir) @@ -186,10 +319,17 @@ func openReader(ctx context.Context, dir string, storage toolfs.StorageOptions) func dumpCommand(storage *toolfs.StorageOptions) *cobra.Command { var ( tableID uint64 + tableName string + accountID uint32 + databaseID uint64 tsStr string output string + outputDir string + jobs int metaComments bool header bool + loadScript bool + noLoad bool rowOrder string ) @@ -207,7 +347,10 @@ Examples: mo-tool ckp dump --table-id=12345 /path/to/ckp # dump to stdout mo-tool ckp dump --table-id=12345 -o users.csv . # dump to file mo-tool ckp dump --table-id=12345 --ts=1749001234567890:1 . # dump at specific TS - mo-tool ckp dump --table-id=12345 --header --meta-comments .`, + mo-tool ckp dump --table-id=12345 --load-script -o /tmp/ . + mo-tool ckp dump --database-id=9001 --table=users -o users.csv . + mo-tool ckp dump --database-id=9001 --output-dir=/tmp/test-dump --jobs=4 . + mo-tool ckp dump --database-id=9001 --load-script -o /tmp/test-dump .`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { dir := "." @@ -215,8 +358,36 @@ Examples: dir = args[0] } - if tableID == 0 { - return fmt.Errorf("--table-id is required") + accountIDSet := cmd.Flags().Changed("account-id") + tableName = strings.TrimSpace(tableName) + batchDump := tableID == 0 && tableName == "" + databaseIDSet := cmd.Flags().Changed("database-id") + if tableID == 0 && tableName == "" && !databaseIDSet && !accountIDSet { + return fmt.Errorf("--table-id, --table, or at least one of --database-id/--account-id is required") + } + if noLoad && !loadScript { + return fmt.Errorf("--no-load requires --load-script") + } + if tableID != 0 && tableName != "" { + return fmt.Errorf("--table-id cannot be combined with --table") + } + if tableID != 0 && (databaseIDSet || accountIDSet || (outputDir != "" && !loadScript)) { + return fmt.Errorf("--table-id cannot be combined with --database-id, --account-id, or --output-dir") + } + if tableName != "" && outputDir != "" && !loadScript { + return fmt.Errorf("--output-dir is only valid for --database-id/--account-id batch dumps") + } + if batchDump && output != "" && !loadScript { + return fmt.Errorf("--output cannot be used when dumping by --database-id or --account-id; use --output-dir") + } + if loadScript && output == "" { + return fmt.Errorf("--output/-o directory is required with --load-script") + } + if loadScript && outputDir == "" { + outputDir = output + } + if batchDump && outputDir == "" { + return fmt.Errorf("--output-dir is required when dumping by --database-id or --account-id") } logFile, err := setupLogFile() @@ -232,25 +403,14 @@ Examples: } defer reader.Close() - // Determine snapshot TS - snapshotTS := types.TS{} - if tsStr != "" { - snapshotTS, err = parseTS(tsStr) - if err != nil { - return fmt.Errorf("parse --ts: %w", err) - } - } else { - // Use the latest available TS - info := reader.Info() - if !info.LatestTS.IsEmpty() { - snapshotTS = info.LatestTS - } + snapshotTS, err := resolveSnapshotTS(ctx, reader, tsStr) + if err != nil { + return fmt.Errorf("resolve --ts: %w", err) } - // Open output writer var w = cmd.OutOrStdout() var outFile *os.File - if output != "" { + if output != "" && !loadScript { outFile, err = os.Create(output) if err != nil { return fmt.Errorf("create output file: %w", err) @@ -264,13 +424,96 @@ Examples: return err } + if tableName != "" { + var accountFilter *uint32 + if accountIDSet { + accountFilter = &accountID + } + var dbFilter *uint64 + if databaseIDSet { + dbFilter = &databaseID + } + table, err := resolveTableByName(ctx, reader, snapshotTS, tableName, dbFilter, accountFilter) + if err != nil { + return err + } + tableID = table.TableID + } + effectiveHeader := header || (loadScript && !noLoad) + + if batchDump { + var accountFilter *uint32 + if accountIDSet { + accountFilter = &accountID + } + var dbFilter *uint64 + if databaseIDSet { + dbFilter = &databaseID + } + tables, err := reader.ListCatalogTables(ctx, snapshotTS, checkpointtool.TableListOptions{ + AccountID: accountFilter, + DatabaseID: dbFilter, + }) + if err != nil { + return fmt.Errorf("list checkpoint catalog tables: %w", err) + } + if len(tables) == 0 { + return fmt.Errorf("no checkpoint tables match database-id-set=%v database-id=%d account-id-set=%v account-id=%d", databaseIDSet, databaseID, accountIDSet, accountID) + } + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return fmt.Errorf("create output dir: %w", err) + } + if !noLoad { + if jobs < 1 { + jobs = 1 + } + if jobs > len(tables) { + jobs = len(tables) + } + if err := dumpTablesConcurrently(ctx, reader, tables, snapshotTS, outputDir, jobs, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout()); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Dumped %d tables to %s\n", len(tables), outputDir) + } + if loadScript { + scriptPath, err := writeRestoreScript(ctx, reader, tables, snapshotTS, output, outputDir, !noLoad, effectiveHeader) + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Restore script written to %s\n", scriptPath) + } + return nil + } + + var tableEntry checkpointtool.TableCatalogEntry + if loadScript { + tableEntry, err = resolveTableByID(ctx, reader, snapshotTS, tableID) + if err != nil { + return err + } + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return fmt.Errorf("create output dir: %w", err) + } + if !noLoad { + if err := dumpOneTable(ctx, reader, tableEntry, snapshotTS, outputDir, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout(), &sync.Mutex{}); err != nil { + return err + } + } + scriptPath, err := writeRestoreScript(ctx, reader, []checkpointtool.TableCatalogEntry{tableEntry}, snapshotTS, output, outputDir, !noLoad, effectiveHeader) + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Restore script written to %s\n", scriptPath) + return nil + } + if err := reader.DumpTableCSVComposed( ctx, w, tableID, snapshotTS, checkpointtool.WithCSVMetaComments(metaComments), - checkpointtool.WithCSVHeader(header), + checkpointtool.WithCSVHeader(effectiveHeader), checkpointtool.WithCSVRowOrder(parsedRowOrder), ); err != nil { return fmt.Errorf("dump table %d: %w", tableID, err) @@ -283,20 +526,33 @@ Examples: }, } - cmd.Flags().Uint64Var(&tableID, "table-id", 0, "Table ID to dump (required)") - cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp in 'physical:logical' format (default: latest)") + cmd.Flags().Uint64Var(&tableID, "table-id", 0, "Table ID to dump") + cmd.Flags().StringVar(&tableName, "table", "", "Table name to dump; use --database-id and/or --account-id to disambiguate") + cmd.Flags().Uint32Var(&accountID, "account-id", 0, "Account ID to dump tables for; combine with --database-id to narrow the result") + cmd.Flags().Uint64Var(&databaseID, "database-id", 0, "Database ID to dump tables for") + cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp: physical:logical, physical-logical, RFC3339, or '2006-01-02 15:04:05' in local time (default: latest)") cmd.Flags().StringVarP(&output, "output", "o", "", "Output CSV file path (default: stdout)") + cmd.Flags().StringVar(&outputDir, "output-dir", "", "Output directory for database/account dumps") + cmd.Flags().IntVar(&jobs, "jobs", 1, "Concurrent table dumps for --database-id/--account-id batch dumps") cmd.Flags().BoolVar(&metaComments, "meta-comments", false, "Prepend DDL and row-count comment lines (disabled by default so output can be loaded directly)") cmd.Flags().BoolVar(&header, "header", false, "Include a CSV header row with column names") + cmd.Flags().BoolVar(&loadScript, "load-script", false, "Generate restore.sql with CREATE DATABASE, CREATE TABLE, and LOAD DATA statements; --output/-o is treated as a directory") + cmd.Flags().BoolVar(&noLoad, "no-load", false, "With --load-script, generate only DDL and skip CSV dump and LOAD DATA statements") cmd.Flags().StringVar(&rowOrder, "row-order", string(checkpointtool.CSVRowOrderStorage), "CSV row order: storage (streaming, large-table friendly) or lexical (sort by visible CSV values in memory)") return cmd } -// parseTS parses a timestamp string in "physical:logical" format. func parseTS(s string) (types.TS, error) { - parts := strings.SplitN(s, ":", 2) - if len(parts) == 2 { + s = strings.TrimSpace(s) + for _, sep := range []string{":", "-"} { + parts := strings.SplitN(s, sep, 2) + if len(parts) != 2 { + continue + } + if !allDigits(parts[0]) || !allDigits(parts[1]) { + continue + } physical, err := strconv.ParseInt(parts[0], 10, 64) if err != nil { return types.TS{}, fmt.Errorf("invalid physical in timestamp: %w", err) @@ -307,7 +563,340 @@ func parseTS(s string) (types.TS, error) { } return types.BuildTS(physical, uint32(logical)), nil } - return types.TS{}, fmt.Errorf("timestamp must be in 'physical:logical' format, got %q", s) + + for _, layout := range []string{ + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999", + "2006-01-02 15:04:05", + "2006-01-02 15:04", + "2006-01-02", + } { + t, err := time.ParseInLocation(layout, s, time.Local) + if err == nil { + return types.BuildTS(t.UnixNano(), 0), nil + } + } + return types.TS{}, fmt.Errorf("timestamp must be physical:logical, physical-logical, RFC3339, or local time, got %q", s) +} + +func resolveSnapshotTS(ctx context.Context, reader *checkpointtool.CheckpointReader, tsStr string) (types.TS, error) { + info := reader.Info() + if strings.TrimSpace(tsStr) == "" { + if info.LatestTS.IsEmpty() { + return types.TS{}, fmt.Errorf("no checkpoint timestamp is available") + } + if err := reader.ValidateSnapshot(ctx, info.LatestTS); err != nil { + return types.TS{}, err + } + return info.LatestTS, nil + } + + ts, err := parseTS(tsStr) + if err != nil { + return types.TS{}, err + } + if ts.IsEmpty() { + return types.TS{}, fmt.Errorf("timestamp must not be empty") + } + if !info.EarliestTS.IsEmpty() && ts.LT(&info.EarliestTS) { + return types.TS{}, fmt.Errorf("timestamp %s is earlier than earliest checkpoint %s", ts.ToString(), info.EarliestTS.ToString()) + } + if !info.LatestTS.IsEmpty() && info.LatestTS.LT(&ts) { + return types.TS{}, fmt.Errorf("timestamp %s is newer than latest checkpoint %s", ts.ToString(), info.LatestTS.ToString()) + } + if err := reader.ValidateSnapshot(ctx, ts); err != nil { + return types.TS{}, err + } + return ts, nil +} + +func allDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func resolveTableByName( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + snapshotTS types.TS, + tableName string, + databaseID *uint64, + accountID *uint32, +) (checkpointtool.TableCatalogEntry, error) { + tables, err := reader.ListCatalogTables(ctx, snapshotTS, checkpointtool.TableListOptions{ + AccountID: accountID, + DatabaseID: databaseID, + }) + if err != nil { + return checkpointtool.TableCatalogEntry{}, fmt.Errorf("list checkpoint catalog tables: %w", err) + } + var matches []checkpointtool.TableCatalogEntry + for _, table := range tables { + if table.TableName == tableName { + matches = append(matches, table) + } + } + if len(matches) == 0 { + return checkpointtool.TableCatalogEntry{}, fmt.Errorf("table %q not found in checkpoint catalog", tableName) + } + if len(matches) > 1 { + var names []string + for _, table := range matches { + names = append(names, fmt.Sprintf("account=%d database=%s table-id=%d", table.AccountID, table.DatabaseName, table.TableID)) + } + return checkpointtool.TableCatalogEntry{}, fmt.Errorf("table %q is ambiguous; use --database-id/--account-id/--table-id (%s)", tableName, strings.Join(names, "; ")) + } + return matches[0], nil +} + +func resolveTableByID( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + snapshotTS types.TS, + tableID uint64, +) (checkpointtool.TableCatalogEntry, error) { + tables, err := reader.ListCatalogTables(ctx, snapshotTS, checkpointtool.TableListOptions{ + IncludeViews: true, + }) + if err != nil { + return checkpointtool.TableCatalogEntry{}, fmt.Errorf("list checkpoint catalog tables: %w", err) + } + for _, table := range tables { + if table.TableID == tableID { + return table, nil + } + } + return checkpointtool.TableCatalogEntry{}, fmt.Errorf("table %d not found in checkpoint catalog", tableID) +} + +func writeRestoreScript( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + tables []checkpointtool.TableCatalogEntry, + snapshotTS types.TS, + scriptDir string, + csvRoot string, + includeLoad bool, + csvHasHeader bool, +) (string, error) { + if err := os.MkdirAll(scriptDir, 0o755); err != nil { + return "", fmt.Errorf("create script dir: %w", err) + } + scriptPath := filepath.Join(scriptDir, "restore.sql") + f, err := os.Create(scriptPath) + if err != nil { + return "", fmt.Errorf("create restore script: %w", err) + } + defer f.Close() + + currentDB := "" + for _, table := range tables { + if table.DatabaseName == "" { + return "", fmt.Errorf("table %d has empty database name in checkpoint catalog", table.TableID) + } + if table.DatabaseName != currentDB { + if currentDB != "" { + if _, err := fmt.Fprintln(f); err != nil { + return "", err + } + } + if _, err := fmt.Fprintf(f, "CREATE DATABASE IF NOT EXISTS %s;\n", quoteSQLIdent(table.DatabaseName)); err != nil { + return "", err + } + if _, err := fmt.Fprintf(f, "USE %s;\n\n", quoteSQLIdent(table.DatabaseName)); err != nil { + return "", err + } + currentDB = table.DatabaseName + } + + ddl, err := reader.ShowCreateTable(ctx, table.TableID, snapshotTS) + if err != nil { + return "", fmt.Errorf("show create table %d: %w", table.TableID, err) + } + if _, err := fmt.Fprintln(f, strings.TrimRight(ddl, " ;\n\t")); err != nil { + return "", err + } + if _, err := fmt.Fprintln(f, ";"); err != nil { + return "", err + } + if includeLoad { + if _, err := fmt.Fprintf(f, "\nLOAD DATA INFILE %s\n", quoteSQLString(tableCSVPath(csvRoot, table))); err != nil { + return "", err + } + if _, err := fmt.Fprintf(f, "INTO TABLE %s\n", quoteSQLIdent(table.TableName)); err != nil { + return "", err + } + if _, err := fmt.Fprintln(f, "FIELDS TERMINATED BY ','"); err != nil { + return "", err + } + if _, err := fmt.Fprintln(f, "ENCLOSED BY '\"'"); err != nil { + return "", err + } + if _, err := fmt.Fprintln(f, "LINES TERMINATED BY '\\n'"); err != nil { + return "", err + } + if csvHasHeader { + if _, err := fmt.Fprintln(f, "IGNORE 1 LINES"); err != nil { + return "", err + } + } + if _, err := fmt.Fprintln(f, ";"); err != nil { + return "", err + } + } + if _, err := fmt.Fprintln(f); err != nil { + return "", err + } + } + if err := f.Close(); err != nil { + return "", fmt.Errorf("close restore script: %w", err) + } + return scriptPath, nil +} + +func quoteSQLIdent(s string) string { + return "`" + strings.ReplaceAll(s, "`", "``") + "`" +} + +func quoteSQLString(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `'`, `''`) + return "'" + s + "'" +} + +func dumpTablesConcurrently( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + tables []checkpointtool.TableCatalogEntry, + snapshotTS types.TS, + outputDir string, + jobs int, + rowOrder checkpointtool.CSVRowOrder, + metaComments bool, + header bool, + out io.Writer, +) error { + tableCh := make(chan checkpointtool.TableCatalogEntry) + errCh := make(chan error, 1) + var outMu sync.Mutex + var wg sync.WaitGroup + + worker := func() { + defer wg.Done() + for table := range tableCh { + if err := dumpOneTable(ctx, reader, table, snapshotTS, outputDir, rowOrder, metaComments, header, out, &outMu); err != nil { + select { + case errCh <- err: + default: + } + return + } + } + } + for i := 0; i < jobs; i++ { + wg.Add(1) + go worker() + } + for _, table := range tables { + select { + case err := <-errCh: + close(tableCh) + wg.Wait() + return err + case tableCh <- table: + } + } + close(tableCh) + wg.Wait() + select { + case err := <-errCh: + return err + default: + return nil + } +} + +func tableCSVPath(outputDir string, table checkpointtool.TableCatalogEntry) string { + return filepath.Join( + outputDir, + fmt.Sprintf("account_%d", table.AccountID), + fmt.Sprintf("db_%d", table.DatabaseID), + fmt.Sprintf("%s_%d.csv", safePathPart(table.TableName), table.TableID), + ) +} + +func dumpOneTable( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + table checkpointtool.TableCatalogEntry, + snapshotTS types.TS, + outputDir string, + rowOrder checkpointtool.CSVRowOrder, + metaComments bool, + header bool, + out io.Writer, + outMu *sync.Mutex, +) error { + filePath := tableCSVPath(outputDir, table) + tableDir := filepath.Dir(filePath) + if err := os.MkdirAll(tableDir, 0o755); err != nil { + return fmt.Errorf("create table output dir: %w", err) + } + outFile, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("create output file for table %d: %w", table.TableID, err) + } + err = reader.DumpTableCSVComposed( + ctx, + outFile, + table.TableID, + snapshotTS, + checkpointtool.WithCSVMetaComments(metaComments), + checkpointtool.WithCSVHeader(header), + checkpointtool.WithCSVRowOrder(rowOrder), + ) + closeErr := outFile.Close() + if err != nil { + return fmt.Errorf("dump table %d (%s.%s): %w", table.TableID, table.DatabaseName, table.TableName, err) + } + if closeErr != nil { + return fmt.Errorf("close output file for table %d: %w", table.TableID, closeErr) + } + outMu.Lock() + fmt.Fprintf(out, "Table %d %s.%s dumped to %s\n", table.TableID, table.DatabaseName, table.TableName, filePath) + outMu.Unlock() + return nil +} + +func safePathPart(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "_" + } + var b strings.Builder + for _, r := range s { + switch { + case r >= 'a' && r <= 'z': + b.WriteRune(r) + case r >= 'A' && r <= 'Z': + b.WriteRune(r) + case r >= '0' && r <= '9': + b.WriteRune(r) + case r == '_' || r == '-' || r == '.': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return b.String() } // showCreateTableCommand implements the "ckp show-create-table" subcommand @@ -359,17 +948,9 @@ Examples: } defer reader.Close() - snapshotTS := types.TS{} - if tsStr != "" { - snapshotTS, err = parseTS(tsStr) - if err != nil { - return fmt.Errorf("parse --ts: %w", err) - } - } else { - info := reader.Info() - if !info.LatestTS.IsEmpty() { - snapshotTS = info.LatestTS - } + snapshotTS, err := resolveSnapshotTS(ctx, reader, tsStr) + if err != nil { + return fmt.Errorf("resolve --ts: %w", err) } ddl, err := reader.ShowCreateTable(ctx, tableID, snapshotTS) @@ -383,7 +964,7 @@ Examples: } cmd.Flags().Uint64Var(&tableID, "table-id", 0, "Table ID to show CREATE TABLE for (required)") - cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp in 'physical:logical' format (default: latest)") + cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp: physical:logical, physical-logical, RFC3339, or local time (default: latest)") return cmd } diff --git a/docs/design/checkpoint_tool_feature_analysis.md b/docs/design/checkpoint_tool_feature_analysis.md new file mode 100644 index 0000000000000..c4738bad70b67 --- /dev/null +++ b/docs/design/checkpoint_tool_feature_analysis.md @@ -0,0 +1,714 @@ +# Checkpoint CSV Dump 工具使用文档 + +## 概述 + +`mo-tool ckp` 是从 MatrixOne checkpoint 数据离线导出 CSV 的命令行工具。支持以表、database、租户（account）为单位导出，mo-data 可以是本地目录或远程 S3/MinIO，可指定时间戳导出历史快照数据。 + +--- + +## 一、构建 + +```bash +make mo-tool +``` + +构建产物为 `./mo-tool`。 + +--- + +## 二、以表为单位 dump + +### 2.1 命令 + +```bash +./mo-tool ckp dump --table-id= [选项] +``` + +### 2.2 参数说明 + +| 参数 | 必需 | 说明 | +|------|------|------| +| `--table-id` | 是 | MatrixOne 内部 table_id，可从 `mo_tables` 系统表或交互式 viewer 中获取 | +| `--ts` | 否 | 快照时间戳。不指定则使用最新的 checkpoint 时间戳 | +| `-o` / `--output` | 否 | 输出路径。纯 CSV 模式指定文件路径，`--load-script` 模式指定目录（工具自动生成文件名）。不指定则输出到 stdout | +| `--header` | 否 | 在 CSV 第一行输出列名头行 | +| `--meta-comments` | 否 | 在 CSV 文件头部输出 DDL 和行数统计注释（以 `--` 开头） | +| `--row-order` | 否 | 行排序方式：`storage`（默认，流式按存储顺序）或 `lexical`（按可见列字典序排序） | +| `--load-script` | 否 | 切换为 LOAD 脚本输出模式：生成包含 CREATE DATABASE + CREATE TABLE + LOAD DATA 的 SQL 文件 | +| `--no-load` | 否 | 配合 `--load-script` 使用，跳过 LOAD DATA 语句，只输出 DDL | + +### 2.3 示例 + +**导出到文件**： + +```bash +./mo-tool ckp dump --table-id=272535 --header -o /tmp/employees.csv /path/to/mo-data +``` + +**导出到 stdout**： + +```bash +./mo-tool ckp dump --table-id=272535 --header /path/to/mo-data +``` + +**指定时间戳导出**： + +```bash +./mo-tool ckp dump --table-id=272535 --ts=1781084792256166694:1 --header -o /tmp/employees.csv /path/to/mo-data +``` + +**生成 LOAD 脚本（含建库建表 + LOAD DATA）**： + +```bash +./mo-tool ckp dump --table-id=272535 --load-script -o /tmp/ /path/to/mo-data +``` + +**带 DDL 元数据注释导出**： + +```bash +./mo-tool ckp dump --table-id=272535 --header --meta-comments -o /tmp/employees.csv /path/to/mo-data +``` + +此时输出文件头部会包含： + +```text +-- CREATE TABLE employees (id INT PRIMARY KEY, name VARCHAR(100), ...) +-- Database: test_ckp +-- Table: employees +-- Visible rows: 103 (deleted: 0, physical: 103) +id,name,age,salary,department,hire_date,is_active +... +``` + +### 2.4 输出位置 + +| 模式 | `-o` 含义 | 输出文件 | +|------|----------|---------| +| 纯 CSV（默认） | 文件路径 | `-o /tmp/employees.csv` → `/tmp/employees.csv` | +| `--load-script` | 目录路径 | `-o /tmp/` → `/tmp/restore.sql` | + +未指定 `-o` 时输出到 stdout。 + +--- + +## 三、以 database 为单位 dump + +### 3.1 命令 + +```bash +./mo-tool ckp dump --database-id= --output-dir= [选项] +``` + +### 3.2 参数说明 + +| 参数 | 必需 | 说明 | +|------|------|------| +| `--database-id` | 是 | Database ID（uint64），可从 `mo_database` 系统表或 `ckp list --type=databases` 获取 | +| `--output-dir` | 是 | 输出根目录。该目录会自动创建 | +| `--ts` | 否 | 快照时间戳，不指定则用最新 | +| `--header` | 否 | 在每个 CSV 文件第一行输出列名头行 | +| `--meta-comments` | 否 | 在每个 CSV 文件头部输出 DDL 和行数统计注释 | +| `--row-order` | 否 | 行排序方式 | +| `--load-script` | 否 | 切换为 LOAD 脚本输出模式：生成单个 SQL 文件，包含 CREATE DATABASE + 所有表的 CREATE TABLE + LOAD DATA | +| `--no-load` | 否 | 配合 `--load-script` 使用，跳过 LOAD DATA 语句，只输出 DDL | + +### 3.3 示例 + +**导出指定 database ID 下的所有表**： + +```bash +./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header /path/to/mo-data +``` + +**生成该 database 的 LOAD 恢复脚本**： + +```bash +./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data +``` + +### 3.4 输出目录结构 + +以 database 为单位 dump 时，CSV 文件按以下目录结构组织： + +```text +/ +├── restore.sql # 如果指定了 --load-script，生成恢复脚本 +└── account_/ + └── db_/ + ├── _.csv + ├── _.csv + └── ... +``` + +具体示例： + +**dump CSV + LOAD 脚本**： + +```bash +./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header \ + --load-script -o ./dump_out/ /path/to/mo-data +``` + +```text +dump_out/ +├── restore.sql +└── account_7/ + └── db_9001/ + ├── employees_272535.csv + ├── departments_272536.csv + └── alter_compat_272538.csv +``` + +其中 `restore.sql` 的内容： + +```sql +CREATE DATABASE IF NOT EXISTS test_ckp; +USE test_ckp; + +CREATE TABLE employees ( + id INT PRIMARY KEY, + name VARCHAR(100), + ... +); + +LOAD DATA INFILE 'dump_out/account_7/db_9001/employees_272535.csv' +INTO TABLE employees +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; + +CREATE TABLE departments (...); + +LOAD DATA INFILE 'dump_out/account_7/db_9001/departments_272536.csv' +INTO TABLE departments +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; +``` + +**只 dump CSV（不含 LOAD 脚本）**： + +```bash +./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header /path/to/mo-data +``` + +```text +dump_out/ +└── account_7/ + └── db_9001/ + ├── employees_272535.csv + ├── departments_272536.csv + └── alter_compat_272538.csv +``` + +目录命名规则： + +- **account 目录**：`account_`，其中 `account_id` 为十进制数字。系统租户的 account_id 为 `0` +- **database 目录**：`db_`，其中 `database_id` 为十进制数字 +- **CSV 文件名**：`_.csv`，其中 `table_name` 为表名，`table_id` 为内部数字 ID。表名中的特殊字符会被替换为下划线 + +> **注意**：`relkind='v'` 的视图不会被导出。 + +### 3.5 命令输出 + +执行过程中，每成功导出一个表会打印一行信息： + +```text +Table 272535 test_ckp.employees dumped to dump_out/account_7/db_9001/employees_272535.csv +Table 272536 test_ckp.departments dumped to dump_out/account_7/db_9001/departments_272536.csv +Dumped 2 tables to dump_out +``` + +--- + +## 四、以租户（account）为单位 dump + +### 4.1 命令 + +```bash +./mo-tool ckp dump --account-id= --output-dir= [选项] +``` + +### 4.2 参数说明 + +| 参数 | 必需 | 说明 | +|------|------|------| +| `--account-id` | 是 | 租户 ID（uint32） | +| `--output-dir` | 是 | 输出根目录 | +| `--ts` | 否 | 快照时间戳 | +| `--header` | 否 | CSV 列名头行 | +| `--meta-comments` | 否 | DDL 和行数统计注释 | +| `--row-order` | 否 | 行排序方式 | +| `--load-script` | 否 | 切换为 LOAD 脚本输出模式：生成单个 SQL 文件，包含所有 database 的 CREATE DATABASE + CREATE TABLE + LOAD DATA | +| `--no-load` | 否 | 配合 `--load-script` 使用，跳过 LOAD DATA 语句，只输出 DDL | + +### 4.3 示例 + +**导出某个租户的所有表**： + +```bash +./mo-tool ckp dump --account-id=7 --output-dir=./dump_out --header /path/to/mo-data +``` + +**生成该租户的 LOAD 恢复脚本**： + +```bash +./mo-tool ckp dump --account-id=7 --load-script -o /tmp/ /path/to/mo-data +``` + +### 4.4 输出目录结构 + +以租户为单位 dump 时，目录结构与 database 级别一致： + +```text +/ +├── restore.sql # 如果指定了 --load-script，生成恢复脚本 +└── account_/ + ├── db_/ + │ ├── _.csv + │ └── ... + ├── db_/ + │ └── ... + └── ... +``` + +具体示例： + +```bash +./mo-tool ckp dump --account-id=7 --output-dir=./dump_out --header \ + --load-script -o ./dump_out/ /path/to/mo-data +``` + +```text +dump_out/ +├── restore.sql +└── account_7/ + ├── db_9001/ + │ ├── employees_272535.csv + │ └── alter_compat_272538.csv + └── db_9002/ + ├── orders_272540.csv + ├── lineitem_272541.csv + └── ... +``` + +`restore.sql` 包含该租户下所有 database 的建表和数据导入语句。 + +--- + +## 五、本地 mo-data + +直接传入本地 mo-data 目录路径即可，适用于 checkpoint 数据存储在本地磁盘的场景。 + +```bash +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv /path/to/mo-data +``` + +内部使用 `fileservice.NewLocalFS` 创建本地文件服务，直接读取 checkpoint 元数据和数据对象文件。 + +--- + +## 六、远程 S3/MinIO mo-data + +当 checkpoint 数据存储在 S3 或 MinIO 对象存储上时，支持两种远程访问方式。 + +### 6.1 方式一：通过 MO 配置文件 + +使用 MatrixOne 的 TOML 配置文件（如 `tn.toml`），其中的 `[fileservice]` 段包含对象存储的连接信息： + +```bash +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED +``` + +- `--fs-config`：MO 配置文件路径 +- `--fs-name`：使用的 fileservice 名称（默认为 `SHARED`），对应配置中 `[fileservice]` 段的 `name` 字段 + +### 6.2 方式二：直接指定 S3 参数 + +不依赖 MO 配置文件，直接传入对象存储的连接参数： + +```bash +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 +``` + +S3 参数格式（逗号分隔的 key=value 对）： + +| 参数 | 必需 | 说明 | +|------|------|------| +| `bucket` | 是 | S3 bucket 名称 | +| `endpoint` | 是 | S3/MinIO API 地址 | +| `key-prefix` | 是 | mo-data 在 bucket 中的路径前缀 | +| `key-id` | 是 | 访问密钥 ID | +| `key-secret` | 是 | 访问密钥 Secret | +| `region` | 否 | S3 region | + +`--backend` 可选值： +- `S3`：AWS S3 或兼容 S3 协议的服务（默认） +- `MINIO`：MinIO 对象存储 + +### 6.3 远程读取机制 + +远程读取使用 `lazyCacheFS` 实现按需缓存： + +1. 首次访问远程文件时，下载到本地临时目录 +2. 后续读取同一文件直接命中本地缓存 +3. 绕过 MO 内部的 checksum 格式验证，直接按 OS file range 读取 +4. 工具退出时自动清理临时缓存目录 + +checkpoint 元数据列表（`List` 操作）直接对远程服务执行，不下载全部文件。 + +### 6.4 远程 dump 示例 + +```bash +# 远程单表 dump +./mo-tool ckp dump --table-id=272535 --header -o /tmp/remote_table.csv \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED + +# 远程 database dump +./mo-tool ckp dump --database-id=9001 --output-dir=./remote_dump --header \ + --fs-config etc/launch-minio-local/tn.toml + +# 远程 ckp info +./mo-tool ckp info --fs-config etc/launch-minio-local/tn.toml --fs-name SHARED +``` + +--- + +## 七、CSV 格式说明 + +### 7.1 格式规范 + +导出的 CSV 文件兼容 MySQL/MariaDB/MatrixOne 的 `LOAD DATA` 语句，具体格式： + +| 特性 | 约定 | +|------|------| +| 字段分隔符 | `,`（逗号） | +| 字段引用符 | `"`（双引号） | +| 转义符 | `\`（反斜杠） | +| 行分隔符 | `\n`（换行） | +| NULL 表示 | `\N`（反斜杠+大写 N） | +| 字符串类型 | 总是用双引号括起（VARCHAR, CHAR, TEXT, BLOB, JSON, GEOMETRY, DATALINK 等） | +| 数值类型 | 不使用引号（INT, BIGINT, FLOAT, DOUBLE, DECIMAL, BOOL 等） | +| 日期时间类型 | 不使用引号（DATE, TIME, DATETIME, TIMESTAMP） | + +### 7.2 特殊值处理 + +| 场景 | CSV 输出 | +|------|----------| +| NULL 值 | `\N` | +| 包含双引号的字符串（如 `a"b`） | `"a""b"` | +| 包含反斜杠的字符串（如 `a\b`） | `"a\\b"` | +| 包含逗号的字符串（如 `a,b`） | `"a,b"` | +| 包含换行的字符串 | `"a\nb"` | + +### 7.3 CSV 输出示例 + +```csv +id,name,age,salary,department,hire_date,is_active +1,"Alice",30,75000.00,"Engineering",2020-01-15,true +2,"Bob",\N,\N,\N,\N,\N +3,"Charlie",\N,65000.00,\N,2021-03-10,false +4,"David""The Boss""",42,120000.00,"Management",2019-06-01,true +``` + +--- + +## 八、LOAD DATA 回灌 + +### 8.1 推荐方式：使用 `--load-script` + +`--load-script` 一步生成完整的 SQL 恢复脚本（CREATE DATABASE + CREATE TABLE + LOAD DATA），在目标 MO 实例中直接执行即可： + +```bash +# 生成恢复脚本 +./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data + +# 在目标 MO 实例中执行 +mysql -h 127.0.0.1 -P 6001 -u root -p111 < /tmp/restore.sql +``` + +详见第十一章。 + +### 8.2 手动回灌（不使用 `--load-script`） + +如果不使用 `--load-script`，手动回灌流程如下： + +**步骤 1：建表** + +```sql +CREATE TABLE test_ckp.employees_load ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); +``` + +**步骤 2：LOAD DATA** + +```sql +LOAD DATA INFILE '/tmp/employees.csv' +INTO TABLE test_ckp.employees_load +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; -- 如果 dump 时使用了 --header +``` + +LOAD DATA 参数对应关系： + +| CSV 格式特性 | LOAD DATA 参数 | +|-------------|---------------| +| 逗号分隔字段 | `FIELDS TERMINATED BY ','` | +| 双引号括起字符串 | `ENCLOSED BY '"'` | +| 反斜杠转义 | 默认（MySQL/MO 默认转义符为 `\`） | +| 换行分隔 | `LINES TERMINATED BY '\n'` | +| NULL 为 `\N` | 默认（MySQL/MO 默认 NULL 表示即 `\N`） | +| 有 header 行 | `IGNORE 1 LINES` | +| 无 header 行 | 不需要 `IGNORE 1 LINES` | + +### 8.3 数据校验 + +```sql +-- 对比原始行数和导入行数 +SELECT (SELECT COUNT(*) FROM test_ckp.employees) AS original_rows, + (SELECT COUNT(*) FROM test_ckp.employees_load) AS loaded_rows; + +-- 对比数据差异 +SELECT COUNT(*) AS diff_rows +FROM ( + SELECT * FROM test_ckp.employees + EXCEPT + SELECT * FROM test_ckp.employees_load +) AS diff; +``` + +--- + +## 九、CSV 行排序选项 + +### 9.1 storage 顺序（默认） + +```bash +./mo-tool ckp dump --table-id=272535 --row-order=storage --header -o /tmp/table.csv /path/to/mo-data +``` + +- 按物理存储顺序输出，流式写入 +- 内存占用小，适合大表 +- 行顺序与存储布局一致，不可预测 + +### 9.2 lexical 顺序 + +```bash +./mo-tool ckp dump --table-id=272535 --row-order=lexical --header -o /tmp/table.csv /path/to/mo-data +``` + +- 按所有可见列的字典序排序后输出 +- 需要将全部行加载到内存中排序 +- 适合需要确定性输出顺序的场景 +- 不适合超大表 + +--- + +## 十、交互式浏览器 + +除了命令行 dump，`mo-tool ckp` 还提供了交互式 TUI 浏览器，用于探索 checkpoint 内容： + +```bash +./mo-tool ckp view /path/to/mo-data +``` + +交互式浏览器支持以下操作： + +- **浏览 checkpoint 列表**：查看所有 GCKP/ICKP 条目及其时间范围 +- **浏览表列表**：选择一个 checkpoint 条目后，查看其中包含的所有表（可按 account 筛选） +- **浏览对象详情**：选择一个表后，查看其 data/tombstone 对象列表 +- **浏览逻辑表视图**：查看 tombstone 过滤后的实际数据行 +- **打开对象文件**：深入到单个数据对象中查看原始内容 + +远程 checkpoint 也可使用交互式浏览器： + +```bash +./mo-tool ckp view --fs-config etc/launch-minio-local/tn.toml --fs-name SHARED +``` + +--- + +## 十一、checkpoint 信息查看 + +### 11.1 查看 checkpoint 摘要 + +```bash +./mo-tool ckp info /path/to/mo-data +``` + +输出 checkpoint 的条目类型统计和时间范围。 + +### 11.2 生成 LOAD 脚本（`--load-script`） + +dump 命令支持 `--load-script` 选项，生成一个可直接在目标 MatrixOne 实例中执行的 SQL 脚本，一行命令即可完成数据恢复。 + +脚本包含以下内容（按顺序）： + +1. **CREATE DATABASE** — 重建 database +2. **USE database** — 切换到目标 database +3. **CREATE TABLE**（每个表一条） — 重建表结构，DDL 从 checkpoint 的 `mo_tables.rel_createsql` 解析 +4. **LOAD DATA**（每个 CSV 文件一条，可选）— 导入数据 + +#### 单表生成 LOAD 脚本 + +```bash +./mo-tool ckp dump --table-id=272535 --load-script -o /tmp/ /path/to/mo-data +``` + +生成的 `restore.sql` 示例： + +```sql +CREATE DATABASE IF NOT EXISTS test_ckp; +USE test_ckp; + +CREATE TABLE employees ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); + +LOAD DATA INFILE '/tmp/employees_272535.csv' +INTO TABLE employees +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; +``` + +#### database 级别生成 LOAD 脚本 + +```bash +./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data +``` + +生成的脚本包含该 database 下所有表的 DDL 和 LOAD DATA 语句。CSV 文件输出到脚本同级目录。 + +#### 租户级别生成 LOAD 脚本 + +```bash +./mo-tool ckp dump --account-id=7 --load-script -o /tmp/ /path/to/mo-data +``` + +生成的脚本包含该租户下所有 database 和表的 DDL 及 LOAD DATA 语句。 + +#### LOAD DATA 可选 + +```bash +# 只生成 DDL，不含 LOAD DATA（仅建库建表） +./mo-tool ckp dump --database-id=9001 --load-script --no-load -o /tmp/ /path/to/mo-data +``` + +`--no-load` 跳过 LOAD DATA 语句，只输出 CREATE DATABASE + CREATE TABLE。 + +--- + +## 十二、常用命令速查 + +```bash +# 构建 +make mo-tool + +# 查看 checkpoint 摘要 +./mo-tool ckp info /path/to/mo-data + +# 单表 dump 到文件 +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv /path/to/mo-data + +# 单表 dump 到 stdout +./mo-tool ckp dump --table-id=272535 --header /path/to/mo-data + +# 指定时间戳单表 dump +./mo-tool ckp dump --table-id=272535 --ts=1781084792256166694:1 --header -o /tmp/table.csv /path/to/mo-data + +# database 级别批量 dump +./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header /path/to/mo-data + +# 租户级别批量 dump +./mo-tool ckp dump --account-id=7 --output-dir=./dump_out --header /path/to/mo-data + +# 生成 LOAD 脚本（单表/database/租户） +./mo-tool ckp dump --table-id=272535 --load-script -o /tmp/ /path/to/mo-data +./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data +./mo-tool ckp dump --account-id=7 --load-script -o /tmp/ /path/to/mo-data +./mo-tool ckp dump --database-id=9001 --load-script --no-load -o /tmp/ /path/to/mo-data + +# dump 系统表 +./mo-tool ckp dump --table-id=2 -o /tmp/mo_tables.csv /path/to/mo-data # mo_tables +./mo-tool ckp dump --table-id=3 -o /tmp/mo_columns.csv /path/to/mo-data # mo_columns + +# 远程 S3/MinIO dump（方式一：配置文件） +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ + --fs-config etc/launch-minio-local/tn.toml --fs-name SHARED + +# 远程 S3/MinIO dump（方式二：直接指定参数） +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 + +# 交互式浏览器 +./mo-tool ckp view /path/to/mo-data +``` + +--- + +## 十三、恢复流程模板 + +### 推荐方式：`--load-script` 一键恢复 + +```bash +# 1. 从 checkpoint 生成恢复脚本 +./mo-tool ckp dump --database-id= --load-script --header -o ./ /path/to/mo-data + +# 2. 在目标 MO 实例执行 +mysql -h -P -u -p < restore.sql +``` + +生成的 `restore.sql` 结构： + +```sql +CREATE DATABASE IF NOT EXISTS ; +USE ; + +CREATE TABLE (...); +LOAD DATA INFILE '' INTO TABLE ...; + +CREATE TABLE (...); +LOAD DATA INFILE '' INTO TABLE ...; +``` + +### 手动方式 + +```sql +-- 建表 +CREATE TABLE . ( + ... +); + +-- 导入 CSV（dump 时使用了 --header） +LOAD DATA INFILE '' +INTO TABLE .
+FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; +``` diff --git a/docs/design/checkpoint_tool_fragility.md b/docs/design/checkpoint_tool_fragility.md new file mode 100644 index 0000000000000..4d764bf8b0d8e --- /dev/null +++ b/docs/design/checkpoint_tool_fragility.md @@ -0,0 +1,263 @@ +# Checkpoint Tool 脆弱性分析：上游依赖与变更风险 + +本文档梳理 `mo-tool ckp` 离线 checkpoint 工具链中对 MatrixOne 内核私有 API / 数据格式的各项依赖，分析它们在不相关代码变更下被意外打破的风险。 + +--- + +## 1. 系统表 Schema 的硬编码副本 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:165-244` (`builtinColumnsForLayout`) + +**问题**：`mo_tables`（20 列）、`mo_columns`（27 列）、`mo_database`（10 列）三张系统表的完整列名、类型、Position 被逐列手写了一份，作为 checkpoint 数据中 catalog 元数据不可用时的 fallback。 + +```go +case catalog.MO_TABLES_ID: + cols := []builtinColumnDef{ + {Name: "rel_id", SQLType: "BIGINT", Position: 0}, + {Name: "relname", SQLType: "VARCHAR(5000)", Position: 1}, + // ... 共 20+ 列 + } +``` + +**触发条件**： +- `catalog` 包中 `MoTablesSchema` / `MoColumnsSchema` 增删或重命名任一列（例如新增 `rel_foo`、删除 `extra_info`） +- 该变更不会导致编译错误——因为 `builtinColumnsForLayout` 是手写的独立结构体，不引用 `catalog` 包的 schema 变量 + +**后果**：checkpoint 数据中如果 header 是泛型名（`col_0`, `col_1`），按位置取值时会拿到错误的列值；`builtinTableSchemaForLayout` 返回的 Column 列表与实际物理数据不对齐。 + +--- + +## 2. Catalog Layout 检测依赖列数 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:357-370` (`inferCatalogLayout`) + +**问题**：纯靠**列数**判断一份 checkpoint 是当前 layout 还是 3.0-dev 旧 layout： + +```go +switch dataWidth { +case len(schema): // 列数完全匹配 + return layout, 0 +case len(schema) + 1: // 多一列也接受，带 offset=1 + return layout, 1 +} +return currentCatalogLayout, 0 // 都不匹配 → 假设当前版本 +``` + +已知的两个 layout 差异： +| | `mo_tables` 列数 | `mo_columns` 列数 | +|---|---|---| +| current | 20 | 27 | +| 3.0-dev | 19（少 `rel_logical_id`） | 25（少 `attr_has_generated`, `attr_generated`） | + +**触发条件**： +- 未来的 layout 变体恰好与已知 layout 的列数相同（比如新版本加一列又删一列，列数不变但顺序变了） +- 列数对不上时静默 fallback 到 `currentCatalogLayout`，不报错 + +**后果**：`fallbackCatalogColIndex` 用 layout 的固定列顺序定位，列映射全错。对 `mo_tables` 的影响是取不到正确的 `rel_createsql`、`relname`；对 `mo_columns` 的影响是 `att_seqnum` / `att_is_hidden` 位置错误，导致 CSV 列全部错位或隐藏列未能过滤。 + +--- + +## 3. `logicalViewMetaCols = 3` 硬编码 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:34` + +**问题**：`LogicalTableView` 固定有 3 个 meta 列（`object`, `block`, `row`），所有数据列的偏移计算都依赖这个常量： + +```go +const logicalViewMetaCols = 3 + +// 在 table_dump.go 中大量使用： +dataRow := fullRow[logicalViewMetaCols:] +dataWidth := len(view.Headers) - logicalViewMetaCols +dataIdx := logicalViewMetaCols + pos +``` + +`LogicalTableView` 的 headers 在 `logical_table.go:43` 构造： +```go +view := &LogicalTableView{ + Headers: []string{"object", "block", "row"}, + // ... +} +``` + +**触发条件**：`BuildLogicalTableView` 的 header 构造逻辑变更（增减 meta 列，如新增 `segment` 列） + +**后果**：所有 `table_dump.go` 中的列偏移计算全部偏移，`buildSchemaFromMoTablesRow` 和 `buildColumnsFromMoColumnsRows` 拿到错误的数据列值。 + +--- + +## 4. 内核私有 API 直接依赖 + +### 4.1 `checkpoint.ReadEntriesFromMeta` + +**位置**：`pkg/vm/engine/tae/db/checkpoint/meta.go:45` + +工具调用方式： +```go +checkpoint.ReadEntriesFromMeta( + ctx, "", ioutil.GetCheckpointDir(), name, 0, nil, r.mp, r.fs, +) +``` + +**风险**： +- 7 参数函数，`sid=""`, `verbose=0`, `onEachEntry=nil` 对工具无意义 +- 内部绑定 meta 文件的二进制格式（注释中有明确的 attr/types 表格） +- meta 文件格式变更 → 函数行为改变，无需 deprecation + +### 4.2 `logtail.NewCKPReaderWithTableID_V2` + +**位置**：`pkg/vm/engine/tae/logtail/ckp_reader.go:237` + +**风险**： +- `_V2` 后缀说明已发生过一次破坏性 API 变更 +- 配套的 `ReadMeta()` 有隐式调用顺序要求（注释："must called after ReadMeta"） +- `ConsumeCheckpointWithTableID` 的回调签名如果改变，编译失败 + +### 4.3 `ioutil.GetCheckpointDir()` + +**位置**：`pkg/objectio/ioutil/` + +**风险**： +- 固定返回 `"ckp/"` 字符串 +- 如果 MO 调整 checkpoint 目录结构 → 工具找不到任何 checkpoint 文件 + +### 4.4 `ioutil.ListTSRangeFiles` / `ckputil.ListCKPMetaNames` + +**风险**： +- 依赖 checkpoint 目录下的文件命名约定 +- 如果文件命名规则变化（扩展名、前缀等），过滤逻辑失效 + +### 4.5 `ckputil.TableObjectsAttrs` + +**位置**：`pkg/vm/engine/ckputil/types.go:66` + +```go +var TableObjectsAttrs = []string{ + "account_id", "db_id", "table_id", "object_type", + "id", "create_ts", "delete_ts", ... +} +``` + +工具在 `checkpoint_reader.go:471` 直接引用： +```go +columns := ckputil.TableObjectsAttrs +``` + +**风险**： +- 包级 `var`（非 const），无编译期保护 +- 属性名或顺序变化 → 工具输出的列名不变但值对不上 + +### 4.6 `objectioutil.FindTombstonesOfObject` / `GetTombstonesByBlockId` + +**位置**：`pkg/objectio/ioutil/` + +工具在 `logical_table.go` 中使用： +```go +selection, err := objectioutil.FindTombstonesOfObject(ctx, objectID, tombstoneStats, r.fs) +// ... +err := objectioutil.GetTombstonesByBlockId(ctx, snapshotTS, &blockID, getTombstone, &mask, r.fs) +``` + +**风险**： +- 深度依赖 objectio 的 bitmap、ObjectId、ObjectStats 等内部类型 +- 如果 objectio 重构 bitmap 表示方式 → tombstone 过滤逻辑完全失效 +- `getTombstone` 回调闭包跨层传递 `tombstoneStats` 切片，控制流不直观 + +--- + +## 5. 字符串列名匹配的静默 Fallback + +**位置**：`pkg/tools/checkpointtool/table_dump.go:373-384` (`fallbackCatalogColIndex`) + +**问题**：列位置解析有两步 fallback，失败时静默退化为按位置猜测： + +``` +① view.Headers 中按字符串匹配列名（如 "relname", "attnum"） + ↓ 失败 +② inferCatalogLayout 判断 layout → 按 layout 固定顺序定位 + ↓ 仍找不到 +③ 返回 -1 → 上层以 0 作为默认值，拿到错误的列 +``` + +**触发条件**： +- `catalog` 包改了列名（如 `rel_createsql` → `rel_create_sql`）→ 步骤① 失败 +- 新版 checkpoint 的 header 变成了 `col_N` 泛型名（无字符串可匹配）→ 直接走步骤② +- 步骤② 的 layout 检测也失败（见第 2 节）→ 走步骤③ + +**后果**：不是报错而是**静默输出错误的 CSV**。用户在不知情的情况下拿到列值错位的导出结果。 + +--- + +## 6. `knownCatalogLayouts` 只覆盖两个版本 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:150-152` + +```go +func knownCatalogLayouts() []catalogLayout { + return []catalogLayout{currentCatalogLayout, legacy3CatalogLayout} +} +``` + +**问题**：只认识 "current" 和 "3.0-dev" 两种 layout。如果 MO 4.0 又改了一次 catalog schema，就需要手动加第三种。否则 `inferCatalogLayout` fallback 到 `currentCatalogLayout`，列位置全错。 + +--- + +## 7. Catalog Layout 假设差异列只在末尾 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:53-57` + +```go +legacy3CatalogLayout = catalogLayout{ + moTablesSchema: catalog.MoTablesSchema[:len(catalog.MoTablesSchema)-1], // 切掉末尾 1 列 + moColumnsSchema: catalog.MoColumnsSchema[:len(catalog.MoColumnsSchema)-2], // 切掉末尾 2 列 +} +``` + +**前提假设**：新版本只在 `MoColumnsSchema` / `MoTablesSchema` **末尾**追加列。如果未来在**中间**插入列或调整列顺序，用"切末尾"方式构造的旧 layout schema 就会产生错位（前半段对齐、后半段全错）。 + +--- + +## 8. `ObjectEntryInfo` 的可见性判断依赖 `objectio.ObjectEntry.Visible(ts)` + +**位置**：`pkg/tools/checkpointtool/logical_table.go:218` + +```go +if obj.Visible(snapshotTS) { + visible = append(visible, entry) +} +``` + +**问题**：`objectio.ObjectEntry.Visible()` 的语义（基于 CreateTime / DeleteTime）是 MO 内部约定。如果可见性判断逻辑被调整（例如引入新的时间维度），工具的 tombstone 过滤和逻辑表输出就会不一致。 + +--- + +## 影响矩阵 + +| 上游变更 | 影响范围 | 是否静默 | +|---|---|---| +| `catalog` 包增删列 | schema 解析 → CSV 列全错 | ✅ 静默（编译通过） | +| 新增 catalog layout | 列位置偏移 | ✅ 静默（fallback 到 current） | +| `logical_table.go` 改 header | 所有数据列偏移 | ✅ 静默（`logicalViewMetaCols` 仍是 3） | +| `checkpoint` 包 meta 格式 | 无法加载 entry | ❌ 报错 | +| `logtail.NewCKPReader*` 改名 | 编译失败 | ❌ 编译错误 | +| `ioutil.GetCheckpointDir()` 改路径 | 找不到 checkpoint | ❌ 报错 | +| `ckputil.TableObjectsAttrs` 变动 | `ReadRangeData` 列名错误 | ✅ 静默 | +| `objectioutil` tombstone API 变更 | tombstone 过滤失效 | ❌ 可能编译或运行时错误 | +| `catalog` 列名修改 | 字符串匹配失败 → fallback 到手写位置 | ✅ 静默 | +| MO 列顺序调整（中间插入） | 旧 layout schema 错位 | ✅ 静默 | + +--- + +## 缓解建议 + +1. **消除 schema 硬编码**：`builtinColumnsForLayout` 应改为从 `catalog.MoTablesSchema` / `catalog.MoColumnsSchema` 动态生成，而非手写一份副本。 + +2. **在 checkpoint 数据中嵌入 layout 版本号**：让 `mo_tables` 数据中携带 catalog schema 版本标记，替代列数推断。 + +3. **将 `logicalViewMetaCols` 改为由 `LogicalTableView` 自身提供**：`LogicalTableView.MetaWidth()` 方法，而非在 `table_dump.go` 中硬编码常量 `3`。 + +4. **对所有内核 API 调用加一层抽象接口**：定义 `CheckpointSource` / `ObjectReader` 等接口，将 `checkpoint.ReadEntriesFromMeta`、`logtail.NewCKPReader*` 等调用隔离在 adapter 层内。 + +5. **在 `inferCatalogLayout` 无法匹配时发出 warning**：而非静默 fallback 到 `currentCatalogLayout`。 + +6. **对列名匹配失败增加告警**：`fallbackCatalogColIndex` 返回 `-1` 时记录日志，便于排查。 diff --git a/etc/launch-minio-local/log.toml b/etc/launch-minio-local/log.toml index 6753ed7d58772..0ff8d14184b26 100644 --- a/etc/launch-minio-local/log.toml +++ b/etc/launch-minio-local/log.toml @@ -1,4 +1,3 @@ -# service node type, [DN|CN|LOG] service-type = "LOG" data-dir = "./etc/launch-minio-local/mo-data" diff --git a/etc/launch-minio-local/tn.toml b/etc/launch-minio-local/tn.toml index 65fdbff27e59c..b455547cc6971 100644 --- a/etc/launch-minio-local/tn.toml +++ b/etc/launch-minio-local/tn.toml @@ -55,11 +55,11 @@ fileservice = "SHARED" log-backend = "logservice" [tn.Ckp] -flush-interval = "60s" -min-count = 100 -scan-interval = "5s" -incremental-interval = "180s" -global-min-count = 60 +flush-interval = "10s" +min-count = 5 +scan-interval = "2s" +incremental-interval = "30s" +global-min-count = 3 [tn.LogtailServer] rpc-max-message-size = "16KiB" diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index b141200a26cb3..992c681cd3334 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -386,6 +386,24 @@ func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { return view, nil } +// ValidateSnapshot checks that ts can compose a catalog-backed checkpoint view. +func (r *CheckpointReader) ValidateSnapshot(ctx context.Context, ts types.TS) error { + view, err := r.ComposeAt(ts) + if err != nil { + return err + } + if len(view.Tables) == 0 { + return moerr.NewInternalErrorf(ctx, "checkpoint snapshot %s is not usable: no tables are available", ts.ToString()) + } + if _, ok := view.Tables[moTablesID]; !ok { + return moerr.NewInternalErrorf(ctx, "checkpoint snapshot %s is not usable: mo_tables catalog is not available; wait for a global checkpoint or choose a later timestamp", ts.ToString()) + } + if _, ok := view.Tables[moColumnsID]; !ok { + return moerr.NewInternalErrorf(ctx, "checkpoint snapshot %s is not usable: mo_columns catalog is not available; wait for a global checkpoint or choose a later timestamp", ts.ToString()) + } + return nil +} + // isDataFileNotFound checks if err indicates a checkpoint data file was GC'd/missing. func isDataFileNotFound(err error) bool { if err == nil { @@ -467,16 +485,7 @@ func (r *CheckpointReader) ReadRangeData(entry *checkpoint.CheckpointEntry, rng return nil, nil, nil } - // Use checkpoint schema for column names - columns := ckputil.TableObjectsAttrs - if len(columns) == 0 { - // Fallback: generate column names from vector count - bat := bats[0] - columns = make([]string, len(bat.Vecs)) - for i := range columns { - columns[i] = fmt.Sprintf("col_%d", i) - } - } + columns := checkpointRangeColumns(len(bats[0].Vecs)) // Extract rows within the range startRow := uint32(rng.Start.GetRowOffset()) @@ -495,14 +504,23 @@ func (r *CheckpointReader) ReadRangeData(entry *checkpoint.CheckpointEntry, rng } } - // Trim columns to match actual vector count - if len(bats) > 0 && len(columns) > len(bats[0].Vecs) { - columns = columns[:len(bats[0].Vecs)] - } - return columns, rows, nil } +func checkpointRangeColumns(width int) []string { + if width <= 0 { + return nil + } + columns := append([]string(nil), ckputil.TableObjectsAttrs...) + if len(columns) > width { + return columns[:width] + } + for len(columns) < width { + columns = append(columns, fmt.Sprintf("col_%d", len(columns))) + } + return columns +} + // vecValueToString converts a vector value at index to string func vecValueToString(vec *vector.Vector, idx int) string { if vec.IsNull(uint64(idx)) { diff --git a/pkg/tools/checkpointtool/logical_table.go b/pkg/tools/checkpointtool/logical_table.go index b30646fc8c8b9..3e16e9e430ceb 100644 --- a/pkg/tools/checkpointtool/logical_table.go +++ b/pkg/tools/checkpointtool/logical_table.go @@ -21,6 +21,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/objectio" objectioutil "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" "github.com/matrixorigin/matrixone/pkg/tools/objecttool" @@ -39,13 +40,10 @@ func (r *CheckpointReader) BuildLogicalTableView( dataEntries []*ObjectEntryInfo, tombEntries []*ObjectEntryInfo, ) (*LogicalTableView, error) { - view := &LogicalTableView{ - Headers: []string{"object", "block", "row"}, - Rows: make([][]string, 0), - } + view := newLogicalTableView() stats, err := r.scanLogicalTable(ctx, snapshotTS, dataEntries, tombEntries, func(cols []objecttool.ColInfo) error { - if len(view.Headers) != 3 { + if len(view.Headers) != view.MetaWidth() { return nil } for _, col := range cols { @@ -55,7 +53,7 @@ func (r *CheckpointReader) BuildLogicalTableView( return nil }, func(objShort string, blockIdx int, rowIdx int, values []string, _ []bool) error { - row := make([]string, 0, len(values)+3) + row := make([]string, 0, len(values)+view.MetaWidth()) row = append(row, objShort, fmt.Sprintf("%d", blockIdx), fmt.Sprintf("%d", rowIdx)) row = append(row, values...) view.Rows = append(view.Rows, row) @@ -127,14 +125,33 @@ func (r *CheckpointReader) scanLogicalTable( stats.PhysicalRows += bat.RowCount() + commitTSVec, releaseCommitTS, err := reader.ReadBlockCommitTS(ctx, uint32(blockIdx)) + if err != nil { + release() + _ = reader.Close() + return stats, err + } + var commitTSs []types.TS + if commitTSVec != nil { + commitTSs = vector.MustFixedColWithTypeCheck[types.TS](commitTSVec) + } + deleteMask, err := r.buildDeleteMaskForBlock(ctx, &snapshotTS, entry.ObjectStats, uint16(blockIdx), relevantTombstones) if err != nil { + if releaseCommitTS != nil { + releaseCommitTS() + } release() _ = reader.Close() return stats, err } for rowIdx := 0; rowIdx < bat.RowCount(); rowIdx++ { + if commitTSs != nil && rowIdx < len(commitTSs) && + !commitTSVec.IsNull(uint64(rowIdx)) && + commitTSs[rowIdx].GT(&snapshotTS) { + continue + } if deleteMask.IsValid() && deleteMask.Contains(uint64(rowIdx)) { stats.DeletedRows++ continue @@ -150,6 +167,9 @@ func (r *CheckpointReader) scanLogicalTable( if deleteMask.IsValid() { deleteMask.Release() } + if releaseCommitTS != nil { + releaseCommitTS() + } release() _ = reader.Close() return stats, err @@ -161,6 +181,9 @@ func (r *CheckpointReader) scanLogicalTable( if deleteMask.IsValid() { deleteMask.Release() } + if releaseCommitTS != nil { + releaseCommitTS() + } release() } diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 8dc7962a0f3a1..210c299c8fc38 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -44,12 +44,29 @@ type catalogLayout struct { moColumnsSchema []string } +type catalogLayoutMatch struct { + layout catalogLayout + offset int +} + var ( currentCatalogLayout = catalogLayout{ name: "current", moTablesSchema: append([]string(nil), catalog.MoTablesSchema...), moColumnsSchema: append([]string(nil), catalog.MoColumnsSchema...), } + preCPKLayout = catalogLayout{ + name: "pre-cpk", + moTablesSchema: catalogSchemaWithout( + catalog.MoTablesSchema, + catalog.SystemRelAttr_ExtraInfo, + catalog.SystemRelAttr_CPKey, + ), + moColumnsSchema: catalogSchemaWithout( + catalog.MoColumnsSchema, + catalog.SystemColAttr_CPKey, + ), + } legacy3CatalogLayout = catalogLayout{ name: "3.0-dev", moTablesSchema: append([]string(nil), catalog.MoTablesSchema[:len(catalog.MoTablesSchema)-1]...), @@ -73,6 +90,21 @@ type TableSchema struct { CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql } +type TableCatalogEntry struct { + TableID uint64 + AccountID uint32 + DatabaseID uint64 + DatabaseName string + TableName string + RelKind string +} + +type TableListOptions struct { + AccountID *uint32 + DatabaseID *uint64 + IncludeViews bool +} + type CSVRowOrder string const ( @@ -148,7 +180,7 @@ type builtinColumnDef struct { } func knownCatalogLayouts() []catalogLayout { - return []catalogLayout{currentCatalogLayout, legacy3CatalogLayout} + return []catalogLayout{preCPKLayout, currentCatalogLayout, legacy3CatalogLayout} } func schemaForLayout(layout catalogLayout, tableID uint64) []string { @@ -162,6 +194,21 @@ func schemaForLayout(layout catalogLayout, tableID uint64) []string { } } +func catalogSchemaWithout(schema []string, names ...string) []string { + excluded := make(map[string]struct{}, len(names)) + for _, name := range names { + excluded[name] = struct{}{} + } + filtered := make([]string, 0, len(schema)) + for _, name := range schema { + if _, ok := excluded[name]; ok { + continue + } + filtered = append(filtered, name) + } + return filtered +} + func builtinColumnsForLayout(layout catalogLayout, tableID uint64) []builtinColumnDef { switch tableID { case catalog.MO_DATABASE_ID: @@ -225,11 +272,11 @@ func builtinColumnsForLayout(layout catalogLayout, tableID uint64) []builtinColu {Name: "att_is_auto_increment", SQLType: "TINYINT", Position: 16}, {Name: "att_comment", SQLType: "VARCHAR(2048)", Position: 17}, {Name: "att_is_hidden", SQLType: "TINYINT", Position: 18}, - {Name: "att_has_update", SQLType: "TINYINT", Position: 19}, - {Name: "att_update", SQLType: "VARCHAR(2048)", Position: 20}, - {Name: "att_is_clusterby", SQLType: "TINYINT", Position: 21}, - {Name: "att_seqnum", SQLType: "SMALLINT UNSIGNED", Position: 22}, - {Name: "att_enum", SQLType: "VARCHAR", Position: 23}, + {Name: catalog.SystemColAttr_HasUpdate, SQLType: "TINYINT", Position: 19}, + {Name: catalog.SystemColAttr_Update, SQLType: "VARCHAR(2048)", Position: 20}, + {Name: catalog.SystemColAttr_IsClusterBy, SQLType: "TINYINT", Position: 21}, + {Name: catalog.SystemColAttr_Seqnum, SQLType: "SMALLINT UNSIGNED", Position: 22}, + {Name: catalog.SystemColAttr_EnumValues, SQLType: "VARCHAR", Position: 23}, {Name: catalog.SystemColAttr_CPKey, SQLType: "VARCHAR(65535)", Position: 24, Hidden: true}, } if layout.name != legacy3CatalogLayout.name { @@ -311,22 +358,26 @@ func inferBuiltinCatalogLayout( switch tableID { case moTablesID: if moTablesView != nil { - layout, _ := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID) - return layout + if layout, _, ok := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID); ok { + return layout + } } case moColumnsID: if moColumnsView != nil { - layout, _ := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID) - return layout + if layout, _, ok := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID); ok { + return layout + } } case catalog.MO_DATABASE_ID: if moTablesView != nil { - layout, _ := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID) - return layout + if layout, _, ok := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID); ok { + return layout + } } if moColumnsView != nil { - layout, _ := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID) - return layout + if layout, _, ok := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID); ok { + return layout + } } } return currentCatalogLayout @@ -354,27 +405,57 @@ func mergeBuiltinSchemaFallback(schema *TableSchema, builtin *TableSchema, table return schema } -func inferCatalogLayout(dataWidth int, tableID uint64) (catalogLayout, int) { +func inferCatalogLayout(dataWidth int, tableID uint64) (catalogLayout, int, bool) { + for _, offset := range []int{0, 1, 2} { + for _, layout := range knownCatalogLayouts() { + schema := schemaForLayout(layout, tableID) + if len(schema) == 0 { + continue + } + if dataWidth == len(schema)+offset { + return layout, offset, true + } + } + } + return catalogLayout{}, 0, false +} + +func catalogLayoutMatches(dataWidth int, tableID uint64) []catalogLayoutMatch { + var matches []catalogLayoutMatch for _, layout := range knownCatalogLayouts() { schema := schemaForLayout(layout, tableID) if len(schema) == 0 { continue } - switch dataWidth { - case len(schema): - return layout, 0 - case len(schema) + 1: - return layout, 1 + for _, offset := range []int{0, 1, 2} { + if dataWidth >= len(schema)+offset { + matches = append(matches, catalogLayoutMatch{ + layout: layout, + offset: offset, + }) + } } } - return currentCatalogLayout, 0 + return matches } func fallbackCatalogColIndex(view *LogicalTableView, tableID uint64, colName string) int { if idx := view.columnDataIndex(colName); idx >= 0 { return idx } - layout, offset := inferCatalogLayout(len(view.Headers)-logicalViewMetaCols, tableID) + layout, offset, ok := inferCatalogLayout(len(view.Headers)-logicalViewMetaCols, tableID) + if !ok { + return -1 + } + for i, name := range schemaForLayout(layout, tableID) { + if name == colName { + return i + offset + } + } + return -1 +} + +func catalogColIndexForLayout(layout catalogLayout, tableID uint64, colName string, offset int) int { for i, name := range schemaForLayout(layout, tableID) { if name == colName { return i + offset @@ -550,6 +631,44 @@ func (r *CheckpointReader) DumpTableCSVComposed( return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries, resolveCSVExportOptions(opts)) } +func (r *CheckpointReader) ListCatalogTables( + ctx context.Context, + snapshotTS types.TS, + opts TableListOptions, +) ([]TableCatalogEntry, error) { + moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) + if err != nil { + return nil, err + } + tables := buildCatalogTablesFromMoTablesRows(moTablesView) + filtered := make([]TableCatalogEntry, 0, len(tables)) + for _, table := range tables { + if opts.AccountID != nil && table.AccountID != *opts.AccountID { + continue + } + if opts.DatabaseID != nil && table.DatabaseID != *opts.DatabaseID { + continue + } + if !opts.IncludeViews && table.RelKind != "" && table.RelKind != "r" { + continue + } + filtered = append(filtered, table) + } + sort.Slice(filtered, func(i, j int) bool { + if filtered[i].AccountID != filtered[j].AccountID { + return filtered[i].AccountID < filtered[j].AccountID + } + if filtered[i].DatabaseName != filtered[j].DatabaseName { + return filtered[i].DatabaseName < filtered[j].DatabaseName + } + if filtered[i].TableName != filtered[j].TableName { + return filtered[i].TableName < filtered[j].TableName + } + return filtered[i].TableID < filtered[j].TableID + }) + return filtered, nil +} + func (r *CheckpointReader) streamTableCSV( ctx context.Context, w io.Writer, @@ -906,35 +1025,224 @@ func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *Table relNameIdx := fallbackCatalogColIndex(view, moTablesID, "relname") if relNameIdx < len(dataRow) { - schema.TableName = dataRow[relNameIdx] + schema.TableName = strings.Clone(dataRow[relNameIdx]) } relDBIdx := fallbackCatalogColIndex(view, moTablesID, "reldatabase") if relDBIdx < len(dataRow) { - schema.DatabaseName = dataRow[relDBIdx] + schema.DatabaseName = strings.Clone(dataRow[relDBIdx]) } createSQLIdx := fallbackCatalogColIndex(view, moTablesID, "rel_createsql") if createSQLIdx < len(dataRow) { - schema.CreateSQL = dataRow[createSQLIdx] + schema.CreateSQL = strings.Clone(dataRow[createSQLIdx]) } return schema } +func findCreateSQLFromMoTables(view *LogicalTableView, tableID uint64) string { + tableIDStr := fmt.Sprintf("%d", tableID) + try := func(relIDCol, createSQLCol int) string { + if relIDCol < 0 || createSQLCol < 0 { + return "" + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewMetaCols:] + if relIDCol >= len(row) || createSQLCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr && row[createSQLCol] != "" { + return strings.Clone(row[createSQLCol]) + } + } + return "" + } + + relIDCol := fallbackCatalogColIndex(view, moTablesID, "rel_id") + createSQLCol := fallbackCatalogColIndex(view, moTablesID, "rel_createsql") + if ddl := try(relIDCol, createSQLCol); ddl != "" { + return ddl + } + + dataWidth := len(view.Headers) - logicalViewMetaCols + for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { + relIDCol = catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset) + createSQLCol = catalogColIndexForLayout(match.layout, moTablesID, "rel_createsql", match.offset) + if ddl := try(relIDCol, createSQLCol); ddl != "" { + return ddl + } + } + return "" +} + +func buildCatalogTablesFromMoTablesRows(view *LogicalTableView) []TableCatalogEntry { + var best []TableCatalogEntry + try := func(relIDCol, relNameCol, relDBCol, relDBIDCol, relKindCol, accountIDCol int) { + tables := buildCatalogTablesFromMoTablesRowsAt( + view, + relIDCol, + relNameCol, + relDBCol, + relDBIDCol, + relKindCol, + accountIDCol, + ) + if len(tables) > len(best) { + best = tables + } + } + + try( + fallbackCatalogColIndex(view, moTablesID, "rel_id"), + fallbackCatalogColIndex(view, moTablesID, "relname"), + fallbackCatalogColIndex(view, moTablesID, "reldatabase"), + fallbackCatalogColIndex(view, moTablesID, "reldatabase_id"), + fallbackCatalogColIndex(view, moTablesID, "relkind"), + fallbackCatalogColIndex(view, moTablesID, "account_id"), + ) + dataWidth := len(view.Headers) - logicalViewMetaCols + for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { + try( + catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "relname", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "reldatabase", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "reldatabase_id", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "relkind", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "account_id", match.offset), + ) + } + return best +} + +func buildCatalogTablesFromMoTablesRowsAt( + view *LogicalTableView, + relIDCol int, + relNameCol int, + relDBCol int, + relDBIDCol int, + relKindCol int, + accountIDCol int, +) []TableCatalogEntry { + if relIDCol < 0 || relNameCol < 0 || relDBCol < 0 { + return nil + } + + seen := make(map[uint64]struct{}) + tables := make([]TableCatalogEntry, 0) + for _, fullRow := range view.Rows { + row := fullRow[logicalViewMetaCols:] + if relIDCol >= len(row) || relNameCol >= len(row) || relDBCol >= len(row) { + continue + } + tableID, err := strconv.ParseUint(row[relIDCol], 10, 64) + if err != nil || tableID == 0 { + continue + } + if _, ok := seen[tableID]; ok { + continue + } + entry := TableCatalogEntry{ + TableID: tableID, + AccountID: 0, + DatabaseName: strings.Clone(row[relDBCol]), + TableName: strings.Clone(row[relNameCol]), + } + if !validCatalogName(entry.DatabaseName) || !validCatalogName(entry.TableName) { + continue + } + if relDBIDCol >= 0 && relDBIDCol < len(row) { + if dbID, err := strconv.ParseUint(row[relDBIDCol], 10, 64); err == nil { + entry.DatabaseID = dbID + } + } + if relKindCol >= 0 && relKindCol < len(row) { + entry.RelKind = strings.Clone(row[relKindCol]) + if !validRelKind(entry.RelKind) { + continue + } + } + if accountIDCol >= 0 && accountIDCol < len(row) { + if accountID, err := strconv.ParseUint(row[accountIDCol], 10, 32); err == nil { + entry.AccountID = uint32(accountID) + } + } + seen[tableID] = struct{}{} + tables = append(tables, entry) + } + return tables +} + +func validCatalogName(s string) bool { + if s == "" { + return false + } + for _, r := range s { + switch { + case r >= 'a' && r <= 'z': + case r >= 'A' && r <= 'Z': + case r >= '0' && r <= '9': + case r == '_' || r == '-' || r == '$': + default: + return false + } + } + return true +} + +func validRelKind(s string) bool { + switch s { + case "", "r", "v", "e", "cluster", "external", "view": + return true + default: + return false + } +} + // buildColumnsFromMoColumnsRows builds a sorted column list from mo_columns data rows // filtered for a specific tableID and excluding hidden columns. // Uses column name lookup from view headers to handle hidden column offsets. // mo_columns columns: att_relname_id, att_relname, attname, atttyp, attnum, ..., att_is_hidden func buildColumnsFromMoColumnsRows(view *LogicalTableView, tableID uint64) []TableColumn { - tableIDStr := fmt.Sprintf("%d", tableID) - relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") typCol := fallbackCatalogColIndex(view, moColumnsID, "atttyp") numCol := fallbackCatalogColIndex(view, moColumnsID, "attnum") hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") - seqNumCol := fallbackCatalogColIndex(view, moColumnsID, "att_seqnum") + seqNumCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Seqnum) + + if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { + if cols := buildColumnsFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol, seqNumCol); len(cols) > 0 { + return cols + } + } + dataWidth := len(view.Headers) - logicalViewMetaCols + for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { + relnameIDCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_relname_id", match.offset) + nameCol = catalogColIndexForLayout(match.layout, moColumnsID, "attname", match.offset) + typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) + numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) + hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) + seqNumCol = catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Seqnum, match.offset) + if cols := buildColumnsFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol, seqNumCol); len(cols) > 0 { + return cols + } + } + + return nil +} + +func buildColumnsFromMoColumnsRowsAt( + view *LogicalTableView, + tableID uint64, + relnameIDCol int, + nameCol int, + typCol int, + numCol int, + hiddenCol int, + seqNumCol int, +) []TableColumn { + tableIDStr := fmt.Sprintf("%d", tableID) var cols []TableColumn for _, fullRow := range view.Rows { row := fullRow[logicalViewMetaCols:] @@ -1012,16 +1320,8 @@ func (r *CheckpointReader) ShowCreateTable( // 1. Try mo_tables.rel_createsql moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) if err == nil && moTablesView != nil { - relIDCol := fallbackCatalogColIndex(moTablesView, moTablesID, "rel_id") - createSQLCol := fallbackCatalogColIndex(moTablesView, moTablesID, "rel_createsql") - for _, fullRow := range moTablesView.Rows { - row := fullRow[logicalViewMetaCols:] - if relIDCol < len(row) && row[relIDCol] == fmt.Sprintf("%d", tableID) { - if createSQLCol < len(row) && row[createSQLCol] != "" { - return row[createSQLCol], nil - } - break - } + if ddl := findCreateSQLFromMoTables(moTablesView, tableID); ddl != "" { + return ddl, nil } } @@ -1038,11 +1338,15 @@ func (r *CheckpointReader) ShowCreateTable( switch tableID { case moTablesID: if moTablesView != nil { - layout, _ = inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID) + if inferred, _, ok := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID); ok { + layout = inferred + } } case moColumnsID: if moColumnsView != nil { - layout, _ = inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID) + if inferred, _, ok := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID); ok { + layout = inferred + } } } if ddl := hardcodedCreateTableForLayout(tableID, layout); ddl != "" { @@ -1066,13 +1370,43 @@ func getTableName(view *LogicalTableView, tableID uint64) string { // buildCreateTableFromMoColumns reconstructs a CREATE TABLE DDL from mo_columns data. func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) string { - tableIDStr := fmt.Sprintf("%d", tableID) relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") typCol := fallbackCatalogColIndex(view, moColumnsID, "atttyp") numCol := fallbackCatalogColIndex(view, moColumnsID, "attnum") hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") + if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + return ddl + } + } + + dataWidth := len(view.Headers) - logicalViewMetaCols + for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { + relnameIDCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_relname_id", match.offset) + nameCol = catalogColIndexForLayout(match.layout, moColumnsID, "attname", match.offset) + typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) + numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) + hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + return ddl + } + } + + return "" +} + +func buildCreateTableFromMoColumnsAt( + view *LogicalTableView, + tableID uint64, + relnameIDCol int, + nameCol int, + typCol int, + numCol int, + hiddenCol int, +) string { + tableIDStr := fmt.Sprintf("%d", tableID) type colInfo struct { name string sqlType string diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index a399a33f59adc..0ea53feb8affb 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -226,7 +226,7 @@ func TestBuildColumnsFromMoColumnsRows(t *testing.T) { "att_relname_id", "att_relname", "attname", "atttyp", "attnum", "col_9", "col_10", "col_11", "col_12", "col_13", "col_14", "col_15", "col_16", "col_17", "att_is_hidden", - "col_19", "col_20", "col_21", "att_seqnum", + "col_19", "col_20", "col_21", "attr_seqnum", }, Rows: [][]string{ {"obj1", "0", "0", "", "", "", "", "12345", "my_table", "id", "BIGINT", "1", "", "", "", "", "", "", "", "", "", "0", "", "", "", "0"}, @@ -270,6 +270,113 @@ func TestBuildColumnsFromMoColumnsRows_FallbackToAttnumWhenSeqnumMissing(t *test assert.Equal(t, 1, cols[1].PhysicalPosition) } +func TestBuildColumnsFromMoColumnsRows_GenericPreCPKWithPhysicalPrefix(t *testing.T) { + const ( + tableID = uint64(12345) + offset = 1 + ) + + headers := []string{"object", "block", "row"} + for i := 0; i < len(preCPKLayout.moColumnsSchema)+offset; i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + + row := func(metaRow, name, typ, attnum, hidden, seqnum string) []string { + data := make([]string, len(preCPKLayout.moColumnsSchema)+offset) + setCatalogValue := func(colName, value string) { + idx := catalogColIndexForLayout(preCPKLayout, moColumnsID, colName, offset) + require.GreaterOrEqual(t, idx, 0) + data[idx] = value + } + data[0] = "physical-prefix" + setCatalogValue("att_relname_id", fmt.Sprintf("%d", tableID)) + setCatalogValue("attname", name) + setCatalogValue("atttyp", typ) + setCatalogValue("attnum", attnum) + setCatalogValue("att_is_hidden", hidden) + setCatalogValue("attr_seqnum", seqnum) + return append([]string{"obj1", "0", metaRow}, data...) + } + + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("0", "id", "INT", "1", "0", "0"), + row("1", "name", "VARCHAR(100)", "2", "0", "1"), + row("2", "__mo_rowid", "ROWID", "0", "1", "2"), + }, + } + + cols := buildColumnsFromMoColumnsRows(view, tableID) + require.Len(t, cols, 2) + assert.Equal(t, "id", cols[0].Name) + assert.Equal(t, 0, cols[0].PhysicalPosition) + assert.Equal(t, "name", cols[1].Name) + assert.Equal(t, 1, cols[1].PhysicalPosition) + + ddl := buildCreateTableFromMoColumns(view, tableID) + assert.Contains(t, ddl, "`id` INT") + assert.Contains(t, ddl, "`name` VARCHAR(100)") + assert.NotContains(t, ddl, "__mo_rowid") +} + +func TestFindCreateSQLFromMoTables_GenericWithTrailingColumns(t *testing.T) { + headers := []string{"object", "block", "row"} + for i := 0; i < len(preCPKLayout.moTablesSchema)+2; i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + row := make([]string, len(headers)) + row[0], row[1], row[2] = "obj1", "0", "0" + row[logicalViewMetaCols+0] = "12345" + row[logicalViewMetaCols+1] = "employees" + row[logicalViewMetaCols+2] = "test_ckp" + row[logicalViewMetaCols+7] = "CREATE TABLE employees (id INT)" + row[len(row)-2] = "tail-1" + row[len(row)-1] = "tail-2" + + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{row}, + } + + assert.Equal(t, "CREATE TABLE employees (id INT)", findCreateSQLFromMoTables(view, 12345)) +} + +func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testing.T) { + headers := []string{"object", "block", "row"} + for i := 0; i < len(preCPKLayout.moTablesSchema)+2; i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + row := func(relID, relName, dbName, dbID, relKind, accountID string) []string { + data := make([]string, len(preCPKLayout.moTablesSchema)+2) + data[0] = relID + data[1] = relName + data[2] = dbName + data[3] = dbID + data[5] = relKind + data[11] = accountID + return append([]string{"obj1", "0", relID}, data...) + } + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("1001", "orders", "tpch_10g", "9001", "r", "7"), + row("1002", "revenue0", "tpch_10g", "9001", "v", "7"), + }, + } + + tables := buildCatalogTablesFromMoTablesRows(view) + require.Len(t, tables, 2) + assert.Equal(t, uint64(1001), tables[0].TableID) + assert.Equal(t, uint32(7), tables[0].AccountID) + assert.Equal(t, uint64(9001), tables[0].DatabaseID) + assert.Equal(t, "tpch_10g", tables[0].DatabaseName) + assert.Equal(t, "orders", tables[0].TableName) + assert.Equal(t, "r", tables[0].RelKind) + assert.Equal(t, "revenue0", tables[1].TableName) + assert.Equal(t, "v", tables[1].RelKind) +} + func TestInferCatalogLayout(t *testing.T) { tests := []struct { name string @@ -277,22 +384,38 @@ func TestInferCatalogLayout(t *testing.T) { tableID uint64 layout string offset int + ok bool }{ - {name: "current_tables", dataWidth: len(catalog.MoTablesSchema), tableID: moTablesID, layout: currentCatalogLayout.name, offset: 0}, - {name: "current_tables_fakepk", dataWidth: len(catalog.MoTablesSchema) + 1, tableID: moTablesID, layout: currentCatalogLayout.name, offset: 1}, - {name: "legacy3_tables", dataWidth: len(legacy3CatalogLayout.moTablesSchema), tableID: moTablesID, layout: legacy3CatalogLayout.name, offset: 0}, - {name: "legacy3_columns", dataWidth: len(legacy3CatalogLayout.moColumnsSchema), tableID: moColumnsID, layout: legacy3CatalogLayout.name, offset: 0}, + {name: "current_tables", dataWidth: len(catalog.MoTablesSchema), tableID: moTablesID, layout: currentCatalogLayout.name, offset: 0, ok: true}, + {name: "current_tables_fakepk", dataWidth: len(catalog.MoTablesSchema) + 1, tableID: moTablesID, layout: currentCatalogLayout.name, offset: 1, ok: true}, + {name: "current_columns_fakepk", dataWidth: len(catalog.MoColumnsSchema) + 1, tableID: moColumnsID, layout: currentCatalogLayout.name, offset: 1, ok: true}, + {name: "pre_cpk_tables", dataWidth: len(preCPKLayout.moTablesSchema), tableID: moTablesID, layout: preCPKLayout.name, offset: 0, ok: true}, + {name: "pre_cpk_columns", dataWidth: len(preCPKLayout.moColumnsSchema), tableID: moColumnsID, layout: preCPKLayout.name, offset: 0, ok: true}, + {name: "legacy3_tables", dataWidth: len(legacy3CatalogLayout.moTablesSchema), tableID: moTablesID, layout: legacy3CatalogLayout.name, offset: 0, ok: true}, + {name: "legacy3_columns", dataWidth: len(legacy3CatalogLayout.moColumnsSchema), tableID: moColumnsID, layout: legacy3CatalogLayout.name, offset: 0, ok: true}, + {name: "unknown_tables", dataWidth: 7, tableID: moTablesID, ok: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - layout, offset := inferCatalogLayout(tt.dataWidth, tt.tableID) + layout, offset, ok := inferCatalogLayout(tt.dataWidth, tt.tableID) + assert.Equal(t, tt.ok, ok) + if !tt.ok { + return + } assert.Equal(t, tt.layout, layout.name) assert.Equal(t, tt.offset, offset) }) } } +func TestFallbackCatalogColIndex_UnknownLayoutDoesNotGuessCurrent(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3", "col_4", "col_5", "col_6"}, + } + assert.Equal(t, -1, fallbackCatalogColIndex(view, moTablesID, "rel_createsql")) +} + // TestLogicalTableViewWithSchema_mergeHeaders merges schema column names using position-based mapping. func TestLogicalTableViewWithSchema_mergeHeaders(t *testing.T) { schema := &TableSchema{ @@ -484,7 +607,7 @@ func TestColumnSchemaRoundTrip(t *testing.T) { "att_relname_id", "att_relname", "attname", "atttyp", "attnum", "col_9", "col_10", "col_11", "col_12", "col_13", "col_14", "col_15", "col_16", "col_17", "att_is_hidden", - "col_19", "col_20", "col_21", "att_seqnum", + "col_19", "col_20", "col_21", "attr_seqnum", }, Rows: [][]string{ {"obj1", "0", "0", "", "", "", "", "12345", "my_table", "a", "INT", "1", "", "", "", "", "", "", "", "", "", "0", "", "", "", "0"}, diff --git a/pkg/tools/checkpointtool/types.go b/pkg/tools/checkpointtool/types.go index 1cf030edeee99..73beb619c0942 100644 --- a/pkg/tools/checkpointtool/types.go +++ b/pkg/tools/checkpointtool/types.go @@ -91,3 +91,47 @@ type LogicalTableView struct { DeletedRows int VisibleRows int } + +var logicalTableViewMetaHeaders = []string{"object", "block", "row"} + +func newLogicalTableView() *LogicalTableView { + return &LogicalTableView{ + Headers: append([]string(nil), logicalTableViewMetaHeaders...), + Rows: make([][]string, 0), + } +} + +func (v *LogicalTableView) MetaWidth() int { + if v == nil { + return len(logicalTableViewMetaHeaders) + } + limit := len(logicalTableViewMetaHeaders) + if len(v.Headers) < limit { + limit = len(v.Headers) + } + for i := 0; i < limit; i++ { + if v.Headers[i] != logicalTableViewMetaHeaders[i] { + return i + } + } + return limit +} + +func (v *LogicalTableView) DataWidth() int { + if v == nil { + return 0 + } + width := len(v.Headers) - v.MetaWidth() + if width < 0 { + return 0 + } + return width +} + +func (v *LogicalTableView) DataRow(fullRow []string) []string { + metaWidth := v.MetaWidth() + if len(fullRow) < metaWidth { + return nil + } + return fullRow[metaWidth:] +} diff --git a/pkg/tools/objecttool/interactive/state.go b/pkg/tools/objecttool/interactive/state.go index 16bf45c4d15d8..0bb280b0bc7f3 100644 --- a/pkg/tools/objecttool/interactive/state.go +++ b/pkg/tools/objecttool/interactive/state.go @@ -1154,8 +1154,9 @@ func (s *State) Columns() []objecttool.ColInfo { cols := make([]objecttool.ColInfo, len(s.metaCols)) for i, col := range s.metaCols { cols[i] = objecttool.ColInfo{ - Idx: col.Idx, - Type: col.Type, + Idx: col.Idx, + SeqNum: col.Idx, + Type: col.Type, } } return cols @@ -1170,8 +1171,9 @@ func (s *State) expandColumns(cols []objecttool.ColInfo) []objecttool.ColInfo { // Replace source column with expanded columns for j := range exp.NewCols { result = append(result, objecttool.ColInfo{ - Idx: uint16(len(result)), - Type: exp.NewTypes[j], + Idx: uint16(len(result)), + SeqNum: uint16(len(result)), + Type: exp.NewTypes[j], }) } } else { diff --git a/pkg/tools/objecttool/object_reader.go b/pkg/tools/objecttool/object_reader.go index 9c45f28d89057..c29f3164fa5aa 100644 --- a/pkg/tools/objecttool/object_reader.go +++ b/pkg/tools/objecttool/object_reader.go @@ -47,8 +47,9 @@ type ObjectInfo struct { // ColInfo contains column information type ColInfo struct { - Idx uint16 - Type types.Type + Idx uint16 + SeqNum uint16 + Type types.Type } // ObjectReader reads object files @@ -142,8 +143,9 @@ func buildColInfo(meta objectio.ObjectDataMeta) []ColInfo { for i := uint16(0); i < colCount; i++ { colMeta := blockMeta.ColumnMeta(i) cols[i] = ColInfo{ - Idx: i, - Type: types.T(colMeta.DataType()).ToType(), + Idx: i, + SeqNum: i, + Type: types.T(colMeta.DataType()).ToType(), } } } @@ -176,7 +178,7 @@ func (r *ObjectReader) ReadBlock(ctx context.Context, blockIdx uint32) (*batch.B colIdxs := make([]uint16, len(r.cols)) colTypes := make([]types.Type, len(r.cols)) for i := range r.cols { - colIdxs[i] = uint16(i) + colIdxs[i] = r.cols[i].SeqNum colTypes[i] = r.cols[i].Type } @@ -205,6 +207,45 @@ func (r *ObjectReader) ReadBlock(ctx context.Context, blockIdx uint32) (*batch.B return bat, release, nil } +// ReadBlockCommitTS reads the hidden commit timestamp column for a block. +// Objects that do not carry commit timestamps return a nil vector. +func (r *ObjectReader) ReadBlockCommitTS(ctx context.Context, blockIdx uint32) (*vector.Vector, func(), error) { + if blockIdx >= r.info.BlockCount { + return nil, nil, moerr.NewInternalErrorf(ctx, "block index %d out of range [0, %d)", blockIdx, r.info.BlockCount) + } + + ioVectors, err := r.objReader.ReadOneBlock( + ctx, + []uint16{objectio.SEQNUM_COMMITTS}, + []types.Type{types.T_TS.ToType()}, + uint16(blockIdx), + r.mp, + ) + if err != nil { + return nil, nil, err + } + + release := func() { + objectio.ReleaseIOVector(&ioVectors) + } + + obj, err := objectio.Decode(ioVectors.Entries[0].CachedData.Bytes()) + if err != nil { + release() + return nil, nil, err + } + vec := obj.(*vector.Vector) + if vec.GetType().Oid != types.T_TS { + release() + return nil, nil, moerr.NewInternalErrorf(ctx, "commit TS column type mismatch: expected TS, got %s", vec.GetType().String()) + } + if vec.GetNulls().GetCardinality() == vec.Length() { + release() + return nil, func() {}, nil + } + return vec, release, nil +} + // BlockCount returns block count func (r *ObjectReader) BlockCount() uint32 { return r.info.BlockCount diff --git a/pkg/tools/toolfs/lazy_cache_fs.go b/pkg/tools/toolfs/lazy_cache_fs.go index bdc14f40604b7..61f27838e5d2f 100644 --- a/pkg/tools/toolfs/lazy_cache_fs.go +++ b/pkg/tools/toolfs/lazy_cache_fs.go @@ -16,19 +16,21 @@ package toolfs import ( "context" + "io" "iter" "os" "path/filepath" "strings" "sync" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/fileservice" ) type lazyCacheFS struct { name string remote fileservice.FileService - local fileservice.FileService + local *fileservice.LocalFS root string mu sync.Mutex @@ -60,21 +62,26 @@ func (l *lazyCacheFS) Write(ctx context.Context, vector fileservice.IOVector) er if err := l.remote.Write(ctx, vector); err != nil { return err } - return l.local.Write(ctx, vector) + normalized := stripServiceName(vector.FilePath, l.name) + _ = os.Remove(filepath.Join(l.root, filepath.FromSlash(normalized))) + l.mu.Lock() + delete(l.caching, normalized) + l.mu.Unlock() + return nil } func (l *lazyCacheFS) Read(ctx context.Context, vector *fileservice.IOVector) error { if err := l.ensureCached(ctx, vector.FilePath); err != nil { return err } - return l.local.Read(ctx, vector) + return l.readCachedFile(ctx, vector) } func (l *lazyCacheFS) ReadCache(ctx context.Context, vector *fileservice.IOVector) error { if err := l.ensureCached(ctx, vector.FilePath); err != nil { return err } - return l.local.ReadCache(ctx, vector) + return l.readCachedFile(ctx, vector) } func (l *lazyCacheFS) List(ctx context.Context, dirPath string) iter.Seq2[*fileservice.DirEntry, error] { @@ -169,6 +176,52 @@ func (l *lazyCacheFS) cacheFile(ctx context.Context, normalized string, localPat return nil } +func (l *lazyCacheFS) readCachedFile(ctx context.Context, vector *fileservice.IOVector) error { + normalized := stripServiceName(vector.FilePath, l.name) + localPath := filepath.Join(l.root, filepath.FromSlash(normalized)) + + file, err := os.Open(localPath) + if os.IsNotExist(err) { + return moerr.NewFileNotFoundNoCtx(normalized) + } + if err != nil { + return err + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return err + } + fileSize := stat.Size() + + for i, entry := range vector.Entries { + if entry.Size == 0 { + return moerr.NewEmptyRangeNoCtx(normalized) + } + if entry.Offset < 0 { + return moerr.NewUnexpectedEOFNoCtx(normalized) + } + if entry.Size < 0 { + entry.Size = fileSize - entry.Offset + if entry.Size < 0 { + return moerr.NewUnexpectedEOFNoCtx(normalized) + } + } + if _, err := file.Seek(entry.Offset, io.SeekStart); err != nil { + return err + } + if err := entry.ReadFromOSFile(ctx, file, l.local); err != nil { + if err == io.ErrUnexpectedEOF { + return moerr.NewUnexpectedEOFNoCtx(normalized) + } + return err + } + vector.Entries[i] = entry + } + return nil +} + func stripServiceName(path string, service string) string { prefix := strings.ToUpper(service) + ":" if strings.HasPrefix(strings.ToUpper(path), prefix) { diff --git a/pkg/tools/toolfs/lazy_cache_fs_test.go b/pkg/tools/toolfs/lazy_cache_fs_test.go new file mode 100644 index 0000000000000..a3f6185ff34b8 --- /dev/null +++ b/pkg/tools/toolfs/lazy_cache_fs_test.go @@ -0,0 +1,72 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toolfs + +import ( + "bytes" + "context" + "encoding/binary" + "testing" + + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/stretchr/testify/require" +) + +const objectIOMagic = 0xFFFFFFFF + +func TestLazyCacheFSReadsMagicPrefixedRemoteObjectsWithoutLocalChecksum(t *testing.T) { + ctx := context.Background() + remote, err := fileservice.NewMemoryFS("SHARED", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + + payload := make([]byte, 4, 32) + binary.LittleEndian.PutUint32(payload, objectIOMagic) + payload = append(payload, []byte("checkpoint-object-bytes")...) + + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "ckp/meta", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: int64(len(payload)), + Data: payload, + }}, + })) + + fs, _, err := newLazyCacheFS(ctx, remote) + require.NoError(t, err) + defer fs.Close(ctx) + + vec := &fileservice.IOVector{ + FilePath: "SHARED:ckp/meta", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: -1, + }}, + } + require.NoError(t, fs.Read(ctx, vec)) + require.Equal(t, payload, vec.Entries[0].Data) + + var buf bytes.Buffer + cacheVec := &fileservice.IOVector{ + FilePath: "ckp/meta", + Entries: []fileservice.IOEntry{{ + Offset: 4, + Size: 10, + WriterForRead: &buf, + }}, + } + require.NoError(t, fs.ReadCache(ctx, cacheVec)) + require.Equal(t, payload[4:14], buf.Bytes()) +} From 143b0320fd34d73488a3ddba38ffe063ecf30cd8 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 11 Jun 2026 16:42:50 +0800 Subject: [PATCH 07/76] fix: EarliestTS skip empty start/end, cover end; pipeline CSV streaming - EarliestTS/LatestTS: skip entries with empty start/end timestamps - EarliestTS also considers end timestamps, not just start - LatestTS: skip empty end values - Add parallel CSV pipeline with memory backpressure for storage mode - Fix block read memory leak: release now cleans batch vectors Co-Authored-By: Claude Fable 5 --- pkg/tools/checkpointtool/checkpoint_reader.go | 15 +- pkg/tools/checkpointtool/table_dump.go | 747 +++++++++++++++++- pkg/tools/objecttool/object_reader.go | 22 +- 3 files changed, 765 insertions(+), 19 deletions(-) diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index 992c681cd3334..ca308f0e41604 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -148,11 +148,18 @@ func (r *CheckpointReader) Info() *CheckpointInfo { } start := e.GetStart() end := e.GetEnd() - if info.EarliestTS.IsEmpty() || start.LT(&info.EarliestTS) { - info.EarliestTS = start + if !start.IsEmpty() { + if info.EarliestTS.IsEmpty() || start.LT(&info.EarliestTS) { + info.EarliestTS = start + } } - if end.GT(&info.LatestTS) { - info.LatestTS = end + if !end.IsEmpty() { + if info.LatestTS.IsEmpty() || end.GT(&info.LatestTS) { + info.LatestTS = end + } + if info.EarliestTS.IsEmpty() || end.LT(&info.EarliestTS) { + info.EarliestTS = end + } } } return info diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 210c299c8fc38..91319f0673cd7 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -15,17 +15,25 @@ package checkpointtool import ( + "bytes" "context" "fmt" "io" "os" + "runtime" "sort" "strconv" "strings" + "sync" + "sync/atomic" + "time" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/tools/objecttool" ) @@ -38,6 +46,16 @@ const ( moColumnsID = uint64(catalog.MO_COLUMNS_ID) ) +const ( + csvPipelineQueueCapacity = 2 + csvPipelineReaderMax = 4 + csvPipelineReportEvery = 10 * time.Second + csvPipelineMinFreeMemory = 1 << 30 + csvPipelineFreeRatio = 10 + csvPipelineMemoryPoll = 200 * time.Millisecond + csvPipelineWorkerMemory = 256 << 20 +) + type catalogLayout struct { name string moTablesSchema []string @@ -677,13 +695,24 @@ func (r *CheckpointReader) streamTableCSV( dataEntries, tombEntries []*ObjectEntryInfo, options CSVExportOptions, ) error { - tmpFile, err := os.CreateTemp("", "mo-tool-table-dump-*.csv") - if err != nil { - return err + if !options.IncludeMetadata && options.RowOrder == CSVRowOrderStorage { + return r.streamTableCSVPipeline(ctx, w, schema, snapshotTS, dataEntries, tombEntries, options) + } + + needsBuffer := options.IncludeMetadata || options.RowOrder == CSVRowOrderLexical + var output io.Writer = w + var tmpFile *os.File + if needsBuffer { + var err error + tmpFile, err = os.CreateTemp("", "mo-tool-table-dump-*.csv") + if err != nil { + return err + } + tmpName := tmpFile.Name() + defer os.Remove(tmpName) + defer tmpFile.Close() + output = tmpFile } - tmpName := tmpFile.Name() - defer os.Remove(tmpName) - defer tmpFile.Close() header := make([]string, 0, len(schema.Columns)) physicalPositions := make([]int, 0, len(schema.Columns)) @@ -698,7 +727,7 @@ func (r *CheckpointReader) streamTableCSV( var projectedTypes []types.Type if options.IncludeHeader { // header is emitted after we know output mode but before rows - if err := writeSQLLoadCSVRow(tmpFile, nil, header, nil); err != nil { + if err := writeSQLLoadCSVRow(output, nil, header, nil); err != nil { return err } } @@ -711,7 +740,7 @@ func (r *CheckpointReader) streamTableCSV( lexicalRows = append(lexicalRows, exportedCSVRow{values: row, nulls: rowNulls}) return nil } - return writeSQLLoadCSVRow(tmpFile, projectedTypes, row, rowNulls) + return writeSQLLoadCSVRow(output, projectedTypes, row, rowNulls) } stats, err := r.scanLogicalTable(ctx, snapshotTS, dataEntries, tombEntries, @@ -727,12 +756,15 @@ func (r *CheckpointReader) streamTableCSV( if options.RowOrder == CSVRowOrderLexical { sortCSVRowsLexical(lexicalRows) for _, row := range lexicalRows { - if err := writeSQLLoadCSVRow(tmpFile, projectedTypes, row.values, row.nulls); err != nil { + if err := writeSQLLoadCSVRow(output, projectedTypes, row.values, row.nulls); err != nil { return err } } } + if !needsBuffer { + return nil + } if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { return err } @@ -745,6 +777,670 @@ func (r *CheckpointReader) streamTableCSV( return err } +type csvPipelineChunk struct { + objectIdx int + blockIdx int + rows int + data []byte +} + +type csvPipelineObjectJob struct { + objectIdx int + entry *ObjectEntryInfo +} + +type csvPipelineBlock struct { + objectIdx int + blockIdx int + entry *ObjectEntryInfo + projectedTypes []types.Type + relevantTombstones []objectio.ObjectStats + bat *batch.Batch + release func() + commitTSVec *vector.Vector + releaseCommitTS func() +} + +func (b *csvPipelineBlock) releaseBlock() { + if b.releaseCommitTS != nil { + b.releaseCommitTS() + b.releaseCommitTS = nil + } + if b.release != nil { + b.release() + b.release = nil + } +} + +type csvPipelineCounters struct { + totalObjects int64 + processedObjects atomic.Int64 + readQueuedBlocks atomic.Int64 + readQueuedRows atomic.Int64 + readBatches atomic.Int64 + readRows atomic.Int64 + processedBatches atomic.Int64 + visibleRows atomic.Int64 + physicalRows atomic.Int64 + queuedBatches atomic.Int64 + queuedBytes atomic.Int64 + writtenBatches atomic.Int64 + writtenBytes atomic.Int64 + memoryWaits atomic.Int64 + memoryAvailable atomic.Int64 + memoryFloor atomic.Int64 + readNanos atomic.Int64 + processNanos atomic.Int64 + writeNanos atomic.Int64 +} + +type csvPipelineWorkerPlan struct { + readerWorkers int + processorWorkers int + readQueueCapacity int + cpuLimit int + memoryLimit int + memoryTotal uint64 + memoryFree uint64 + memoryFloor uint64 + memoryPerWork uint64 +} + +func (r *CheckpointReader) streamTableCSVPipeline( + ctx context.Context, + w io.Writer, + schema *TableSchema, + snapshotTS types.TS, + dataEntries, tombEntries []*ObjectEntryInfo, + options CSVExportOptions, +) error { + header := make([]string, 0, len(schema.Columns)) + physicalPositions := make([]int, 0, len(schema.Columns)) + for _, col := range schema.Columns { + header = append(header, col.Name) + physicalPos := col.PhysicalPosition + if physicalPos < 0 { + physicalPos = col.Position + } + physicalPositions = append(physicalPositions, physicalPos) + } + if options.IncludeHeader { + if err := writeSQLLoadCSVRow(w, nil, header, nil); err != nil { + return err + } + } + + visibleDataEntries := visibleObjectEntries(dataEntries, snapshotTS) + visibleTombEntries := visibleObjectEntries(tombEntries, snapshotTS) + tombstoneStats := dedupeObjectStats(visibleTombEntries) + counters := &csvPipelineCounters{totalObjects: int64(len(visibleDataEntries))} + workerPlan := csvPipelineWorkerCount(len(visibleDataEntries)) + counters.memoryFloor.Store(int64(workerPlan.memoryFloor)) + counters.memoryAvailable.Store(int64(workerPlan.memoryFree)) + + fmt.Fprintf(os.Stderr, + "csv pipeline: start objects=%d reader_mode=in_processor processor_workers=%d writer_workers=1 cpu_limit=%d memory_limit=%d memory_total=%d memory_free=%d memory_floor=%d memory_per_worker=%d write_queue_capacity=%d report_every=%s\n", + len(visibleDataEntries), + workerPlan.processorWorkers, + workerPlan.cpuLimit, + workerPlan.memoryLimit, + workerPlan.memoryTotal, + workerPlan.memoryFree, + workerPlan.memoryFloor, + workerPlan.memoryPerWork, + csvPipelineQueueCapacity, + csvPipelineReportEvery, + ) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + chunks := make(chan csvPipelineChunk, csvPipelineQueueCapacity) + writerDone := make(chan error, 1) + reportDone := make(chan struct{}) + defer close(reportDone) + go reportCSVPipeline(ctx, reportDone, counters) + go writeCSVChunks(ctx, w, chunks, counters, cancel, writerDone) + + producerErr := r.produceCSVChunks(ctx, chunks, counters, snapshotTS, visibleDataEntries, tombstoneStats, physicalPositions, workerPlan) + close(chunks) + writerErr := <-writerDone + printCSVPipelineReport("finish", counters) + if producerErr != nil { + return producerErr + } + return writerErr +} + +func (r *CheckpointReader) produceCSVChunks( + ctx context.Context, + chunks chan<- csvPipelineChunk, + counters *csvPipelineCounters, + snapshotTS types.TS, + visibleDataEntries []*ObjectEntryInfo, + tombstoneStats []objectio.ObjectStats, + physicalPositions []int, + workerPlan csvPipelineWorkerPlan, +) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + objectJobs := make(chan csvPipelineObjectJob, workerPlan.processorWorkers) + var processorWG sync.WaitGroup + var errOnce sync.Once + var workerErr error + setErr := func(err error) { + if err == nil { + return + } + errOnce.Do(func() { + workerErr = err + cancel() + }) + } + + processorWG.Add(workerPlan.processorWorkers) + for workerID := 0; workerID < workerPlan.processorWorkers; workerID++ { + go func() { + defer processorWG.Done() + for job := range objectJobs { + if err := ctx.Err(); err != nil { + setErr(err) + return + } + if err := r.processCSVObjectChunks(ctx, chunks, counters, snapshotTS, tombstoneStats, physicalPositions, job); err != nil { + setErr(err) + return + } + } + }() + } + + for objectIdx, entry := range visibleDataEntries { + select { + case objectJobs <- csvPipelineObjectJob{objectIdx: objectIdx, entry: entry}: + case <-ctx.Done(): + close(objectJobs) + processorWG.Wait() + if workerErr != nil { + return workerErr + } + return ctx.Err() + } + } + close(objectJobs) + processorWG.Wait() + if workerErr != nil { + return workerErr + } + return ctx.Err() +} + +func (r *CheckpointReader) processCSVObjectChunks( + ctx context.Context, + chunks chan<- csvPipelineChunk, + counters *csvPipelineCounters, + snapshotTS types.TS, + tombstoneStats []objectio.ObjectStats, + physicalPositions []int, + job csvPipelineObjectJob, +) error { + entry := job.entry + if err := ctx.Err(); err != nil { + return err + } + objName := entry.ObjectStats.ObjectName().String() + reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) + if err != nil { + if isDataFileNotFound(err) { + counters.processedObjects.Add(1) + return nil + } + return err + } + defer reader.Close() + + projectedTypes := buildProjectedTypes(reader.Columns(), physicalPositions) + relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.ObjectStats.ObjectName().ObjectId(), tombstoneStats) + if err != nil { + return err + } + + for blockIdx := 0; blockIdx < int(entry.ObjectStats.BlkCnt()); blockIdx++ { + if err := ctx.Err(); err != nil { + return err + } + if err := waitForCSVMemory(ctx, counters); err != nil { + return err + } + + start := time.Now() + bat, release, err := reader.ReadBlock(ctx, uint32(blockIdx)) + if err != nil { + return err + } + commitTSVec, releaseCommitTS, err := reader.ReadBlockCommitTS(ctx, uint32(blockIdx)) + counters.readNanos.Add(time.Since(start).Nanoseconds()) + if err != nil { + release() + return err + } + if bat.RowCount() == 0 { + if releaseCommitTS != nil { + releaseCommitTS() + } + release() + continue + } + + block := csvPipelineBlock{ + objectIdx: job.objectIdx, + blockIdx: blockIdx, + entry: entry, + projectedTypes: projectedTypes, + relevantTombstones: relevantTombstones, + bat: bat, + release: release, + commitTSVec: commitTSVec, + releaseCommitTS: releaseCommitTS, + } + counters.readBatches.Add(1) + counters.readRows.Add(int64(bat.RowCount())) + chunk, err := r.buildCSVChunkForBlock(ctx, block, snapshotTS, physicalPositions, counters) + if err != nil { + return err + } + if len(chunk.data) == 0 { + continue + } + counters.queuedBatches.Add(1) + counters.queuedBytes.Add(int64(len(chunk.data))) + select { + case chunks <- chunk: + case <-ctx.Done(): + counters.queuedBatches.Add(-1) + counters.queuedBytes.Add(-int64(len(chunk.data))) + return ctx.Err() + } + } + + counters.processedObjects.Add(1) + return nil +} + +func (r *CheckpointReader) readCSVObjectBlocks( + ctx context.Context, + blocks chan<- csvPipelineBlock, + counters *csvPipelineCounters, + snapshotTS types.TS, + tombstoneStats []objectio.ObjectStats, + physicalPositions []int, + job csvPipelineObjectJob, +) error { + entry := job.entry + if err := ctx.Err(); err != nil { + return err + } + objName := entry.ObjectStats.ObjectName().String() + reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) + if err != nil { + if isDataFileNotFound(err) { + counters.processedObjects.Add(1) + return nil + } + return err + } + + projectedTypes := buildProjectedTypes(reader.Columns(), physicalPositions) + relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.ObjectStats.ObjectName().ObjectId(), tombstoneStats) + if err != nil { + _ = reader.Close() + return err + } + + for blockIdx := 0; blockIdx < int(entry.ObjectStats.BlkCnt()); blockIdx++ { + if err := ctx.Err(); err != nil { + _ = reader.Close() + return err + } + if err := waitForCSVMemory(ctx, counters); err != nil { + _ = reader.Close() + return err + } + + start := time.Now() + bat, release, err := reader.ReadBlock(ctx, uint32(blockIdx)) + if err != nil { + _ = reader.Close() + return err + } + commitTSVec, releaseCommitTS, err := reader.ReadBlockCommitTS(ctx, uint32(blockIdx)) + counters.readNanos.Add(time.Since(start).Nanoseconds()) + if err != nil { + release() + _ = reader.Close() + return err + } + if bat.RowCount() == 0 { + if releaseCommitTS != nil { + releaseCommitTS() + } + release() + continue + } + + block := csvPipelineBlock{ + objectIdx: job.objectIdx, + blockIdx: blockIdx, + entry: entry, + projectedTypes: projectedTypes, + relevantTombstones: relevantTombstones, + bat: bat, + release: release, + commitTSVec: commitTSVec, + releaseCommitTS: releaseCommitTS, + } + counters.readBatches.Add(1) + counters.readRows.Add(int64(bat.RowCount())) + counters.readQueuedBlocks.Add(1) + counters.readQueuedRows.Add(int64(bat.RowCount())) + select { + case blocks <- block: + case <-ctx.Done(): + counters.readQueuedBlocks.Add(-1) + counters.readQueuedRows.Add(-int64(bat.RowCount())) + block.releaseBlock() + _ = reader.Close() + return ctx.Err() + } + } + + if err := reader.Close(); err != nil { + return err + } + counters.processedObjects.Add(1) + return nil +} + +func (r *CheckpointReader) processCSVBlockChunk( + ctx context.Context, + chunks chan<- csvPipelineChunk, + counters *csvPipelineCounters, + snapshotTS types.TS, + physicalPositions []int, + block csvPipelineBlock, +) error { + if err := ctx.Err(); err != nil { + block.releaseBlock() + return err + } + chunk, err := r.buildCSVChunkForBlock(ctx, block, snapshotTS, physicalPositions, counters) + if err != nil { + return err + } + if len(chunk.data) == 0 { + return nil + } + counters.queuedBatches.Add(1) + counters.queuedBytes.Add(int64(len(chunk.data))) + select { + case chunks <- chunk: + return nil + case <-ctx.Done(): + counters.queuedBatches.Add(-1) + counters.queuedBytes.Add(-int64(len(chunk.data))) + return ctx.Err() + } +} + +func csvPipelineWorkerCount(objects int) csvPipelineWorkerPlan { + cpuLimit := runtime.GOMAXPROCS(0) + if cpuLimit < 1 { + cpuLimit = 1 + } + + total, free, ok := readSystemMemory() + floor := csvPipelineMemoryFloorFromTotal(total, ok) + memoryLimit := cpuLimit + if ok { + memoryLimit = 1 + if free > floor { + budget := free - floor + memoryLimit = int(budget / csvPipelineWorkerMemory) + if memoryLimit < 1 { + memoryLimit = 1 + } + } + } + processorWorkers := cpuLimit + if memoryLimit < processorWorkers { + processorWorkers = memoryLimit + } + if objects > 0 && objects < processorWorkers { + processorWorkers = objects + } + if processorWorkers < 1 { + processorWorkers = 1 + } + + readerWorkers := csvPipelineReaderMax + if objects > 0 && objects < readerWorkers { + readerWorkers = objects + } + if processorWorkers < readerWorkers { + readerWorkers = processorWorkers + } + if readerWorkers < 1 { + readerWorkers = 1 + } + readQueueCapacity := readerWorkers * 2 + if processorWorkers < readQueueCapacity { + readQueueCapacity = processorWorkers + } + if readQueueCapacity < 1 { + readQueueCapacity = 1 + } + return csvPipelineWorkerPlan{ + readerWorkers: readerWorkers, + processorWorkers: processorWorkers, + readQueueCapacity: readQueueCapacity, + cpuLimit: cpuLimit, + memoryLimit: memoryLimit, + memoryTotal: total, + memoryFree: free, + memoryFloor: floor, + memoryPerWork: csvPipelineWorkerMemory, + } +} + +func csvPipelineMemoryFloorFromTotal(total uint64, ok bool) uint64 { + if !ok || total == 0 { + return csvPipelineMinFreeMemory + } + floor := total / csvPipelineFreeRatio + if floor < csvPipelineMinFreeMemory { + return csvPipelineMinFreeMemory + } + return floor +} + +func waitForCSVMemory(ctx context.Context, counters *csvPipelineCounters) error { + floor := uint64(counters.memoryFloor.Load()) + for { + _, available, ok := readSystemMemory() + if !ok { + return nil + } + counters.memoryAvailable.Store(int64(available)) + if available >= floor { + return nil + } + counters.memoryWaits.Add(1) + timer := time.NewTimer(csvPipelineMemoryPoll) + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + } + } +} + +func readSystemMemory() (total uint64, available uint64, ok bool) { + data, err := os.ReadFile("/proc/meminfo") + if err != nil { + return 0, 0, false + } + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + value, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + continue + } + bytes := value * 1024 + switch fields[0] { + case "MemTotal:": + total = bytes + case "MemAvailable:": + available = bytes + } + } + return total, available, total > 0 && available > 0 +} + +func (r *CheckpointReader) buildCSVChunkForBlock( + ctx context.Context, + block csvPipelineBlock, + snapshotTS types.TS, + physicalPositions []int, + counters *csvPipelineCounters, +) (csvPipelineChunk, error) { + start := time.Now() + defer func() { + counters.processNanos.Add(time.Since(start).Nanoseconds()) + }() + defer block.releaseBlock() + + bat := block.bat + if bat.RowCount() == 0 { + return csvPipelineChunk{}, nil + } + counters.physicalRows.Add(int64(bat.RowCount())) + + var commitTSs []types.TS + if block.commitTSVec != nil { + commitTSs = vector.MustFixedColWithTypeCheck[types.TS](block.commitTSVec) + } + + deleteMask, err := r.buildDeleteMaskForBlock(ctx, &snapshotTS, block.entry.ObjectStats, uint16(block.blockIdx), block.relevantTombstones) + if err != nil { + return csvPipelineChunk{}, err + } + if deleteMask.IsValid() { + defer deleteMask.Release() + } + + var buf bytes.Buffer + rows := 0 + for rowIdx := 0; rowIdx < bat.RowCount(); rowIdx++ { + if commitTSs != nil && rowIdx < len(commitTSs) && + !block.commitTSVec.IsNull(uint64(rowIdx)) && + commitTSs[rowIdx].GT(&snapshotTS) { + continue + } + if deleteMask.IsValid() && deleteMask.Contains(uint64(rowIdx)) { + continue + } + if err := writeProjectedCSVRowFromVecs(&buf, block.projectedTypes, bat.Vecs, physicalPositions, rowIdx); err != nil { + return csvPipelineChunk{}, err + } + rows++ + } + + if rows == 0 { + return csvPipelineChunk{}, nil + } + counters.processedBatches.Add(1) + counters.visibleRows.Add(int64(rows)) + return csvPipelineChunk{ + objectIdx: block.objectIdx, + blockIdx: block.blockIdx, + rows: rows, + data: buf.Bytes(), + }, nil +} + +func writeCSVChunks( + ctx context.Context, + w io.Writer, + chunks <-chan csvPipelineChunk, + counters *csvPipelineCounters, + cancel context.CancelFunc, + done chan<- error, +) { + for chunk := range chunks { + if err := ctx.Err(); err != nil { + done <- err + return + } + start := time.Now() + if _, err := w.Write(chunk.data); err != nil { + cancel() + done <- err + return + } + counters.writeNanos.Add(time.Since(start).Nanoseconds()) + counters.queuedBatches.Add(-1) + counters.queuedBytes.Add(-int64(len(chunk.data))) + counters.writtenBatches.Add(1) + counters.writtenBytes.Add(int64(len(chunk.data))) + } + done <- nil +} + +func reportCSVPipeline(ctx context.Context, done <-chan struct{}, counters *csvPipelineCounters) { + ticker := time.NewTicker(csvPipelineReportEvery) + defer ticker.Stop() + for { + select { + case <-ticker.C: + printCSVPipelineReport("progress", counters) + case <-done: + return + case <-ctx.Done(): + return + } + } +} + +func printCSVPipelineReport(stage string, counters *csvPipelineCounters) { + fmt.Fprintf(os.Stderr, + "csv pipeline: %s objects=%d/%d read_batches=%d read_queue_blocks=%d read_queue_rows=%d processed_batches=%d write_queue_batches=%d write_queue_bytes=%d visible_rows=%d physical_rows=%d written_batches=%d written_bytes=%d read_ms=%d process_ms=%d write_ms=%d mem_available=%d mem_floor=%d mem_waits=%d\n", + stage, + counters.processedObjects.Load(), + counters.totalObjects, + counters.readBatches.Load(), + counters.readQueuedBlocks.Load(), + counters.readQueuedRows.Load(), + counters.processedBatches.Load(), + counters.queuedBatches.Load(), + counters.queuedBytes.Load(), + counters.visibleRows.Load(), + counters.physicalRows.Load(), + counters.writtenBatches.Load(), + counters.writtenBytes.Load(), + counters.readNanos.Load()/int64(time.Millisecond), + counters.processNanos.Load()/int64(time.Millisecond), + counters.writeNanos.Load()/int64(time.Millisecond), + counters.memoryAvailable.Load(), + counters.memoryFloor.Load(), + counters.memoryWaits.Load(), + ) +} + // WriteCSV writes a LogicalTableView with the given schema as CSV to w. func WriteCSV(w io.Writer, schema *TableSchema, view *LogicalTableView, opts ...CSVExportOption) error { options := resolveCSVExportOptions(opts) @@ -845,6 +1541,39 @@ func writeSQLLoadCSVRow(w io.Writer, colTypes []types.Type, fields []string, nul return err } +func writeProjectedCSVRowFromVecs( + w io.Writer, + colTypes []types.Type, + vecs []*vector.Vector, + physicalPositions []int, + rowIdx int, +) error { + for i, pos := range physicalPositions { + if i > 0 { + if _, err := io.WriteString(w, ","); err != nil { + return err + } + } + typ := types.Type{} + if i < len(colTypes) { + typ = colTypes[i] + } + field := "" + isNull := true + if pos >= 0 && pos < len(vecs) && vecs[pos] != nil { + isNull = vecs[pos].IsNull(uint64(rowIdx)) + if !isNull { + field = vecValueToString(vecs[pos], rowIdx) + } + } + if err := writeSQLLoadCSVField(w, typ, field, isNull); err != nil { + return err + } + } + _, err := io.WriteString(w, "\n") + return err +} + func writeSQLLoadCSVField(w io.Writer, typ types.Type, field string, isNull bool) error { if isNull { _, err := io.WriteString(w, `\N`) diff --git a/pkg/tools/objecttool/object_reader.go b/pkg/tools/objecttool/object_reader.go index c29f3164fa5aa..a98fa89dc0384 100644 --- a/pkg/tools/objecttool/object_reader.go +++ b/pkg/tools/objecttool/object_reader.go @@ -188,7 +188,7 @@ func (r *ObjectReader) ReadBlock(ctx context.Context, blockIdx uint32) (*batch.B return nil, nil, err } - release := func() { + releaseIOVector := func() { objectio.ReleaseIOVector(&ioVectors) } @@ -197,13 +197,18 @@ func (r *ObjectReader) ReadBlock(ctx context.Context, blockIdx uint32) (*batch.B for i := range colIdxs { obj, err := objectio.Decode(ioVectors.Entries[i].CachedData.Bytes()) if err != nil { - release() + releaseIOVector() return nil, nil, err } bat.Vecs[i] = obj.(*vector.Vector) bat.SetRowCount(bat.Vecs[i].Length()) } + release := func() { + bat.Clean(r.mp) + releaseIOVector() + } + return bat, release, nil } @@ -225,24 +230,29 @@ func (r *ObjectReader) ReadBlockCommitTS(ctx context.Context, blockIdx uint32) ( return nil, nil, err } - release := func() { + releaseIOVector := func() { objectio.ReleaseIOVector(&ioVectors) } obj, err := objectio.Decode(ioVectors.Entries[0].CachedData.Bytes()) if err != nil { - release() + releaseIOVector() return nil, nil, err } vec := obj.(*vector.Vector) if vec.GetType().Oid != types.T_TS { - release() + releaseIOVector() return nil, nil, moerr.NewInternalErrorf(ctx, "commit TS column type mismatch: expected TS, got %s", vec.GetType().String()) } if vec.GetNulls().GetCardinality() == vec.Length() { - release() + vec.Free(r.mp) + releaseIOVector() return nil, func() {}, nil } + release := func() { + vec.Free(r.mp) + releaseIOVector() + } return vec, release, nil } From cbaecb7957972d31bc48f70294cd1fa9508884c3 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 11 Jun 2026 17:17:50 +0800 Subject: [PATCH 08/76] fix: catalog table listing with schema-projected mo_tables; normalize DDL names - ListCatalogTables: merge catalog entries from raw and schema-projected mo_tables views - add logicalViewDataOffset for header detection without hardcoding meta width - normalizeCreateTableDDLName: replace catalog DDL table name with real name - add SQL token parser for CREATE TABLE name normalization (CREATE TABLE IF NOT EXISTS) - add checkpoint_test.go Co-Authored-By: Claude Fable 5 --- cmd/mo-object-tool/ckp/checkpoint.go | 101 ++++++++++++++ cmd/mo-object-tool/ckp/checkpoint_test.go | 71 ++++++++++ pkg/tools/checkpointtool/table_dump.go | 155 +++++++++++++++++++--- 3 files changed, 306 insertions(+), 21 deletions(-) create mode 100644 cmd/mo-object-tool/ckp/checkpoint_test.go diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 5fa3c42eae5ee..6dff44e304175 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -721,6 +721,7 @@ func writeRestoreScript( if err != nil { return "", fmt.Errorf("show create table %d: %w", table.TableID, err) } + ddl = normalizeCreateTableDDLName(ddl, table) if _, err := fmt.Fprintln(f, strings.TrimRight(ddl, " ;\n\t")); err != nil { return "", err } @@ -772,6 +773,106 @@ func quoteSQLString(s string) string { return "'" + s + "'" } +func normalizeCreateTableDDLName(ddl string, table checkpointtool.TableCatalogEntry) string { + nameStart, nameEnd, ok := createTableNameRange(ddl) + if !ok { + return ddl + } + target := quoteSQLIdent(table.TableName) + if table.DatabaseName != "" { + target = quoteSQLIdent(table.DatabaseName) + "." + target + } + return ddl[:nameStart] + target + ddl[nameEnd:] +} + +func createTableNameRange(sql string) (int, int, bool) { + i, ok := consumeSQLKeyword(sql, 0, "create") + if !ok { + return 0, 0, false + } + i, ok = consumeSQLKeyword(sql, i, "table") + if !ok { + return 0, 0, false + } + if next, ok := consumeSQLKeyword(sql, i, "if"); ok { + if next, ok = consumeSQLKeyword(sql, next, "not"); ok { + if next, ok = consumeSQLKeyword(sql, next, "exists"); ok { + i = next + } + } + } + i = skipSQLSpace(sql, i) + nameStart := i + i, ok = consumeSQLIdentifier(sql, i) + if !ok { + return 0, 0, false + } + if j := skipSQLSpace(sql, i); j < len(sql) && sql[j] == '.' { + j = skipSQLSpace(sql, j+1) + if end, ok := consumeSQLIdentifier(sql, j); ok { + i = end + } + } + return nameStart, i, true +} + +func consumeSQLKeyword(sql string, i int, keyword string) (int, bool) { + i = skipSQLSpace(sql, i) + if len(sql)-i < len(keyword) || !strings.EqualFold(sql[i:i+len(keyword)], keyword) { + return i, false + } + end := i + len(keyword) + if end < len(sql) && isSQLIdentByte(sql[end]) { + return i, false + } + return end, true +} + +func skipSQLSpace(sql string, i int) int { + for i < len(sql) { + switch sql[i] { + case ' ', '\t', '\n', '\r': + i++ + default: + return i + } + } + return i +} + +func consumeSQLIdentifier(sql string, i int) (int, bool) { + if i >= len(sql) { + return i, false + } + if sql[i] == '`' { + i++ + for i < len(sql) { + if sql[i] != '`' { + i++ + continue + } + if i+1 < len(sql) && sql[i+1] == '`' { + i += 2 + continue + } + return i + 1, true + } + return i, false + } + start := i + for i < len(sql) && isSQLIdentByte(sql[i]) { + i++ + } + return i, i > start +} + +func isSQLIdentByte(b byte) bool { + return b == '_' || b == '$' || + (b >= '0' && b <= '9') || + (b >= 'a' && b <= 'z') || + (b >= 'A' && b <= 'Z') +} + func dumpTablesConcurrently( ctx context.Context, reader *checkpointtool.CheckpointReader, diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go new file mode 100644 index 0000000000000..698fcec6e8e6f --- /dev/null +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -0,0 +1,71 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ckp + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" + "github.com/stretchr/testify/assert" +) + +func TestNormalizeCreateTableDDLName(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "compat_ckp", + TableName: "employees", + } + + tests := []struct { + name string + ddl string + want string + }{ + { + name: "plain table name", + ddl: "CREATE TABLE employees_copy_123 (id INT)", + want: "CREATE TABLE `compat_ckp`.`employees` (id INT)", + }, + { + name: "qualified table name", + ddl: "CREATE TABLE `compat_ckp`.`employees_copy_123` (id INT)", + want: "CREATE TABLE `compat_ckp`.`employees` (id INT)", + }, + { + name: "if not exists", + ddl: "CREATE TABLE IF NOT EXISTS old_name (id INT)", + want: "CREATE TABLE IF NOT EXISTS `compat_ckp`.`employees` (id INT)", + }, + { + name: "escaped target name", + ddl: "CREATE TABLE old_name (id INT)", + want: "CREATE TABLE `compat``ckp`.`employees` (id INT)", + }, + } + + tests[3].want = "CREATE TABLE `compat``ckp`.`employees` (id INT)" + tests[3].ddl = "CREATE TABLE old_name (id INT)" + tests[3].name = "escaped database name" + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.name == "escaped database name" { + table.DatabaseName = "compat`ckp" + } else { + table.DatabaseName = "compat_ckp" + } + assert.Equal(t, tt.want, normalizeCreateTableDDLName(tt.ddl, table)) + }) + } +} diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 91319f0673cd7..38f95b16bc14d 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -17,6 +17,7 @@ package checkpointtool import ( "bytes" "context" + "encoding/csv" "fmt" "io" "os" @@ -461,7 +462,8 @@ func fallbackCatalogColIndex(view *LogicalTableView, tableID uint64, colName str if idx := view.columnDataIndex(colName); idx >= 0 { return idx } - layout, offset, ok := inferCatalogLayout(len(view.Headers)-logicalViewMetaCols, tableID) + dataOffset := logicalViewDataOffset(view) + layout, offset, ok := inferCatalogLayout(len(view.Headers)-dataOffset, tableID) if !ok { return -1 } @@ -512,7 +514,7 @@ func (r *CheckpointReader) ReadTableSchema( // Resolve column positions by name (handles hidden column offset) relIDCol := fallbackCatalogColIndex(moTablesView, moTablesID, "rel_id") for _, row := range moTablesView.Rows { - dataRow := row[logicalViewMetaCols:] + dataRow := row[logicalViewDataOffset(moTablesView):] if relIDCol < 0 || relIDCol >= len(dataRow) { continue } @@ -654,11 +656,21 @@ func (r *CheckpointReader) ListCatalogTables( snapshotTS types.TS, opts TableListOptions, ) ([]TableCatalogEntry, error) { + tables, projectedErr := r.listCatalogTablesFromProjectedMoTables(ctx, snapshotTS) moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) - if err != nil { + if err != nil && projectedErr != nil { return nil, err } - tables := buildCatalogTablesFromMoTablesRows(moTablesView) + if projectedErr != nil || len(tables) == 0 { + tables = nil + } + if moTablesView != nil { + tables = mergeCatalogTableEntries(tables, buildCatalogTablesFromMoTablesRows(moTablesView)) + if schema := r.ReadTableSchema(ctx, moTablesID, snapshotTS, moTablesView); len(schema.Columns) > 0 { + projected := MergeLogicalViewWithSchema(moTablesView, schema) + tables = mergeCatalogTableEntries(tables, buildCatalogTablesFromMoTablesRows(projected)) + } + } filtered := make([]TableCatalogEntry, 0, len(tables)) for _, table := range tables { if opts.AccountID != nil && table.AccountID != *opts.AccountID { @@ -687,6 +699,88 @@ func (r *CheckpointReader) ListCatalogTables( return filtered, nil } +func (r *CheckpointReader) listCatalogTablesFromProjectedMoTables( + ctx context.Context, + snapshotTS types.TS, +) ([]TableCatalogEntry, error) { + var buf bytes.Buffer + if err := r.DumpTableCSVComposed(ctx, &buf, moTablesID, snapshotTS, WithCSVHeader(true), WithCSVMetaComments(false)); err != nil { + return nil, err + } + reader := csv.NewReader(bytes.NewReader(buf.Bytes())) + reader.FieldsPerRecord = -1 + records, err := reader.ReadAll() + if err != nil { + return nil, err + } + if len(records) == 0 { + return nil, nil + } + return buildCatalogTablesFromCSVRecords(records[0], records[1:]), nil +} + +func buildCatalogTablesFromCSVRecords(header []string, rows [][]string) []TableCatalogEntry { + index := make(map[string]int, len(header)) + for i, name := range header { + index[name] = i + } + col := func(name string) int { + if idx, ok := index[name]; ok { + return idx + } + return -1 + } + return buildCatalogTablesFromMoTablesRowsAt( + &LogicalTableView{ + Headers: header, + Rows: rows, + }, + col("rel_id"), + col("relname"), + col("reldatabase"), + col("reldatabase_id"), + col("relkind"), + col("account_id"), + ) +} + +func mergeCatalogTableEntries(base []TableCatalogEntry, extra []TableCatalogEntry) []TableCatalogEntry { + if len(extra) == 0 { + return base + } + seen := make(map[uint64]int, len(base)+len(extra)) + for i, table := range base { + seen[table.TableID] = i + } + for _, table := range extra { + if idx, ok := seen[table.TableID]; ok { + if betterCatalogTableEntry(table, base[idx]) { + base[idx] = table + } + continue + } + seen[table.TableID] = len(base) + base = append(base, table) + } + return base +} + +func betterCatalogTableEntry(candidate, current TableCatalogEntry) bool { + if !validCatalogName(current.TableName) && validCatalogName(candidate.TableName) { + return true + } + if !validCatalogName(current.DatabaseName) && validCatalogName(candidate.DatabaseName) { + return true + } + if current.DatabaseID == 0 && candidate.DatabaseID != 0 { + return true + } + if current.RelKind == "" && candidate.RelKind != "" { + return true + } + return false +} + func (r *CheckpointReader) streamTableCSV( ctx context.Context, w io.Writer, @@ -1694,7 +1788,7 @@ func writeCSVMetadata(w io.Writer, schema *TableSchema, stats logicalTableStats) // and filters data rows to only include visible (non-hidden) columns by their physical position. // Returns a new LogicalTableView with merged headers and filtered data rows. func MergeLogicalViewWithSchema(view *LogicalTableView, schema *TableSchema) *LogicalTableView { - dataWidth := len(view.Headers) - logicalViewMetaCols + dataWidth := len(view.Headers) - logicalViewDataOffset(view) if dataWidth < 0 { dataWidth = 0 } @@ -1725,10 +1819,11 @@ func MergeLogicalViewWithSchema(view *LogicalTableView, schema *TableSchema) *Lo // Extract data rows: pick only visible columns by their physical position newRows := make([][]string, len(view.Rows)) + dataOffset := logicalViewDataOffset(view) for i, row := range view.Rows { newRow := make([]string, len(colMap)) for j, pos := range colMap { - dataIdx := logicalViewMetaCols + pos + dataIdx := dataOffset + pos if dataIdx < len(row) { newRow[j] = row[dataIdx] } @@ -1750,7 +1845,7 @@ func MergeLogicalViewWithSchema(view *LogicalTableView, schema *TableSchema) *Lo // mo_tables columns: rel_id, relname, reldatabase, reldatabase_id, ..., rel_createsql func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *TableSchema { schema := &TableSchema{} - dataRow := fullRow[logicalViewMetaCols:] + dataRow := fullRow[logicalViewDataOffset(view):] relNameIdx := fallbackCatalogColIndex(view, moTablesID, "relname") if relNameIdx < len(dataRow) { @@ -1776,7 +1871,7 @@ func findCreateSQLFromMoTables(view *LogicalTableView, tableID uint64) string { return "" } for _, fullRow := range view.Rows { - row := fullRow[logicalViewMetaCols:] + row := fullRow[logicalViewDataOffset(view):] if relIDCol >= len(row) || createSQLCol >= len(row) { continue } @@ -1793,7 +1888,7 @@ func findCreateSQLFromMoTables(view *LogicalTableView, tableID uint64) string { return ddl } - dataWidth := len(view.Headers) - logicalViewMetaCols + dataWidth := len(view.Headers) - logicalViewDataOffset(view) for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { relIDCol = catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset) createSQLCol = catalogColIndexForLayout(match.layout, moTablesID, "rel_createsql", match.offset) @@ -1805,7 +1900,8 @@ func findCreateSQLFromMoTables(view *LogicalTableView, tableID uint64) string { } func buildCatalogTablesFromMoTablesRows(view *LogicalTableView) []TableCatalogEntry { - var best []TableCatalogEntry + seen := make(map[uint64]struct{}) + merged := make([]TableCatalogEntry, 0) try := func(relIDCol, relNameCol, relDBCol, relDBIDCol, relKindCol, accountIDCol int) { tables := buildCatalogTablesFromMoTablesRowsAt( view, @@ -1816,8 +1912,12 @@ func buildCatalogTablesFromMoTablesRows(view *LogicalTableView) []TableCatalogEn relKindCol, accountIDCol, ) - if len(tables) > len(best) { - best = tables + for _, table := range tables { + if _, ok := seen[table.TableID]; ok { + continue + } + seen[table.TableID] = struct{}{} + merged = append(merged, table) } } @@ -1829,7 +1929,7 @@ func buildCatalogTablesFromMoTablesRows(view *LogicalTableView) []TableCatalogEn fallbackCatalogColIndex(view, moTablesID, "relkind"), fallbackCatalogColIndex(view, moTablesID, "account_id"), ) - dataWidth := len(view.Headers) - logicalViewMetaCols + dataWidth := len(view.Headers) - logicalViewDataOffset(view) for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { try( catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset), @@ -1840,7 +1940,7 @@ func buildCatalogTablesFromMoTablesRows(view *LogicalTableView) []TableCatalogEn catalogColIndexForLayout(match.layout, moTablesID, "account_id", match.offset), ) } - return best + return merged } func buildCatalogTablesFromMoTablesRowsAt( @@ -1859,7 +1959,7 @@ func buildCatalogTablesFromMoTablesRowsAt( seen := make(map[uint64]struct{}) tables := make([]TableCatalogEntry, 0) for _, fullRow := range view.Rows { - row := fullRow[logicalViewMetaCols:] + row := fullRow[logicalViewDataOffset(view):] if relIDCol >= len(row) || relNameCol >= len(row) || relDBCol >= len(row) { continue } @@ -1945,7 +2045,7 @@ func buildColumnsFromMoColumnsRows(view *LogicalTableView, tableID uint64) []Tab } } - dataWidth := len(view.Headers) - logicalViewMetaCols + dataWidth := len(view.Headers) - logicalViewDataOffset(view) for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { relnameIDCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_relname_id", match.offset) nameCol = catalogColIndexForLayout(match.layout, moColumnsID, "attname", match.offset) @@ -1974,7 +2074,7 @@ func buildColumnsFromMoColumnsRowsAt( tableIDStr := fmt.Sprintf("%d", tableID) var cols []TableColumn for _, fullRow := range view.Rows { - row := fullRow[logicalViewMetaCols:] + row := fullRow[logicalViewDataOffset(view):] // Skip hidden columns if hiddenCol >= 0 && hiddenCol < len(row) { @@ -2111,7 +2211,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) strin } } - dataWidth := len(view.Headers) - logicalViewMetaCols + dataWidth := len(view.Headers) - logicalViewDataOffset(view) for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { relnameIDCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_relname_id", match.offset) nameCol = catalogColIndexForLayout(match.layout, moColumnsID, "attname", match.offset) @@ -2144,7 +2244,7 @@ func buildCreateTableFromMoColumnsAt( var cols []colInfo for _, fullRow := range view.Rows { - row := fullRow[logicalViewMetaCols:] + row := fullRow[logicalViewDataOffset(view):] if hiddenCol >= 0 && hiddenCol < len(row) { if row[hiddenCol] == "1" || row[hiddenCol] == "true" { continue @@ -2215,10 +2315,23 @@ func hardcodedCreateTableForLayout(tableID uint64, layout catalogLayout) string // columnDataIndex returns the data-column index (0-based, after stripping meta columns) // for the named column, or -1 if not found. func (v *LogicalTableView) columnDataIndex(colName string) int { - for i := logicalViewMetaCols; i < len(v.Headers); i++ { + dataOffset := logicalViewDataOffset(v) + for i := dataOffset; i < len(v.Headers); i++ { if v.Headers[i] == colName { - return i - logicalViewMetaCols + return i - dataOffset } } return -1 } + +func logicalViewDataOffset(view *LogicalTableView) int { + if view == nil || len(view.Headers) < logicalViewMetaCols { + return 0 + } + for i, h := range logicalTableViewMetaHeaders { + if view.Headers[i] != h { + return 0 + } + } + return logicalViewMetaCols +} From 985b1feb41363da5f0cceeec67c60e05aee3f2e8 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 12 Jun 2026 11:37:39 +0800 Subject: [PATCH 09/76] fix: add parallel 'true' to LOAD DATA in restore.sql Co-Authored-By: Claude Fable 5 --- cmd/mo-object-tool/ckp/checkpoint.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 6dff44e304175..776f1813e21fe 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -735,6 +735,9 @@ func writeRestoreScript( if _, err := fmt.Fprintf(f, "INTO TABLE %s\n", quoteSQLIdent(table.TableName)); err != nil { return "", err } + if _, err := fmt.Fprintln(f, "parallel 'true'"); err != nil { + return "", err + } if _, err := fmt.Fprintln(f, "FIELDS TERMINATED BY ','"); err != nil { return "", err } From 9c9f4c7da1440e8f73520bfa739b6710af877cf5 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 12 Jun 2026 11:43:49 +0800 Subject: [PATCH 10/76] fix: move parallel 'true' after LINES TERMINATED BY Co-Authored-By: Claude Fable 5 --- cmd/mo-object-tool/ckp/checkpoint.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 776f1813e21fe..3bfe2d68f097f 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -735,9 +735,6 @@ func writeRestoreScript( if _, err := fmt.Fprintf(f, "INTO TABLE %s\n", quoteSQLIdent(table.TableName)); err != nil { return "", err } - if _, err := fmt.Fprintln(f, "parallel 'true'"); err != nil { - return "", err - } if _, err := fmt.Fprintln(f, "FIELDS TERMINATED BY ','"); err != nil { return "", err } @@ -747,6 +744,9 @@ func writeRestoreScript( if _, err := fmt.Fprintln(f, "LINES TERMINATED BY '\\n'"); err != nil { return "", err } + if _, err := fmt.Fprintln(f, "parallel 'true'"); err != nil { + return "", err + } if csvHasHeader { if _, err := fmt.Fprintln(f, "IGNORE 1 LINES"); err != nil { return "", err From 9ec570dda7537c198dc758b1ca6c4bb25e773152 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 12 Jun 2026 13:56:34 +0800 Subject: [PATCH 11/76] fix: move parallel 'true' after IGNORE N LINES clause Co-Authored-By: Claude Fable 5 --- cmd/mo-object-tool/ckp/checkpoint.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 3bfe2d68f097f..d44d4b11dc7a6 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -20,6 +20,7 @@ import ( "io" "os" "path/filepath" + "runtime/pprof" "sort" "strconv" "strings" @@ -331,6 +332,7 @@ func dumpCommand(storage *toolfs.StorageOptions) *cobra.Command { loadScript bool noLoad bool rowOrder string + cpuProfile string ) cmd := &cobra.Command{ @@ -358,6 +360,18 @@ Examples: dir = args[0] } + if cpuProfile != "" { + f, err := os.Create(cpuProfile) + if err != nil { + return fmt.Errorf("create cpuprofile: %w", err) + } + defer f.Close() + if err := pprof.StartCPUProfile(f); err != nil { + return fmt.Errorf("start cpuprofile: %w", err) + } + defer pprof.StopCPUProfile() + } + accountIDSet := cmd.Flags().Changed("account-id") tableName = strings.TrimSpace(tableName) batchDump := tableID == 0 && tableName == "" @@ -539,6 +553,7 @@ Examples: cmd.Flags().BoolVar(&loadScript, "load-script", false, "Generate restore.sql with CREATE DATABASE, CREATE TABLE, and LOAD DATA statements; --output/-o is treated as a directory") cmd.Flags().BoolVar(&noLoad, "no-load", false, "With --load-script, generate only DDL and skip CSV dump and LOAD DATA statements") cmd.Flags().StringVar(&rowOrder, "row-order", string(checkpointtool.CSVRowOrderStorage), "CSV row order: storage (streaming, large-table friendly) or lexical (sort by visible CSV values in memory)") + cmd.Flags().StringVar(&cpuProfile, "cpuprofile", "", "Write CPU profile to file") return cmd } @@ -744,14 +759,14 @@ func writeRestoreScript( if _, err := fmt.Fprintln(f, "LINES TERMINATED BY '\\n'"); err != nil { return "", err } - if _, err := fmt.Fprintln(f, "parallel 'true'"); err != nil { - return "", err - } if csvHasHeader { if _, err := fmt.Fprintln(f, "IGNORE 1 LINES"); err != nil { return "", err } } + if _, err := fmt.Fprintln(f, "parallel 'true'"); err != nil { + return "", err + } if _, err := fmt.Fprintln(f, ";"); err != nil { return "", err } From 7155277553bf8f95bf0c593eb3351502c4cef17d Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 12 Jun 2026 14:39:24 +0800 Subject: [PATCH 12/76] perf: speed up checkpoint CSV dump --- pkg/tools/checkpointtool/table_dump.go | 229 +++++++++++++++++++- pkg/tools/checkpointtool/table_dump_test.go | 47 ++++ 2 files changed, 265 insertions(+), 11 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 38f95b16bc14d..302e75c4484fb 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -1636,7 +1636,7 @@ func writeSQLLoadCSVRow(w io.Writer, colTypes []types.Type, fields []string, nul } func writeProjectedCSVRowFromVecs( - w io.Writer, + w *bytes.Buffer, colTypes []types.Type, vecs []*vector.Vector, physicalPositions []int, @@ -1652,20 +1652,227 @@ func writeProjectedCSVRowFromVecs( if i < len(colTypes) { typ = colTypes[i] } - field := "" - isNull := true - if pos >= 0 && pos < len(vecs) && vecs[pos] != nil { - isNull = vecs[pos].IsNull(uint64(rowIdx)) - if !isNull { - field = vecValueToString(vecs[pos], rowIdx) + if err := writeSQLLoadCSVFieldFromVec(w, typ, vecs, pos, rowIdx); err != nil { + return err + } + } + w.WriteByte('\n') + return nil +} + +func writeSQLLoadCSVFieldFromVec( + w *bytes.Buffer, + typ types.Type, + vecs []*vector.Vector, + pos int, + rowIdx int, +) error { + if pos < 0 || pos >= len(vecs) || vecs[pos] == nil || vecs[pos].IsNull(uint64(rowIdx)) { + _, err := io.WriteString(w, `\N`) + return err + } + vec := vecs[pos] + rowIdx = vectorRowIndex(vec, rowIdx) + if shouldQuoteSQLLoadType(typ) { + w.WriteByte('"') + appendCSVQuotedVecValue(w, vec, rowIdx) + w.WriteByte('"') + return nil + } + appendCSVUnquotedVecValue(w, typ, vec, rowIdx) + return nil +} + +func vectorRowIndex(vec *vector.Vector, rowIdx int) int { + if vec.IsConst() { + return 0 + } + return rowIdx +} + +func appendCSVQuotedVecValue(w *bytes.Buffer, vec *vector.Vector, rowIdx int) { + switch vec.GetType().Oid { + case types.T_char, types.T_varchar, types.T_blob, types.T_text, + types.T_binary, types.T_varbinary, types.T_datalink, types.T_json: + appendEscapedSQLLoadBytes(w, vec.GetBytesAt(rowIdx), '"') + default: + appendEscapedSQLLoadString(w, vecValueToString(vec, rowIdx), '"') + } +} + +func appendCSVUnquotedVecValue(w *bytes.Buffer, typ types.Type, vec *vector.Vector, rowIdx int) { + switch vec.GetType().Oid { + case types.T_bool: + if vector.MustFixedColWithTypeCheck[bool](vec)[rowIdx] { + w.WriteString("true") + } else { + w.WriteString("false") + } + case types.T_int8: + appendCSVInt(w, int64(vector.MustFixedColWithTypeCheck[int8](vec)[rowIdx])) + case types.T_int16: + appendCSVInt(w, int64(vector.MustFixedColWithTypeCheck[int16](vec)[rowIdx])) + case types.T_int32: + appendCSVInt(w, int64(vector.MustFixedColWithTypeCheck[int32](vec)[rowIdx])) + case types.T_int64: + appendCSVInt(w, vector.MustFixedColWithTypeCheck[int64](vec)[rowIdx]) + case types.T_uint8: + appendCSVUint(w, uint64(vector.MustFixedColWithTypeCheck[uint8](vec)[rowIdx])) + case types.T_uint16: + appendCSVUint(w, uint64(vector.MustFixedColWithTypeCheck[uint16](vec)[rowIdx])) + case types.T_uint32: + appendCSVUint(w, uint64(vector.MustFixedColWithTypeCheck[uint32](vec)[rowIdx])) + case types.T_uint64: + appendCSVUint(w, vector.MustFixedColWithTypeCheck[uint64](vec)[rowIdx]) + case types.T_float32: + appendCSVFloat(w, float64(vector.MustFixedColWithTypeCheck[float32](vec)[rowIdx]), 32) + case types.T_float64: + appendCSVFloat(w, vector.MustFixedColWithTypeCheck[float64](vec)[rowIdx], 64) + case types.T_decimal64: + appendDecimal64(w, vector.MustFixedColWithTypeCheck[types.Decimal64](vec)[rowIdx], vec.GetType().Scale) + case types.T_decimal128: + appendDecimal128(w, vector.MustFixedColWithTypeCheck[types.Decimal128](vec)[rowIdx], vec.GetType().Scale) + case types.T_date: + appendDate(w, vector.MustFixedColWithTypeCheck[types.Date](vec)[rowIdx]) + default: + w.WriteString(vecValueToString(vec, rowIdx)) + } +} + +func appendEscapedSQLLoadString(w *bytes.Buffer, s string, enclosed byte) { + for i := 0; i < len(s); i++ { + switch s[i] { + case '\\': + w.WriteByte('\\') + w.WriteByte('\\') + case enclosed: + if enclosed != 0 && enclosed != '\\' { + w.WriteByte(enclosed) + w.WriteByte(enclosed) + } else { + w.WriteByte(s[i]) } + default: + w.WriteByte(s[i]) } - if err := writeSQLLoadCSVField(w, typ, field, isNull); err != nil { - return err + } +} + +func appendEscapedSQLLoadBytes(w *bytes.Buffer, data []byte, enclosed byte) { + for _, b := range data { + switch b { + case '\\': + w.WriteByte('\\') + w.WriteByte('\\') + case enclosed: + if enclosed != 0 && enclosed != '\\' { + w.WriteByte(enclosed) + w.WriteByte(enclosed) + } else { + w.WriteByte(b) + } + default: + w.WriteByte(b) } } - _, err := io.WriteString(w, "\n") - return err +} + +func appendDecimal64(w *bytes.Buffer, value types.Decimal64, scale int32) { + if value.Sign() { + w.WriteByte('-') + value = value.Minus() + } + v := uint64(value) + if scale <= 0 { + appendCSVUint(w, v) + return + } + pow := uint64(1) + for i := int32(0); i < scale; i++ { + pow *= 10 + } + intPart := v / pow + fracPart := v % pow + appendCSVUint(w, intPart) + w.WriteByte('.') + var scratch [32]byte + frac := strconv.AppendUint(scratch[:0], fracPart, 10) + for i := len(frac); i < int(scale); i++ { + w.WriteByte('0') + } + w.Write(frac) +} + +func appendDecimal128(w *bytes.Buffer, value types.Decimal128, scale int32) { + if value.Sign() { + w.WriteByte('-') + value = value.Minus() + } + var scratch [80]byte + i := len(scratch) + ten := types.Decimal128{B0_63: types.Pow10[1]} + one := types.Decimal128{B0_63: 1} + for value.B64_127 != 0 || value.B0_63 != 0 { + digit, _ := value.Mod128(ten) + i-- + scratch[i] = byte(digit.B0_63) + '0' + value, _ = value.Div128(ten) + if digit.B0_63 >= 5 { + value, _ = value.Sub128(one) + } + scale-- + if scale == 0 { + i-- + scratch[i] = '.' + } + } + for scale > 0 { + i-- + scratch[i] = '0' + scale-- + if scale == 0 { + i-- + scratch[i] = '.' + } + } + if scale == 0 { + i-- + scratch[i] = '0' + } + w.Write(scratch[i:]) +} + +func appendDate(w *bytes.Buffer, value types.Date) { + year, month, day, _ := value.Calendar(true) + appendZeroPaddedInt(w, int64(year), 4) + w.WriteByte('-') + appendZeroPaddedInt(w, int64(month), 2) + w.WriteByte('-') + appendZeroPaddedInt(w, int64(day), 2) +} + +func appendZeroPaddedInt(w *bytes.Buffer, value int64, width int) { + var scratch [32]byte + buf := strconv.AppendInt(scratch[:0], value, 10) + for i := len(buf); i < width; i++ { + w.WriteByte('0') + } + w.Write(buf) +} + +func appendCSVInt(w *bytes.Buffer, value int64) { + var scratch [32]byte + w.Write(strconv.AppendInt(scratch[:0], value, 10)) +} + +func appendCSVUint(w *bytes.Buffer, value uint64) { + var scratch [32]byte + w.Write(strconv.AppendUint(scratch[:0], value, 10)) +} + +func appendCSVFloat(w *bytes.Buffer, value float64, bitSize int) { + var scratch [32]byte + w.Write(strconv.AppendFloat(scratch[:0], value, 'g', -1, bitSize)) } func writeSQLLoadCSVField(w io.Writer, typ types.Type, field string, isNull bool) error { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 0ea53feb8affb..1e1703ba25536 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -22,7 +22,9 @@ import ( "testing" "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/sql/util/csvparser" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -107,6 +109,51 @@ func TestWriteCSV_withData(t *testing.T) { assert.Equal(t, `4,"NULL",NULL`, dataLines[4]) } +func TestWriteProjectedCSVRowFromVecsFastPath(t *testing.T) { + mp := mpool.MustNewZero() + vecInt := vector.NewVec(types.T_int64.ToType()) + vecDecimal := vector.NewVec(types.T_decimal64.ToTypeWithScale(2)) + vecDecimal128 := vector.NewVec(types.T_decimal128.ToTypeWithScale(5)) + vecDate := vector.NewVec(types.T_date.ToType()) + vecString := vector.NewVec(types.T_varchar.ToType()) + vecNull := vector.NewVec(types.T_int32.ToType()) + defer vecInt.Free(mp) + defer vecDecimal.Free(mp) + defer vecDecimal128.Free(mp) + defer vecDate.Free(mp) + defer vecString.Free(mp) + defer vecNull.Free(mp) + + require.NoError(t, vector.AppendFixed(vecInt, int64(-42), false, mp)) + decimal, err := types.ParseDecimal64("-123.40", 18, 2) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(vecDecimal, decimal, false, mp)) + decimal128, err := types.ParseDecimal128("123456789012345.67890", 30, 5) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(vecDecimal128, decimal128, false, mp)) + require.NoError(t, vector.AppendFixed(vecDate, types.DateFromCalendar(2024, 6, 1), false, mp)) + require.NoError(t, vector.AppendBytes(vecString, []byte(`a"b\c`), false, mp)) + require.NoError(t, vector.AppendFixed(vecNull, int32(0), true, mp)) + + var buf bytes.Buffer + err = writeProjectedCSVRowFromVecs( + &buf, + []types.Type{ + types.T_int64.ToType(), + types.T_decimal64.ToTypeWithScale(2), + types.T_decimal128.ToTypeWithScale(5), + types.T_date.ToType(), + types.T_varchar.ToType(), + types.T_int32.ToType(), + }, + []*vector.Vector{vecInt, vecDecimal, vecDecimal128, vecDate, vecString, vecNull}, + []int{0, 1, 2, 3, 4, 5}, + 0, + ) + require.NoError(t, err) + assert.Equal(t, "-42,-123.40,123456789012345.67890,2024-06-01,\"a\"\"b\\\\c\",\\N\n", buf.String()) +} + // TestWriteCSV_withCreateSQLHeader tests the header comment from CreateSQL. func TestWriteCSV_withCreateSQLHeader(t *testing.T) { schema := &TableSchema{ From f8cd540266084f2e311f590c53e8ce4b11564fef Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 12 Jun 2026 16:31:59 +0800 Subject: [PATCH 13/76] fix checkpoint dump restore indexes --- cmd/mo-object-tool/ckp/checkpoint.go | 181 ++++++++- cmd/mo-object-tool/ckp/checkpoint_test.go | 15 + pkg/tools/checkpointtool/checkpoint_reader.go | 71 ++++ pkg/tools/checkpointtool/table_dump.go | 359 ++++++++++++++++++ pkg/tools/checkpointtool/table_dump_test.go | 22 ++ 5 files changed, 637 insertions(+), 11 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index d44d4b11dc7a6..da817a8f00ffe 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -484,7 +484,11 @@ Examples: if jobs > len(tables) { jobs = len(tables) } - if err := dumpTablesConcurrently(ctx, reader, tables, snapshotTS, outputDir, jobs, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout()); err != nil { + dumpPlans, err := prepareTableDumpPlans(ctx, reader, tables, snapshotTS) + if err != nil { + return err + } + if err := dumpTablesConcurrently(ctx, reader, dumpPlans, snapshotTS, outputDir, jobs, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout()); err != nil { return err } fmt.Fprintf(cmd.OutOrStdout(), "Dumped %d tables to %s\n", len(tables), outputDir) @@ -509,7 +513,11 @@ Examples: return fmt.Errorf("create output dir: %w", err) } if !noLoad { - if err := dumpOneTable(ctx, reader, tableEntry, snapshotTS, outputDir, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout(), &sync.Mutex{}); err != nil { + dumpData, err := reader.PrepareTableDumpData(ctx, tableEntry.TableID, snapshotTS) + if err != nil { + return fmt.Errorf("prepare table %d (%s.%s): %w", tableEntry.TableID, tableEntry.DatabaseName, tableEntry.TableName, err) + } + if err := dumpOneTable(ctx, reader, tableDumpPlan{table: tableEntry, data: dumpData}, snapshotTS, outputDir, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout(), &sync.Mutex{}); err != nil { return err } } @@ -743,6 +751,16 @@ func writeRestoreScript( if _, err := fmt.Fprintln(f, ";"); err != nil { return "", err } + indexDDLs, err := reader.ShowCreateIndexStatements(ctx, table.TableID, table.TableName, snapshotTS) + if err != nil { + return "", fmt.Errorf("show create indexes for table %d: %w", table.TableID, err) + } + indexDDLs = filterExistingIndexDDLs(ddl, indexDDLs) + for _, indexDDL := range indexDDLs { + if _, err := fmt.Fprintln(f, strings.TrimRight(indexDDL, " \n\t")); err != nil { + return "", err + } + } if includeLoad { if _, err := fmt.Fprintf(f, "\nLOAD DATA INFILE %s\n", quoteSQLString(tableCSVPath(csvRoot, table))); err != nil { return "", err @@ -803,6 +821,106 @@ func normalizeCreateTableDDLName(ddl string, table checkpointtool.TableCatalogEn return ddl[:nameStart] + target + ddl[nameEnd:] } +func ddlTableName(ddl string, tableID uint64) string { + nameStart, nameEnd, ok := createTableNameRange(ddl) + if !ok { + return strconv.FormatUint(tableID, 10) + } + name := strings.TrimSpace(ddl[nameStart:nameEnd]) + if dot := strings.LastIndex(name, "."); dot >= 0 { + name = name[dot+1:] + } + if unquoted, ok := unquoteSQLIdent(name); ok && unquoted != "" { + return unquoted + } + return strings.Trim(name, "`") +} + +func unquoteSQLIdent(s string) (string, bool) { + s = strings.TrimSpace(s) + if len(s) < 2 || s[0] != '`' || s[len(s)-1] != '`' { + return s, false + } + return strings.ReplaceAll(s[1:len(s)-1], "``", "`"), true +} + +func filterExistingIndexDDLs(createDDL string, indexDDLs []string) []string { + if len(indexDDLs) == 0 { + return nil + } + filtered := make([]string, 0, len(indexDDLs)) + for _, indexDDL := range indexDDLs { + name := generatedIndexDDLName(indexDDL) + if name != "" && createTableHasIndex(createDDL, name) { + continue + } + filtered = append(filtered, indexDDL) + } + return filtered +} + +func generatedIndexDDLName(indexDDL string) string { + upper := strings.ToUpper(indexDDL) + for _, marker := range []string{" ADD UNIQUE KEY ", " ADD KEY ", " ADD INDEX "} { + i := strings.Index(upper, marker) + if i < 0 { + continue + } + name := strings.TrimSpace(indexDDL[i+len(marker):]) + if unquoted, ok := readLeadingSQLIdent(name); ok { + return unquoted + } + } + return "" +} + +func createTableHasIndex(createDDL string, name string) bool { + quoted := quoteSQLIdent(name) + upper := strings.ToUpper(createDDL) + for _, marker := range []string{"KEY " + quoted, "INDEX " + quoted, "CONSTRAINT " + quoted} { + if strings.Contains(upper, strings.ToUpper(marker)) { + return true + } + } + lowerDDL := strings.ToLower(createDDL) + lowerName := strings.ToLower(name) + for _, marker := range []string{"key " + lowerName, "index " + lowerName, "constraint " + lowerName} { + if strings.Contains(lowerDDL, marker) { + return true + } + } + return false +} + +func readLeadingSQLIdent(s string) (string, bool) { + s = strings.TrimSpace(s) + if s == "" { + return "", false + } + if s[0] == '`' { + end := 1 + for end < len(s) { + if s[end] == '`' { + if end+1 < len(s) && s[end+1] == '`' { + end += 2 + continue + } + return strings.ReplaceAll(s[1:end], "``", "`"), true + } + end++ + } + return "", false + } + end := 0 + for end < len(s) && isSQLIdentByte(s[end]) { + end++ + } + if end == 0 { + return "", false + } + return s[:end], true +} + func createTableNameRange(sql string) (int, int, bool) { i, ok := consumeSQLKeyword(sql, 0, "create") if !ok { @@ -891,11 +1009,42 @@ func isSQLIdentByte(b byte) bool { (b >= 'A' && b <= 'Z') } -func dumpTablesConcurrently( +type tableDumpPlan struct { + table checkpointtool.TableCatalogEntry + data *checkpointtool.TableDumpData +} + +func prepareTableDumpPlans( ctx context.Context, reader *checkpointtool.CheckpointReader, tables []checkpointtool.TableCatalogEntry, snapshotTS types.TS, +) ([]tableDumpPlan, error) { + tableIDs := make([]uint64, 0, len(tables)) + for _, table := range tables { + tableIDs = append(tableIDs, table.TableID) + } + dumpDataByTable, err := reader.PrepareTableDumpDataForTables(ctx, tableIDs, snapshotTS) + if err != nil { + return nil, err + } + + plans := make([]tableDumpPlan, 0, len(tables)) + for _, table := range tables { + data := dumpDataByTable[table.TableID] + if data == nil { + return nil, fmt.Errorf("prepare table %d (%s.%s): missing object list", table.TableID, table.DatabaseName, table.TableName) + } + plans = append(plans, tableDumpPlan{table: table, data: data}) + } + return plans, nil +} + +func dumpTablesConcurrently( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + plans []tableDumpPlan, + snapshotTS types.TS, outputDir string, jobs int, rowOrder checkpointtool.CSVRowOrder, @@ -903,15 +1052,16 @@ func dumpTablesConcurrently( header bool, out io.Writer, ) error { - tableCh := make(chan checkpointtool.TableCatalogEntry) + tableCh := make(chan tableDumpPlan) errCh := make(chan error, 1) var outMu sync.Mutex var wg sync.WaitGroup worker := func() { defer wg.Done() - for table := range tableCh { - if err := dumpOneTable(ctx, reader, table, snapshotTS, outputDir, rowOrder, metaComments, header, out, &outMu); err != nil { + workerReader := reader.Fork(ctx) + for plan := range tableCh { + if err := dumpOneTable(ctx, workerReader, plan, snapshotTS, outputDir, rowOrder, metaComments, header, out, &outMu); err != nil { select { case errCh <- err: default: @@ -924,13 +1074,13 @@ func dumpTablesConcurrently( wg.Add(1) go worker() } - for _, table := range tables { + for _, plan := range plans { select { case err := <-errCh: close(tableCh) wg.Wait() return err - case tableCh <- table: + case tableCh <- plan: } } close(tableCh) @@ -955,7 +1105,7 @@ func tableCSVPath(outputDir string, table checkpointtool.TableCatalogEntry) stri func dumpOneTable( ctx context.Context, reader *checkpointtool.CheckpointReader, - table checkpointtool.TableCatalogEntry, + plan tableDumpPlan, snapshotTS types.TS, outputDir string, rowOrder checkpointtool.CSVRowOrder, @@ -964,6 +1114,7 @@ func dumpOneTable( out io.Writer, outMu *sync.Mutex, ) error { + table := plan.table filePath := tableCSVPath(outputDir, table) tableDir := filepath.Dir(filePath) if err := os.MkdirAll(tableDir, 0o755); err != nil { @@ -973,10 +1124,10 @@ func dumpOneTable( if err != nil { return fmt.Errorf("create output file for table %d: %w", table.TableID, err) } - err = reader.DumpTableCSVComposed( + err = reader.DumpPreparedTableCSV( ctx, outFile, - table.TableID, + plan.data, snapshotTS, checkpointtool.WithCSVMetaComments(metaComments), checkpointtool.WithCSVHeader(header), @@ -1078,6 +1229,14 @@ Examples: } fmt.Fprintln(cmd.OutOrStdout(), ddl) + indexDDLs, err := reader.ShowCreateIndexStatements(ctx, tableID, ddlTableName(ddl, tableID), snapshotTS) + if err != nil { + return fmt.Errorf("show create indexes for table %d: %w", tableID, err) + } + indexDDLs = filterExistingIndexDDLs(ddl, indexDDLs) + for _, indexDDL := range indexDDLs { + fmt.Fprintln(cmd.OutOrStdout(), indexDDL) + } return nil }, } diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 698fcec6e8e6f..5d34cd8fc6acb 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -69,3 +69,18 @@ func TestNormalizeCreateTableDDLName(t *testing.T) { }) } } + +func TestFilterExistingIndexDDLs(t *testing.T) { + createDDL := "CREATE TABLE `items_gist` (\n" + + " `id` int NOT NULL,\n" + + " `embedding` vecf32(960) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`),\n" + + " KEY `ivf_2000` USING ivfflat(`embedding`) lists=2000 op_type 'vector_l2_ops'\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `items_gist` ADD KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops' ;", + "ALTER TABLE `items_gist` ADD KEY `new_idx`(`id`);", + } + + assert.Equal(t, []string{"ALTER TABLE `items_gist` ADD KEY `new_idx`(`id`);"}, filterExistingIndexDDLs(createDDL, indexDDLs)) +} diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index ca308f0e41604..98cba381e2813 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -95,6 +95,22 @@ func (r *CheckpointReader) FS() fileservice.FileService { return r.fs } +// Fork returns a lightweight reader that shares the file service and already +// loaded checkpoint entries, but uses an independent memory pool and does not +// own the file service. +func (r *CheckpointReader) Fork(ctx context.Context) *CheckpointReader { + if ctx == nil { + ctx = r.ctx + } + return &CheckpointReader{ + ctx: ctx, + fs: r.fs, + dir: r.dir, + entries: r.entries, + mp: mpool.MustNewZero(), + } +} + func (r *CheckpointReader) loadEntries() error { names, err := ckputil.ListCKPMetaNames(r.ctx, r.fs) if err != nil { @@ -474,6 +490,61 @@ func (r *CheckpointReader) GetObjectEntries(entry *checkpoint.CheckpointEntry, t return dataEntries, tombEntries, nil } +// GetObjectEntriesForTables reads detailed object entries for multiple tables +// from one checkpoint entry in a single pass. +func (r *CheckpointReader) GetObjectEntriesForTables( + entry *checkpoint.CheckpointEntry, + tableIDs map[uint64]struct{}, +) (map[uint64][]*ObjectEntryInfo, map[uint64][]*ObjectEntryInfo, error) { + loc := entry.GetLocation() + if loc.IsEmpty() { + return nil, nil, nil + } + + _, err := r.fs.StatFile(r.ctx, loc.Name().String()) + if err != nil { + if isDataFileNotFound(err) { + return nil, nil, moerr.NewFileNotFoundErrorf(r.ctx, "checkpoint data file not found (may have been GC'd): %s", loc.Name().String()) + } + return nil, nil, err + } + + dataByTable := make(map[uint64][]*ObjectEntryInfo) + tombByTable := make(map[uint64][]*ObjectEntryInfo) + reader := logtail.NewCKPReader(entry.GetVersion(), loc, r.mp, r.fs) + if err := reader.ReadMeta(r.ctx); err != nil { + return nil, nil, err + } + err = reader.ForEachRow(r.ctx, func( + _ uint32, + _, tid uint64, + objectType int8, + objectStats objectio.ObjectStats, + create, delete types.TS, + _ types.Rowid, + ) error { + if _, ok := tableIDs[tid]; !ok { + return nil + } + info := &ObjectEntryInfo{ + ObjectStats: objectStats, + CreateTime: create, + DeleteTime: delete, + } + switch objectType { + case ckputil.ObjectType_Data: + dataByTable[tid] = append(dataByTable[tid], info) + case ckputil.ObjectType_Tombstone: + tombByTable[tid] = append(tombByTable[tid], info) + } + return nil + }) + if err != nil { + return nil, nil, err + } + return dataByTable, tombByTable, nil +} + // ReadRangeData reads actual data from a range and returns column names and row data as strings func (r *CheckpointReader) ReadRangeData(entry *checkpoint.CheckpointEntry, rng ckputil.TableRange) ([]string, [][]string, error) { objName := rng.ObjectStats.ObjectName().String() diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 302e75c4484fb..875d5a6a9059d 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -142,6 +142,28 @@ type exportedCSVRow struct { nulls []bool } +// TableDumpData contains the checkpoint metadata needed to dump one table. +type TableDumpData struct { + TableID uint64 + Schema *TableSchema + DataEntries []*ObjectEntryInfo + TombEntries []*ObjectEntryInfo +} + +type indexDDLColumn struct { + name string + ordinal int +} + +type indexDDLInfo struct { + name string + indexType string + algo string + algoParams string + comment string + columns map[string]indexDDLColumn +} + type CSVExportOption func(*CSVExportOptions) func defaultCSVExportOptions() CSVExportOptions { @@ -651,6 +673,141 @@ func (r *CheckpointReader) DumpTableCSVComposed( return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries, resolveCSVExportOptions(opts)) } +// PrepareTableDumpData resolves the table schema and object lists once so batch +// dumps can avoid repeatedly composing checkpoint metadata per table worker. +func (r *CheckpointReader) PrepareTableDumpData( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) (*TableDumpData, error) { + dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) + if err != nil { + return nil, err + } + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) + if len(schema.Columns) == 0 { + return nil, moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) + } + return &TableDumpData{ + TableID: tableID, + Schema: cloneTableSchema(schema), + DataEntries: dataEntries, + TombEntries: tombEntries, + }, nil +} + +// PrepareTableDumpDataForTables resolves schemas and object lists for multiple +// tables by composing the checkpoint once and scanning each selected checkpoint +// entry once. +func (r *CheckpointReader) PrepareTableDumpDataForTables( + ctx context.Context, + tableIDs []uint64, + snapshotTS types.TS, +) (map[uint64]*TableDumpData, error) { + composed, err := r.ComposeAt(snapshotTS) + if err != nil { + return nil, err + } + + tableSet := make(map[uint64]struct{}, len(tableIDs)) + result := make(map[uint64]*TableDumpData, len(tableIDs)) + for _, tableID := range tableIDs { + if _, ok := tableSet[tableID]; ok { + continue + } + tbl, ok := composed.Tables[tableID] + if !ok || (len(tbl.DataRanges) == 0 && len(tbl.TombRanges) == 0) { + return nil, moerr.NewInternalErrorf(ctx, "table %d not found in checkpoint at ts %s", tableID, snapshotTS.ToString()) + } + tableSet[tableID] = struct{}{} + result[tableID] = &TableDumpData{TableID: tableID} + } + + entryRefs := make([]*EntryInfo, 0, len(composed.Incrementals)+1) + if composed.BaseEntry != nil { + entryRefs = append(entryRefs, composed.BaseEntry) + } + entryRefs = append(entryRefs, composed.Incrementals...) + + for _, ref := range entryRefs { + e := r.entries[ref.Index] + dataByTable, tombByTable, err := r.GetObjectEntriesForTables(e, tableSet) + if err != nil { + if isDataFileNotFound(err) { + continue + } + return nil, err + } + for tableID, entries := range dataByTable { + result[tableID].DataEntries = append(result[tableID].DataEntries, entries...) + } + for tableID, entries := range tombByTable { + result[tableID].TombEntries = append(result[tableID].TombEntries, entries...) + } + } + + for tableID, data := range result { + if len(data.DataEntries) == 0 { + return nil, moerr.NewInternalErrorf(ctx, "no data entries for table %d", tableID) + } + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) + if len(schema.Columns) == 0 { + return nil, moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) + } + data.Schema = cloneTableSchema(schema) + } + return result, nil +} + +// DumpPreparedTableCSV writes a table using metadata previously resolved by +// PrepareTableDumpData. +func (r *CheckpointReader) DumpPreparedTableCSV( + ctx context.Context, + w io.Writer, + data *TableDumpData, + snapshotTS types.TS, + opts ...CSVExportOption, +) error { + if data == nil { + return moerr.NewInternalError(ctx, "missing prepared table dump data") + } + if data.Schema == nil || len(data.Schema.Columns) == 0 { + return moerr.NewInternalErrorf(ctx, "cannot resolve visible columns for table %d from checkpoint metadata", data.TableID) + } + return r.streamTableCSV(ctx, w, data.Schema, snapshotTS, data.DataEntries, data.TombEntries, resolveCSVExportOptions(opts)) +} + +func cloneTableSchema(schema *TableSchema) *TableSchema { + if schema == nil { + return nil + } + clone := &TableSchema{ + TableName: strings.Clone(schema.TableName), + DatabaseName: strings.Clone(schema.DatabaseName), + CreateSQL: strings.Clone(schema.CreateSQL), + } + if len(schema.Columns) > 0 { + clone.Columns = make([]TableColumn, len(schema.Columns)) + for i, col := range schema.Columns { + clone.Columns[i] = TableColumn{ + Name: strings.Clone(col.Name), + SQLType: strings.Clone(col.SQLType), + Position: col.Position, + PhysicalPosition: col.PhysicalPosition, + } + } + } + return clone +} + func (r *CheckpointReader) ListCatalogTables( ctx context.Context, snapshotTS types.TS, @@ -2396,6 +2553,208 @@ func (r *CheckpointReader) ShowCreateTable( ) } +// ShowCreateIndexStatements returns ALTER TABLE statements for secondary indexes +// recorded in mo_catalog.mo_indexes. CREATE TABLE reconstruction from mo_columns +// cannot see these rows, so restore scripts need to apply them separately. +func (r *CheckpointReader) ShowCreateIndexStatements( + ctx context.Context, + tableID uint64, + tableName string, + snapshotTS types.TS, +) ([]string, error) { + moIndexesTableID, ok, err := r.findCatalogTableID(ctx, snapshotTS, catalog.MO_INDEXES) + if err != nil { + return nil, err + } + if !ok { + return nil, nil + } + + view, err := r.getTableLogicalView(ctx, moIndexesTableID, snapshotTS) + if err != nil || view == nil { + return nil, err + } + schema := r.ReadTableSchema(ctx, moIndexesTableID, snapshotTS, view) + if len(schema.Columns) > 0 { + view = MergeLogicalViewWithSchema(view, schema) + } + return buildCreateIndexStatementsFromMoIndexes(view, tableID, tableName) +} + +func (r *CheckpointReader) findCatalogTableID( + ctx context.Context, + snapshotTS types.TS, + tableName string, +) (uint64, bool, error) { + tables, err := r.ListCatalogTables(ctx, snapshotTS, TableListOptions{IncludeViews: true}) + if err != nil { + return 0, false, fmt.Errorf("list catalog tables: %w", err) + } + for _, table := range tables { + if table.TableName != tableName { + continue + } + if table.DatabaseName == "" || table.DatabaseName == catalog.MO_CATALOG { + return table.TableID, true, nil + } + } + return 0, false, nil +} + +func buildCreateIndexStatementsFromMoIndexes( + view *LogicalTableView, + tableID uint64, + tableName string, +) ([]string, error) { + if view == nil { + return nil, nil + } + tableIDCol := view.columnDataIndex("table_id") + nameCol := view.columnDataIndex("name") + typeCol := view.columnDataIndex("type") + algoCol := view.columnDataIndex(catalog.IndexAlgoName) + paramsCol := view.columnDataIndex(catalog.IndexAlgoParams) + commentCol := view.columnDataIndex("comment") + columnNameCol := view.columnDataIndex("column_name") + ordinalCol := view.columnDataIndex("ordinal_position") + hiddenCol := view.columnDataIndex("hidden") + if tableIDCol < 0 || nameCol < 0 || columnNameCol < 0 { + return nil, nil + } + + byName := make(map[string]*indexDDLInfo) + tableIDStr := strconv.FormatUint(tableID, 10) + for _, row := range view.Rows { + if tableIDCol >= len(row) || row[tableIDCol] != tableIDStr { + continue + } + if hiddenCol >= 0 && hiddenCol < len(row) && isTruthyCatalogValue(row[hiddenCol]) { + continue + } + if nameCol >= len(row) || columnNameCol >= len(row) { + continue + } + name := row[nameCol] + colName := row[columnNameCol] + if name == "" || strings.EqualFold(name, "PRIMARY") || colName == "" || catalog.IsAlias(colName) { + continue + } + info := byName[name] + if info == nil { + info = &indexDDLInfo{name: name, columns: make(map[string]indexDDLColumn)} + byName[name] = info + } + if typeCol >= 0 && typeCol < len(row) && info.indexType == "" { + info.indexType = row[typeCol] + } + if algoCol >= 0 && algoCol < len(row) && info.algo == "" { + info.algo = row[algoCol] + } + if paramsCol >= 0 && paramsCol < len(row) && info.algoParams == "" { + info.algoParams = row[paramsCol] + } + if commentCol >= 0 && commentCol < len(row) && info.comment == "" { + info.comment = row[commentCol] + } + ordinal := len(info.columns) + 1 + if ordinalCol >= 0 && ordinalCol < len(row) { + if parsed, err := strconv.Atoi(row[ordinalCol]); err == nil { + ordinal = parsed + } + } + if existing, ok := info.columns[colName]; !ok || ordinal < existing.ordinal { + info.columns[colName] = indexDDLColumn{name: colName, ordinal: ordinal} + } + } + + names := make([]string, 0, len(byName)) + for name, info := range byName { + if len(info.columns) > 0 { + names = append(names, name) + } + } + sort.Strings(names) + + statements := make([]string, 0, len(names)) + for _, name := range names { + stmt, err := renderCreateIndexStatement(tableName, byName[name]) + if err != nil { + return nil, err + } + if stmt != "" { + statements = append(statements, stmt) + } + } + return statements, nil +} + +func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, error) { + if info == nil || tableName == "" || len(info.columns) == 0 { + return "", nil + } + cols := make([]indexDDLColumn, 0, len(info.columns)) + for _, col := range info.columns { + cols = append(cols, col) + } + sort.Slice(cols, func(i, j int) bool { + if cols[i].ordinal != cols[j].ordinal { + return cols[i].ordinal < cols[j].ordinal + } + return cols[i].name < cols[j].name + }) + + var sb strings.Builder + sb.WriteString("ALTER TABLE ") + sb.WriteString(quoteDDLIdent(tableName)) + sb.WriteString(" ADD ") + if strings.EqualFold(info.indexType, "UNIQUE") { + sb.WriteString("UNIQUE ") + } + sb.WriteString("KEY ") + sb.WriteString(quoteDDLIdent(info.name)) + if !catalog.IsNullIndexAlgo(info.algo) { + sb.WriteString(" USING ") + sb.WriteString(info.algo) + } + sb.WriteString("(") + for i, col := range cols { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(col.name)) + } + sb.WriteString(")") + if strings.TrimSpace(info.algoParams) != "" { + params, err := catalog.IndexParamsToStringList(info.algoParams) + if err != nil { + return "", err + } + if strings.TrimSpace(params) != "" { + sb.WriteString(params) + } + } + if info.comment != "" { + sb.WriteString(" COMMENT ") + sb.WriteString(quoteDDLString(info.comment)) + } + sb.WriteString(";") + return sb.String(), nil +} + +func quoteDDLIdent(s string) string { + return "`" + strings.ReplaceAll(s, "`", "``") + "`" +} + +func quoteDDLString(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `'`, `''`) + return "'" + s + "'" +} + +func isTruthyCatalogValue(s string) bool { + return s == "1" || strings.EqualFold(s, "true") +} + // getTableName tries to get the table name for a tableID from the LogicalTableView's // mo_tables data if available, falling back to the table ID as string. func getTableName(view *LogicalTableView, tableID uint64) string { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 1e1703ba25536..2f201dd76417a 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -596,6 +596,28 @@ func TestMergeLogicalViewWithSchema_LargeSparsePhysicalPositions(t *testing.T) { assert.Equal(t, []string{"v0", "v63", "v127"}, merged.Rows[0]) } +func TestBuildCreateIndexStatementsFromMoIndexes_IVFFlat(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{ + "table_id", "name", "type", "algo", "algo_params", "comment", + "column_name", "ordinal_position", "hidden", + }, + Rows: [][]string{ + {"272535", "PRIMARY", "PRIMARY", "", "", "", "id", "1", "0"}, + {"272535", "ivf_2000", "MULTIPLE", "ivfflat", `{"lists":"2000","op_type":"vector_l2_ops"}`, "", "embedding", "1", "0"}, + {"272535", "ivf_2000", "MULTIPLE", "ivfflat", `{"lists":"2000","op_type":"vector_l2_ops"}`, "", "embedding", "1", "0"}, + {"272535", "ivf_2000", "MULTIPLE", "ivfflat", `{"lists":"2000","op_type":"vector_l2_ops"}`, "", "__mo_alias_embedding", "1", "0"}, + {"999999", "other_idx", "MULTIPLE", "", "", "", "v", "1", "0"}, + }, + } + + stmts, err := buildCreateIndexStatementsFromMoIndexes(view, 272535, "items_gist") + require.NoError(t, err) + require.Equal(t, []string{ + "ALTER TABLE `items_gist` ADD KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops' ;", + }, stmts) +} + func TestWriteCSV_LexicalRowOrder(t *testing.T) { schema := &TableSchema{ TableName: "sorted", From c5e338bc749f731842ab7756e022a6eeb632f839 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 12 Jun 2026 17:02:09 +0800 Subject: [PATCH 14/76] feat checkpoint dump to object storage --- cmd/mo-object-tool/ckp/checkpoint.go | 181 ++++++++++++++++++---- cmd/mo-object-tool/ckp/checkpoint_test.go | 6 + pkg/tools/checkpointtool/table_dump.go | 32 +++- 3 files changed, 182 insertions(+), 37 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index da817a8f00ffe..f377a3413631e 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "os" + "path" "path/filepath" "runtime/pprof" "sort" @@ -29,6 +30,7 @@ import ( "time" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool/interactive" @@ -69,6 +71,13 @@ func addStorageFlags(cmd *cobra.Command, storage *toolfs.StorageOptions) { cmd.PersistentFlags().StringVar(&storage.Backend, "backend", "", "remote backend for --s3: S3 or MINIO") } +func addOutputStorageFlags(cmd *cobra.Command, storage *toolfs.StorageOptions) { + cmd.Flags().StringVar(&storage.FSConfig, "out-fs-config", "", "MO config TOML containing fileservice settings for dump output") + cmd.Flags().StringVar(&storage.FSName, "out-fs-name", "SHARED", "fileservice name to use from --out-fs-config") + cmd.Flags().StringVar(&storage.S3, "out-s3", "", "S3 arguments for dump output, for example bucket=...,endpoint=...,region=...,key-prefix=...,key-id=...,key-secret=...") + cmd.Flags().StringVar(&storage.Backend, "out-backend", "", "remote backend for --out-s3: S3 or MINIO") +} + func setupLogFile() (*os.File, error) { homeDir, err := os.UserHomeDir() if err != nil { @@ -312,6 +321,101 @@ func openReader(ctx context.Context, dir string, storage toolfs.StorageOptions) return checkpointtool.OpenWithFS(ctx, fs, display, checkpointtool.WithCloseFS()) } +type dumpOutput struct { + fs fileservice.FileService + remote bool +} + +func openDumpOutput(ctx context.Context, storage toolfs.StorageOptions) (*dumpOutput, error) { + if !storage.IsRemote() { + return &dumpOutput{}, nil + } + fs, display, err := toolfs.Open(ctx, storage) + if err != nil { + return nil, err + } + logutil.Infof("using fileservice %s for dump output", display) + return &dumpOutput{fs: fs, remote: true}, nil +} + +func (o *dumpOutput) Close(ctx context.Context) { + if o != nil && o.fs != nil { + o.fs.Close(ctx) + } +} + +func (o *dumpOutput) MkdirAll(dir string) error { + if o == nil || !o.remote { + return os.MkdirAll(dir, 0o755) + } + return nil +} + +func (o *dumpOutput) Create(ctx context.Context, filePath string) (io.WriteCloser, error) { + if o == nil || !o.remote { + return os.Create(filePath) + } + return newFileServiceWriteCloser(ctx, o.fs, filePath) +} + +type fileServiceWriteCloser struct { + pw *io.PipeWriter + done chan error + closeOnce sync.Once + closeErr error +} + +func newFileServiceWriteCloser(ctx context.Context, fs fileservice.FileService, filePath string) (io.WriteCloser, error) { + pr, pw := io.Pipe() + w := &fileServiceWriteCloser{ + pw: pw, + done: make(chan error, 1), + } + go func() { + var err error + defer func() { + if recovered := recover(); recovered != nil { + err = fmt.Errorf("write object %s panic: %v", filePath, recovered) + _ = pr.CloseWithError(err) + } + w.done <- err + }() + err = fs.Write(ctx, fileservice.IOVector{ + FilePath: cleanObjectPath(filePath), + Entries: []fileservice.IOEntry{{ + ReaderForWrite: pr, + Size: -1, + }}, + }) + if err != nil { + _ = pr.CloseWithError(err) + } + }() + return w, nil +} + +func (w *fileServiceWriteCloser) Write(p []byte) (int, error) { + return w.pw.Write(p) +} + +func (w *fileServiceWriteCloser) Close() error { + w.closeOnce.Do(func() { + err := w.pw.Close() + writeErr := <-w.done + if err != nil { + w.closeErr = err + return + } + w.closeErr = writeErr + }) + return w.closeErr +} + +func cleanObjectPath(filePath string) string { + filePath = filepath.ToSlash(filePath) + return strings.TrimPrefix(path.Clean(filePath), "/") +} + // dumpCommand implements the "ckp dump" subcommand for offline CSV export. // // Usage: @@ -319,20 +423,21 @@ func openReader(ctx context.Context, dir string, storage toolfs.StorageOptions) // mo-tool ckp dump --table-id=12345 [--ts=...] [--output=table.csv] [directory] func dumpCommand(storage *toolfs.StorageOptions) *cobra.Command { var ( - tableID uint64 - tableName string - accountID uint32 - databaseID uint64 - tsStr string - output string - outputDir string - jobs int - metaComments bool - header bool - loadScript bool - noLoad bool - rowOrder string - cpuProfile string + tableID uint64 + tableName string + accountID uint32 + databaseID uint64 + tsStr string + output string + outputDir string + jobs int + metaComments bool + header bool + loadScript bool + noLoad bool + rowOrder string + cpuProfile string + outputStorage toolfs.StorageOptions ) cmd := &cobra.Command{ @@ -352,7 +457,8 @@ Examples: mo-tool ckp dump --table-id=12345 --load-script -o /tmp/ . mo-tool ckp dump --database-id=9001 --table=users -o users.csv . mo-tool ckp dump --database-id=9001 --output-dir=/tmp/test-dump --jobs=4 . - mo-tool ckp dump --database-id=9001 --load-script -o /tmp/test-dump .`, + mo-tool ckp dump --database-id=9001 --load-script -o /tmp/test-dump . + mo-tool ckp dump --table-id=12345 -o dump/users.csv --out-s3='endpoint=...,bucket=...,key-prefix=...,key-id=...,key-secret=...' .`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { dir := "." @@ -416,6 +522,11 @@ Examples: return fmt.Errorf("open checkpoint dir: %w", err) } defer reader.Close() + dumpOut, err := openDumpOutput(ctx, outputStorage) + if err != nil { + return fmt.Errorf("open dump output fileservice: %w", err) + } + defer dumpOut.Close(ctx) snapshotTS, err := resolveSnapshotTS(ctx, reader, tsStr) if err != nil { @@ -423,9 +534,9 @@ Examples: } var w = cmd.OutOrStdout() - var outFile *os.File + var outFile io.WriteCloser if output != "" && !loadScript { - outFile, err = os.Create(output) + outFile, err = dumpOut.Create(ctx, output) if err != nil { return fmt.Errorf("create output file: %w", err) } @@ -474,7 +585,7 @@ Examples: if len(tables) == 0 { return fmt.Errorf("no checkpoint tables match database-id-set=%v database-id=%d account-id-set=%v account-id=%d", databaseIDSet, databaseID, accountIDSet, accountID) } - if err := os.MkdirAll(outputDir, 0o755); err != nil { + if err := dumpOut.MkdirAll(outputDir); err != nil { return fmt.Errorf("create output dir: %w", err) } if !noLoad { @@ -488,13 +599,13 @@ Examples: if err != nil { return err } - if err := dumpTablesConcurrently(ctx, reader, dumpPlans, snapshotTS, outputDir, jobs, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout()); err != nil { + if err := dumpTablesConcurrently(ctx, reader, dumpOut, dumpPlans, snapshotTS, outputDir, jobs, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout()); err != nil { return err } fmt.Fprintf(cmd.OutOrStdout(), "Dumped %d tables to %s\n", len(tables), outputDir) } if loadScript { - scriptPath, err := writeRestoreScript(ctx, reader, tables, snapshotTS, output, outputDir, !noLoad, effectiveHeader) + scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, tables, snapshotTS, output, outputDir, !noLoad, effectiveHeader) if err != nil { return err } @@ -509,7 +620,7 @@ Examples: if err != nil { return err } - if err := os.MkdirAll(outputDir, 0o755); err != nil { + if err := dumpOut.MkdirAll(outputDir); err != nil { return fmt.Errorf("create output dir: %w", err) } if !noLoad { @@ -517,11 +628,11 @@ Examples: if err != nil { return fmt.Errorf("prepare table %d (%s.%s): %w", tableEntry.TableID, tableEntry.DatabaseName, tableEntry.TableName, err) } - if err := dumpOneTable(ctx, reader, tableDumpPlan{table: tableEntry, data: dumpData}, snapshotTS, outputDir, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout(), &sync.Mutex{}); err != nil { + if err := dumpOneTable(ctx, reader, dumpOut, tableDumpPlan{table: tableEntry, data: dumpData}, snapshotTS, outputDir, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout(), &sync.Mutex{}); err != nil { return err } } - scriptPath, err := writeRestoreScript(ctx, reader, []checkpointtool.TableCatalogEntry{tableEntry}, snapshotTS, output, outputDir, !noLoad, effectiveHeader) + scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, []checkpointtool.TableCatalogEntry{tableEntry}, snapshotTS, output, outputDir, !noLoad, effectiveHeader) if err != nil { return err } @@ -562,6 +673,7 @@ Examples: cmd.Flags().BoolVar(&noLoad, "no-load", false, "With --load-script, generate only DDL and skip CSV dump and LOAD DATA statements") cmd.Flags().StringVar(&rowOrder, "row-order", string(checkpointtool.CSVRowOrderStorage), "CSV row order: storage (streaming, large-table friendly) or lexical (sort by visible CSV values in memory)") cmd.Flags().StringVar(&cpuProfile, "cpuprofile", "", "Write CPU profile to file") + addOutputStorageFlags(cmd, &outputStorage) return cmd } @@ -703,6 +815,7 @@ func resolveTableByID( func writeRestoreScript( ctx context.Context, reader *checkpointtool.CheckpointReader, + dumpOut *dumpOutput, tables []checkpointtool.TableCatalogEntry, snapshotTS types.TS, scriptDir string, @@ -710,11 +823,11 @@ func writeRestoreScript( includeLoad bool, csvHasHeader bool, ) (string, error) { - if err := os.MkdirAll(scriptDir, 0o755); err != nil { + if err := dumpOut.MkdirAll(scriptDir); err != nil { return "", fmt.Errorf("create script dir: %w", err) } - scriptPath := filepath.Join(scriptDir, "restore.sql") - f, err := os.Create(scriptPath) + scriptPath := outputPathJoin(scriptDir, "restore.sql") + f, err := dumpOut.Create(ctx, scriptPath) if err != nil { return "", fmt.Errorf("create restore script: %w", err) } @@ -1043,6 +1156,7 @@ func prepareTableDumpPlans( func dumpTablesConcurrently( ctx context.Context, reader *checkpointtool.CheckpointReader, + dumpOut *dumpOutput, plans []tableDumpPlan, snapshotTS types.TS, outputDir string, @@ -1061,7 +1175,7 @@ func dumpTablesConcurrently( defer wg.Done() workerReader := reader.Fork(ctx) for plan := range tableCh { - if err := dumpOneTable(ctx, workerReader, plan, snapshotTS, outputDir, rowOrder, metaComments, header, out, &outMu); err != nil { + if err := dumpOneTable(ctx, workerReader, dumpOut, plan, snapshotTS, outputDir, rowOrder, metaComments, header, out, &outMu); err != nil { select { case errCh <- err: default: @@ -1094,7 +1208,7 @@ func dumpTablesConcurrently( } func tableCSVPath(outputDir string, table checkpointtool.TableCatalogEntry) string { - return filepath.Join( + return outputPathJoin( outputDir, fmt.Sprintf("account_%d", table.AccountID), fmt.Sprintf("db_%d", table.DatabaseID), @@ -1102,9 +1216,14 @@ func tableCSVPath(outputDir string, table checkpointtool.TableCatalogEntry) stri ) } +func outputPathJoin(elem ...string) string { + return path.Join(elem...) +} + func dumpOneTable( ctx context.Context, reader *checkpointtool.CheckpointReader, + dumpOut *dumpOutput, plan tableDumpPlan, snapshotTS types.TS, outputDir string, @@ -1116,11 +1235,11 @@ func dumpOneTable( ) error { table := plan.table filePath := tableCSVPath(outputDir, table) - tableDir := filepath.Dir(filePath) - if err := os.MkdirAll(tableDir, 0o755); err != nil { + tableDir := path.Dir(filePath) + if err := dumpOut.MkdirAll(tableDir); err != nil { return fmt.Errorf("create table output dir: %w", err) } - outFile, err := os.Create(filePath) + outFile, err := dumpOut.Create(ctx, filePath) if err != nil { return fmt.Errorf("create output file for table %d: %w", table.TableID, err) } diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 5d34cd8fc6acb..f02e201d73431 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -84,3 +84,9 @@ func TestFilterExistingIndexDDLs(t *testing.T) { assert.Equal(t, []string{"ALTER TABLE `items_gist` ADD KEY `new_idx`(`id`);"}, filterExistingIndexDDLs(createDDL, indexDDLs)) } + +func TestCleanObjectPath(t *testing.T) { + assert.Equal(t, "dump/account_1/t.csv", cleanObjectPath("dump/account_1/t.csv")) + assert.Equal(t, "tmp/dump/t.csv", cleanObjectPath("/tmp/dump/t.csv")) + assert.Equal(t, "dump/t.csv", cleanObjectPath("dump//nested/../t.csv")) +} diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 875d5a6a9059d..581a240894367 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -2570,17 +2570,37 @@ func (r *CheckpointReader) ShowCreateIndexStatements( return nil, nil } - view, err := r.getTableLogicalView(ctx, moIndexesTableID, snapshotTS) - if err != nil || view == nil { + view, err := r.dumpCatalogTableView(ctx, moIndexesTableID, snapshotTS) + if err != nil { return nil, err } - schema := r.ReadTableSchema(ctx, moIndexesTableID, snapshotTS, view) - if len(schema.Columns) > 0 { - view = MergeLogicalViewWithSchema(view, schema) - } return buildCreateIndexStatementsFromMoIndexes(view, tableID, tableName) } +func (r *CheckpointReader) dumpCatalogTableView( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) (*LogicalTableView, error) { + var buf bytes.Buffer + if err := r.DumpTableCSVComposed(ctx, &buf, tableID, snapshotTS, WithCSVHeader(true), WithCSVMetaComments(false)); err != nil { + return nil, err + } + reader := csv.NewReader(bytes.NewReader(buf.Bytes())) + reader.FieldsPerRecord = -1 + records, err := reader.ReadAll() + if err != nil { + return nil, err + } + if len(records) == 0 { + return &LogicalTableView{}, nil + } + return &LogicalTableView{ + Headers: records[0], + Rows: records[1:], + }, nil +} + func (r *CheckpointReader) findCatalogTableID( ctx context.Context, snapshotTS types.TS, From 5a1d59c8db6116a6f9c6c039bbae787fed806635 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 10:01:43 +0800 Subject: [PATCH 15/76] update --- cmd/mo-object-tool/ckp/checkpoint.go | 173 ++++++++++++++++++++-- cmd/mo-object-tool/ckp/checkpoint_test.go | 49 ++++++ pkg/tools/checkpointtool/table_dump.go | 3 +- 3 files changed, 210 insertions(+), 15 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index f377a3413631e..2acf390511786 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -858,22 +858,20 @@ func writeRestoreScript( return "", fmt.Errorf("show create table %d: %w", table.TableID, err) } ddl = normalizeCreateTableDDLName(ddl, table) + indexDDLs, err := reader.ShowCreateIndexStatements(ctx, table.TableID, table.TableName, snapshotTS) + if err != nil { + return "", fmt.Errorf("show create indexes for table %d: %w", table.TableID, err) + } + ddl, err = mergeCreateTableIndexDDLs(ddl, indexDDLs) + if err != nil { + return "", fmt.Errorf("merge create table indexes for table %d: %w", table.TableID, err) + } if _, err := fmt.Fprintln(f, strings.TrimRight(ddl, " ;\n\t")); err != nil { return "", err } if _, err := fmt.Fprintln(f, ";"); err != nil { return "", err } - indexDDLs, err := reader.ShowCreateIndexStatements(ctx, table.TableID, table.TableName, snapshotTS) - if err != nil { - return "", fmt.Errorf("show create indexes for table %d: %w", table.TableID, err) - } - indexDDLs = filterExistingIndexDDLs(ddl, indexDDLs) - for _, indexDDL := range indexDDLs { - if _, err := fmt.Fprintln(f, strings.TrimRight(indexDDL, " \n\t")); err != nil { - return "", err - } - } if includeLoad { if _, err := fmt.Fprintf(f, "\nLOAD DATA INFILE %s\n", quoteSQLString(tableCSVPath(csvRoot, table))); err != nil { return "", err @@ -934,6 +932,153 @@ func normalizeCreateTableDDLName(ddl string, table checkpointtool.TableCatalogEn return ddl[:nameStart] + target + ddl[nameEnd:] } +func mergeCreateTableIndexDDLs(createDDL string, indexDDLs []string) (string, error) { + indexDDLs = filterExistingIndexDDLs(createDDL, indexDDLs) + if len(indexDDLs) == 0 { + return createDDL, nil + } + clauses := make([]string, 0, len(indexDDLs)) + for _, indexDDL := range indexDDLs { + clause, ok := alterTableAddClause(indexDDL) + if !ok { + return "", fmt.Errorf("unsupported index DDL %q", indexDDL) + } + clauses = append(clauses, clause) + } + return injectCreateTableClauses(createDDL, clauses) +} + +func alterTableAddClause(sql string) (string, bool) { + i, ok := consumeSQLKeyword(sql, 0, "alter") + if !ok { + return "", false + } + i, ok = consumeSQLKeyword(sql, i, "table") + if !ok { + return "", false + } + i = skipSQLSpace(sql, i) + i, ok = consumeSQLIdentifier(sql, i) + if !ok { + return "", false + } + if j := skipSQLSpace(sql, i); j < len(sql) && sql[j] == '.' { + j = skipSQLSpace(sql, j+1) + if end, ok := consumeSQLIdentifier(sql, j); ok { + i = end + } + } + i, ok = consumeSQLKeyword(sql, i, "add") + if !ok { + return "", false + } + clause := trimSQLStatementTerminator(sql[i:]) + _, hasUnique := consumeSQLKeyword(clause, 0, "unique") + _, hasKey := consumeSQLKeyword(clause, 0, "key") + _, hasIndex := consumeSQLKeyword(clause, 0, "index") + if !hasUnique && !hasKey && !hasIndex { + return "", false + } + return clause, clause != "" +} + +func trimSQLStatementTerminator(sql string) string { + sql = strings.TrimSpace(sql) + for strings.HasSuffix(sql, ";") { + sql = strings.TrimSpace(strings.TrimSuffix(sql, ";")) + } + return sql +} + +func injectCreateTableClauses(createDDL string, clauses []string) (string, error) { + open, close, ok := createTableDefinitionParens(createDDL) + if !ok { + return "", fmt.Errorf("cannot locate CREATE TABLE definition") + } + body := createDDL[open+1 : close] + multiline := strings.Contains(body, "\n") + if multiline { + indent := inferCreateTableClauseIndent(body) + insert := ",\n" + indent + strings.Join(clauses, ",\n"+indent) + return createDDL[:close] + insert + createDDL[close:], nil + } + separator := ", " + if strings.TrimSpace(body) == "" { + separator = "" + } + return createDDL[:close] + separator + strings.Join(clauses, ", ") + createDDL[close:], nil +} + +func inferCreateTableClauseIndent(body string) string { + lines := strings.Split(body, "\n") + for _, line := range lines[1:] { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + return line[:len(line)-len(strings.TrimLeft(line, " \t"))] + } + return " " +} + +func createTableDefinitionParens(sql string) (int, int, bool) { + _, nameEnd, ok := createTableNameRange(sql) + if !ok { + return 0, 0, false + } + open := skipSQLSpace(sql, nameEnd) + if open >= len(sql) || sql[open] != '(' { + return 0, 0, false + } + depth := 0 + inBacktick := false + inString := byte(0) + for i := open; i < len(sql); i++ { + ch := sql[i] + if inBacktick { + if ch == '`' { + if i+1 < len(sql) && sql[i+1] == '`' { + i++ + continue + } + inBacktick = false + } + continue + } + if inString != 0 { + if ch == '\\' && i+1 < len(sql) { + i++ + continue + } + if ch == inString { + if i+1 < len(sql) && sql[i+1] == inString { + i++ + continue + } + inString = 0 + } + continue + } + switch ch { + case '`': + inBacktick = true + case '\'', '"': + inString = ch + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + return open, i, true + } + if depth < 0 { + return 0, 0, false + } + } + } + return 0, 0, false +} + func ddlTableName(ddl string, tableID uint64) string { nameStart, nameEnd, ok := createTableNameRange(ddl) if !ok { @@ -1347,15 +1492,15 @@ Examples: return fmt.Errorf("show create table %d: %w", tableID, err) } - fmt.Fprintln(cmd.OutOrStdout(), ddl) indexDDLs, err := reader.ShowCreateIndexStatements(ctx, tableID, ddlTableName(ddl, tableID), snapshotTS) if err != nil { return fmt.Errorf("show create indexes for table %d: %w", tableID, err) } - indexDDLs = filterExistingIndexDDLs(ddl, indexDDLs) - for _, indexDDL := range indexDDLs { - fmt.Fprintln(cmd.OutOrStdout(), indexDDL) + ddl, err = mergeCreateTableIndexDDLs(ddl, indexDDLs) + if err != nil { + return fmt.Errorf("merge create table indexes for table %d: %w", tableID, err) } + fmt.Fprintln(cmd.OutOrStdout(), ddl) return nil }, } diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index f02e201d73431..7f939e6adef20 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -85,6 +85,55 @@ func TestFilterExistingIndexDDLs(t *testing.T) { assert.Equal(t, []string{"ALTER TABLE `items_gist` ADD KEY `new_idx`(`id`);"}, filterExistingIndexDDLs(createDDL, indexDDLs)) } +func TestMergeCreateTableIndexDDLs(t *testing.T) { + createDDL := "CREATE TABLE `ann`.`items_gist` (\n" + + " `id` int NOT NULL,\n" + + " `embedding` vecf32(960) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`)\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `items_gist` ADD KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops' ;", + } + want := "CREATE TABLE `ann`.`items_gist` (\n" + + " `id` int NOT NULL,\n" + + " `embedding` vecf32(960) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`),\n" + + " KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops'\n" + + ")" + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + assert.NoError(t, err) + assert.Equal(t, want, got) +} + +func TestMergeCreateTableIndexDDLsSingleLine(t *testing.T) { + createDDL := "CREATE TABLE `ann`.`items_sift` (id int primary key, embedding vecf32(128))" + indexDDLs := []string{ + "ALTER TABLE `items_sift` ADD KEY `ivf_500` USING ivfflat(`embedding`) lists = 500 op_type 'vector_l2_ops' ;", + } + want := "CREATE TABLE `ann`.`items_sift` (id int primary key, embedding vecf32(128), KEY `ivf_500` USING ivfflat(`embedding`) lists = 500 op_type 'vector_l2_ops')" + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + assert.NoError(t, err) + assert.Equal(t, want, got) +} + +func TestMergeCreateTableIndexDDLsSkipsExistingIndex(t *testing.T) { + createDDL := "CREATE TABLE `items_gist` (\n" + + " `id` int NOT NULL,\n" + + " `embedding` vecf32(960) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`),\n" + + " KEY `ivf_2000` USING ivfflat(`embedding`) lists=2000 op_type 'vector_l2_ops'\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `items_gist` ADD KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops' ;", + } + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + assert.NoError(t, err) + assert.Equal(t, createDDL, got) +} + func TestCleanObjectPath(t *testing.T) { assert.Equal(t, "dump/account_1/t.csv", cleanObjectPath("dump/account_1/t.csv")) assert.Equal(t, "tmp/dump/t.csv", cleanObjectPath("/tmp/dump/t.csv")) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 581a240894367..51b4e248e925e 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -2555,7 +2555,8 @@ func (r *CheckpointReader) ShowCreateTable( // ShowCreateIndexStatements returns ALTER TABLE statements for secondary indexes // recorded in mo_catalog.mo_indexes. CREATE TABLE reconstruction from mo_columns -// cannot see these rows, so restore scripts need to apply them separately. +// cannot see these rows, so callers that need complete DDL should merge or apply +// these definitions. func (r *CheckpointReader) ShowCreateIndexStatements( ctx context.Context, tableID uint64, From 513808380794d20c098e70f322c1a320791ea1e7 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 10:11:53 +0800 Subject: [PATCH 16/76] fix ckp dump restore load paths for s3 output --- cmd/mo-object-tool/ckp/checkpoint.go | 88 ++++++++++++++++++++++- cmd/mo-object-tool/ckp/checkpoint_test.go | 38 ++++++++++ pkg/tools/toolfs/storage.go | 12 +++- pkg/tools/toolfs/storage_test.go | 12 ++++ 4 files changed, 145 insertions(+), 5 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 2acf390511786..3c8b6cea456db 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -527,6 +527,10 @@ Examples: return fmt.Errorf("open dump output fileservice: %w", err) } defer dumpOut.Close(ctx) + loadPathResolver, err := newLoadDataPathResolver(outputStorage) + if err != nil { + return err + } snapshotTS, err := resolveSnapshotTS(ctx, reader, tsStr) if err != nil { @@ -605,7 +609,7 @@ Examples: fmt.Fprintf(cmd.OutOrStdout(), "Dumped %d tables to %s\n", len(tables), outputDir) } if loadScript { - scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, tables, snapshotTS, output, outputDir, !noLoad, effectiveHeader) + scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, tables, snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) if err != nil { return err } @@ -632,7 +636,7 @@ Examples: return err } } - scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, []checkpointtool.TableCatalogEntry{tableEntry}, snapshotTS, output, outputDir, !noLoad, effectiveHeader) + scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, []checkpointtool.TableCatalogEntry{tableEntry}, snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) if err != nil { return err } @@ -820,6 +824,7 @@ func writeRestoreScript( snapshotTS types.TS, scriptDir string, csvRoot string, + loadPathResolver loadDataPathResolver, includeLoad bool, csvHasHeader bool, ) (string, error) { @@ -873,7 +878,7 @@ func writeRestoreScript( return "", err } if includeLoad { - if _, err := fmt.Fprintf(f, "\nLOAD DATA INFILE %s\n", quoteSQLString(tableCSVPath(csvRoot, table))); err != nil { + if _, err := fmt.Fprintf(f, "\n%s\n", loadPathResolver.loadDataSource(csvRoot, table)); err != nil { return "", err } if _, err := fmt.Fprintf(f, "INTO TABLE %s\n", quoteSQLIdent(table.TableName)); err != nil { @@ -1361,6 +1366,83 @@ func tableCSVPath(outputDir string, table checkpointtool.TableCatalogEntry) stri ) } +type loadDataPathResolver struct { + s3Args fileservice.ObjectStorageArguments + s3Backend string +} + +func newLoadDataPathResolver(storage toolfs.StorageOptions) (loadDataPathResolver, error) { + if storage.S3 == "" { + return loadDataPathResolver{}, nil + } + backend := strings.ToUpper(storage.Backend) + if backend == "" { + backend = "S3" + } + if backend != "S3" && backend != "MINIO" { + return loadDataPathResolver{}, nil + } + args, err := toolfs.ParseS3Arguments(storage.S3, storage.FSName) + if err != nil { + return loadDataPathResolver{}, fmt.Errorf("parse output S3 arguments: %w", err) + } + if args.Bucket == "" { + return loadDataPathResolver{}, fmt.Errorf("parse output S3 arguments: missing bucket") + } + return loadDataPathResolver{s3Args: args, s3Backend: backend}, nil +} + +func (r loadDataPathResolver) loadDataSource(outputDir string, table checkpointtool.TableCatalogEntry) string { + csvPath := tableCSVPath(outputDir, table) + if r.s3Args.Bucket == "" { + return "LOAD DATA INFILE " + quoteSQLString(csvPath) + } + + options := []string{ + "bucket", r.s3Args.Bucket, + "filepath", outputS3ObjectKey(r.s3Args.KeyPrefix, csvPath), + } + if r.s3Args.Endpoint != "" { + options = append(options, "endpoint", r.s3Args.Endpoint) + } + if r.s3Args.Region != "" { + options = append(options, "region", r.s3Args.Region) + } + if r.s3Args.KeyID != "" { + options = append(options, "access_key_id", r.s3Args.KeyID) + } + if r.s3Args.KeySecret != "" { + options = append(options, "secret_access_key", r.s3Args.KeySecret) + } + if r.s3Args.RoleARN != "" { + options = append(options, "role_arn", r.s3Args.RoleARN) + } + if r.s3Args.ExternalID != "" { + options = append(options, "external_id", r.s3Args.ExternalID) + } + if r.s3Args.IsMinio || r.s3Backend == "MINIO" { + options = append(options, "provider", "minio") + } + return "LOAD DATA URL s3option{" + formatLoadDataOptions(options) + "}" +} + +func outputS3ObjectKey(keyPrefix string, csvPath string) string { + keyPrefix = strings.Trim(keyPrefix, "/") + csvPath = strings.TrimLeft(csvPath, "/") + if keyPrefix == "" { + return csvPath + } + return path.Join(keyPrefix, csvPath) +} + +func formatLoadDataOptions(options []string) string { + parts := make([]string, 0, len(options)/2) + for i := 0; i+1 < len(options); i += 2 { + parts = append(parts, quoteSQLString(options[i])+"="+quoteSQLString(options[i+1])) + } + return strings.Join(parts, ", ") +} + func outputPathJoin(elem ...string) string { return path.Join(elem...) } diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 7f939e6adef20..dd8bac2f4c7a1 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -18,7 +18,9 @@ import ( "testing" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNormalizeCreateTableDDLName(t *testing.T) { @@ -139,3 +141,39 @@ func TestCleanObjectPath(t *testing.T) { assert.Equal(t, "tmp/dump/t.csv", cleanObjectPath("/tmp/dump/t.csv")) assert.Equal(t, "dump/t.csv", cleanObjectPath("dump//nested/../t.csv")) } + +func TestLoadDataPathResolverLocal(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + AccountID: 0, + DatabaseID: 272577, + TableID: 272578, + TableName: "bmsql_config", + } + + resolver, err := newLoadDataPathResolver(toolfs.StorageOptions{}) + require.NoError(t, err) + assert.Equal(t, + "LOAD DATA INFILE 'bmsql_config/account_0/db_272577/bmsql_config_272578.csv'", + resolver.loadDataSource("bmsql_config", table), + ) +} + +func TestLoadDataPathResolverS3(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + AccountID: 0, + DatabaseID: 272577, + TableID: 272578, + TableName: "bmsql_config", + } + + resolver, err := newLoadDataPathResolver(toolfs.StorageOptions{ + S3: "bucket=mo-nightly-gz-1308875761,endpoint=https://cos.ap-guangzhou.myqcloud.com,region=ap-guangzhou," + + "key-prefix=/ckp-dump-test/tpcc_100_20260612_174916/bmsql_config/,key-id=xxx,key-secret=yyy", + Backend: "S3", + }) + require.NoError(t, err) + assert.Equal(t, + "LOAD DATA URL s3option{'bucket'='mo-nightly-gz-1308875761', 'filepath'='ckp-dump-test/tpcc_100_20260612_174916/bmsql_config/bmsql_config/account_0/db_272577/bmsql_config_272578.csv', 'endpoint'='https://cos.ap-guangzhou.myqcloud.com', 'region'='ap-guangzhou', 'access_key_id'='xxx', 'secret_access_key'='yyy'}", + resolver.loadDataSource("bmsql_config", table), + ) +} diff --git a/pkg/tools/toolfs/storage.go b/pkg/tools/toolfs/storage.go index 732d68803bc4a..d82074adb37b4 100644 --- a/pkg/tools/toolfs/storage.go +++ b/pkg/tools/toolfs/storage.go @@ -60,8 +60,8 @@ func Open(ctx context.Context, opts StorageOptions) (fileservice.FileService, st return nil, "", moerr.NewInvalidInputNoCtx("missing --s3 arguments") } - args := fileservice.ObjectStorageArguments{Name: opts.FSName} - if err := args.SetFromString(splitArgs(opts.S3)); err != nil { + args, err := ParseS3Arguments(opts.S3, opts.FSName) + if err != nil { return nil, "", err } cfg := fileservice.Config{ @@ -145,6 +145,14 @@ func openFromConfig(ctx context.Context, path string, fsName string) (fileservic ) } +func ParseS3Arguments(s string, fsName string) (fileservice.ObjectStorageArguments, error) { + args := fileservice.ObjectStorageArguments{Name: fsName} + if err := args.SetFromString(splitArgs(s)); err != nil { + return fileservice.ObjectStorageArguments{}, err + } + return args, nil +} + func splitArgs(s string) []string { parts := strings.Split(s, ",") ret := make([]string, 0, len(parts)) diff --git a/pkg/tools/toolfs/storage_test.go b/pkg/tools/toolfs/storage_test.go index 99419129ae62c..0f7bf2212a4d5 100644 --- a/pkg/tools/toolfs/storage_test.go +++ b/pkg/tools/toolfs/storage_test.go @@ -75,6 +75,18 @@ func TestOpenFromConfigFallsBackToDataDir(t *testing.T) { assert.Contains(t, display, dir) } +func TestParseS3Arguments(t *testing.T) { + args, err := ParseS3Arguments("bucket=b,endpoint=http://minio:9000,prefix=/dump/,key-id=k,key-secret=s", "OUT") + require.NoError(t, err) + + assert.Equal(t, "OUT", args.Name) + assert.Equal(t, "b", args.Bucket) + assert.Equal(t, "http://minio:9000", args.Endpoint) + assert.Equal(t, "/dump/", args.KeyPrefix) + assert.Equal(t, "k", args.KeyID) + assert.Equal(t, "s", args.KeySecret) +} + func writeConfig(t *testing.T, content string) string { t.Helper() path := filepath.Join(t.TempDir(), "tn.toml") From fefe3cf7c7f6eef1ca32628476ed518a445d076a Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 11:13:23 +0800 Subject: [PATCH 17/76] feat(ckp): remove --table dump by name, set --jobs default to 5 Remove the single-table dump by name (--table flag) functionality from ckp dump. The resolveTableByName function and all related validation logic are removed. Also change the default --jobs value from 1 to 5 for batch dump concurrency. Co-Authored-By: Claude Fable 5 --- cmd/mo-object-tool/ckp/checkpoint.go | 66 ++-------------------------- 1 file changed, 4 insertions(+), 62 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 3c8b6cea456db..15faca9cf5e85 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -424,7 +424,6 @@ func cleanObjectPath(filePath string) string { func dumpCommand(storage *toolfs.StorageOptions) *cobra.Command { var ( tableID uint64 - tableName string accountID uint32 databaseID uint64 tsStr string @@ -455,7 +454,6 @@ Examples: mo-tool ckp dump --table-id=12345 -o users.csv . # dump to file mo-tool ckp dump --table-id=12345 --ts=1749001234567890:1 . # dump at specific TS mo-tool ckp dump --table-id=12345 --load-script -o /tmp/ . - mo-tool ckp dump --database-id=9001 --table=users -o users.csv . mo-tool ckp dump --database-id=9001 --output-dir=/tmp/test-dump --jobs=4 . mo-tool ckp dump --database-id=9001 --load-script -o /tmp/test-dump . mo-tool ckp dump --table-id=12345 -o dump/users.csv --out-s3='endpoint=...,bucket=...,key-prefix=...,key-id=...,key-secret=...' .`, @@ -479,24 +477,17 @@ Examples: } accountIDSet := cmd.Flags().Changed("account-id") - tableName = strings.TrimSpace(tableName) - batchDump := tableID == 0 && tableName == "" + batchDump := tableID == 0 databaseIDSet := cmd.Flags().Changed("database-id") - if tableID == 0 && tableName == "" && !databaseIDSet && !accountIDSet { - return fmt.Errorf("--table-id, --table, or at least one of --database-id/--account-id is required") + if tableID == 0 && !databaseIDSet && !accountIDSet { + return fmt.Errorf("--table-id, or at least one of --database-id/--account-id is required") } if noLoad && !loadScript { return fmt.Errorf("--no-load requires --load-script") } - if tableID != 0 && tableName != "" { - return fmt.Errorf("--table-id cannot be combined with --table") - } if tableID != 0 && (databaseIDSet || accountIDSet || (outputDir != "" && !loadScript)) { return fmt.Errorf("--table-id cannot be combined with --database-id, --account-id, or --output-dir") } - if tableName != "" && outputDir != "" && !loadScript { - return fmt.Errorf("--output-dir is only valid for --database-id/--account-id batch dumps") - } if batchDump && output != "" && !loadScript { return fmt.Errorf("--output cannot be used when dumping by --database-id or --account-id; use --output-dir") } @@ -553,21 +544,6 @@ Examples: return err } - if tableName != "" { - var accountFilter *uint32 - if accountIDSet { - accountFilter = &accountID - } - var dbFilter *uint64 - if databaseIDSet { - dbFilter = &databaseID - } - table, err := resolveTableByName(ctx, reader, snapshotTS, tableName, dbFilter, accountFilter) - if err != nil { - return err - } - tableID = table.TableID - } effectiveHeader := header || (loadScript && !noLoad) if batchDump { @@ -664,13 +640,12 @@ Examples: } cmd.Flags().Uint64Var(&tableID, "table-id", 0, "Table ID to dump") - cmd.Flags().StringVar(&tableName, "table", "", "Table name to dump; use --database-id and/or --account-id to disambiguate") cmd.Flags().Uint32Var(&accountID, "account-id", 0, "Account ID to dump tables for; combine with --database-id to narrow the result") cmd.Flags().Uint64Var(&databaseID, "database-id", 0, "Database ID to dump tables for") cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp: physical:logical, physical-logical, RFC3339, or '2006-01-02 15:04:05' in local time (default: latest)") cmd.Flags().StringVarP(&output, "output", "o", "", "Output CSV file path (default: stdout)") cmd.Flags().StringVar(&outputDir, "output-dir", "", "Output directory for database/account dumps") - cmd.Flags().IntVar(&jobs, "jobs", 1, "Concurrent table dumps for --database-id/--account-id batch dumps") + cmd.Flags().IntVar(&jobs, "jobs", 5, "Concurrent table dumps for --database-id/--account-id batch dumps") cmd.Flags().BoolVar(&metaComments, "meta-comments", false, "Prepend DDL and row-count comment lines (disabled by default so output can be loaded directly)") cmd.Flags().BoolVar(&header, "header", false, "Include a CSV header row with column names") cmd.Flags().BoolVar(&loadScript, "load-script", false, "Generate restore.sql with CREATE DATABASE, CREATE TABLE, and LOAD DATA statements; --output/-o is treated as a directory") @@ -762,39 +737,6 @@ func allDigits(s string) bool { return true } -func resolveTableByName( - ctx context.Context, - reader *checkpointtool.CheckpointReader, - snapshotTS types.TS, - tableName string, - databaseID *uint64, - accountID *uint32, -) (checkpointtool.TableCatalogEntry, error) { - tables, err := reader.ListCatalogTables(ctx, snapshotTS, checkpointtool.TableListOptions{ - AccountID: accountID, - DatabaseID: databaseID, - }) - if err != nil { - return checkpointtool.TableCatalogEntry{}, fmt.Errorf("list checkpoint catalog tables: %w", err) - } - var matches []checkpointtool.TableCatalogEntry - for _, table := range tables { - if table.TableName == tableName { - matches = append(matches, table) - } - } - if len(matches) == 0 { - return checkpointtool.TableCatalogEntry{}, fmt.Errorf("table %q not found in checkpoint catalog", tableName) - } - if len(matches) > 1 { - var names []string - for _, table := range matches { - names = append(names, fmt.Sprintf("account=%d database=%s table-id=%d", table.AccountID, table.DatabaseName, table.TableID)) - } - return checkpointtool.TableCatalogEntry{}, fmt.Errorf("table %q is ambiguous; use --database-id/--account-id/--table-id (%s)", tableName, strings.Join(names, "; ")) - } - return matches[0], nil -} func resolveTableByID( ctx context.Context, From 72e35a17e6477100a638a01fd662c1f3d4f7531b Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 11:28:37 +0800 Subject: [PATCH 18/76] fix(ckp): rebuild show create from mo_columns --- pkg/tools/checkpointtool/table_dump.go | 103 ++++++++++++++++---- pkg/tools/checkpointtool/table_dump_test.go | 49 ++++++++++ 2 files changed, 132 insertions(+), 20 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 51b4e248e925e..e65cc67443d73 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -2263,6 +2263,59 @@ func findCreateSQLFromMoTables(view *LogicalTableView, tableID uint64) string { return "" } +func findTableNameFromMoTables(view *LogicalTableView, tableID uint64) string { + tableIDStr := fmt.Sprintf("%d", tableID) + try := func(relIDCol, relNameCol int) string { + if relIDCol < 0 || relNameCol < 0 { + return "" + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || relNameCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr && row[relNameCol] != "" { + return strings.Clone(row[relNameCol]) + } + } + return "" + } + + relIDCol := fallbackCatalogColIndex(view, moTablesID, "rel_id") + relNameCol := fallbackCatalogColIndex(view, moTablesID, "relname") + if name := try(relIDCol, relNameCol); name != "" { + return name + } + + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { + relIDCol = catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset) + relNameCol = catalogColIndexForLayout(match.layout, moTablesID, "relname", match.offset) + if name := try(relIDCol, relNameCol); name != "" { + return name + } + } + return "" +} + +func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView *LogicalTableView) string { + tableName := "" + if moTablesView != nil { + tableName = findTableNameFromMoTables(moTablesView, tableID) + } + if moColumnsView != nil { + if ddl := buildCreateTableFromMoColumns(moColumnsView, tableID, tableName); ddl != "" { + return ddl + } + } + if moTablesView != nil { + if ddl := findCreateSQLFromMoTables(moTablesView, tableID); ddl != "" { + return ddl + } + } + return "" +} + func buildCatalogTablesFromMoTablesRows(view *LogicalTableView) []TableCatalogEntry { seen := make(map[uint64]struct{}) merged := make([]TableCatalogEntry, 0) @@ -2496,10 +2549,13 @@ func buildColumnsFromMoColumnsRowsAt( // the checkpoint's mo_tables and mo_columns system tables (GCKP + following ICKPs). // // Priority: -// 1. mo_tables.rel_createsql — the original CREATE TABLE SQL (most accurate) -// 2. Reconstructed from mo_columns (attname, atttyp, attnum, att_is_hidden) +// 1. Reconstructed from mo_columns (attname, atttyp, attnum, att_is_hidden) +// 2. mo_tables.rel_createsql — fallback when mo_columns metadata is unavailable // 3. Hardcoded built-in table schemas (for core system tables like mo_tables, mo_columns, etc.) // +// ALTER TABLE updates mo_columns but does not rewrite mo_tables.rel_createsql, so +// mo_columns is the authoritative source for the current visible column set. +// // NOTE: // We intentionally do not synthesize a CREATE TABLE from raw physical object column // types when mo_tables/mo_columns metadata is unavailable. Physical object columns may @@ -2510,20 +2566,21 @@ func (r *CheckpointReader) ShowCreateTable( tableID uint64, snapshotTS types.TS, ) (string, error) { - // 1. Try mo_tables.rel_createsql + // Read mo_tables first so the mo_columns path can still render the table name + // while taking the current column set from mo_columns. moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) - if err == nil && moTablesView != nil { - if ddl := findCreateSQLFromMoTables(moTablesView, tableID); ddl != "" { - return ddl, nil - } + if err != nil { + moTablesView = nil } - // 2. Reconstruct from mo_columns + // 1. Reconstruct from mo_columns. moColumnsView, err := r.getTableLogicalView(ctx, moColumnsID, snapshotTS) - if err == nil && moColumnsView != nil { - if ddl := buildCreateTableFromMoColumns(moColumnsView, tableID); ddl != "" { - return ddl, nil - } + if err != nil { + moColumnsView = nil + } + + if ddl := createTableDDLFromCatalogViews(tableID, moTablesView, moColumnsView); ddl != "" { + return ddl, nil } // 3. Hardcoded built-in table schemas @@ -2785,7 +2842,12 @@ func getTableName(view *LogicalTableView, tableID uint64) string { } // buildCreateTableFromMoColumns reconstructs a CREATE TABLE DDL from mo_columns data. -func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) string { +func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, tableNames ...string) string { + tableName := fmt.Sprintf("%d", tableID) + if len(tableNames) > 0 && tableNames[0] != "" { + tableName = tableNames[0] + } + relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") typCol := fallbackCatalogColIndex(view, moColumnsID, "atttyp") @@ -2793,7 +2855,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) strin hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -2805,7 +2867,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) strin typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -2816,6 +2878,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64) strin func buildCreateTableFromMoColumnsAt( view *LogicalTableView, tableID uint64, + tableName string, relnameIDCol int, nameCol int, typCol int, @@ -2864,16 +2927,16 @@ func buildCreateTableFromMoColumnsAt( var sb strings.Builder sb.WriteString("CREATE TABLE ") - sb.WriteString(fmt.Sprintf("`%d`", tableID)) + sb.WriteString(quoteDDLIdent(tableName)) sb.WriteString(" (\n") for i, c := range cols { - sb.WriteString(" `") if c.name != "" { - sb.WriteString(c.name) + sb.WriteString(" ") + sb.WriteString(quoteDDLIdent(c.name)) } else { - sb.WriteString(fmt.Sprintf("col_%d", i)) + sb.WriteString(" ") + sb.WriteString(quoteDDLIdent(fmt.Sprintf("col_%d", i))) } - sb.WriteString("`") if c.sqlType != "" { sb.WriteString(" ") sb.WriteString(c.sqlType) diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 2f201dd76417a..722450e24f22b 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -389,6 +389,55 @@ func TestFindCreateSQLFromMoTables_GenericWithTrailingColumns(t *testing.T) { assert.Equal(t, "CREATE TABLE employees (id INT)", findCreateSQLFromMoTables(view, 12345)) } +func TestCreateTableDDLFromCatalogViews_PrefersMoColumnsOverStaleCreateSQL(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "col_4", "col_5", "col_6", "rel_createsql", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "12345", "t", "db", "100", + "", "", "", "CREATE TABLE t (a INT, b VARCHAR(100))", + }, + }, + } + moColumnsView := &LogicalTableView{ + Headers: []string{"object", "block", "row", "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden"}, + Rows: [][]string{ + {"obj1", "0", "0", "12345", "a", "INT", "1", "0"}, + {"obj1", "0", "1", "12345", "c", "BIGINT", "2", "0"}, + }, + } + + ddl := createTableDDLFromCatalogViews(12345, moTablesView, moColumnsView) + assert.Contains(t, ddl, "CREATE TABLE `t`") + assert.Contains(t, ddl, "`a` INT") + assert.Contains(t, ddl, "`c` BIGINT") + assert.NotContains(t, ddl, "`b`") +} + +func TestCreateTableDDLFromCatalogViews_FallsBackToRelCreateSQL(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "col_4", "col_5", "col_6", "rel_createsql", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "12345", "employees", "db", "100", + "", "", "", "CREATE TABLE employees (id INT)", + }, + }, + } + + assert.Equal(t, "CREATE TABLE employees (id INT)", createTableDDLFromCatalogViews(12345, moTablesView, nil)) +} + func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testing.T) { headers := []string{"object", "block", "row"} for i := 0; i < len(preCPKLayout.moTablesSchema)+2; i++ { From 15e3a54ecbf9ecd03811006666e9814fc5a52647 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 11:35:55 +0800 Subject: [PATCH 19/76] fix multipart upload small final part --- pkg/fileservice/aws_sdk_v2.go | 48 ++++++++++++++++++++++------ pkg/fileservice/parallel_sdk_test.go | 35 ++++++++++++++------ pkg/fileservice/qcloud_sdk.go | 48 ++++++++++++++++++++++------ 3 files changed, 101 insertions(+), 30 deletions(-) diff --git a/pkg/fileservice/aws_sdk_v2.go b/pkg/fileservice/aws_sdk_v2.go index 49bff5d58ad5a..a6fe508d34dc4 100644 --- a/pkg/fileservice/aws_sdk_v2.go +++ b/pkg/fileservice/aws_sdk_v2.go @@ -656,14 +656,9 @@ func (a *AwsSDKv2) WriteMultipartParallel( } } - if !sendJob(firstBufPtr, firstBuf, firstN) { - close(jobCh) - wg.Wait() - if firstErr != nil { - return firstErr - } - return ctx.Err() - } + pendingBufPtr := firstBufPtr + pendingBuf := firstBuf + pendingN := firstN for { nextBufPtr, nextBuf, nextN, readErr := readChunk() @@ -683,12 +678,45 @@ func (a *AwsSDKv2) WriteMultipartParallel( } break } - if !sendJob(nextBufPtr, nextBuf, nextN) { + if readErr != nil && errors.Is(readErr, io.EOF) { + if int64(pendingN)+int64(nextN) <= maxMultipartPartSize { + merged := make([]byte, pendingN+nextN) + copy(merged, pendingBuf[:pendingN]) + copy(merged[pendingN:], nextBuf[:nextN]) + bufPool.Put(pendingBufPtr) + bufPool.Put(nextBufPtr) + if !sendJob(nil, merged, len(merged)) { + break + } + } else { + if !sendJob(pendingBufPtr, pendingBuf, pendingN) { + if nextBufPtr != nil { + bufPool.Put(nextBufPtr) + } + break + } + if !sendJob(nextBufPtr, nextBuf, nextN) { + break + } + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } - if readErr != nil && errors.Is(readErr, io.EOF) { + if !sendJob(pendingBufPtr, pendingBuf, pendingN) { + if nextBufPtr != nil { + bufPool.Put(nextBufPtr) + } break } + pendingBufPtr = nextBufPtr + pendingBuf = nextBuf + pendingN = nextN + } + + if pendingN > 0 { + _ = sendJob(pendingBufPtr, pendingBuf, pendingN) } close(jobCh) diff --git a/pkg/fileservice/parallel_sdk_test.go b/pkg/fileservice/parallel_sdk_test.go index 5a7d699fa5ace..c24b51dbbb7b5 100644 --- a/pkg/fileservice/parallel_sdk_test.go +++ b/pkg/fileservice/parallel_sdk_test.go @@ -184,8 +184,11 @@ func TestAwsParallelMultipartSuccess(t *testing.T) { if err != nil { t.Fatalf("write failed: %v, parts=%d, complete=%s", err, len(state.parts), string(state.completeBody)) } - if len(state.parts) != 2 { - t.Fatalf("expected 2 parts, got %d", len(state.parts)) + if len(state.parts) != 1 { + t.Fatalf("expected merged final part, got %d parts", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) } if len(state.completeBody) == 0 { t.Fatalf("complete body not recorded") @@ -294,8 +297,11 @@ func TestAwsParallelMultipartUnknownSize(t *testing.T) { }); err != nil { t.Fatalf("write failed: %v", err) } - if len(state.parts) != 2 { - t.Fatalf("expected multipart upload with unknown size") + if len(state.parts) != 1 { + t.Fatalf("expected multipart upload with merged final part, got %d parts", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) } } @@ -353,8 +359,11 @@ func TestAwsWriteLargeNonSeekableFallsBackToMultipart(t *testing.T) { if state.putCount != 0 { t.Fatalf("expected multipart fallback instead of raw put, got %d put requests", state.putCount) } - if len(state.parts) != 2 { - t.Fatalf("expected 2 multipart parts, got %d", len(state.parts)) + if len(state.parts) != 1 { + t.Fatalf("expected merged multipart part, got %d", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) } if len(state.completeBody) == 0 { t.Fatalf("expected multipart complete request") @@ -575,8 +584,11 @@ func TestCOSParallelMultipartSuccess(t *testing.T) { if err != nil { t.Fatalf("write failed: %v, parts=%d, complete=%s", err, len(state.parts), string(state.completeBody)) } - if len(state.parts) != 2 { - t.Fatalf("expected 2 parts, got %d", len(state.parts)) + if len(state.parts) != 1 { + t.Fatalf("expected merged final part, got %d parts", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) } if len(state.completeBody) == 0 { t.Fatalf("complete body not recorded") @@ -683,8 +695,11 @@ func TestCOSParallelMultipartUnknownSize(t *testing.T) { }); err != nil { t.Fatalf("write failed: %v", err) } - if len(state.parts) != 2 { - t.Fatalf("expected multipart upload with unknown size") + if len(state.parts) != 1 { + t.Fatalf("expected multipart upload with merged final part, got %d parts", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) } } diff --git a/pkg/fileservice/qcloud_sdk.go b/pkg/fileservice/qcloud_sdk.go index 165199aa37563..30ab44a9625fa 100644 --- a/pkg/fileservice/qcloud_sdk.go +++ b/pkg/fileservice/qcloud_sdk.go @@ -479,14 +479,9 @@ func (a *QCloudSDK) WriteMultipartParallel( } } - if !sendJob(firstBufPtr, firstBuf, firstN) { - close(jobCh) - wg.Wait() - if firstErr != nil { - return firstErr - } - return ctx.Err() - } + pendingBufPtr := firstBufPtr + pendingBuf := firstBuf + pendingN := firstN for { nextBufPtr, nextBuf, nextN, readErr := readChunk() @@ -506,12 +501,45 @@ func (a *QCloudSDK) WriteMultipartParallel( } break } - if !sendJob(nextBufPtr, nextBuf, nextN) { + if readErr != nil && errors.Is(readErr, io.EOF) { + if int64(pendingN)+int64(nextN) <= maxMultipartPartSize { + merged := make([]byte, pendingN+nextN) + copy(merged, pendingBuf[:pendingN]) + copy(merged[pendingN:], nextBuf[:nextN]) + bufPool.Put(pendingBufPtr) + bufPool.Put(nextBufPtr) + if !sendJob(nil, merged, len(merged)) { + break + } + } else { + if !sendJob(pendingBufPtr, pendingBuf, pendingN) { + if nextBufPtr != nil { + bufPool.Put(nextBufPtr) + } + break + } + if !sendJob(nextBufPtr, nextBuf, nextN) { + break + } + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } - if readErr != nil && errors.Is(readErr, io.EOF) { + if !sendJob(pendingBufPtr, pendingBuf, pendingN) { + if nextBufPtr != nil { + bufPool.Put(nextBufPtr) + } break } + pendingBufPtr = nextBufPtr + pendingBuf = nextBuf + pendingN = nextN + } + + if pendingN > 0 { + _ = sendJob(pendingBufPtr, pendingBuf, pendingN) } close(jobCh) From 69ad9bc0844a49e0238ac4eab7517b80f701e393 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 12:05:21 +0800 Subject: [PATCH 20/76] fix(fileservice): avoid qcloud multipart upload starvation --- pkg/fileservice/parallel_sdk_test.go | 101 ++++++++++++++++++++++++ pkg/fileservice/parallel_upload_test.go | 33 ++++++++ pkg/fileservice/qcloud_sdk.go | 13 ++- pkg/fileservice/s3_fs.go | 6 +- 4 files changed, 145 insertions(+), 8 deletions(-) diff --git a/pkg/fileservice/parallel_sdk_test.go b/pkg/fileservice/parallel_sdk_test.go index c24b51dbbb7b5..b69f17a2ebed5 100644 --- a/pkg/fileservice/parallel_sdk_test.go +++ b/pkg/fileservice/parallel_sdk_test.go @@ -520,6 +520,15 @@ func newMockCOSServer(t *testing.T, failPart int) (*httptest.Server, *cosServerS } state.mu.Lock() state.completeBody = append([]byte{}, body...) + for partNum := 1; partNum < len(state.parts); partNum++ { + if len(state.parts[partNum]) < int(minMultipartPartSize) { + state.mu.Unlock() + w.WriteHeader(http.StatusBadRequest) + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(`EntityTooSmallYour proposed upload is smaller than the minimum allowed object size.`)) + return + } + } state.mu.Unlock() w.Header().Set("Content-Type", "application/xml") state.respBody = `locbucketobjectetag` @@ -703,6 +712,50 @@ func TestCOSParallelMultipartUnknownSize(t *testing.T) { } } +func TestCOSParallelMultipartPipeSmallTail(t *testing.T) { + server, state := newMockCOSServer(t, 0) + defer server.Close() + state.uploadID = "cos-pipe-small-tail" + + sdk := newTestCOSClient(t, server) + pr, pw := io.Pipe() + writeErrCh := make(chan error, 1) + go func() { + chunk := bytes.Repeat([]byte("x"), 1<<20) + for i := 0; i < 10; i++ { + if _, err := pw.Write(chunk); err != nil { + writeErrCh <- err + return + } + } + if _, err := pw.Write([]byte("tail")); err != nil { + writeErrCh <- err + return + } + writeErrCh <- pw.Close() + }() + + err := sdk.WriteMultipartParallel(context.Background(), "object", pr, nil, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 2, + }) + if writeErr := <-writeErrCh; writeErr != nil { + t.Fatalf("pipe writer failed: %v", writeErr) + } + if err != nil { + t.Fatalf("write failed: %v, parts=%d, complete=%s", err, len(state.parts), string(state.completeBody)) + } + if len(state.parts) != 2 { + t.Fatalf("expected 2 parts, got %d", len(state.parts)) + } + if len(state.parts[1]) != int(minMultipartPartSize) { + t.Fatalf("expected first part size %d, got %d", minMultipartPartSize, len(state.parts[1])) + } + if len(state.parts[2]) != int(minMultipartPartSize)+len("tail") { + t.Fatalf("expected merged final part size %d, got %d", int(minMultipartPartSize)+len("tail"), len(state.parts[2])) + } +} + func TestCOSMultipartEmptyReader(t *testing.T) { server, state := newMockCOSServer(t, 0) defer server.Close() @@ -740,3 +793,51 @@ func TestCOSMultipartCreateFail(t *testing.T) { t.Fatalf("expected create multipart error") } } + +func TestCOSParallelMultipartDoesNotDeadlockOnTinyGlobalPool(t *testing.T) { + server, state := newMockCOSServer(t, 0) + defer server.Close() + state.uploadID = "cos-no-deadlock" + + sdk := newTestCOSClient(t, server) + data := bytes.Repeat([]byte("n"), int(minMultipartPartSize*2)) + size := int64(len(data)) + + oldPool := parallelUploadPool + tinyPool, err := ants.NewPool(1) + if err != nil { + t.Fatalf("create ants pool: %v", err) + } + parallelUploadPool = tinyPool + defer func() { + tinyPool.Release() + parallelUploadPool = oldPool + }() + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { + done <- sdk.WriteMultipartParallel(ctx, "object", bytes.NewReader(data), &size, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 2, + }) + }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("write failed: %v", err) + } + case <-ctx.Done(): + t.Fatalf("multipart upload timed out, likely deadlocked") + } + + if len(state.parts) != 2 { + t.Fatalf("expected 2 parts, got %d", len(state.parts)) + } + if len(state.completeBody) == 0 { + t.Fatalf("expected multipart complete request") + } +} diff --git a/pkg/fileservice/parallel_upload_test.go b/pkg/fileservice/parallel_upload_test.go index e3e3fb435931b..2be8c7a0e9e0e 100644 --- a/pkg/fileservice/parallel_upload_test.go +++ b/pkg/fileservice/parallel_upload_test.go @@ -277,6 +277,39 @@ func TestS3FSWriteUnknownSizeDefaultNoParallel(t *testing.T) { } } +func TestS3FSWriteContextParallelModeOverridesFSMode(t *testing.T) { + storage := &mockParallelStorage{supports: true} + fs := &S3FS{ + name: "s3", + storage: storage, + ioMerger: NewIOMerger(), + asyncUpdate: true, + parallelMode: ParallelAuto, + } + + reader := strings.NewReader("streamed-data") + vector := IOVector{ + FilePath: "unknown-context-off", + Entries: []IOEntry{ + {Offset: 0, Size: -1, ReaderForWrite: reader}, + }, + } + + ctx := WithParallelMode(context.Background(), ParallelOff) + if err := fs.Write(ctx, vector); err != nil { + t.Fatalf("write failed: %v", err) + } + if storage.mpCalled != 0 { + t.Fatalf("parallel write should be disabled by context override") + } + if storage.writeCalled != 1 { + t.Fatalf("expected single write, got %d", storage.writeCalled) + } + if string(storage.lastData) != "streamed-data" { + t.Fatalf("unexpected data %q", string(storage.lastData)) + } +} + func TestS3FSSmallSizeSkipsParallel(t *testing.T) { storage := &mockParallelStorage{supports: true} fs := &S3FS{ diff --git a/pkg/fileservice/qcloud_sdk.go b/pkg/fileservice/qcloud_sdk.go index 30ab44a9625fa..86c481ba83a5a 100644 --- a/pkg/fileservice/qcloud_sdk.go +++ b/pkg/fileservice/qcloud_sdk.go @@ -404,9 +404,11 @@ func (a *QCloudSDK) WriteMultipartParallel( jobCh := make(chan partJob, options.Concurrency*2) - startWorker := func() error { + startWorker := func() { wg.Add(1) - return getParallelUploadPool().Submit(func() { + // Use per-upload goroutines so concurrent multipart uploads do not starve + // each other on the small global parallelUploadPool. + go func() { defer wg.Done() for job := range jobCh { if ctx.Err() != nil { @@ -442,14 +444,11 @@ func (a *QCloudSDK) WriteMultipartParallel( }) partsLock.Unlock() } - }) + }() } for i := 0; i < options.Concurrency; i++ { - if submitErr := startWorker(); submitErr != nil { - setErr(submitErr) - break - } + startWorker() } sendJob := func(bufPtr *[]byte, buf []byte, n int) bool { diff --git a/pkg/fileservice/s3_fs.go b/pkg/fileservice/s3_fs.go index 023ac35ca88f2..e30e65168d82b 100644 --- a/pkg/fileservice/s3_fs.go +++ b/pkg/fileservice/s3_fs.go @@ -461,7 +461,11 @@ func (s *S3FS) write(ctx context.Context, vector IOVector) (bytesWritten int, er } key := s.pathToKey(path.File) enableParallel := false - switch s.parallelMode { + parallelMode := s.parallelMode + if mode, ok := parallelModeFromContext(ctx); ok { + parallelMode = mode + } + switch parallelMode { case ParallelForce: enableParallel = true case ParallelAuto: From 65197f1c35512549ede03b30cb650a7fe1fc96fa Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 14:22:15 +0800 Subject: [PATCH 21/76] fix(ckp): use parallel multipart for remote dump streams --- cmd/mo-object-tool/ckp/checkpoint.go | 3 ++- pkg/fileservice/parallel_upload_test.go | 33 +++++++++++++++++++++++++ pkg/fileservice/s3_fs.go | 7 ++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 15faca9cf5e85..a587658ecc363 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -380,6 +380,8 @@ func newFileServiceWriteCloser(ctx context.Context, fs fileservice.FileService, } w.done <- err }() + ctx = fileservice.WithParallelMode(ctx, fileservice.ParallelAuto) + logutil.Infof("streaming dump output to fileservice with parallel-mode=auto: %s", cleanObjectPath(filePath)) err = fs.Write(ctx, fileservice.IOVector{ FilePath: cleanObjectPath(filePath), Entries: []fileservice.IOEntry{{ @@ -737,7 +739,6 @@ func allDigits(s string) bool { return true } - func resolveTableByID( ctx context.Context, reader *checkpointtool.CheckpointReader, diff --git a/pkg/fileservice/parallel_upload_test.go b/pkg/fileservice/parallel_upload_test.go index 2be8c7a0e9e0e..bc45dc5ccff73 100644 --- a/pkg/fileservice/parallel_upload_test.go +++ b/pkg/fileservice/parallel_upload_test.go @@ -310,6 +310,39 @@ func TestS3FSWriteContextParallelModeOverridesFSMode(t *testing.T) { } } +func TestS3FSWriteContextParallelAutoEnablesUnknownSize(t *testing.T) { + storage := &mockParallelStorage{supports: true} + fs := &S3FS{ + name: "s3", + storage: storage, + ioMerger: NewIOMerger(), + asyncUpdate: true, + parallelMode: ParallelOff, + } + + reader := strings.NewReader("streamed-data") + vector := IOVector{ + FilePath: "unknown-context-auto", + Entries: []IOEntry{ + {Offset: 0, Size: -1, ReaderForWrite: reader}, + }, + } + + ctx := WithParallelMode(context.Background(), ParallelAuto) + if err := fs.Write(ctx, vector); err != nil { + t.Fatalf("write failed: %v", err) + } + if storage.mpCalled != 1 { + t.Fatalf("parallel write should be enabled by context auto override") + } + if storage.writeCalled != 0 { + t.Fatalf("unexpected single write") + } + if storage.lastSize != nil { + t.Fatalf("size hint should be nil for unknown-size stream") + } +} + func TestS3FSSmallSizeSkipsParallel(t *testing.T) { storage := &mockParallelStorage{supports: true} fs := &S3FS{ diff --git a/pkg/fileservice/s3_fs.go b/pkg/fileservice/s3_fs.go index e30e65168d82b..b27878b1589d7 100644 --- a/pkg/fileservice/s3_fs.go +++ b/pkg/fileservice/s3_fs.go @@ -473,6 +473,13 @@ func (s *S3FS) write(ctx context.Context, vector IOVector) (bytesWritten int, er enableParallel = true } } + if size == nil { + logutil.Info("s3 write unknown-size stream", + zap.String("key", key), + zap.Uint8("parallel-mode", uint8(parallelMode)), + zap.Bool("parallel-enabled", enableParallel), + ) + } if pmw, ok := s.storage.(ParallelMultipartWriter); ok && pmw.SupportsParallelMultipart() && enableParallel { From 7b57204ad28d8647be29cf953fffb07084221673 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 14:49:45 +0800 Subject: [PATCH 22/76] fix(toolfs): bound remote lazy cache size --- pkg/tools/toolfs/lazy_cache_fs.go | 197 +++++++++++++++++++++++-- pkg/tools/toolfs/lazy_cache_fs_test.go | 92 ++++++++++++ 2 files changed, 280 insertions(+), 9 deletions(-) diff --git a/pkg/tools/toolfs/lazy_cache_fs.go b/pkg/tools/toolfs/lazy_cache_fs.go index 61f27838e5d2f..5601790a9053b 100644 --- a/pkg/tools/toolfs/lazy_cache_fs.go +++ b/pkg/tools/toolfs/lazy_cache_fs.go @@ -20,6 +20,8 @@ import ( "iter" "os" "path/filepath" + "sort" + "strconv" "strings" "sync" @@ -27,14 +29,27 @@ import ( "github.com/matrixorigin/matrixone/pkg/fileservice" ) +const defaultLazyCacheMaxBytes int64 = 8 << 30 + +type lazyCacheEntry struct { + path string + size int64 + lastAccess uint64 +} + type lazyCacheFS struct { name string remote fileservice.FileService local *fileservice.LocalFS root string - mu sync.Mutex - caching map[string]*sync.Once + mu sync.Mutex + caching map[string]*sync.Once + entries map[string]lazyCacheEntry + reservations map[string]int64 + usedBytes int64 + maxBytes int64 + accessCounter uint64 } func newLazyCacheFS(ctx context.Context, remote fileservice.FileService) (fileservice.FileService, string, error) { @@ -48,11 +63,14 @@ func newLazyCacheFS(ctx context.Context, remote fileservice.FileService) (filese return nil, "", err } return &lazyCacheFS{ - name: remote.Name(), - remote: remote, - local: local, - root: root, - caching: make(map[string]*sync.Once), + name: remote.Name(), + remote: remote, + local: local, + root: root, + caching: make(map[string]*sync.Once), + entries: make(map[string]lazyCacheEntry), + reservations: make(map[string]int64), + maxBytes: lazyCacheMaxBytesFromEnv(), }, root, nil } @@ -66,6 +84,7 @@ func (l *lazyCacheFS) Write(ctx context.Context, vector fileservice.IOVector) er _ = os.Remove(filepath.Join(l.root, filepath.FromSlash(normalized))) l.mu.Lock() delete(l.caching, normalized) + l.removeCacheEntryLocked(normalized) l.mu.Unlock() return nil } @@ -90,6 +109,13 @@ func (l *lazyCacheFS) List(ctx context.Context, dirPath string) iter.Seq2[*files func (l *lazyCacheFS) Delete(ctx context.Context, filePaths ...string) error { _ = l.local.Delete(ctx, filePaths...) + l.mu.Lock() + for _, filePath := range filePaths { + normalized := stripServiceName(filePath, l.name) + delete(l.caching, normalized) + l.removeCacheEntryLocked(normalized) + } + l.mu.Unlock() return l.remote.Delete(ctx, filePaths...) } @@ -114,7 +140,8 @@ func (l *lazyCacheFS) Close(ctx context.Context) { func (l *lazyCacheFS) ensureCached(ctx context.Context, filePath string) error { normalized := stripServiceName(filePath, l.name) localPath := filepath.Join(l.root, filepath.FromSlash(normalized)) - if _, err := os.Stat(localPath); err == nil { + if stat, err := os.Stat(localPath); err == nil { + l.touchCacheEntry(normalized, localPath, stat.Size()) return nil } @@ -143,6 +170,15 @@ func (l *lazyCacheFS) getOnce(path string) *sync.Once { } func (l *lazyCacheFS) cacheFile(ctx context.Context, normalized string, localPath string) error { + committed := false + defer func() { + if !committed { + l.releaseCacheReservation(normalized) + } + }() + if stat, err := l.remote.StatFile(ctx, normalized); err == nil && stat != nil && stat.Size > 0 { + l.reserveCacheSpace(stat.Size, normalized) + } if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { return err } @@ -170,9 +206,16 @@ func (l *lazyCacheFS) cacheFile(ctx context.Context, normalized string, localPat if err := tmp.Close(); err != nil { return err } + stat, err := os.Stat(tmpName) + if err != nil { + return err + } + l.reserveCacheSpace(stat.Size(), normalized) if err := os.Rename(tmpName, localPath); err != nil { return err } + l.touchCacheEntry(normalized, localPath, stat.Size()) + committed = true return nil } @@ -182,7 +225,13 @@ func (l *lazyCacheFS) readCachedFile(ctx context.Context, vector *fileservice.IO file, err := os.Open(localPath) if os.IsNotExist(err) { - return moerr.NewFileNotFoundNoCtx(normalized) + if err := l.ensureCached(ctx, normalized); err != nil { + return err + } + file, err = os.Open(localPath) + if os.IsNotExist(err) { + return moerr.NewFileNotFoundNoCtx(normalized) + } } if err != nil { return err @@ -194,6 +243,7 @@ func (l *lazyCacheFS) readCachedFile(ctx context.Context, vector *fileservice.IO return err } fileSize := stat.Size() + l.touchCacheEntry(normalized, localPath, fileSize) for i, entry := range vector.Entries { if entry.Size == 0 { @@ -222,6 +272,135 @@ func (l *lazyCacheFS) readCachedFile(ctx context.Context, vector *fileservice.IO return nil } +func (l *lazyCacheFS) touchCacheEntry(normalized string, localPath string, size int64) { + l.mu.Lock() + defer l.mu.Unlock() + l.accessCounter++ + if reservation, ok := l.reservations[normalized]; ok { + l.usedBytes -= reservation + delete(l.reservations, normalized) + } + if old, ok := l.entries[normalized]; ok { + l.usedBytes -= old.size + } + l.entries[normalized] = lazyCacheEntry{ + path: localPath, + size: size, + lastAccess: l.accessCounter, + } + l.usedBytes += size +} + +func (l *lazyCacheFS) reserveCacheSpace(needed int64, protected string) { + if l.maxBytes <= 0 || needed <= 0 { + return + } + l.mu.Lock() + defer l.mu.Unlock() + reserved := l.reservations[protected] + if needed <= reserved { + return + } + delta := needed - reserved + l.evictCacheLocked(delta, protected) + l.reservations[protected] = needed + l.usedBytes += delta +} + +func (l *lazyCacheFS) evictCacheLocked(needed int64, protected string) { + if l.usedBytes+needed <= l.maxBytes { + return + } + entries := make([]lazyCacheEntry, 0, len(l.entries)) + for key, entry := range l.entries { + if key == protected { + continue + } + entries = append(entries, entry) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].lastAccess < entries[j].lastAccess + }) + for _, entry := range entries { + if l.usedBytes+needed <= l.maxBytes { + return + } + _ = os.Remove(entry.path) + for key, cached := range l.entries { + if cached.path == entry.path { + delete(l.entries, key) + delete(l.caching, key) + l.usedBytes -= cached.size + break + } + } + } +} + +func (l *lazyCacheFS) removeCacheEntryLocked(normalized string) { + if reservation, ok := l.reservations[normalized]; ok { + l.usedBytes -= reservation + delete(l.reservations, normalized) + } + if entry, ok := l.entries[normalized]; ok { + l.usedBytes -= entry.size + delete(l.entries, normalized) + } +} + +func (l *lazyCacheFS) releaseCacheReservation(normalized string) { + l.mu.Lock() + defer l.mu.Unlock() + if reservation, ok := l.reservations[normalized]; ok { + l.usedBytes -= reservation + delete(l.reservations, normalized) + } +} + +func lazyCacheMaxBytesFromEnv() int64 { + value := strings.TrimSpace(os.Getenv("MO_TOOL_REMOTE_CACHE_SIZE")) + if value == "" { + return defaultLazyCacheMaxBytes + } + if n, ok := parseLazyCacheSize(value); ok && n > 0 { + return n + } + return defaultLazyCacheMaxBytes +} + +func parseLazyCacheSize(value string) (int64, bool) { + value = strings.TrimSpace(strings.ToLower(value)) + if value == "" { + return 0, false + } + multiplier := int64(1) + for _, suffix := range []struct { + s string + m int64 + }{ + {s: "gib", m: 1 << 30}, + {s: "gb", m: 1 << 30}, + {s: "g", m: 1 << 30}, + {s: "mib", m: 1 << 20}, + {s: "mb", m: 1 << 20}, + {s: "m", m: 1 << 20}, + {s: "kib", m: 1 << 10}, + {s: "kb", m: 1 << 10}, + {s: "k", m: 1 << 10}, + } { + if strings.HasSuffix(value, suffix.s) { + multiplier = suffix.m + value = strings.TrimSpace(strings.TrimSuffix(value, suffix.s)) + break + } + } + n, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, false + } + return n * multiplier, true +} + func stripServiceName(path string, service string) string { prefix := strings.ToUpper(service) + ":" if strings.HasPrefix(strings.ToUpper(path), prefix) { diff --git a/pkg/tools/toolfs/lazy_cache_fs_test.go b/pkg/tools/toolfs/lazy_cache_fs_test.go index a3f6185ff34b8..6f1f381f50eb8 100644 --- a/pkg/tools/toolfs/lazy_cache_fs_test.go +++ b/pkg/tools/toolfs/lazy_cache_fs_test.go @@ -18,6 +18,8 @@ import ( "bytes" "context" "encoding/binary" + "os" + "path/filepath" "testing" "github.com/matrixorigin/matrixone/pkg/fileservice" @@ -70,3 +72,93 @@ func TestLazyCacheFSReadsMagicPrefixedRemoteObjectsWithoutLocalChecksum(t *testi require.NoError(t, fs.ReadCache(ctx, cacheVec)) require.Equal(t, payload[4:14], buf.Bytes()) } + +func TestLazyCacheFSEvictsOldEntriesWhenOverLimit(t *testing.T) { + t.Setenv("MO_TOOL_REMOTE_CACHE_SIZE", "32") + + ctx := context.Background() + remote, err := fileservice.NewMemoryFS("SHARED", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + + first := bytes.Repeat([]byte("a"), 24) + second := bytes.Repeat([]byte("b"), 24) + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "ckp/first", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: int64(len(first)), + Data: first, + }}, + })) + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "ckp/second", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: int64(len(second)), + Data: second, + }}, + })) + + fs, root, err := newLazyCacheFS(ctx, remote) + require.NoError(t, err) + defer fs.Close(ctx) + + readAll := func(path string) []byte { + vec := &fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: -1, + }}, + } + require.NoError(t, fs.Read(ctx, vec)) + return vec.Entries[0].Data + } + + require.Equal(t, first, readAll("ckp/first")) + require.FileExists(t, filepath.Join(root, "ckp/first")) + + require.Equal(t, second, readAll("ckp/second")) + require.NoFileExists(t, filepath.Join(root, "ckp/first")) + require.FileExists(t, filepath.Join(root, "ckp/second")) + + require.Equal(t, first, readAll("ckp/first")) + require.FileExists(t, filepath.Join(root, "ckp/first")) + require.NoFileExists(t, filepath.Join(root, "ckp/second")) +} + +func TestParseLazyCacheSize(t *testing.T) { + tests := []struct { + value string + want int64 + ok bool + }{ + {value: "42", want: 42, ok: true}, + {value: "2k", want: 2 << 10, ok: true}, + {value: "3MB", want: 3 << 20, ok: true}, + {value: "4GiB", want: 4 << 30, ok: true}, + {value: "", ok: false}, + {value: "bad", ok: false}, + } + + for _, tt := range tests { + got, ok := parseLazyCacheSize(tt.value) + require.Equal(t, tt.ok, ok) + if tt.ok { + require.Equal(t, tt.want, got) + } + } +} + +func TestLazyCacheMaxBytesFromEnvDefault(t *testing.T) { + old := os.Getenv("MO_TOOL_REMOTE_CACHE_SIZE") + t.Cleanup(func() { + if old == "" { + _ = os.Unsetenv("MO_TOOL_REMOTE_CACHE_SIZE") + } else { + _ = os.Setenv("MO_TOOL_REMOTE_CACHE_SIZE", old) + } + }) + _ = os.Unsetenv("MO_TOOL_REMOTE_CACHE_SIZE") + require.Equal(t, defaultLazyCacheMaxBytes, lazyCacheMaxBytesFromEnv()) +} From 64baae337ba0ecb8b2da822749618913b3a92653 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 15:16:43 +0800 Subject: [PATCH 23/76] fix(ckp): fallback index DDL merge for restore script --- cmd/mo-object-tool/ckp/checkpoint.go | 29 ++++++++++++- cmd/mo-object-tool/ckp/checkpoint_test.go | 53 +++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index a587658ecc363..93e4841824c57 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -893,7 +893,11 @@ func mergeCreateTableIndexDDLs(createDDL string, indexDDLs []string) (string, er } clauses = append(clauses, clause) } - return injectCreateTableClauses(createDDL, clauses) + ddl, err := injectCreateTableClauses(createDDL, clauses) + if err == nil { + return ddl, nil + } + return appendIndexDDLsAfterCreateTable(createDDL, indexDDLs), nil } func alterTableAddClause(sql string) (string, bool) { @@ -957,6 +961,16 @@ func injectCreateTableClauses(createDDL string, clauses []string) (string, error return createDDL[:close] + separator + strings.Join(clauses, ", ") + createDDL[close:], nil } +func appendIndexDDLsAfterCreateTable(createDDL string, indexDDLs []string) string { + var sb strings.Builder + sb.WriteString(strings.TrimRight(createDDL, " ;\n\t")) + for _, indexDDL := range indexDDLs { + sb.WriteString(";\n") + sb.WriteString(strings.TrimRight(strings.TrimSpace(indexDDL), " ;\n\t")) + } + return sb.String() +} + func inferCreateTableClauseIndent(body string) string { lines := strings.Split(body, "\n") for _, line := range lines[1:] { @@ -1132,6 +1146,19 @@ func createTableNameRange(sql string) (int, int, bool) { if !ok { return 0, 0, false } + for { + next, ok := consumeSQLKeyword(sql, i, "temporary") + if ok { + i = next + continue + } + next, ok = consumeSQLKeyword(sql, i, "cluster") + if ok { + i = next + continue + } + break + } i, ok = consumeSQLKeyword(sql, i, "table") if !ok { return 0, 0, false diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index dd8bac2f4c7a1..0fce6edc90801 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -136,6 +136,59 @@ func TestMergeCreateTableIndexDDLsSkipsExistingIndex(t *testing.T) { assert.Equal(t, createDDL, got) } +func TestMergeCreateTableIndexDDLsWithConstraintsAndComments(t *testing.T) { + createDDL := "CREATE TABLE `ckp_constraints`.`parent` (\n" + + " `id` INT NOT NULL PRIMARY KEY,\n" + + " `code` VARCHAR(20) NOT NULL UNIQUE,\n" + + " `note` VARCHAR(100) DEFAULT 'parent-default' COMMENT 'parent note'\n" + + ") COMMENT='parent table comment'" + indexDDLs := []string{ + "ALTER TABLE `parent` ADD KEY `idx_parent_note`(`note`);", + } + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + require.NoError(t, err) + assert.Contains(t, got, "KEY `idx_parent_note`(`note`)") + assert.Contains(t, got, ") COMMENT='parent table comment'") +} + +func TestMergeCreateTableIndexDDLsWithAutoIncrementAndClusterTable(t *testing.T) { + createDDL := "CREATE CLUSTER TABLE `ckp_constraints`.`t_auto_inc` (\n" + + " `id` BIGINT NOT NULL AUTO_INCREMENT,\n" + + " `note` VARCHAR(64) COMMENT 'note (with parens)',\n" + + " PRIMARY KEY (`id`)\n" + + ") COMMENT='auto increment table'" + indexDDLs := []string{ + "ALTER TABLE `t_auto_inc` ADD KEY `idx_auto_inc_note`(`note`);", + } + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + require.NoError(t, err) + assert.Contains(t, got, "KEY `idx_auto_inc_note`(`note`)") + assert.Contains(t, got, "COMMENT='auto increment table'") +} + +func TestMergeCreateTableIndexDDLsFallsBackToSeparateAlter(t *testing.T) { + createDDL := "CREATE TABLE `parent` LIKE `parent_template`" + indexDDLs := []string{ + "ALTER TABLE `parent` ADD KEY `idx_parent_note`(`note`);", + } + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + require.NoError(t, err) + assert.Equal(t, "CREATE TABLE `parent` LIKE `parent_template`;\nALTER TABLE `parent` ADD KEY `idx_parent_note`(`note`)", got) +} + +func TestNormalizeCreateTableDDLNameWithCreateTableModifiers(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_constraints", + TableName: "parent", + } + + got := normalizeCreateTableDDLName("CREATE CLUSTER TABLE old_parent (id INT)", table) + assert.Equal(t, "CREATE CLUSTER TABLE `ckp_constraints`.`parent` (id INT)", got) +} + func TestCleanObjectPath(t *testing.T) { assert.Equal(t, "dump/account_1/t.csv", cleanObjectPath("dump/account_1/t.csv")) assert.Equal(t, "tmp/dump/t.csv", cleanObjectPath("/tmp/dump/t.csv")) From 1167c861edadb4944ffab40a4cf3646b31bf0667 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 15:24:44 +0800 Subject: [PATCH 24/76] fix(ckp): allow dumps for catalog-only tables --- pkg/tools/checkpointtool/table_dump.go | 61 +++++++++++++-------- pkg/tools/checkpointtool/table_dump_test.go | 35 ++++++++++++ 2 files changed, 73 insertions(+), 23 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index e65cc67443d73..4d20407431f5d 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -658,10 +658,6 @@ func (r *CheckpointReader) DumpTableCSVComposed( snapshotTS types.TS, opts ...CSVExportOption, ) error { - dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) - if err != nil { - return err - } schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) if len(schema.Columns) == 0 { return moerr.NewInternalErrorf( @@ -670,6 +666,14 @@ func (r *CheckpointReader) DumpTableCSVComposed( tableID, ) } + dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) + if err != nil { + if !isTableDataUnavailable(err) { + return err + } + dataEntries = nil + tombEntries = nil + } return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries, resolveCSVExportOptions(opts)) } @@ -680,10 +684,6 @@ func (r *CheckpointReader) PrepareTableDumpData( tableID uint64, snapshotTS types.TS, ) (*TableDumpData, error) { - dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) - if err != nil { - return nil, err - } schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) if len(schema.Columns) == 0 { return nil, moerr.NewInternalErrorf( @@ -692,6 +692,14 @@ func (r *CheckpointReader) PrepareTableDumpData( tableID, ) } + dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) + if err != nil { + if !isTableDataUnavailable(err) { + return nil, err + } + dataEntries = nil + tombEntries = nil + } return &TableDumpData{ TableID: tableID, Schema: cloneTableSchema(schema), @@ -719,12 +727,19 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( if _, ok := tableSet[tableID]; ok { continue } - tbl, ok := composed.Tables[tableID] - if !ok || (len(tbl.DataRanges) == 0 && len(tbl.TombRanges) == 0) { - return nil, moerr.NewInternalErrorf(ctx, "table %d not found in checkpoint at ts %s", tableID, snapshotTS.ToString()) + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) + if len(schema.Columns) == 0 { + return nil, moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) } tableSet[tableID] = struct{}{} - result[tableID] = &TableDumpData{TableID: tableID} + result[tableID] = &TableDumpData{ + TableID: tableID, + Schema: cloneTableSchema(schema), + } } entryRefs := make([]*EntryInfo, 0, len(composed.Incrementals)+1) @@ -751,22 +766,22 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( } for tableID, data := range result { - if len(data.DataEntries) == 0 { - return nil, moerr.NewInternalErrorf(ctx, "no data entries for table %d", tableID) + if data.Schema == nil || len(data.Schema.Columns) == 0 { + return nil, moerr.NewInternalErrorf(ctx, "cannot resolve visible columns for table %d from checkpoint metadata", tableID) } - schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) - if len(schema.Columns) == 0 { - return nil, moerr.NewInternalErrorf( - ctx, - "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", - tableID, - ) - } - data.Schema = cloneTableSchema(schema) } return result, nil } +func isTableDataUnavailable(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "not found in checkpoint") || + strings.Contains(msg, "no data entries for table") +} + // DumpPreparedTableCSV writes a table using metadata previously resolved by // PrepareTableDumpData. func (r *CheckpointReader) DumpPreparedTableCSV( diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 722450e24f22b..eefe18d08d58d 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -63,6 +63,41 @@ func TestWriteCSV_empty(t *testing.T) { assert.Equal(t, "id,name", lines[csvStart]) } +func TestDumpPreparedTableCSV_EmptyTableWritesHeader(t *testing.T) { + reader := &CheckpointReader{} + data := &TableDumpData{ + TableID: 334019, + Schema: &TableSchema{ + TableName: "t_empty", + DatabaseName: "ckp_tables", + Columns: []TableColumn{ + {Name: "id", SQLType: "INT", Position: 1, PhysicalPosition: 0}, + {Name: "name", SQLType: "VARCHAR(64)", Position: 2, PhysicalPosition: 1}, + }, + }, + } + + var buf bytes.Buffer + err := reader.DumpPreparedTableCSV( + context.Background(), + &buf, + data, + types.TS{}, + WithCSVMetaComments(false), + WithCSVHeader(true), + ) + require.NoError(t, err) + assert.Equal(t, "id,name\n", buf.String()) +} + +func TestIsTableDataUnavailable(t *testing.T) { + assert.False(t, isTableDataUnavailable(assert.AnError)) + assert.False(t, isTableDataUnavailable(nil)) + assert.True(t, isTableDataUnavailable(fmt.Errorf("internal error: table 334019 not found in checkpoint at ts 1-0"))) + assert.True(t, isTableDataUnavailable(fmt.Errorf("internal error: no data entries for table 334019"))) + assert.False(t, isTableDataUnavailable(fmt.Errorf("compose checkpoint failed"))) +} + // TestWriteCSV_withData tests writing data rows to CSV. func TestWriteCSV_withData(t *testing.T) { schema := &TableSchema{ From 583011010a1e4a1a19b3e67a6b05c2812f751d33 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 15:29:39 +0800 Subject: [PATCH 25/76] fix(ckp): fallback for binary mo_columns types --- pkg/tools/checkpointtool/table_dump.go | 24 +++++++++++++++++ pkg/tools/checkpointtool/table_dump_test.go | 30 +++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 4d20407431f5d..5d780d58db38a 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -2931,6 +2931,9 @@ func buildCreateTableFromMoColumnsAt( if typCol >= 0 && typCol < len(row) { c.sqlType = row[typCol] } + if !isPrintableSQLType(c.sqlType) { + return "" + } cols = append(cols, c) } @@ -2966,6 +2969,27 @@ func buildCreateTableFromMoColumnsAt( return sb.String() } +func isPrintableSQLType(sqlType string) bool { + sqlType = strings.TrimSpace(sqlType) + if sqlType == "" { + return false + } + hasLetter := false + for _, r := range sqlType { + switch { + case r >= 'A' && r <= 'Z': + hasLetter = true + case r >= 'a' && r <= 'z': + hasLetter = true + case r >= '0' && r <= '9': + case strings.ContainsRune(" ()_,'+-.", r): + default: + return false + } + } + return hasLetter +} + // hardcodedCreateTable returns the CREATE TABLE DDL for core built-in system tables. // These tables' schemas are known at compile time and may not appear in the checkpoint's // mo_tables/mo_columns (due to minimal deployments). diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index eefe18d08d58d..aaf4876bf69f7 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -473,6 +473,36 @@ func TestCreateTableDDLFromCatalogViews_FallsBackToRelCreateSQL(t *testing.T) { assert.Equal(t, "CREATE TABLE employees (id INT)", createTableDDLFromCatalogViews(12345, moTablesView, nil)) } +func TestCreateTableDDLFromCatalogViews_FallsBackToRelCreateSQLWhenMoColumnTypesAreBinary(t *testing.T) { + const relCreateSQL = "CREATE TABLE `ckp_constraints`.`parent` (`id` INT NOT NULL PRIMARY KEY, `code` VARCHAR(20) NOT NULL UNIQUE, `note` VARCHAR(100) DEFAULT 'parent-default' COMMENT 'parent note') COMMENT='parent table comment'" + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "col_4", "col_5", "col_6", "rel_createsql", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "333999", "parent", "ckp_constraints", "333997", + "", "", "", relCreateSQL, + }, + }, + } + moColumnsView := &LogicalTableView{ + Headers: []string{"object", "block", "row", "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden"}, + Rows: [][]string{ + {"obj1", "0", "0", "333999", "id", string([]byte{'A', 'L', 0, 0xff, 'X'}), "1", "0"}, + {"obj1", "0", "1", "333999", "code", "VARCHAR(20)", "2", "0"}, + }, + } + + ddl := createTableDDLFromCatalogViews(333999, moTablesView, moColumnsView) + assert.Equal(t, relCreateSQL, ddl) + assert.NotContains(t, ddl, "\x00") + assert.NotContains(t, ddl, "\ufffd") +} + func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testing.T) { headers := []string{"object", "block", "row"} for i := 0; i < len(preCPKLayout.moTablesSchema)+2; i++ { From 34e774412e615893bea81c1ebf838335be919d7b Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 15:35:38 +0800 Subject: [PATCH 26/76] fix(ckp): skip corrupt create sql candidates --- pkg/tools/checkpointtool/table_dump.go | 25 ++++++++++++-- pkg/tools/checkpointtool/table_dump_test.go | 36 +++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 5d780d58db38a..d9e1e6f377da3 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -2238,7 +2238,9 @@ func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *Table createSQLIdx := fallbackCatalogColIndex(view, moTablesID, "rel_createsql") if createSQLIdx < len(dataRow) { - schema.CreateSQL = strings.Clone(dataRow[createSQLIdx]) + if isPrintableCreateTableSQL(dataRow[createSQLIdx]) { + schema.CreateSQL = strings.Clone(dataRow[createSQLIdx]) + } } return schema } @@ -2254,7 +2256,7 @@ func findCreateSQLFromMoTables(view *LogicalTableView, tableID uint64) string { if relIDCol >= len(row) || createSQLCol >= len(row) { continue } - if row[relIDCol] == tableIDStr && row[createSQLCol] != "" { + if row[relIDCol] == tableIDStr && isPrintableCreateTableSQL(row[createSQLCol]) { return strings.Clone(row[createSQLCol]) } } @@ -2990,6 +2992,25 @@ func isPrintableSQLType(sqlType string) bool { return hasLetter } +func isPrintableCreateTableSQL(ddl string) bool { + ddl = strings.TrimSpace(ddl) + if ddl == "" { + return false + } + for _, r := range ddl { + switch { + case r == '\n' || r == '\r' || r == '\t': + case r >= 0x20 && r <= 0x7e: + default: + return false + } + } + upper := strings.ToUpper(ddl) + return strings.HasPrefix(upper, "CREATE TABLE ") || + strings.HasPrefix(upper, "CREATE TEMPORARY TABLE ") || + strings.HasPrefix(upper, "CREATE CLUSTER TABLE ") +} + // hardcodedCreateTable returns the CREATE TABLE DDL for core built-in system tables. // These tables' schemas are known at compile time and may not appear in the checkpoint's // mo_tables/mo_columns (due to minimal deployments). diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index aaf4876bf69f7..9aa4d11f0e469 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -424,6 +424,42 @@ func TestFindCreateSQLFromMoTables_GenericWithTrailingColumns(t *testing.T) { assert.Equal(t, "CREATE TABLE employees (id INT)", findCreateSQLFromMoTables(view, 12345)) } +func TestFindCreateSQLFromMoTables_SkipsBinaryCreateSQLCandidates(t *testing.T) { + headers := []string{"object", "block", "row"} + for i := 0; i < len(currentCatalogLayout.moTablesSchema)+2; i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + headers[logicalViewMetaCols+0] = "rel_id" + headers[logicalViewMetaCols+7] = "rel_createsql" + row := make([]string, len(headers)) + row[0], row[1], row[2] = "obj1", "0", "0" + + row[logicalViewMetaCols+0] = "334019" + row[logicalViewMetaCols+7] = string([]byte{0xff, '/', 0, 0xfe, '0'}) + + offset := 2 + row[logicalViewMetaCols+offset+0] = "334019" + row[logicalViewMetaCols+offset+7] = "CREATE TABLE `ckp_tables`.`t_empty` (`id` INT)" + + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{row}, + } + + assert.Equal(t, "CREATE TABLE `ckp_tables`.`t_empty` (`id` INT)", findCreateSQLFromMoTables(view, 334019)) +} + +func TestFindCreateSQLFromMoTables_RejectsBinaryCreateSQL(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "rel_id", "relname", "reldatabase", "reldatabase_id", "col_4", "col_5", "col_6", "rel_createsql"}, + Rows: [][]string{ + {"obj1", "0", "0", "334019", "t_empty", "ckp_tables", "334017", "", "", "", string([]byte{0xff, '/', 0, 0xfe, '0'})}, + }, + } + + assert.Empty(t, findCreateSQLFromMoTables(view, 334019)) +} + func TestCreateTableDDLFromCatalogViews_PrefersMoColumnsOverStaleCreateSQL(t *testing.T) { moTablesView := &LogicalTableView{ Headers: []string{ From 700bd0d851d931735cb701ee5202f5252ecaa29e Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 15:42:16 +0800 Subject: [PATCH 27/76] fix(ckp): reuse prepared schema for restore ddl --- cmd/mo-object-tool/ckp/checkpoint.go | 54 ++++++++++++++++----- pkg/tools/checkpointtool/table_dump.go | 25 ++++++++++ pkg/tools/checkpointtool/table_dump_test.go | 28 +++++++++++ 3 files changed, 94 insertions(+), 13 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 93e4841824c57..d2c4c764c8b86 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -570,6 +570,13 @@ Examples: if err := dumpOut.MkdirAll(outputDir); err != nil { return fmt.Errorf("create output dir: %w", err) } + var dumpPlans []tableDumpPlan + if loadScript || !noLoad { + dumpPlans, err = prepareTableDumpPlans(ctx, reader, tables, snapshotTS) + if err != nil { + return err + } + } if !noLoad { if jobs < 1 { jobs = 1 @@ -577,17 +584,13 @@ Examples: if jobs > len(tables) { jobs = len(tables) } - dumpPlans, err := prepareTableDumpPlans(ctx, reader, tables, snapshotTS) - if err != nil { - return err - } if err := dumpTablesConcurrently(ctx, reader, dumpOut, dumpPlans, snapshotTS, outputDir, jobs, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout()); err != nil { return err } fmt.Fprintf(cmd.OutOrStdout(), "Dumped %d tables to %s\n", len(tables), outputDir) } if loadScript { - scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, tables, snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) + scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, tables, dumpDataByTableID(dumpPlans), snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) if err != nil { return err } @@ -605,16 +608,16 @@ Examples: if err := dumpOut.MkdirAll(outputDir); err != nil { return fmt.Errorf("create output dir: %w", err) } + dumpData, err := reader.PrepareTableDumpData(ctx, tableEntry.TableID, snapshotTS) + if err != nil { + return fmt.Errorf("prepare table %d (%s.%s): %w", tableEntry.TableID, tableEntry.DatabaseName, tableEntry.TableName, err) + } if !noLoad { - dumpData, err := reader.PrepareTableDumpData(ctx, tableEntry.TableID, snapshotTS) - if err != nil { - return fmt.Errorf("prepare table %d (%s.%s): %w", tableEntry.TableID, tableEntry.DatabaseName, tableEntry.TableName, err) - } if err := dumpOneTable(ctx, reader, dumpOut, tableDumpPlan{table: tableEntry, data: dumpData}, snapshotTS, outputDir, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout(), &sync.Mutex{}); err != nil { return err } } - scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, []checkpointtool.TableCatalogEntry{tableEntry}, snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) + scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, []checkpointtool.TableCatalogEntry{tableEntry}, map[uint64]*checkpointtool.TableDumpData{tableEntry.TableID: dumpData}, snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) if err != nil { return err } @@ -764,6 +767,7 @@ func writeRestoreScript( reader *checkpointtool.CheckpointReader, dumpOut *dumpOutput, tables []checkpointtool.TableCatalogEntry, + dumpDataByTable map[uint64]*checkpointtool.TableDumpData, snapshotTS types.TS, scriptDir string, csvRoot string, @@ -801,9 +805,22 @@ func writeRestoreScript( currentDB = table.DatabaseName } - ddl, err := reader.ShowCreateTable(ctx, table.TableID, snapshotTS) - if err != nil { - return "", fmt.Errorf("show create table %d: %w", table.TableID, err) + ddl := "" + if dumpData := dumpDataByTable[table.TableID]; dumpData != nil { + schema := dumpData.Schema + if schema != nil { + schemaCopy := *schema + schemaCopy.TableName = table.TableName + schemaCopy.DatabaseName = table.DatabaseName + ddl = checkpointtool.RenderCreateTableDDLFromSchema(&schemaCopy) + } + } + if ddl == "" { + var err error + ddl, err = reader.ShowCreateTable(ctx, table.TableID, snapshotTS) + if err != nil { + return "", fmt.Errorf("show create table %d: %w", table.TableID, err) + } } ddl = normalizeCreateTableDDLName(ddl, table) indexDDLs, err := reader.ShowCreateIndexStatements(ctx, table.TableID, table.TableName, snapshotTS) @@ -1247,6 +1264,17 @@ type tableDumpPlan struct { data *checkpointtool.TableDumpData } +func dumpDataByTableID(plans []tableDumpPlan) map[uint64]*checkpointtool.TableDumpData { + if len(plans) == 0 { + return nil + } + byID := make(map[uint64]*checkpointtool.TableDumpData, len(plans)) + for _, plan := range plans { + byID[plan.table.TableID] = plan.data + } + return byID +} + func prepareTableDumpPlans( ctx context.Context, reader *checkpointtool.CheckpointReader, diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index d9e1e6f377da3..7fa8d9897121c 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -391,6 +391,31 @@ func renderCreateTableDDL(tableName string, cols []TableColumn) string { return sb.String() } +// RenderCreateTableDDLFromSchema renders a CREATE TABLE statement from resolved +// checkpoint schema metadata. Visible columns are preferred because ALTER TABLE +// updates mo_columns but does not rewrite mo_tables.rel_createsql. +func RenderCreateTableDDLFromSchema(schema *TableSchema) string { + if schema == nil { + return "" + } + if schema.TableName != "" && len(schema.Columns) > 0 { + canRenderColumns := true + for _, col := range schema.Columns { + if col.Name == "" || !isPrintableSQLType(col.SQLType) { + canRenderColumns = false + break + } + } + if canRenderColumns { + return renderCreateTableDDL(schema.TableName, schema.Columns) + } + } + if isPrintableCreateTableSQL(schema.CreateSQL) { + return strings.Clone(schema.CreateSQL) + } + return "" +} + func inferBuiltinCatalogLayout( tableID uint64, moTablesView *LogicalTableView, diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 9aa4d11f0e469..b585a5ed3caa4 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -539,6 +539,34 @@ func TestCreateTableDDLFromCatalogViews_FallsBackToRelCreateSQLWhenMoColumnTypes assert.NotContains(t, ddl, "\ufffd") } +func TestRenderCreateTableDDLFromSchema_PrefersColumnsOverCreateSQL(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t", + CreateSQL: "CREATE TABLE t (a INT, b VARCHAR(100))", + Columns: []TableColumn{ + {Name: "a", SQLType: "INT", Position: 1}, + {Name: "c", SQLType: "BIGINT", Position: 2}, + }, + }) + + assert.Contains(t, ddl, "`a` INT") + assert.Contains(t, ddl, "`c` BIGINT") + assert.NotContains(t, ddl, "`b`") +} + +func TestRenderCreateTableDDLFromSchema_FallsBackToCreateSQLWhenColumnTypesAreBinary(t *testing.T) { + const relCreateSQL = "CREATE TABLE t (a INT)" + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t", + CreateSQL: relCreateSQL, + Columns: []TableColumn{ + {Name: "a", SQLType: string([]byte{'A', 0, 0xff})}, + }, + }) + + assert.Equal(t, relCreateSQL, ddl) +} + func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testing.T) { headers := []string{"object", "block", "row"} for i := 0; i < len(preCPKLayout.moTablesSchema)+2; i++ { From 3eb760e21d27356e9a26be781ab56d71acfbe333 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 15:53:39 +0800 Subject: [PATCH 28/76] fix(ckp): decode mo_columns attr types for ddl --- cmd/mo-object-tool/ckp/checkpoint.go | 5 +- pkg/tools/checkpointtool/table_dump.go | 101 +++++++------------- pkg/tools/checkpointtool/table_dump_test.go | 96 +++++-------------- 3 files changed, 61 insertions(+), 141 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index d2c4c764c8b86..35e95a072ef81 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -1531,9 +1531,8 @@ func showCreateTableCommand(storage *toolfs.StorageOptions) *cobra.Command { Long: `Display the CREATE TABLE SQL for a given table by reading the checkpoint's mo_tables and mo_columns system tables (GCKP + following ICKPs). -The DDL is resolved from mo_tables.rel_createsql if available, otherwise -reconstructed from mo_columns visible column definitions, with hardcoded -fallbacks for core built-in system tables. +The DDL is reconstructed from mo_columns visible column definitions, with +hardcoded fallbacks for core built-in system tables. Examples: mo-tool ckp show-create-table --table-id=12345 /path/to/ckp diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 7fa8d9897121c..d3c40636a0b77 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -392,28 +392,20 @@ func renderCreateTableDDL(tableName string, cols []TableColumn) string { } // RenderCreateTableDDLFromSchema renders a CREATE TABLE statement from resolved -// checkpoint schema metadata. Visible columns are preferred because ALTER TABLE -// updates mo_columns but does not rewrite mo_tables.rel_createsql. +// checkpoint column metadata. func RenderCreateTableDDLFromSchema(schema *TableSchema) string { if schema == nil { return "" } - if schema.TableName != "" && len(schema.Columns) > 0 { - canRenderColumns := true - for _, col := range schema.Columns { - if col.Name == "" || !isPrintableSQLType(col.SQLType) { - canRenderColumns = false - break - } - } - if canRenderColumns { - return renderCreateTableDDL(schema.TableName, schema.Columns) - } + if schema.TableName == "" || len(schema.Columns) == 0 { + return "" } - if isPrintableCreateTableSQL(schema.CreateSQL) { - return strings.Clone(schema.CreateSQL) + for _, col := range schema.Columns { + if col.Name == "" || !isPrintableSQLType(col.SQLType) { + return "" + } } - return "" + return renderCreateTableDDL(schema.TableName, schema.Columns) } func inferBuiltinCatalogLayout( @@ -2270,41 +2262,6 @@ func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *Table return schema } -func findCreateSQLFromMoTables(view *LogicalTableView, tableID uint64) string { - tableIDStr := fmt.Sprintf("%d", tableID) - try := func(relIDCol, createSQLCol int) string { - if relIDCol < 0 || createSQLCol < 0 { - return "" - } - for _, fullRow := range view.Rows { - row := fullRow[logicalViewDataOffset(view):] - if relIDCol >= len(row) || createSQLCol >= len(row) { - continue - } - if row[relIDCol] == tableIDStr && isPrintableCreateTableSQL(row[createSQLCol]) { - return strings.Clone(row[createSQLCol]) - } - } - return "" - } - - relIDCol := fallbackCatalogColIndex(view, moTablesID, "rel_id") - createSQLCol := fallbackCatalogColIndex(view, moTablesID, "rel_createsql") - if ddl := try(relIDCol, createSQLCol); ddl != "" { - return ddl - } - - dataWidth := len(view.Headers) - logicalViewDataOffset(view) - for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { - relIDCol = catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset) - createSQLCol = catalogColIndexForLayout(match.layout, moTablesID, "rel_createsql", match.offset) - if ddl := try(relIDCol, createSQLCol); ddl != "" { - return ddl - } - } - return "" -} - func findTableNameFromMoTables(view *LogicalTableView, tableID uint64) string { tableIDStr := fmt.Sprintf("%d", tableID) try := func(relIDCol, relNameCol int) string { @@ -2350,11 +2307,6 @@ func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView return ddl } } - if moTablesView != nil { - if ddl := findCreateSQLFromMoTables(moTablesView, tableID); ddl != "" { - return ddl - } - } return "" } @@ -2572,7 +2524,11 @@ func buildColumnsFromMoColumnsRowsAt( col.Name = row[nameCol] } if typCol >= 0 && typCol < len(row) { - col.SQLType = row[typCol] + sqlType, ok := decodeMoColumnSQLType(row[typCol]) + if !ok { + return nil + } + col.SQLType = sqlType } cols = append(cols, col) @@ -2591,9 +2547,8 @@ func buildColumnsFromMoColumnsRowsAt( // the checkpoint's mo_tables and mo_columns system tables (GCKP + following ICKPs). // // Priority: -// 1. Reconstructed from mo_columns (attname, atttyp, attnum, att_is_hidden) -// 2. mo_tables.rel_createsql — fallback when mo_columns metadata is unavailable -// 3. Hardcoded built-in table schemas (for core system tables like mo_tables, mo_columns, etc.) +// 1. Reconstructed from mo_columns (attname, decoded atttyp, attnum, att_is_hidden) +// 2. Hardcoded built-in table schemas (for core system tables like mo_tables, mo_columns, etc.) // // ALTER TABLE updates mo_columns but does not rewrite mo_tables.rel_createsql, so // mo_columns is the authoritative source for the current visible column set. @@ -2956,10 +2911,11 @@ func buildCreateTableFromMoColumnsAt( c.name = row[nameCol] } if typCol >= 0 && typCol < len(row) { - c.sqlType = row[typCol] - } - if !isPrintableSQLType(c.sqlType) { - return "" + sqlType, ok := decodeMoColumnSQLType(row[typCol]) + if !ok { + return "" + } + c.sqlType = sqlType } cols = append(cols, c) } @@ -3017,6 +2973,23 @@ func isPrintableSQLType(sqlType string) bool { return hasLetter } +func decodeMoColumnSQLType(raw string) (string, bool) { + if raw == "" { + return "", false + } + var typ types.Type + if err := types.Decode([]byte(raw), &typ); err == nil && typ.Oid != types.T_any { + sqlType := typ.DescString() + if isPrintableSQLType(sqlType) { + return sqlType, true + } + } + if isPrintableSQLType(raw) { + return raw, true + } + return "", false +} + func isPrintableCreateTableSQL(ddl string) bool { ddl = strings.TrimSpace(ddl) if ddl == "" { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index b585a5ed3caa4..ea50a26839439 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -30,6 +30,13 @@ import ( "github.com/stretchr/testify/require" ) +func encodedSQLType(t *testing.T, typ types.Type) string { + t.Helper() + data, err := types.Encode(&typ) + require.NoError(t, err) + return string(data) +} + // TestWriteCSV_empty tests writing an empty logical view to CSV. func TestWriteCSV_empty(t *testing.T) { schema := &TableSchema{ @@ -402,64 +409,6 @@ func TestBuildColumnsFromMoColumnsRows_GenericPreCPKWithPhysicalPrefix(t *testin assert.NotContains(t, ddl, "__mo_rowid") } -func TestFindCreateSQLFromMoTables_GenericWithTrailingColumns(t *testing.T) { - headers := []string{"object", "block", "row"} - for i := 0; i < len(preCPKLayout.moTablesSchema)+2; i++ { - headers = append(headers, fmt.Sprintf("col_%d", i)) - } - row := make([]string, len(headers)) - row[0], row[1], row[2] = "obj1", "0", "0" - row[logicalViewMetaCols+0] = "12345" - row[logicalViewMetaCols+1] = "employees" - row[logicalViewMetaCols+2] = "test_ckp" - row[logicalViewMetaCols+7] = "CREATE TABLE employees (id INT)" - row[len(row)-2] = "tail-1" - row[len(row)-1] = "tail-2" - - view := &LogicalTableView{ - Headers: headers, - Rows: [][]string{row}, - } - - assert.Equal(t, "CREATE TABLE employees (id INT)", findCreateSQLFromMoTables(view, 12345)) -} - -func TestFindCreateSQLFromMoTables_SkipsBinaryCreateSQLCandidates(t *testing.T) { - headers := []string{"object", "block", "row"} - for i := 0; i < len(currentCatalogLayout.moTablesSchema)+2; i++ { - headers = append(headers, fmt.Sprintf("col_%d", i)) - } - headers[logicalViewMetaCols+0] = "rel_id" - headers[logicalViewMetaCols+7] = "rel_createsql" - row := make([]string, len(headers)) - row[0], row[1], row[2] = "obj1", "0", "0" - - row[logicalViewMetaCols+0] = "334019" - row[logicalViewMetaCols+7] = string([]byte{0xff, '/', 0, 0xfe, '0'}) - - offset := 2 - row[logicalViewMetaCols+offset+0] = "334019" - row[logicalViewMetaCols+offset+7] = "CREATE TABLE `ckp_tables`.`t_empty` (`id` INT)" - - view := &LogicalTableView{ - Headers: headers, - Rows: [][]string{row}, - } - - assert.Equal(t, "CREATE TABLE `ckp_tables`.`t_empty` (`id` INT)", findCreateSQLFromMoTables(view, 334019)) -} - -func TestFindCreateSQLFromMoTables_RejectsBinaryCreateSQL(t *testing.T) { - view := &LogicalTableView{ - Headers: []string{"object", "block", "row", "rel_id", "relname", "reldatabase", "reldatabase_id", "col_4", "col_5", "col_6", "rel_createsql"}, - Rows: [][]string{ - {"obj1", "0", "0", "334019", "t_empty", "ckp_tables", "334017", "", "", "", string([]byte{0xff, '/', 0, 0xfe, '0'})}, - }, - } - - assert.Empty(t, findCreateSQLFromMoTables(view, 334019)) -} - func TestCreateTableDDLFromCatalogViews_PrefersMoColumnsOverStaleCreateSQL(t *testing.T) { moTablesView := &LogicalTableView{ Headers: []string{ @@ -478,8 +427,8 @@ func TestCreateTableDDLFromCatalogViews_PrefersMoColumnsOverStaleCreateSQL(t *te moColumnsView := &LogicalTableView{ Headers: []string{"object", "block", "row", "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden"}, Rows: [][]string{ - {"obj1", "0", "0", "12345", "a", "INT", "1", "0"}, - {"obj1", "0", "1", "12345", "c", "BIGINT", "2", "0"}, + {"obj1", "0", "0", "12345", "a", encodedSQLType(t, types.T_int32.ToType()), "1", "0"}, + {"obj1", "0", "1", "12345", "c", encodedSQLType(t, types.T_int64.ToType()), "2", "0"}, }, } @@ -490,7 +439,7 @@ func TestCreateTableDDLFromCatalogViews_PrefersMoColumnsOverStaleCreateSQL(t *te assert.NotContains(t, ddl, "`b`") } -func TestCreateTableDDLFromCatalogViews_FallsBackToRelCreateSQL(t *testing.T) { +func TestCreateTableDDLFromCatalogViews_DoesNotFallbackToRelCreateSQL(t *testing.T) { moTablesView := &LogicalTableView{ Headers: []string{ "object", "block", "row", @@ -506,11 +455,10 @@ func TestCreateTableDDLFromCatalogViews_FallsBackToRelCreateSQL(t *testing.T) { }, } - assert.Equal(t, "CREATE TABLE employees (id INT)", createTableDDLFromCatalogViews(12345, moTablesView, nil)) + assert.Empty(t, createTableDDLFromCatalogViews(12345, moTablesView, nil)) } -func TestCreateTableDDLFromCatalogViews_FallsBackToRelCreateSQLWhenMoColumnTypesAreBinary(t *testing.T) { - const relCreateSQL = "CREATE TABLE `ckp_constraints`.`parent` (`id` INT NOT NULL PRIMARY KEY, `code` VARCHAR(20) NOT NULL UNIQUE, `note` VARCHAR(100) DEFAULT 'parent-default' COMMENT 'parent note') COMMENT='parent table comment'" +func TestCreateTableDDLFromCatalogViews_DecodesMoColumnTypes(t *testing.T) { moTablesView := &LogicalTableView{ Headers: []string{ "object", "block", "row", @@ -521,22 +469,23 @@ func TestCreateTableDDLFromCatalogViews_FallsBackToRelCreateSQLWhenMoColumnTypes { "obj1", "0", "0", "333999", "parent", "ckp_constraints", "333997", - "", "", "", relCreateSQL, + "", "", "", "CREATE TABLE `ckp_constraints`.`parent` (`old` INT)", }, }, } moColumnsView := &LogicalTableView{ Headers: []string{"object", "block", "row", "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden"}, Rows: [][]string{ - {"obj1", "0", "0", "333999", "id", string([]byte{'A', 'L', 0, 0xff, 'X'}), "1", "0"}, - {"obj1", "0", "1", "333999", "code", "VARCHAR(20)", "2", "0"}, + {"obj1", "0", "0", "333999", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "0"}, + {"obj1", "0", "1", "333999", "code", encodedSQLType(t, types.New(types.T_varchar, 20, 0)), "2", "0"}, }, } ddl := createTableDDLFromCatalogViews(333999, moTablesView, moColumnsView) - assert.Equal(t, relCreateSQL, ddl) - assert.NotContains(t, ddl, "\x00") - assert.NotContains(t, ddl, "\ufffd") + assert.Contains(t, ddl, "CREATE TABLE `parent`") + assert.Contains(t, ddl, "`id` INT") + assert.Contains(t, ddl, "`code` VARCHAR(20)") + assert.NotContains(t, ddl, "`old`") } func TestRenderCreateTableDDLFromSchema_PrefersColumnsOverCreateSQL(t *testing.T) { @@ -554,17 +503,16 @@ func TestRenderCreateTableDDLFromSchema_PrefersColumnsOverCreateSQL(t *testing.T assert.NotContains(t, ddl, "`b`") } -func TestRenderCreateTableDDLFromSchema_FallsBackToCreateSQLWhenColumnTypesAreBinary(t *testing.T) { - const relCreateSQL = "CREATE TABLE t (a INT)" +func TestRenderCreateTableDDLFromSchema_DoesNotFallbackToCreateSQLWhenColumnTypesAreBinary(t *testing.T) { ddl := RenderCreateTableDDLFromSchema(&TableSchema{ TableName: "t", - CreateSQL: relCreateSQL, + CreateSQL: "CREATE TABLE t (a INT)", Columns: []TableColumn{ {Name: "a", SQLType: string([]byte{'A', 0, 0xff})}, }, }) - assert.Equal(t, relCreateSQL, ddl) + assert.Empty(t, ddl) } func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testing.T) { From c092eaccee6f785bb5be9d8dc85f2b0bda639818 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 16:00:36 +0800 Subject: [PATCH 29/76] chore(ckp): add schema resolution debug logs --- pkg/tools/checkpointtool/table_dump.go | 85 ++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index d3c40636a0b77..f76c788496fe5 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "encoding/csv" + "encoding/hex" "fmt" "io" "os" @@ -2450,8 +2451,23 @@ func buildColumnsFromMoColumnsRows(view *LogicalTableView, tableID uint64) []Tab hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") seqNumCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Seqnum) + ckpDebugSchemaf( + "mo_columns schema resolve start table=%d rows=%d headers=%d data_width=%d fallback_cols relname_id=%d name=%d typ=%d num=%d hidden=%d seqnum=%d", + tableID, + len(view.Rows), + len(view.Headers), + len(view.Headers)-logicalViewDataOffset(view), + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + ) + if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { if cols := buildColumnsFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol, seqNumCol); len(cols) > 0 { + ckpDebugSchemaf("mo_columns schema resolve success table=%d source=fallback cols=%d", tableID, len(cols)) return cols } } @@ -2464,11 +2480,25 @@ func buildColumnsFromMoColumnsRows(view *LogicalTableView, tableID uint64) []Tab numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) seqNumCol = catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Seqnum, match.offset) + ckpDebugSchemaf( + "mo_columns schema try table=%d layout=%s offset=%d relname_id=%d name=%d typ=%d num=%d hidden=%d seqnum=%d", + tableID, + match.layout.name, + match.offset, + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + ) if cols := buildColumnsFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol, seqNumCol); len(cols) > 0 { + ckpDebugSchemaf("mo_columns schema resolve success table=%d source=%s offset=%d cols=%d", tableID, match.layout.name, match.offset, len(cols)) return cols } } + ckpDebugSchemaf("mo_columns schema resolve failed table=%d", tableID) return nil } @@ -2484,6 +2514,7 @@ func buildColumnsFromMoColumnsRowsAt( ) []TableColumn { tableIDStr := fmt.Sprintf("%d", tableID) var cols []TableColumn + matchedRows := 0 for _, fullRow := range view.Rows { row := fullRow[logicalViewDataOffset(view):] @@ -2501,6 +2532,7 @@ func buildColumnsFromMoColumnsRowsAt( if row[relnameIDCol] != tableIDStr { continue } + matchedRows++ pos := len(cols) if numCol >= 0 && numCol < len(row) { @@ -2526,6 +2558,20 @@ func buildColumnsFromMoColumnsRowsAt( if typCol >= 0 && typCol < len(row) { sqlType, ok := decodeMoColumnSQLType(row[typCol]) if !ok { + ckpDebugSchemaf( + "mo_columns type decode failed table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d column=%q raw_len=%d raw_hex=%s", + tableID, + matchedRows, + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + col.Name, + len(row[typCol]), + debugHexPrefix(row[typCol], 32), + ) return nil } col.SQLType = sqlType @@ -2539,6 +2585,19 @@ func buildColumnsFromMoColumnsRowsAt( return cols[i].Position < cols[j].Position }) + if matchedRows > 0 && len(cols) == 0 { + ckpDebugSchemaf( + "mo_columns matched rows but produced no visible columns table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d", + tableID, + matchedRows, + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + ) + } return cols } @@ -2913,6 +2972,18 @@ func buildCreateTableFromMoColumnsAt( if typCol >= 0 && typCol < len(row) { sqlType, ok := decodeMoColumnSQLType(row[typCol]) if !ok { + ckpDebugSchemaf( + "mo_columns ddl type decode failed table=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d column=%q raw_len=%d raw_hex=%s", + tableID, + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + c.name, + len(row[typCol]), + debugHexPrefix(row[typCol], 32), + ) return "" } c.sqlType = sqlType @@ -2990,6 +3061,20 @@ func decodeMoColumnSQLType(raw string) (string, bool) { return "", false } +func ckpDebugSchemaf(format string, args ...any) { + if os.Getenv("MO_TOOL_CKP_DEBUG_SCHEMA") == "" { + return + } + fmt.Fprintf(os.Stderr, "ckp schema debug: "+format+"\n", args...) +} + +func debugHexPrefix(s string, limit int) string { + if limit > 0 && len(s) > limit { + s = s[:limit] + } + return hex.EncodeToString([]byte(s)) +} + func isPrintableCreateTableSQL(ddl string) bool { ddl = strings.TrimSpace(ddl) if ddl == "" { From c2a196fc3225cf341472779b27a9f5bcba68562e Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 16:05:32 +0800 Subject: [PATCH 30/76] fix(ckp): detect encoded column types across mo_columns rows --- pkg/tools/checkpointtool/table_dump.go | 76 ++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index f76c788496fe5..317b79a3ffb1d 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -2557,9 +2557,12 @@ func buildColumnsFromMoColumnsRowsAt( } if typCol >= 0 && typCol < len(row) { sqlType, ok := decodeMoColumnSQLType(row[typCol]) + if !ok { + sqlType, ok = detectMoColumnSQLType(row) + } if !ok { ckpDebugSchemaf( - "mo_columns type decode failed table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d column=%q raw_len=%d raw_hex=%s", + "mo_columns type decode failed table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d column=%q raw_len=%d raw_hex=%s row=%s", tableID, matchedRows, relnameIDCol, @@ -2571,9 +2574,13 @@ func buildColumnsFromMoColumnsRowsAt( col.Name, len(row[typCol]), debugHexPrefix(row[typCol], 32), + debugRowCells(row, 40), ) return nil } + if typCol >= 0 && typCol < len(row) && row[typCol] != "" && !isDecodedMoColumnSQLType(row[typCol]) { + ckpDebugSchemaf("mo_columns type detected from alternate cell table=%d column=%q declared_typ_col=%d sql_type=%s row=%s", tableID, col.Name, typCol, sqlType, debugRowCells(row, 40)) + } col.SQLType = sqlType } @@ -2971,9 +2978,12 @@ func buildCreateTableFromMoColumnsAt( } if typCol >= 0 && typCol < len(row) { sqlType, ok := decodeMoColumnSQLType(row[typCol]) + if !ok { + sqlType, ok = detectMoColumnSQLType(row) + } if !ok { ckpDebugSchemaf( - "mo_columns ddl type decode failed table=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d column=%q raw_len=%d raw_hex=%s", + "mo_columns ddl type decode failed table=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d column=%q raw_len=%d raw_hex=%s row=%s", tableID, relnameIDCol, nameCol, @@ -2983,9 +2993,13 @@ func buildCreateTableFromMoColumnsAt( c.name, len(row[typCol]), debugHexPrefix(row[typCol], 32), + debugRowCells(row, 40), ) return "" } + if typCol >= 0 && typCol < len(row) && row[typCol] != "" && !isDecodedMoColumnSQLType(row[typCol]) { + ckpDebugSchemaf("mo_columns ddl type detected from alternate cell table=%d column=%q declared_typ_col=%d sql_type=%s row=%s", tableID, c.name, typCol, sqlType, debugRowCells(row, 40)) + } c.sqlType = sqlType } cols = append(cols, c) @@ -3048,19 +3062,56 @@ func decodeMoColumnSQLType(raw string) (string, bool) { if raw == "" { return "", false } + if sqlType, ok := decodeMoColumnEncodedSQLType(raw); ok { + return sqlType, true + } + if isPrintableSQLType(raw) { + return raw, true + } + return "", false +} + +func decodeMoColumnEncodedSQLType(raw string) (string, bool) { var typ types.Type + if len(raw) < typ.ProtoSize() { + return "", false + } if err := types.Decode([]byte(raw), &typ); err == nil && typ.Oid != types.T_any { sqlType := typ.DescString() if isPrintableSQLType(sqlType) { return sqlType, true } } - if isPrintableSQLType(raw) { - return raw, true - } return "", false } +func isDecodedMoColumnSQLType(raw string) bool { + _, ok := decodeMoColumnEncodedSQLType(raw) + return ok +} + +func detectMoColumnSQLType(row []string) (string, bool) { + found := "" + foundCol := -1 + for i, cell := range row { + sqlType, ok := decodeMoColumnEncodedSQLType(cell) + if !ok { + continue + } + if found != "" { + ckpDebugSchemaf("mo_columns alternate type detection ambiguous first_col=%d first_type=%s next_col=%d next_type=%s row=%s", foundCol, found, i, sqlType, debugRowCells(row, 40)) + return "", false + } + found = sqlType + foundCol = i + } + if found == "" { + return "", false + } + ckpDebugSchemaf("mo_columns alternate type detected col=%d sql_type=%s", foundCol, found) + return found, true +} + func ckpDebugSchemaf(format string, args ...any) { if os.Getenv("MO_TOOL_CKP_DEBUG_SCHEMA") == "" { return @@ -3075,6 +3126,21 @@ func debugHexPrefix(s string, limit int) string { return hex.EncodeToString([]byte(s)) } +func debugRowCells(row []string, limit int) string { + parts := make([]string, 0, len(row)) + for i, cell := range row { + parts = append(parts, fmt.Sprintf("%d:len=%d hex=%s text=%q", i, len(cell), debugHexPrefix(cell, limit), debugTextPrefix(cell, limit))) + } + return strings.Join(parts, " | ") +} + +func debugTextPrefix(s string, limit int) string { + if limit > 0 && len(s) > limit { + s = s[:limit] + } + return s +} + func isPrintableCreateTableSQL(ddl string) bool { ddl = strings.TrimSpace(ddl) if ddl == "" { From c6462e6913ae3a45849945d2dce72611c65c118b Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 16:14:07 +0800 Subject: [PATCH 31/76] fix(ckp): use object metadata for missing column types --- pkg/tools/checkpointtool/table_dump.go | 222 ++++++++++++++++--------- 1 file changed, 139 insertions(+), 83 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 317b79a3ffb1d..8b9d4d46eeafa 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -151,6 +151,24 @@ type TableDumpData struct { TombEntries []*ObjectEntryInfo } +var moIndexesHeaders = []string{ + "id", + "table_id", + "database_id", + "name", + "type", + catalog.IndexAlgoName, + catalog.IndexAlgoTableType, + catalog.IndexAlgoParams, + "is_visible", + "hidden", + "comment", + "column_name", + "ordinal_position", + "options", + "index_table_name", +} + type indexDDLColumn struct { name string ordinal int @@ -585,6 +603,68 @@ func (r *CheckpointReader) ReadTableSchema( return schema } +func (r *CheckpointReader) applyObjectColumnTypes( + ctx context.Context, + snapshotTS types.TS, + schema *TableSchema, + dataEntries []*ObjectEntryInfo, +) { + if schema == nil || len(schema.Columns) == 0 || len(dataEntries) == 0 { + return + } + visibleEntries := visibleObjectEntries(dataEntries, snapshotTS) + for _, entry := range visibleEntries { + objName := entry.ObjectStats.ObjectName().String() + reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) + if err != nil { + continue + } + cols := reader.Columns() + _ = reader.Close() + if len(cols) == 0 { + continue + } + typeBySeqNum := make(map[int]string, len(cols)) + for i, col := range cols { + sqlType := col.Type.DescString() + if !isPrintableSQLType(sqlType) { + continue + } + typeBySeqNum[int(col.SeqNum)] = sqlType + if _, ok := typeBySeqNum[int(col.Idx)]; !ok { + typeBySeqNum[int(col.Idx)] = sqlType + } + if _, ok := typeBySeqNum[i]; !ok { + typeBySeqNum[i] = sqlType + } + } + filled := 0 + for i := range schema.Columns { + if isPrintableSQLType(schema.Columns[i].SQLType) { + continue + } + if sqlType, ok := typeBySeqNum[schema.Columns[i].PhysicalPosition]; ok { + schema.Columns[i].SQLType = sqlType + filled++ + } + } + ckpDebugSchemaf("object column types applied table=%s object=%s filled=%d cols=%d", schema.TableName, objName, filled, len(schema.Columns)) + return + } +} + +func schemaHasColumnTypes(schema *TableSchema) bool { + if schema == nil || len(schema.Columns) == 0 { + return false + } + for _, col := range schema.Columns { + if !isPrintableSQLType(col.SQLType) { + return false + } + } + return true +} + // getTableLogicalView composes the checkpoint view at snapshotTS and builds a logical // table view for the given tableID by aggregating data/tomb entries across all // relevant checkpoint entries (GCKP + ICKPs). @@ -656,7 +736,8 @@ func (r *CheckpointReader) DumpTableCSV( opts ...CSVExportOption, ) error { schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) - if len(schema.Columns) == 0 { + r.applyObjectColumnTypes(ctx, snapshotTS, schema, dataEntries) + if !schemaHasColumnTypes(schema) { return moerr.NewInternalErrorf( ctx, "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", @@ -676,14 +757,6 @@ func (r *CheckpointReader) DumpTableCSVComposed( snapshotTS types.TS, opts ...CSVExportOption, ) error { - schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) - if len(schema.Columns) == 0 { - return moerr.NewInternalErrorf( - ctx, - "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", - tableID, - ) - } dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) if err != nil { if !isTableDataUnavailable(err) { @@ -692,6 +765,15 @@ func (r *CheckpointReader) DumpTableCSVComposed( dataEntries = nil tombEntries = nil } + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) + r.applyObjectColumnTypes(ctx, snapshotTS, schema, dataEntries) + if !schemaHasColumnTypes(schema) { + return moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) + } return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries, resolveCSVExportOptions(opts)) } @@ -702,14 +784,6 @@ func (r *CheckpointReader) PrepareTableDumpData( tableID uint64, snapshotTS types.TS, ) (*TableDumpData, error) { - schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) - if len(schema.Columns) == 0 { - return nil, moerr.NewInternalErrorf( - ctx, - "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", - tableID, - ) - } dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) if err != nil { if !isTableDataUnavailable(err) { @@ -718,6 +792,15 @@ func (r *CheckpointReader) PrepareTableDumpData( dataEntries = nil tombEntries = nil } + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) + r.applyObjectColumnTypes(ctx, snapshotTS, schema, dataEntries) + if !schemaHasColumnTypes(schema) { + return nil, moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) + } return &TableDumpData{ TableID: tableID, Schema: cloneTableSchema(schema), @@ -784,7 +867,8 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( } for tableID, data := range result { - if data.Schema == nil || len(data.Schema.Columns) == 0 { + r.applyObjectColumnTypes(ctx, snapshotTS, data.Schema, data.DataEntries) + if !schemaHasColumnTypes(data.Schema) { return nil, moerr.NewInternalErrorf(ctx, "cannot resolve visible columns for table %d from checkpoint metadata", tableID) } } @@ -812,7 +896,7 @@ func (r *CheckpointReader) DumpPreparedTableCSV( if data == nil { return moerr.NewInternalError(ctx, "missing prepared table dump data") } - if data.Schema == nil || len(data.Schema.Columns) == 0 { + if !schemaHasColumnTypes(data.Schema) { return moerr.NewInternalErrorf(ctx, "cannot resolve visible columns for table %d from checkpoint metadata", data.TableID) } return r.streamTableCSV(ctx, w, data.Schema, snapshotTS, data.DataEntries, data.TombEntries, resolveCSVExportOptions(opts)) @@ -2556,13 +2640,11 @@ func buildColumnsFromMoColumnsRowsAt( col.Name = row[nameCol] } if typCol >= 0 && typCol < len(row) { - sqlType, ok := decodeMoColumnSQLType(row[typCol]) - if !ok { - sqlType, ok = detectMoColumnSQLType(row) - } - if !ok { + if sqlType, ok := decodeMoColumnSQLType(row[typCol]); ok { + col.SQLType = sqlType + } else { ckpDebugSchemaf( - "mo_columns type decode failed table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d column=%q raw_len=%d raw_hex=%s row=%s", + "mo_columns type decode unavailable table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d column=%q raw_len=%d raw_hex=%s", tableID, matchedRows, relnameIDCol, @@ -2574,14 +2656,8 @@ func buildColumnsFromMoColumnsRowsAt( col.Name, len(row[typCol]), debugHexPrefix(row[typCol], 32), - debugRowCells(row, 40), ) - return nil } - if typCol >= 0 && typCol < len(row) && row[typCol] != "" && !isDecodedMoColumnSQLType(row[typCol]) { - ckpDebugSchemaf("mo_columns type detected from alternate cell table=%d column=%q declared_typ_col=%d sql_type=%s row=%s", tableID, col.Name, typCol, sqlType, debugRowCells(row, 40)) - } - col.SQLType = sqlType } cols = append(cols, col) @@ -2642,7 +2718,12 @@ func (r *CheckpointReader) ShowCreateTable( moColumnsView = nil } - if ddl := createTableDDLFromCatalogViews(tableID, moTablesView, moColumnsView); ddl != "" { + dataEntries, _, dataErr := r.getTableEntriesAt(ctx, tableID, snapshotTS) + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) + if dataErr == nil { + r.applyObjectColumnTypes(ctx, snapshotTS, schema, dataEntries) + } + if ddl := RenderCreateTableDDLFromSchema(schema); ddl != "" { return ddl, nil } @@ -2691,13 +2772,37 @@ func (r *CheckpointReader) ShowCreateIndexStatements( return nil, nil } - view, err := r.dumpCatalogTableView(ctx, moIndexesTableID, snapshotTS) + view, err := r.dumpCatalogTableViewWithHeaders(ctx, moIndexesTableID, snapshotTS, moIndexesHeaders) if err != nil { return nil, err } return buildCreateIndexStatementsFromMoIndexes(view, tableID, tableName) } +func (r *CheckpointReader) dumpCatalogTableViewWithHeaders( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, + headers []string, +) (*LogicalTableView, error) { + view, err := r.getTableLogicalView(ctx, tableID, snapshotTS) + if err != nil { + return nil, err + } + if view == nil { + return &LogicalTableView{}, nil + } + fixedHeaders := append([]string(nil), logicalTableViewMetaHeaders...) + fixedHeaders = append(fixedHeaders, headers...) + if len(view.Headers) > len(fixedHeaders) { + for i := len(fixedHeaders); i < len(view.Headers); i++ { + fixedHeaders = append(fixedHeaders, fmt.Sprintf("col_%d", i-len(logicalTableViewMetaHeaders))) + } + } + view.Headers = fixedHeaders + return view, nil +} + func (r *CheckpointReader) dumpCatalogTableView( ctx context.Context, tableID uint64, @@ -2978,12 +3083,9 @@ func buildCreateTableFromMoColumnsAt( } if typCol >= 0 && typCol < len(row) { sqlType, ok := decodeMoColumnSQLType(row[typCol]) - if !ok { - sqlType, ok = detectMoColumnSQLType(row) - } if !ok { ckpDebugSchemaf( - "mo_columns ddl type decode failed table=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d column=%q raw_len=%d raw_hex=%s row=%s", + "mo_columns ddl type decode failed table=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d column=%q raw_len=%d raw_hex=%s", tableID, relnameIDCol, nameCol, @@ -2993,13 +3095,9 @@ func buildCreateTableFromMoColumnsAt( c.name, len(row[typCol]), debugHexPrefix(row[typCol], 32), - debugRowCells(row, 40), ) return "" } - if typCol >= 0 && typCol < len(row) && row[typCol] != "" && !isDecodedMoColumnSQLType(row[typCol]) { - ckpDebugSchemaf("mo_columns ddl type detected from alternate cell table=%d column=%q declared_typ_col=%d sql_type=%s row=%s", tableID, c.name, typCol, sqlType, debugRowCells(row, 40)) - } c.sqlType = sqlType } cols = append(cols, c) @@ -3085,33 +3183,6 @@ func decodeMoColumnEncodedSQLType(raw string) (string, bool) { return "", false } -func isDecodedMoColumnSQLType(raw string) bool { - _, ok := decodeMoColumnEncodedSQLType(raw) - return ok -} - -func detectMoColumnSQLType(row []string) (string, bool) { - found := "" - foundCol := -1 - for i, cell := range row { - sqlType, ok := decodeMoColumnEncodedSQLType(cell) - if !ok { - continue - } - if found != "" { - ckpDebugSchemaf("mo_columns alternate type detection ambiguous first_col=%d first_type=%s next_col=%d next_type=%s row=%s", foundCol, found, i, sqlType, debugRowCells(row, 40)) - return "", false - } - found = sqlType - foundCol = i - } - if found == "" { - return "", false - } - ckpDebugSchemaf("mo_columns alternate type detected col=%d sql_type=%s", foundCol, found) - return found, true -} - func ckpDebugSchemaf(format string, args ...any) { if os.Getenv("MO_TOOL_CKP_DEBUG_SCHEMA") == "" { return @@ -3126,21 +3197,6 @@ func debugHexPrefix(s string, limit int) string { return hex.EncodeToString([]byte(s)) } -func debugRowCells(row []string, limit int) string { - parts := make([]string, 0, len(row)) - for i, cell := range row { - parts = append(parts, fmt.Sprintf("%d:len=%d hex=%s text=%q", i, len(cell), debugHexPrefix(cell, limit), debugTextPrefix(cell, limit))) - } - return strings.Join(parts, " | ") -} - -func debugTextPrefix(s string, limit int) string { - if limit > 0 && len(s) > limit { - s = s[:limit] - } - return s -} - func isPrintableCreateTableSQL(ddl string) bool { ddl = strings.TrimSpace(ddl) if ddl == "" { From 51b8e56f8f0954ac88468fd874a17c72ab90c16b Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 16:31:04 +0800 Subject: [PATCH 32/76] chore(ckp): log raw mo_columns attr types --- pkg/tools/checkpointtool/checkpoint_reader.go | 5 + pkg/tools/checkpointtool/table_dump.go | 157 ++++++++---------- 2 files changed, 78 insertions(+), 84 deletions(-) diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index 98cba381e2813..0c0f7789a2e2c 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -615,6 +615,11 @@ func vecValueToString(vec *vector.Vector, idx int) string { return values[rowIdx].String2(vec.GetType().Scale) } } + switch vec.GetType().Oid { + case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, + types.T_json, types.T_blob, types.T_text, types.T_datalink, types.T_geometry: + return string(vec.GetBytesAt(idx)) + } value := vec.RowToString(idx) if value == "null" { return "NULL" diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 8b9d4d46eeafa..2c892d6d3633b 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -603,68 +603,6 @@ func (r *CheckpointReader) ReadTableSchema( return schema } -func (r *CheckpointReader) applyObjectColumnTypes( - ctx context.Context, - snapshotTS types.TS, - schema *TableSchema, - dataEntries []*ObjectEntryInfo, -) { - if schema == nil || len(schema.Columns) == 0 || len(dataEntries) == 0 { - return - } - visibleEntries := visibleObjectEntries(dataEntries, snapshotTS) - for _, entry := range visibleEntries { - objName := entry.ObjectStats.ObjectName().String() - reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) - if err != nil { - continue - } - cols := reader.Columns() - _ = reader.Close() - if len(cols) == 0 { - continue - } - typeBySeqNum := make(map[int]string, len(cols)) - for i, col := range cols { - sqlType := col.Type.DescString() - if !isPrintableSQLType(sqlType) { - continue - } - typeBySeqNum[int(col.SeqNum)] = sqlType - if _, ok := typeBySeqNum[int(col.Idx)]; !ok { - typeBySeqNum[int(col.Idx)] = sqlType - } - if _, ok := typeBySeqNum[i]; !ok { - typeBySeqNum[i] = sqlType - } - } - filled := 0 - for i := range schema.Columns { - if isPrintableSQLType(schema.Columns[i].SQLType) { - continue - } - if sqlType, ok := typeBySeqNum[schema.Columns[i].PhysicalPosition]; ok { - schema.Columns[i].SQLType = sqlType - filled++ - } - } - ckpDebugSchemaf("object column types applied table=%s object=%s filled=%d cols=%d", schema.TableName, objName, filled, len(schema.Columns)) - return - } -} - -func schemaHasColumnTypes(schema *TableSchema) bool { - if schema == nil || len(schema.Columns) == 0 { - return false - } - for _, col := range schema.Columns { - if !isPrintableSQLType(col.SQLType) { - return false - } - } - return true -} - // getTableLogicalView composes the checkpoint view at snapshotTS and builds a logical // table view for the given tableID by aggregating data/tomb entries across all // relevant checkpoint entries (GCKP + ICKPs). @@ -736,8 +674,7 @@ func (r *CheckpointReader) DumpTableCSV( opts ...CSVExportOption, ) error { schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) - r.applyObjectColumnTypes(ctx, snapshotTS, schema, dataEntries) - if !schemaHasColumnTypes(schema) { + if len(schema.Columns) == 0 { return moerr.NewInternalErrorf( ctx, "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", @@ -766,8 +703,7 @@ func (r *CheckpointReader) DumpTableCSVComposed( tombEntries = nil } schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) - r.applyObjectColumnTypes(ctx, snapshotTS, schema, dataEntries) - if !schemaHasColumnTypes(schema) { + if len(schema.Columns) == 0 { return moerr.NewInternalErrorf( ctx, "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", @@ -793,8 +729,7 @@ func (r *CheckpointReader) PrepareTableDumpData( tombEntries = nil } schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) - r.applyObjectColumnTypes(ctx, snapshotTS, schema, dataEntries) - if !schemaHasColumnTypes(schema) { + if len(schema.Columns) == 0 { return nil, moerr.NewInternalErrorf( ctx, "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", @@ -867,8 +802,7 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( } for tableID, data := range result { - r.applyObjectColumnTypes(ctx, snapshotTS, data.Schema, data.DataEntries) - if !schemaHasColumnTypes(data.Schema) { + if data.Schema == nil || len(data.Schema.Columns) == 0 { return nil, moerr.NewInternalErrorf(ctx, "cannot resolve visible columns for table %d from checkpoint metadata", tableID) } } @@ -896,7 +830,7 @@ func (r *CheckpointReader) DumpPreparedTableCSV( if data == nil { return moerr.NewInternalError(ctx, "missing prepared table dump data") } - if !schemaHasColumnTypes(data.Schema) { + if data.Schema == nil || len(data.Schema.Columns) == 0 { return moerr.NewInternalErrorf(ctx, "cannot resolve visible columns for table %d from checkpoint metadata", data.TableID) } return r.streamTableCSV(ctx, w, data.Schema, snapshotTS, data.DataEntries, data.TombEntries, resolveCSVExportOptions(opts)) @@ -2599,6 +2533,7 @@ func buildColumnsFromMoColumnsRowsAt( tableIDStr := fmt.Sprintf("%d", tableID) var cols []TableColumn matchedRows := 0 + typeDecodeFailures := 0 for _, fullRow := range view.Rows { row := fullRow[logicalViewDataOffset(view):] @@ -2643,8 +2578,9 @@ func buildColumnsFromMoColumnsRowsAt( if sqlType, ok := decodeMoColumnSQLType(row[typCol]); ok { col.SQLType = sqlType } else { + typeDecodeFailures++ ckpDebugSchemaf( - "mo_columns type decode unavailable table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d column=%q raw_len=%d raw_hex=%s", + "mo_columns atttyp decode failed table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d column=%q %s", tableID, matchedRows, relnameIDCol, @@ -2654,15 +2590,29 @@ func buildColumnsFromMoColumnsRowsAt( hiddenCol, seqNumCol, col.Name, - len(row[typCol]), - debugHexPrefix(row[typCol], 32), + debugMoColumnTypeCell(row[typCol], fullRow), ) } } - cols = append(cols, col) } + if typeDecodeFailures > 0 { + ckpDebugSchemaf( + "mo_columns schema rejected table=%d matched_rows=%d type_decode_failures=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d", + tableID, + matchedRows, + typeDecodeFailures, + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + ) + return nil + } + // Sort by position sort.Slice(cols, func(i, j int) bool { return cols[i].Position < cols[j].Position @@ -2718,12 +2668,7 @@ func (r *CheckpointReader) ShowCreateTable( moColumnsView = nil } - dataEntries, _, dataErr := r.getTableEntriesAt(ctx, tableID, snapshotTS) - schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) - if dataErr == nil { - r.applyObjectColumnTypes(ctx, snapshotTS, schema, dataEntries) - } - if ddl := RenderCreateTableDDLFromSchema(schema); ddl != "" { + if ddl := createTableDDLFromCatalogViews(tableID, moTablesView, moColumnsView); ddl != "" { return ddl, nil } @@ -3085,7 +3030,7 @@ func buildCreateTableFromMoColumnsAt( sqlType, ok := decodeMoColumnSQLType(row[typCol]) if !ok { ckpDebugSchemaf( - "mo_columns ddl type decode failed table=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d column=%q raw_len=%d raw_hex=%s", + "mo_columns ddl atttyp decode failed table=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d column=%q %s", tableID, relnameIDCol, nameCol, @@ -3093,8 +3038,7 @@ func buildCreateTableFromMoColumnsAt( numCol, hiddenCol, c.name, - len(row[typCol]), - debugHexPrefix(row[typCol], 32), + debugMoColumnTypeCell(row[typCol], fullRow), ) return "" } @@ -3197,6 +3141,51 @@ func debugHexPrefix(s string, limit int) string { return hex.EncodeToString([]byte(s)) } +func debugMoColumnTypeCell(raw string, fullRow []string) string { + var typ types.Type + decodeErr := "" + oid := "" + width := int32(0) + scale := int32(0) + desc := "" + if len(raw) >= typ.ProtoSize() { + if err := types.Decode([]byte(raw), &typ); err != nil { + decodeErr = err.Error() + } else { + decodeErr = "" + oid = typ.Oid.String() + width = typ.Width + scale = typ.Scale + desc = typ.DescString() + } + } + objectName := "" + blockIdx := "" + rowIdx := "" + if len(fullRow) > 0 { + objectName = fullRow[0] + } + if len(fullRow) > 1 { + blockIdx = fullRow[1] + } + if len(fullRow) > 2 { + rowIdx = fullRow[2] + } + return fmt.Sprintf( + "raw_len=%d raw_hex=%s decode_err=%s oid=%s width=%d scale=%d desc=%q object=%s block=%s row=%s", + len(raw), + debugHexPrefix(raw, 64), + decodeErr, + oid, + width, + scale, + desc, + objectName, + blockIdx, + rowIdx, + ) +} + func isPrintableCreateTableSQL(ddl string) bool { ddl = strings.TrimSpace(ddl) if ddl == "" { From f7ebb55c1634885c1d66b7b231809da5ac706ec6 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 16:42:31 +0800 Subject: [PATCH 33/76] fix(ckp): read object columns by seqnum --- pkg/tools/checkpointtool/logical_table.go | 76 +++++++++++++++++- pkg/tools/checkpointtool/table_dump.go | 96 ++++++++++++++++++----- pkg/tools/checkpointtool/types.go | 1 + pkg/tools/objecttool/object_reader.go | 35 +++++++-- 4 files changed, 179 insertions(+), 29 deletions(-) diff --git a/pkg/tools/checkpointtool/logical_table.go b/pkg/tools/checkpointtool/logical_table.go index 3e16e9e430ceb..15d4a5c472037 100644 --- a/pkg/tools/checkpointtool/logical_table.go +++ b/pkg/tools/checkpointtool/logical_table.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "sort" + "strings" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" @@ -49,6 +50,7 @@ func (r *CheckpointReader) BuildLogicalTableView( for _, col := range cols { view.Headers = append(view.Headers, fmt.Sprintf("col_%d", col.Idx)) view.ColTypes = append(view.ColTypes, col.Type) + view.ColSeqNums = append(view.ColSeqNums, col.SeqNum) } return nil }, @@ -86,6 +88,7 @@ func (r *CheckpointReader) scanLogicalTable( visibleTombEntries := visibleObjectEntries(tombEntries, snapshotTS) tombstoneStats := dedupeObjectStats(visibleTombEntries) columnsSent := false + var canonicalSeqNums []uint16 for _, entry := range visibleDataEntries { objName := entry.ObjectStats.ObjectName().String() @@ -96,9 +99,12 @@ func (r *CheckpointReader) scanLogicalTable( } return stats, err } + debugLogicalObjectColumns(entry.ObjectStats, reader) + cols := reader.Columns() if !columnsSent && onColumns != nil { - if err := onColumns(reader.Columns()); err != nil { + canonicalSeqNums = columnSeqNums(cols) + if err := onColumns(cols); err != nil { _ = reader.Close() return stats, err } @@ -163,6 +169,9 @@ func (r *CheckpointReader) scanLogicalTable( nulls[i] = vec.IsNull(uint64(rowIdx)) values[i] = vecValueToString(vec, rowIdx) } + if len(canonicalSeqNums) > 0 { + values, nulls = alignRowValuesBySeqNums(cols, canonicalSeqNums, values, nulls) + } if err := onRow(entry.ObjectStats.ObjectName().Short().ShortString(), blockIdx, rowIdx, values, nulls); err != nil { if deleteMask.IsValid() { deleteMask.Release() @@ -195,6 +204,71 @@ func (r *CheckpointReader) scanLogicalTable( return stats, nil } +func columnSeqNums(cols []objecttool.ColInfo) []uint16 { + seqNums := make([]uint16, len(cols)) + for i, col := range cols { + seqNums[i] = col.SeqNum + } + return seqNums +} + +func alignRowValuesBySeqNums( + cols []objecttool.ColInfo, + targetSeqNums []uint16, + values []string, + nulls []bool, +) ([]string, []bool) { + if len(cols) == 0 || len(targetSeqNums) == 0 { + return values, nulls + } + indexBySeqNum := make(map[uint16]int, len(cols)) + for idx, col := range cols { + indexBySeqNum[col.SeqNum] = idx + } + alignedValues := make([]string, len(targetSeqNums)) + alignedNulls := make([]bool, len(targetSeqNums)) + for targetIdx, seqNum := range targetSeqNums { + sourceIdx, ok := indexBySeqNum[seqNum] + if !ok || sourceIdx >= len(values) { + alignedNulls[targetIdx] = true + continue + } + alignedValues[targetIdx] = values[sourceIdx] + if sourceIdx < len(nulls) { + alignedNulls[targetIdx] = nulls[sourceIdx] + } + } + return alignedValues, alignedNulls +} + +func debugLogicalObjectColumns(stats objectio.ObjectStats, reader *objecttool.ObjectReader) { + if reader == nil { + return + } + info := reader.Info() + meta := reader.Meta() + header := meta.BlockHeader() + var cols strings.Builder + for i, col := range reader.Columns() { + if i > 0 { + cols.WriteString(",") + } + cols.WriteString(fmt.Sprintf("%d->seq%d:%s", col.Idx, col.SeqNum, col.Type.Oid.String())) + } + ckpDebugSchemaf( + "object columns object=%s stats_appendable=%v meta_appendable=%v blocks=%d rows=%d column_count=%d meta_column_count=%d max_seqnum=%d columns=[%s]", + stats.ObjectName().Short().ShortString(), + stats.GetAppendable(), + info != nil && info.IsAppendable, + meta.BlockCount(), + header.Rows(), + header.ColumnCount(), + header.MetaColumnCount(), + header.MaxSeqnum(), + cols.String(), + ) +} + func dedupeObjectStats(entries []*ObjectEntryInfo) []objectio.ObjectStats { seen := make(map[string]struct{}) stats := make([]objectio.ObjectStats, 0, len(entries)) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 2c892d6d3633b..e3fd031547906 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -615,7 +615,30 @@ func (r *CheckpointReader) getTableLogicalView( if err != nil { return nil, err } - return r.BuildLogicalTableView(ctx, snapshotTS, allData, allTomb) + view, err := r.BuildLogicalTableView(ctx, snapshotTS, allData, allTomb) + if err != nil { + return nil, err + } + applyCatalogColumnHeaders(view, tableID) + return view, nil +} + +func applyCatalogColumnHeaders(view *LogicalTableView, tableID uint64) { + if view == nil || len(view.ColSeqNums) == 0 { + return + } + schema := schemaForLayout(currentCatalogLayout, tableID) + if len(schema) == 0 { + return + } + offset := logicalViewDataOffset(view) + for dataIdx, seqNum := range view.ColSeqNums { + headerIdx := offset + dataIdx + if headerIdx >= len(view.Headers) || int(seqNum) >= len(schema) { + continue + } + view.Headers[headerIdx] = schema[seqNum] + } } func (r *CheckpointReader) getTableEntriesAt( @@ -1017,15 +1040,16 @@ func (r *CheckpointReader) streamTableCSV( } header := make([]string, 0, len(schema.Columns)) - physicalPositions := make([]int, 0, len(schema.Columns)) + columnSeqNums := make([]int, 0, len(schema.Columns)) for _, col := range schema.Columns { header = append(header, col.Name) - physicalPos := col.PhysicalPosition - if physicalPos < 0 { - physicalPos = col.Position + seqNum := col.PhysicalPosition + if seqNum < 0 { + seqNum = col.Position } - physicalPositions = append(physicalPositions, physicalPos) + columnSeqNums = append(columnSeqNums, seqNum) } + physicalPositions := append([]int(nil), columnSeqNums...) var projectedTypes []types.Type if options.IncludeHeader { // header is emitted after we know output mode but before rows @@ -1047,6 +1071,7 @@ func (r *CheckpointReader) streamTableCSV( stats, err := r.scanLogicalTable(ctx, snapshotTS, dataEntries, tombEntries, func(cols []objecttool.ColInfo) error { + physicalPositions = dataIndexesForSeqNums(cols, columnSeqNums) projectedTypes = buildProjectedTypes(cols, physicalPositions) return nil }, @@ -1157,14 +1182,14 @@ func (r *CheckpointReader) streamTableCSVPipeline( options CSVExportOptions, ) error { header := make([]string, 0, len(schema.Columns)) - physicalPositions := make([]int, 0, len(schema.Columns)) + columnSeqNums := make([]int, 0, len(schema.Columns)) for _, col := range schema.Columns { header = append(header, col.Name) - physicalPos := col.PhysicalPosition - if physicalPos < 0 { - physicalPos = col.Position + seqNum := col.PhysicalPosition + if seqNum < 0 { + seqNum = col.Position } - physicalPositions = append(physicalPositions, physicalPos) + columnSeqNums = append(columnSeqNums, seqNum) } if options.IncludeHeader { if err := writeSQLLoadCSVRow(w, nil, header, nil); err != nil { @@ -1204,7 +1229,7 @@ func (r *CheckpointReader) streamTableCSVPipeline( go reportCSVPipeline(ctx, reportDone, counters) go writeCSVChunks(ctx, w, chunks, counters, cancel, writerDone) - producerErr := r.produceCSVChunks(ctx, chunks, counters, snapshotTS, visibleDataEntries, tombstoneStats, physicalPositions, workerPlan) + producerErr := r.produceCSVChunks(ctx, chunks, counters, snapshotTS, visibleDataEntries, tombstoneStats, columnSeqNums, workerPlan) close(chunks) writerErr := <-writerDone printCSVPipelineReport("finish", counters) @@ -1221,7 +1246,7 @@ func (r *CheckpointReader) produceCSVChunks( snapshotTS types.TS, visibleDataEntries []*ObjectEntryInfo, tombstoneStats []objectio.ObjectStats, - physicalPositions []int, + columnSeqNums []int, workerPlan csvPipelineWorkerPlan, ) error { ctx, cancel := context.WithCancel(ctx) @@ -1250,7 +1275,7 @@ func (r *CheckpointReader) produceCSVChunks( setErr(err) return } - if err := r.processCSVObjectChunks(ctx, chunks, counters, snapshotTS, tombstoneStats, physicalPositions, job); err != nil { + if err := r.processCSVObjectChunks(ctx, chunks, counters, snapshotTS, tombstoneStats, columnSeqNums, job); err != nil { setErr(err) return } @@ -1284,7 +1309,7 @@ func (r *CheckpointReader) processCSVObjectChunks( counters *csvPipelineCounters, snapshotTS types.TS, tombstoneStats []objectio.ObjectStats, - physicalPositions []int, + columnSeqNums []int, job csvPipelineObjectJob, ) error { entry := job.entry @@ -1302,6 +1327,7 @@ func (r *CheckpointReader) processCSVObjectChunks( } defer reader.Close() + physicalPositions := dataIndexesForSeqNums(reader.Columns(), columnSeqNums) projectedTypes := buildProjectedTypes(reader.Columns(), physicalPositions) relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.ObjectStats.ObjectName().ObjectId(), tombstoneStats) if err != nil { @@ -1376,7 +1402,7 @@ func (r *CheckpointReader) readCSVObjectBlocks( counters *csvPipelineCounters, snapshotTS types.TS, tombstoneStats []objectio.ObjectStats, - physicalPositions []int, + columnSeqNums []int, job csvPipelineObjectJob, ) error { entry := job.entry @@ -1393,6 +1419,7 @@ func (r *CheckpointReader) readCSVObjectBlocks( return err } + physicalPositions := dataIndexesForSeqNums(reader.Columns(), columnSeqNums) projectedTypes := buildProjectedTypes(reader.Columns(), physicalPositions) relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.ObjectStats.ObjectName().ObjectId(), tombstoneStats) if err != nil { @@ -1823,6 +1850,20 @@ func buildProjectedTypes(cols []objecttool.ColInfo, physicalPositions []int) []t return projected } +func dataIndexesForSeqNums(cols []objecttool.ColInfo, seqNums []int) []int { + indexes := make([]int, len(seqNums)) + for i, seqNum := range seqNums { + indexes[i] = seqNum + for dataIdx, col := range cols { + if int(col.SeqNum) == seqNum { + indexes[i] = dataIdx + break + } + } + } + return indexes +} + func writeSQLLoadCSVRow(w io.Writer, colTypes []types.Type, fields []string, nulls []bool) error { for i, field := range fields { if i > 0 { @@ -2225,11 +2266,14 @@ func MergeLogicalViewWithSchema(view *LogicalTableView, schema *TableSchema) *Lo for _, col := range schema.Columns { newHeaders = append(newHeaders, col.Name) - physicalPos := col.PhysicalPosition - if physicalPos < 0 { - physicalPos = col.Position + dataPos := dataIndexForSeqNum(view, col.PhysicalPosition) + if dataPos < 0 { + dataPos = col.PhysicalPosition } - colMap = append(colMap, physicalPos) + if dataPos < 0 { + dataPos = col.Position + } + colMap = append(colMap, dataPos) } // Extract data rows: pick only visible columns by their physical position @@ -2255,6 +2299,18 @@ func MergeLogicalViewWithSchema(view *LogicalTableView, schema *TableSchema) *Lo } } +func dataIndexForSeqNum(view *LogicalTableView, seqNum int) int { + if view == nil || seqNum < 0 { + return -1 + } + for idx, candidate := range view.ColSeqNums { + if int(candidate) == seqNum { + return idx + } + } + return -1 +} + // buildSchemaFromMoTablesRow extracts a TableSchema from a mo_tables data row. // Uses column name lookup from view headers to handle hidden column offsets. // mo_tables columns: rel_id, relname, reldatabase, reldatabase_id, ..., rel_createsql diff --git a/pkg/tools/checkpointtool/types.go b/pkg/tools/checkpointtool/types.go index 73beb619c0942..0d9882059d9b1 100644 --- a/pkg/tools/checkpointtool/types.go +++ b/pkg/tools/checkpointtool/types.go @@ -87,6 +87,7 @@ type LogicalTableView struct { Headers []string Rows [][]string ColTypes []types.Type // column types for data columns (after meta cols) + ColSeqNums []uint16 // object seqnums for data columns (after meta cols) PhysicalRows int DeletedRows int VisibleRows int diff --git a/pkg/tools/objecttool/object_reader.go b/pkg/tools/objecttool/object_reader.go index a98fa89dc0384..c46403918a957 100644 --- a/pkg/tools/objecttool/object_reader.go +++ b/pkg/tools/objecttool/object_reader.go @@ -120,9 +120,10 @@ func OpenWithFS(ctx context.Context, fs fileservice.FileService, fileName string func buildObjectInfo(path string, meta objectio.ObjectDataMeta) *ObjectInfo { info := &ObjectInfo{ - Path: path, - BlockCount: meta.BlockCount(), - ColCount: meta.BlockHeader().ColumnCount(), + Path: path, + BlockCount: meta.BlockCount(), + ColCount: meta.BlockHeader().ColumnCount(), + IsAppendable: meta.BlockHeader().Appendable(), } // Calculate total row count @@ -136,17 +137,35 @@ func buildObjectInfo(path string, meta objectio.ObjectDataMeta) *ObjectInfo { func buildColInfo(meta objectio.ObjectDataMeta) []ColInfo { colCount := meta.BlockHeader().ColumnCount() cols := make([]ColInfo, colCount) + filled := make([]bool, colCount) // Get column types from first block if meta.BlockCount() > 0 { blockMeta := meta.GetBlockMeta(0) - for i := uint16(0); i < colCount; i++ { - colMeta := blockMeta.ColumnMeta(i) - cols[i] = ColInfo{ - Idx: i, - SeqNum: i, + metaColCount := blockMeta.GetMetaColumnCount() + for seqNum := uint16(0); seqNum < metaColCount; seqNum++ { + colMeta := blockMeta.ColumnMeta(seqNum) + idx := colMeta.Idx() + if idx >= colCount || colMeta.Location().OriginSize() == 0 { + continue + } + cols[idx] = ColInfo{ + Idx: idx, + SeqNum: seqNum, Type: types.T(colMeta.DataType()).ToType(), } + filled[idx] = true + } + } + + for i := uint16(0); i < colCount; i++ { + if filled[i] { + continue + } + cols[i] = ColInfo{ + Idx: i, + SeqNum: i, + Type: types.T_any.ToType(), } } From f21e78fb361d6a1e3db8fa36b07f404ca9bab9ff Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 17:12:11 +0800 Subject: [PATCH 34/76] fix(ckp): restore catalog table ddl attributes --- pkg/tools/checkpointtool/table_dump.go | 584 +++++++++++++++++--- pkg/tools/checkpointtool/table_dump_test.go | 91 +++ 2 files changed, 583 insertions(+), 92 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index e3fd031547906..89c64fe9b1db9 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -36,7 +36,9 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/tools/objecttool" + "github.com/matrixorigin/matrixone/pkg/vm/engine" ) const ( @@ -100,6 +102,23 @@ type TableColumn struct { SQLType string // SQL type string (e.g. "BIGINT", "VARCHAR(100)") Position int // SQL ordinal position PhysicalPosition int // physical/object column position + Unsigned bool + NotNull bool + Default string + HasDefault bool + OnUpdate string + Generated string + GeneratedStored bool + Comment string + ConstraintType string + AutoIncrement bool + ClusterBy bool + EnumValues string +} + +type TableUniqueKey struct { + Name string + Columns []string } // TableSchema holds the decoded schema for one user table. @@ -108,6 +127,8 @@ type TableSchema struct { DatabaseName string Columns []TableColumn // sorted by Position CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql + Comment string + UniqueKeys []TableUniqueKey } type TableCatalogEntry struct { @@ -384,32 +405,200 @@ func builtinTableSchemaForLayout(layout catalogLayout, tableID uint64) *TableSch } func renderCreateTableDDL(tableName string, cols []TableColumn) string { + return renderCreateTableDDLWithComment(tableName, cols, "") +} + +func renderCreateTableDDLWithComment(tableName string, cols []TableColumn, comment string, uniqueKeys ...TableUniqueKey) string { if tableName == "" || len(cols) == 0 { return "" } var sb strings.Builder - sb.WriteString("CREATE TABLE `") - sb.WriteString(tableName) - sb.WriteString("` (\n") + primaryCols := primaryKeyColumns(cols) + uniqueKeys = normalizedUniqueKeys(uniqueKeys) + sb.WriteString("CREATE TABLE ") + sb.WriteString(quoteDDLIdent(tableName)) + sb.WriteString(" (\n") for i, col := range cols { - sb.WriteString(" `") - sb.WriteString(col.Name) - sb.WriteString("`") + sb.WriteString(" ") + sb.WriteString(quoteDDLIdent(col.Name)) if col.SQLType != "" { sb.WriteString(" ") sb.WriteString(col.SQLType) + if col.Unsigned && !strings.Contains(strings.ToUpper(col.SQLType), "UNSIGNED") { + sb.WriteString(" UNSIGNED") + } } - if i < len(cols)-1 { + appendColumnDDLAttributes(&sb, col) + if i < len(cols)-1 || len(primaryCols) > 0 || len(uniqueKeys) > 0 { sb.WriteString(",\n") } else { sb.WriteString("\n") } } - sb.WriteString(");") + if len(primaryCols) > 0 { + sb.WriteString(" PRIMARY KEY (") + for i, col := range primaryCols { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(col.Name)) + } + if len(uniqueKeys) > 0 { + sb.WriteString("),\n") + } else { + sb.WriteString(")\n") + } + } + for i, key := range uniqueKeys { + if len(key.Columns) == 0 { + continue + } + sb.WriteString(" UNIQUE KEY ") + sb.WriteString(quoteDDLIdent(key.Name)) + sb.WriteString("(") + for i, col := range key.Columns { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(col)) + } + if i < len(uniqueKeys)-1 { + sb.WriteString("),\n") + } else { + sb.WriteString(")\n") + } + } + sb.WriteString(")") + if comment != "" { + sb.WriteString(" COMMENT=") + sb.WriteString(quoteDDLString(comment)) + } + if clusterCols := clusterByColumns(cols); len(clusterCols) > 0 { + sb.WriteString(" CLUSTER BY (") + for i, col := range clusterCols { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(col.Name)) + } + sb.WriteString(")") + } + sb.WriteString(";") return sb.String() } +func appendColumnDDLAttributes(sb *strings.Builder, col TableColumn) { + if col.Generated != "" { + if col.NotNull { + sb.WriteString(" NOT NULL") + } + sb.WriteString(" GENERATED ALWAYS AS (") + sb.WriteString(col.Generated) + sb.WriteString(")") + if col.GeneratedStored { + sb.WriteString(" STORED") + } else { + sb.WriteString(" VIRTUAL") + } + } else { + if col.NotNull || col.AutoIncrement { + sb.WriteString(" NOT NULL") + } + if col.AutoIncrement { + sb.WriteString(" AUTO_INCREMENT") + } + if col.HasDefault { + if strings.TrimSpace(col.Default) == "" { + sb.WriteString(" DEFAULT NULL") + } else { + sb.WriteString(" DEFAULT ") + sb.WriteString(formatDDLDefault(col.Default)) + } + } + if col.OnUpdate != "" { + sb.WriteString(" ON UPDATE ") + sb.WriteString(formatDDLDefault(col.OnUpdate)) + } + } + if col.Comment != "" { + sb.WriteString(" COMMENT ") + sb.WriteString(quoteDDLString(col.Comment)) + } +} + +func primaryKeyColumns(cols []TableColumn) []TableColumn { + primary := make([]TableColumn, 0, 1) + for _, col := range cols { + if strings.EqualFold(col.ConstraintType, "p") { + primary = append(primary, col) + } + } + sort.Slice(primary, func(i, j int) bool { + return primary[i].Position < primary[j].Position + }) + return primary +} + +func clusterByColumns(cols []TableColumn) []TableColumn { + clusterBy := make([]TableColumn, 0, 1) + for _, col := range cols { + if col.ClusterBy { + clusterBy = append(clusterBy, col) + } + } + sort.Slice(clusterBy, func(i, j int) bool { + return clusterBy[i].Position < clusterBy[j].Position + }) + return clusterBy +} + +func normalizedUniqueKeys(keys []TableUniqueKey) []TableUniqueKey { + seen := make(map[string]struct{}, len(keys)) + out := make([]TableUniqueKey, 0, len(keys)) + for _, key := range keys { + cols := make([]string, 0, len(key.Columns)) + for _, col := range key.Columns { + col = strings.TrimSpace(col) + if col == "" { + continue + } + cols = append(cols, col) + } + if len(cols) == 0 { + continue + } + name := strings.TrimSpace(key.Name) + if name == "" { + name = cols[0] + } + signature := strings.ToLower(name) + "\x00" + strings.ToLower(strings.Join(cols, "\x00")) + if _, ok := seen[signature]; ok { + continue + } + seen[signature] = struct{}{} + out = append(out, TableUniqueKey{Name: name, Columns: cols}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Name != out[j].Name { + return out[i].Name < out[j].Name + } + return strings.Join(out[i].Columns, "\x00") < strings.Join(out[j].Columns, "\x00") + }) + return out +} + +func formatDDLDefault(defaultExpr string) string { + defaultExpr = strings.TrimSpace(defaultExpr) + if defaultExpr == "" { + return "NULL" + } + if len(defaultExpr) >= 2 && defaultExpr[0] == '\'' && defaultExpr[len(defaultExpr)-1] == '\'' { + return "'" + strings.ReplaceAll(defaultExpr[1:len(defaultExpr)-1], "'", "''") + "'" + } + return defaultExpr +} + // RenderCreateTableDDLFromSchema renders a CREATE TABLE statement from resolved // checkpoint column metadata. func RenderCreateTableDDLFromSchema(schema *TableSchema) string { @@ -419,12 +608,27 @@ func RenderCreateTableDDLFromSchema(schema *TableSchema) string { if schema.TableName == "" || len(schema.Columns) == 0 { return "" } + if schema.Comment != "" && !isPrintableSQLText(schema.Comment) { + return "" + } for _, col := range schema.Columns { if col.Name == "" || !isPrintableSQLType(col.SQLType) { return "" } + if col.Default != "" && !isPrintableDDLExpression(col.Default) { + return "" + } + if col.OnUpdate != "" && !isPrintableDDLExpression(col.OnUpdate) { + return "" + } + if col.Generated != "" && !isPrintableDDLExpression(col.Generated) { + return "" + } + if col.Comment != "" && !isPrintableSQLText(col.Comment) { + return "" + } } - return renderCreateTableDDL(schema.TableName, schema.Columns) + return renderCreateTableDDLWithComment(schema.TableName, schema.Columns, schema.Comment, schema.UniqueKeys...) } func inferBuiltinCatalogLayout( @@ -867,6 +1071,7 @@ func cloneTableSchema(schema *TableSchema) *TableSchema { TableName: strings.Clone(schema.TableName), DatabaseName: strings.Clone(schema.DatabaseName), CreateSQL: strings.Clone(schema.CreateSQL), + Comment: strings.Clone(schema.Comment), } if len(schema.Columns) > 0 { clone.Columns = make([]TableColumn, len(schema.Columns)) @@ -876,6 +1081,30 @@ func cloneTableSchema(schema *TableSchema) *TableSchema { SQLType: strings.Clone(col.SQLType), Position: col.Position, PhysicalPosition: col.PhysicalPosition, + Unsigned: col.Unsigned, + NotNull: col.NotNull, + Default: strings.Clone(col.Default), + HasDefault: col.HasDefault, + OnUpdate: strings.Clone(col.OnUpdate), + Generated: strings.Clone(col.Generated), + GeneratedStored: col.GeneratedStored, + Comment: strings.Clone(col.Comment), + ConstraintType: strings.Clone(col.ConstraintType), + AutoIncrement: col.AutoIncrement, + ClusterBy: col.ClusterBy, + EnumValues: strings.Clone(col.EnumValues), + } + } + } + if len(schema.UniqueKeys) > 0 { + clone.UniqueKeys = make([]TableUniqueKey, len(schema.UniqueKeys)) + for i, key := range schema.UniqueKeys { + clone.UniqueKeys[i].Name = strings.Clone(key.Name) + if len(key.Columns) > 0 { + clone.UniqueKeys[i].Columns = make([]string, len(key.Columns)) + for j, col := range key.Columns { + clone.UniqueKeys[i].Columns[j] = strings.Clone(col) + } } } } @@ -2319,21 +2548,29 @@ func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *Table dataRow := fullRow[logicalViewDataOffset(view):] relNameIdx := fallbackCatalogColIndex(view, moTablesID, "relname") - if relNameIdx < len(dataRow) { + if relNameIdx >= 0 && relNameIdx < len(dataRow) { schema.TableName = strings.Clone(dataRow[relNameIdx]) } relDBIdx := fallbackCatalogColIndex(view, moTablesID, "reldatabase") - if relDBIdx < len(dataRow) { + if relDBIdx >= 0 && relDBIdx < len(dataRow) { schema.DatabaseName = strings.Clone(dataRow[relDBIdx]) } createSQLIdx := fallbackCatalogColIndex(view, moTablesID, "rel_createsql") - if createSQLIdx < len(dataRow) { + if createSQLIdx >= 0 && createSQLIdx < len(dataRow) { if isPrintableCreateTableSQL(dataRow[createSQLIdx]) { schema.CreateSQL = strings.Clone(dataRow[createSQLIdx]) } } + commentIdx := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment) + if commentIdx >= 0 && commentIdx < len(dataRow) && isPrintableSQLText(dataRow[commentIdx]) { + schema.Comment = strings.Clone(dataRow[commentIdx]) + } + constraintIdx := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint) + if constraintIdx >= 0 && constraintIdx < len(dataRow) { + schema.UniqueKeys = decodeUniqueKeysFromMoTablesConstraint(dataRow[constraintIdx]) + } return schema } @@ -2372,13 +2609,55 @@ func findTableNameFromMoTables(view *LogicalTableView, tableID uint64) string { return "" } +func findTableCommentFromMoTables(view *LogicalTableView, tableID uint64) string { + tableIDStr := fmt.Sprintf("%d", tableID) + relIDCol := fallbackCatalogColIndex(view, moTablesID, "rel_id") + commentCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment) + if relIDCol < 0 || commentCol < 0 { + return "" + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || commentCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr && isPrintableSQLText(row[commentCol]) { + return strings.Clone(row[commentCol]) + } + } + return "" +} + +func findUniqueKeysFromMoTables(view *LogicalTableView, tableID uint64) []TableUniqueKey { + tableIDStr := fmt.Sprintf("%d", tableID) + relIDCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_ID) + constraintCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint) + if relIDCol < 0 || constraintCol < 0 { + return nil + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || constraintCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr { + return decodeUniqueKeysFromMoTablesConstraint(row[constraintCol]) + } + } + return nil +} + func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView *LogicalTableView) string { tableName := "" + tableComment := "" + uniqueKeys := []TableUniqueKey(nil) if moTablesView != nil { tableName = findTableNameFromMoTables(moTablesView, tableID) + tableComment = findTableCommentFromMoTables(moTablesView, tableID) + uniqueKeys = findUniqueKeysFromMoTables(moTablesView, tableID) } if moColumnsView != nil { - if ddl := buildCreateTableFromMoColumns(moColumnsView, tableID, tableName); ddl != "" { + if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, uniqueKeys); ddl != "" { return ddl } } @@ -2590,6 +2869,19 @@ func buildColumnsFromMoColumnsRowsAt( var cols []TableColumn matchedRows := 0 typeDecodeFailures := 0 + notNullCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_NullAbility) + hasDefaultCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_HasExpr) + defaultCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_DefaultExpr) + constraintCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_ConstraintType) + unsignedCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsUnsigned) + autoIncrementCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsAutoIncrement) + commentCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Comment) + hasUpdateCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_HasUpdate) + updateCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Update) + clusterByCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsClusterBy) + enumCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_EnumValues) + hasGeneratedCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_HasGenerated) + generatedCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Generated) for _, fullRow := range view.Rows { row := fullRow[logicalViewDataOffset(view):] @@ -2650,6 +2942,43 @@ func buildColumnsFromMoColumnsRowsAt( ) } } + if notNullCol >= 0 && notNullCol < len(row) { + col.NotNull = isTruthyCatalogValue(row[notNullCol]) + } + if hasDefaultCol >= 0 && hasDefaultCol < len(row) { + col.HasDefault = isTruthyCatalogValue(row[hasDefaultCol]) + } + if defaultCol >= 0 && defaultCol < len(row) { + if defaultExpr := decodeMoColumnDefault(row[defaultCol]); defaultExpr != "" { + col.Default = defaultExpr + } + } + if constraintCol >= 0 && constraintCol < len(row) { + col.ConstraintType = row[constraintCol] + } + if unsignedCol >= 0 && unsignedCol < len(row) { + col.Unsigned = isTruthyCatalogValue(row[unsignedCol]) + } + if autoIncrementCol >= 0 && autoIncrementCol < len(row) { + col.AutoIncrement = isTruthyCatalogValue(row[autoIncrementCol]) + } + if commentCol >= 0 && commentCol < len(row) { + col.Comment = row[commentCol] + } + if hasUpdateCol >= 0 && hasUpdateCol < len(row) && isTruthyCatalogValue(row[hasUpdateCol]) && + updateCol >= 0 && updateCol < len(row) { + col.OnUpdate = decodeMoColumnOnUpdate(row[updateCol]) + } + if clusterByCol >= 0 && clusterByCol < len(row) { + col.ClusterBy = isTruthyCatalogValue(row[clusterByCol]) + } + if enumCol >= 0 && enumCol < len(row) { + col.EnumValues = row[enumCol] + } + if hasGeneratedCol >= 0 && hasGeneratedCol < len(row) && isTruthyCatalogValue(row[hasGeneratedCol]) && + generatedCol >= 0 && generatedCol < len(row) { + col.Generated, col.GeneratedStored = decodeMoColumnGenerated(row[generatedCol]) + } cols = append(cols, col) } @@ -2866,17 +3195,28 @@ func buildCreateIndexStatementsFromMoIndexes( ordinalCol := view.columnDataIndex("ordinal_position") hiddenCol := view.columnDataIndex("hidden") if tableIDCol < 0 || nameCol < 0 || columnNameCol < 0 { + ckpDebugSchemaf( + "mo_indexes headers unavailable table=%d headers=%v table_id_col=%d name_col=%d column_name_col=%d", + tableID, + view.Headers, + tableIDCol, + nameCol, + columnNameCol, + ) return nil, nil } byName := make(map[string]*indexDDLInfo) tableIDStr := strconv.FormatUint(tableID, 10) + matchedRows := 0 + hiddenRows := 0 for _, row := range view.Rows { if tableIDCol >= len(row) || row[tableIDCol] != tableIDStr { continue } + matchedRows++ if hiddenCol >= 0 && hiddenCol < len(row) && isTruthyCatalogValue(row[hiddenCol]) { - continue + hiddenRows++ } if nameCol >= len(row) || columnNameCol >= len(row) { continue @@ -2932,6 +3272,15 @@ func buildCreateIndexStatementsFromMoIndexes( statements = append(statements, stmt) } } + ckpDebugSchemaf( + "mo_indexes resolved table=%d rows=%d hidden_rows=%d indexes=%d names=%v statements=%d", + tableID, + matchedRows, + hiddenRows, + len(byName), + names, + len(statements), + ) return statements, nil } @@ -2988,6 +3337,53 @@ func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, e return sb.String(), nil } +func decodeUniqueKeysFromMoTablesConstraint(raw string) (keys []TableUniqueKey) { + if raw == "" { + return nil + } + defer func() { + if r := recover(); r != nil { + ckpDebugSchemaf("mo_tables constraint decode panic len=%d hex=%s panic=%v", len(raw), debugHexPrefix(raw, 64), r) + keys = nil + } + }() + c := &engine.ConstraintDef{} + if err := c.UnmarshalBinary([]byte(raw)); err != nil { + ckpDebugSchemaf("mo_tables constraint decode failed len=%d hex=%s err=%v", len(raw), debugHexPrefix(raw, 64), err) + return nil + } + for _, ct := range c.Cts { + indexDef, ok := ct.(*engine.IndexDef) + if !ok || indexDef == nil { + continue + } + for _, index := range indexDef.Indexes { + if index == nil || !index.Unique || strings.EqualFold(index.IndexName, "PRIMARY") { + continue + } + cols := make([]string, 0, len(index.Parts)) + for _, part := range index.Parts { + part = strings.TrimSpace(part) + if part == "" || catalog.IsAlias(part) { + continue + } + cols = append(cols, part) + } + if len(cols) == 0 { + continue + } + name := index.IndexName + if name == "" { + name = cols[0] + } + keys = append(keys, TableUniqueKey{Name: name, Columns: cols}) + } + } + keys = normalizedUniqueKeys(keys) + ckpDebugSchemaf("mo_tables constraint unique keys decoded count=%d", len(keys)) + return keys +} + func quoteDDLIdent(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } @@ -3016,7 +3412,20 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, table if len(tableNames) > 0 && tableNames[0] != "" { tableName = tableNames[0] } + tableComment := "" + if len(tableNames) > 1 && tableNames[1] != "" { + tableComment = tableNames[1] + } + return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, nil) +} +func buildCreateTableFromMoColumnsWithOptions( + view *LogicalTableView, + tableID uint64, + tableName string, + tableComment string, + uniqueKeys []TableUniqueKey, +) string { relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") typCol := fallbackCatalogColIndex(view, moColumnsID, "atttyp") @@ -3024,7 +3433,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, table hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -3036,7 +3445,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, table typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -3048,91 +3457,20 @@ func buildCreateTableFromMoColumnsAt( view *LogicalTableView, tableID uint64, tableName string, + tableComment string, + uniqueKeys []TableUniqueKey, relnameIDCol int, nameCol int, typCol int, numCol int, hiddenCol int, ) string { - tableIDStr := fmt.Sprintf("%d", tableID) - type colInfo struct { - name string - sqlType string - position int - } - var cols []colInfo - - for _, fullRow := range view.Rows { - row := fullRow[logicalViewDataOffset(view):] - if hiddenCol >= 0 && hiddenCol < len(row) { - if row[hiddenCol] == "1" || row[hiddenCol] == "true" { - continue - } - } - if relnameIDCol < 0 || relnameIDCol >= len(row) || row[relnameIDCol] != tableIDStr { - continue - } - pos := len(cols) - if numCol >= 0 && numCol < len(row) { - if n, err := strconv.Atoi(row[numCol]); err == nil { - pos = n - } - } - c := colInfo{position: pos} - if nameCol >= 0 && nameCol < len(row) { - c.name = row[nameCol] - } - if typCol >= 0 && typCol < len(row) { - sqlType, ok := decodeMoColumnSQLType(row[typCol]) - if !ok { - ckpDebugSchemaf( - "mo_columns ddl atttyp decode failed table=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d column=%q %s", - tableID, - relnameIDCol, - nameCol, - typCol, - numCol, - hiddenCol, - c.name, - debugMoColumnTypeCell(row[typCol], fullRow), - ) - return "" - } - c.sqlType = sqlType - } - cols = append(cols, c) - } - + seqNumCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Seqnum) + cols := buildColumnsFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol, seqNumCol) if len(cols) == 0 { return "" } - - sort.Slice(cols, func(i, j int) bool { return cols[i].position < cols[j].position }) - - var sb strings.Builder - sb.WriteString("CREATE TABLE ") - sb.WriteString(quoteDDLIdent(tableName)) - sb.WriteString(" (\n") - for i, c := range cols { - if c.name != "" { - sb.WriteString(" ") - sb.WriteString(quoteDDLIdent(c.name)) - } else { - sb.WriteString(" ") - sb.WriteString(quoteDDLIdent(fmt.Sprintf("col_%d", i))) - } - if c.sqlType != "" { - sb.WriteString(" ") - sb.WriteString(c.sqlType) - } - if i < len(cols)-1 { - sb.WriteString(",\n") - } else { - sb.WriteString("\n") - } - } - sb.WriteString(");") - return sb.String() + return renderCreateTableDDLWithComment(tableName, cols, tableComment, uniqueKeys...) } func isPrintableSQLType(sqlType string) bool { @@ -3169,6 +3507,48 @@ func decodeMoColumnSQLType(raw string) (string, bool) { return "", false } +func decodeMoColumnDefault(raw string) string { + if raw == "" { + return "" + } + var def plan.Default + if err := types.Decode([]byte(raw), &def); err == nil { + return strings.TrimSpace(def.OriginString) + } + if isPrintableDDLExpression(raw) { + return strings.TrimSpace(raw) + } + return "" +} + +func decodeMoColumnOnUpdate(raw string) string { + if raw == "" { + return "" + } + var update plan.OnUpdate + if err := types.Decode([]byte(raw), &update); err == nil { + return strings.TrimSpace(update.OriginString) + } + if isPrintableDDLExpression(raw) { + return strings.TrimSpace(raw) + } + return "" +} + +func decodeMoColumnGenerated(raw string) (string, bool) { + if raw == "" { + return "", false + } + var generated plan.GeneratedCol + if err := types.Decode([]byte(raw), &generated); err == nil { + return strings.TrimSpace(generated.OriginString), generated.IsStored + } + if isPrintableDDLExpression(raw) { + return strings.TrimSpace(raw), false + } + return "", false +} + func decodeMoColumnEncodedSQLType(raw string) (string, bool) { var typ types.Type if len(raw) < typ.ProtoSize() { @@ -3183,6 +3563,26 @@ func decodeMoColumnEncodedSQLType(raw string) (string, bool) { return "", false } +func isPrintableDDLExpression(expr string) bool { + expr = strings.TrimSpace(expr) + if expr == "" { + return true + } + return isPrintableSQLText(expr) +} + +func isPrintableSQLText(s string) bool { + for _, r := range s { + switch { + case r == '\n' || r == '\r' || r == '\t': + case r >= 0x20 && r <= 0x7e: + default: + return false + } + } + return true +} + func ckpDebugSchemaf(format string, args ...any) { if os.Getenv("MO_TOOL_CKP_DEBUG_SCHEMA") == "" { return diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index ea50a26839439..1b13f1676047e 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -25,7 +25,9 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/sql/util/csvparser" + "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -37,6 +39,33 @@ func encodedSQLType(t *testing.T, typ types.Type) string { return string(data) } +func encodedDefault(t *testing.T, origin string, nullable bool) string { + t.Helper() + data, err := types.Encode(&plan.Default{OriginString: origin, NullAbility: nullable}) + require.NoError(t, err) + return string(data) +} + +func encodedUniqueConstraint(t *testing.T, name string, columns ...string) string { + t.Helper() + def := &engine.ConstraintDef{ + Cts: []engine.Constraint{ + &engine.IndexDef{ + Indexes: []*plan.IndexDef{ + { + IndexName: name, + Parts: columns, + Unique: true, + }, + }, + }, + }, + } + data, err := def.MarshalBinary() + require.NoError(t, err) + return string(data) +} + // TestWriteCSV_empty tests writing an empty logical view to CSV. func TestWriteCSV_empty(t *testing.T) { schema := &TableSchema{ @@ -488,6 +517,68 @@ func TestCreateTableDDLFromCatalogViews_DecodesMoColumnTypes(t *testing.T) { assert.NotContains(t, ddl, "`old`") } +func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "relpersistence", "relkind", "rel_comment", "rel_createsql", "constraint", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "333999", "parent", "ckp_constraints", "333997", + "", "r", "parent table comment", "CREATE TABLE `ckp_constraints`.`parent` (`old` INT)", encodedUniqueConstraint(t, "code", "code"), + }, + }, + } + + headers := append([]string{"object", "block", "row"}, catalog.MoColumnsSchema...) + row := func(name, typ, attnum, notNull, hasDefault, defaultExpr, constraintType, comment, seqnum string) []string { + data := make([]string, len(catalog.MoColumnsSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoColumnsSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_columns header %s", colName) + } + set(catalog.SystemColAttr_RelID, "333999") + set(catalog.SystemColAttr_RelName, "parent") + set(catalog.SystemColAttr_Name, name) + set(catalog.SystemColAttr_Type, typ) + set(catalog.SystemColAttr_Num, attnum) + set(catalog.SystemColAttr_NullAbility, notNull) + set(catalog.SystemColAttr_HasExpr, hasDefault) + set(catalog.SystemColAttr_DefaultExpr, defaultExpr) + set(catalog.SystemColAttr_ConstraintType, constraintType) + set(catalog.SystemColAttr_Comment, comment) + set(catalog.SystemColAttr_IsHidden, "0") + set(catalog.SystemColAttr_Seqnum, seqnum) + return append([]string{"obj1", "0", attnum}, data...) + } + moColumnsView := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("id", encodedSQLType(t, types.T_int32.ToType()), "1", "1", "0", "", "p", "", "0"), + row("code", encodedSQLType(t, types.New(types.T_varchar, 20, 0)), "2", "1", "0", "", "", "", "1"), + row("note", encodedSQLType(t, types.New(types.T_varchar, 100, 0)), "3", "0", "1", encodedDefault(t, "'parent-default'", true), "", "parent note", "2"), + }, + } + + ddl := createTableDDLFromCatalogViews(333999, moTablesView, moColumnsView) + assert.Contains(t, ddl, "CREATE TABLE `parent`") + assert.Contains(t, ddl, "`id` INT NOT NULL") + assert.Contains(t, ddl, "`code` VARCHAR(20) NOT NULL") + assert.Contains(t, ddl, "`note` VARCHAR(100) DEFAULT 'parent-default' COMMENT 'parent note'") + assert.Contains(t, ddl, "PRIMARY KEY (`id`)") + assert.Contains(t, ddl, "UNIQUE KEY `code`(`code`)") + assert.Contains(t, ddl, "COMMENT='parent table comment'") + assert.NotContains(t, ddl, "`old`") +} + func TestRenderCreateTableDDLFromSchema_PrefersColumnsOverCreateSQL(t *testing.T) { ddl := RenderCreateTableDDLFromSchema(&TableSchema{ TableName: "t", From 90fcfc20e8fce236aa3dbe1a32c6a50567bd7576 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 17:22:01 +0800 Subject: [PATCH 35/76] fix(ckp): restore non-unique catalog indexes --- pkg/tools/checkpointtool/table_dump.go | 18 ++++++++++++------ pkg/tools/checkpointtool/table_dump_test.go | 17 ++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 89c64fe9b1db9..d848348df7f2b 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -119,6 +119,7 @@ type TableColumn struct { type TableUniqueKey struct { Name string Columns []string + Unique bool } // TableSchema holds the decoded schema for one user table. @@ -454,7 +455,11 @@ func renderCreateTableDDLWithComment(tableName string, cols []TableColumn, comme if len(key.Columns) == 0 { continue } - sb.WriteString(" UNIQUE KEY ") + sb.WriteString(" ") + if key.Unique { + sb.WriteString("UNIQUE ") + } + sb.WriteString("KEY ") sb.WriteString(quoteDDLIdent(key.Name)) sb.WriteString("(") for i, col := range key.Columns { @@ -572,12 +577,12 @@ func normalizedUniqueKeys(keys []TableUniqueKey) []TableUniqueKey { if name == "" { name = cols[0] } - signature := strings.ToLower(name) + "\x00" + strings.ToLower(strings.Join(cols, "\x00")) + signature := strconv.FormatBool(key.Unique) + "\x00" + strings.ToLower(name) + "\x00" + strings.ToLower(strings.Join(cols, "\x00")) if _, ok := seen[signature]; ok { continue } seen[signature] = struct{}{} - out = append(out, TableUniqueKey{Name: name, Columns: cols}) + out = append(out, TableUniqueKey{Name: name, Columns: cols, Unique: key.Unique}) } sort.Slice(out, func(i, j int) bool { if out[i].Name != out[j].Name { @@ -1100,6 +1105,7 @@ func cloneTableSchema(schema *TableSchema) *TableSchema { clone.UniqueKeys = make([]TableUniqueKey, len(schema.UniqueKeys)) for i, key := range schema.UniqueKeys { clone.UniqueKeys[i].Name = strings.Clone(key.Name) + clone.UniqueKeys[i].Unique = key.Unique if len(key.Columns) > 0 { clone.UniqueKeys[i].Columns = make([]string, len(key.Columns)) for j, col := range key.Columns { @@ -3358,7 +3364,7 @@ func decodeUniqueKeysFromMoTablesConstraint(raw string) (keys []TableUniqueKey) continue } for _, index := range indexDef.Indexes { - if index == nil || !index.Unique || strings.EqualFold(index.IndexName, "PRIMARY") { + if index == nil || strings.EqualFold(index.IndexName, "PRIMARY") { continue } cols := make([]string, 0, len(index.Parts)) @@ -3376,11 +3382,11 @@ func decodeUniqueKeysFromMoTablesConstraint(raw string) (keys []TableUniqueKey) if name == "" { name = cols[0] } - keys = append(keys, TableUniqueKey{Name: name, Columns: cols}) + keys = append(keys, TableUniqueKey{Name: name, Columns: cols, Unique: index.Unique}) } } keys = normalizedUniqueKeys(keys) - ckpDebugSchemaf("mo_tables constraint unique keys decoded count=%d", len(keys)) + ckpDebugSchemaf("mo_tables constraint indexes decoded count=%d", len(keys)) return keys } diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 1b13f1676047e..0d93f38b2c78a 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -46,18 +46,12 @@ func encodedDefault(t *testing.T, origin string, nullable bool) string { return string(data) } -func encodedUniqueConstraint(t *testing.T, name string, columns ...string) string { +func encodedIndexConstraint(t *testing.T, indexes ...*plan.IndexDef) string { t.Helper() def := &engine.ConstraintDef{ Cts: []engine.Constraint{ &engine.IndexDef{ - Indexes: []*plan.IndexDef{ - { - IndexName: name, - Parts: columns, - Unique: true, - }, - }, + Indexes: indexes, }, }, } @@ -528,7 +522,11 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test { "obj1", "0", "0", "333999", "parent", "ckp_constraints", "333997", - "", "r", "parent table comment", "CREATE TABLE `ckp_constraints`.`parent` (`old` INT)", encodedUniqueConstraint(t, "code", "code"), + "", "r", "parent table comment", "CREATE TABLE `ckp_constraints`.`parent` (`old` INT)", encodedIndexConstraint( + t, + &plan.IndexDef{IndexName: "code", Parts: []string{"code"}, Unique: true}, + &plan.IndexDef{IndexName: "idx_parent_note", Parts: []string{"note"}}, + ), }, }, } @@ -575,6 +573,7 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test assert.Contains(t, ddl, "`note` VARCHAR(100) DEFAULT 'parent-default' COMMENT 'parent note'") assert.Contains(t, ddl, "PRIMARY KEY (`id`)") assert.Contains(t, ddl, "UNIQUE KEY `code`(`code`)") + assert.Contains(t, ddl, "KEY `idx_parent_note`(`note`)") assert.Contains(t, ddl, "COMMENT='parent table comment'") assert.NotContains(t, ddl, "`old`") } From ebbe27c9db831ac801d9bd2d825b9bfeba0575ad Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 17:37:39 +0800 Subject: [PATCH 36/76] fix(ckp): restore table partition clause --- pkg/tools/checkpointtool/table_dump.go | 132 ++++++++++++++++++-- pkg/tools/checkpointtool/table_dump_test.go | 15 ++- 2 files changed, 138 insertions(+), 9 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index d848348df7f2b..c1c1abc2b89bc 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -130,6 +130,7 @@ type TableSchema struct { CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql Comment string UniqueKeys []TableUniqueKey + Partition string } type TableCatalogEntry struct { @@ -191,6 +192,15 @@ var moIndexesHeaders = []string{ "index_table_name", } +var moPartitionMetadataHeaders = []string{ + "table_id", + "table_name", + "database_name", + "partition_method", + "partition_description", + "partition_count", +} + type indexDDLColumn struct { name string ordinal int @@ -410,6 +420,10 @@ func renderCreateTableDDL(tableName string, cols []TableColumn) string { } func renderCreateTableDDLWithComment(tableName string, cols []TableColumn, comment string, uniqueKeys ...TableUniqueKey) string { + return renderCreateTableDDLFull(tableName, cols, comment, "", uniqueKeys...) +} + +func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment string, partition string, uniqueKeys ...TableUniqueKey) string { if tableName == "" || len(cols) == 0 { return "" } @@ -479,6 +493,10 @@ func renderCreateTableDDLWithComment(tableName string, cols []TableColumn, comme sb.WriteString(" COMMENT=") sb.WriteString(quoteDDLString(comment)) } + if partition != "" { + sb.WriteString(" ") + sb.WriteString(partition) + } if clusterCols := clusterByColumns(cols); len(clusterCols) > 0 { sb.WriteString(" CLUSTER BY (") for i, col := range clusterCols { @@ -633,7 +651,7 @@ func RenderCreateTableDDLFromSchema(schema *TableSchema) string { return "" } } - return renderCreateTableDDLWithComment(schema.TableName, schema.Columns, schema.Comment, schema.UniqueKeys...) + return renderCreateTableDDLFull(schema.TableName, schema.Columns, schema.Comment, schema.Partition, schema.UniqueKeys...) } func inferBuiltinCatalogLayout( @@ -808,6 +826,11 @@ func (r *CheckpointReader) ReadTableSchema( layout := inferBuiltinCatalogLayout(tableID, moTablesView, moColumnsView) schema = mergeBuiltinSchemaFallback(schema, builtinTableSchemaForLayout(layout, tableID), tableID) } + if schema.Partition == "" { + if partitionClause := r.readPartitionClause(ctx, tableID, snapshotTS); partitionClause != "" { + schema.Partition = partitionClause + } + } return schema } @@ -1077,6 +1100,7 @@ func cloneTableSchema(schema *TableSchema) *TableSchema { DatabaseName: strings.Clone(schema.DatabaseName), CreateSQL: strings.Clone(schema.CreateSQL), Comment: strings.Clone(schema.Comment), + Partition: strings.Clone(schema.Partition), } if len(schema.Columns) > 0 { clone.Columns = make([]TableColumn, len(schema.Columns)) @@ -2653,17 +2677,21 @@ func findUniqueKeysFromMoTables(view *LogicalTableView, tableID uint64) []TableU return nil } -func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView *LogicalTableView) string { +func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView *LogicalTableView, partitionMetadataView ...*LogicalTableView) string { tableName := "" tableComment := "" uniqueKeys := []TableUniqueKey(nil) + partition := "" if moTablesView != nil { tableName = findTableNameFromMoTables(moTablesView, tableID) tableComment = findTableCommentFromMoTables(moTablesView, tableID) uniqueKeys = findUniqueKeysFromMoTables(moTablesView, tableID) } + if len(partitionMetadataView) > 0 && partitionMetadataView[0] != nil { + partition = buildPartitionClauseFromMetadata(partitionMetadataView[0], tableID) + } if moColumnsView != nil { - if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, uniqueKeys); ddl != "" { + if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, partition, uniqueKeys); ddl != "" { return ddl } } @@ -3059,7 +3087,12 @@ func (r *CheckpointReader) ShowCreateTable( moColumnsView = nil } - if ddl := createTableDDLFromCatalogViews(tableID, moTablesView, moColumnsView); ddl != "" { + var partitionMetadataView *LogicalTableView + if view, err := r.getPartitionMetadataView(ctx, snapshotTS); err == nil { + partitionMetadataView = view + } + + if ddl := createTableDDLFromCatalogViews(tableID, moTablesView, moColumnsView, partitionMetadataView); ddl != "" { return ddl, nil } @@ -3115,6 +3148,32 @@ func (r *CheckpointReader) ShowCreateIndexStatements( return buildCreateIndexStatementsFromMoIndexes(view, tableID, tableName) } +func (r *CheckpointReader) getPartitionMetadataView( + ctx context.Context, + snapshotTS types.TS, +) (*LogicalTableView, error) { + partitionMetadataTableID, ok, err := r.findCatalogTableID(ctx, snapshotTS, catalog.MOPartitionMetadata) + if err != nil { + return nil, err + } + if !ok { + return nil, nil + } + return r.dumpCatalogTableViewWithHeaders(ctx, partitionMetadataTableID, snapshotTS, moPartitionMetadataHeaders) +} + +func (r *CheckpointReader) readPartitionClause( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) string { + view, err := r.getPartitionMetadataView(ctx, snapshotTS) + if err != nil || view == nil { + return "" + } + return buildPartitionClauseFromMetadata(view, tableID) +} + func (r *CheckpointReader) dumpCatalogTableViewWithHeaders( ctx context.Context, tableID uint64, @@ -3290,6 +3349,61 @@ func buildCreateIndexStatementsFromMoIndexes( return statements, nil } +func buildPartitionClauseFromMetadata(view *LogicalTableView, tableID uint64) string { + if view == nil { + return "" + } + tableIDCol := view.columnDataIndex("table_id") + methodCol := view.columnDataIndex("partition_method") + descriptionCol := view.columnDataIndex("partition_description") + countCol := view.columnDataIndex("partition_count") + if tableIDCol < 0 || methodCol < 0 || descriptionCol < 0 { + ckpDebugSchemaf( + "partition metadata headers unavailable table=%d headers=%v table_id_col=%d method_col=%d description_col=%d count_col=%d", + tableID, + view.Headers, + tableIDCol, + methodCol, + descriptionCol, + countCol, + ) + return "" + } + tableIDStr := strconv.FormatUint(tableID, 10) + for _, row := range view.Rows { + if tableIDCol >= len(row) || row[tableIDCol] != tableIDStr { + continue + } + if methodCol >= len(row) || descriptionCol >= len(row) { + continue + } + method := strings.TrimSpace(row[methodCol]) + description := strings.TrimSpace(row[descriptionCol]) + if description == "" || !isPrintableSQLText(description) { + return "" + } + clause := "partition by " + description + if isAutoPartitionCountMethod(method) && countCol >= 0 && countCol < len(row) { + if count, err := strconv.Atoi(strings.TrimSpace(row[countCol])); err == nil && count > 0 { + clause += fmt.Sprintf(" partitions %d", count) + } + } + ckpDebugSchemaf("partition metadata resolved table=%d method=%q description=%q clause=%q", tableID, method, description, clause) + return clause + } + ckpDebugSchemaf("partition metadata not found table=%d rows=%d", tableID, len(view.Rows)) + return "" +} + +func isAutoPartitionCountMethod(method string) bool { + switch strings.ToLower(strings.TrimSpace(method)) { + case "key", "linearkey", "hash", "linearhash": + return true + default: + return false + } +} + func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, error) { if info == nil || tableName == "" || len(info.columns) == 0 { return "", nil @@ -3422,7 +3536,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, table if len(tableNames) > 1 && tableNames[1] != "" { tableComment = tableNames[1] } - return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, nil) + return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, "", nil) } func buildCreateTableFromMoColumnsWithOptions( @@ -3430,6 +3544,7 @@ func buildCreateTableFromMoColumnsWithOptions( tableID uint64, tableName string, tableComment string, + partition string, uniqueKeys []TableUniqueKey, ) string { relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") @@ -3439,7 +3554,7 @@ func buildCreateTableFromMoColumnsWithOptions( hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -3451,7 +3566,7 @@ func buildCreateTableFromMoColumnsWithOptions( typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -3464,6 +3579,7 @@ func buildCreateTableFromMoColumnsAt( tableID uint64, tableName string, tableComment string, + partition string, uniqueKeys []TableUniqueKey, relnameIDCol int, nameCol int, @@ -3476,7 +3592,7 @@ func buildCreateTableFromMoColumnsAt( if len(cols) == 0 { return "" } - return renderCreateTableDDLWithComment(tableName, cols, tableComment, uniqueKeys...) + return renderCreateTableDDLFull(tableName, cols, tableComment, partition, uniqueKeys...) } func isPrintableSQLType(sqlType string) bool { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 0d93f38b2c78a..9ba598cd785c8 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -565,8 +565,20 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test row("note", encodedSQLType(t, types.New(types.T_varchar, 100, 0)), "3", "0", "1", encodedDefault(t, "'parent-default'", true), "", "parent note", "2"), }, } + partitionMetadataView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "table_id", "table_name", "database_name", "partition_method", "partition_description", "partition_count", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "333999", "parent", "ckp_constraints", "Key", "key algorithm = 2 (`id`, `tenant_id`)", "4", + }, + }, + } - ddl := createTableDDLFromCatalogViews(333999, moTablesView, moColumnsView) + ddl := createTableDDLFromCatalogViews(333999, moTablesView, moColumnsView, partitionMetadataView) assert.Contains(t, ddl, "CREATE TABLE `parent`") assert.Contains(t, ddl, "`id` INT NOT NULL") assert.Contains(t, ddl, "`code` VARCHAR(20) NOT NULL") @@ -575,6 +587,7 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test assert.Contains(t, ddl, "UNIQUE KEY `code`(`code`)") assert.Contains(t, ddl, "KEY `idx_parent_note`(`note`)") assert.Contains(t, ddl, "COMMENT='parent table comment'") + assert.Contains(t, ddl, "partition by key algorithm = 2 (`id`, `tenant_id`) partitions 4") assert.NotContains(t, ddl, "`old`") } From 63d77f4b4a1a3d3fa2dd36816d8e99287a7f2c92 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 17:46:33 +0800 Subject: [PATCH 37/76] fix(ckp): restore primary keys from constraints --- pkg/tools/checkpointtool/table_dump.go | 113 +++++++++++++++++--- pkg/tools/checkpointtool/table_dump_test.go | 93 ++++++++++++++-- 2 files changed, 181 insertions(+), 25 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index c1c1abc2b89bc..5963a399544a6 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -130,6 +130,7 @@ type TableSchema struct { CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql Comment string UniqueKeys []TableUniqueKey + PrimaryKey []string Partition string } @@ -420,16 +421,16 @@ func renderCreateTableDDL(tableName string, cols []TableColumn) string { } func renderCreateTableDDLWithComment(tableName string, cols []TableColumn, comment string, uniqueKeys ...TableUniqueKey) string { - return renderCreateTableDDLFull(tableName, cols, comment, "", uniqueKeys...) + return renderCreateTableDDLFull(tableName, cols, comment, "", nil, uniqueKeys...) } -func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment string, partition string, uniqueKeys ...TableUniqueKey) string { +func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment string, partition string, primaryKey []string, uniqueKeys ...TableUniqueKey) string { if tableName == "" || len(cols) == 0 { return "" } var sb strings.Builder - primaryCols := primaryKeyColumns(cols) + primaryCols := primaryKeyColumns(cols, primaryKey) uniqueKeys = normalizedUniqueKeys(uniqueKeys) sb.WriteString("CREATE TABLE ") sb.WriteString(quoteDDLIdent(tableName)) @@ -550,16 +551,22 @@ func appendColumnDDLAttributes(sb *strings.Builder, col TableColumn) { } } -func primaryKeyColumns(cols []TableColumn) []TableColumn { - primary := make([]TableColumn, 0, 1) +func primaryKeyColumns(cols []TableColumn, primaryKey []string) []TableColumn { + if len(primaryKey) == 0 { + return nil + } + byName := make(map[string]TableColumn, len(cols)) for _, col := range cols { - if strings.EqualFold(col.ConstraintType, "p") { - primary = append(primary, col) + byName[strings.ToLower(col.Name)] = col + } + primary := make([]TableColumn, 0, 1) + for i, name := range primaryKey { + col, ok := byName[strings.ToLower(name)] + if !ok { + col = TableColumn{Name: name, Position: i + 1} } + primary = append(primary, col) } - sort.Slice(primary, func(i, j int) bool { - return primary[i].Position < primary[j].Position - }) return primary } @@ -651,7 +658,7 @@ func RenderCreateTableDDLFromSchema(schema *TableSchema) string { return "" } } - return renderCreateTableDDLFull(schema.TableName, schema.Columns, schema.Comment, schema.Partition, schema.UniqueKeys...) + return renderCreateTableDDLFull(schema.TableName, schema.Columns, schema.Comment, schema.Partition, schema.PrimaryKey, schema.UniqueKeys...) } func inferBuiltinCatalogLayout( @@ -1102,6 +1109,12 @@ func cloneTableSchema(schema *TableSchema) *TableSchema { Comment: strings.Clone(schema.Comment), Partition: strings.Clone(schema.Partition), } + if len(schema.PrimaryKey) > 0 { + clone.PrimaryKey = make([]string, len(schema.PrimaryKey)) + for i, col := range schema.PrimaryKey { + clone.PrimaryKey[i] = strings.Clone(col) + } + } if len(schema.Columns) > 0 { clone.Columns = make([]TableColumn, len(schema.Columns)) for i, col := range schema.Columns { @@ -2600,6 +2613,7 @@ func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *Table constraintIdx := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint) if constraintIdx >= 0 && constraintIdx < len(dataRow) { schema.UniqueKeys = decodeUniqueKeysFromMoTablesConstraint(dataRow[constraintIdx]) + schema.PrimaryKey = decodePrimaryKeyFromMoTablesConstraint(dataRow[constraintIdx]) } return schema } @@ -2677,21 +2691,42 @@ func findUniqueKeysFromMoTables(view *LogicalTableView, tableID uint64) []TableU return nil } +func findPrimaryKeyFromMoTables(view *LogicalTableView, tableID uint64) []string { + tableIDStr := fmt.Sprintf("%d", tableID) + relIDCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_ID) + constraintCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint) + if relIDCol < 0 || constraintCol < 0 { + return nil + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || constraintCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr { + return decodePrimaryKeyFromMoTablesConstraint(row[constraintCol]) + } + } + return nil +} + func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView *LogicalTableView, partitionMetadataView ...*LogicalTableView) string { tableName := "" tableComment := "" uniqueKeys := []TableUniqueKey(nil) + primaryKey := []string(nil) partition := "" if moTablesView != nil { tableName = findTableNameFromMoTables(moTablesView, tableID) tableComment = findTableCommentFromMoTables(moTablesView, tableID) uniqueKeys = findUniqueKeysFromMoTables(moTablesView, tableID) + primaryKey = findPrimaryKeyFromMoTables(moTablesView, tableID) } if len(partitionMetadataView) > 0 && partitionMetadataView[0] != nil { partition = buildPartitionClauseFromMetadata(partitionMetadataView[0], tableID) } if moColumnsView != nil { - if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, partition, uniqueKeys); ddl != "" { + if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys); ddl != "" { return ddl } } @@ -3504,6 +3539,50 @@ func decodeUniqueKeysFromMoTablesConstraint(raw string) (keys []TableUniqueKey) return keys } +func decodePrimaryKeyFromMoTablesConstraint(raw string) (cols []string) { + if raw == "" { + return nil + } + defer func() { + if r := recover(); r != nil { + ckpDebugSchemaf("mo_tables primary key decode panic len=%d hex=%s panic=%v", len(raw), debugHexPrefix(raw, 64), r) + cols = nil + } + }() + c := &engine.ConstraintDef{} + if err := c.UnmarshalBinary([]byte(raw)); err != nil { + ckpDebugSchemaf("mo_tables primary key decode failed len=%d hex=%s err=%v", len(raw), debugHexPrefix(raw, 64), err) + return nil + } + for _, ct := range c.Cts { + pkDef, ok := ct.(*engine.PrimaryKeyDef) + if !ok || pkDef == nil || pkDef.Pkey == nil { + continue + } + pk := pkDef.Pkey + if catalog.IsFakePkName(pk.PkeyColName) { + continue + } + for _, name := range pk.Names { + name = strings.TrimSpace(name) + if name != "" && !catalog.IsAlias(name) && !catalog.IsFakePkName(name) && name != catalog.CPrimaryKeyColName { + cols = append(cols, name) + } + } + if len(cols) == 0 { + name := strings.TrimSpace(pk.PkeyColName) + if name != "" && !catalog.IsAlias(name) && !catalog.IsFakePkName(name) && name != catalog.CPrimaryKeyColName { + cols = append(cols, name) + } + } + if len(cols) > 0 { + ckpDebugSchemaf("mo_tables primary key decoded cols=%v", cols) + return cols + } + } + return nil +} + func quoteDDLIdent(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } @@ -3536,7 +3615,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, table if len(tableNames) > 1 && tableNames[1] != "" { tableComment = tableNames[1] } - return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, "", nil) + return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, "", nil, nil) } func buildCreateTableFromMoColumnsWithOptions( @@ -3545,6 +3624,7 @@ func buildCreateTableFromMoColumnsWithOptions( tableName string, tableComment string, partition string, + primaryKey []string, uniqueKeys []TableUniqueKey, ) string { relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") @@ -3554,7 +3634,7 @@ func buildCreateTableFromMoColumnsWithOptions( hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -3566,7 +3646,7 @@ func buildCreateTableFromMoColumnsWithOptions( typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -3580,6 +3660,7 @@ func buildCreateTableFromMoColumnsAt( tableName string, tableComment string, partition string, + primaryKey []string, uniqueKeys []TableUniqueKey, relnameIDCol int, nameCol int, @@ -3592,7 +3673,7 @@ func buildCreateTableFromMoColumnsAt( if len(cols) == 0 { return "" } - return renderCreateTableDDLFull(tableName, cols, tableComment, partition, uniqueKeys...) + return renderCreateTableDDLFull(tableName, cols, tableComment, partition, primaryKey, uniqueKeys...) } func isPrintableSQLType(sqlType string) bool { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 9ba598cd785c8..b825c3b9261fe 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -46,20 +46,37 @@ func encodedDefault(t *testing.T, origin string, nullable bool) string { return string(data) } -func encodedIndexConstraint(t *testing.T, indexes ...*plan.IndexDef) string { +func encodedConstraint(t *testing.T, constraints ...engine.Constraint) string { t.Helper() def := &engine.ConstraintDef{ - Cts: []engine.Constraint{ - &engine.IndexDef{ - Indexes: indexes, - }, - }, + Cts: constraints, } data, err := def.MarshalBinary() require.NoError(t, err) return string(data) } +func encodedIndexConstraint(t *testing.T, indexes ...*plan.IndexDef) string { + t.Helper() + return encodedConstraint(t, &engine.IndexDef{Indexes: indexes}) +} + +func encodedPrimaryKeyConstraint(t *testing.T, names ...string) engine.Constraint { + t.Helper() + pkeyColName := "" + if len(names) == 1 { + pkeyColName = names[0] + } else if len(names) > 1 { + pkeyColName = catalog.CPrimaryKeyColName + } + return &engine.PrimaryKeyDef{ + Pkey: &plan.PrimaryKeyDef{ + PkeyColName: pkeyColName, + Names: names, + }, + } +} + // TestWriteCSV_empty tests writing an empty logical view to CSV. func TestWriteCSV_empty(t *testing.T) { schema := &TableSchema{ @@ -522,10 +539,13 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test { "obj1", "0", "0", "333999", "parent", "ckp_constraints", "333997", - "", "r", "parent table comment", "CREATE TABLE `ckp_constraints`.`parent` (`old` INT)", encodedIndexConstraint( + "", "r", "parent table comment", "CREATE TABLE `ckp_constraints`.`parent` (`old` INT)", encodedConstraint( t, - &plan.IndexDef{IndexName: "code", Parts: []string{"code"}, Unique: true}, - &plan.IndexDef{IndexName: "idx_parent_note", Parts: []string{"note"}}, + encodedPrimaryKeyConstraint(t, "id"), + &engine.IndexDef{Indexes: []*plan.IndexDef{ + {IndexName: "code", Parts: []string{"code"}, Unique: true}, + {IndexName: "idx_parent_note", Parts: []string{"note"}}, + }}, ), }, }, @@ -591,6 +611,61 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test assert.NotContains(t, ddl, "`old`") } +func TestCreateTableDDLFromCatalogViews_IgnoresMoColumnsPrimaryWithoutConstraint(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "relpersistence", "relkind", "rel_comment", "rel_createsql", "constraint", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "334100", "t_int_signed", "ckp_types", "334099", + "", "r", "", "", "", + }, + }, + } + + headers := append([]string{"object", "block", "row"}, catalog.MoColumnsSchema...) + row := func(name, typ, attnum, notNull, constraintType, seqnum string) []string { + data := make([]string, len(catalog.MoColumnsSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoColumnsSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_columns header %s", colName) + } + set(catalog.SystemColAttr_RelID, "334100") + set(catalog.SystemColAttr_RelName, "t_int_signed") + set(catalog.SystemColAttr_Name, name) + set(catalog.SystemColAttr_Type, typ) + set(catalog.SystemColAttr_Num, attnum) + set(catalog.SystemColAttr_NullAbility, notNull) + set(catalog.SystemColAttr_ConstraintType, constraintType) + set(catalog.SystemColAttr_IsHidden, "0") + set(catalog.SystemColAttr_Seqnum, seqnum) + return append([]string{"obj1", "0", attnum}, data...) + } + moColumnsView := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("id", encodedSQLType(t, types.T_int32.ToType()), "1", "1", "p", "0"), + row("c_tiny", encodedSQLType(t, types.T_int8.ToType()), "2", "0", "", "1"), + row("c_small", encodedSQLType(t, types.T_int16.ToType()), "3", "0", "", "2"), + row("c_int", encodedSQLType(t, types.T_int32.ToType()), "4", "0", "", "3"), + row("c_big", encodedSQLType(t, types.T_int64.ToType()), "5", "0", "", "4"), + }, + } + + ddl := createTableDDLFromCatalogViews(334100, moTablesView, moColumnsView) + assert.Contains(t, ddl, "`id` INT NOT NULL") + assert.NotContains(t, ddl, "PRIMARY KEY") +} + func TestRenderCreateTableDDLFromSchema_PrefersColumnsOverCreateSQL(t *testing.T) { ddl := RenderCreateTableDDLFromSchema(&TableSchema{ TableName: "t", From 46a53ae12c31808f9712edcdd343e5d6274561b8 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 15 Jun 2026 17:54:32 +0800 Subject: [PATCH 38/76] fix(ckp): avoid recursive catalog lookup --- pkg/tools/checkpointtool/table_dump.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 5963a399544a6..a12bd8218aee5 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -3262,10 +3262,11 @@ func (r *CheckpointReader) findCatalogTableID( snapshotTS types.TS, tableName string, ) (uint64, bool, error) { - tables, err := r.ListCatalogTables(ctx, snapshotTS, TableListOptions{IncludeViews: true}) + moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) if err != nil { - return 0, false, fmt.Errorf("list catalog tables: %w", err) + return 0, false, fmt.Errorf("read mo_tables: %w", err) } + tables := buildCatalogTablesFromMoTablesRows(moTablesView) for _, table := range tables { if table.TableName != tableName { continue From 59d4e24e193e7e92a17b93600d08261b615733e8 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Tue, 16 Jun 2026 10:07:09 +0800 Subject: [PATCH 39/76] fix(ckp): preserve vector index options --- pkg/tools/checkpointtool/table_dump.go | 73 ++++++++++++++++----- pkg/tools/checkpointtool/table_dump_test.go | 5 ++ 2 files changed, 60 insertions(+), 18 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index a12bd8218aee5..c1ff5e6c97cdd 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -117,9 +117,12 @@ type TableColumn struct { } type TableUniqueKey struct { - Name string - Columns []string - Unique bool + Name string + Columns []string + Unique bool + Algo string + AlgoParams string + Comment string } // TableSchema holds the decoded schema for one user table. @@ -476,6 +479,7 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri } sb.WriteString("KEY ") sb.WriteString(quoteDDLIdent(key.Name)) + appendIndexAlgorithmDDL(&sb, key.Algo) sb.WriteString("(") for i, col := range key.Columns { if i > 0 { @@ -483,10 +487,14 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri } sb.WriteString(quoteDDLIdent(col)) } + sb.WriteString(")") + if err := appendIndexTrailingOptionsDDL(&sb, key.AlgoParams, key.Comment); err != nil { + ckpDebugSchemaf("mo_tables constraint index options skipped index=%s algo=%q params=%q err=%v", key.Name, key.Algo, key.AlgoParams, err) + } if i < len(uniqueKeys)-1 { - sb.WriteString("),\n") + sb.WriteString(",\n") } else { - sb.WriteString(")\n") + sb.WriteString("\n") } } sb.WriteString(")") @@ -607,7 +615,14 @@ func normalizedUniqueKeys(keys []TableUniqueKey) []TableUniqueKey { continue } seen[signature] = struct{}{} - out = append(out, TableUniqueKey{Name: name, Columns: cols, Unique: key.Unique}) + out = append(out, TableUniqueKey{ + Name: name, + Columns: cols, + Unique: key.Unique, + Algo: strings.TrimSpace(key.Algo), + AlgoParams: strings.TrimSpace(key.AlgoParams), + Comment: strings.TrimSpace(key.Comment), + }) } sort.Slice(out, func(i, j int) bool { if out[i].Name != out[j].Name { @@ -1143,6 +1158,9 @@ func cloneTableSchema(schema *TableSchema) *TableSchema { for i, key := range schema.UniqueKeys { clone.UniqueKeys[i].Name = strings.Clone(key.Name) clone.UniqueKeys[i].Unique = key.Unique + clone.UniqueKeys[i].Algo = strings.Clone(key.Algo) + clone.UniqueKeys[i].AlgoParams = strings.Clone(key.AlgoParams) + clone.UniqueKeys[i].Comment = strings.Clone(key.Comment) if len(key.Columns) > 0 { clone.UniqueKeys[i].Columns = make([]string, len(key.Columns)) for j, col := range key.Columns { @@ -3464,10 +3482,7 @@ func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, e } sb.WriteString("KEY ") sb.WriteString(quoteDDLIdent(info.name)) - if !catalog.IsNullIndexAlgo(info.algo) { - sb.WriteString(" USING ") - sb.WriteString(info.algo) - } + appendIndexAlgorithmDDL(&sb, info.algo) sb.WriteString("(") for i, col := range cols { if i > 0 { @@ -3476,21 +3491,36 @@ func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, e sb.WriteString(quoteDDLIdent(col.name)) } sb.WriteString(")") - if strings.TrimSpace(info.algoParams) != "" { - params, err := catalog.IndexParamsToStringList(info.algoParams) + if err := appendIndexTrailingOptionsDDL(&sb, info.algoParams, info.comment); err != nil { + return "", err + } + sb.WriteString(";") + return sb.String(), nil +} + +func appendIndexAlgorithmDDL(sb *strings.Builder, algo string) { + algo = strings.TrimSpace(algo) + if !catalog.IsNullIndexAlgo(algo) { + sb.WriteString(" USING ") + sb.WriteString(algo) + } +} + +func appendIndexTrailingOptionsDDL(sb *strings.Builder, algoParams, comment string) error { + if strings.TrimSpace(algoParams) != "" { + params, err := catalog.IndexParamsToStringList(algoParams) if err != nil { - return "", err + return err } if strings.TrimSpace(params) != "" { sb.WriteString(params) } } - if info.comment != "" { + if comment != "" { sb.WriteString(" COMMENT ") - sb.WriteString(quoteDDLString(info.comment)) + sb.WriteString(quoteDDLString(comment)) } - sb.WriteString(";") - return sb.String(), nil + return nil } func decodeUniqueKeysFromMoTablesConstraint(raw string) (keys []TableUniqueKey) { @@ -3532,7 +3562,14 @@ func decodeUniqueKeysFromMoTablesConstraint(raw string) (keys []TableUniqueKey) if name == "" { name = cols[0] } - keys = append(keys, TableUniqueKey{Name: name, Columns: cols, Unique: index.Unique}) + keys = append(keys, TableUniqueKey{ + Name: name, + Columns: cols, + Unique: index.Unique, + Algo: index.IndexAlgo, + AlgoParams: index.IndexAlgoParams, + Comment: index.Comment, + }) } } keys = normalizedUniqueKeys(keys) diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index b825c3b9261fe..0816a58575a1b 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -545,6 +545,8 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test &engine.IndexDef{Indexes: []*plan.IndexDef{ {IndexName: "code", Parts: []string{"code"}, Unique: true}, {IndexName: "idx_parent_note", Parts: []string{"note"}}, + {IndexName: "ivf_500", Parts: []string{"embedding"}, IndexAlgo: "ivfflat", IndexAlgoParams: `{"lists":"500","op_type":"vector_l2_ops"}`}, + {IndexName: "ivf_2000", Parts: []string{"embedding"}, IndexAlgo: "ivfflat", IndexAlgoParams: `{"lists":"2000","op_type":"vector_l2_ops"}`}, }}, ), }, @@ -583,6 +585,7 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test row("id", encodedSQLType(t, types.T_int32.ToType()), "1", "1", "0", "", "p", "", "0"), row("code", encodedSQLType(t, types.New(types.T_varchar, 20, 0)), "2", "1", "0", "", "", "", "1"), row("note", encodedSQLType(t, types.New(types.T_varchar, 100, 0)), "3", "0", "1", encodedDefault(t, "'parent-default'", true), "", "parent note", "2"), + row("embedding", encodedSQLType(t, types.New(types.T_array_float32, 128, 0)), "4", "0", "0", "", "", "", "3"), }, } partitionMetadataView := &LogicalTableView{ @@ -606,6 +609,8 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test assert.Contains(t, ddl, "PRIMARY KEY (`id`)") assert.Contains(t, ddl, "UNIQUE KEY `code`(`code`)") assert.Contains(t, ddl, "KEY `idx_parent_note`(`note`)") + assert.Contains(t, ddl, "KEY `ivf_500` USING ivfflat(`embedding`) lists = 500 op_type 'vector_l2_ops'") + assert.Contains(t, ddl, "KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops'") assert.Contains(t, ddl, "COMMENT='parent table comment'") assert.Contains(t, ddl, "partition by key algorithm = 2 (`id`, `tenant_id`) partitions 4") assert.NotContains(t, ddl, "`old`") From c9058ecf4016087929357f6dca17c9f65c99fc44 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Tue, 16 Jun 2026 10:44:53 +0800 Subject: [PATCH 40/76] fix(ckp): align catalog headers by seqnum --- pkg/tools/checkpointtool/table_dump.go | 44 ++++++++++++++++++--- pkg/tools/checkpointtool/table_dump_test.go | 20 ++++++++++ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index c1ff5e6c97cdd..083cecd080416 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -3240,15 +3240,47 @@ func (r *CheckpointReader) dumpCatalogTableViewWithHeaders( if view == nil { return &LogicalTableView{}, nil } - fixedHeaders := append([]string(nil), logicalTableViewMetaHeaders...) - fixedHeaders = append(fixedHeaders, headers...) - if len(view.Headers) > len(fixedHeaders) { - for i := len(fixedHeaders); i < len(view.Headers); i++ { - fixedHeaders = append(fixedHeaders, fmt.Sprintf("col_%d", i-len(logicalTableViewMetaHeaders))) + applyCatalogHeadersBySeqNums(view, headers) + return view, nil +} + +func applyCatalogHeadersBySeqNums(view *LogicalTableView, headers []string) { + if view == nil { + return + } + dataOffset := logicalViewDataOffset(view) + if dataOffset == 0 && len(view.Headers) < logicalViewMetaCols { + dataOffset = logicalViewMetaCols + } + fixedHeaders := make([]string, len(view.Headers)) + if len(fixedHeaders) < dataOffset { + fixedHeaders = make([]string, dataOffset) + } + copy(fixedHeaders, logicalTableViewMetaHeaders) + for i := dataOffset; i < len(fixedHeaders); i++ { + fixedHeaders[i] = fmt.Sprintf("col_%d", i-dataOffset) + } + + mapped := 0 + for dataIdx, seqNum := range view.ColSeqNums { + headerIdx := dataOffset + dataIdx + if headerIdx >= len(fixedHeaders) || int(seqNum) >= len(headers) { + continue + } + fixedHeaders[headerIdx] = headers[seqNum] + mapped++ + } + if mapped == 0 { + for i, header := range headers { + headerIdx := dataOffset + i + if headerIdx >= len(fixedHeaders) { + fixedHeaders = append(fixedHeaders, header) + continue + } + fixedHeaders[headerIdx] = header } } view.Headers = fixedHeaders - return view, nil } func (r *CheckpointReader) dumpCatalogTableView( diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 0816a58575a1b..434140d714c1c 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -616,6 +616,26 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test assert.NotContains(t, ddl, "`old`") } +func TestBuildPartitionClauseFromMetadata_UsesSeqNumsWithHiddenColumn(t *testing.T) { + view := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, "col_0", "col_1", "col_2", "col_3", "col_4", "col_5", "col_6"), + ColSeqNums: []uint16{ + 6, // hidden/cpkey column before visible catalog columns + 0, 1, 2, 3, 4, 5, + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "hidden-key", + "334023", "t_hash_partition", "ckp_tables", "Hash", "hash (`id`)", "4", + }, + }, + } + applyCatalogHeadersBySeqNums(view, moPartitionMetadataHeaders) + + assert.Equal(t, "partition by hash (`id`) partitions 4", buildPartitionClauseFromMetadata(view, 334023)) +} + func TestCreateTableDDLFromCatalogViews_IgnoresMoColumnsPrimaryWithoutConstraint(t *testing.T) { moTablesView := &LogicalTableView{ Headers: []string{ From 0a349f63826d0ccdb51ddc1705300acdf58ce0ee Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Tue, 16 Jun 2026 10:56:50 +0800 Subject: [PATCH 41/76] fix(ckp): recover partition metadata by row shape --- pkg/tools/checkpointtool/table_dump.go | 81 +++++++++++++++++---- pkg/tools/checkpointtool/table_dump_test.go | 14 ++++ 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 083cecd080416..e1289258e7039 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -3453,34 +3453,87 @@ func buildPartitionClauseFromMetadata(view *LogicalTableView, tableID uint64) st descriptionCol, countCol, ) - return "" } tableIDStr := strconv.FormatUint(tableID, 10) + if clause := buildPartitionClauseFromMetadataAt(view, tableIDStr, tableIDCol, methodCol, descriptionCol, countCol); clause != "" { + return clause + } + if clause := buildPartitionClauseFromMetadataByRowShape(view, tableIDStr); clause != "" { + return clause + } + ckpDebugSchemaf("partition metadata not found table=%d rows=%d headers=%v", tableID, len(view.Rows), view.Headers) + return "" +} + +func buildPartitionClauseFromMetadataAt(view *LogicalTableView, tableIDStr string, tableIDCol, methodCol, descriptionCol, countCol int) string { + if tableIDCol < 0 || methodCol < 0 || descriptionCol < 0 { + return "" + } + dataOffset := logicalViewDataOffset(view) for _, row := range view.Rows { - if tableIDCol >= len(row) || row[tableIDCol] != tableIDStr { + if len(row) < dataOffset { continue } - if methodCol >= len(row) || descriptionCol >= len(row) { + dataRow := row[dataOffset:] + if tableIDCol >= len(dataRow) || dataRow[tableIDCol] != tableIDStr { continue } - method := strings.TrimSpace(row[methodCol]) - description := strings.TrimSpace(row[descriptionCol]) - if description == "" || !isPrintableSQLText(description) { - return "" + if methodCol >= len(dataRow) || descriptionCol >= len(dataRow) { + continue + } + return renderPartitionClauseFromMetadataCells(tableIDStr, strings.TrimSpace(dataRow[methodCol]), strings.TrimSpace(dataRow[descriptionCol]), cellAt(dataRow, countCol)) + } + return "" +} + +func buildPartitionClauseFromMetadataByRowShape(view *LogicalTableView, tableIDStr string) string { + dataOffset := logicalViewDataOffset(view) + for _, row := range view.Rows { + if len(row) < dataOffset { + continue } - clause := "partition by " + description - if isAutoPartitionCountMethod(method) && countCol >= 0 && countCol < len(row) { - if count, err := strconv.Atoi(strings.TrimSpace(row[countCol])); err == nil && count > 0 { - clause += fmt.Sprintf(" partitions %d", count) + dataRow := row[dataOffset:] + for tableIDCol, cell := range dataRow { + if cell != tableIDStr { + continue + } + methodCol := tableIDCol + 3 + descriptionCol := tableIDCol + 4 + countCol := tableIDCol + 5 + if descriptionCol >= len(dataRow) { + continue + } + clause := renderPartitionClauseFromMetadataCells(tableIDStr, strings.TrimSpace(cellAt(dataRow, methodCol)), strings.TrimSpace(cellAt(dataRow, descriptionCol)), cellAt(dataRow, countCol)) + if clause != "" { + ckpDebugSchemaf("partition metadata resolved by row shape table=%s table_id_col=%d clause=%q", tableIDStr, tableIDCol, clause) + return clause } } - ckpDebugSchemaf("partition metadata resolved table=%d method=%q description=%q clause=%q", tableID, method, description, clause) - return clause } - ckpDebugSchemaf("partition metadata not found table=%d rows=%d", tableID, len(view.Rows)) return "" } +func renderPartitionClauseFromMetadataCells(tableIDStr, method, description, countText string) string { + if description == "" || !isPrintableSQLText(description) { + return "" + } + clause := "partition by " + description + if isAutoPartitionCountMethod(method) { + if count, err := strconv.Atoi(strings.TrimSpace(countText)); err == nil && count > 0 { + clause += fmt.Sprintf(" partitions %d", count) + } + } + ckpDebugSchemaf("partition metadata resolved table=%s method=%q description=%q clause=%q", tableIDStr, method, description, clause) + return clause +} + +func cellAt(row []string, idx int) string { + if idx < 0 || idx >= len(row) { + return "" + } + return row[idx] +} + func isAutoPartitionCountMethod(method string) bool { switch strings.ToLower(strings.TrimSpace(method)) { case "key", "linearkey", "hash", "linearhash": diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 434140d714c1c..9fee2f2e699e3 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -636,6 +636,20 @@ func TestBuildPartitionClauseFromMetadata_UsesSeqNumsWithHiddenColumn(t *testing assert.Equal(t, "partition by hash (`id`) partitions 4", buildPartitionClauseFromMetadata(view, 334023)) } +func TestBuildPartitionClauseFromMetadata_FallsBackToRowShape(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3", "col_4", "col_5"}, + Rows: [][]string{ + { + "obj1", "0", "0", + "272577", "t_hash_partition", "ckp_tables", "Hash", "hash (`id`)", "4", + }, + }, + } + + assert.Equal(t, "partition by hash (`id`) partitions 4", buildPartitionClauseFromMetadata(view, 272577)) +} + func TestCreateTableDDLFromCatalogViews_IgnoresMoColumnsPrimaryWithoutConstraint(t *testing.T) { moTablesView := &LogicalTableView{ Headers: []string{ From 84c1e7cc703e8f31caf689b9fa09103648329cc0 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Tue, 16 Jun 2026 16:45:19 +0800 Subject: [PATCH 42/76] fix(ckp): dump partition table data --- pkg/tools/checkpointtool/table_dump.go | 248 ++++++++++++++++++-- pkg/tools/checkpointtool/table_dump_test.go | 27 +++ 2 files changed, 258 insertions(+), 17 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index e1289258e7039..b96b28896677e 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -205,6 +205,16 @@ var moPartitionMetadataHeaders = []string{ "partition_count", } +var moPartitionTablesHeaders = []string{ + "partition_id", + "partition_table_name", + "primary_table_id", + "partition_name", + "partition_ordinal_position", + "partition_expression_str", + "partition_expression", +} + type indexDDLColumn struct { name string ordinal int @@ -971,23 +981,11 @@ func (r *CheckpointReader) DumpTableCSVComposed( snapshotTS types.TS, opts ...CSVExportOption, ) error { - dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) + data, err := r.PrepareTableDumpData(ctx, tableID, snapshotTS) if err != nil { - if !isTableDataUnavailable(err) { - return err - } - dataEntries = nil - tombEntries = nil - } - schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) - if len(schema.Columns) == 0 { - return moerr.NewInternalErrorf( - ctx, - "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", - tableID, - ) + return err } - return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries, resolveCSVExportOptions(opts)) + return r.DumpPreparedTableCSV(ctx, w, data, snapshotTS, opts...) } // PrepareTableDumpData resolves the table schema and object lists once so batch @@ -1013,6 +1011,25 @@ func (r *CheckpointReader) PrepareTableDumpData( tableID, ) } + partitionTableIDs, err := r.readPartitionTableIDs(ctx, snapshotTS, tableID) + if err != nil { + return nil, err + } + for _, partitionTableID := range partitionTableIDs { + partitionData, partitionTomb, err := r.getTableEntriesAt(ctx, partitionTableID, snapshotTS) + if err != nil { + if !isTableDataUnavailable(err) { + return nil, err + } + ckpDebugSchemaf("partition table data unavailable primary=%d partition=%d err=%v", tableID, partitionTableID, err) + continue + } + dataEntries = append(dataEntries, partitionData...) + tombEntries = append(tombEntries, partitionTomb...) + } + if len(partitionTableIDs) > 0 { + ckpDebugSchemaf("partition table dump data resolved primary=%d partitions=%v data_entries=%d tomb_entries=%d", tableID, partitionTableIDs, len(dataEntries), len(tombEntries)) + } return &TableDumpData{ TableID: tableID, Schema: cloneTableSchema(schema), @@ -1036,6 +1053,8 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( tableSet := make(map[uint64]struct{}, len(tableIDs)) result := make(map[uint64]*TableDumpData, len(tableIDs)) + partitionToPrimary := make(map[uint64]uint64) + partitionIDsByPrimary := make(map[uint64][]uint64) for _, tableID := range tableIDs { if _, ok := tableSet[tableID]; ok { continue @@ -1054,6 +1073,17 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( Schema: cloneTableSchema(schema), } } + partitionTableMap, err := r.readPartitionTableIDsForPrimaries(ctx, snapshotTS, tableSet) + if err != nil { + return nil, err + } + for primaryID, partitionIDs := range partitionTableMap { + for _, partitionID := range partitionIDs { + tableSet[partitionID] = struct{}{} + partitionToPrimary[partitionID] = primaryID + } + partitionIDsByPrimary[primaryID] = append(partitionIDsByPrimary[primaryID], partitionIDs...) + } entryRefs := make([]*EntryInfo, 0, len(composed.Incrementals)+1) if composed.BaseEntry != nil { @@ -1071,10 +1101,24 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( return nil, err } for tableID, entries := range dataByTable { - result[tableID].DataEntries = append(result[tableID].DataEntries, entries...) + targetID := tableID + if primaryID, ok := partitionToPrimary[tableID]; ok { + targetID = primaryID + } + if result[targetID] == nil { + continue + } + result[targetID].DataEntries = append(result[targetID].DataEntries, entries...) } for tableID, entries := range tombByTable { - result[tableID].TombEntries = append(result[tableID].TombEntries, entries...) + targetID := tableID + if primaryID, ok := partitionToPrimary[tableID]; ok { + targetID = primaryID + } + if result[targetID] == nil { + continue + } + result[targetID].TombEntries = append(result[targetID].TombEntries, entries...) } } @@ -1082,6 +1126,9 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( if data.Schema == nil || len(data.Schema.Columns) == 0 { return nil, moerr.NewInternalErrorf(ctx, "cannot resolve visible columns for table %d from checkpoint metadata", tableID) } + if partitionIDs := partitionIDsByPrimary[tableID]; len(partitionIDs) > 0 { + ckpDebugSchemaf("partition table dump data resolved primary=%d partitions=%v data_entries=%d tomb_entries=%d", tableID, partitionIDs, len(data.DataEntries), len(data.TombEntries)) + } } return result, nil } @@ -3215,6 +3262,48 @@ func (r *CheckpointReader) getPartitionMetadataView( return r.dumpCatalogTableViewWithHeaders(ctx, partitionMetadataTableID, snapshotTS, moPartitionMetadataHeaders) } +func (r *CheckpointReader) getPartitionTablesView( + ctx context.Context, + snapshotTS types.TS, +) (*LogicalTableView, error) { + partitionTablesID, ok, err := r.findCatalogTableID(ctx, snapshotTS, catalog.MOPartitionTables) + if err != nil { + return nil, err + } + if !ok { + return nil, nil + } + return r.dumpCatalogTableViewWithHeaders(ctx, partitionTablesID, snapshotTS, moPartitionTablesHeaders) +} + +func (r *CheckpointReader) readPartitionTableIDs( + ctx context.Context, + snapshotTS types.TS, + primaryTableID uint64, +) ([]uint64, error) { + view, err := r.getPartitionTablesView(ctx, snapshotTS) + if err != nil || view == nil { + return nil, err + } + primarySet := map[uint64]struct{}{primaryTableID: {}} + return buildPartitionTableIDMap(view, primarySet)[primaryTableID], nil +} + +func (r *CheckpointReader) readPartitionTableIDsForPrimaries( + ctx context.Context, + snapshotTS types.TS, + primaryTableIDs map[uint64]struct{}, +) (map[uint64][]uint64, error) { + if len(primaryTableIDs) == 0 { + return nil, nil + } + view, err := r.getPartitionTablesView(ctx, snapshotTS) + if err != nil || view == nil { + return nil, err + } + return buildPartitionTableIDMap(view, primaryTableIDs), nil +} + func (r *CheckpointReader) readPartitionClause( ctx context.Context, tableID uint64, @@ -3465,6 +3554,118 @@ func buildPartitionClauseFromMetadata(view *LogicalTableView, tableID uint64) st return "" } +type partitionTableRef struct { + id uint64 + ordinal int + name string +} + +func buildPartitionTableIDMap(view *LogicalTableView, primaryTableIDs map[uint64]struct{}) map[uint64][]uint64 { + result := make(map[uint64][]uint64) + if view == nil || len(primaryTableIDs) == 0 { + return result + } + partitionIDCol := view.columnDataIndex("partition_id") + partitionNameCol := view.columnDataIndex("partition_table_name") + primaryIDCol := view.columnDataIndex("primary_table_id") + ordinalCol := view.columnDataIndex("partition_ordinal_position") + byPrimary := make(map[uint64][]partitionTableRef) + if partitionIDCol >= 0 && primaryIDCol >= 0 { + addPartitionTableRefsAt(view, primaryTableIDs, byPrimary, partitionIDCol, primaryIDCol, partitionNameCol, ordinalCol) + } + if len(byPrimary) == 0 { + addPartitionTableRefsByRowShape(view, primaryTableIDs, byPrimary) + } + for primaryID, refs := range byPrimary { + sort.Slice(refs, func(i, j int) bool { + if refs[i].ordinal != refs[j].ordinal { + return refs[i].ordinal < refs[j].ordinal + } + if refs[i].name != refs[j].name { + return refs[i].name < refs[j].name + } + return refs[i].id < refs[j].id + }) + ids := make([]uint64, 0, len(refs)) + seen := make(map[uint64]struct{}, len(refs)) + for _, ref := range refs { + if ref.id == 0 { + continue + } + if _, ok := seen[ref.id]; ok { + continue + } + seen[ref.id] = struct{}{} + ids = append(ids, ref.id) + } + if len(ids) > 0 { + result[primaryID] = ids + ckpDebugSchemaf("partition table ids resolved primary=%d partitions=%v", primaryID, ids) + } + } + return result +} + +func addPartitionTableRefsAt(view *LogicalTableView, primaryTableIDs map[uint64]struct{}, out map[uint64][]partitionTableRef, partitionIDCol, primaryIDCol, partitionNameCol, ordinalCol int) { + dataOffset := logicalViewDataOffset(view) + for _, row := range view.Rows { + if len(row) < dataOffset { + continue + } + dataRow := row[dataOffset:] + if partitionIDCol >= len(dataRow) || primaryIDCol >= len(dataRow) { + continue + } + partitionID, ok := parseUintCell(dataRow[partitionIDCol]) + if !ok { + continue + } + primaryID, ok := parseUintCell(dataRow[primaryIDCol]) + if !ok { + continue + } + if _, wanted := primaryTableIDs[primaryID]; !wanted { + continue + } + out[primaryID] = append(out[primaryID], partitionTableRef{ + id: partitionID, + ordinal: parseIntCellDefault(cellAt(dataRow, ordinalCol), len(out[primaryID])), + name: cellAt(dataRow, partitionNameCol), + }) + } +} + +func addPartitionTableRefsByRowShape(view *LogicalTableView, primaryTableIDs map[uint64]struct{}, out map[uint64][]partitionTableRef) { + dataOffset := logicalViewDataOffset(view) + for _, row := range view.Rows { + if len(row) < dataOffset { + continue + } + dataRow := row[dataOffset:] + for primaryIDCol, cell := range dataRow { + primaryID, ok := parseUintCell(cell) + if !ok { + continue + } + if _, wanted := primaryTableIDs[primaryID]; !wanted { + continue + } + partitionIDCol := primaryIDCol - 2 + partitionNameCol := primaryIDCol - 1 + ordinalCol := primaryIDCol + 2 + partitionID, ok := parseUintCell(cellAt(dataRow, partitionIDCol)) + if !ok { + continue + } + out[primaryID] = append(out[primaryID], partitionTableRef{ + id: partitionID, + ordinal: parseIntCellDefault(cellAt(dataRow, ordinalCol), len(out[primaryID])), + name: cellAt(dataRow, partitionNameCol), + }) + } + } +} + func buildPartitionClauseFromMetadataAt(view *LogicalTableView, tableIDStr string, tableIDCol, methodCol, descriptionCol, countCol int) string { if tableIDCol < 0 || methodCol < 0 || descriptionCol < 0 { return "" @@ -3534,6 +3735,19 @@ func cellAt(row []string, idx int) string { return row[idx] } +func parseUintCell(cell string) (uint64, bool) { + value, err := strconv.ParseUint(strings.TrimSpace(cell), 10, 64) + return value, err == nil && value != 0 +} + +func parseIntCellDefault(cell string, fallback int) int { + value, err := strconv.Atoi(strings.TrimSpace(cell)) + if err != nil { + return fallback + } + return value +} + func isAutoPartitionCountMethod(method string) bool { switch strings.ToLower(strings.TrimSpace(method)) { case "key", "linearkey", "hash", "linearhash": diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 9fee2f2e699e3..ad468cd647175 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -650,6 +650,33 @@ func TestBuildPartitionClauseFromMetadata_FallsBackToRowShape(t *testing.T) { assert.Equal(t, "partition by hash (`id`) partitions 4", buildPartitionClauseFromMetadata(view, 272577)) } +func TestBuildPartitionTableIDMap(t *testing.T) { + view := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionTablesHeaders...), + Rows: [][]string{ + {"obj1", "0", "0", "272578", "%!%p0%!%t_hash_partition", "272577", "p0", "0", "", ""}, + {"obj1", "0", "1", "272579", "%!%p1%!%t_hash_partition", "272577", "p1", "1", "", ""}, + {"obj1", "0", "2", "272580", "%!%p0%!%other", "999999", "p0", "0", "", ""}, + }, + } + + got := buildPartitionTableIDMap(view, map[uint64]struct{}{272577: {}}) + assert.Equal(t, []uint64{272578, 272579}, got[272577]) +} + +func TestBuildPartitionTableIDMap_FallsBackToRowShape(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3", "col_4", "col_5"}, + Rows: [][]string{ + {"obj1", "0", "1", "334040", "%!%p1%!%t_hash_partition", "334039", "p1", "1", "", ""}, + {"obj1", "0", "0", "334041", "%!%p0%!%t_hash_partition", "334039", "p0", "0", "", ""}, + }, + } + + got := buildPartitionTableIDMap(view, map[uint64]struct{}{334039: {}}) + assert.Equal(t, []uint64{334041, 334040}, got[334039]) +} + func TestCreateTableDDLFromCatalogViews_IgnoresMoColumnsPrimaryWithoutConstraint(t *testing.T) { moTablesView := &LogicalTableView{ Headers: []string{ From 7ff851767631b24ae963d3440bb4fc2bd1eb2d72 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 09:48:56 +0800 Subject: [PATCH 43/76] fix ckp list external table visibility --- cmd/mo-object-tool/ckp/checkpoint.go | 4 ++-- pkg/tools/checkpointtool/table_dump.go | 11 ++++++++++- pkg/tools/checkpointtool/table_dump_test.go | 11 +++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 35e95a072ef81..71faecd3770d5 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -195,7 +195,7 @@ func listCommand(storage *toolfs.StorageOptions) *cobra.Command { Short: "List checkpoint catalog tables", Long: `List table metadata from the checkpoint catalog. -By default this lists ordinary tables. Use --include-views to include views, +By default this lists tables and external tables. Use --include-views to include views, and use --database-id or --account-id to narrow the result.`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -246,7 +246,7 @@ and use --database-id or --account-id to narrow the result.`, } cmd.Flags().Uint32Var(&accountID, "account-id", 0, "Account ID to list") cmd.Flags().Uint64Var(&databaseID, "database-id", 0, "Database ID to list") - cmd.Flags().BoolVar(&includeViews, "include-views", false, "Include views and non-table relations") + cmd.Flags().BoolVar(&includeViews, "include-views", false, "Include views") cmd.Flags().StringVar(&listType, "type", "tables", "List type: tables, databases, or accounts") cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp: physical:logical, physical-logical, RFC3339, or local time (default: latest)") return cmd diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index b96b28896677e..0b2b257981362 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -1247,7 +1247,7 @@ func (r *CheckpointReader) ListCatalogTables( if opts.DatabaseID != nil && table.DatabaseID != *opts.DatabaseID { continue } - if !opts.IncludeViews && table.RelKind != "" && table.RelKind != "r" { + if !opts.IncludeViews && isViewRelKind(table.RelKind) { continue } filtered = append(filtered, table) @@ -2926,6 +2926,15 @@ func validRelKind(s string) bool { } } +func isViewRelKind(s string) bool { + switch strings.ToLower(strings.TrimSpace(s)) { + case "v", "view": + return true + default: + return false + } +} + // buildColumnsFromMoColumnsRows builds a sorted column list from mo_columns data rows // filtered for a specific tableID and excluding hidden columns. // Uses column name lookup from view headers to handle hidden column offsets. diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index ad468cd647175..fb99d2c27099b 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -794,6 +794,17 @@ func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testin assert.Equal(t, "v", tables[1].RelKind) } +func TestIsViewRelKind(t *testing.T) { + assert.False(t, isViewRelKind("")) + assert.False(t, isViewRelKind("r")) + assert.False(t, isViewRelKind("e")) + assert.False(t, isViewRelKind("external")) + assert.False(t, isViewRelKind("cluster")) + assert.True(t, isViewRelKind("v")) + assert.True(t, isViewRelKind("view")) + assert.True(t, isViewRelKind(" VIEW ")) +} + func TestInferCatalogLayout(t *testing.T) { tests := []struct { name string From 10a1f1d316218be6a6a9e70f28742f7cf4194804 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 09:52:07 +0800 Subject: [PATCH 44/76] include views in ckp list output --- cmd/mo-object-tool/ckp/checkpoint.go | 6 +++--- pkg/tools/checkpointtool/table_dump.go | 18 +++++----------- pkg/tools/checkpointtool/table_dump_test.go | 24 +++++++++++++-------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 71faecd3770d5..bd30fc6dcc16b 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -195,8 +195,8 @@ func listCommand(storage *toolfs.StorageOptions) *cobra.Command { Short: "List checkpoint catalog tables", Long: `List table metadata from the checkpoint catalog. -By default this lists tables and external tables. Use --include-views to include views, -and use --database-id or --account-id to narrow the result.`, +By default this lists all catalog relations. Use --database-id or --account-id +to narrow the result.`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { dir := "." @@ -246,7 +246,7 @@ and use --database-id or --account-id to narrow the result.`, } cmd.Flags().Uint32Var(&accountID, "account-id", 0, "Account ID to list") cmd.Flags().Uint64Var(&databaseID, "database-id", 0, "Database ID to list") - cmd.Flags().BoolVar(&includeViews, "include-views", false, "Include views") + cmd.Flags().BoolVar(&includeViews, "include-views", false, "Deprecated no-op; views are listed by default") cmd.Flags().StringVar(&listType, "type", "tables", "List type: tables, databases, or accounts") cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp: physical:logical, physical-logical, RFC3339, or local time (default: latest)") return cmd diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 0b2b257981362..dad88f5cbea98 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -1239,6 +1239,10 @@ func (r *CheckpointReader) ListCatalogTables( tables = mergeCatalogTableEntries(tables, buildCatalogTablesFromMoTablesRows(projected)) } } + return filterCatalogTablesForList(tables, opts), nil +} + +func filterCatalogTablesForList(tables []TableCatalogEntry, opts TableListOptions) []TableCatalogEntry { filtered := make([]TableCatalogEntry, 0, len(tables)) for _, table := range tables { if opts.AccountID != nil && table.AccountID != *opts.AccountID { @@ -1247,9 +1251,6 @@ func (r *CheckpointReader) ListCatalogTables( if opts.DatabaseID != nil && table.DatabaseID != *opts.DatabaseID { continue } - if !opts.IncludeViews && isViewRelKind(table.RelKind) { - continue - } filtered = append(filtered, table) } sort.Slice(filtered, func(i, j int) bool { @@ -1264,7 +1265,7 @@ func (r *CheckpointReader) ListCatalogTables( } return filtered[i].TableID < filtered[j].TableID }) - return filtered, nil + return filtered } func (r *CheckpointReader) listCatalogTablesFromProjectedMoTables( @@ -2926,15 +2927,6 @@ func validRelKind(s string) bool { } } -func isViewRelKind(s string) bool { - switch strings.ToLower(strings.TrimSpace(s)) { - case "v", "view": - return true - default: - return false - } -} - // buildColumnsFromMoColumnsRows builds a sorted column list from mo_columns data rows // filtered for a specific tableID and excluding hidden columns. // Uses column name lookup from view headers to handle hidden column offsets. diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index fb99d2c27099b..22c717908e1f4 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -794,15 +794,21 @@ func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testin assert.Equal(t, "v", tables[1].RelKind) } -func TestIsViewRelKind(t *testing.T) { - assert.False(t, isViewRelKind("")) - assert.False(t, isViewRelKind("r")) - assert.False(t, isViewRelKind("e")) - assert.False(t, isViewRelKind("external")) - assert.False(t, isViewRelKind("cluster")) - assert.True(t, isViewRelKind("v")) - assert.True(t, isViewRelKind("view")) - assert.True(t, isViewRelKind(" VIEW ")) +func TestListCatalogTablesDoesNotFilterRelKinds(t *testing.T) { + tables := []TableCatalogEntry{ + {TableID: 1, TableName: "orders", DatabaseName: "db1", RelKind: "r"}, + {TableID: 2, TableName: "ext_orders", DatabaseName: "db1", RelKind: "e"}, + {TableID: 3, TableName: "orders_v", DatabaseName: "db1", RelKind: "v"}, + {TableID: 4, TableName: "cluster_orders", DatabaseName: "db1", RelKind: "cluster"}, + } + filtered := filterCatalogTablesForList(tables, TableListOptions{}) + + require.Len(t, filtered, 4) + names := make([]string, 0, len(filtered)) + for _, table := range filtered { + names = append(names, table.TableName) + } + assert.ElementsMatch(t, []string{"orders", "ext_orders", "orders_v", "cluster_orders"}, names) } func TestInferCatalogLayout(t *testing.T) { From 54352d39d242c4c6eb7334a1ae03ee927dcafe36 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 10:05:02 +0800 Subject: [PATCH 45/76] ignore missing optional catalog tables in ckp dump --- pkg/tools/checkpointtool/table_dump.go | 10 ++++++++++ pkg/tools/checkpointtool/table_dump_test.go | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index dad88f5cbea98..7bc7b4bd0fde3 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -3325,6 +3325,9 @@ func (r *CheckpointReader) dumpCatalogTableViewWithHeaders( ) (*LogicalTableView, error) { view, err := r.getTableLogicalView(ctx, tableID, snapshotTS) if err != nil { + if isMissingCheckpointTableError(err, tableID) { + return &LogicalTableView{}, nil + } return nil, err } if view == nil { @@ -3334,6 +3337,13 @@ func (r *CheckpointReader) dumpCatalogTableViewWithHeaders( return view, nil } +func isMissingCheckpointTableError(err error, tableID uint64) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), fmt.Sprintf("table %d not found in checkpoint", tableID)) +} + func applyCatalogHeadersBySeqNums(view *LogicalTableView, headers []string) { if view == nil { return diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 22c717908e1f4..d343235001dca 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -811,6 +812,13 @@ func TestListCatalogTablesDoesNotFilterRelKinds(t *testing.T) { assert.ElementsMatch(t, []string{"orders", "ext_orders", "orders_v", "cluster_orders"}, names) } +func TestIsMissingCheckpointTableError(t *testing.T) { + err := moerr.NewInternalErrorf(context.Background(), "table %d not found in checkpoint at ts %s", uint64(272419), "1-0") + assert.True(t, isMissingCheckpointTableError(err, 272419)) + assert.False(t, isMissingCheckpointTableError(err, 272420)) + assert.False(t, isMissingCheckpointTableError(nil, 272419)) +} + func TestInferCatalogLayout(t *testing.T) { tests := []struct { name string From ee0e253657f08abd653c45cc99c3dd55a8090e39 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 10:19:24 +0800 Subject: [PATCH 46/76] include initial incremental checkpoint in ckp view --- pkg/tools/checkpointtool/checkpoint_reader.go | 12 +++++++++++- pkg/tools/checkpointtool/checkpoint_reader_test.go | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index 0c0f7789a2e2c..93026dc5f4a5f 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -387,7 +387,7 @@ func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { for i, e := range r.entries { start := e.GetStart() end := e.GetEnd() - if e.IsIncremental() && start.GT(&baseEnd) && end.LE(&ts) { + if e.IsIncremental() && shouldIncludeIncrementalCheckpoint(start, end, baseEnd, ts, baseEntry != nil) { tables, err := r.GetTables(e) if err != nil { if isDataFileNotFound(err) { @@ -409,6 +409,16 @@ func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { return view, nil } +func shouldIncludeIncrementalCheckpoint(start, end, baseEnd, ts types.TS, hasBase bool) bool { + if !end.LE(&ts) { + return false + } + if hasBase { + return start.GT(&baseEnd) + } + return start.GE(&baseEnd) +} + // ValidateSnapshot checks that ts can compose a catalog-backed checkpoint view. func (r *CheckpointReader) ValidateSnapshot(ctx context.Context, ts types.TS) error { view, err := r.ComposeAt(ts) diff --git a/pkg/tools/checkpointtool/checkpoint_reader_test.go b/pkg/tools/checkpointtool/checkpoint_reader_test.go index 25e5efc9cd003..8686f6f4a1c61 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader_test.go +++ b/pkg/tools/checkpointtool/checkpoint_reader_test.go @@ -21,6 +21,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -74,3 +75,16 @@ func TestVecValueToString_SQLLikeFormatting(t *testing.T) { require.NoError(t, vector.AppendFixed(nullVec, int64(0), true, mp)) require.Equal(t, "NULL", vecValueToString(nullVec, 0)) } + +func TestShouldIncludeIncrementalCheckpointWithoutBase(t *testing.T) { + zero := types.TS{} + ts1 := types.BuildTS(1, 0) + ts2 := types.BuildTS(2, 0) + + assert.True(t, shouldIncludeIncrementalCheckpoint(zero, ts1, zero, ts1, false)) + assert.True(t, shouldIncludeIncrementalCheckpoint(ts1, ts2, zero, ts2, false)) + assert.False(t, shouldIncludeIncrementalCheckpoint(ts1, ts2, zero, ts1, false)) + + assert.False(t, shouldIncludeIncrementalCheckpoint(zero, ts1, zero, ts1, true)) + assert.True(t, shouldIncludeIncrementalCheckpoint(ts1, ts2, zero, ts2, true)) +} From 6909cf4211238feb93e0f44301fac6a4cff96df4 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 10:33:22 +0800 Subject: [PATCH 47/76] use absolute local paths in ckp restore scripts --- cmd/mo-object-tool/ckp/checkpoint.go | 3 +++ cmd/mo-object-tool/ckp/checkpoint_test.go | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index bd30fc6dcc16b..50550a0572aab 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -1393,6 +1393,9 @@ func newLoadDataPathResolver(storage toolfs.StorageOptions) (loadDataPathResolve func (r loadDataPathResolver) loadDataSource(outputDir string, table checkpointtool.TableCatalogEntry) string { csvPath := tableCSVPath(outputDir, table) if r.s3Args.Bucket == "" { + if absPath, err := filepath.Abs(csvPath); err == nil { + csvPath = absPath + } return "LOAD DATA INFILE " + quoteSQLString(csvPath) } diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 0fce6edc90801..4f5f9938bc2d8 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -15,6 +15,8 @@ package ckp import ( + "path/filepath" + "strings" "testing" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" @@ -205,10 +207,10 @@ func TestLoadDataPathResolverLocal(t *testing.T) { resolver, err := newLoadDataPathResolver(toolfs.StorageOptions{}) require.NoError(t, err) - assert.Equal(t, - "LOAD DATA INFILE 'bmsql_config/account_0/db_272577/bmsql_config_272578.csv'", - resolver.loadDataSource("bmsql_config", table), - ) + loadSource := resolver.loadDataSource("bmsql_config", table) + assert.True(t, strings.HasPrefix(loadSource, "LOAD DATA INFILE '")) + assert.Contains(t, loadSource, "/bmsql_config/account_0/db_272577/bmsql_config_272578.csv'") + assert.True(t, filepath.IsAbs(strings.TrimSuffix(strings.TrimPrefix(loadSource, "LOAD DATA INFILE '"), "'"))) } func TestLoadDataPathResolverS3(t *testing.T) { From 243b0a21be92a4cdfa9c14472689402a38ac6c21 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 10:47:34 +0800 Subject: [PATCH 48/76] restore temp table display names in ckp tool --- pkg/tools/checkpointtool/table_dump.go | 20 ++++++++++++++ pkg/tools/checkpointtool/table_dump_test.go | 29 +++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 7bc7b4bd0fde3..cfd2feafdf9cb 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -35,6 +35,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/tools/objecttool" @@ -2895,12 +2896,31 @@ func buildCatalogTablesFromMoTablesRowsAt( entry.AccountID = uint32(accountID) } } + entry.TableName = displayTableName(entry.DatabaseName, entry.TableName) seen[tableID] = struct{}{} tables = append(tables, entry) } return tables } +func displayTableName(databaseName, tableName string) string { + if !defines.IsTempTableName(tableName) { + return tableName + } + prefix := defines.TempTableNamePrefix + rest := strings.TrimPrefix(tableName, prefix) + sessionEnd := strings.IndexByte(rest, '_') + if sessionEnd <= 0 || sessionEnd+1 >= len(rest) { + return tableName + } + afterSession := rest[sessionEnd+1:] + dbPrefix := databaseName + "_" + if strings.HasPrefix(afterSession, dbPrefix) && len(afterSession) > len(dbPrefix) { + return afterSession[len(dbPrefix):] + } + return tableName +} + func validCatalogName(s string) bool { if s == "" { return false diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index d343235001dca..f2fd440a3bb58 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -795,6 +795,35 @@ func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testin assert.Equal(t, "v", tables[1].RelKind) } +func TestBuildCatalogTablesFromMoTablesRows_UsesTemporaryDisplayName(t *testing.T) { + headers := []string{"object", "block", "row"} + for i := 0; i < len(preCPKLayout.moTablesSchema); i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + data := make([]string, len(preCPKLayout.moTablesSchema)) + data[0] = "1003" + data[1] = "__mo_tmp_123e4567e89b12d3a456426614174000_ckp_temp_t_session_only" + data[2] = "ckp_temp" + data[3] = "9002" + data[5] = "r" + data[11] = "7" + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{append([]string{"obj1", "0", "0"}, data...)}, + } + + tables := buildCatalogTablesFromMoTablesRows(view) + require.Len(t, tables, 1) + assert.Equal(t, "t_session_only", tables[0].TableName) +} + +func TestDisplayTableName(t *testing.T) { + assert.Equal(t, "orders", displayTableName("db1", "orders")) + assert.Equal(t, "t1", displayTableName("db1", "__mo_tmp_123e4567e89b12d3a456426614174000_db1_t1")) + assert.Equal(t, "__mo_tmp_123e4567e89b12d3a456426614174000_other_t1", displayTableName("db1", "__mo_tmp_123e4567e89b12d3a456426614174000_other_t1")) + assert.Equal(t, "__mo_tmp_bad", displayTableName("db1", "__mo_tmp_bad")) +} + func TestListCatalogTablesDoesNotFilterRelKinds(t *testing.T) { tables := []TableCatalogEntry{ {TableID: 1, TableName: "orders", DatabaseName: "db1", RelKind: "r"}, From 6682230156010ec331a200f00948d562a15746ce Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 10:58:30 +0800 Subject: [PATCH 49/76] fix ckp restore ddl for enum time and fulltext --- pkg/tools/checkpointtool/table_dump.go | 82 +++++++++++++++++++-- pkg/tools/checkpointtool/table_dump_test.go | 45 +++++++++++ 2 files changed, 120 insertions(+), 7 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index cfd2feafdf9cb..bf8b579458943 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -452,10 +452,10 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri for i, col := range cols { sb.WriteString(" ") sb.WriteString(quoteDDLIdent(col.Name)) - if col.SQLType != "" { + if sqlType := renderColumnSQLType(col); sqlType != "" { sb.WriteString(" ") - sb.WriteString(col.SQLType) - if col.Unsigned && !strings.Contains(strings.ToUpper(col.SQLType), "UNSIGNED") { + sb.WriteString(sqlType) + if col.Unsigned && !strings.Contains(strings.ToUpper(sqlType), "UNSIGNED") { sb.WriteString(" UNSIGNED") } } @@ -531,6 +531,38 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri return sb.String() } +func renderColumnSQLType(col TableColumn) string { + sqlType := strings.TrimSpace(col.SQLType) + enumValues := strings.TrimSpace(col.EnumValues) + if enumValues == "" { + return sqlType + } + upperType := strings.ToUpper(sqlType) + switch { + case upperType == "ENUM": + return renderEnumSetSQLType("ENUM", enumValues) + case upperType == "SET": + return renderEnumSetSQLType("SET", enumValues) + default: + return sqlType + } +} + +func renderEnumSetSQLType(kind string, enumValues string) string { + values := strings.TrimSpace(enumValues) + if values == "" { + return kind + } + upper := strings.ToUpper(values) + if strings.HasPrefix(upper, kind+"(") { + return values + } + if strings.HasPrefix(values, "(") { + return kind + values + } + return kind + "(" + values + ")" +} + func appendColumnDDLAttributes(sb *strings.Builder, col TableColumn) { if col.Generated != "" { if col.NotNull { @@ -3807,12 +3839,21 @@ func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, e sb.WriteString("ALTER TABLE ") sb.WriteString(quoteDDLIdent(tableName)) sb.WriteString(" ADD ") - if strings.EqualFold(info.indexType, "UNIQUE") { + fullText := isFullTextIndex(info) + if fullText { + sb.WriteString("FULLTEXT ") + } else if strings.EqualFold(info.indexType, "UNIQUE") { sb.WriteString("UNIQUE ") } - sb.WriteString("KEY ") + if fullText { + sb.WriteString("INDEX ") + } else { + sb.WriteString("KEY ") + } sb.WriteString(quoteDDLIdent(info.name)) - appendIndexAlgorithmDDL(&sb, info.algo) + if !fullText { + appendIndexAlgorithmDDL(&sb, info.algo) + } sb.WriteString("(") for i, col := range cols { if i > 0 { @@ -3828,6 +3869,13 @@ func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, e return sb.String(), nil } +func isFullTextIndex(info *indexDDLInfo) bool { + if info == nil { + return false + } + return catalog.IsFullTextIndexAlgo(info.algo) || strings.EqualFold(info.indexType, "FULLTEXT") +} + func appendIndexAlgorithmDDL(sb *strings.Builder, algo string) { algo = strings.TrimSpace(algo) if !catalog.IsNullIndexAlgo(algo) { @@ -4126,7 +4174,7 @@ func decodeMoColumnEncodedSQLType(raw string) (string, bool) { return "", false } if err := types.Decode([]byte(raw), &typ); err == nil && typ.Oid != types.T_any { - sqlType := typ.DescString() + sqlType := sqlTypeFromMOType(typ) if isPrintableSQLType(sqlType) { return sqlType, true } @@ -4134,6 +4182,26 @@ func decodeMoColumnEncodedSQLType(raw string) (string, bool) { return "", false } +func sqlTypeFromMOType(typ types.Type) string { + switch typ.Oid { + case types.T_time: + return scaledSQLType("TIME", typ.Scale) + case types.T_datetime: + return scaledSQLType("DATETIME", typ.Scale) + case types.T_timestamp: + return scaledSQLType("TIMESTAMP", typ.Scale) + default: + return typ.DescString() + } +} + +func scaledSQLType(name string, scale int32) string { + if scale <= 0 { + return name + } + return fmt.Sprintf("%s(%d)", name, scale) +} + func isPrintableDDLExpression(expr string) bool { expr = strings.TrimSpace(expr) if expr == "" { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index f2fd440a3bb58..7babe2dff42a1 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -760,6 +760,33 @@ func TestRenderCreateTableDDLFromSchema_DoesNotFallbackToCreateSQLWhenColumnType assert.Empty(t, ddl) } +func TestRenderCreateTableDDLFromSchema_EnumSetValues(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t_enum_set", + Columns: []TableColumn{ + {Name: "c_enum", SQLType: "ENUM", EnumValues: "'red','blue'", Position: 1}, + {Name: "c_set", SQLType: "SET", EnumValues: "('x','y')", Position: 2}, + }, + }) + + assert.Contains(t, ddl, "`c_enum` ENUM('red','blue')") + assert.Contains(t, ddl, "`c_set` SET('x','y')") +} + +func TestDecodeMoColumnEncodedSQLType_TemporalScale(t *testing.T) { + sqlType, ok := decodeMoColumnEncodedSQLType(encodedSQLType(t, types.New(types.T_time, 0, 6))) + require.True(t, ok) + assert.Equal(t, "TIME(6)", sqlType) + + sqlType, ok = decodeMoColumnEncodedSQLType(encodedSQLType(t, types.New(types.T_datetime, 0, 3))) + require.True(t, ok) + assert.Equal(t, "DATETIME(3)", sqlType) + + sqlType, ok = decodeMoColumnEncodedSQLType(encodedSQLType(t, types.New(types.T_timestamp, 0, 6))) + require.True(t, ok) + assert.Equal(t, "TIMESTAMP(6)", sqlType) +} + func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testing.T) { headers := []string{"object", "block", "row"} for i := 0; i < len(preCPKLayout.moTablesSchema)+2; i++ { @@ -1042,6 +1069,24 @@ func TestBuildCreateIndexStatementsFromMoIndexes_IVFFlat(t *testing.T) { }, stmts) } +func TestBuildCreateIndexStatementsFromMoIndexes_FullText(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{ + "table_id", "name", "type", "algo", "algo_params", "comment", + "column_name", "ordinal_position", "hidden", + }, + Rows: [][]string{ + {"272535", "idx_doc", "MULTIPLE", "fulltext", "", "", "doc", "1", "0"}, + }, + } + + stmts, err := buildCreateIndexStatementsFromMoIndexes(view, 272535, "t_fulltext") + require.NoError(t, err) + require.Equal(t, []string{ + "ALTER TABLE `t_fulltext` ADD FULLTEXT INDEX `idx_doc`(`doc`);", + }, stmts) +} + func TestWriteCSV_LexicalRowOrder(t *testing.T) { schema := &TableSchema{ TableName: "sorted", From a88ae0d1070e180634cd5d66060b4cda20dc3e76 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 11:52:46 +0800 Subject: [PATCH 50/76] restore external tables from ckp create sql --- cmd/mo-object-tool/ckp/checkpoint.go | 69 +++++++++++++++------ cmd/mo-object-tool/ckp/checkpoint_test.go | 39 +++++++++++- pkg/tools/checkpointtool/table_dump.go | 3 +- pkg/tools/checkpointtool/table_dump_test.go | 4 ++ 4 files changed, 93 insertions(+), 22 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 50550a0572aab..ac27d0975cc0e 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -805,24 +805,10 @@ func writeRestoreScript( currentDB = table.DatabaseName } - ddl := "" - if dumpData := dumpDataByTable[table.TableID]; dumpData != nil { - schema := dumpData.Schema - if schema != nil { - schemaCopy := *schema - schemaCopy.TableName = table.TableName - schemaCopy.DatabaseName = table.DatabaseName - ddl = checkpointtool.RenderCreateTableDDLFromSchema(&schemaCopy) - } - } - if ddl == "" { - var err error - ddl, err = reader.ShowCreateTable(ctx, table.TableID, snapshotTS) - if err != nil { - return "", fmt.Errorf("show create table %d: %w", table.TableID, err) - } + ddl, err := restoreCreateTableDDL(ctx, reader, table, dumpDataByTable[table.TableID], snapshotTS) + if err != nil { + return "", err } - ddl = normalizeCreateTableDDLName(ddl, table) indexDDLs, err := reader.ShowCreateIndexStatements(ctx, table.TableID, table.TableName, snapshotTS) if err != nil { return "", fmt.Errorf("show create indexes for table %d: %w", table.TableID, err) @@ -837,7 +823,7 @@ func writeRestoreScript( if _, err := fmt.Fprintln(f, ";"); err != nil { return "", err } - if includeLoad { + if shouldWriteLoadData(includeLoad, table) { if _, err := fmt.Fprintf(f, "\n%s\n", loadPathResolver.loadDataSource(csvRoot, table)); err != nil { return "", err } @@ -875,6 +861,48 @@ func writeRestoreScript( return scriptPath, nil } +func restoreCreateTableDDL( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + table checkpointtool.TableCatalogEntry, + dumpData *checkpointtool.TableDumpData, + snapshotTS types.TS, +) (string, error) { + ddl := "" + if dumpData != nil && dumpData.Schema != nil { + schema := dumpData.Schema + if isExternalRelation(table) && strings.TrimSpace(schema.CreateSQL) != "" { + ddl = schema.CreateSQL + } else { + schemaCopy := *schema + schemaCopy.TableName = table.TableName + schemaCopy.DatabaseName = table.DatabaseName + ddl = checkpointtool.RenderCreateTableDDLFromSchema(&schemaCopy) + } + } + if ddl == "" { + var err error + ddl, err = reader.ShowCreateTable(ctx, table.TableID, snapshotTS) + if err != nil { + return "", fmt.Errorf("show create table %d: %w", table.TableID, err) + } + } + return normalizeCreateTableDDLName(ddl, table), nil +} + +func shouldWriteLoadData(includeLoad bool, table checkpointtool.TableCatalogEntry) bool { + return includeLoad && !isExternalRelation(table) +} + +func isExternalRelation(table checkpointtool.TableCatalogEntry) bool { + switch strings.ToLower(strings.TrimSpace(table.RelKind)) { + case "e", "external": + return true + default: + return false + } +} + func quoteSQLIdent(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } @@ -1174,6 +1202,11 @@ func createTableNameRange(sql string) (int, int, bool) { i = next continue } + next, ok = consumeSQLKeyword(sql, i, "external") + if ok { + i = next + continue + } break } i, ok = consumeSQLKeyword(sql, i, "table") diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 4f5f9938bc2d8..384ca6f0c343b 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -15,10 +15,12 @@ package ckp import ( + "context" "path/filepath" "strings" "testing" + "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/stretchr/testify/assert" @@ -51,6 +53,11 @@ func TestNormalizeCreateTableDDLName(t *testing.T) { ddl: "CREATE TABLE IF NOT EXISTS old_name (id INT)", want: "CREATE TABLE IF NOT EXISTS `compat_ckp`.`employees` (id INT)", }, + { + name: "external table", + ddl: "CREATE EXTERNAL TABLE old_name (id INT) INFILE {'filepath'='/tmp/data.csv','format'='csv'}", + want: "CREATE EXTERNAL TABLE `compat_ckp`.`employees` (id INT) INFILE {'filepath'='/tmp/data.csv','format'='csv'}", + }, { name: "escaped target name", ddl: "CREATE TABLE old_name (id INT)", @@ -58,9 +65,9 @@ func TestNormalizeCreateTableDDLName(t *testing.T) { }, } - tests[3].want = "CREATE TABLE `compat``ckp`.`employees` (id INT)" - tests[3].ddl = "CREATE TABLE old_name (id INT)" - tests[3].name = "escaped database name" + tests[4].want = "CREATE TABLE `compat``ckp`.`employees` (id INT)" + tests[4].ddl = "CREATE TABLE old_name (id INT)" + tests[4].name = "escaped database name" for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -74,6 +81,32 @@ func TestNormalizeCreateTableDDLName(t *testing.T) { } } +func TestRestoreCreateTableDDLUsesExternalCreateSQL(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_external", + TableName: "ext_csv_local", + RelKind: "e", + } + dumpData := &checkpointtool.TableDumpData{ + Schema: &checkpointtool.TableSchema{ + TableName: "stale_name", + CreateSQL: "CREATE EXTERNAL TABLE stale_name (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}", + Columns: []checkpointtool.TableColumn{{Name: "id", SQLType: "INT", Position: 1}}, + }, + } + + ddl, err := restoreCreateTableDDL(context.Background(), nil, table, dumpData, types.TS{}) + require.NoError(t, err) + assert.Equal(t, "CREATE EXTERNAL TABLE `ckp_external`.`ext_csv_local` (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}", ddl) +} + +func TestShouldWriteLoadDataSkipsExternalRelations(t *testing.T) { + assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "e"})) + assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "external"})) + assert.True(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "r"})) + assert.False(t, shouldWriteLoadData(false, checkpointtool.TableCatalogEntry{RelKind: "r"})) +} + func TestFilterExistingIndexDDLs(t *testing.T) { createDDL := "CREATE TABLE `items_gist` (\n" + " `id` int NOT NULL,\n" + diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index bf8b579458943..6721f31c6acdb 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -4297,7 +4297,8 @@ func isPrintableCreateTableSQL(ddl string) bool { upper := strings.ToUpper(ddl) return strings.HasPrefix(upper, "CREATE TABLE ") || strings.HasPrefix(upper, "CREATE TEMPORARY TABLE ") || - strings.HasPrefix(upper, "CREATE CLUSTER TABLE ") + strings.HasPrefix(upper, "CREATE CLUSTER TABLE ") || + strings.HasPrefix(upper, "CREATE EXTERNAL TABLE ") } // hardcodedCreateTable returns the CREATE TABLE DDL for core built-in system tables. diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 7babe2dff42a1..c9f1d903cb6ed 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -760,6 +760,10 @@ func TestRenderCreateTableDDLFromSchema_DoesNotFallbackToCreateSQLWhenColumnType assert.Empty(t, ddl) } +func TestIsPrintableCreateTableSQLAcceptsExternalTable(t *testing.T) { + assert.True(t, isPrintableCreateTableSQL("CREATE EXTERNAL TABLE ext_csv (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}")) +} + func TestRenderCreateTableDDLFromSchema_EnumSetValues(t *testing.T) { ddl := RenderCreateTableDDLFromSchema(&TableSchema{ TableName: "t_enum_set", From df53e118a55f4b02aa84bd6d3e7fbd6e89338f4e Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 14:03:39 +0800 Subject: [PATCH 51/76] package external table sources in ckp dump --- cmd/mo-object-tool/ckp/checkpoint.go | 139 ++++++++++++++++++++++ cmd/mo-object-tool/ckp/checkpoint_test.go | 37 ++++++ 2 files changed, 176 insertions(+) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index ac27d0975cc0e..78beca4e55490 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -809,6 +809,12 @@ func writeRestoreScript( if err != nil { return "", err } + if isExternalRelation(table) { + ddl, err = packageExternalTableSource(ctx, dumpOut, csvRoot, table, ddl) + if err != nil { + return "", err + } + } indexDDLs, err := reader.ShowCreateIndexStatements(ctx, table.TableID, table.TableName, snapshotTS) if err != nil { return "", fmt.Errorf("show create indexes for table %d: %w", table.TableID, err) @@ -903,6 +909,139 @@ func isExternalRelation(table checkpointtool.TableCatalogEntry) bool { } } +func packageExternalTableSource( + ctx context.Context, + dumpOut *dumpOutput, + outputDir string, + table checkpointtool.TableCatalogEntry, + ddl string, +) (string, error) { + sourcePath, valueStart, valueEnd, ok := externalTableFilepathValueRange(ddl) + if !ok { + return "", fmt.Errorf("external table %d (%s.%s): CREATE SQL does not contain INFILE filepath", table.TableID, table.DatabaseName, table.TableName) + } + if dumpOut != nil && dumpOut.remote { + return "", fmt.Errorf("external table %d (%s.%s): packaging local external source is not supported for remote dump output", table.TableID, table.DatabaseName, table.TableName) + } + + destPath := externalSourcePath(outputDir, table, filepath.Base(sourcePath)) + if err := dumpOut.MkdirAll(path.Dir(destPath)); err != nil { + return "", fmt.Errorf("create external source output dir: %w", err) + } + if err := copyLocalFileToDumpOutput(ctx, dumpOut, sourcePath, destPath); err != nil { + return "", fmt.Errorf("copy external source for table %d (%s.%s): %w", table.TableID, table.DatabaseName, table.TableName, err) + } + restorePath := destPath + if absPath, err := filepath.Abs(destPath); err == nil { + restorePath = absPath + } + return ddl[:valueStart] + quoteSQLString(restorePath) + ddl[valueEnd:], nil +} + +func copyLocalFileToDumpOutput(ctx context.Context, dumpOut *dumpOutput, sourcePath, destPath string) error { + in, err := os.Open(sourcePath) + if err != nil { + return err + } + defer in.Close() + + out, err := dumpOut.Create(ctx, destPath) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + closeErr := out.Close() + if copyErr != nil { + return copyErr + } + return closeErr +} + +func externalSourcePath(outputDir string, table checkpointtool.TableCatalogEntry, sourceName string) string { + sourceName = safePathPart(sourceName) + if sourceName == "_" || sourceName == "." || sourceName == string(filepath.Separator) { + sourceName = safePathPart(table.TableName) + ".data" + } + return outputPathJoin( + outputDir, + "external_sources", + fmt.Sprintf("account_%d", table.AccountID), + fmt.Sprintf("db_%d", table.DatabaseID), + fmt.Sprintf("%s_%d_%s", safePathPart(table.TableName), table.TableID, sourceName), + ) +} + +func externalTableFilepathValueRange(sql string) (string, int, int, bool) { + lower := strings.ToLower(sql) + searchFrom := 0 + for { + idx := strings.Index(lower[searchFrom:], "filepath") + if idx < 0 { + return "", 0, 0, false + } + idx += searchFrom + i := idx + len("filepath") + for i < len(sql) { + ch := sql[i] + if ch == '=' { + break + } + if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' && ch != '\'' && ch != '"' { + searchFrom = i + 1 + break + } + i++ + } + if searchFrom > idx { + continue + } + if i >= len(sql) || sql[i] != '=' { + return "", 0, 0, false + } + i = skipSQLSpace(sql, i+1) + valueStart := i + value, valueEnd, ok := readSQLStringOrBareValue(sql, i) + if !ok { + return "", 0, 0, false + } + return value, valueStart, valueEnd, true + } +} + +func readSQLStringOrBareValue(sql string, start int) (string, int, bool) { + if start >= len(sql) { + return "", 0, false + } + if sql[start] == '\'' || sql[start] == '"' { + quote := sql[start] + var b strings.Builder + for i := start + 1; i < len(sql); i++ { + ch := sql[i] + if ch == quote { + if i+1 < len(sql) && sql[i+1] == quote { + b.WriteByte(quote) + i++ + continue + } + return b.String(), i + 1, true + } + if ch == '\\' && i+1 < len(sql) { + i++ + b.WriteByte(sql[i]) + continue + } + b.WriteByte(ch) + } + return "", 0, false + } + end := start + for end < len(sql) && sql[end] != ',' && sql[end] != '}' && sql[end] != ')' && sql[end] != ' ' && sql[end] != '\t' && sql[end] != '\n' && sql[end] != '\r' { + end++ + } + value := strings.TrimSpace(sql[start:end]) + return value, end, value != "" +} + func quoteSQLIdent(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 384ca6f0c343b..1ec125fc3be17 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -16,6 +16,7 @@ package ckp import ( "context" + "os" "path/filepath" "strings" "testing" @@ -107,6 +108,42 @@ func TestShouldWriteLoadDataSkipsExternalRelations(t *testing.T) { assert.False(t, shouldWriteLoadData(false, checkpointtool.TableCatalogEntry{RelKind: "r"})) } +func TestPackageExternalTableSourceCopiesAndRewritesFilepath(t *testing.T) { + tmpDir := t.TempDir() + sourcePath := filepath.Join(tmpDir, "local_ext_people.csv") + require.NoError(t, os.WriteFile(sourcePath, []byte("id,name\n1,a\n"), 0o644)) + + table := checkpointtool.TableCatalogEntry{ + AccountID: 0, + DatabaseID: 272731, + TableID: 272732, + DatabaseName: "ckp_external", + TableName: "ext_csv_local", + RelKind: "e", + } + ddl := "CREATE EXTERNAL TABLE ext_csv_local (id INT, name VARCHAR(50)) INFILE {'filepath'='" + sourcePath + "', 'format'='csv'}" + + got, err := packageExternalTableSource(context.Background(), &dumpOutput{}, filepath.Join(tmpDir, "dump"), table, ddl) + require.NoError(t, err) + assert.NotContains(t, got, sourcePath) + + copiedPath, _, _, ok := externalTableFilepathValueRange(got) + require.True(t, ok) + assert.True(t, filepath.IsAbs(copiedPath)) + assert.Contains(t, copiedPath, filepath.Join("external_sources", "account_0", "db_272731")) + data, err := os.ReadFile(copiedPath) + require.NoError(t, err) + assert.Equal(t, "id,name\n1,a\n", string(data)) +} + +func TestExternalTableFilepathValueRange(t *testing.T) { + ddl := "CREATE EXTERNAL TABLE t (id INT) INFILE {'format'='csv', 'filepath'='/tmp/a.csv'}" + value, start, end, ok := externalTableFilepathValueRange(ddl) + require.True(t, ok) + assert.Equal(t, "/tmp/a.csv", value) + assert.Equal(t, "CREATE EXTERNAL TABLE t (id INT) INFILE {'format'='csv', 'filepath'='/new.csv'}", ddl[:start]+quoteSQLString("/new.csv")+ddl[end:]) +} + func TestFilterExistingIndexDDLs(t *testing.T) { createDDL := "CREATE TABLE `items_gist` (\n" + " `id` int NOT NULL,\n" + From ff890db1b0a03932d3f41d0386f08f990bbf243d Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 14:14:41 +0800 Subject: [PATCH 52/76] fix ckp load csv encoding for special types --- pkg/tools/checkpointtool/table_dump.go | 48 +++++++++++++++++- pkg/tools/checkpointtool/table_dump_test.go | 55 +++++++++++++++++++++ 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 6721f31c6acdb..e42eefc6960ed 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -2313,8 +2313,12 @@ func vectorRowIndex(vec *vector.Vector, rowIdx int) int { func appendCSVQuotedVecValue(w *bytes.Buffer, vec *vector.Vector, rowIdx int) { switch vec.GetType().Oid { case types.T_char, types.T_varchar, types.T_blob, types.T_text, - types.T_binary, types.T_varbinary, types.T_datalink, types.T_json: + types.T_binary, types.T_varbinary, types.T_datalink: appendEscapedSQLLoadBytes(w, vec.GetBytesAt(rowIdx), '"') + case types.T_json: + appendEscapedSQLLoadString(w, types.DecodeJson(vec.GetBytesAt(rowIdx)).String(), '"') + case types.T_bit: + appendEscapedSQLLoadBytes(w, bitValueToLoadBytes(vector.MustFixedColWithTypeCheck[uint64](vec)[rowIdx], vec.GetType().Width), '"') default: appendEscapedSQLLoadString(w, vecValueToString(vec, rowIdx), '"') } @@ -2354,6 +2358,12 @@ func appendCSVUnquotedVecValue(w *bytes.Buffer, typ types.Type, vec *vector.Vect appendDecimal128(w, vector.MustFixedColWithTypeCheck[types.Decimal128](vec)[rowIdx], vec.GetType().Scale) case types.T_date: appendDate(w, vector.MustFixedColWithTypeCheck[types.Date](vec)[rowIdx]) + case types.T_time: + w.WriteString(vector.MustFixedColWithTypeCheck[types.Time](vec)[rowIdx].String2(vec.GetType().Scale)) + case types.T_datetime: + w.WriteString(vector.MustFixedColWithTypeCheck[types.Datetime](vec)[rowIdx].String2(vec.GetType().Scale)) + case types.T_timestamp: + w.WriteString(vector.MustFixedColWithTypeCheck[types.Timestamp](vec)[rowIdx].String2(time.Local, vec.GetType().Scale)) default: w.WriteString(vecValueToString(vec, rowIdx)) } @@ -2381,6 +2391,24 @@ func appendEscapedSQLLoadString(w *bytes.Buffer, s string, enclosed byte) { func appendEscapedSQLLoadBytes(w *bytes.Buffer, data []byte, enclosed byte) { for _, b := range data { switch b { + case 0: + w.WriteByte('\\') + w.WriteByte('0') + case '\b': + w.WriteByte('\\') + w.WriteByte('b') + case '\n': + w.WriteByte('\\') + w.WriteByte('n') + case '\r': + w.WriteByte('\\') + w.WriteByte('r') + case '\t': + w.WriteByte('\\') + w.WriteByte('t') + case 0x1a: + w.WriteByte('\\') + w.WriteByte('Z') case '\\': w.WriteByte('\\') w.WriteByte('\\') @@ -2397,6 +2425,22 @@ func appendEscapedSQLLoadBytes(w *bytes.Buffer, data []byte, enclosed byte) { } } +func bitValueToLoadBytes(value uint64, width int32) []byte { + byteLen := int((width + 7) / 8) + if byteLen <= 0 { + byteLen = 1 + } + if byteLen > 8 { + byteLen = 8 + } + out := make([]byte, byteLen) + for i := byteLen - 1; i >= 0; i-- { + out[i] = byte(value) + value >>= 8 + } + return out +} + func appendDecimal64(w *bytes.Buffer, value types.Decimal64, scale int32) { if value.Sign() { w.WriteByte('-') @@ -2529,7 +2573,7 @@ func shouldQuoteSQLLoadType(typ types.Type) bool { switch typ.Oid { case types.T_char, types.T_varchar, types.T_blob, types.T_text, types.T_binary, types.T_varbinary, types.T_datalink, types.T_json, - types.T_geometry, types.T_array_float32, types.T_array_float64: + types.T_geometry, types.T_array_float32, types.T_array_float64, types.T_bit: return true default: return false diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index c9f1d903cb6ed..4fcf19f1adede 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -20,6 +20,7 @@ import ( "fmt" "strings" "testing" + "time" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" @@ -1376,6 +1377,60 @@ func TestWriteSQLLoadCSVRow_RoundTripsThroughCSVParser(t *testing.T) { assert.True(t, row[2].HasStringQuote) } +func TestWriteSQLLoadCSVFieldFromVec_BitUsesPackedBytes(t *testing.T) { + mp := mpool.MustNewZero() + vec := vector.NewVec(types.New(types.T_bit, 1, 0)) + require.NoError(t, vector.AppendFixed(vec, uint64(1), false, mp)) + + var buf bytes.Buffer + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *vec.GetType(), []*vector.Vector{vec}, 0, 0)) + assert.Equal(t, "\"\x01\"", buf.String()) +} + +func TestWriteSQLLoadCSVFieldFromVec_JSONUsesVisibleText(t *testing.T) { + mp := mpool.MustNewZero() + vec := vector.NewVec(types.T_json.ToType()) + bj, err := types.ParseStringToByteJson(`{"name":"mo","n":1}`) + require.NoError(t, err) + stored, err := types.EncodeJson(bj) + require.NoError(t, err) + require.NoError(t, vector.AppendBytes(vec, stored, false, mp)) + + var buf bytes.Buffer + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *vec.GetType(), []*vector.Vector{vec}, 0, 0)) + assert.Equal(t, `"{""n"": 1, ""name"": ""mo""}"`, buf.String()) +} + +func TestWriteSQLLoadCSVFieldFromVec_TemporalKeepsScale(t *testing.T) { + mp := mpool.MustNewZero() + timeVec := vector.NewVec(types.New(types.T_time, 0, 6)) + datetimeVec := vector.NewVec(types.New(types.T_datetime, 0, 3)) + timestampVec := vector.NewVec(types.New(types.T_timestamp, 0, 6)) + require.NoError(t, vector.AppendFixed(timeVec, types.TimeFromClock(false, 11, 22, 33, 123456), false, mp)) + require.NoError(t, vector.AppendFixed(datetimeVec, types.DatetimeFromClock(2024, 1, 2, 3, 4, 5, 123000), false, mp)) + require.NoError(t, vector.AppendFixed(timestampVec, types.FromClockZone(time.Local, 2024, 1, 2, 3, 4, 5, 123456), false, mp)) + + var buf bytes.Buffer + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *timeVec.GetType(), []*vector.Vector{timeVec}, 0, 0)) + assert.Equal(t, "11:22:33.123456", buf.String()) + buf.Reset() + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *datetimeVec.GetType(), []*vector.Vector{datetimeVec}, 0, 0)) + assert.Equal(t, "2024-01-02 03:04:05.123", buf.String()) + buf.Reset() + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *timestampVec.GetType(), []*vector.Vector{timestampVec}, 0, 0)) + assert.Equal(t, "2024-01-02 03:04:05.123456", buf.String()) +} + +func TestWriteSQLLoadCSVFieldFromVec_BinaryEscapesControlBytes(t *testing.T) { + mp := mpool.MustNewZero() + vec := vector.NewVec(types.T_blob.ToType()) + require.NoError(t, vector.AppendBytes(vec, []byte{'a', '\n', 0, '\r', '\t', 0x1a, '"', '\\'}, false, mp)) + + var buf bytes.Buffer + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *vec.GetType(), []*vector.Vector{vec}, 0, 0)) + assert.Equal(t, `"a\n\0\r\t\Z""\\"`, buf.String()) +} + func TestParseCSVRowOrder(t *testing.T) { order, err := ParseCSVRowOrder("storage") require.NoError(t, err) From 1f13865dc27d4ac8ee904e1e692df236e9f473e9 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 14:30:05 +0800 Subject: [PATCH 53/76] restore external table ddl from rel_createsql params --- cmd/mo-object-tool/ckp/checkpoint.go | 258 ++++++++++++++++++-- cmd/mo-object-tool/ckp/checkpoint_test.go | 40 ++- pkg/tools/checkpointtool/table_dump.go | 94 +++++-- pkg/tools/checkpointtool/table_dump_test.go | 35 +++ 4 files changed, 386 insertions(+), 41 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 78beca4e55490..b759485a9ffd7 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -15,7 +15,9 @@ package ckp import ( + "bytes" "context" + "encoding/json" "fmt" "io" "os" @@ -32,6 +34,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool/interactive" "github.com/matrixorigin/matrixone/pkg/tools/toolfs" @@ -779,12 +782,8 @@ func writeRestoreScript( return "", fmt.Errorf("create script dir: %w", err) } scriptPath := outputPathJoin(scriptDir, "restore.sql") - f, err := dumpOut.Create(ctx, scriptPath) - if err != nil { - return "", fmt.Errorf("create restore script: %w", err) - } - defer f.Close() + var script bytes.Buffer currentDB := "" for _, table := range tables { if table.DatabaseName == "" { @@ -792,14 +791,14 @@ func writeRestoreScript( } if table.DatabaseName != currentDB { if currentDB != "" { - if _, err := fmt.Fprintln(f); err != nil { + if _, err := fmt.Fprintln(&script); err != nil { return "", err } } - if _, err := fmt.Fprintf(f, "CREATE DATABASE IF NOT EXISTS %s;\n", quoteSQLIdent(table.DatabaseName)); err != nil { + if _, err := fmt.Fprintf(&script, "CREATE DATABASE IF NOT EXISTS %s;\n", quoteSQLIdent(table.DatabaseName)); err != nil { return "", err } - if _, err := fmt.Fprintf(f, "USE %s;\n\n", quoteSQLIdent(table.DatabaseName)); err != nil { + if _, err := fmt.Fprintf(&script, "USE %s;\n\n", quoteSQLIdent(table.DatabaseName)); err != nil { return "", err } currentDB = table.DatabaseName @@ -823,47 +822,56 @@ func writeRestoreScript( if err != nil { return "", fmt.Errorf("merge create table indexes for table %d: %w", table.TableID, err) } - if _, err := fmt.Fprintln(f, strings.TrimRight(ddl, " ;\n\t")); err != nil { + if _, err := fmt.Fprintln(&script, strings.TrimRight(ddl, " ;\n\t")); err != nil { return "", err } - if _, err := fmt.Fprintln(f, ";"); err != nil { + if _, err := fmt.Fprintln(&script, ";"); err != nil { return "", err } if shouldWriteLoadData(includeLoad, table) { - if _, err := fmt.Fprintf(f, "\n%s\n", loadPathResolver.loadDataSource(csvRoot, table)); err != nil { + if _, err := fmt.Fprintf(&script, "\n%s\n", loadPathResolver.loadDataSource(csvRoot, table)); err != nil { return "", err } - if _, err := fmt.Fprintf(f, "INTO TABLE %s\n", quoteSQLIdent(table.TableName)); err != nil { + if _, err := fmt.Fprintf(&script, "INTO TABLE %s\n", quoteSQLIdent(table.TableName)); err != nil { return "", err } - if _, err := fmt.Fprintln(f, "FIELDS TERMINATED BY ','"); err != nil { + if _, err := fmt.Fprintln(&script, "FIELDS TERMINATED BY ','"); err != nil { return "", err } - if _, err := fmt.Fprintln(f, "ENCLOSED BY '\"'"); err != nil { + if _, err := fmt.Fprintln(&script, "ENCLOSED BY '\"'"); err != nil { return "", err } - if _, err := fmt.Fprintln(f, "LINES TERMINATED BY '\\n'"); err != nil { + if _, err := fmt.Fprintln(&script, "LINES TERMINATED BY '\\n'"); err != nil { return "", err } if csvHasHeader { - if _, err := fmt.Fprintln(f, "IGNORE 1 LINES"); err != nil { + if _, err := fmt.Fprintln(&script, "IGNORE 1 LINES"); err != nil { return "", err } } - if _, err := fmt.Fprintln(f, "parallel 'true'"); err != nil { + if _, err := fmt.Fprintln(&script, "parallel 'true'"); err != nil { return "", err } - if _, err := fmt.Fprintln(f, ";"); err != nil { + if _, err := fmt.Fprintln(&script, ";"); err != nil { return "", err } } - if _, err := fmt.Fprintln(f); err != nil { + if _, err := fmt.Fprintln(&script); err != nil { return "", err } } + + f, err := dumpOut.Create(ctx, scriptPath) + if err != nil { + return "", fmt.Errorf("create restore script: %w", err) + } + _, writeErr := io.Copy(f, &script) if err := f.Close(); err != nil { return "", fmt.Errorf("close restore script: %w", err) } + if writeErr != nil { + return "", fmt.Errorf("write restore script: %w", writeErr) + } return scriptPath, nil } @@ -877,8 +885,15 @@ func restoreCreateTableDDL( ddl := "" if dumpData != nil && dumpData.Schema != nil { schema := dumpData.Schema - if isExternalRelation(table) && strings.TrimSpace(schema.CreateSQL) != "" { - ddl = schema.CreateSQL + if isExternalRelation(table) { + if strings.TrimSpace(schema.CreateSQL) == "" { + return "", fmt.Errorf("external table %d (%s.%s): missing external table parameter JSON in checkpoint metadata", table.TableID, table.DatabaseName, table.TableName) + } + var err error + ddl, err = renderExternalCreateTableDDLFromParamJSON(table, schema) + if err != nil { + return "", err + } } else { schemaCopy := *schema schemaCopy.TableName = table.TableName @@ -896,6 +911,197 @@ func restoreCreateTableDDL( return normalizeCreateTableDDLName(ddl, table), nil } +func renderExternalCreateTableDDLFromParamJSON(table checkpointtool.TableCatalogEntry, schema *checkpointtool.TableSchema) (string, error) { + var param tree.ExternParam + if err := json.Unmarshal([]byte(schema.CreateSQL), ¶m); err != nil { + return "", fmt.Errorf("external table %d (%s.%s): decode external table parameter JSON: %w", table.TableID, table.DatabaseName, table.TableName, err) + } + applyExternalParamOptions(¶m) + + schemaCopy := *schema + schemaCopy.TableName = table.TableName + schemaCopy.DatabaseName = table.DatabaseName + schemaCopy.CreateSQL = "" + base := checkpointtool.RenderCreateTableDDLFromSchema(&schemaCopy) + if base == "" { + return "", fmt.Errorf("external table %d (%s.%s): cannot render external table columns from checkpoint metadata", table.TableID, table.DatabaseName, table.TableName) + } + base = strings.TrimRight(base, " ;\n\t") + if strings.HasPrefix(strings.ToUpper(base), "CREATE TABLE ") { + base = "CREATE EXTERNAL TABLE " + strings.TrimSpace(base[len("CREATE TABLE "):]) + } else { + return "", fmt.Errorf("external table %d (%s.%s): unexpected generated DDL: %s", table.TableID, table.DatabaseName, table.TableName, summarizeSQLForError(base)) + } + return base + renderExternalParamClause(¶m), nil +} + +func applyExternalParamOptions(param *tree.ExternParam) { + for i := 0; i+1 < len(param.Option); i += 2 { + key := strings.ToLower(param.Option[i]) + value := param.Option[i+1] + switch key { + case "filepath": + param.Filepath = value + case "compression": + param.CompressType = value + case "format": + param.Format = strings.ToLower(value) + case "jsondata": + param.JsonData = strings.ToLower(value) + case "hive_partitioning": + param.HivePartitioning = strings.EqualFold(value, "true") + case "hive_partition_columns": + if value != "" { + param.HivePartitionCols = strings.Split(value, ",") + } + case "endpoint": + ensureExternalS3Param(param).Endpoint = value + case "region": + ensureExternalS3Param(param).Region = value + case "access_key_id": + ensureExternalS3Param(param).APIKey = value + case "secret_access_key": + ensureExternalS3Param(param).APISecret = value + case "bucket": + ensureExternalS3Param(param).Bucket = value + case "provider": + ensureExternalS3Param(param).Provider = value + case "role_arn": + ensureExternalS3Param(param).RoleArn = value + case "external_id": + ensureExternalS3Param(param).ExternalId = value + } + } + if param.CompressType == "" { + param.CompressType = "auto" + } + if param.Format == "" { + param.Format = "csv" + } +} + +func ensureExternalS3Param(param *tree.ExternParam) *tree.S3Parameter { + if param.S3Param == nil { + param.S3Param = &tree.S3Parameter{} + } + return param.S3Param +} + +func renderExternalParamClause(param *tree.ExternParam) string { + if param.ScanType == tree.S3 { + return renderExternalS3Clause(param) + } + return renderExternalInfileClause(param) +} + +func renderExternalInfileClause(param *tree.ExternParam) string { + options := []string{ + "filepath", param.Filepath, + "compression", param.CompressType, + "format", param.Format, + } + if param.JsonData != "" { + options = append(options, "jsondata", param.JsonData) + } + if param.HivePartitioning { + options = append(options, "hive_partitioning", "true") + if len(param.HivePartitionCols) > 0 { + options = append(options, "hive_partition_columns", strings.Join(param.HivePartitionCols, ",")) + } + } + return " INFILE {" + formatExternalOptions(options) + "}" + renderExternalTailClause(param) +} + +func renderExternalS3Clause(param *tree.ExternParam) string { + options := make([]string, 0, 20) + if param.S3Param != nil { + options = appendExternalOptionIfSet(options, "endpoint", param.S3Param.Endpoint) + options = appendExternalOptionIfSet(options, "region", param.S3Param.Region) + options = appendExternalOptionIfSet(options, "access_key_id", param.S3Param.APIKey) + options = appendExternalOptionIfSet(options, "secret_access_key", param.S3Param.APISecret) + options = appendExternalOptionIfSet(options, "bucket", param.S3Param.Bucket) + } + options = appendExternalOptionIfSet(options, "filepath", param.Filepath) + if param.S3Param != nil { + options = appendExternalOptionIfSet(options, "provider", param.S3Param.Provider) + options = appendExternalOptionIfSet(options, "role_arn", param.S3Param.RoleArn) + options = appendExternalOptionIfSet(options, "external_id", param.S3Param.ExternalId) + } + options = appendExternalOptionIfSet(options, "compression", param.CompressType) + options = appendExternalOptionIfSet(options, "format", param.Format) + options = appendExternalOptionIfSet(options, "jsondata", param.JsonData) + return " URL s3option {" + formatExternalOptions(options) + "}" + renderExternalTailClause(param) +} + +func appendExternalOptionIfSet(options []string, key, value string) []string { + if value == "" { + return options + } + return append(options, key, value) +} + +func formatExternalOptions(options []string) string { + parts := make([]string, 0, len(options)/2) + for i := 0; i+1 < len(options); i += 2 { + parts = append(parts, quoteSQLString(options[i])+"="+quoteSQLString(options[i+1])) + } + return strings.Join(parts, ", ") +} + +func renderExternalTailClause(param *tree.ExternParam) string { + if param.Tail == nil { + return "" + } + var sb strings.Builder + if param.Tail.Fields != nil { + var fields []string + if param.Tail.Fields.Terminated != nil { + fields = append(fields, "TERMINATED BY "+quoteExternalLiteral(param.Tail.Fields.Terminated.Value)) + } + if param.Tail.Fields.EnclosedBy != nil { + fields = append(fields, "ENCLOSED BY "+quoteExternalLiteral(byteSQLString(param.Tail.Fields.EnclosedBy.Value))) + } + if param.Tail.Fields.EscapedBy != nil { + fields = append(fields, "ESCAPED BY "+quoteExternalLiteral(byteSQLString(param.Tail.Fields.EscapedBy.Value))) + } + if len(fields) > 0 { + sb.WriteString(" FIELDS ") + sb.WriteString(strings.Join(fields, " ")) + } + } + if param.Tail.Lines != nil { + var lines []string + if param.Tail.Lines.StartingBy != "" { + lines = append(lines, "STARTING BY "+quoteExternalLiteral(param.Tail.Lines.StartingBy)) + } + if param.Tail.Lines.TerminatedBy != nil { + lines = append(lines, "TERMINATED BY "+quoteExternalLiteral(param.Tail.Lines.TerminatedBy.Value)) + } + if len(lines) > 0 { + sb.WriteString(" LINES ") + sb.WriteString(strings.Join(lines, " ")) + } + } + if param.Tail.IgnoredLines > 0 { + fmt.Fprintf(&sb, " IGNORE %d LINES", param.Tail.IgnoredLines) + } + return sb.String() +} + +func quoteExternalLiteral(s string) string { + s = strings.ReplaceAll(s, "\n", `\n`) + s = strings.ReplaceAll(s, "\r", `\r`) + s = strings.ReplaceAll(s, "\t", `\t`) + return quoteSQLString(s) +} + +func byteSQLString(value byte) string { + if value == 0 { + return "" + } + return string([]byte{value}) +} + func shouldWriteLoadData(includeLoad bool, table checkpointtool.TableCatalogEntry) bool { return includeLoad && !isExternalRelation(table) } @@ -918,7 +1124,7 @@ func packageExternalTableSource( ) (string, error) { sourcePath, valueStart, valueEnd, ok := externalTableFilepathValueRange(ddl) if !ok { - return "", fmt.Errorf("external table %d (%s.%s): CREATE SQL does not contain INFILE filepath", table.TableID, table.DatabaseName, table.TableName) + return "", fmt.Errorf("external table %d (%s.%s): CREATE EXTERNAL TABLE SQL does not contain INFILE filepath: %s", table.TableID, table.DatabaseName, table.TableName, summarizeSQLForError(ddl)) } if dumpOut != nil && dumpOut.remote { return "", fmt.Errorf("external table %d (%s.%s): packaging local external source is not supported for remote dump output", table.TableID, table.DatabaseName, table.TableName) @@ -1042,6 +1248,14 @@ func readSQLStringOrBareValue(sql string, start int) (string, int, bool) { return value, end, value != "" } +func summarizeSQLForError(sql string) string { + sql = strings.Join(strings.Fields(sql), " ") + if len(sql) > 240 { + return sql[:240] + "..." + } + return sql +} + func quoteSQLIdent(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 1ec125fc3be17..909d6a37d5386 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -16,12 +16,14 @@ package ckp import ( "context" + "encoding/json" "os" "path/filepath" "strings" "testing" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/stretchr/testify/assert" @@ -88,17 +90,51 @@ func TestRestoreCreateTableDDLUsesExternalCreateSQL(t *testing.T) { TableName: "ext_csv_local", RelKind: "e", } + paramJSON, err := json.Marshal(&tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + Option: []string{"filepath", "/tmp/ext.csv", "format", "csv", "compression", "none"}, + Tail: &tree.TailParameter{ + Fields: &tree.Fields{ + Terminated: &tree.Terminated{Value: ","}, + EnclosedBy: &tree.EnclosedBy{Value: '"'}, + }, + Lines: &tree.Lines{ + TerminatedBy: &tree.Terminated{Value: "\n"}, + }, + IgnoredLines: 1, + }, + }, + }) + require.NoError(t, err) dumpData := &checkpointtool.TableDumpData{ Schema: &checkpointtool.TableSchema{ TableName: "stale_name", - CreateSQL: "CREATE EXTERNAL TABLE stale_name (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}", + CreateSQL: string(paramJSON), Columns: []checkpointtool.TableColumn{{Name: "id", SQLType: "INT", Position: 1}}, }, } ddl, err := restoreCreateTableDDL(context.Background(), nil, table, dumpData, types.TS{}) require.NoError(t, err) - assert.Equal(t, "CREATE EXTERNAL TABLE `ckp_external`.`ext_csv_local` (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}", ddl) + assert.Equal(t, "CREATE EXTERNAL TABLE `ckp_external`.`ext_csv_local` (\n `id` INT\n) INFILE {'filepath'='/tmp/ext.csv', 'compression'='none', 'format'='csv'} FIELDS TERMINATED BY ',' ENCLOSED BY '\"' LINES TERMINATED BY '\\\\n' IGNORE 1 LINES", ddl) +} + +func TestRestoreCreateTableDDLExternalRequiresCreateSQL(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_external", + TableName: "ext_csv_local", + RelKind: "e", + } + dumpData := &checkpointtool.TableDumpData{ + Schema: &checkpointtool.TableSchema{ + TableName: "ext_csv_local", + Columns: []checkpointtool.TableColumn{{Name: "id", SQLType: "INT", Position: 1}}, + }, + } + + _, err := restoreCreateTableDDL(context.Background(), nil, table, dumpData, types.TS{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing external table parameter JSON") } func TestShouldWriteLoadDataSkipsExternalRelations(t *testing.T) { diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index e42eefc6960ed..3f664630179df 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -861,17 +861,8 @@ func (r *CheckpointReader) ReadTableSchema( if err != nil { // Don't log - expected for system tables or when mo_tables not in checkpoint } else if moTablesView != nil { - // Resolve column positions by name (handles hidden column offset) - relIDCol := fallbackCatalogColIndex(moTablesView, moTablesID, "rel_id") - for _, row := range moTablesView.Rows { - dataRow := row[logicalViewDataOffset(moTablesView):] - if relIDCol < 0 || relIDCol >= len(dataRow) { - continue - } - if dataRow[relIDCol] == fmt.Sprintf("%d", tableID) { - schema = buildSchemaFromMoTablesRow(moTablesView, row) - break - } + if moTablesSchema := findTableSchemaFromMoTables(moTablesView, tableID); moTablesSchema != nil { + schema = moTablesSchema } } @@ -2730,30 +2721,45 @@ func dataIndexForSeqNum(view *LogicalTableView, seqNum int) int { // Uses column name lookup from view headers to handle hidden column offsets. // mo_tables columns: rel_id, relname, reldatabase, reldatabase_id, ..., rel_createsql func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *TableSchema { + return buildSchemaFromMoTablesRowAt( + view, + fullRow, + fallbackCatalogColIndex(view, moTablesID, "relname"), + fallbackCatalogColIndex(view, moTablesID, "reldatabase"), + fallbackCatalogColIndex(view, moTablesID, "rel_createsql"), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint), + ) +} + +func buildSchemaFromMoTablesRowAt( + view *LogicalTableView, + fullRow []string, + relNameIdx int, + relDBIdx int, + createSQLIdx int, + commentIdx int, + constraintIdx int, +) *TableSchema { schema := &TableSchema{} dataRow := fullRow[logicalViewDataOffset(view):] - relNameIdx := fallbackCatalogColIndex(view, moTablesID, "relname") if relNameIdx >= 0 && relNameIdx < len(dataRow) { schema.TableName = strings.Clone(dataRow[relNameIdx]) } - relDBIdx := fallbackCatalogColIndex(view, moTablesID, "reldatabase") if relDBIdx >= 0 && relDBIdx < len(dataRow) { schema.DatabaseName = strings.Clone(dataRow[relDBIdx]) } - createSQLIdx := fallbackCatalogColIndex(view, moTablesID, "rel_createsql") if createSQLIdx >= 0 && createSQLIdx < len(dataRow) { - if isPrintableCreateTableSQL(dataRow[createSQLIdx]) { + if isPrintableCreateTableSQL(dataRow[createSQLIdx]) || isPrintableExternalParamJSON(dataRow[createSQLIdx]) { schema.CreateSQL = strings.Clone(dataRow[createSQLIdx]) } } - commentIdx := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment) if commentIdx >= 0 && commentIdx < len(dataRow) && isPrintableSQLText(dataRow[commentIdx]) { schema.Comment = strings.Clone(dataRow[commentIdx]) } - constraintIdx := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint) if constraintIdx >= 0 && constraintIdx < len(dataRow) { schema.UniqueKeys = decodeUniqueKeysFromMoTablesConstraint(dataRow[constraintIdx]) schema.PrimaryKey = decodePrimaryKeyFromMoTablesConstraint(dataRow[constraintIdx]) @@ -2761,6 +2767,49 @@ func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *Table return schema } +func findTableSchemaFromMoTables(view *LogicalTableView, tableID uint64) *TableSchema { + tableIDStr := fmt.Sprintf("%d", tableID) + try := func(relIDCol, relNameCol, relDBCol, createSQLCol, commentCol, constraintCol int) *TableSchema { + if relIDCol < 0 { + return nil + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || row[relIDCol] != tableIDStr { + continue + } + return buildSchemaFromMoTablesRowAt(view, fullRow, relNameCol, relDBCol, createSQLCol, commentCol, constraintCol) + } + return nil + } + + if schema := try( + fallbackCatalogColIndex(view, moTablesID, "rel_id"), + fallbackCatalogColIndex(view, moTablesID, "relname"), + fallbackCatalogColIndex(view, moTablesID, "reldatabase"), + fallbackCatalogColIndex(view, moTablesID, "rel_createsql"), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint), + ); schema != nil { + return schema + } + + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { + if schema := try( + catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "relname", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "reldatabase", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "rel_createsql", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_Comment, match.offset), + catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_Constraint, match.offset), + ); schema != nil { + return schema + } + } + return nil +} + func findTableNameFromMoTables(view *LogicalTableView, tableID uint64) string { tableIDStr := fmt.Sprintf("%d", tableID) try := func(relIDCol, relNameCol int) string { @@ -4345,6 +4394,17 @@ func isPrintableCreateTableSQL(ddl string) bool { strings.HasPrefix(upper, "CREATE EXTERNAL TABLE ") } +func isPrintableExternalParamJSON(raw string) bool { + raw = strings.TrimSpace(raw) + if raw == "" || raw[0] != '{' || raw[len(raw)-1] != '}' { + return false + } + if !isPrintableSQLText(raw) { + return false + } + return strings.Contains(raw, `"ExParamConst"`) || strings.Contains(raw, `"Filepath"`) || strings.Contains(raw, `"Option"`) +} + // hardcodedCreateTable returns the CREATE TABLE DDL for core built-in system tables. // These tables' schemas are known at compile time and may not appear in the checkpoint's // mo_tables/mo_columns (due to minimal deployments). diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 4fcf19f1adede..647365c15283a 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -765,6 +765,41 @@ func TestIsPrintableCreateTableSQLAcceptsExternalTable(t *testing.T) { assert.True(t, isPrintableCreateTableSQL("CREATE EXTERNAL TABLE ext_csv (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}")) } +func TestIsPrintableExternalParamJSON(t *testing.T) { + assert.True(t, isPrintableExternalParamJSON(`{"ExParamConst":{"Option":["filepath","/tmp/ext.csv","format","csv"]}}`)) + assert.False(t, isPrintableExternalParamJSON(`CREATE EXTERNAL TABLE ext_csv (id INT)`)) +} + +func TestFindTableSchemaFromMoTablesUsesCatalogLayoutFallback(t *testing.T) { + schema := schemaForLayout(currentCatalogLayout, moTablesID) + headers := append([]string{}, logicalTableViewMetaHeaders...) + for range schema { + headers = append(headers, "") + } + row := make([]string, len(headers)) + row[0], row[1], row[2] = "obj", "0", "0" + data := row[logicalViewMetaCols:] + for i, name := range schema { + switch name { + case "rel_id": + data[i] = "272746" + case "relname": + data[i] = "ext_csv_local" + case "reldatabase": + data[i] = "ckp25010_external" + case "rel_createsql": + data[i] = `{"ExParamConst":{"Option":["filepath","/tmp/ext.csv","format","csv"]}}` + } + } + + got := findTableSchemaFromMoTables(&LogicalTableView{Headers: headers, Rows: [][]string{row}}, 272746) + require.NotNil(t, got) + assert.Equal(t, "ext_csv_local", got.TableName) + assert.Equal(t, "ckp25010_external", got.DatabaseName) + assert.Contains(t, got.CreateSQL, "ExParamConst") + assert.Contains(t, got.CreateSQL, "filepath") +} + func TestRenderCreateTableDDLFromSchema_EnumSetValues(t *testing.T) { ddl := RenderCreateTableDDLFromSchema(&TableSchema{ TableName: "t_enum_set", From 4ad067322e6df0d612026c94b41276ddd0512048 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 14:42:52 +0800 Subject: [PATCH 54/76] fix ckp restore ddl for fulltext enum set --- cmd/mo-object-tool/ckp/checkpoint.go | 15 +++- cmd/mo-object-tool/ckp/checkpoint_test.go | 34 ++++++++ pkg/tools/checkpointtool/table_dump.go | 87 ++++++++++++++++++--- pkg/tools/checkpointtool/table_dump_test.go | 10 +-- 4 files changed, 129 insertions(+), 17 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index b759485a9ffd7..8e72db17d577a 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -1326,6 +1326,9 @@ func alterTableAddClause(sql string) (string, bool) { _, hasUnique := consumeSQLKeyword(clause, 0, "unique") _, hasKey := consumeSQLKeyword(clause, 0, "key") _, hasIndex := consumeSQLKeyword(clause, 0, "index") + if _, hasFullText := consumeSQLKeyword(clause, 0, "fulltext"); hasFullText { + hasKey = true + } if !hasUnique && !hasKey && !hasIndex { return "", false } @@ -1350,7 +1353,15 @@ func injectCreateTableClauses(createDDL string, clauses []string) (string, error if multiline { indent := inferCreateTableClauseIndent(body) insert := ",\n" + indent + strings.Join(clauses, ",\n"+indent) - return createDDL[:close] + insert + createDDL[close:], nil + insertAt := close + for insertAt > open+1 { + ch := createDDL[insertAt-1] + if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' { + break + } + insertAt-- + } + return createDDL[:insertAt] + insert + createDDL[insertAt:], nil } separator := ", " if strings.TrimSpace(body) == "" { @@ -1479,7 +1490,7 @@ func filterExistingIndexDDLs(createDDL string, indexDDLs []string) []string { func generatedIndexDDLName(indexDDL string) string { upper := strings.ToUpper(indexDDL) - for _, marker := range []string{" ADD UNIQUE KEY ", " ADD KEY ", " ADD INDEX "} { + for _, marker := range []string{" ADD FULLTEXT KEY ", " ADD FULLTEXT INDEX ", " ADD UNIQUE KEY ", " ADD KEY ", " ADD INDEX "} { i := strings.Index(upper, marker) if i < 0 { continue diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 909d6a37d5386..7bae444b129b6 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -216,6 +216,40 @@ func TestMergeCreateTableIndexDDLs(t *testing.T) { assert.Equal(t, want, got) } +func TestMergeCreateTableIndexDDLsFullText(t *testing.T) { + createDDL := "CREATE TABLE `ckp_constraints`.`t_fulltext` (\n" + + " `id` BIGINT NOT NULL,\n" + + " `doc` TEXT DEFAULT NULL,\n" + + " PRIMARY KEY (`id`)\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `t_fulltext` ADD FULLTEXT KEY `idx_doc`(`doc`);", + } + want := "CREATE TABLE `ckp_constraints`.`t_fulltext` (\n" + + " `id` BIGINT NOT NULL,\n" + + " `doc` TEXT DEFAULT NULL,\n" + + " PRIMARY KEY (`id`),\n" + + " FULLTEXT KEY `idx_doc`(`doc`)\n" + + ")" + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + assert.NoError(t, err) + assert.Equal(t, want, got) +} + +func TestFilterExistingIndexDDLsFullText(t *testing.T) { + createDDL := "CREATE TABLE `ckp_constraints`.`t_fulltext` (\n" + + " `id` BIGINT NOT NULL,\n" + + " `doc` TEXT DEFAULT NULL,\n" + + " FULLTEXT KEY `idx_doc`(`doc`)\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `t_fulltext` ADD FULLTEXT KEY `idx_doc`(`doc`);", + } + + assert.Empty(t, filterExistingIndexDDLs(createDDL, indexDDLs)) +} + func TestMergeCreateTableIndexDDLsSingleLine(t *testing.T) { createDDL := "CREATE TABLE `ann`.`items_sift` (id int primary key, embedding vecf32(128))" indexDDLs := []string{ diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 3f664630179df..6e96d23f6e6f7 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -485,12 +485,17 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri continue } sb.WriteString(" ") - if key.Unique { + fullText := isFullTextKey(key) + if fullText { + sb.WriteString("FULLTEXT ") + } else if key.Unique { sb.WriteString("UNIQUE ") } sb.WriteString("KEY ") sb.WriteString(quoteDDLIdent(key.Name)) - appendIndexAlgorithmDDL(&sb, key.Algo) + if !fullText { + appendIndexAlgorithmDDL(&sb, key.Algo) + } sb.WriteString("(") for i, col := range key.Columns { if i > 0 { @@ -541,7 +546,7 @@ func renderColumnSQLType(col TableColumn) string { switch { case upperType == "ENUM": return renderEnumSetSQLType("ENUM", enumValues) - case upperType == "SET": + case upperType == "SET", upperType == "BIGINT UNSIGNED": return renderEnumSetSQLType("SET", enumValues) default: return sqlType @@ -558,9 +563,71 @@ func renderEnumSetSQLType(kind string, enumValues string) string { return values } if strings.HasPrefix(values, "(") { - return kind + values + return kind + renderEnumSetValueList(values) + } + return kind + renderEnumSetValueList(values) +} + +func renderEnumSetValueList(values string) string { + parts := splitEnumSetValues(stripEnumSetValueParens(values)) + quoted := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if isQuotedSQLString(part) { + quoted = append(quoted, part) + continue + } + quoted = append(quoted, quoteDDLString(part)) + } + return "(" + strings.Join(quoted, ",") + ")" +} + +func stripEnumSetValueParens(values string) string { + values = strings.TrimSpace(values) + if len(values) >= 2 && values[0] == '(' && values[len(values)-1] == ')' { + return strings.TrimSpace(values[1 : len(values)-1]) + } + return values +} + +func splitEnumSetValues(values string) []string { + var parts []string + start := 0 + inString := byte(0) + for i := 0; i < len(values); i++ { + ch := values[i] + if inString != 0 { + if ch == '\\' && i+1 < len(values) { + i++ + continue + } + if ch == inString { + if i+1 < len(values) && values[i+1] == inString { + i++ + continue + } + inString = 0 + } + continue + } + switch ch { + case '\'', '"': + inString = ch + case ',': + parts = append(parts, values[start:i]) + start = i + 1 + } } - return kind + "(" + values + ")" + parts = append(parts, values[start:]) + return parts +} + +func isQuotedSQLString(s string) bool { + s = strings.TrimSpace(s) + return len(s) >= 2 && ((s[0] == '\'' && s[len(s)-1] == '\'') || (s[0] == '"' && s[len(s)-1] == '"')) } func appendColumnDDLAttributes(sb *strings.Builder, col TableColumn) { @@ -3938,11 +4005,7 @@ func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, e } else if strings.EqualFold(info.indexType, "UNIQUE") { sb.WriteString("UNIQUE ") } - if fullText { - sb.WriteString("INDEX ") - } else { - sb.WriteString("KEY ") - } + sb.WriteString("KEY ") sb.WriteString(quoteDDLIdent(info.name)) if !fullText { appendIndexAlgorithmDDL(&sb, info.algo) @@ -3969,6 +4032,10 @@ func isFullTextIndex(info *indexDDLInfo) bool { return catalog.IsFullTextIndexAlgo(info.algo) || strings.EqualFold(info.indexType, "FULLTEXT") } +func isFullTextKey(key TableUniqueKey) bool { + return catalog.IsFullTextIndexAlgo(key.Algo) +} + func appendIndexAlgorithmDDL(sb *strings.Builder, algo string) { algo = strings.TrimSpace(algo) if !catalog.IsNullIndexAlgo(algo) { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 647365c15283a..220226e42e1a8 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -804,13 +804,13 @@ func TestRenderCreateTableDDLFromSchema_EnumSetValues(t *testing.T) { ddl := RenderCreateTableDDLFromSchema(&TableSchema{ TableName: "t_enum_set", Columns: []TableColumn{ - {Name: "c_enum", SQLType: "ENUM", EnumValues: "'red','blue'", Position: 1}, - {Name: "c_set", SQLType: "SET", EnumValues: "('x','y')", Position: 2}, + {Name: "c_enum", SQLType: "ENUM", EnumValues: "red,green,blue", Position: 1}, + {Name: "c_set", SQLType: "BIGINT UNSIGNED", EnumValues: "a,b,c", Position: 2}, }, }) - assert.Contains(t, ddl, "`c_enum` ENUM('red','blue')") - assert.Contains(t, ddl, "`c_set` SET('x','y')") + assert.Contains(t, ddl, "`c_enum` ENUM('red','green','blue')") + assert.Contains(t, ddl, "`c_set` SET('a','b','c')") } func TestDecodeMoColumnEncodedSQLType_TemporalScale(t *testing.T) { @@ -1123,7 +1123,7 @@ func TestBuildCreateIndexStatementsFromMoIndexes_FullText(t *testing.T) { stmts, err := buildCreateIndexStatementsFromMoIndexes(view, 272535, "t_fulltext") require.NoError(t, err) require.Equal(t, []string{ - "ALTER TABLE `t_fulltext` ADD FULLTEXT INDEX `idx_doc`(`doc`);", + "ALTER TABLE `t_fulltext` ADD FULLTEXT KEY `idx_doc`(`doc`);", }, stmts) } From 1cd8ecefa494c61675c76c7ef0cca87608a8b511 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 15:11:35 +0800 Subject: [PATCH 55/76] support year in load data external parser --- pkg/sql/colexec/external/external.go | 11 +++- pkg/sql/colexec/external/external_test.go | 65 +++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/pkg/sql/colexec/external/external.go b/pkg/sql/colexec/external/external.go index 232692136a4f0..89ed150a4134c 100644 --- a/pkg/sql/colexec/external/external.go +++ b/pkg/sql/colexec/external/external.go @@ -1381,6 +1381,15 @@ func getColData(bat *batch.Batch, line []csvparser.Field, rowIdx int, param *Ext if err := vector.AppendFixed(vec, d, false, mp); err != nil { return err } + case types.T_year: + d, err := types.ParseMoYear(field.Val) + if err != nil { + logutil.Errorf("parse field[%v] err:%v", field.Val, err) + return moerr.NewInternalErrorf(param.Ctx, "the input value '%v' is not Year type for column %d", field.Val, colIdx) + } + if err := vector.AppendFixed(vec, d, false, mp); err != nil { + return err + } case types.T_uuid: d, err := types.ParseUuid(field.Val) if err != nil { @@ -1391,7 +1400,7 @@ func getColData(bat *batch.Batch, line []csvparser.Field, rowIdx int, param *Ext return err } default: - return moerr.NewInternalErrorf(param.Ctx, "the value type %d is not support now", param.Cols[rowIdx].Typ.Id) + return moerr.NewInternalErrorf(param.Ctx, "the value type %d is not support now", param.Cols[colIdx].Typ.Id) } return nil } diff --git a/pkg/sql/colexec/external/external_test.go b/pkg/sql/colexec/external/external_test.go index 47e9e48f411a5..719a562cb956b 100644 --- a/pkg/sql/colexec/external/external_test.go +++ b/pkg/sql/colexec/external/external_test.go @@ -1128,6 +1128,71 @@ func Test_getMOCSVReader(t *testing.T) { require.Equal(t, nil, err) } +func TestGetColDataYear(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + + bat := batch.NewWithSize(1) + bat.Vecs[0] = vector.NewVec(types.T_year.ToType()) + defer bat.Clean(proc.Mp()) + + param := &ExternalParam{ + ExParamConst: ExParamConst{ + Cols: []*plan.ColDef{{ + Name: "c_year", + Typ: plan.Type{Id: int32(types.T_year)}, + }}, + Ctx: context.Background(), + Extern: &tree.ExternParam{}, + }, + } + + err := getColData( + bat, + []csvparser.Field{{Val: "2024"}}, + 0, + param, + proc.Mp(), + plan.ExternAttr{ColName: "c_year", ColIndex: 0, ColFieldIndex: 0}, + proc, + ) + require.NoError(t, err) + require.Equal(t, []types.MoYear{2024}, vector.MustFixedColWithTypeCheck[types.MoYear](bat.Vecs[0])) +} + +func TestGetColDataUnsupportedTypeReportsColumnIndex(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + + bat := batch.NewWithSize(2) + bat.Vecs[0] = vector.NewVec(types.T_int32.ToType()) + bat.Vecs[1] = vector.NewVec(types.T_TS.ToType()) + defer bat.Clean(proc.Mp()) + + param := &ExternalParam{ + ExParamConst: ExParamConst{ + Cols: []*plan.ColDef{ + {Name: "id", Typ: plan.Type{Id: int32(types.T_int32)}}, + {Name: "unsupported", Typ: plan.Type{Id: int32(types.T_TS)}}, + }, + Ctx: context.Background(), + Extern: &tree.ExternParam{}, + }, + } + + err := getColData( + bat, + []csvparser.Field{{Val: "1"}}, + 0, + param, + proc.Mp(), + plan.ExternAttr{ColName: "unsupported", ColIndex: 1, ColFieldIndex: 0}, + proc, + ) + require.Error(t, err) + require.Contains(t, err.Error(), "the value type 100 is not support now") +} + // Test_getColData_VecDimensionCheck tests that getColData correctly validates // vector dimension against the column definition vecf32(N)/vecf64(N) during LOAD DATA. // This is the regression test for the external.go part of https://github.com/matrixorigin/matrixone/issues/23872 From 86e50a03e139a3c12de70ecdb6885afcc3b6a340 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 15:44:13 +0800 Subject: [PATCH 56/76] restore foreign keys in ckp table ddl --- pkg/tools/checkpointtool/table_dump.go | 328 +++++++++++++++++++- pkg/tools/checkpointtool/table_dump_test.go | 74 +++++ 2 files changed, 393 insertions(+), 9 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 6e96d23f6e6f7..b3f995e0381a9 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -126,6 +126,16 @@ type TableUniqueKey struct { Comment string } +type TableForeignKey struct { + Name string + Columns []string + ReferDatabase string + ReferTable string + ReferColumns []string + OnDelete plan.ForeignKeyDef_RefAction + OnUpdate plan.ForeignKeyDef_RefAction +} + // TableSchema holds the decoded schema for one user table. type TableSchema struct { TableName string @@ -135,6 +145,7 @@ type TableSchema struct { Comment string UniqueKeys []TableUniqueKey PrimaryKey []string + ForeignKeys []TableForeignKey Partition string } @@ -439,6 +450,10 @@ func renderCreateTableDDLWithComment(tableName string, cols []TableColumn, comme } func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment string, partition string, primaryKey []string, uniqueKeys ...TableUniqueKey) string { + return renderCreateTableDDLFullWithForeignKeys(tableName, cols, comment, partition, primaryKey, uniqueKeys, nil) +} + +func renderCreateTableDDLFullWithForeignKeys(tableName string, cols []TableColumn, comment string, partition string, primaryKey []string, uniqueKeys []TableUniqueKey, foreignKeys []TableForeignKey) string { if tableName == "" || len(cols) == 0 { return "" } @@ -446,6 +461,12 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri var sb strings.Builder primaryCols := primaryKeyColumns(cols, primaryKey) uniqueKeys = normalizedUniqueKeys(uniqueKeys) + foreignKeys = normalizedForeignKeys(foreignKeys) + extraClauses := 0 + if len(primaryCols) > 0 { + extraClauses++ + } + extraClauses += len(uniqueKeys) + len(foreignKeys) sb.WriteString("CREATE TABLE ") sb.WriteString(quoteDDLIdent(tableName)) sb.WriteString(" (\n") @@ -460,7 +481,7 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri } } appendColumnDDLAttributes(&sb, col) - if i < len(cols)-1 || len(primaryCols) > 0 || len(uniqueKeys) > 0 { + if i < len(cols)-1 || extraClauses > 0 { sb.WriteString(",\n") } else { sb.WriteString("\n") @@ -474,7 +495,7 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri } sb.WriteString(quoteDDLIdent(col.Name)) } - if len(uniqueKeys) > 0 { + if len(uniqueKeys) > 0 || len(foreignKeys) > 0 { sb.WriteString("),\n") } else { sb.WriteString(")\n") @@ -507,7 +528,16 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri if err := appendIndexTrailingOptionsDDL(&sb, key.AlgoParams, key.Comment); err != nil { ckpDebugSchemaf("mo_tables constraint index options skipped index=%s algo=%q params=%q err=%v", key.Name, key.Algo, key.AlgoParams, err) } - if i < len(uniqueKeys)-1 { + if i < len(uniqueKeys)-1 || len(foreignKeys) > 0 { + sb.WriteString(",\n") + } else { + sb.WriteString("\n") + } + } + for i, key := range foreignKeys { + sb.WriteString(" ") + sb.WriteString(renderForeignKeyClause(key)) + if i < len(foreignKeys)-1 { sb.WriteString(",\n") } else { sb.WriteString("\n") @@ -743,6 +773,98 @@ func normalizedUniqueKeys(keys []TableUniqueKey) []TableUniqueKey { return out } +func normalizedForeignKeys(keys []TableForeignKey) []TableForeignKey { + seen := make(map[string]struct{}, len(keys)) + out := make([]TableForeignKey, 0, len(keys)) + for _, key := range keys { + cols := normalizedNameList(key.Columns) + referCols := normalizedNameList(key.ReferColumns) + if len(cols) == 0 || len(cols) != len(referCols) || strings.TrimSpace(key.ReferTable) == "" { + continue + } + name := strings.TrimSpace(key.Name) + if name == "" { + name = cols[0] + } + signature := strings.ToLower(name) + "\x00" + strings.ToLower(strings.Join(cols, "\x00")) + "\x00" + + strings.ToLower(key.ReferDatabase) + "\x00" + strings.ToLower(key.ReferTable) + "\x00" + + strings.ToLower(strings.Join(referCols, "\x00")) + if _, ok := seen[signature]; ok { + continue + } + seen[signature] = struct{}{} + out = append(out, TableForeignKey{ + Name: name, + Columns: cols, + ReferDatabase: strings.TrimSpace(key.ReferDatabase), + ReferTable: strings.TrimSpace(key.ReferTable), + ReferColumns: referCols, + OnDelete: key.OnDelete, + OnUpdate: key.OnUpdate, + }) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Name != out[j].Name { + return out[i].Name < out[j].Name + } + return strings.Join(out[i].Columns, "\x00") < strings.Join(out[j].Columns, "\x00") + }) + return out +} + +func normalizedNameList(names []string) []string { + out := make([]string, 0, len(names)) + for _, name := range names { + name = strings.TrimSpace(name) + if name != "" && !catalog.IsAlias(name) { + out = append(out, name) + } + } + return out +} + +func renderForeignKeyClause(key TableForeignKey) string { + var sb strings.Builder + sb.WriteString("CONSTRAINT ") + sb.WriteString(quoteDDLIdent(key.Name)) + sb.WriteString(" FOREIGN KEY (") + appendDDLIdentList(&sb, key.Columns) + sb.WriteString(") REFERENCES ") + sb.WriteString(quoteDDLIdent(key.ReferTable)) + sb.WriteString(" (") + appendDDLIdentList(&sb, key.ReferColumns) + sb.WriteString(")") + sb.WriteString(" ON DELETE ") + sb.WriteString(renderFKAction(key.OnDelete)) + sb.WriteString(" ON UPDATE ") + sb.WriteString(renderFKAction(key.OnUpdate)) + return sb.String() +} + +func appendDDLIdentList(sb *strings.Builder, names []string) { + for i, name := range names { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(name)) + } +} + +func renderFKAction(action plan.ForeignKeyDef_RefAction) string { + switch action { + case plan.ForeignKeyDef_CASCADE: + return "CASCADE" + case plan.ForeignKeyDef_SET_NULL: + return "SET NULL" + case plan.ForeignKeyDef_SET_DEFAULT: + return "SET DEFAULT" + case plan.ForeignKeyDef_NO_ACTION: + return "NO ACTION" + default: + return "RESTRICT" + } +} + func formatDDLDefault(defaultExpr string) string { defaultExpr = strings.TrimSpace(defaultExpr) if defaultExpr == "" { @@ -783,7 +905,7 @@ func RenderCreateTableDDLFromSchema(schema *TableSchema) string { return "" } } - return renderCreateTableDDLFull(schema.TableName, schema.Columns, schema.Comment, schema.Partition, schema.PrimaryKey, schema.UniqueKeys...) + return renderCreateTableDDLFullWithForeignKeys(schema.TableName, schema.Columns, schema.Comment, schema.Partition, schema.PrimaryKey, schema.UniqueKeys, schema.ForeignKeys) } func inferBuiltinCatalogLayout( @@ -943,6 +1065,9 @@ func (r *CheckpointReader) ReadTableSchema( if len(cols) > 0 { schema.Columns = cols } + if moTablesView != nil { + schema.ForeignKeys = findForeignKeysFromCatalogViews(moTablesView, moColumnsView, tableID) + } } if len(schema.Columns) == 0 { @@ -2969,23 +3094,132 @@ func findPrimaryKeyFromMoTables(view *LogicalTableView, tableID uint64) []string return nil } +func findForeignKeysFromCatalogViews(moTablesView, moColumnsView *LogicalTableView, tableID uint64) []TableForeignKey { + if moTablesView == nil || moColumnsView == nil { + return nil + } + tableByID := mapTablesByID(moTablesView) + colNamesByTableID := mapColumnNamesByTableIDAndColID(moColumnsView) + constraint := findMoTablesConstraint(moTablesView, tableID) + return decodeForeignKeysFromMoTablesConstraint(constraint, tableID, tableByID, colNamesByTableID) +} + +func mapTablesByID(view *LogicalTableView) map[uint64]TableCatalogEntry { + entries := buildCatalogTablesFromMoTablesRows(view) + out := make(map[uint64]TableCatalogEntry, len(entries)) + for _, entry := range entries { + out[entry.TableID] = entry + } + return out +} + +func mapColumnNamesByTableIDAndColID(view *LogicalTableView) map[uint64]map[uint64]string { + result := make(map[uint64]map[uint64]string) + add := func(relIDCol, uniqNameCol, nameCol, hiddenCol int) { + if relIDCol < 0 || uniqNameCol < 0 || nameCol < 0 { + return + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || uniqNameCol >= len(row) || nameCol >= len(row) { + continue + } + if hiddenCol >= 0 && hiddenCol < len(row) && isTruthyCatalogValue(row[hiddenCol]) { + continue + } + tableID, ok := parseUintCell(row[relIDCol]) + if !ok { + continue + } + colID, err := strconv.ParseUint(strings.TrimSpace(row[uniqNameCol]), 10, 64) + if err != nil { + continue + } + name := strings.TrimSpace(row[nameCol]) + if name == "" || catalog.IsAlias(name) { + continue + } + if result[tableID] == nil { + result[tableID] = make(map[uint64]string) + } + result[tableID][colID] = name + } + } + + add( + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_RelID), + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_UniqName), + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Name), + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsHidden), + ) + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { + add( + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_RelID, match.offset), + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_UniqName, match.offset), + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Name, match.offset), + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_IsHidden, match.offset), + ) + } + return result +} + +func findMoTablesConstraint(view *LogicalTableView, tableID uint64) string { + tableIDStr := fmt.Sprintf("%d", tableID) + try := func(relIDCol, constraintCol int) string { + if relIDCol < 0 || constraintCol < 0 { + return "" + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || constraintCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr { + return row[constraintCol] + } + } + return "" + } + if raw := try( + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_ID), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint), + ); raw != "" { + return raw + } + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { + if raw := try( + catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_ID, match.offset), + catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_Constraint, match.offset), + ); raw != "" { + return raw + } + } + return "" +} + func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView *LogicalTableView, partitionMetadataView ...*LogicalTableView) string { tableName := "" tableComment := "" uniqueKeys := []TableUniqueKey(nil) primaryKey := []string(nil) + foreignKeys := []TableForeignKey(nil) partition := "" if moTablesView != nil { tableName = findTableNameFromMoTables(moTablesView, tableID) tableComment = findTableCommentFromMoTables(moTablesView, tableID) uniqueKeys = findUniqueKeysFromMoTables(moTablesView, tableID) primaryKey = findPrimaryKeyFromMoTables(moTablesView, tableID) + if moColumnsView != nil { + foreignKeys = findForeignKeysFromCatalogViews(moTablesView, moColumnsView, tableID) + } } if len(partitionMetadataView) > 0 && partitionMetadataView[0] != nil { partition = buildPartitionClauseFromMetadata(partitionMetadataView[0], tableID) } if moColumnsView != nil { - if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys); ddl != "" { + if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys); ddl != "" { return ddl } } @@ -4159,6 +4393,80 @@ func decodePrimaryKeyFromMoTablesConstraint(raw string) (cols []string) { return nil } +func decodeForeignKeysFromMoTablesConstraint( + raw string, + tableID uint64, + tableByID map[uint64]TableCatalogEntry, + colNamesByTableID map[uint64]map[uint64]string, +) (keys []TableForeignKey) { + if raw == "" { + return nil + } + defer func() { + if r := recover(); r != nil { + ckpDebugSchemaf("mo_tables foreign key decode panic len=%d hex=%s panic=%v", len(raw), debugHexPrefix(raw, 64), r) + keys = nil + } + }() + c := &engine.ConstraintDef{} + if err := c.UnmarshalBinary([]byte(raw)); err != nil { + ckpDebugSchemaf("mo_tables foreign key decode failed len=%d hex=%s err=%v", len(raw), debugHexPrefix(raw, 64), err) + return nil + } + childColsByID := colNamesByTableID[tableID] + for _, ct := range c.Cts { + fkDef, ok := ct.(*engine.ForeignKeyDef) + if !ok || fkDef == nil { + continue + } + for _, fk := range fkDef.Fkeys { + if fk == nil || len(fk.Cols) == 0 || len(fk.Cols) != len(fk.ForeignCols) { + continue + } + parent, ok := tableByID[fk.ForeignTbl] + if !ok || parent.TableName == "" { + continue + } + cols := namesForColumnIDs(childColsByID, fk.Cols) + referCols := namesForColumnIDs(colNamesByTableID[fk.ForeignTbl], fk.ForeignCols) + if len(cols) == 0 || len(cols) != len(referCols) { + continue + } + name := strings.TrimSpace(fk.Name) + if name == "" { + name = cols[0] + } + keys = append(keys, TableForeignKey{ + Name: name, + Columns: cols, + ReferDatabase: parent.DatabaseName, + ReferTable: parent.TableName, + ReferColumns: referCols, + OnDelete: fk.OnDelete, + OnUpdate: fk.OnUpdate, + }) + } + } + keys = normalizedForeignKeys(keys) + ckpDebugSchemaf("mo_tables foreign keys decoded count=%d", len(keys)) + return keys +} + +func namesForColumnIDs(byID map[uint64]string, ids []uint64) []string { + if len(byID) == 0 || len(ids) == 0 { + return nil + } + names := make([]string, 0, len(ids)) + for _, id := range ids { + name := strings.TrimSpace(byID[id]) + if name == "" { + return nil + } + names = append(names, name) + } + return names +} + func quoteDDLIdent(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } @@ -4191,7 +4499,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, table if len(tableNames) > 1 && tableNames[1] != "" { tableComment = tableNames[1] } - return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, "", nil, nil) + return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, "", nil, nil, nil) } func buildCreateTableFromMoColumnsWithOptions( @@ -4202,6 +4510,7 @@ func buildCreateTableFromMoColumnsWithOptions( partition string, primaryKey []string, uniqueKeys []TableUniqueKey, + foreignKeys []TableForeignKey, ) string { relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") @@ -4210,7 +4519,7 @@ func buildCreateTableFromMoColumnsWithOptions( hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -4222,7 +4531,7 @@ func buildCreateTableFromMoColumnsWithOptions( typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -4238,6 +4547,7 @@ func buildCreateTableFromMoColumnsAt( partition string, primaryKey []string, uniqueKeys []TableUniqueKey, + foreignKeys []TableForeignKey, relnameIDCol int, nameCol int, typCol int, @@ -4249,7 +4559,7 @@ func buildCreateTableFromMoColumnsAt( if len(cols) == 0 { return "" } - return renderCreateTableDDLFull(tableName, cols, tableComment, partition, primaryKey, uniqueKeys...) + return renderCreateTableDDLFullWithForeignKeys(tableName, cols, tableComment, partition, primaryKey, uniqueKeys, foreignKeys) } func isPrintableSQLType(sqlType string) bool { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 220226e42e1a8..233311eb3157b 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -618,6 +618,80 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test assert.NotContains(t, ddl, "`old`") } +func TestCreateTableDDLFromCatalogViews_IncludesForeignKeys(t *testing.T) { + moTablesHeaders := append([]string{"object", "block", "row"}, catalog.MoTablesSchema...) + tableRow := func(relID, relName, dbName, dbID, constraint string) []string { + data := make([]string, len(catalog.MoTablesSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoTablesSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_tables header %s", colName) + } + set(catalog.SystemRelAttr_ID, relID) + set(catalog.SystemRelAttr_Name, relName) + set(catalog.SystemRelAttr_DBName, dbName) + set(catalog.SystemRelAttr_DBID, dbID) + set(catalog.SystemRelAttr_Kind, "r") + set(catalog.SystemRelAttr_Constraint, constraint) + return append([]string{"obj", "0", relID}, data...) + } + moTablesView := &LogicalTableView{ + Headers: moTablesHeaders, + Rows: [][]string{ + tableRow("100", "parent", "ckp_constraints", "10", encodedConstraint(t, encodedPrimaryKeyConstraint(t, "id"))), + tableRow("101", "child_cascade", "ckp_constraints", "10", encodedConstraint(t, + &engine.ForeignKeyDef{Fkeys: []*plan.ForeignKeyDef{{ + Name: "fk_child_cascade_parent", + Cols: []uint64{1}, + ForeignTbl: 100, + ForeignCols: []uint64{0}, + OnDelete: plan.ForeignKeyDef_CASCADE, + OnUpdate: plan.ForeignKeyDef_RESTRICT, + }}}, + )), + }, + } + + moColumnsHeaders := append([]string{"object", "block", "row"}, catalog.MoColumnsSchema...) + columnRow := func(relID, relName, colID, name, typ, attnum, notNull string) []string { + data := make([]string, len(catalog.MoColumnsSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoColumnsSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_columns header %s", colName) + } + set(catalog.SystemColAttr_UniqName, colID) + set(catalog.SystemColAttr_RelID, relID) + set(catalog.SystemColAttr_RelName, relName) + set(catalog.SystemColAttr_Name, name) + set(catalog.SystemColAttr_Type, typ) + set(catalog.SystemColAttr_Num, attnum) + set(catalog.SystemColAttr_NullAbility, notNull) + set(catalog.SystemColAttr_IsHidden, "0") + set(catalog.SystemColAttr_Seqnum, attnum) + return append([]string{"obj", "0", attnum}, data...) + } + moColumnsView := &LogicalTableView{ + Headers: moColumnsHeaders, + Rows: [][]string{ + columnRow("100", "parent", "0", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "1"), + columnRow("101", "child_cascade", "0", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "1"), + columnRow("101", "child_cascade", "1", "parent_id", encodedSQLType(t, types.T_int32.ToType()), "2", "0"), + }, + } + + ddl := createTableDDLFromCatalogViews(101, moTablesView, moColumnsView) + assert.Contains(t, ddl, "CONSTRAINT `fk_child_cascade_parent` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT") +} + func TestBuildPartitionClauseFromMetadata_UsesSeqNumsWithHiddenColumn(t *testing.T) { view := &LogicalTableView{ Headers: append([]string{"object", "block", "row"}, "col_0", "col_1", "col_2", "col_3", "col_4", "col_5", "col_6"), From c87d7b7edb77215e6260e87945044bdb86933ded Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 16:02:16 +0800 Subject: [PATCH 57/76] restore views as views in ckp dump --- cmd/mo-object-tool/ckp/checkpoint.go | 72 +++++++++++++++++---- cmd/mo-object-tool/ckp/checkpoint_test.go | 58 +++++++++++++++-- pkg/tools/checkpointtool/table_dump.go | 4 +- pkg/tools/checkpointtool/table_dump_test.go | 4 ++ 4 files changed, 121 insertions(+), 17 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 8e72db17d577a..3f76ad9813f0f 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -814,13 +814,15 @@ func writeRestoreScript( return "", err } } - indexDDLs, err := reader.ShowCreateIndexStatements(ctx, table.TableID, table.TableName, snapshotTS) - if err != nil { - return "", fmt.Errorf("show create indexes for table %d: %w", table.TableID, err) - } - ddl, err = mergeCreateTableIndexDDLs(ddl, indexDDLs) - if err != nil { - return "", fmt.Errorf("merge create table indexes for table %d: %w", table.TableID, err) + if !isViewRelation(table) { + indexDDLs, err := reader.ShowCreateIndexStatements(ctx, table.TableID, table.TableName, snapshotTS) + if err != nil { + return "", fmt.Errorf("show create indexes for table %d: %w", table.TableID, err) + } + ddl, err = mergeCreateTableIndexDDLs(ddl, indexDDLs) + if err != nil { + return "", fmt.Errorf("merge create table indexes for table %d: %w", table.TableID, err) + } } if _, err := fmt.Fprintln(&script, strings.TrimRight(ddl, " ;\n\t")); err != nil { return "", err @@ -885,7 +887,15 @@ func restoreCreateTableDDL( ddl := "" if dumpData != nil && dumpData.Schema != nil { schema := dumpData.Schema - if isExternalRelation(table) { + if isViewRelation(table) { + ddl = strings.TrimSpace(schema.CreateSQL) + if ddl == "" { + return "", fmt.Errorf("view %d (%s.%s): missing CREATE VIEW SQL in checkpoint metadata", table.TableID, table.DatabaseName, table.TableName) + } + if !isCreateViewSQL(ddl) { + return "", fmt.Errorf("view %d (%s.%s): rel_createsql is not CREATE VIEW: %s", table.TableID, table.DatabaseName, table.TableName, summarizeSQLForError(ddl)) + } + } else if isExternalRelation(table) { if strings.TrimSpace(schema.CreateSQL) == "" { return "", fmt.Errorf("external table %d (%s.%s): missing external table parameter JSON in checkpoint metadata", table.TableID, table.DatabaseName, table.TableName) } @@ -1103,7 +1113,7 @@ func byteSQLString(value byte) string { } func shouldWriteLoadData(includeLoad bool, table checkpointtool.TableCatalogEntry) bool { - return includeLoad && !isExternalRelation(table) + return includeLoad && !isExternalRelation(table) && !isViewRelation(table) } func isExternalRelation(table checkpointtool.TableCatalogEntry) bool { @@ -1115,6 +1125,20 @@ func isExternalRelation(table checkpointtool.TableCatalogEntry) bool { } } +func isViewRelation(table checkpointtool.TableCatalogEntry) bool { + switch strings.ToLower(strings.TrimSpace(table.RelKind)) { + case "v", "view": + return true + default: + return false + } +} + +func isCreateViewSQL(ddl string) bool { + upper := strings.ToUpper(strings.TrimSpace(ddl)) + return strings.HasPrefix(upper, "CREATE VIEW ") || strings.HasPrefix(upper, "CREATE OR REPLACE VIEW ") +} + func packageExternalTableSource( ctx context.Context, dumpOut *dumpOutput, @@ -1555,6 +1579,11 @@ func createTableNameRange(sql string) (int, int, bool) { if !ok { return 0, 0, false } + if next, ok := consumeSQLKeyword(sql, i, "or"); ok { + if next, ok = consumeSQLKeyword(sql, next, "replace"); ok { + i = next + } + } for { next, ok := consumeSQLKeyword(sql, i, "temporary") if ok { @@ -1573,11 +1602,24 @@ func createTableNameRange(sql string) (int, int, bool) { } break } - i, ok = consumeSQLKeyword(sql, i, "table") - if !ok { + isTable := false + if next, ok := consumeSQLKeyword(sql, i, "table"); ok { + i = next + isTable = true + } else if next, ok := consumeSQLKeyword(sql, i, "view"); ok { + i = next + } else { return 0, 0, false } - if next, ok := consumeSQLKeyword(sql, i, "if"); ok { + if isTable { + if next, ok := consumeSQLKeyword(sql, i, "if"); ok { + if next, ok = consumeSQLKeyword(sql, next, "not"); ok { + if next, ok = consumeSQLKeyword(sql, next, "exists"); ok { + i = next + } + } + } + } else if next, ok := consumeSQLKeyword(sql, i, "if"); ok { if next, ok = consumeSQLKeyword(sql, next, "not"); ok { if next, ok = consumeSQLKeyword(sql, next, "exists"); ok { i = next @@ -1859,6 +1901,12 @@ func dumpOneTable( outMu *sync.Mutex, ) error { table := plan.table + if isViewRelation(table) { + outMu.Lock() + fmt.Fprintf(out, "View %d %s.%s skipped CSV dump\n", table.TableID, table.DatabaseName, table.TableName) + outMu.Unlock() + return nil + } filePath := tableCSVPath(outputDir, table) tableDir := path.Dir(filePath) if err := dumpOut.MkdirAll(tableDir); err != nil { diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 7bae444b129b6..0e04972145855 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -61,6 +61,16 @@ func TestNormalizeCreateTableDDLName(t *testing.T) { ddl: "CREATE EXTERNAL TABLE old_name (id INT) INFILE {'filepath'='/tmp/data.csv','format'='csv'}", want: "CREATE EXTERNAL TABLE `compat_ckp`.`employees` (id INT) INFILE {'filepath'='/tmp/data.csv','format'='csv'}", }, + { + name: "view", + ddl: "CREATE VIEW old_view AS SELECT id FROM src", + want: "CREATE VIEW `compat_ckp`.`employees` AS SELECT id FROM src", + }, + { + name: "or replace view", + ddl: "CREATE OR REPLACE VIEW `old_db`.`old_view` AS SELECT id FROM src", + want: "CREATE OR REPLACE VIEW `compat_ckp`.`employees` AS SELECT id FROM src", + }, { name: "escaped target name", ddl: "CREATE TABLE old_name (id INT)", @@ -68,9 +78,9 @@ func TestNormalizeCreateTableDDLName(t *testing.T) { }, } - tests[4].want = "CREATE TABLE `compat``ckp`.`employees` (id INT)" - tests[4].ddl = "CREATE TABLE old_name (id INT)" - tests[4].name = "escaped database name" + tests[len(tests)-1].want = "CREATE TABLE `compat``ckp`.`employees` (id INT)" + tests[len(tests)-1].ddl = "CREATE TABLE old_name (id INT)" + tests[len(tests)-1].name = "escaped database name" for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -137,9 +147,49 @@ func TestRestoreCreateTableDDLExternalRequiresCreateSQL(t *testing.T) { assert.Contains(t, err.Error(), "missing external table parameter JSON") } -func TestShouldWriteLoadDataSkipsExternalRelations(t *testing.T) { +func TestRestoreCreateTableDDLUsesViewCreateSQL(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_views", + TableName: "v_normal", + RelKind: "v", + } + dumpData := &checkpointtool.TableDumpData{ + Schema: &checkpointtool.TableSchema{ + TableName: "stale_name", + CreateSQL: "CREATE VIEW `old_db`.`old_view` AS SELECT id, name FROM `ckp_tables`.`t_normal`", + Columns: []checkpointtool.TableColumn{{Name: "id", SQLType: "INT", Position: 1}}, + }, + } + + ddl, err := restoreCreateTableDDL(context.Background(), nil, table, dumpData, types.TS{}) + require.NoError(t, err) + assert.Equal(t, "CREATE VIEW `ckp_views`.`v_normal` AS SELECT id, name FROM `ckp_tables`.`t_normal`", ddl) +} + +func TestRestoreCreateTableDDLViewRequiresCreateViewSQL(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_views", + TableName: "v_normal", + RelKind: "v", + } + dumpData := &checkpointtool.TableDumpData{ + Schema: &checkpointtool.TableSchema{ + TableName: "v_normal", + CreateSQL: "CREATE TABLE `v_normal` (`id` INT)", + Columns: []checkpointtool.TableColumn{{Name: "id", SQLType: "INT", Position: 1}}, + }, + } + + _, err := restoreCreateTableDDL(context.Background(), nil, table, dumpData, types.TS{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "rel_createsql is not CREATE VIEW") +} + +func TestShouldWriteLoadDataSkipsExternalAndViewRelations(t *testing.T) { assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "e"})) assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "external"})) + assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "v"})) + assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "view"})) assert.True(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "r"})) assert.False(t, shouldWriteLoadData(false, checkpointtool.TableCatalogEntry{RelKind: "r"})) } diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index b3f995e0381a9..5164b94b5c1a6 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -4768,7 +4768,9 @@ func isPrintableCreateTableSQL(ddl string) bool { return strings.HasPrefix(upper, "CREATE TABLE ") || strings.HasPrefix(upper, "CREATE TEMPORARY TABLE ") || strings.HasPrefix(upper, "CREATE CLUSTER TABLE ") || - strings.HasPrefix(upper, "CREATE EXTERNAL TABLE ") + strings.HasPrefix(upper, "CREATE EXTERNAL TABLE ") || + strings.HasPrefix(upper, "CREATE VIEW ") || + strings.HasPrefix(upper, "CREATE OR REPLACE VIEW ") } func isPrintableExternalParamJSON(raw string) bool { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 233311eb3157b..973d19d113eb2 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -839,6 +839,10 @@ func TestIsPrintableCreateTableSQLAcceptsExternalTable(t *testing.T) { assert.True(t, isPrintableCreateTableSQL("CREATE EXTERNAL TABLE ext_csv (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}")) } +func TestIsPrintableCreateTableSQLAcceptsView(t *testing.T) { + assert.True(t, isPrintableCreateTableSQL("CREATE VIEW `ckp_views`.`v_normal` AS SELECT id FROM `ckp_tables`.`t_normal`")) +} + func TestIsPrintableExternalParamJSON(t *testing.T) { assert.True(t, isPrintableExternalParamJSON(`{"ExParamConst":{"Option":["filepath","/tmp/ext.csv","format","csv"]}}`)) assert.False(t, isPrintableExternalParamJSON(`CREATE EXTERNAL TABLE ext_csv (id INT)`)) From 1b801eeb9903cac68e8d8211896dcd1f991fed21 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 16:07:48 +0800 Subject: [PATCH 58/76] preserve foreign keys in ckp schema clone --- pkg/tools/checkpointtool/table_dump.go | 24 +++++++++++++++++++++ pkg/tools/checkpointtool/table_dump_test.go | 22 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 5164b94b5c1a6..523305363ff0c 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -1432,6 +1432,30 @@ func cloneTableSchema(schema *TableSchema) *TableSchema { } } } + if len(schema.ForeignKeys) > 0 { + clone.ForeignKeys = make([]TableForeignKey, len(schema.ForeignKeys)) + for i, key := range schema.ForeignKeys { + clone.ForeignKeys[i] = TableForeignKey{ + Name: strings.Clone(key.Name), + ReferDatabase: strings.Clone(key.ReferDatabase), + ReferTable: strings.Clone(key.ReferTable), + OnDelete: key.OnDelete, + OnUpdate: key.OnUpdate, + } + if len(key.Columns) > 0 { + clone.ForeignKeys[i].Columns = make([]string, len(key.Columns)) + for j, col := range key.Columns { + clone.ForeignKeys[i].Columns[j] = strings.Clone(col) + } + } + if len(key.ReferColumns) > 0 { + clone.ForeignKeys[i].ReferColumns = make([]string, len(key.ReferColumns)) + for j, col := range key.ReferColumns { + clone.ForeignKeys[i].ReferColumns[j] = strings.Clone(col) + } + } + } + } return clone } diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 973d19d113eb2..e75a22766f72e 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -692,6 +692,28 @@ func TestCreateTableDDLFromCatalogViews_IncludesForeignKeys(t *testing.T) { assert.Contains(t, ddl, "CONSTRAINT `fk_child_cascade_parent` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT") } +func TestCloneTableSchemaCopiesForeignKeys(t *testing.T) { + schema := &TableSchema{ + TableName: "child_restrict", + Columns: []TableColumn{ + {Name: "id", SQLType: "INT", Position: 1, NotNull: true}, + {Name: "parent_id", SQLType: "INT", Position: 2, NotNull: true}, + }, + PrimaryKey: []string{"id"}, + ForeignKeys: []TableForeignKey{{ + Name: "fk_child_restrict_parent", + Columns: []string{"parent_id"}, + ReferTable: "parent", + ReferColumns: []string{"id"}, + OnDelete: plan.ForeignKeyDef_RESTRICT, + OnUpdate: plan.ForeignKeyDef_RESTRICT, + }}, + } + + ddl := RenderCreateTableDDLFromSchema(cloneTableSchema(schema)) + assert.Contains(t, ddl, "CONSTRAINT `fk_child_restrict_parent` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT") +} + func TestBuildPartitionClauseFromMetadata_UsesSeqNumsWithHiddenColumn(t *testing.T) { view := &LogicalTableView{ Headers: append([]string{"object", "block", "row"}, "col_0", "col_1", "col_2", "col_3", "col_4", "col_5", "col_6"), From e84804fc34f50ac41704f8998fe9c45789c6f85b Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 16:16:33 +0800 Subject: [PATCH 59/76] render unsupported array ddl as json in ckp dump --- pkg/tools/checkpointtool/table_dump.go | 5 ++++- pkg/tools/checkpointtool/table_dump_test.go | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 523305363ff0c..728cf45d635ad 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -568,11 +568,14 @@ func renderCreateTableDDLFullWithForeignKeys(tableName string, cols []TableColum func renderColumnSQLType(col TableColumn) string { sqlType := strings.TrimSpace(col.SQLType) + upperType := strings.ToUpper(sqlType) + if strings.HasPrefix(upperType, "ARRAY(") { + return "JSON" + } enumValues := strings.TrimSpace(col.EnumValues) if enumValues == "" { return sqlType } - upperType := strings.ToUpper(sqlType) switch { case upperType == "ENUM": return renderEnumSetSQLType("ENUM", enumValues) diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index e75a22766f72e..f4fdb3a870df4 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -857,6 +857,19 @@ func TestRenderCreateTableDDLFromSchema_DoesNotFallbackToCreateSQLWhenColumnType assert.Empty(t, ddl) } +func TestRenderCreateTableDDLFromSchema_MapsUnsupportedArrayTypeToJSON(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t_array", + Columns: []TableColumn{ + {Name: "id", SQLType: "INT", Position: 1}, + {Name: "tags", SQLType: "ARRAY(VARCHAR(20))", Position: 2}, + }, + }) + + assert.Contains(t, ddl, "`tags` JSON") + assert.NotContains(t, ddl, "ARRAY(VARCHAR(20))") +} + func TestIsPrintableCreateTableSQLAcceptsExternalTable(t *testing.T) { assert.True(t, isPrintableCreateTableSQL("CREATE EXTERNAL TABLE ext_csv (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}")) } From 0b7a36d73cee5d304f6899e8d6a214e9b11b003b Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 17:05:58 +0800 Subject: [PATCH 60/76] fix ckp restore foreign key ordering --- cmd/mo-object-tool/ckp/checkpoint.go | 65 ++++++++++++++++++++- pkg/tools/checkpointtool/table_dump.go | 30 +++++++--- pkg/tools/checkpointtool/table_dump_test.go | 10 ++-- 3 files changed, 91 insertions(+), 14 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 3f76ad9813f0f..da0233ed0ea34 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -785,7 +785,7 @@ func writeRestoreScript( var script bytes.Buffer currentDB := "" - for _, table := range tables { + for _, table := range orderTablesForRestore(tables, dumpDataByTable) { if table.DatabaseName == "" { return "", fmt.Errorf("table %d has empty database name in checkpoint catalog", table.TableID) } @@ -877,6 +877,69 @@ func writeRestoreScript( return scriptPath, nil } +func orderTablesForRestore( + tables []checkpointtool.TableCatalogEntry, + dumpDataByTable map[uint64]*checkpointtool.TableDumpData, +) []checkpointtool.TableCatalogEntry { + if len(tables) <= 1 { + return tables + } + type tableKey struct { + database string + table string + } + keyFor := func(table checkpointtool.TableCatalogEntry) tableKey { + return tableKey{database: table.DatabaseName, table: table.TableName} + } + tableByKey := make(map[tableKey]checkpointtool.TableCatalogEntry, len(tables)) + orderByID := make(map[uint64]int, len(tables)) + for i, table := range tables { + tableByKey[keyFor(table)] = table + orderByID[table.TableID] = i + } + ordered := make([]checkpointtool.TableCatalogEntry, 0, len(tables)) + visiting := make(map[uint64]bool, len(tables)) + visited := make(map[uint64]bool, len(tables)) + var visit func(checkpointtool.TableCatalogEntry) + visit = func(table checkpointtool.TableCatalogEntry) { + if visited[table.TableID] { + return + } + if visiting[table.TableID] { + return + } + visiting[table.TableID] = true + if dumpData := dumpDataByTable[table.TableID]; dumpData != nil && dumpData.Schema != nil { + refs := append([]checkpointtool.TableForeignKey(nil), dumpData.Schema.ForeignKeys...) + sort.SliceStable(refs, func(i, j int) bool { + left, leftOK := tableByKey[tableKey{database: refs[i].ReferDatabase, table: refs[i].ReferTable}] + right, rightOK := tableByKey[tableKey{database: refs[j].ReferDatabase, table: refs[j].ReferTable}] + if leftOK != rightOK { + return leftOK + } + if !leftOK { + return false + } + return orderByID[left.TableID] < orderByID[right.TableID] + }) + for _, fk := range refs { + parent, ok := tableByKey[tableKey{database: fk.ReferDatabase, table: fk.ReferTable}] + if !ok { + continue + } + visit(parent) + } + } + visiting[table.TableID] = false + visited[table.TableID] = true + ordered = append(ordered, table) + } + for _, table := range tables { + visit(table) + } + return ordered +} + func restoreCreateTableDDL( ctx context.Context, reader *checkpointtool.CheckpointReader, diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 728cf45d635ad..d093b8d54d54e 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -3142,13 +3142,13 @@ func mapTablesByID(view *LogicalTableView) map[uint64]TableCatalogEntry { func mapColumnNamesByTableIDAndColID(view *LogicalTableView) map[uint64]map[uint64]string { result := make(map[uint64]map[uint64]string) - add := func(relIDCol, uniqNameCol, nameCol, hiddenCol int) { - if relIDCol < 0 || uniqNameCol < 0 || nameCol < 0 { + add := func(relIDCol, uniqNameCol, numCol, nameCol, hiddenCol int) { + if relIDCol < 0 || nameCol < 0 { return } for _, fullRow := range view.Rows { row := fullRow[logicalViewDataOffset(view):] - if relIDCol >= len(row) || uniqNameCol >= len(row) || nameCol >= len(row) { + if relIDCol >= len(row) || nameCol >= len(row) { continue } if hiddenCol >= 0 && hiddenCol < len(row) && isTruthyCatalogValue(row[hiddenCol]) { @@ -3158,10 +3158,6 @@ func mapColumnNamesByTableIDAndColID(view *LogicalTableView) map[uint64]map[uint if !ok { continue } - colID, err := strconv.ParseUint(strings.TrimSpace(row[uniqNameCol]), 10, 64) - if err != nil { - continue - } name := strings.TrimSpace(row[nameCol]) if name == "" || catalog.IsAlias(name) { continue @@ -3169,13 +3165,25 @@ func mapColumnNamesByTableIDAndColID(view *LogicalTableView) map[uint64]map[uint if result[tableID] == nil { result[tableID] = make(map[uint64]string) } - result[tableID][colID] = name + addColumnNameID := func(col string) { + colID, err := strconv.ParseUint(strings.TrimSpace(col), 10, 64) + if err == nil { + result[tableID][colID] = name + } + } + if uniqNameCol >= 0 && uniqNameCol < len(row) { + addColumnNameID(row[uniqNameCol]) + } + if numCol >= 0 && numCol < len(row) { + addColumnNameID(row[numCol]) + } } } add( fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_RelID), fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_UniqName), + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Num), fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Name), fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsHidden), ) @@ -3184,6 +3192,7 @@ func mapColumnNamesByTableIDAndColID(view *LogicalTableView) map[uint64]map[uint add( catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_RelID, match.offset), catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_UniqName, match.offset), + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Num, match.offset), catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Name, match.offset), catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_IsHidden, match.offset), ) @@ -4444,19 +4453,24 @@ func decodeForeignKeysFromMoTablesConstraint( for _, ct := range c.Cts { fkDef, ok := ct.(*engine.ForeignKeyDef) if !ok || fkDef == nil { + ckpDebugSchemaf("mo_tables foreign key skip table=%d constraint_type=%T", tableID, ct) continue } + ckpDebugSchemaf("mo_tables foreign key scan table=%d fkeys=%d", tableID, len(fkDef.Fkeys)) for _, fk := range fkDef.Fkeys { if fk == nil || len(fk.Cols) == 0 || len(fk.Cols) != len(fk.ForeignCols) { + ckpDebugSchemaf("mo_tables foreign key skip table=%d invalid fk=%v", tableID, fk) continue } parent, ok := tableByID[fk.ForeignTbl] if !ok || parent.TableName == "" { + ckpDebugSchemaf("mo_tables foreign key skip table=%d missing parent=%d", tableID, fk.ForeignTbl) continue } cols := namesForColumnIDs(childColsByID, fk.Cols) referCols := namesForColumnIDs(colNamesByTableID[fk.ForeignTbl], fk.ForeignCols) if len(cols) == 0 || len(cols) != len(referCols) { + ckpDebugSchemaf("mo_tables foreign key skip table=%d cols=%v resolved=%v foreign_table=%d foreign_cols=%v resolved_foreign=%v child_cols_by_id=%v foreign_cols_by_id=%v", tableID, fk.Cols, cols, fk.ForeignTbl, fk.ForeignCols, referCols, childColsByID, colNamesByTableID[fk.ForeignTbl]) continue } name := strings.TrimSpace(fk.Name) diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index f4fdb3a870df4..470c155285eda 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -646,9 +646,9 @@ func TestCreateTableDDLFromCatalogViews_IncludesForeignKeys(t *testing.T) { tableRow("101", "child_cascade", "ckp_constraints", "10", encodedConstraint(t, &engine.ForeignKeyDef{Fkeys: []*plan.ForeignKeyDef{{ Name: "fk_child_cascade_parent", - Cols: []uint64{1}, + Cols: []uint64{2}, ForeignTbl: 100, - ForeignCols: []uint64{0}, + ForeignCols: []uint64{1}, OnDelete: plan.ForeignKeyDef_CASCADE, OnUpdate: plan.ForeignKeyDef_RESTRICT, }}}, @@ -682,9 +682,9 @@ func TestCreateTableDDLFromCatalogViews_IncludesForeignKeys(t *testing.T) { moColumnsView := &LogicalTableView{ Headers: moColumnsHeaders, Rows: [][]string{ - columnRow("100", "parent", "0", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "1"), - columnRow("101", "child_cascade", "0", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "1"), - columnRow("101", "child_cascade", "1", "parent_id", encodedSQLType(t, types.T_int32.ToType()), "2", "0"), + columnRow("100", "parent", "100-id", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "1"), + columnRow("101", "child_cascade", "101-id", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "1"), + columnRow("101", "child_cascade", "101-parent_id", "parent_id", encodedSQLType(t, types.T_int32.ToType()), "2", "0"), }, } From cdc1ef1d49f257c792d40a042f49ee33131a1503 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 17 Jun 2026 17:13:35 +0800 Subject: [PATCH 61/76] restore partition definitions in ckp dump ddl --- pkg/tools/checkpointtool/table_dump.go | 133 ++++++++++++++++---- pkg/tools/checkpointtool/table_dump_test.go | 37 ++++++ 2 files changed, 146 insertions(+), 24 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index d093b8d54d54e..683f500505f71 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -3792,11 +3792,15 @@ func (r *CheckpointReader) readPartitionClause( tableID uint64, snapshotTS types.TS, ) string { - view, err := r.getPartitionMetadataView(ctx, snapshotTS) - if err != nil || view == nil { + metadataView, err := r.getPartitionMetadataView(ctx, snapshotTS) + if err != nil || metadataView == nil { return "" } - return buildPartitionClauseFromMetadata(view, tableID) + tablesView, err := r.getPartitionTablesView(ctx, snapshotTS) + if err != nil { + tablesView = nil + } + return buildPartitionClauseFromMetadata(metadataView, tableID, tablesView) } func (r *CheckpointReader) dumpCatalogTableViewWithHeaders( @@ -4017,10 +4021,14 @@ func buildCreateIndexStatementsFromMoIndexes( return statements, nil } -func buildPartitionClauseFromMetadata(view *LogicalTableView, tableID uint64) string { +func buildPartitionClauseFromMetadata(view *LogicalTableView, tableID uint64, partitionTablesView ...*LogicalTableView) string { if view == nil { return "" } + var tablesView *LogicalTableView + if len(partitionTablesView) > 0 { + tablesView = partitionTablesView[0] + } tableIDCol := view.columnDataIndex("table_id") methodCol := view.columnDataIndex("partition_method") descriptionCol := view.columnDataIndex("partition_description") @@ -4037,10 +4045,10 @@ func buildPartitionClauseFromMetadata(view *LogicalTableView, tableID uint64) st ) } tableIDStr := strconv.FormatUint(tableID, 10) - if clause := buildPartitionClauseFromMetadataAt(view, tableIDStr, tableIDCol, methodCol, descriptionCol, countCol); clause != "" { + if clause := buildPartitionClauseFromMetadataAt(view, tableIDStr, tableIDCol, methodCol, descriptionCol, countCol, tablesView); clause != "" { return clause } - if clause := buildPartitionClauseFromMetadataByRowShape(view, tableIDStr); clause != "" { + if clause := buildPartitionClauseFromMetadataByRowShape(view, tableIDStr, tablesView); clause != "" { return clause } ckpDebugSchemaf("partition metadata not found table=%d rows=%d headers=%v", tableID, len(view.Rows), view.Headers) @@ -4048,9 +4056,11 @@ func buildPartitionClauseFromMetadata(view *LogicalTableView, tableID uint64) st } type partitionTableRef struct { - id uint64 - ordinal int - name string + id uint64 + ordinal int + name string + partitionName string + expressionText string } func buildPartitionTableIDMap(view *LogicalTableView, primaryTableIDs map[uint64]struct{}) map[uint64][]uint64 { @@ -4059,12 +4069,14 @@ func buildPartitionTableIDMap(view *LogicalTableView, primaryTableIDs map[uint64 return result } partitionIDCol := view.columnDataIndex("partition_id") - partitionNameCol := view.columnDataIndex("partition_table_name") + partitionTableNameCol := view.columnDataIndex("partition_table_name") primaryIDCol := view.columnDataIndex("primary_table_id") + partitionNameCol := view.columnDataIndex("partition_name") ordinalCol := view.columnDataIndex("partition_ordinal_position") + expressionCol := view.columnDataIndex("partition_expression_str") byPrimary := make(map[uint64][]partitionTableRef) if partitionIDCol >= 0 && primaryIDCol >= 0 { - addPartitionTableRefsAt(view, primaryTableIDs, byPrimary, partitionIDCol, primaryIDCol, partitionNameCol, ordinalCol) + addPartitionTableRefsAt(view, primaryTableIDs, byPrimary, partitionIDCol, primaryIDCol, partitionTableNameCol, partitionNameCol, ordinalCol, expressionCol) } if len(byPrimary) == 0 { addPartitionTableRefsByRowShape(view, primaryTableIDs, byPrimary) @@ -4099,7 +4111,12 @@ func buildPartitionTableIDMap(view *LogicalTableView, primaryTableIDs map[uint64 return result } -func addPartitionTableRefsAt(view *LogicalTableView, primaryTableIDs map[uint64]struct{}, out map[uint64][]partitionTableRef, partitionIDCol, primaryIDCol, partitionNameCol, ordinalCol int) { +func addPartitionTableRefsAt( + view *LogicalTableView, + primaryTableIDs map[uint64]struct{}, + out map[uint64][]partitionTableRef, + partitionIDCol, primaryIDCol, partitionTableNameCol, partitionNameCol, ordinalCol, expressionCol int, +) { dataOffset := logicalViewDataOffset(view) for _, row := range view.Rows { if len(row) < dataOffset { @@ -4121,9 +4138,11 @@ func addPartitionTableRefsAt(view *LogicalTableView, primaryTableIDs map[uint64] continue } out[primaryID] = append(out[primaryID], partitionTableRef{ - id: partitionID, - ordinal: parseIntCellDefault(cellAt(dataRow, ordinalCol), len(out[primaryID])), - name: cellAt(dataRow, partitionNameCol), + id: partitionID, + ordinal: parseIntCellDefault(cellAt(dataRow, ordinalCol), len(out[primaryID])), + name: cellAt(dataRow, partitionTableNameCol), + partitionName: cellAt(dataRow, partitionNameCol), + expressionText: cellAt(dataRow, expressionCol), }) } } @@ -4144,22 +4163,26 @@ func addPartitionTableRefsByRowShape(view *LogicalTableView, primaryTableIDs map continue } partitionIDCol := primaryIDCol - 2 - partitionNameCol := primaryIDCol - 1 + partitionTableNameCol := primaryIDCol - 1 + partitionNameCol := primaryIDCol + 1 ordinalCol := primaryIDCol + 2 + expressionCol := primaryIDCol + 3 partitionID, ok := parseUintCell(cellAt(dataRow, partitionIDCol)) if !ok { continue } out[primaryID] = append(out[primaryID], partitionTableRef{ - id: partitionID, - ordinal: parseIntCellDefault(cellAt(dataRow, ordinalCol), len(out[primaryID])), - name: cellAt(dataRow, partitionNameCol), + id: partitionID, + ordinal: parseIntCellDefault(cellAt(dataRow, ordinalCol), len(out[primaryID])), + name: cellAt(dataRow, partitionTableNameCol), + partitionName: cellAt(dataRow, partitionNameCol), + expressionText: cellAt(dataRow, expressionCol), }) } } } -func buildPartitionClauseFromMetadataAt(view *LogicalTableView, tableIDStr string, tableIDCol, methodCol, descriptionCol, countCol int) string { +func buildPartitionClauseFromMetadataAt(view *LogicalTableView, tableIDStr string, tableIDCol, methodCol, descriptionCol, countCol int, partitionTablesView *LogicalTableView) string { if tableIDCol < 0 || methodCol < 0 || descriptionCol < 0 { return "" } @@ -4175,12 +4198,12 @@ func buildPartitionClauseFromMetadataAt(view *LogicalTableView, tableIDStr strin if methodCol >= len(dataRow) || descriptionCol >= len(dataRow) { continue } - return renderPartitionClauseFromMetadataCells(tableIDStr, strings.TrimSpace(dataRow[methodCol]), strings.TrimSpace(dataRow[descriptionCol]), cellAt(dataRow, countCol)) + return renderPartitionClauseFromMetadataCells(tableIDStr, strings.TrimSpace(dataRow[methodCol]), strings.TrimSpace(dataRow[descriptionCol]), cellAt(dataRow, countCol), partitionTablesView) } return "" } -func buildPartitionClauseFromMetadataByRowShape(view *LogicalTableView, tableIDStr string) string { +func buildPartitionClauseFromMetadataByRowShape(view *LogicalTableView, tableIDStr string, partitionTablesView *LogicalTableView) string { dataOffset := logicalViewDataOffset(view) for _, row := range view.Rows { if len(row) < dataOffset { @@ -4197,7 +4220,7 @@ func buildPartitionClauseFromMetadataByRowShape(view *LogicalTableView, tableIDS if descriptionCol >= len(dataRow) { continue } - clause := renderPartitionClauseFromMetadataCells(tableIDStr, strings.TrimSpace(cellAt(dataRow, methodCol)), strings.TrimSpace(cellAt(dataRow, descriptionCol)), cellAt(dataRow, countCol)) + clause := renderPartitionClauseFromMetadataCells(tableIDStr, strings.TrimSpace(cellAt(dataRow, methodCol)), strings.TrimSpace(cellAt(dataRow, descriptionCol)), cellAt(dataRow, countCol), partitionTablesView) if clause != "" { ckpDebugSchemaf("partition metadata resolved by row shape table=%s table_id_col=%d clause=%q", tableIDStr, tableIDCol, clause) return clause @@ -4207,7 +4230,7 @@ func buildPartitionClauseFromMetadataByRowShape(view *LogicalTableView, tableIDS return "" } -func renderPartitionClauseFromMetadataCells(tableIDStr, method, description, countText string) string { +func renderPartitionClauseFromMetadataCells(tableIDStr, method, description, countText string, partitionTablesView *LogicalTableView) string { if description == "" || !isPrintableSQLText(description) { return "" } @@ -4217,10 +4240,63 @@ func renderPartitionClauseFromMetadataCells(tableIDStr, method, description, cou clause += fmt.Sprintf(" partitions %d", count) } } + if definitions := renderPartitionDefinitions(tableIDStr, method, partitionTablesView); definitions != "" { + clause += definitions + } ckpDebugSchemaf("partition metadata resolved table=%s method=%q description=%q clause=%q", tableIDStr, method, description, clause) return clause } +func renderPartitionDefinitions(tableIDStr, method string, view *LogicalTableView) string { + if view == nil || !needsExplicitPartitionDefinitions(method) { + return "" + } + primaryID, err := strconv.ParseUint(strings.TrimSpace(tableIDStr), 10, 64) + if err != nil { + return "" + } + refsByPrimary := make(map[uint64][]partitionTableRef) + primarySet := map[uint64]struct{}{primaryID: {}} + partitionIDCol := view.columnDataIndex("partition_id") + partitionTableNameCol := view.columnDataIndex("partition_table_name") + primaryIDCol := view.columnDataIndex("primary_table_id") + partitionNameCol := view.columnDataIndex("partition_name") + ordinalCol := view.columnDataIndex("partition_ordinal_position") + expressionCol := view.columnDataIndex("partition_expression_str") + if partitionIDCol >= 0 && primaryIDCol >= 0 { + addPartitionTableRefsAt(view, primarySet, refsByPrimary, partitionIDCol, primaryIDCol, partitionTableNameCol, partitionNameCol, ordinalCol, expressionCol) + } + if len(refsByPrimary) == 0 { + addPartitionTableRefsByRowShape(view, primarySet, refsByPrimary) + } + refs := refsByPrimary[primaryID] + if len(refs) == 0 { + return "" + } + sort.Slice(refs, func(i, j int) bool { + if refs[i].ordinal != refs[j].ordinal { + return refs[i].ordinal < refs[j].ordinal + } + if refs[i].partitionName != refs[j].partitionName { + return refs[i].partitionName < refs[j].partitionName + } + return refs[i].id < refs[j].id + }) + parts := make([]string, 0, len(refs)) + for _, ref := range refs { + name := strings.TrimSpace(ref.partitionName) + expr := strings.TrimSpace(ref.expressionText) + if name == "" || expr == "" || !isPrintableSQLText(name) || !isPrintableSQLText(expr) { + continue + } + parts = append(parts, fmt.Sprintf("partition %s %s", quoteDDLIdent(name), expr)) + } + if len(parts) == 0 { + return "" + } + return " (\n " + strings.Join(parts, ",\n ") + "\n)" +} + func cellAt(row []string, idx int) string { if idx < 0 || idx >= len(row) { return "" @@ -4250,6 +4326,15 @@ func isAutoPartitionCountMethod(method string) bool { } } +func needsExplicitPartitionDefinitions(method string) bool { + switch strings.ToLower(strings.TrimSpace(method)) { + case "range", "list": + return true + default: + return false + } +} + func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, error) { if info == nil || tableName == "" || len(info.columns) == 0 { return "", nil diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 470c155285eda..4b80c75a16aea 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -748,6 +748,43 @@ func TestBuildPartitionClauseFromMetadata_FallsBackToRowShape(t *testing.T) { assert.Equal(t, "partition by hash (`id`) partitions 4", buildPartitionClauseFromMetadata(view, 272577)) } +func TestBuildPartitionClauseFromMetadata_IncludesRangePartitionDefinitions(t *testing.T) { + metadataView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionMetadataHeaders...), + Rows: [][]string{ + {"obj1", "0", "0", "272882", "t_range_partition", "ckp_partition_options", "Range", "range (`id`)", "4"}, + }, + } + tablesView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionTablesHeaders...), + Rows: [][]string{ + {"obj1", "0", "0", "272883", "%!%p0%!%t_range_partition", "272882", "p0", "0", "values less than (100)", ""}, + {"obj1", "0", "1", "272884", "%!%p1%!%t_range_partition", "272882", "p1", "1", "values less than (200)", ""}, + {"obj1", "0", "2", "272885", "%!%pmax%!%t_range_partition", "272882", "pmax", "2", "values less than MAXVALUE", ""}, + }, + } + + assert.Equal(t, "partition by range (`id`) (\n partition `p0` values less than (100),\n partition `p1` values less than (200),\n partition `pmax` values less than MAXVALUE\n)", buildPartitionClauseFromMetadata(metadataView, 272882, tablesView)) +} + +func TestBuildPartitionClauseFromMetadata_IncludesListPartitionDefinitions(t *testing.T) { + metadataView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionMetadataHeaders...), + Rows: [][]string{ + {"obj1", "0", "0", "272895", "t_list_columns_partition", "ckp_partition_options", "List", "list columns (`category`)", "3"}, + }, + } + tablesView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionTablesHeaders...), + Rows: [][]string{ + {"obj1", "0", "1", "272897", "%!%p_b%!%t_list_columns_partition", "272895", "p_b", "1", "values in ('b')", ""}, + {"obj1", "0", "0", "272896", "%!%p_a%!%t_list_columns_partition", "272895", "p_a", "0", "values in ('a')", ""}, + }, + } + + assert.Equal(t, "partition by list columns (`category`) (\n partition `p_a` values in ('a'),\n partition `p_b` values in ('b')\n)", buildPartitionClauseFromMetadata(metadataView, 272895, tablesView)) +} + func TestBuildPartitionTableIDMap(t *testing.T) { view := &LogicalTableView{ Headers: append([]string{"object", "block", "row"}, moPartitionTablesHeaders...), From 34058a32fab76f44244b45039814a24bbc549b9c Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 18 Jun 2026 14:22:48 +0800 Subject: [PATCH 62/76] update --- cmd/mo-object-tool/ckp/checkpoint.go | 12 + .../checkpoint_tool_feature_analysis.md | 376 +++++++++++++++--- 2 files changed, 334 insertions(+), 54 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index da0233ed0ea34..18a9c933fe730 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "os/signal" "path" "path/filepath" "runtime/pprof" @@ -28,6 +29,7 @@ import ( "strconv" "strings" "sync" + "syscall" "text/tabwriter" "time" @@ -478,6 +480,16 @@ Examples: if err := pprof.StartCPUProfile(f); err != nil { return fmt.Errorf("start cpuprofile: %w", err) } + + // Ensure CPU profile is flushed on SIGINT (Ctrl+C) + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + pprof.StopCPUProfile() + f.Close() + os.Exit(1) + }() defer pprof.StopCPUProfile() } diff --git a/docs/design/checkpoint_tool_feature_analysis.md b/docs/design/checkpoint_tool_feature_analysis.md index c4738bad70b67..ef3f692b29039 100644 --- a/docs/design/checkpoint_tool_feature_analysis.md +++ b/docs/design/checkpoint_tool_feature_analysis.md @@ -14,22 +14,88 @@ make mo-tool 构建产物为 `./mo-tool`。 +### 1.1 构建注意事项 + +- **从 git worktree 构建**：如果从 `git worktree add --detach` 创建的 detached HEAD 状态构建，需要先在 Makefile 中给 `go build` 加上 `-buildvcs=false`，否则会报 `error obtaining VCS status: exit status 128`。具体修改：在 Makefile 的 `build` 目标中将 `go build` 改为 `go build -buildvcs=false`。 +- **构建 mo-service**：`make build` 生成 `./mo-service`，`make debug` 生成 race-detector 版本。 + --- -## 二、以表为单位 dump +## 二、ckp list — 浏览 checkpoint 目录 + +`ckp list` 用于从 checkpoint 中列出数据库、表、租户等元数据，是 dump 前的关键查询命令。 ### 2.1 命令 ```bash -./mo-tool ckp dump --table-id= [选项] +./mo-tool ckp list [directory] [flags] ``` ### 2.2 参数说明 | 参数 | 必需 | 说明 | |------|------|------| -| `--table-id` | 是 | MatrixOne 内部 table_id，可从 `mo_tables` 系统表或交互式 viewer 中获取 | -| `--ts` | 否 | 快照时间戳。不指定则使用最新的 checkpoint 时间戳 | +| `--type` | 否 | 列出类型：`tables`（默认）、`databases` 或 `accounts` | +| `--account-id` | 否 | 按租户 ID 过滤 | +| `--database-id` | 否 | 按数据库 ID 过滤 | +| `--ts` | 否 | 快照时间戳，默认使用 latest | +| `--include-views` | 否 | 包含视图（默认只列普通表） | + +### 2.3 示例 + +```bash +# 列出所有数据库 +./mo-tool ckp list /path/to/mo-data/shared --type databases + +# 列出指定数据库下的所有表 +./mo-tool ckp list /path/to/mo-data/shared --database-id 272528 + +# 列出所有租户 +./mo-tool ckp list /path/to/mo-data/shared --type accounts + +# 远程 S3/MinIO 下列出 +./mo-tool ckp list --fs-config etc/launch-minio-local/tn.toml --type databases +``` + +--- + +### 2.4 ckp show-create-table — 从 checkpoint 获取 DDL + +从 checkpoint 中获取指定表的完整 CREATE TABLE DDL，无需连接数据库实例即可查看表结构。 + +```bash +./mo-tool ckp show-create-table /path/to/mo-data/shared --table-id 272537 +``` + +输出示例： + +```sql +CREATE TABLE `t_all_wide` ( + `id` INT NOT NULL, + `c_tiny` TINYINT DEFAULT NULL, + ... + PRIMARY KEY (`id`) +); +``` + +支持远程访问：`--fs-config` / `--s3` 参数与 `dump` 命令相同。 + +--- + +## 三、以表为单位 dump + +### 3.1 命令 + +```bash +./mo-tool ckp dump --table-id= [选项] +``` + +### 3.2 参数说明 + +| 参数 | 必需 | 说明 | +|------|------|------| +| `--table-id` | 是 | MatrixOne 内部 table_id，可从 `mo_tables` 系统表或 `ckp list` 中获取 | +| `--ts` | 否 | 快照时间戳。不指定则使用最新的 checkpoint 时间戳。支持格式：`physical:logical`（如 `1781084792256166694:1`）、`physical-logical`、RFC3339、或本地时间格式 | | `-o` / `--output` | 否 | 输出路径。纯 CSV 模式指定文件路径，`--load-script` 模式指定目录（工具自动生成文件名）。不指定则输出到 stdout | | `--header` | 否 | 在 CSV 第一行输出列名头行 | | `--meta-comments` | 否 | 在 CSV 文件头部输出 DDL 和行数统计注释（以 `--` 开头） | @@ -37,7 +103,7 @@ make mo-tool | `--load-script` | 否 | 切换为 LOAD 脚本输出模式：生成包含 CREATE DATABASE + CREATE TABLE + LOAD DATA 的 SQL 文件 | | `--no-load` | 否 | 配合 `--load-script` 使用，跳过 LOAD DATA 语句，只输出 DDL | -### 2.3 示例 +### 3.3 示例 **导出到文件**： @@ -80,7 +146,7 @@ id,name,age,salary,department,hire_date,is_active ... ``` -### 2.4 输出位置 +### 3.4 输出位置 | 模式 | `-o` 含义 | 输出文件 | |------|----------|---------| @@ -91,15 +157,15 @@ id,name,age,salary,department,hire_date,is_active --- -## 三、以 database 为单位 dump +## 四、以 database 为单位 dump -### 3.1 命令 +### 4.1 命令 ```bash ./mo-tool ckp dump --database-id= --output-dir= [选项] ``` -### 3.2 参数说明 +### 4.2 参数说明 | 参数 | 必需 | 说明 | |------|------|------| @@ -112,7 +178,7 @@ id,name,age,salary,department,hire_date,is_active | `--load-script` | 否 | 切换为 LOAD 脚本输出模式：生成单个 SQL 文件，包含 CREATE DATABASE + 所有表的 CREATE TABLE + LOAD DATA | | `--no-load` | 否 | 配合 `--load-script` 使用，跳过 LOAD DATA 语句，只输出 DDL | -### 3.3 示例 +### 4.3 示例 **导出指定 database ID 下的所有表**： @@ -126,7 +192,7 @@ id,name,age,salary,department,hire_date,is_active ./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data ``` -### 3.4 输出目录结构 +### 4.4 输出目录结构 以 database 为单位 dump 时，CSV 文件按以下目录结构组织： @@ -211,7 +277,7 @@ dump_out/ > **注意**：`relkind='v'` 的视图不会被导出。 -### 3.5 命令输出 +### 4.5 命令输出 执行过程中，每成功导出一个表会打印一行信息： @@ -223,15 +289,15 @@ Dumped 2 tables to dump_out --- -## 四、以租户（account）为单位 dump +## 五、以租户（account）为单位 dump -### 4.1 命令 +### 5.1 命令 ```bash ./mo-tool ckp dump --account-id= --output-dir= [选项] ``` -### 4.2 参数说明 +### 5.2 参数说明 | 参数 | 必需 | 说明 | |------|------|------| @@ -244,7 +310,7 @@ Dumped 2 tables to dump_out | `--load-script` | 否 | 切换为 LOAD 脚本输出模式：生成单个 SQL 文件，包含所有 database 的 CREATE DATABASE + CREATE TABLE + LOAD DATA | | `--no-load` | 否 | 配合 `--load-script` 使用，跳过 LOAD DATA 语句，只输出 DDL | -### 4.3 示例 +### 5.3 示例 **导出某个租户的所有表**： @@ -258,7 +324,7 @@ Dumped 2 tables to dump_out ./mo-tool ckp dump --account-id=7 --load-script -o /tmp/ /path/to/mo-data ``` -### 4.4 输出目录结构 +### 5.4 输出目录结构 以租户为单位 dump 时，目录结构与 database 级别一致： @@ -298,7 +364,7 @@ dump_out/ --- -## 五、本地 mo-data +## 六、本地 mo-data 直接传入本地 mo-data 目录路径即可，适用于 checkpoint 数据存储在本地磁盘的场景。 @@ -310,11 +376,11 @@ dump_out/ --- -## 六、远程 S3/MinIO mo-data +## 七、远程 S3/MinIO mo-data 当 checkpoint 数据存储在 S3 或 MinIO 对象存储上时，支持两种远程访问方式。 -### 6.1 方式一：通过 MO 配置文件 +### 7.1 方式一：通过 MO 配置文件 使用 MatrixOne 的 TOML 配置文件（如 `tn.toml`），其中的 `[fileservice]` 段包含对象存储的连接信息： @@ -327,7 +393,7 @@ dump_out/ - `--fs-config`：MO 配置文件路径 - `--fs-name`：使用的 fileservice 名称（默认为 `SHARED`），对应配置中 `[fileservice]` 段的 `name` 字段 -### 6.2 方式二：直接指定 S3 参数 +### 7.2 方式二：直接指定 S3 参数 不依赖 MO 配置文件，直接传入对象存储的连接参数： @@ -352,7 +418,7 @@ S3 参数格式（逗号分隔的 key=value 对）： - `S3`：AWS S3 或兼容 S3 协议的服务（默认） - `MINIO`：MinIO 对象存储 -### 6.3 远程读取机制 +### 7.3 远程读取机制 远程读取使用 `lazyCacheFS` 实现按需缓存： @@ -363,7 +429,7 @@ S3 参数格式（逗号分隔的 key=value 对）： checkpoint 元数据列表（`List` 操作）直接对远程服务执行，不下载全部文件。 -### 6.4 远程 dump 示例 +### 7.4 远程 dump 示例 ```bash # 远程单表 dump @@ -379,11 +445,17 @@ checkpoint 元数据列表（`List` 操作）直接对远程服务执行，不 ./mo-tool ckp info --fs-config etc/launch-minio-local/tn.toml --fs-name SHARED ``` +### 7.5 远程访问已知限制 + +- **MINIO backend 数据文件不完整**：当 MatrixOne 以 MINIO 作为 SHARED fileservice 运行时，checkpoint 元数据文件（`ckp/meta_*.ckp`）正常写入，但部分数据对象文件可能因缓存策略未完全 flush 到 MINIO，导致 dump 时出现 `file is not found` 错误。 +- **建议**：对于本地开发/测试场景，使用 DISK backend（见下文 十五、checkpoint 配置建议），将 SHARED 改为本地目录，确保所有数据文件可被 mo-tool 直接读取。 +- **S3 region 参数**：使用 `--backend MINIO` 时必须显式指定 `region`，否则报 `Invalid region` 错误。例如 `region=us-east-1`。 + --- -## 七、CSV 格式说明 +## 八、CSV 格式说明 -### 7.1 格式规范 +### 8.1 格式规范 导出的 CSV 文件兼容 MySQL/MariaDB/MatrixOne 的 `LOAD DATA` 语句，具体格式： @@ -398,7 +470,7 @@ checkpoint 元数据列表（`List` 操作）直接对远程服务执行，不 | 数值类型 | 不使用引号（INT, BIGINT, FLOAT, DOUBLE, DECIMAL, BOOL 等） | | 日期时间类型 | 不使用引号（DATE, TIME, DATETIME, TIMESTAMP） | -### 7.2 特殊值处理 +### 8.2 特殊值处理 | 场景 | CSV 输出 | |------|----------| @@ -408,7 +480,7 @@ checkpoint 元数据列表（`List` 操作）直接对远程服务执行，不 | 包含逗号的字符串（如 `a,b`） | `"a,b"` | | 包含换行的字符串 | `"a\nb"` | -### 7.3 CSV 输出示例 +### 8.3 CSV 输出示例 ```csv id,name,age,salary,department,hire_date,is_active @@ -420,9 +492,9 @@ id,name,age,salary,department,hire_date,is_active --- -## 八、LOAD DATA 回灌 +## 九、LOAD DATA 回灌 -### 8.1 推荐方式：使用 `--load-script` +### 9.1 推荐方式：使用 `--load-script` `--load-script` 一步生成完整的 SQL 恢复脚本（CREATE DATABASE + CREATE TABLE + LOAD DATA），在目标 MO 实例中直接执行即可： @@ -436,7 +508,7 @@ mysql -h 127.0.0.1 -P 6001 -u root -p111 < /tmp/restore.sql 详见第十一章。 -### 8.2 手动回灌（不使用 `--load-script`） +### 9.2 手动回灌（不使用 `--load-script`） 如果不使用 `--load-script`，手动回灌流程如下： @@ -477,7 +549,7 @@ LOAD DATA 参数对应关系： | 有 header 行 | `IGNORE 1 LINES` | | 无 header 行 | 不需要 `IGNORE 1 LINES` | -### 8.3 数据校验 +### 9.3 数据校验 ```sql -- 对比原始行数和导入行数 @@ -495,9 +567,9 @@ FROM ( --- -## 九、CSV 行排序选项 +## 十、CSV 行排序选项 -### 9.1 storage 顺序（默认） +### 10.1 storage 顺序（默认） ```bash ./mo-tool ckp dump --table-id=272535 --row-order=storage --header -o /tmp/table.csv /path/to/mo-data @@ -507,7 +579,7 @@ FROM ( - 内存占用小，适合大表 - 行顺序与存储布局一致，不可预测 -### 9.2 lexical 顺序 +### 10.2 lexical 顺序 ```bash ./mo-tool ckp dump --table-id=272535 --row-order=lexical --header -o /tmp/table.csv /path/to/mo-data @@ -520,7 +592,7 @@ FROM ( --- -## 十、交互式浏览器 +## 十一、交互式浏览器 除了命令行 dump，`mo-tool ckp` 还提供了交互式 TUI 浏览器，用于探索 checkpoint 内容： @@ -544,9 +616,9 @@ FROM ( --- -## 十一、checkpoint 信息查看 +## 十二、checkpoint 信息查看 -### 11.1 查看 checkpoint 摘要 +### 12.1 查看 checkpoint 摘要 ```bash ./mo-tool ckp info /path/to/mo-data @@ -554,7 +626,7 @@ FROM ( 输出 checkpoint 的条目类型统计和时间范围。 -### 11.2 生成 LOAD 脚本（`--load-script`） +### 12.2 生成 LOAD 脚本（`--load-script`） dump 命令支持 `--load-script` 选项，生成一个可直接在目标 MatrixOne 实例中执行的 SQL 脚本，一行命令即可完成数据恢复。 @@ -622,39 +694,59 @@ IGNORE 1 LINES; --- -## 十二、常用命令速查 +## 十三、常用命令速查 ```bash # 构建 make mo-tool +# === ckp info === # 查看 checkpoint 摘要 -./mo-tool ckp info /path/to/mo-data +./mo-tool ckp info /path/to/mo-data/shared +./mo-tool ckp info --fs-config etc/launch-minio-local/tn.toml # 远程 + +# === ckp list === +# 列出所有数据库 +./mo-tool ckp list /path/to/mo-data/shared --type databases +# 列出指定数据库的表 +./mo-tool ckp list /path/to/mo-data/shared --database-id + +# 列出所有租户 +./mo-tool ckp list /path/to/mo-data/shared --type accounts + +# 远程列出 +./mo-tool ckp list --fs-config etc/launch-minio-local/tn.toml --type databases + +# === ckp show-create-table === +# 查看 checkpoint 中表的 DDL +./mo-tool ckp show-create-table /path/to/mo-data/shared --table-id + +# === ckp dump === # 单表 dump 到文件 -./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv /path/to/mo-data +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv /path/to/mo-data/shared # 单表 dump 到 stdout -./mo-tool ckp dump --table-id=272535 --header /path/to/mo-data +./mo-tool ckp dump --table-id=272535 --header /path/to/mo-data/shared # 指定时间戳单表 dump -./mo-tool ckp dump --table-id=272535 --ts=1781084792256166694:1 --header -o /tmp/table.csv /path/to/mo-data +./mo-tool ckp dump --table-id=272535 --ts=1781084792256166694:1 --header -o /tmp/table.csv /path/to/mo-data/shared # database 级别批量 dump -./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header /path/to/mo-data +./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header /path/to/mo-data/shared # 租户级别批量 dump -./mo-tool ckp dump --account-id=7 --output-dir=./dump_out --header /path/to/mo-data +./mo-tool ckp dump --account-id=7 --output-dir=./dump_out --header /path/to/mo-data/shared # 生成 LOAD 脚本（单表/database/租户） -./mo-tool ckp dump --table-id=272535 --load-script -o /tmp/ /path/to/mo-data -./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data -./mo-tool ckp dump --account-id=7 --load-script -o /tmp/ /path/to/mo-data -./mo-tool ckp dump --database-id=9001 --load-script --no-load -o /tmp/ /path/to/mo-data +./mo-tool ckp dump --table-id=272535 --load-script -o /tmp/ /path/to/mo-data/shared +./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data/shared +./mo-tool ckp dump --account-id=7 --load-script -o /tmp/ /path/to/mo-data/shared +./mo-tool ckp dump --database-id=9001 --load-script --no-load -o /tmp/ /path/to/mo-data/shared # dump 系统表 -./mo-tool ckp dump --table-id=2 -o /tmp/mo_tables.csv /path/to/mo-data # mo_tables -./mo-tool ckp dump --table-id=3 -o /tmp/mo_columns.csv /path/to/mo-data # mo_columns +./mo-tool ckp dump --table-id=2 -o /tmp/mo_tables.csv /path/to/mo-data/shared # mo_tables +./mo-tool ckp dump --table-id=3 -o /tmp/mo_columns.csv /path/to/mo-data/shared # mo_columns # 远程 S3/MinIO dump（方式一：配置文件） ./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ @@ -663,15 +755,18 @@ make mo-tool # 远程 S3/MinIO dump（方式二：直接指定参数） ./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ --backend MINIO \ - --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,region=us-east-1,key-prefix=server/data,key-id=minio,key-secret=minio123 # 交互式浏览器 -./mo-tool ckp view /path/to/mo-data +./mo-tool ckp view /path/to/mo-data/shared + +# === 测试数据准备 === +./prepare_ckp_dump_coverage_data.sh --host 127.0.0.1 --port 6001 --user dump --password 111 --db-prefix ckp --drop-existing ``` --- -## 十三、恢复流程模板 +## 十四、恢复流程模板 ### 推荐方式：`--load-script` 一键恢复 @@ -712,3 +807,176 @@ ENCLOSED BY '"' LINES TERMINATED BY '\n' IGNORE 1 LINES; ``` + +--- + +## 十五、测试数据准备 + +### 15.1 prepare_ckp_dump_coverage_data.sh + +仓库根目录下的 `prepare_ckp_dump_coverage_data.sh` 脚本用于快速生成 checkpoint dump 的覆盖测试数据。 + +**功能**：创建 4 个精心设计的测试 database： + +| Database | 内容 | +|---|---| +| `_types` | 各种数据类型及边界值（有符号/无符号整数、浮点/定点数、bool/bit、字符串/JSON、binary/blob、时间类型、enum/set、uuid、vecf32、array、datalink） | +| `_constraints` | 约束场景（外键 RESTRICT/CASCADE/SET NULL、自增列、复合主键/唯一键、特殊标识符、全文索引、向量索引） | +| `_tables` | 表形态（普通表、空表、view、CTAS、LIKE、hash 分区、key 分区、cluster by、特殊表名） | +| `_mvcc_perf` | DML 历史（insert/update/delete）、truncate、alter add column、大规模行表（默认 10000 行）、32 列宽表 | + +**用法**： + +```bash +# 默认时间戳命名（每次生成不同的 database 名） +./prepare_ckp_dump_coverage_data.sh --host 127.0.0.1 --port 6001 --user dump --password 111 + +# 固定 database 名前缀 + 大规模数据，便于反复测试 +./prepare_ckp_dump_coverage_data.sh \ + --host 127.0.0.1 --port 6001 \ + --user dump --password 111 \ + --db-prefix ckp \ + --scale 10000 \ + --drop-existing +``` + +**主要参数**： + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `--db-prefix` | `ckp_cov_` | database 名前缀 | +| `--scale` | `10000`（最大 `1000000`） | 大规模表的行数 | +| `--drop-existing` | false | 先删除同名 database | +| `--generate-only` | false | 只生成 SQL 文件，不连接数据库执行 | +| `--out-dir` | `/tmp/ckp_dump_coverage_` | SQL 文件输出目录 | + +### 15.2 已知兼容性问题 + +不同 MO 版本对 SQL 的支持不同，以下语法可能在特定版本上失败（脚本中 `/11_types_optional.sql` 和 `/21_indexes_optional.sql` 以 `--force` 模式执行，失败不中断）： + +| 语法 | 说明 | +|------|------| +| `ARRAY(VARCHAR(20))` | 3.0-dev 不支持 ARRAY 类型 | +| `LEAST()` | 3.0-dev 不支持 LEAST 函数 | +| `TEMPORARY TABLE` | 3.0-dev 不支持临时表 | +| `CHAR(n)` 函数 | 3.0-dev 不支持 `CHAR()` 函数，需用 `UNHEX()` 替代 | +| `COUNT(*) AS rows` | `rows` 是 3.0-dev 保留关键字，需改别名 | + +--- + +## 十六、checkpoint 配置建议 + +MO 的 checkpoint 行为由 TN 配置中的 `[tn.Ckp]` 段控制。**测试时**，默认配置可能导致 checkpoint 生成过慢，建议使用以下激进配置： + +```toml +# 默认配置（checkpoint 生成慢，不适合测试） +[tn.Ckp] +flush-interval = "60s" +min-count = 100 +scan-interval = "5s" +incremental-interval = "180s" +global-min-count = 60 + +# 测试推荐配置（checkpoint 快速生成） +[tn.Ckp] +flush-interval = "10s" +min-count = 1 +scan-interval = "2s" +incremental-interval = "30s" +global-min-count = 1 +``` + +**关键参数说明**： + +| 参数 | 说明 | +|------|------| +| `flush-interval` | checkpoint flush 间隔 | +| `min-count` | 触发 flush 的最小 log entry 数量 | +| `incremental-interval` | 增量 checkpoint 间隔 | +| `global-min-count` | 触发全局 checkpoint 的最小 log entry 数量。全局 checkpoint 包含完整的 catalog 快照（mo_tables/mo_columns），是 mo-tool dump 数据的前提 | + +**触发 checkpoint 的方式**： +- 自动触发：满足时间或 log count 阈值 +- 关闭触发：优雅关闭 MO（`SIGTERM`）会触发一次 shutdown checkpoint，将未持久化的数据写入 checkpoint + +**SHARED fileservice backend 选择**： + +| backend | 适用场景 | 说明 | +|---------|---------|------| +| `DISK` | 本地开发/测试 | mo-data 直接存储为本地文件，mo-tool 无需额外配置即可读取 | +| `MINIO` / `S3` | 生产环境 | mo-data 存储在对象存储上，需通过 `--fs-config` 或 `--s3` 参数访问（见 七、远程 S3/MinIO mo-data） | + +本地测试推荐使用 DISK backend 的配置： + +```toml +[[fileservice]] +backend = "DISK" +data-dir = "./etc/launch-minio-local/mo-data/shared" +name = "SHARED" +``` + +--- + +## 十七、版本兼容性 + +### 17.1 view_ckp mo-tool 读取 3.0-dev checkpoint + +| 功能 | 状态 | 说明 | +|------|------|------| +| `ckp info` | ✅ | checkpoint 条目统计和时间范围正常 | +| `ckp list --type databases` | ✅ | 系统和用户 database 全部列出 | +| `ckp list --type tables` | ✅ | 表元数据完整 | +| `ckp show-create-table` | ✅ | DDL 正确还原 | +| `ckp dump`（catalog 表） | ✅ | mo_tables/mo_columns 等系统表 dump 正常 | +| `ckp dump`（用户表） | ⚠️ | CSV header 正常生成，但数据行可能为空（取决于 checkpoint 是否完整包含数据对象） | +| `ckp dump --load-script` | ⚠️ | DDL 部分正常，数据导入可能无数据 | + +### 17.2 数据完整性检查 + +如果 dump 用户表得到 0 行数据（`visible_rows=0`），请检查： + +1. **checkpoint 是否包含数据对象**：在 `ckp info` 中确认 `Global ≥ 1`。如果只有 Incremental 条目且缺少对应的 Global checkpoint，数据对象可能未关联到 catalog。 +2. **checkpoint 时间戳**：使用 `--ts` 指定更早或更晚的 checkpoint 时间戳试试。 +3. **mo-data 文件完整性**：检查 `shared` 目录下是否有与 checkpoint meta 文件中引用的 UUID 对应的数据文件。 +4. **SHARED backend**：使用 MINIO 时可能存在缓存未 flush 问题，建议切换到 DISK backend（见 十六、checkpoint 配置建议）。 + +### 17.3 完整测试流程 + +```bash +# 1. 启动 MO（3.0-dev，使用 DISK SHARED backend + 激进 checkpoint 配置） +cd matrixone-3.0-dev +./mo-service -launch etc/launch-minio-local/launch.toml & + +# 2. 等待 MO 就绪 +mysql -h 127.0.0.1 -P 6001 -u dump -p111 -e "SELECT 1" + +# 3. 生成测试数据 +./prepare_ckp_dump_coverage_data.sh \ + --host 127.0.0.1 --port 6001 --user dump --password 111 \ + --db-prefix ckp --scale 10000 --drop-existing + +# 4. 等待 checkpoint 生成（约 1-2 分钟） +sleep 120 + +# 5. 优雅关闭 MO（触发 shutdown checkpoint） +pkill -TERM mo-service +sleep 10 + +# 6. 查看 checkpoint 信息 +./mo-tool ckp info /path/to/mo-data/shared + +# 7. 列出数据库 +./mo-tool ckp list /path/to/mo-data/shared --type databases + +# 8. 列出某个 database 的表 +./mo-tool ckp list /path/to/mo-data/shared --database-id + +# 9. 查看表 DDL +./mo-tool ckp show-create-table /path/to/mo-data/shared --table-id + +# 10. dump 单表 +./mo-tool ckp dump --table-id --header -o /tmp/table.csv /path/to/mo-data/shared + +# 11. dump 整个 database（含 LOAD 脚本） +./mo-tool ckp dump --database-id --load-script -o /tmp/dump_out /path/to/mo-data/shared +``` From c0d240c344f108dbf1c6acfc8e4ba766ce0e59ee Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 18 Jun 2026 15:06:27 +0800 Subject: [PATCH 63/76] preserve array type in ckp restore ddl --- pkg/tools/checkpointtool/table_dump.go | 3 --- pkg/tools/checkpointtool/table_dump_test.go | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 683f500505f71..2c3ef6189fadf 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -569,9 +569,6 @@ func renderCreateTableDDLFullWithForeignKeys(tableName string, cols []TableColum func renderColumnSQLType(col TableColumn) string { sqlType := strings.TrimSpace(col.SQLType) upperType := strings.ToUpper(sqlType) - if strings.HasPrefix(upperType, "ARRAY(") { - return "JSON" - } enumValues := strings.TrimSpace(col.EnumValues) if enumValues == "" { return sqlType diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 4b80c75a16aea..a45649cddec06 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -894,7 +894,7 @@ func TestRenderCreateTableDDLFromSchema_DoesNotFallbackToCreateSQLWhenColumnType assert.Empty(t, ddl) } -func TestRenderCreateTableDDLFromSchema_MapsUnsupportedArrayTypeToJSON(t *testing.T) { +func TestRenderCreateTableDDLFromSchema_PreservesArrayType(t *testing.T) { ddl := RenderCreateTableDDLFromSchema(&TableSchema{ TableName: "t_array", Columns: []TableColumn{ @@ -903,8 +903,8 @@ func TestRenderCreateTableDDLFromSchema_MapsUnsupportedArrayTypeToJSON(t *testin }, }) - assert.Contains(t, ddl, "`tags` JSON") - assert.NotContains(t, ddl, "ARRAY(VARCHAR(20))") + assert.Contains(t, ddl, "`tags` ARRAY(VARCHAR(20))") + assert.NotContains(t, ddl, "`tags` JSON") } func TestIsPrintableCreateTableSQLAcceptsExternalTable(t *testing.T) { From 0440e9d88d98e8560296456a68e1d8e684a056c2 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 18 Jun 2026 15:40:49 +0800 Subject: [PATCH 64/76] preserve typed array ddl in ckp restore --- pkg/tools/checkpointtool/table_dump.go | 13 +++++++++++++ pkg/tools/checkpointtool/table_dump_test.go | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 2c3ef6189fadf..82322da87e50a 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -574,6 +574,11 @@ func renderColumnSQLType(col TableColumn) string { return sqlType } switch { + case upperType == "JSON": + if arrayType := renderTypedArraySQLType(enumValues); arrayType != "" { + return arrayType + } + return sqlType case upperType == "ENUM": return renderEnumSetSQLType("ENUM", enumValues) case upperType == "SET", upperType == "BIGINT UNSIGNED": @@ -583,6 +588,14 @@ func renderColumnSQLType(col TableColumn) string { } } +func renderTypedArraySQLType(enumValues string) string { + values := strings.TrimSpace(enumValues) + if len(values) < len("array(") || !strings.EqualFold(values[:len("array(")], "array(") { + return "" + } + return "ARRAY" + values[len("array"):] +} + func renderEnumSetSQLType(kind string, enumValues string) string { values := strings.TrimSpace(enumValues) if values == "" { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index a45649cddec06..8cbd2f2e378b0 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -907,6 +907,19 @@ func TestRenderCreateTableDDLFromSchema_PreservesArrayType(t *testing.T) { assert.NotContains(t, ddl, "`tags` JSON") } +func TestRenderCreateTableDDLFromSchema_PreservesTypedArrayMetadata(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t_array", + Columns: []TableColumn{ + {Name: "id", SQLType: "INT", Position: 1}, + {Name: "tags", SQLType: "JSON", EnumValues: "array(varchar(20))", Position: 2}, + }, + }) + + assert.Contains(t, ddl, "`tags` ARRAY(varchar(20))") + assert.NotContains(t, ddl, "`tags` JSON") +} + func TestIsPrintableCreateTableSQLAcceptsExternalTable(t *testing.T) { assert.True(t, isPrintableCreateTableSQL("CREATE EXTERNAL TABLE ext_csv (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}")) } From 075d6cd358b268a17e4108c6b8cfad5dcf6b4120 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 18 Jun 2026 16:00:50 +0800 Subject: [PATCH 65/76] list ckp databases from mo_database --- cmd/mo-object-tool/ckp/checkpoint.go | 10 +- pkg/tools/checkpointtool/table_dump.go | 146 ++++++++++++++++++-- pkg/tools/checkpointtool/table_dump_test.go | 29 ++++ 3 files changed, 171 insertions(+), 14 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 18a9c933fe730..268b5f32d4c46 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -235,11 +235,17 @@ to narrow the result.`, if cmd.Flags().Changed("database-id") { dbFilter = &databaseID } - tables, err := reader.ListCatalogTables(ctx, snapshotTS, checkpointtool.TableListOptions{ + opts := checkpointtool.TableListOptions{ AccountID: accountFilter, DatabaseID: dbFilter, IncludeViews: includeViews, - }) + } + var tables []checkpointtool.TableCatalogEntry + if listType == "databases" || listType == "dbs" { + tables, err = reader.ListCatalogDatabases(ctx, snapshotTS, opts) + } else { + tables, err = reader.ListCatalogTables(ctx, snapshotTS, opts) + } if err != nil { return fmt.Errorf("list checkpoint catalog tables: %w", err) } diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 82322da87e50a..a13c22705e7f9 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -47,8 +47,9 @@ const ( logicalViewMetaCols = 3 // System table IDs - moTablesID = uint64(catalog.MO_TABLES_ID) - moColumnsID = uint64(catalog.MO_COLUMNS_ID) + moDatabaseID = uint64(catalog.MO_DATABASE_ID) + moTablesID = uint64(catalog.MO_TABLES_ID) + moColumnsID = uint64(catalog.MO_COLUMNS_ID) ) const ( @@ -62,9 +63,10 @@ const ( ) type catalogLayout struct { - name string - moTablesSchema []string - moColumnsSchema []string + name string + moDatabaseSchema []string + moTablesSchema []string + moColumnsSchema []string } type catalogLayoutMatch struct { @@ -74,12 +76,14 @@ type catalogLayoutMatch struct { var ( currentCatalogLayout = catalogLayout{ - name: "current", - moTablesSchema: append([]string(nil), catalog.MoTablesSchema...), - moColumnsSchema: append([]string(nil), catalog.MoColumnsSchema...), + name: "current", + moDatabaseSchema: append([]string(nil), catalog.MoDatabaseSchema...), + moTablesSchema: append([]string(nil), catalog.MoTablesSchema...), + moColumnsSchema: append([]string(nil), catalog.MoColumnsSchema...), } preCPKLayout = catalogLayout{ - name: "pre-cpk", + name: "pre-cpk", + moDatabaseSchema: catalogSchemaWithout(catalog.MoDatabaseSchema, catalog.SystemDBAttr_CPKey), moTablesSchema: catalogSchemaWithout( catalog.MoTablesSchema, catalog.SystemRelAttr_ExtraInfo, @@ -91,9 +95,10 @@ var ( ), } legacy3CatalogLayout = catalogLayout{ - name: "3.0-dev", - moTablesSchema: append([]string(nil), catalog.MoTablesSchema[:len(catalog.MoTablesSchema)-1]...), - moColumnsSchema: append([]string(nil), catalog.MoColumnsSchema[:len(catalog.MoColumnsSchema)-2]...), + name: "3.0-dev", + moDatabaseSchema: append([]string(nil), catalog.MoDatabaseSchema...), + moTablesSchema: append([]string(nil), catalog.MoTablesSchema[:len(catalog.MoTablesSchema)-1]...), + moColumnsSchema: append([]string(nil), catalog.MoColumnsSchema[:len(catalog.MoColumnsSchema)-2]...), } ) @@ -303,6 +308,8 @@ func knownCatalogLayouts() []catalogLayout { func schemaForLayout(layout catalogLayout, tableID uint64) []string { switch tableID { + case moDatabaseID: + return layout.moDatabaseSchema case moTablesID: return layout.moTablesSchema case moColumnsID: @@ -1495,6 +1502,23 @@ func (r *CheckpointReader) ListCatalogTables( return filterCatalogTablesForList(tables, opts), nil } +func (r *CheckpointReader) ListCatalogDatabases( + ctx context.Context, + snapshotTS types.TS, + opts TableListOptions, +) ([]TableCatalogEntry, error) { + moDatabaseView, err := r.getTableLogicalView(ctx, moDatabaseID, snapshotTS) + if err != nil { + return nil, err + } + databases := buildCatalogDatabasesFromMoDatabaseRows(moDatabaseView) + if schema := r.ReadTableSchema(ctx, moDatabaseID, snapshotTS, moDatabaseView); len(schema.Columns) > 0 { + projected := MergeLogicalViewWithSchema(moDatabaseView, schema) + databases = mergeCatalogDatabaseEntries(databases, buildCatalogDatabasesFromMoDatabaseRows(projected)) + } + return filterCatalogTablesForList(databases, opts), nil +} + func filterCatalogTablesForList(tables []TableCatalogEntry, opts TableListOptions) []TableCatalogEntry { filtered := make([]TableCatalogEntry, 0, len(tables)) for _, table := range tables { @@ -1521,6 +1545,104 @@ func filterCatalogTablesForList(tables []TableCatalogEntry, opts TableListOption return filtered } +func buildCatalogDatabasesFromMoDatabaseRows(view *LogicalTableView) []TableCatalogEntry { + seen := make(map[string]struct{}) + merged := make([]TableCatalogEntry, 0) + try := func(datIDCol, datNameCol, accountIDCol int) { + databases := buildCatalogDatabasesFromMoDatabaseRowsAt(view, datIDCol, datNameCol, accountIDCol) + for _, database := range databases { + key := catalogDatabaseEntryKey(database) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + merged = append(merged, database) + } + } + + try( + fallbackCatalogColIndex(view, moDatabaseID, "dat_id"), + fallbackCatalogColIndex(view, moDatabaseID, "datname"), + fallbackCatalogColIndex(view, moDatabaseID, "account_id"), + ) + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moDatabaseID) { + try( + catalogColIndexForLayout(match.layout, moDatabaseID, "dat_id", match.offset), + catalogColIndexForLayout(match.layout, moDatabaseID, "datname", match.offset), + catalogColIndexForLayout(match.layout, moDatabaseID, "account_id", match.offset), + ) + } + return merged +} + +func buildCatalogDatabasesFromMoDatabaseRowsAt( + view *LogicalTableView, + datIDCol int, + datNameCol int, + accountIDCol int, +) []TableCatalogEntry { + if datIDCol < 0 || datNameCol < 0 { + return nil + } + + seen := make(map[string]struct{}) + databases := make([]TableCatalogEntry, 0) + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if datIDCol >= len(row) || datNameCol >= len(row) { + continue + } + databaseID, err := strconv.ParseUint(row[datIDCol], 10, 64) + if err != nil || databaseID == 0 { + continue + } + entry := TableCatalogEntry{ + AccountID: 0, + DatabaseID: databaseID, + DatabaseName: strings.Clone(row[datNameCol]), + } + if !validCatalogName(entry.DatabaseName) { + continue + } + if accountIDCol >= 0 && accountIDCol < len(row) { + if accountID, err := strconv.ParseUint(row[accountIDCol], 10, 32); err == nil { + entry.AccountID = uint32(accountID) + } + } + key := catalogDatabaseEntryKey(entry) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + databases = append(databases, entry) + } + return databases +} + +func mergeCatalogDatabaseEntries(base []TableCatalogEntry, extra []TableCatalogEntry) []TableCatalogEntry { + if len(extra) == 0 { + return base + } + seen := make(map[string]struct{}, len(base)+len(extra)) + for _, database := range base { + seen[catalogDatabaseEntryKey(database)] = struct{}{} + } + for _, database := range extra { + key := catalogDatabaseEntryKey(database) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + base = append(base, database) + } + return base +} + +func catalogDatabaseEntryKey(entry TableCatalogEntry) string { + return fmt.Sprintf("%d\x00%d\x00%s", entry.AccountID, entry.DatabaseID, strings.ToLower(entry.DatabaseName)) +} + func (r *CheckpointReader) listCatalogTablesFromProjectedMoTables( ctx context.Context, snapshotTS types.TS, diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 8cbd2f2e378b0..ad038b1638227 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -1025,6 +1025,35 @@ func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testin assert.Equal(t, "v", tables[1].RelKind) } +func TestBuildCatalogDatabasesFromMoDatabaseRows_IncludesDatabaseWithoutTables(t *testing.T) { + headers := []string{"object", "block", "row"} + for i := 0; i < len(currentCatalogLayout.moDatabaseSchema); i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + row := func(datID, datName, accountID, datType string) []string { + data := make([]string, len(currentCatalogLayout.moDatabaseSchema)) + data[0] = datID + data[1] = datName + data[7] = accountID + data[8] = datType + return append([]string{"obj1", "0", datID}, data...) + } + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("335900", "ckp_pubsub_sub_all", "415", catalog.SystemDBTypeSubscription), + }, + } + + databases := buildCatalogDatabasesFromMoDatabaseRows(view) + require.Len(t, databases, 1) + assert.Equal(t, uint32(415), databases[0].AccountID) + assert.Equal(t, uint64(335900), databases[0].DatabaseID) + assert.Equal(t, "ckp_pubsub_sub_all", databases[0].DatabaseName) + assert.Empty(t, databases[0].TableName) + assert.Zero(t, databases[0].TableID) +} + func TestBuildCatalogTablesFromMoTablesRows_UsesTemporaryDisplayName(t *testing.T) { headers := []string{"object", "block", "row"} for i := 0; i < len(preCPKLayout.moTablesSchema); i++ { From 24bfedc9e5f2562a0a5db9c96e3bad3d8cf37e9c Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 18 Jun 2026 16:54:11 +0800 Subject: [PATCH 66/76] resolve ckp partition metadata by account --- pkg/tools/checkpointtool/table_dump.go | 155 +++++++++++++++++++++---- 1 file changed, 134 insertions(+), 21 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index a13c22705e7f9..510c8400759fb 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -145,6 +145,7 @@ type TableForeignKey struct { type TableSchema struct { TableName string DatabaseName string + AccountID uint32 Columns []TableColumn // sorted by Position CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql Comment string @@ -1095,7 +1096,7 @@ func (r *CheckpointReader) ReadTableSchema( schema = mergeBuiltinSchemaFallback(schema, builtinTableSchemaForLayout(layout, tableID), tableID) } if schema.Partition == "" { - if partitionClause := r.readPartitionClause(ctx, tableID, snapshotTS); partitionClause != "" { + if partitionClause := r.readPartitionClause(ctx, tableID, snapshotTS, schema.AccountID); partitionClause != "" { schema.Partition = partitionClause } } @@ -1247,7 +1248,7 @@ func (r *CheckpointReader) PrepareTableDumpData( tableID, ) } - partitionTableIDs, err := r.readPartitionTableIDs(ctx, snapshotTS, tableID) + partitionTableIDs, err := r.readPartitionTableIDs(ctx, snapshotTS, tableID, schema.AccountID) if err != nil { return nil, err } @@ -1288,6 +1289,7 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( } tableSet := make(map[uint64]struct{}, len(tableIDs)) + accountByPrimary := make(map[uint64]uint32, len(tableIDs)) result := make(map[uint64]*TableDumpData, len(tableIDs)) partitionToPrimary := make(map[uint64]uint64) partitionIDsByPrimary := make(map[uint64][]uint64) @@ -1304,12 +1306,13 @@ func (r *CheckpointReader) PrepareTableDumpDataForTables( ) } tableSet[tableID] = struct{}{} + accountByPrimary[tableID] = schema.AccountID result[tableID] = &TableDumpData{ TableID: tableID, Schema: cloneTableSchema(schema), } } - partitionTableMap, err := r.readPartitionTableIDsForPrimaries(ctx, snapshotTS, tableSet) + partitionTableMap, err := r.readPartitionTableIDsForPrimaries(ctx, snapshotTS, tableSet, accountByPrimary) if err != nil { return nil, err } @@ -1403,6 +1406,7 @@ func cloneTableSchema(schema *TableSchema) *TableSchema { clone := &TableSchema{ TableName: strings.Clone(schema.TableName), DatabaseName: strings.Clone(schema.DatabaseName), + AccountID: schema.AccountID, CreateSQL: strings.Clone(schema.CreateSQL), Comment: strings.Clone(schema.Comment), Partition: strings.Clone(schema.Partition), @@ -1508,15 +1512,65 @@ func (r *CheckpointReader) ListCatalogDatabases( opts TableListOptions, ) ([]TableCatalogEntry, error) { moDatabaseView, err := r.getTableLogicalView(ctx, moDatabaseID, snapshotTS) + + // Primary path: read databases directly from mo_database. + // This covers databases that have no tables (e.g. empty subscription databases), + // which the mo_tables-based fallback would miss. + if err == nil && moDatabaseView != nil { + databases := buildCatalogDatabasesFromMoDatabaseRows(moDatabaseView) + if schema := r.ReadTableSchema(ctx, moDatabaseID, snapshotTS, moDatabaseView); len(schema.Columns) > 0 { + projected := MergeLogicalViewWithSchema(moDatabaseView, schema) + databases = mergeCatalogDatabaseEntries(databases, buildCatalogDatabasesFromMoDatabaseRows(projected)) + } + // Merge databases derived from mo_tables to fill any gaps that + // mo_database alone may not cover (e.g. when mo_database checkpoint + // data is incomplete). + if moTablesDBs, tablesErr := r.listDatabasesFromMoTables(ctx, snapshotTS); tablesErr == nil { + databases = mergeCatalogDatabaseEntries(databases, moTablesDBs) + } + return filterCatalogTablesForList(databases, opts), nil + } + + // Fallback: when mo_database is not available in the checkpoint (e.g. + // the table has not been flushed to a checkpoint entry yet), derive + // databases from mo_tables. This is the pre-mo_database behavior and + // works whenever mo_tables checkpoint data exists. + databases, tablesErr := r.listDatabasesFromMoTables(ctx, snapshotTS) + if tablesErr != nil { + return nil, err // return the original mo_database error, not the mo_tables one + } + return filterCatalogTablesForList(databases, opts), nil +} + +// listDatabasesFromMoTables derives database catalog entries from mo_tables +// rows. It returns database-level entries (account_id, database_id, database_name) +// with empty TableName and zero TableID. +func (r *CheckpointReader) listDatabasesFromMoTables( + ctx context.Context, + snapshotTS types.TS, +) ([]TableCatalogEntry, error) { + tables, err := r.ListCatalogTables(ctx, snapshotTS, TableListOptions{}) if err != nil { return nil, err } - databases := buildCatalogDatabasesFromMoDatabaseRows(moDatabaseView) - if schema := r.ReadTableSchema(ctx, moDatabaseID, snapshotTS, moDatabaseView); len(schema.Columns) > 0 { - projected := MergeLogicalViewWithSchema(moDatabaseView, schema) - databases = mergeCatalogDatabaseEntries(databases, buildCatalogDatabasesFromMoDatabaseRows(projected)) + seen := make(map[string]struct{}) + var databases []TableCatalogEntry + for _, t := range tables { + key := fmt.Sprintf("%d\x00%d\x00%s", t.AccountID, t.DatabaseID, strings.ToLower(t.DatabaseName)) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + if !validCatalogName(t.DatabaseName) { + continue + } + databases = append(databases, TableCatalogEntry{ + AccountID: t.AccountID, + DatabaseID: t.DatabaseID, + DatabaseName: t.DatabaseName, + }) } - return filterCatalogTablesForList(databases, opts), nil + return databases, nil } func filterCatalogTablesForList(tables []TableCatalogEntry, opts TableListOptions) []TableCatalogEntry { @@ -3080,6 +3134,7 @@ func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *Table fallbackCatalogColIndex(view, moTablesID, "rel_createsql"), fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment), fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint), + fallbackCatalogColIndex(view, moTablesID, "account_id"), ) } @@ -3091,6 +3146,7 @@ func buildSchemaFromMoTablesRowAt( createSQLIdx int, commentIdx int, constraintIdx int, + accountIDIdx int, ) *TableSchema { schema := &TableSchema{} dataRow := fullRow[logicalViewDataOffset(view):] @@ -3115,12 +3171,17 @@ func buildSchemaFromMoTablesRowAt( schema.UniqueKeys = decodeUniqueKeysFromMoTablesConstraint(dataRow[constraintIdx]) schema.PrimaryKey = decodePrimaryKeyFromMoTablesConstraint(dataRow[constraintIdx]) } + if accountIDIdx >= 0 && accountIDIdx < len(dataRow) { + if accountID, err := strconv.ParseUint(dataRow[accountIDIdx], 10, 32); err == nil { + schema.AccountID = uint32(accountID) + } + } return schema } func findTableSchemaFromMoTables(view *LogicalTableView, tableID uint64) *TableSchema { tableIDStr := fmt.Sprintf("%d", tableID) - try := func(relIDCol, relNameCol, relDBCol, createSQLCol, commentCol, constraintCol int) *TableSchema { + try := func(relIDCol, relNameCol, relDBCol, createSQLCol, commentCol, constraintCol, accountIDCol int) *TableSchema { if relIDCol < 0 { return nil } @@ -3129,7 +3190,7 @@ func findTableSchemaFromMoTables(view *LogicalTableView, tableID uint64) *TableS if relIDCol >= len(row) || row[relIDCol] != tableIDStr { continue } - return buildSchemaFromMoTablesRowAt(view, fullRow, relNameCol, relDBCol, createSQLCol, commentCol, constraintCol) + return buildSchemaFromMoTablesRowAt(view, fullRow, relNameCol, relDBCol, createSQLCol, commentCol, constraintCol, accountIDCol) } return nil } @@ -3141,6 +3202,7 @@ func findTableSchemaFromMoTables(view *LogicalTableView, tableID uint64) *TableS fallbackCatalogColIndex(view, moTablesID, "rel_createsql"), fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment), fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint), + fallbackCatalogColIndex(view, moTablesID, "account_id"), ); schema != nil { return schema } @@ -3154,6 +3216,7 @@ func findTableSchemaFromMoTables(view *LogicalTableView, tableID uint64) *TableS catalogColIndexForLayout(match.layout, moTablesID, "rel_createsql", match.offset), catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_Comment, match.offset), catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_Constraint, match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "account_id", match.offset), ); schema != nil { return schema } @@ -3795,6 +3858,15 @@ func (r *CheckpointReader) ShowCreateTable( if err != nil { moTablesView = nil } + var accountID uint32 + if moTablesView != nil { + for _, table := range buildCatalogTablesFromMoTablesRows(moTablesView) { + if table.TableID == tableID { + accountID = table.AccountID + break + } + } + } // 1. Reconstruct from mo_columns. moColumnsView, err := r.getTableLogicalView(ctx, moColumnsID, snapshotTS) @@ -3803,7 +3875,7 @@ func (r *CheckpointReader) ShowCreateTable( } var partitionMetadataView *LogicalTableView - if view, err := r.getPartitionMetadataView(ctx, snapshotTS); err == nil { + if view, err := r.getPartitionMetadataView(ctx, snapshotTS, accountID); err == nil { partitionMetadataView = view } @@ -3866,8 +3938,9 @@ func (r *CheckpointReader) ShowCreateIndexStatements( func (r *CheckpointReader) getPartitionMetadataView( ctx context.Context, snapshotTS types.TS, + accountID uint32, ) (*LogicalTableView, error) { - partitionMetadataTableID, ok, err := r.findCatalogTableID(ctx, snapshotTS, catalog.MOPartitionMetadata) + partitionMetadataTableID, ok, err := r.findCatalogTableIDForAccount(ctx, snapshotTS, catalog.MOPartitionMetadata, accountID) if err != nil { return nil, err } @@ -3880,8 +3953,9 @@ func (r *CheckpointReader) getPartitionMetadataView( func (r *CheckpointReader) getPartitionTablesView( ctx context.Context, snapshotTS types.TS, + accountID uint32, ) (*LogicalTableView, error) { - partitionTablesID, ok, err := r.findCatalogTableID(ctx, snapshotTS, catalog.MOPartitionTables) + partitionTablesID, ok, err := r.findCatalogTableIDForAccount(ctx, snapshotTS, catalog.MOPartitionTables, accountID) if err != nil { return nil, err } @@ -3895,8 +3969,9 @@ func (r *CheckpointReader) readPartitionTableIDs( ctx context.Context, snapshotTS types.TS, primaryTableID uint64, + accountID uint32, ) ([]uint64, error) { - view, err := r.getPartitionTablesView(ctx, snapshotTS) + view, err := r.getPartitionTablesView(ctx, snapshotTS, accountID) if err != nil || view == nil { return nil, err } @@ -3908,27 +3983,46 @@ func (r *CheckpointReader) readPartitionTableIDsForPrimaries( ctx context.Context, snapshotTS types.TS, primaryTableIDs map[uint64]struct{}, + accountByPrimary map[uint64]uint32, ) (map[uint64][]uint64, error) { if len(primaryTableIDs) == 0 { return nil, nil } - view, err := r.getPartitionTablesView(ctx, snapshotTS) - if err != nil || view == nil { - return nil, err + result := make(map[uint64][]uint64) + primaryByAccount := make(map[uint32]map[uint64]struct{}) + for primaryID := range primaryTableIDs { + accountID := accountByPrimary[primaryID] + if primaryByAccount[accountID] == nil { + primaryByAccount[accountID] = make(map[uint64]struct{}) + } + primaryByAccount[accountID][primaryID] = struct{}{} } - return buildPartitionTableIDMap(view, primaryTableIDs), nil + for accountID, primaries := range primaryByAccount { + view, err := r.getPartitionTablesView(ctx, snapshotTS, accountID) + if err != nil { + return nil, err + } + if view == nil { + continue + } + for primaryID, partitionIDs := range buildPartitionTableIDMap(view, primaries) { + result[primaryID] = append(result[primaryID], partitionIDs...) + } + } + return result, nil } func (r *CheckpointReader) readPartitionClause( ctx context.Context, tableID uint64, snapshotTS types.TS, + accountID uint32, ) string { - metadataView, err := r.getPartitionMetadataView(ctx, snapshotTS) + metadataView, err := r.getPartitionMetadataView(ctx, snapshotTS, accountID) if err != nil || metadataView == nil { return "" } - tablesView, err := r.getPartitionTablesView(ctx, snapshotTS) + tablesView, err := r.getPartitionTablesView(ctx, snapshotTS, accountID) if err != nil { tablesView = nil } @@ -4029,19 +4123,38 @@ func (r *CheckpointReader) findCatalogTableID( ctx context.Context, snapshotTS types.TS, tableName string, +) (uint64, bool, error) { + return r.findCatalogTableIDForAccount(ctx, snapshotTS, tableName, 0) +} + +func (r *CheckpointReader) findCatalogTableIDForAccount( + ctx context.Context, + snapshotTS types.TS, + tableName string, + accountID uint32, ) (uint64, bool, error) { moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) if err != nil { return 0, false, fmt.Errorf("read mo_tables: %w", err) } + var fallback uint64 tables := buildCatalogTablesFromMoTablesRows(moTablesView) for _, table := range tables { if table.TableName != tableName { continue } - if table.DatabaseName == "" || table.DatabaseName == catalog.MO_CATALOG { + if table.DatabaseName != "" && table.DatabaseName != catalog.MO_CATALOG { + continue + } + if table.AccountID == accountID { return table.TableID, true, nil } + if accountID == 0 && fallback == 0 { + fallback = table.TableID + } + } + if fallback != 0 { + return fallback, true, nil } return 0, false, nil } From 4b0e684fae424090a3865d3caa6549b7e69b6e9e Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 24 Jun 2026 10:08:33 +0800 Subject: [PATCH 67/76] update --- pkg/tools/checkpointtool/table_dump.go | 117 ++++++++++++++++++-- pkg/tools/checkpointtool/table_dump_test.go | 71 ++++++++++++ 2 files changed, 180 insertions(+), 8 deletions(-) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 510c8400759fb..9aa05cbb55d5e 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -153,6 +153,7 @@ type TableSchema struct { PrimaryKey []string ForeignKeys []TableForeignKey Partition string + ClusterBy []string } type TableCatalogEntry struct { @@ -462,6 +463,10 @@ func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment stri } func renderCreateTableDDLFullWithForeignKeys(tableName string, cols []TableColumn, comment string, partition string, primaryKey []string, uniqueKeys []TableUniqueKey, foreignKeys []TableForeignKey) string { + return renderCreateTableDDLFullWithForeignKeysAndClusterBy(tableName, cols, comment, partition, primaryKey, uniqueKeys, foreignKeys, nil) +} + +func renderCreateTableDDLFullWithForeignKeysAndClusterBy(tableName string, cols []TableColumn, comment string, partition string, primaryKey []string, uniqueKeys []TableUniqueKey, foreignKeys []TableForeignKey, clusterBy []string) string { if tableName == "" || len(cols) == 0 { return "" } @@ -560,7 +565,11 @@ func renderCreateTableDDLFullWithForeignKeys(tableName string, cols []TableColum sb.WriteString(" ") sb.WriteString(partition) } - if clusterCols := clusterByColumns(cols); len(clusterCols) > 0 { + if len(clusterBy) > 0 { + sb.WriteString(" CLUSTER BY (") + appendDDLIdentList(&sb, clusterBy) + sb.WriteString(")") + } else if clusterCols := clusterByColumns(cols); len(clusterCols) > 0 { sb.WriteString(" CLUSTER BY (") for i, col := range clusterCols { if i > 0 { @@ -701,7 +710,7 @@ func appendColumnDDLAttributes(sb *strings.Builder, col TableColumn) { if col.AutoIncrement { sb.WriteString(" AUTO_INCREMENT") } - if col.HasDefault { + if col.HasDefault && !col.AutoIncrement { if strings.TrimSpace(col.Default) == "" { sb.WriteString(" DEFAULT NULL") } else { @@ -891,6 +900,9 @@ func formatDDLDefault(defaultExpr string) string { if defaultExpr == "" { return "NULL" } + if strings.EqualFold(defaultExpr, "null") { + return "NULL" + } if len(defaultExpr) >= 2 && defaultExpr[0] == '\'' && defaultExpr[len(defaultExpr)-1] == '\'' { return "'" + strings.ReplaceAll(defaultExpr[1:len(defaultExpr)-1], "'", "''") + "'" } @@ -926,7 +938,7 @@ func RenderCreateTableDDLFromSchema(schema *TableSchema) string { return "" } } - return renderCreateTableDDLFullWithForeignKeys(schema.TableName, schema.Columns, schema.Comment, schema.Partition, schema.PrimaryKey, schema.UniqueKeys, schema.ForeignKeys) + return renderCreateTableDDLFullWithForeignKeysAndClusterBy(schema.TableName, schema.Columns, schema.Comment, schema.Partition, schema.PrimaryKey, schema.UniqueKeys, schema.ForeignKeys, schema.ClusterBy) } func inferBuiltinCatalogLayout( @@ -1086,6 +1098,7 @@ func (r *CheckpointReader) ReadTableSchema( if len(cols) > 0 { schema.Columns = cols } + schema.ClusterBy = buildClusterByFromMoColumnsRows(moColumnsView, tableID) if moTablesView != nil { schema.ForeignKeys = findForeignKeysFromCatalogViews(moTablesView, moColumnsView, tableID) } @@ -1417,6 +1430,12 @@ func cloneTableSchema(schema *TableSchema) *TableSchema { clone.PrimaryKey[i] = strings.Clone(col) } } + if len(schema.ClusterBy) > 0 { + clone.ClusterBy = make([]string, len(schema.ClusterBy)) + for i, col := range schema.ClusterBy { + clone.ClusterBy[i] = strings.Clone(col) + } + } if len(schema.Columns) > 0 { clone.Columns = make([]TableColumn, len(schema.Columns)) for i, col := range schema.Columns { @@ -3436,6 +3455,7 @@ func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView uniqueKeys := []TableUniqueKey(nil) primaryKey := []string(nil) foreignKeys := []TableForeignKey(nil) + clusterBy := []string(nil) partition := "" if moTablesView != nil { tableName = findTableNameFromMoTables(moTablesView, tableID) @@ -3446,11 +3466,14 @@ func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView foreignKeys = findForeignKeysFromCatalogViews(moTablesView, moColumnsView, tableID) } } + if moColumnsView != nil { + clusterBy = buildClusterByFromMoColumnsRows(moColumnsView, tableID) + } if len(partitionMetadataView) > 0 && partitionMetadataView[0] != nil { partition = buildPartitionClauseFromMetadata(partitionMetadataView[0], tableID) } if moColumnsView != nil { - if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys); ddl != "" { + if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, clusterBy); ddl != "" { return ddl } } @@ -3831,6 +3854,79 @@ func buildColumnsFromMoColumnsRowsAt( return cols } +func buildClusterByFromMoColumnsRows(view *LogicalTableView, tableID uint64) []string { + relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_RelID) + nameCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Name) + clusterByCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsClusterBy) + if relnameIDCol < 0 || nameCol < 0 || clusterByCol < 0 { + return nil + } + if clusterBy := buildClusterByFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, clusterByCol); len(clusterBy) > 0 { + return clusterBy + } + + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { + relnameIDCol = catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_RelID, match.offset) + nameCol = catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Name, match.offset) + clusterByCol = catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_IsClusterBy, match.offset) + if clusterBy := buildClusterByFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, clusterByCol); len(clusterBy) > 0 { + return clusterBy + } + } + return nil +} + +func buildClusterByFromMoColumnsRowsAt( + view *LogicalTableView, + tableID uint64, + relnameIDCol int, + nameCol int, + clusterByCol int, +) []string { + if relnameIDCol < 0 || nameCol < 0 || clusterByCol < 0 { + return nil + } + tableIDStr := fmt.Sprintf("%d", tableID) + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relnameIDCol >= len(row) || nameCol >= len(row) || clusterByCol >= len(row) { + continue + } + if row[relnameIDCol] != tableIDStr || !isTruthyCatalogValue(row[clusterByCol]) { + continue + } + name := strings.TrimSpace(row[nameCol]) + if name == "" { + continue + } + if names := splitCompositeClusterByColumnName(name); len(names) > 0 { + return names + } + return []string{name} + } + return nil +} + +func splitCompositeClusterByColumnName(name string) []string { + if !strings.HasPrefix(name, catalog.PrefixCBColName) { + return nil + } + var names []string + for next := len(catalog.PrefixCBColName); next < len(name); { + if next+3 > len(name) { + return nil + } + strLen, err := strconv.Atoi(name[next : next+3]) + if err != nil || strLen <= 0 || next+3+strLen > len(name) { + return nil + } + names = append(names, name[next+3:next+3+strLen]) + next += strLen + 3 + } + return names +} + // getDataColumnsFromView extracts only the data columns (skipping object/block/row meta columns). // ShowCreateTable returns the CREATE TABLE DDL for a given tableID by reading // the checkpoint's mo_tables and mo_columns system tables (GCKP + following ICKPs). @@ -4870,7 +4966,7 @@ func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, table if len(tableNames) > 1 && tableNames[1] != "" { tableComment = tableNames[1] } - return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, "", nil, nil, nil) + return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, "", nil, nil, nil, nil) } func buildCreateTableFromMoColumnsWithOptions( @@ -4882,7 +4978,11 @@ func buildCreateTableFromMoColumnsWithOptions( primaryKey []string, uniqueKeys []TableUniqueKey, foreignKeys []TableForeignKey, + clusterBy []string, ) string { + if clusterBy == nil { + clusterBy = buildClusterByFromMoColumnsRows(view, tableID) + } relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") typCol := fallbackCatalogColIndex(view, moColumnsID, "atttyp") @@ -4890,7 +4990,7 @@ func buildCreateTableFromMoColumnsWithOptions( hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, clusterBy, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -4902,7 +5002,7 @@ func buildCreateTableFromMoColumnsWithOptions( typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) - if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, clusterBy, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { return ddl } } @@ -4919,6 +5019,7 @@ func buildCreateTableFromMoColumnsAt( primaryKey []string, uniqueKeys []TableUniqueKey, foreignKeys []TableForeignKey, + clusterBy []string, relnameIDCol int, nameCol int, typCol int, @@ -4930,7 +5031,7 @@ func buildCreateTableFromMoColumnsAt( if len(cols) == 0 { return "" } - return renderCreateTableDDLFullWithForeignKeys(tableName, cols, tableComment, partition, primaryKey, uniqueKeys, foreignKeys) + return renderCreateTableDDLFullWithForeignKeysAndClusterBy(tableName, cols, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, clusterBy) } func isPrintableSQLType(sqlType string) bool { diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index ad038b1638227..9a2edd54fb8ec 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -618,6 +618,65 @@ func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *test assert.NotContains(t, ddl, "`old`") } +func TestCreateTableDDLFromCatalogViews_PreservesCompositeClusterByFromMoColumns(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "relpersistence", "relkind", "rel_comment", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "272528", "statement_info", "system", "272500", + "", "r", "record each statement and stats info[mo_no_del_hint]", + }, + }, + } + + moColumnsView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, catalog.MoColumnsSchema...), + Rows: [][]string{ + moColumnsTestRow(t, "272528", "statement_info", "request_at", encodedSQLType(t, types.New(types.T_datetime, 0, 6)), "1", "1", "0"), + moColumnsTestRow(t, "272528", "statement_info", "account_id", encodedSQLType(t, types.T_uint32.ToType()), "2", "0", "1"), + moColumnsTestRow(t, "272528", "statement_info", "__mo_cbkey_010request_at010account_id", encodedSQLType(t, types.T_varchar.ToType()), "3", "0", "2", catalog.SystemColAttr_IsHidden, "1", catalog.SystemColAttr_IsClusterBy, "1"), + }, + } + + ddl := createTableDDLFromCatalogViews(272528, moTablesView, moColumnsView) + assert.Contains(t, ddl, "CREATE TABLE `statement_info`") + assert.Contains(t, ddl, "`request_at` DATETIME(6) NOT NULL") + assert.Contains(t, ddl, "`account_id` INT UNSIGNED") + assert.NotContains(t, ddl, "__mo_cbkey_010request_at010account_id") + assert.Contains(t, ddl, "COMMENT='record each statement and stats info[mo_no_del_hint]' CLUSTER BY (`request_at`, `account_id`)") +} + +func moColumnsTestRow(t *testing.T, relID, relName, name, typ, attnum, notNull, seqnum string, extra ...string) []string { + t.Helper() + data := make([]string, len(catalog.MoColumnsSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoColumnsSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_columns header %s", colName) + } + set(catalog.SystemColAttr_RelID, relID) + set(catalog.SystemColAttr_RelName, relName) + set(catalog.SystemColAttr_Name, name) + set(catalog.SystemColAttr_Type, typ) + set(catalog.SystemColAttr_Num, attnum) + set(catalog.SystemColAttr_NullAbility, notNull) + set(catalog.SystemColAttr_IsHidden, "0") + set(catalog.SystemColAttr_Seqnum, seqnum) + for i := 0; i+1 < len(extra); i += 2 { + set(extra[i], extra[i+1]) + } + return append([]string{"obj1", "0", attnum}, data...) +} + func TestCreateTableDDLFromCatalogViews_IncludesForeignKeys(t *testing.T) { moTablesHeaders := append([]string{"object", "block", "row"}, catalog.MoTablesSchema...) tableRow := func(relID, relName, dbName, dbID, constraint string) []string { @@ -976,6 +1035,18 @@ func TestRenderCreateTableDDLFromSchema_EnumSetValues(t *testing.T) { assert.Contains(t, ddl, "`c_set` SET('a','b','c')") } +func TestRenderCreateTableDDLFromSchema_AutoIncrementSkipsDefaultNull(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t_auto", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT UNSIGNED", Position: 1, AutoIncrement: true, HasDefault: true}, + }, + }) + + assert.Contains(t, ddl, "`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT") + assert.NotContains(t, ddl, "AUTO_INCREMENT DEFAULT NULL") +} + func TestDecodeMoColumnEncodedSQLType_TemporalScale(t *testing.T) { sqlType, ok := decodeMoColumnEncodedSQLType(encodedSQLType(t, types.New(types.T_time, 0, 6))) require.True(t, ok) From a0de8dd97eee3c920775b0fc019c18141c3806fa Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 24 Jun 2026 16:03:23 +0800 Subject: [PATCH 68/76] add diagnostic logging to checkpoint loading path Add logging to ListTSRangeFiles, ListCKPMetaNames, and loadEntries to help diagnose 'no checkpoint timestamp is available' errors: - ListTSRangeFiles: log dir, total/valid/skipped file counts - ListCKPMetaNames: log total/meta/nonMeta counts and meta file names - loadEntries: log dir, meta files found, entries per file with start/end/type/state, dedup stats, and final usable checkpoints Co-Authored-By: Claude --- pkg/objectio/ioutil/funcs.go | 6 +++++ pkg/tools/checkpointtool/checkpoint_reader.go | 24 +++++++++++++++++++ pkg/vm/engine/ckputil/funcs.go | 11 +++++++++ 3 files changed, 41 insertions(+) diff --git a/pkg/objectio/ioutil/funcs.go b/pkg/objectio/ioutil/funcs.go index 6aa8522dc49db..61b0cdf8c1671 100644 --- a/pkg/objectio/ioutil/funcs.go +++ b/pkg/objectio/ioutil/funcs.go @@ -23,6 +23,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" @@ -41,13 +42,18 @@ func ListTSRangeFiles( ); err != nil { return } + skipped := 0 for _, entry := range entries { if !entry.IsDir { if file := DecodeTSRangeFile(entry.Name); file.IsValid() { files = append(files, file) + } else { + skipped++ } } } + logutil.Infof("[ListTSRangeFiles] dir=%s totalEntries=%d valid=%d skipped(invalidName)=%d", + dir, len(entries), len(files), skipped) return } diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index 93026dc5f4a5f..be0306e16153d 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -26,6 +26,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" @@ -112,25 +113,38 @@ func (r *CheckpointReader) Fork(ctx context.Context) *CheckpointReader { } func (r *CheckpointReader) loadEntries() error { + logutil.Infof("[loadEntries] reading checkpoint dir: %s", r.dir) names, err := ckputil.ListCKPMetaNames(r.ctx, r.fs) if err != nil { return err } if len(names) == 0 { + logutil.Infof("[loadEntries] no checkpoint meta files found in dir=%s", r.dir) return nil } + logutil.Infof("[loadEntries] found %d meta file(s) to read", len(names)) + totalRead := 0 for _, name := range names { entries, err := checkpoint.ReadEntriesFromMeta( r.ctx, "", ioutil.GetCheckpointDir(), name, 0, nil, r.mp, r.fs, ) if err != nil { + logutil.Infof("[loadEntries] failed to read meta file %s: %v", name, err) return err } + logutil.Infof("[loadEntries] meta file=%s entries=%d", name, len(entries)) + for _, e := range entries { + logutil.Infof("[loadEntries] entry: start=%s end=%s type=%s state=%d", + e.GetStart().ToString(), e.GetEnd().ToString(), e.GetType().String(), e.GetState()) + } r.entries = append(r.entries, entries...) + totalRead += len(entries) } + logutil.Infof("[loadEntries] total raw entries read: %d", totalRead) // Deduplicate by (start, end, type, location) + removed := 0 seen := make(map[string]bool) unique := make([]*checkpoint.CheckpointEntry, 0, len(r.entries)) for _, e := range r.entries { @@ -138,15 +152,25 @@ func (r *CheckpointReader) loadEntries() error { if !seen[key] { seen[key] = true unique = append(unique, e) + } else { + removed++ } } r.entries = unique + logutil.Infof("[loadEntries] dedup: removed=%d remaining=%d", removed, len(unique)) // Sort by end timestamp (newest first) sort.Slice(r.entries, func(i, j int) bool { ei, ej := r.entries[i].GetEnd(), r.entries[j].GetEnd() return ei.GT(&ej) }) + + // Log final usable entries + for _, e := range r.entries { + logutil.Infof("[loadEntries] usable ckp: start=%s end=%s type=%s", + e.GetStart().ToString(), e.GetEnd().ToString(), e.GetType().String()) + } + return nil } diff --git a/pkg/vm/engine/ckputil/funcs.go b/pkg/vm/engine/ckputil/funcs.go index 2dd7e953017e7..972ce60f96af8 100644 --- a/pkg/vm/engine/ckputil/funcs.go +++ b/pkg/vm/engine/ckputil/funcs.go @@ -19,6 +19,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" ) @@ -54,9 +55,19 @@ func ListCKPMetaNames( ); err != nil { return } + nonMeta := 0 for _, tsFile := range tsFiles { if tsFile.IsMetadataFile() { files = append(files, tsFile.GetName()) + } else { + nonMeta++ + } + } + logutil.Infof("[ListCKPMetaNames] totalTSRangeFiles=%d metaFiles=%d nonMeta=%d", + len(tsFiles), len(files), nonMeta) + if len(files) > 0 { + for _, f := range files { + logutil.Infof("[ListCKPMetaNames] metaFile: %s", f) } } return From 5d15243c29c3064a5783ba1cc1ecbe425ecfdbbf Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 24 Jun 2026 16:06:58 +0800 Subject: [PATCH 69/76] keep diagnostic logging only in tool path (loadEntries) Revert logutil from shared library functions (ListTSRangeFiles, ListCKPMetaNames) which are also called by the MO server. Instead, add raw file listing diagnostics directly in loadEntries() which is only invoked by mo-tool ckp commands. The raw listing shows every file in the ckp/ directory and whether it is a valid TSRangeFile and/or metadata file, making it easy to identify why checkpoint entries are not being loaded. Co-Authored-By: Claude --- pkg/objectio/ioutil/funcs.go | 6 --- pkg/tools/checkpointtool/checkpoint_reader.go | 43 ++++++++++++++++++- pkg/vm/engine/ckputil/funcs.go | 11 ----- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/pkg/objectio/ioutil/funcs.go b/pkg/objectio/ioutil/funcs.go index 61b0cdf8c1671..6aa8522dc49db 100644 --- a/pkg/objectio/ioutil/funcs.go +++ b/pkg/objectio/ioutil/funcs.go @@ -23,7 +23,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/fileservice" - "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" @@ -42,18 +41,13 @@ func ListTSRangeFiles( ); err != nil { return } - skipped := 0 for _, entry := range entries { if !entry.IsDir { if file := DecodeTSRangeFile(entry.Name); file.IsValid() { files = append(files, file) - } else { - skipped++ } } } - logutil.Infof("[ListTSRangeFiles] dir=%s totalEntries=%d valid=%d skipped(invalidName)=%d", - dir, len(entries), len(files), skipped) return } diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index be0306e16153d..fe12d7d8a4997 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -113,7 +113,32 @@ func (r *CheckpointReader) Fork(ctx context.Context) *CheckpointReader { } func (r *CheckpointReader) loadEntries() error { - logutil.Infof("[loadEntries] reading checkpoint dir: %s", r.dir) + ckpDir := ioutil.GetCheckpointDir() + logutil.Infof("[loadEntries] reading checkpoint dir: %s (display dir: %s)", ckpDir, r.dir) + + // Diagnostic: list all raw files in the checkpoint directory + rawEntries, listErr := fileservice.SortedList(r.fs.List(r.ctx, ckpDir)) + if listErr != nil { + logutil.Infof("[loadEntries] failed to list dir %s: %v", ckpDir, listErr) + } else { + logutil.Infof("[loadEntries] raw listing: total=%d dirs=%d files=%d", + len(rawEntries), countDirs(rawEntries), countFiles(rawEntries)) + for _, e := range rawEntries { + if !e.IsDir { + valid := "" + if f := ioutil.DecodeTSRangeFile(e.Name); f.IsValid() { + valid = " [valid tsrange]" + if f.IsMetadataFile() { + valid += " [meta]" + } else { + valid += " [non-meta ext=" + f.GetExt() + "]" + } + } + logutil.Infof("[loadEntries] file: %s%s", e.Name, valid) + } + } + } + names, err := ckputil.ListCKPMetaNames(r.ctx, r.fs) if err != nil { return err @@ -127,7 +152,7 @@ func (r *CheckpointReader) loadEntries() error { totalRead := 0 for _, name := range names { entries, err := checkpoint.ReadEntriesFromMeta( - r.ctx, "", ioutil.GetCheckpointDir(), name, 0, nil, r.mp, r.fs, + r.ctx, "", ckpDir, name, 0, nil, r.mp, r.fs, ) if err != nil { logutil.Infof("[loadEntries] failed to read meta file %s: %v", name, err) @@ -174,6 +199,20 @@ func (r *CheckpointReader) loadEntries() error { return nil } +func countDirs(entries []fileservice.DirEntry) int { + n := 0 + for _, e := range entries { + if e.IsDir { + n++ + } + } + return n +} + +func countFiles(entries []fileservice.DirEntry) int { + return len(entries) - countDirs(entries) +} + // Info returns checkpoint summary func (r *CheckpointReader) Info() *CheckpointInfo { info := &CheckpointInfo{Dir: r.dir, TotalEntries: len(r.entries)} diff --git a/pkg/vm/engine/ckputil/funcs.go b/pkg/vm/engine/ckputil/funcs.go index 972ce60f96af8..2dd7e953017e7 100644 --- a/pkg/vm/engine/ckputil/funcs.go +++ b/pkg/vm/engine/ckputil/funcs.go @@ -19,7 +19,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/fileservice" - "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" ) @@ -55,19 +54,9 @@ func ListCKPMetaNames( ); err != nil { return } - nonMeta := 0 for _, tsFile := range tsFiles { if tsFile.IsMetadataFile() { files = append(files, tsFile.GetName()) - } else { - nonMeta++ - } - } - logutil.Infof("[ListCKPMetaNames] totalTSRangeFiles=%d metaFiles=%d nonMeta=%d", - len(tsFiles), len(files), nonMeta) - if len(files) > 0 { - for _, f := range files { - logutil.Infof("[ListCKPMetaNames] metaFile: %s", f) } } return From d6f4fdf2b819f2a212b7d7bf141c42657ef97a79 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 24 Jun 2026 16:46:26 +0800 Subject: [PATCH 70/76] update error message --- cmd/mo-object-tool/ckp/checkpoint.go | 64 +++++++++++++++------ cmd/mo-object-tool/ckp/checkpoint_test.go | 10 ++++ pkg/tools/checkpointtool/table_dump.go | 17 +++++- pkg/tools/checkpointtool/table_dump_test.go | 10 ++++ 4 files changed, 80 insertions(+), 21 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 268b5f32d4c46..f374743d9828f 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -1834,20 +1835,30 @@ func dumpTablesConcurrently( header bool, out io.Writer, ) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + tableCh := make(chan tableDumpPlan) - errCh := make(chan error, 1) var outMu sync.Mutex var wg sync.WaitGroup + var errMu sync.Mutex + var dumpErr error + recordErr := func(err error) { + if err == nil { + return + } + errMu.Lock() + dumpErr = preferRealError(dumpErr, err) + errMu.Unlock() + cancel() + } worker := func() { defer wg.Done() workerReader := reader.Fork(ctx) for plan := range tableCh { if err := dumpOneTable(ctx, workerReader, dumpOut, plan, snapshotTS, outputDir, rowOrder, metaComments, header, out, &outMu); err != nil { - select { - case errCh <- err: - default: - } + recordErr(err) return } } @@ -1856,23 +1867,22 @@ func dumpTablesConcurrently( wg.Add(1) go worker() } +sendPlans: for _, plan := range plans { select { - case err := <-errCh: - close(tableCh) - wg.Wait() - return err case tableCh <- plan: + case <-ctx.Done(): + break sendPlans } } close(tableCh) wg.Wait() - select { - case err := <-errCh: - return err - default: - return nil + errMu.Lock() + defer errMu.Unlock() + if dumpErr != nil { + return dumpErr } + return ctx.Err() } func tableCSVPath(outputDir string, table checkpointtool.TableCatalogEntry) string { @@ -2007,11 +2017,8 @@ func dumpOneTable( checkpointtool.WithCSVRowOrder(rowOrder), ) closeErr := outFile.Close() - if err != nil { - return fmt.Errorf("dump table %d (%s.%s): %w", table.TableID, table.DatabaseName, table.TableName, err) - } - if closeErr != nil { - return fmt.Errorf("close output file for table %d: %w", table.TableID, closeErr) + if combinedErr := dumpTableError(err, closeErr); combinedErr != nil { + return fmt.Errorf("dump table %d (%s.%s): %w", table.TableID, table.DatabaseName, table.TableName, combinedErr) } outMu.Lock() fmt.Fprintf(out, "Table %d %s.%s dumped to %s\n", table.TableID, table.DatabaseName, table.TableName, filePath) @@ -2019,6 +2026,25 @@ func dumpOneTable( return nil } +func dumpTableError(dumpErr, closeErr error) error { + return preferRealError(dumpErr, closeErr) +} + +func preferRealError(current, next error) error { + switch { + case current == nil: + return next + case next == nil: + return current + case errors.Is(current, context.Canceled) && !errors.Is(next, context.Canceled): + return next + case errors.Is(next, context.Canceled) && !errors.Is(current, context.Canceled): + return current + default: + return errors.Join(current, next) + } +} + func safePathPart(s string) string { s = strings.TrimSpace(s) if s == "" { diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 0e04972145855..8c6a069a32c55 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -17,6 +17,7 @@ package ckp import ( "context" "encoding/json" + "errors" "os" "path/filepath" "strings" @@ -30,6 +31,15 @@ import ( "github.com/stretchr/testify/require" ) +func TestPreferRealErrorKeepsNonCanceledRootCause(t *testing.T) { + root := errors.New("s3 write failed") + + assert.ErrorIs(t, preferRealError(context.Canceled, root), root) + assert.ErrorIs(t, preferRealError(root, context.Canceled), root) + assert.ErrorIs(t, preferRealError(context.Canceled, context.Canceled), context.Canceled) + assert.NoError(t, preferRealError(nil, nil)) +} + func TestNormalizeCreateTableDDLName(t *testing.T) { table := checkpointtool.TableCatalogEntry{ DatabaseName: "compat_ckp", diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 9aa05cbb55d5e..1e067fbad1d4b 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -19,6 +19,7 @@ import ( "context" "encoding/csv" "encoding/hex" + "errors" "fmt" "io" "os" @@ -2019,10 +2020,22 @@ func (r *CheckpointReader) streamTableCSVPipeline( close(chunks) writerErr := <-writerDone printCSVPipelineReport("finish", counters) - if producerErr != nil { + return csvPipelineError(producerErr, writerErr) +} + +func csvPipelineError(producerErr, writerErr error) error { + switch { + case producerErr == nil: + return writerErr + case writerErr == nil: return producerErr + case errors.Is(producerErr, context.Canceled) && !errors.Is(writerErr, context.Canceled): + return writerErr + case errors.Is(writerErr, context.Canceled) && !errors.Is(producerErr, context.Canceled): + return producerErr + default: + return errors.Join(producerErr, writerErr) } - return writerErr } func (r *CheckpointReader) produceCSVChunks( diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 9a2edd54fb8ec..1ff899752f832 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -17,6 +17,7 @@ package checkpointtool import ( "bytes" "context" + "errors" "fmt" "strings" "testing" @@ -34,6 +35,15 @@ import ( "github.com/stretchr/testify/require" ) +func TestCSVPipelineErrorKeepsNonCanceledRootCause(t *testing.T) { + root := errors.New("remote read failed") + + assert.ErrorIs(t, csvPipelineError(context.Canceled, root), root) + assert.ErrorIs(t, csvPipelineError(root, context.Canceled), root) + assert.ErrorIs(t, csvPipelineError(context.Canceled, context.Canceled), context.Canceled) + assert.NoError(t, csvPipelineError(nil, nil)) +} + func encodedSQLType(t *testing.T, typ types.Type) string { t.Helper() data, err := types.Encode(&typ) From 10f3fc7b5c462a80657d2a4cf36b2dc00668d0d6 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 26 Jun 2026 14:50:21 +0800 Subject: [PATCH 71/76] fix ckp dump s3 retry errors --- cmd/mo-object-tool/ckp/checkpoint.go | 35 ++++++++++++- cmd/mo-object-tool/ckp/checkpoint_test.go | 18 +++++++ pkg/fileservice/error.go | 2 + pkg/fileservice/parallel_sdk_test.go | 61 +++++++++++++++++++++++ pkg/fileservice/qcloud_sdk_test.go | 7 +++ 5 files changed, 122 insertions(+), 1 deletion(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index f374743d9828f..7ae633bfb69b8 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -1848,7 +1848,9 @@ func dumpTablesConcurrently( return } errMu.Lock() - dumpErr = preferRealError(dumpErr, err) + if !isContextCanceledOnly(err) { + dumpErr = preferRealError(dumpErr, err) + } errMu.Unlock() cancel() } @@ -2027,6 +2029,9 @@ func dumpOneTable( } func dumpTableError(dumpErr, closeErr error) error { + if dumpErr != nil && closeErr != nil && dumpErr.Error() == closeErr.Error() { + return dumpErr + } return preferRealError(dumpErr, closeErr) } @@ -2045,6 +2050,34 @@ func preferRealError(current, next error) error { } } +func isContextCanceledOnly(err error) bool { + if err == nil || !errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + type multiUnwrapper interface { + Unwrap() []error + } + if unwrapped, ok := err.(multiUnwrapper); ok { + errs := unwrapped.Unwrap() + if len(errs) == 0 { + return false + } + for _, inner := range errs { + if !isContextCanceledOnly(inner) { + return false + } + } + return true + } + type unwrapper interface { + Unwrap() error + } + if unwrapped, ok := err.(unwrapper); ok { + return isContextCanceledOnly(unwrapped.Unwrap()) + } + return true +} + func safePathPart(s string) string { s = strings.TrimSpace(s) if s == "" { diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 8c6a069a32c55..2f6c4b6b582b3 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "path/filepath" "strings" @@ -40,6 +41,23 @@ func TestPreferRealErrorKeepsNonCanceledRootCause(t *testing.T) { assert.NoError(t, preferRealError(nil, nil)) } +func TestDumpTableErrorDedupesPipeWriteAndCloseError(t *testing.T) { + err := errors.New("s3 write failed") + + assert.Equal(t, err, dumpTableError(err, errors.New("s3 write failed"))) +} + +func TestIsContextCanceledOnly(t *testing.T) { + root := errors.New("s3 write failed") + + assert.True(t, isContextCanceledOnly(context.Canceled)) + assert.True(t, isContextCanceledOnly(fmt.Errorf("dump table 1: %w", context.Canceled))) + assert.True(t, isContextCanceledOnly(errors.Join(context.Canceled, fmt.Errorf("worker: %w", context.Canceled)))) + assert.False(t, isContextCanceledOnly(root)) + assert.False(t, isContextCanceledOnly(context.DeadlineExceeded)) + assert.False(t, isContextCanceledOnly(errors.Join(context.Canceled, root))) +} + func TestNormalizeCreateTableDDLName(t *testing.T) { table := checkpointtool.TableCatalogEntry{ DatabaseName: "compat_ckp", diff --git a/pkg/fileservice/error.go b/pkg/fileservice/error.go index f76ed61b2264e..afd7a2e79e71a 100644 --- a/pkg/fileservice/error.go +++ b/pkg/fileservice/error.go @@ -74,6 +74,8 @@ func IsRetryableError(err error) bool { strings.Contains(str, "connection reset by peer") || strings.Contains(str, "connection timed out") || strings.Contains(str, "dial tcp: lookup") || + strings.Contains(str, "server closed idle connection") || + strings.Contains(str, "server closed connection") || strings.Contains(str, "i/o timeout") || strings.Contains(str, "write: broken pipe") || strings.Contains(str, "TLS handshake timeout") || diff --git a/pkg/fileservice/parallel_sdk_test.go b/pkg/fileservice/parallel_sdk_test.go index 4f909ec6c4520..30892bac9348c 100644 --- a/pkg/fileservice/parallel_sdk_test.go +++ b/pkg/fileservice/parallel_sdk_test.go @@ -720,6 +720,67 @@ func TestCOSMultipartUploadPartError(t *testing.T) { } } +func TestCOSMultipartUploadPartRetriesServerClosedIdleConnection(t *testing.T) { + server, state := newMockCOSServer(t, 0) + defer server.Close() + state.uploadID = "cos-uid-retry-idle-conn" + + baseClient := server.Client() + baseTransport := baseClient.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + transport := &cosUploadPartIdleConnTransport{ + base: baseTransport, + part: "1", + } + baseClient.Transport = transport + + baseURL, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("parse url: %v", err) + } + client := costypes.NewClient( + &costypes.BaseURL{BucketURL: baseURL}, + baseClient, + ) + client.Conf.EnableCRC = false + sdk := &QCloudSDK{ + name: "cos-retry-idle-conn-test", + client: client, + } + + data := bytes.Repeat([]byte("r"), int(minMultipartPartSize+1)) + size := int64(len(data)) + if err := sdk.WriteMultipartParallel(context.Background(), "object", bytes.NewReader(data), &size, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 1, + }); err != nil { + t.Fatalf("write failed after transient upload-part error: %v", err) + } + if transport.uploadPartCalls.Load() < 2 { + t.Fatalf("expected upload part retry, got %d calls", transport.uploadPartCalls.Load()) + } + if !state.completed.Load() { + t.Fatalf("expected multipart upload complete") + } +} + +type cosUploadPartIdleConnTransport struct { + base http.RoundTripper + part string + uploadPartCalls atomic.Int32 +} + +func (t *cosUploadPartIdleConnTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method == http.MethodPut && req.URL.Query().Get("partNumber") == t.part { + if t.uploadPartCalls.Add(1) == 1 { + return nil, fmt.Errorf("http: server closed idle connection") + } + } + return t.base.RoundTrip(req) +} + func TestCOSParallelMultipartUnknownSize(t *testing.T) { server, state := newMockCOSServer(t, 0) defer server.Close() diff --git a/pkg/fileservice/qcloud_sdk_test.go b/pkg/fileservice/qcloud_sdk_test.go index e7ab010ea80b7..62c3a5e70aa5d 100644 --- a/pkg/fileservice/qcloud_sdk_test.go +++ b/pkg/fileservice/qcloud_sdk_test.go @@ -183,6 +183,13 @@ func TestWrappedNetTimeoutRetryable(t *testing.T) { } } +func TestServerClosedIdleConnectionRetryable(t *testing.T) { + err := fmt.Errorf(`Put "https://bucket.cos.ap-guangzhou.myqcloud.com/object?partNumber=716&uploadId=id": http: server closed idle connection`) + if !IsRetryableError(err) { + t.Fatalf("expected server closed idle connection to be retryable") + } +} + type timeoutError struct{} func (timeoutError) Error() string { From 15cf34b60317860ae84eb6db317096e925d6ae71 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 2 Jul 2026 16:48:54 +0800 Subject: [PATCH 72/76] fix checkpoint tool sca errors --- pkg/sql/colexec/external/external.go | 9 --------- pkg/tools/checkpointtool/table_dump.go | 4 ++-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/pkg/sql/colexec/external/external.go b/pkg/sql/colexec/external/external.go index e3001ef0eb9f5..6d5f566c0cecf 100644 --- a/pkg/sql/colexec/external/external.go +++ b/pkg/sql/colexec/external/external.go @@ -1432,15 +1432,6 @@ func getColData(bat *batch.Batch, line []csvparser.Field, rowIdx int, param *Ext if err := vector.AppendFixed(vec, d, false, mp); err != nil { return err } - case types.T_year: - d, err := types.ParseMoYear(field.Val) - if err != nil { - logutil.Errorf("parse field[%v] err:%v", field.Val, err) - return moerr.NewInternalErrorf(param.Ctx, "the input value '%v' is not Year type for column %d", field.Val, colIdx) - } - if err := vector.AppendFixed(vec, d, false, mp); err != nil { - return err - } case types.T_uuid: d, err := types.ParseUuid(field.Val) if err != nil { diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 1e067fbad1d4b..4fad8e44eca9d 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -284,7 +284,7 @@ func ParseCSVRowOrder(s string) (CSVRowOrder, error) { case CSVRowOrderLexical: return CSVRowOrderLexical, nil default: - return "", fmt.Errorf("unsupported row order %q (supported: %s, %s)", s, CSVRowOrderStorage, CSVRowOrderLexical) + return "", moerr.NewInvalidInputf(context.Background(), "unsupported row order %q (supported: %s, %s)", s, CSVRowOrderStorage, CSVRowOrderLexical) } } @@ -4244,7 +4244,7 @@ func (r *CheckpointReader) findCatalogTableIDForAccount( ) (uint64, bool, error) { moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) if err != nil { - return 0, false, fmt.Errorf("read mo_tables: %w", err) + return 0, false, moerr.NewInternalErrorf(ctx, "read mo_tables: %v", err) } var fallback uint64 tables := buildCatalogTablesFromMoTablesRows(moTablesView) From e168dc6191c568e280d42132ac369248926de0d0 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 3 Jul 2026 11:07:38 +0800 Subject: [PATCH 73/76] fix checkpoint tool static checks --- cmd/mo-object-tool/ckp/checkpoint.go | 4 - pkg/tools/checkpointtool/table_dump.go | 192 +------------------- pkg/tools/checkpointtool/table_dump_test.go | 32 +--- pkg/tools/toolfs/storage.go | 2 +- 4 files changed, 9 insertions(+), 221 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 7ae633bfb69b8..ea5a163b6f451 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -386,10 +386,6 @@ func newFileServiceWriteCloser(ctx context.Context, fs fileservice.FileService, go func() { var err error defer func() { - if recovered := recover(); recovered != nil { - err = fmt.Errorf("write object %s panic: %v", filePath, recovered) - _ = pr.CloseWithError(err) - } w.done <- err }() ctx = fileservice.WithParallelMode(ctx, fileservice.ParallelAuto) diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index 4fad8e44eca9d..f54617904c551 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -1090,9 +1090,6 @@ func (r *CheckpointReader) ReadTableSchema( } // Try to read mo_columns from the checkpoint (table 3) - if schema.TableName == fmt.Sprintf("%d", tableID) { - // Couldn't get table name from mo_tables; still try for columns - } moColumnsView, err := r.getTableLogicalView(ctx, moColumnsID, snapshotTS) if err == nil && moColumnsView != nil { cols := buildColumnsFromMoColumnsRows(moColumnsView, tableID) @@ -1174,7 +1171,7 @@ func (r *CheckpointReader) getTableEntriesAt( type entryRef struct { entry *EntryInfo } - var entryRefs []entryRef + entryRefs := make([]entryRef, 0, 1+len(composed.Incrementals)) if composed.BaseEntry != nil { entryRefs = append(entryRefs, entryRef{entry: composed.BaseEntry}) } @@ -1574,7 +1571,7 @@ func (r *CheckpointReader) listDatabasesFromMoTables( return nil, err } seen := make(map[string]struct{}) - var databases []TableCatalogEntry + databases := make([]TableCatalogEntry, 0, len(tables)) for _, t := range tables { key := fmt.Sprintf("%d\x00%d\x00%s", t.AccountID, t.DatabaseID, strings.ToLower(t.DatabaseName)) if _, ok := seen[key]; ok { @@ -1812,7 +1809,7 @@ func (r *CheckpointReader) streamTableCSV( } needsBuffer := options.IncludeMetadata || options.RowOrder == CSVRowOrderLexical - var output io.Writer = w + output := w var tmpFile *os.File if needsBuffer { var err error @@ -2195,132 +2192,6 @@ func (r *CheckpointReader) processCSVObjectChunks( return nil } -func (r *CheckpointReader) readCSVObjectBlocks( - ctx context.Context, - blocks chan<- csvPipelineBlock, - counters *csvPipelineCounters, - snapshotTS types.TS, - tombstoneStats []objectio.ObjectStats, - columnSeqNums []int, - job csvPipelineObjectJob, -) error { - entry := job.entry - if err := ctx.Err(); err != nil { - return err - } - objName := entry.ObjectStats.ObjectName().String() - reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) - if err != nil { - if isDataFileNotFound(err) { - counters.processedObjects.Add(1) - return nil - } - return err - } - - physicalPositions := dataIndexesForSeqNums(reader.Columns(), columnSeqNums) - projectedTypes := buildProjectedTypes(reader.Columns(), physicalPositions) - relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.ObjectStats.ObjectName().ObjectId(), tombstoneStats) - if err != nil { - _ = reader.Close() - return err - } - - for blockIdx := 0; blockIdx < int(entry.ObjectStats.BlkCnt()); blockIdx++ { - if err := ctx.Err(); err != nil { - _ = reader.Close() - return err - } - if err := waitForCSVMemory(ctx, counters); err != nil { - _ = reader.Close() - return err - } - - start := time.Now() - bat, release, err := reader.ReadBlock(ctx, uint32(blockIdx)) - if err != nil { - _ = reader.Close() - return err - } - commitTSVec, releaseCommitTS, err := reader.ReadBlockCommitTS(ctx, uint32(blockIdx)) - counters.readNanos.Add(time.Since(start).Nanoseconds()) - if err != nil { - release() - _ = reader.Close() - return err - } - if bat.RowCount() == 0 { - if releaseCommitTS != nil { - releaseCommitTS() - } - release() - continue - } - - block := csvPipelineBlock{ - objectIdx: job.objectIdx, - blockIdx: blockIdx, - entry: entry, - projectedTypes: projectedTypes, - relevantTombstones: relevantTombstones, - bat: bat, - release: release, - commitTSVec: commitTSVec, - releaseCommitTS: releaseCommitTS, - } - counters.readBatches.Add(1) - counters.readRows.Add(int64(bat.RowCount())) - counters.readQueuedBlocks.Add(1) - counters.readQueuedRows.Add(int64(bat.RowCount())) - select { - case blocks <- block: - case <-ctx.Done(): - counters.readQueuedBlocks.Add(-1) - counters.readQueuedRows.Add(-int64(bat.RowCount())) - block.releaseBlock() - _ = reader.Close() - return ctx.Err() - } - } - - if err := reader.Close(); err != nil { - return err - } - counters.processedObjects.Add(1) - return nil -} - -func (r *CheckpointReader) processCSVBlockChunk( - ctx context.Context, - chunks chan<- csvPipelineChunk, - counters *csvPipelineCounters, - snapshotTS types.TS, - physicalPositions []int, - block csvPipelineBlock, -) error { - if err := ctx.Err(); err != nil { - block.releaseBlock() - return err - } - chunk, err := r.buildCSVChunkForBlock(ctx, block, snapshotTS, physicalPositions, counters) - if err != nil { - return err - } - if len(chunk.data) == 0 { - return nil - } - counters.queuedBatches.Add(1) - counters.queuedBytes.Add(int64(len(chunk.data))) - select { - case chunks <- chunk: - return nil - case <-ctx.Done(): - counters.queuedBatches.Add(-1) - counters.queuedBytes.Add(-int64(len(chunk.data))) - return ctx.Err() - } -} - func csvPipelineWorkerCount(objects int) csvPipelineWorkerPlan { cpuLimit := runtime.GOMAXPROCS(0) if cpuLimit < 1 { @@ -3087,11 +2958,6 @@ func writeCSVMetadata(w io.Writer, schema *TableSchema, stats logicalTableStats) // and filters data rows to only include visible (non-hidden) columns by their physical position. // Returns a new LogicalTableView with merged headers and filtered data rows. func MergeLogicalViewWithSchema(view *LogicalTableView, schema *TableSchema) *LogicalTableView { - dataWidth := len(view.Headers) - logicalViewDataOffset(view) - if dataWidth < 0 { - dataWidth = 0 - } - // Without schema columns we cannot safely distinguish visible columns from hidden ones. if len(schema.Columns) == 0 { return &LogicalTableView{ @@ -3714,7 +3580,7 @@ func buildColumnsFromMoColumnsRowsAt( seqNumCol int, ) []TableColumn { tableIDStr := fmt.Sprintf("%d", tableID) - var cols []TableColumn + cols := make([]TableColumn, 0, len(view.Rows)) matchedRows := 0 typeDecodeFailures := 0 notNullCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_NullAbility) @@ -4204,30 +4070,6 @@ func applyCatalogHeadersBySeqNums(view *LogicalTableView, headers []string) { view.Headers = fixedHeaders } -func (r *CheckpointReader) dumpCatalogTableView( - ctx context.Context, - tableID uint64, - snapshotTS types.TS, -) (*LogicalTableView, error) { - var buf bytes.Buffer - if err := r.DumpTableCSVComposed(ctx, &buf, tableID, snapshotTS, WithCSVHeader(true), WithCSVMetaComments(false)); err != nil { - return nil, err - } - reader := csv.NewReader(bytes.NewReader(buf.Bytes())) - reader.FieldsPerRecord = -1 - records, err := reader.ReadAll() - if err != nil { - return nil, err - } - if len(records) == 0 { - return &LogicalTableView{}, nil - } - return &LogicalTableView{ - Headers: records[0], - Rows: records[1:], - }, nil -} - func (r *CheckpointReader) findCatalogTableID( ctx context.Context, snapshotTS types.TS, @@ -4774,12 +4616,6 @@ func decodeUniqueKeysFromMoTablesConstraint(raw string) (keys []TableUniqueKey) if raw == "" { return nil } - defer func() { - if r := recover(); r != nil { - ckpDebugSchemaf("mo_tables constraint decode panic len=%d hex=%s panic=%v", len(raw), debugHexPrefix(raw, 64), r) - keys = nil - } - }() c := &engine.ConstraintDef{} if err := c.UnmarshalBinary([]byte(raw)); err != nil { ckpDebugSchemaf("mo_tables constraint decode failed len=%d hex=%s err=%v", len(raw), debugHexPrefix(raw, 64), err) @@ -4828,12 +4664,6 @@ func decodePrimaryKeyFromMoTablesConstraint(raw string) (cols []string) { if raw == "" { return nil } - defer func() { - if r := recover(); r != nil { - ckpDebugSchemaf("mo_tables primary key decode panic len=%d hex=%s panic=%v", len(raw), debugHexPrefix(raw, 64), r) - cols = nil - } - }() c := &engine.ConstraintDef{} if err := c.UnmarshalBinary([]byte(raw)); err != nil { ckpDebugSchemaf("mo_tables primary key decode failed len=%d hex=%s err=%v", len(raw), debugHexPrefix(raw, 64), err) @@ -4877,12 +4707,6 @@ func decodeForeignKeysFromMoTablesConstraint( if raw == "" { return nil } - defer func() { - if r := recover(); r != nil { - ckpDebugSchemaf("mo_tables foreign key decode panic len=%d hex=%s panic=%v", len(raw), debugHexPrefix(raw, 64), r) - keys = nil - } - }() c := &engine.ConstraintDef{} if err := c.UnmarshalBinary([]byte(raw)); err != nil { ckpDebugSchemaf("mo_tables foreign key decode failed len=%d hex=%s err=%v", len(raw), debugHexPrefix(raw, 64), err) @@ -4961,14 +4785,6 @@ func isTruthyCatalogValue(s string) bool { return s == "1" || strings.EqualFold(s, "true") } -// getTableName tries to get the table name for a tableID from the LogicalTableView's -// mo_tables data if available, falling back to the table ID as string. -func getTableName(view *LogicalTableView, tableID uint64) string { - // view here is the table's own data view, not mo_tables. - // We can't get the name from it, so use table ID - return fmt.Sprintf("%d", tableID) -} - // buildCreateTableFromMoColumns reconstructs a CREATE TABLE DDL from mo_columns data. func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, tableNames ...string) string { tableName := fmt.Sprintf("%d", tableID) diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index 1ff899752f832..a66cff4296d14 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -68,11 +68,6 @@ func encodedConstraint(t *testing.T, constraints ...engine.Constraint) string { return string(data) } -func encodedIndexConstraint(t *testing.T, indexes ...*plan.IndexDef) string { - t.Helper() - return encodedConstraint(t, &engine.IndexDef{Indexes: indexes}) -} - func encodedPrimaryKeyConstraint(t *testing.T, names ...string) engine.Constraint { t.Helper() pkeyColName := "" @@ -1430,25 +1425,6 @@ func TestWriteCSV_LexicalRowOrder(t *testing.T) { }, lines) } -// makeColumnRow creates a mock mo_columns row with specific values at given indexes. -func makeColumnRow(indexValuePairs ...any) []string { - // 27 columns in mo_columns - row := make([]string, 27) - for i := range row { - row[i] = "" - } - for i := 0; i < len(indexValuePairs); i += 2 { - idx := indexValuePairs[i].(int) - val := indexValuePairs[i+1].(string) - row[idx] = val - } - // set a default att_relname_id if not provided - if row[4] == "" { - row[4] = "12345" - } - return row -} - // TestColumnSchemaRoundTrip tests the full pipeline: rows → columns → schema → CSV header. func TestColumnSchemaRoundTrip(t *testing.T) { moColumnsView := &LogicalTableView{ @@ -1580,7 +1556,7 @@ func TestBuiltinTableSchemaForLayout_CurrentVisibleColumnsOnly(t *testing.T) { assert.Equal(t, "mo_catalog", schema.DatabaseName) assert.NotEmpty(t, schema.CreateSQL) - var names []string + names := make([]string, 0, len(schema.Columns)) for _, col := range schema.Columns { names = append(names, col.Name) assert.Equal(t, col.Position, col.PhysicalPosition) @@ -1593,7 +1569,7 @@ func TestBuiltinTableSchemaForLayout_CurrentVisibleColumnsOnly(t *testing.T) { func TestBuiltinTableSchemaForLayout_Legacy3Compatibility(t *testing.T) { tablesSchema := builtinTableSchemaForLayout(legacy3CatalogLayout, catalog.MO_TABLES_ID) require.NotNil(t, tablesSchema) - var tableNames []string + tableNames := make([]string, 0, len(tablesSchema.Columns)) for _, col := range tablesSchema.Columns { tableNames = append(tableNames, col.Name) } @@ -1603,7 +1579,7 @@ func TestBuiltinTableSchemaForLayout_Legacy3Compatibility(t *testing.T) { columnsSchema := builtinTableSchemaForLayout(legacy3CatalogLayout, catalog.MO_COLUMNS_ID) require.NotNil(t, columnsSchema) - var columnNames []string + columnNames := make([]string, 0, len(columnsSchema.Columns)) for _, col := range columnsSchema.Columns { columnNames = append(columnNames, col.Name) } @@ -1621,7 +1597,7 @@ func TestReadTableSchema_BuiltinFallbackForSystemTable(t *testing.T) { assert.Equal(t, "mo_catalog", schema.DatabaseName) require.NotEmpty(t, schema.Columns) - var names []string + names := make([]string, 0, len(schema.Columns)) for _, col := range schema.Columns { names = append(names, col.Name) } diff --git a/pkg/tools/toolfs/storage.go b/pkg/tools/toolfs/storage.go index d82074adb37b4..406ca1d0e448e 100644 --- a/pkg/tools/toolfs/storage.go +++ b/pkg/tools/toolfs/storage.go @@ -135,7 +135,7 @@ func openFromConfig(ctx context.Context, path string, fsName string) (fileservic return fs, fmt.Sprintf("%s:%s", path, cfg.DataDir), nil } - var names []string + names := make([]string, 0, len(cfg.FileService)) for _, fsCfg := range cfg.FileService { names = append(names, fsCfg.Name) } From 3332df1721b8437cb0a2715deab8ad157c5b9791 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 3 Jul 2026 17:25:40 +0800 Subject: [PATCH 74/76] test: improve checkpoint tool coverage --- .../objecttool/interactive/unified_model.go | 6 +- .../interactive/unified_model_test.go | 189 ++++++++++++++++++ pkg/tools/toolfs/lazy_cache_fs_test.go | 105 ++++++++++ pkg/tools/toolfs/storage_test.go | 39 ++++ 4 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 pkg/tools/objecttool/interactive/unified_model_test.go diff --git a/pkg/tools/objecttool/interactive/unified_model.go b/pkg/tools/objecttool/interactive/unified_model.go index e5aa06bc84ac9..582aac292ceba 100644 --- a/pkg/tools/objecttool/interactive/unified_model.go +++ b/pkg/tools/objecttool/interactive/unified_model.go @@ -33,6 +33,10 @@ type ObjectUnifiedModel struct { cmdInput string } +var newObjectProgram = func(m tea.Model) *tea.Program { + return tea.NewProgram(m, tea.WithAltScreen()) +} + // NewObjectUnifiedModel creates a new unified model for object viewing func NewObjectUnifiedModel(ctx context.Context, reader *objecttool.ObjectReader, opts *ViewOptions) *ObjectUnifiedModel { state := NewState(ctx, reader) @@ -227,7 +231,7 @@ func runUnifiedWithReader( m := NewObjectUnifiedModel(ctx, reader, opts) for { - p := tea.NewProgram(m, tea.WithAltScreen()) + p := newObjectProgram(m) finalModel, err := p.Run() if err != nil { return err diff --git a/pkg/tools/objecttool/interactive/unified_model_test.go b/pkg/tools/objecttool/interactive/unified_model_test.go new file mode 100644 index 0000000000000..025da5a66367a --- /dev/null +++ b/pkg/tools/objecttool/interactive/unified_model_test.go @@ -0,0 +1,189 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package interactive + +import ( + "context" + "io" + "os" + "strings" + "testing" + + tea "github.com/charmbracelet/bubbletea" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/tools/objecttool" + "github.com/stretchr/testify/require" +) + +func TestObjectUnifiedModelUpdateAndCommandMode(t *testing.T) { + dir := t.TempDir() + objectPath := createTestObjectFileWithRowsAndCols(t, dir, "unified_model.obj", 3, 3) + + reader, err := objecttool.Open(context.Background(), objectPath) + require.NoError(t, err) + defer reader.Close() + + model := NewObjectUnifiedModel(context.Background(), reader, &ViewOptions{ + StartRow: 1, + EndRow: 2, + ColumnNames: map[uint16]string{0: "id"}, + ColumnExpander: &ColumnExpander{ + SourceCol: 2, + NewCols: []string{"left", "right"}, + NewTypes: []types.Type{types.T_varchar.ToType(), types.T_varchar.ToType()}, + ExpandFunc: func(any) []any { + return []any{"l", "r"} + }, + }, + ObjectNameCol: 0, + BaseDir: dir, + CustomOverview: func([][]string) string { return "overview" }, + }) + _ = model.Init() + + updated, cmd := model.Update(tea.WindowSizeMsg{Width: 100, Height: 30}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + + updated, cmd = model.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("m")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Equal(t, ViewModeBlkMeta, model.state.viewMode) + + updated, cmd = model.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("M")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Equal(t, ViewModeObjMeta, model.state.viewMode) + + updated, cmd = model.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("d")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Equal(t, ViewModeData, model.state.viewMode) + + updated, cmd = model.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(":")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.True(t, model.cmdMode) + + updated, cmd = model.handleCommandMode(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("2")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Equal(t, "2", model.cmdInput) + + updated, cmd = model.handleCommandMode(tea.KeyMsg{Type: tea.KeyBackspace}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Empty(t, model.cmdInput) + + updated, cmd = model.handleCommandMode(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("1")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + updated, cmd = model.handleCommandMode(tea.KeyMsg{Type: tea.KeyEnter}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.False(t, model.cmdMode) + require.Equal(t, int64(1), model.state.GlobalRowOffset()) + + model.state.objectToOpen = "nested.obj" + require.Equal(t, dir+"/nested.obj", model.GetObjectToOpen()) + model.ClearObjectToOpen() + require.Empty(t, model.GetObjectToOpen()) + require.Contains(t, model.View(), "overview") +} + +func TestObjectUnifiedEntrypointsReturnOpenErrors(t *testing.T) { + ctx := context.Background() + require.Error(t, Run("/path/that/does/not/exist.obj")) + + fs, err := fileservice.NewMemoryFS(defines.LocalFileServiceName, fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + require.Error(t, RunWithFS(ctx, fs, "missing.obj")) + require.Error(t, RunUnifiedWithFS(ctx, fs, "missing.obj", nil)) +} + +func TestRunUnifiedWithFSSuccessQuitsFromInput(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + createTestObjectFileWithRowsAndCols(t, dir, "run_unified.obj", 2, 2) + objectPath := dir + string(os.PathSeparator) + "run_unified.obj" + + oldNewObjectProgram := newObjectProgram + t.Cleanup(func() { newObjectProgram = oldNewObjectProgram }) + newObjectProgram = func(m tea.Model) *tea.Program { + return tea.NewProgram(m, tea.WithInput(strings.NewReader("q")), tea.WithOutput(io.Discard)) + } + + fs, err := fileservice.NewFileService(ctx, fileservice.Config{ + Name: defines.LocalFileServiceName, + Backend: "DISK", + DataDir: dir, + Cache: fileservice.DisabledCacheConfig, + }, nil) + require.NoError(t, err) + defer fs.Close(ctx) + + require.NoError(t, RunUnifiedWithFS(ctx, fs, "run_unified.obj", &ViewOptions{ + StartRow: 0, + EndRow: 1, + })) + require.NoError(t, RunUnified(ctx, objectPath, nil)) +} + +func TestStateColumnsForMetadataAndExpandedData(t *testing.T) { + state := &State{ + viewMode: ViewModeObjMeta, + metaCols: []ColInfo{ + {Idx: 0, Name: "Property", Type: types.T_varchar.ToType()}, + {Idx: 1, Name: "Value", Type: types.T_varchar.ToType()}, + }, + } + require.Len(t, state.Columns(), 2) + + state.viewMode = ViewModeData + state.colExpander = &ColumnExpander{ + SourceCol: 1, + NewCols: []string{"a", "b"}, + NewTypes: []types.Type{types.T_int32.ToType(), types.T_varchar.ToType()}, + } + cols := state.expandColumns([]objecttool.ColInfo{ + {Idx: 0, SeqNum: 0, Type: types.T_int32.ToType()}, + {Idx: 1, SeqNum: 1, Type: types.T_varchar.ToType()}, + {Idx: 2, SeqNum: 2, Type: types.T_bool.ToType()}, + }) + require.Len(t, cols, 4) + require.Equal(t, uint16(2), cols[2].Idx) + require.Equal(t, types.T_varchar, cols[2].Type.Oid) + require.Equal(t, uint16(3), cols[3].Idx) +} + +func TestParseNumber(t *testing.T) { + var n int64 + rest, err := parseNumber("123abc", &n) + require.NoError(t, err) + require.Equal(t, int64(123), n) + require.Equal(t, "abc", rest) + + rest, err = parseNumber("", &n) + require.NoError(t, err) + require.Empty(t, rest) + require.Equal(t, int64(0), n) +} + +func TestRunUnifiedOpenErrorForDirectory(t *testing.T) { + dir := t.TempDir() + require.Error(t, RunUnified(context.Background(), dir+string(os.PathSeparator)+"missing.obj", nil)) +} diff --git a/pkg/tools/toolfs/lazy_cache_fs_test.go b/pkg/tools/toolfs/lazy_cache_fs_test.go index 6f1f381f50eb8..ee13ccb29b949 100644 --- a/pkg/tools/toolfs/lazy_cache_fs_test.go +++ b/pkg/tools/toolfs/lazy_cache_fs_test.go @@ -127,6 +127,111 @@ func TestLazyCacheFSEvictsOldEntriesWhenOverLimit(t *testing.T) { require.NoFileExists(t, filepath.Join(root, "ckp/second")) } +func TestLazyCacheFSDelegatesMetadataAndInvalidatesCachedWrites(t *testing.T) { + ctx := context.Background() + remote, err := fileservice.NewMemoryFS("SHARED", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "dir/file", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: 3, + Data: []byte("old"), + }}, + })) + + fs, root, err := newLazyCacheFS(ctx, remote) + require.NoError(t, err) + defer fs.Close(ctx) + require.Equal(t, "SHARED", fs.Name()) + require.NotNil(t, fs.Cost()) + + require.NoError(t, fs.PrefetchFile(ctx, "SHARED:dir/file")) + require.FileExists(t, filepath.Join(root, "dir/file")) + + stat, err := fs.StatFile(ctx, "dir/file") + require.NoError(t, err) + require.Equal(t, int64(3), stat.Size) + + var listed []string + for entry, err := range fs.List(ctx, "dir") { + require.NoError(t, err) + listed = append(listed, entry.Name) + } + require.Contains(t, listed, "file") + + require.NoError(t, remote.Delete(ctx, "dir/file")) + require.NoError(t, fs.Write(ctx, fileservice.IOVector{ + FilePath: "SHARED:dir/file", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: 3, + Data: []byte("new"), + }}, + })) + require.NoFileExists(t, filepath.Join(root, "dir/file")) + + vec := &fileservice.IOVector{ + FilePath: "dir/file", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: -1, + }}, + } + require.NoError(t, fs.Read(ctx, vec)) + require.Equal(t, []byte("new"), vec.Entries[0].Data) + + require.NoError(t, fs.Delete(ctx, "SHARED:dir/file")) + require.NoFileExists(t, filepath.Join(root, "dir/file")) + _, err = fs.StatFile(ctx, "dir/file") + require.Error(t, err) +} + +func TestLazyCacheFSReadErrors(t *testing.T) { + ctx := context.Background() + remote, err := fileservice.NewMemoryFS("SHARED", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: 4, + Data: []byte("data"), + }}, + })) + + fs, _, err := newLazyCacheFS(ctx, remote) + require.NoError(t, err) + defer fs.Close(ctx) + + require.Error(t, fs.PrefetchFile(ctx, "missing")) + + require.Error(t, fs.Read(ctx, &fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{Offset: 0, Size: 0}}, + })) + require.Error(t, fs.Read(ctx, &fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{Offset: -1, Size: 1}}, + })) + require.Error(t, fs.Read(ctx, &fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{Offset: 10, Size: -1}}, + })) + + var buf bytes.Buffer + require.Error(t, fs.Read(ctx, &fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{ + Offset: 3, + Size: 8, + WriterForRead: &buf, + }}, + })) +} + func TestParseLazyCacheSize(t *testing.T) { tests := []struct { value string diff --git a/pkg/tools/toolfs/storage_test.go b/pkg/tools/toolfs/storage_test.go index 0f7bf2212a4d5..a7c6d194e0a26 100644 --- a/pkg/tools/toolfs/storage_test.go +++ b/pkg/tools/toolfs/storage_test.go @@ -75,6 +75,45 @@ func TestOpenFromConfigFallsBackToDataDir(t *testing.T) { assert.Contains(t, display, dir) } +func TestStorageOptionsIsRemote(t *testing.T) { + assert.False(t, (StorageOptions{}).IsRemote()) + assert.True(t, StorageOptions{FSConfig: "tn.toml"}.IsRemote()) + assert.True(t, StorageOptions{S3: "bucket=b"}.IsRemote()) + assert.True(t, StorageOptions{Backend: "minio"}.IsRemote()) +} + +func TestOpenRejectsInvalidRemoteOptions(t *testing.T) { + ctx := context.Background() + + _, _, err := Open(ctx, StorageOptions{Backend: "disk", S3: "bucket=b"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported backend") + + _, _, err = Open(ctx, StorageOptions{Backend: "s3"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing --s3") + + _, _, err = Open(ctx, StorageOptions{S3: "not-an-argument"}) + require.Error(t, err) +} + +func TestOpenFromConfigReportsMissingFileService(t *testing.T) { + dir := t.TempDir() + cfgPath := writeConfig(t, ` +[[fileservice]] +backend = "DISK" +data-dir = "`+dir+`" +name = "LOCAL" +`) + + _, _, err := Open(context.Background(), StorageOptions{ + FSConfig: cfgPath, + FSName: "MISSING", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "available: LOCAL") +} + func TestParseS3Arguments(t *testing.T) { args, err := ParseS3Arguments("bucket=b,endpoint=http://minio:9000,prefix=/dump/,key-id=k,key-secret=s", "OUT") require.NoError(t, err) From 5c0a0029a487aab5af7feabf410dd90509d042b8 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Fri, 3 Jul 2026 17:57:58 +0800 Subject: [PATCH 75/76] fix checkpoint dump restore edge cases --- cmd/mo-object-tool/ckp/checkpoint.go | 17 ++- pkg/fileservice/aws_sdk_v2.go | 12 ++ pkg/fileservice/parallel_sdk_test.go | 92 +++++++++++++ pkg/fileservice/qcloud_sdk.go | 12 ++ pkg/tools/checkpointtool/checkpoint_reader.go | 24 +++- pkg/tools/checkpointtool/table_dump.go | 127 +++++++++++++++--- pkg/tools/checkpointtool/table_dump_test.go | 58 ++++++++ 7 files changed, 311 insertions(+), 31 deletions(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index ea5a163b6f451..d206668318b33 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -555,7 +555,11 @@ Examples: if err != nil { return fmt.Errorf("create output file: %w", err) } - defer outFile.Close() + defer func() { + if outFile != nil { + _ = outFile.Close() + } + }() w = outFile } @@ -643,7 +647,7 @@ Examples: return nil } - if err := reader.DumpTableCSVComposed( + dumpErr := reader.DumpTableCSVComposed( ctx, w, tableID, @@ -651,8 +655,13 @@ Examples: checkpointtool.WithCSVMetaComments(metaComments), checkpointtool.WithCSVHeader(effectiveHeader), checkpointtool.WithCSVRowOrder(parsedRowOrder), - ); err != nil { - return fmt.Errorf("dump table %d: %w", tableID, err) + ) + if outFile != nil { + dumpErr = errors.Join(dumpErr, outFile.Close()) + outFile = nil + } + if dumpErr != nil { + return fmt.Errorf("dump table %d: %w", tableID, dumpErr) } if output != "" { diff --git a/pkg/fileservice/aws_sdk_v2.go b/pkg/fileservice/aws_sdk_v2.go index d54572f0d54cb..6217a6a281844 100644 --- a/pkg/fileservice/aws_sdk_v2.go +++ b/pkg/fileservice/aws_sdk_v2.go @@ -692,6 +692,9 @@ func (a *AwsSDKv2) WriteMultipartParallel( bufPool.Put(pendingBufPtr) bufPool.Put(nextBufPtr) if !sendJob(nil, merged, len(merged)) { + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } } else { @@ -699,8 +702,14 @@ func (a *AwsSDKv2) WriteMultipartParallel( if nextBufPtr != nil { bufPool.Put(nextBufPtr) } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 if !sendJob(nextBufPtr, nextBuf, nextN) { break } @@ -714,6 +723,9 @@ func (a *AwsSDKv2) WriteMultipartParallel( if nextBufPtr != nil { bufPool.Put(nextBufPtr) } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } pendingBufPtr = nextBufPtr diff --git a/pkg/fileservice/parallel_sdk_test.go b/pkg/fileservice/parallel_sdk_test.go index 30892bac9348c..ad1917fee4096 100644 --- a/pkg/fileservice/parallel_sdk_test.go +++ b/pkg/fileservice/parallel_sdk_test.go @@ -59,6 +59,28 @@ func (r *failAfterBytesReader) Read(p []byte) (int, error) { return n, err } +type waitAfterBytesReader struct { + r io.Reader + readSoFar int64 + waitAfter int64 + waitCh <-chan struct{} + timeout time.Duration +} + +func (r *waitAfterBytesReader) Read(p []byte) (int, error) { + if r.readSoFar >= r.waitAfter && r.waitCh != nil { + select { + case <-r.waitCh: + r.waitCh = nil + case <-time.After(r.timeout): + return 0, fmt.Errorf("timed out waiting after %d bytes", r.waitAfter) + } + } + n, err := r.r.Read(p) + r.readSoFar += int64(n) + return n, err +} + func newMockAWSServer(t *testing.T, failPart int32) (*httptest.Server, *awsServerState) { t.Helper() state := &awsServerState{ @@ -87,6 +109,9 @@ func newMockAWSServer(t *testing.T, failPart int32) (*httptest.Server, *awsServe pn, _ := strconv.Atoi(partStr) body, _ := io.ReadAll(r.Body) if state.failPart > 0 && int32(pn) == state.failPart { + if state.failPartSeen != nil { + state.failPartOnce.Do(func() { close(state.failPartSeen) }) + } w.WriteHeader(http.StatusInternalServerError) return } @@ -144,6 +169,8 @@ type awsServerState struct { parts map[int32][]byte completeBody []byte failPart int32 + failPartSeen chan struct{} + failPartOnce sync.Once failComplete bool failCreate bool uploadID string @@ -307,6 +334,36 @@ func TestAwsMultipartUploadPartError(t *testing.T) { } } +func TestAwsMultipartSendJobFailureAfterPendingChunk(t *testing.T) { + server, state := newMockAWSServer(t, 1) + defer server.Close() + state.uploadID = "uid-pending-fail" + state.failPartSeen = make(chan struct{}) + + sdk := newTestAWSClient(t, server) + data := bytes.Repeat([]byte("q"), int(minMultipartPartSize*3)) + reader := &waitAfterBytesReader{ + r: bytes.NewReader(data), + waitAfter: minMultipartPartSize * 2, + waitCh: state.failPartSeen, + timeout: 3 * time.Second, + } + size := int64(len(data)) + err := sdk.WriteMultipartParallel(context.Background(), "object", reader, &size, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 1, + }) + if err == nil { + t.Fatalf("expected upload part error") + } + if !state.aborted.Load() { + t.Fatalf("expected abort request") + } + if len(state.completeBody) > 0 { + t.Fatalf("expected no complete body") + } +} + func TestAwsParallelMultipartUnknownSize(t *testing.T) { server, state := newMockAWSServer(t, 0) defer server.Close() @@ -532,6 +589,9 @@ func newMockCOSServer(t *testing.T, failPart int) (*httptest.Server, *cosServerS pn, _ := strconv.Atoi(partStr) body, _ := io.ReadAll(r.Body) if state.failPart > 0 && pn == state.failPart { + if state.failPartSeen != nil { + state.failPartOnce.Do(func() { close(state.failPartSeen) }) + } w.WriteHeader(state.failPartStatus) return } @@ -575,6 +635,8 @@ type cosServerState struct { parts map[int][]byte completeBody []byte failPart int + failPartSeen chan struct{} + failPartOnce sync.Once failPartStatus int failComplete bool failCompleteStatus int @@ -720,6 +782,36 @@ func TestCOSMultipartUploadPartError(t *testing.T) { } } +func TestCOSMultipartSendJobFailureAfterPendingChunk(t *testing.T) { + server, state := newMockCOSServer(t, 1) + defer server.Close() + state.uploadID = "cos-uid-pending-fail" + state.failPartSeen = make(chan struct{}) + + sdk := newTestCOSClient(t, server) + data := bytes.Repeat([]byte("s"), int(minMultipartPartSize*3)) + reader := &waitAfterBytesReader{ + r: bytes.NewReader(data), + waitAfter: minMultipartPartSize * 2, + waitCh: state.failPartSeen, + timeout: 3 * time.Second, + } + size := int64(len(data)) + err := sdk.WriteMultipartParallel(context.Background(), "object", reader, &size, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 1, + }) + if err == nil { + t.Fatalf("expected upload part error") + } + if !state.aborted.Load() { + t.Fatalf("expected abort request") + } + if state.completed.Load() || len(state.completeBody) > 0 { + t.Fatalf("expected no complete multipart upload") + } +} + func TestCOSMultipartUploadPartRetriesServerClosedIdleConnection(t *testing.T) { server, state := newMockCOSServer(t, 0) defer server.Close() diff --git a/pkg/fileservice/qcloud_sdk.go b/pkg/fileservice/qcloud_sdk.go index a42b8c3b2861e..9104d3e57d9b1 100644 --- a/pkg/fileservice/qcloud_sdk.go +++ b/pkg/fileservice/qcloud_sdk.go @@ -545,6 +545,9 @@ func (a *QCloudSDK) WriteMultipartParallel( bufPool.Put(pendingBufPtr) bufPool.Put(nextBufPtr) if !sendJob(nil, merged, len(merged)) { + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } } else { @@ -552,8 +555,14 @@ func (a *QCloudSDK) WriteMultipartParallel( if nextBufPtr != nil { bufPool.Put(nextBufPtr) } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 if !sendJob(nextBufPtr, nextBuf, nextN) { break } @@ -567,6 +576,9 @@ func (a *QCloudSDK) WriteMultipartParallel( if nextBufPtr != nil { bufPool.Put(nextBufPtr) } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } pendingBufPtr = nextBufPtr diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index fe12d7d8a4997..47bf1c00001c8 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -42,6 +42,9 @@ type CheckpointReader struct { entries []*checkpoint.CheckpointEntry mp *mpool.MPool closeFS bool + + getTablesForTest func(*CheckpointReader, *checkpoint.CheckpointEntry) ([]*TableInfo, error) + getObjectEntriesForTest func(*CheckpointReader, *checkpoint.CheckpointEntry, uint64) ([]*ObjectEntryInfo, []*ObjectEntryInfo, error) } // Option configures CheckpointReader @@ -104,11 +107,13 @@ func (r *CheckpointReader) Fork(ctx context.Context) *CheckpointReader { ctx = r.ctx } return &CheckpointReader{ - ctx: ctx, - fs: r.fs, - dir: r.dir, - entries: r.entries, - mp: mpool.MustNewZero(), + ctx: ctx, + fs: r.fs, + dir: r.dir, + entries: r.entries, + mp: mpool.MustNewZero(), + getTablesForTest: r.getTablesForTest, + getObjectEntriesForTest: r.getObjectEntriesForTest, } } @@ -343,6 +348,9 @@ func (r *CheckpointReader) GetAccounts(entry *checkpoint.CheckpointEntry) ([]*Ac // GetTables extracts tables from an entry func (r *CheckpointReader) GetTables(entry *checkpoint.CheckpointEntry) ([]*TableInfo, error) { + if r.getTablesForTest != nil { + return r.getTablesForTest(r, entry) + } ranges, err := r.GetTableRanges(entry) if err != nil { return nil, err @@ -421,8 +429,7 @@ func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { // GC may have cleaned up older GCKP data files; skip those and use the next one. var baseEntry *checkpoint.CheckpointEntry var baseEntryIdx int - for i := len(r.entries) - 1; i >= 0; i-- { - e := r.entries[i] + for i, e := range r.entries { end := e.GetEnd() if e.IsGlobal() && end.LE(&ts) { tables, err := r.GetTables(e) @@ -519,6 +526,9 @@ func (r *CheckpointReader) Close() error { // GetObjectEntries reads detailed object entries with timestamps for a table func (r *CheckpointReader) GetObjectEntries(entry *checkpoint.CheckpointEntry, tableID uint64) ([]*ObjectEntryInfo, []*ObjectEntryInfo, error) { + if r.getObjectEntriesForTest != nil { + return r.getObjectEntriesForTest(r, entry, tableID) + } loc := entry.GetLocation() if loc.IsEmpty() { return nil, nil, nil diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go index f54617904c551..ed3491f6bca6f 100644 --- a/pkg/tools/checkpointtool/table_dump.go +++ b/pkg/tools/checkpointtool/table_dump.go @@ -1184,7 +1184,10 @@ func (r *CheckpointReader) getTableEntriesAt( e := r.entries[ref.entry.Index] dataEntries, tombEntries, err := r.GetObjectEntries(e, tableID) if err != nil { - continue + if isDataFileNotFound(err) { + continue + } + return nil, nil, err } allData = append(allData, dataEntries...) allTomb = append(allTomb, tombEntries...) @@ -1889,10 +1892,11 @@ func (r *CheckpointReader) streamTableCSV( } type csvPipelineChunk struct { - objectIdx int - blockIdx int - rows int - data []byte + objectIdx int + blockIdx int + rows int + data []byte + objectDone bool } type csvPipelineObjectJob struct { @@ -2108,6 +2112,19 @@ func (r *CheckpointReader) processCSVObjectChunks( columnSeqNums []int, job csvPipelineObjectJob, ) error { + sendChunk := func(chunk csvPipelineChunk) error { + select { + case chunks <- chunk: + return nil + case <-ctx.Done(): + if len(chunk.data) > 0 { + counters.queuedBatches.Add(-1) + counters.queuedBytes.Add(-int64(len(chunk.data))) + } + return ctx.Err() + } + } + entry := job.entry if err := ctx.Err(); err != nil { return err @@ -2117,7 +2134,7 @@ func (r *CheckpointReader) processCSVObjectChunks( if err != nil { if isDataFileNotFound(err) { counters.processedObjects.Add(1) - return nil + return sendChunk(csvPipelineChunk{objectIdx: job.objectIdx, objectDone: true}) } return err } @@ -2179,17 +2196,13 @@ func (r *CheckpointReader) processCSVObjectChunks( } counters.queuedBatches.Add(1) counters.queuedBytes.Add(int64(len(chunk.data))) - select { - case chunks <- chunk: - case <-ctx.Done(): - counters.queuedBatches.Add(-1) - counters.queuedBytes.Add(-int64(len(chunk.data))) - return ctx.Err() + if err := sendChunk(chunk); err != nil { + return err } } counters.processedObjects.Add(1) - return nil + return sendChunk(csvPipelineChunk{objectIdx: job.objectIdx, objectDone: true}) } func csvPipelineWorkerCount(objects int) csvPipelineWorkerPlan { @@ -2380,22 +2393,96 @@ func writeCSVChunks( cancel context.CancelFunc, done chan<- error, ) { - for chunk := range chunks { - if err := ctx.Err(); err != nil { - done <- err - return - } + type chunkKey struct { + objectIdx int + blockIdx int + } + pending := make(map[chunkKey]csvPipelineChunk) + objectDone := make(map[int]bool) + nextObjectIdx := 0 + nextBlockIdx := 0 + + writeChunk := func(chunk csvPipelineChunk) error { start := time.Now() if _, err := w.Write(chunk.data); err != nil { cancel() - done <- err - return + return err } counters.writeNanos.Add(time.Since(start).Nanoseconds()) counters.queuedBatches.Add(-1) counters.queuedBytes.Add(-int64(len(chunk.data))) counters.writtenBatches.Add(1) counters.writtenBytes.Add(int64(len(chunk.data))) + return nil + } + + nextPendingBlock := func(objectIdx, minBlockIdx int) (int, bool) { + found := false + next := 0 + for key := range pending { + if key.objectIdx != objectIdx || key.blockIdx < minBlockIdx { + continue + } + if !found || key.blockIdx < next { + found = true + next = key.blockIdx + } + } + return next, found + } + + flush := func() error { + for { + key := chunkKey{objectIdx: nextObjectIdx, blockIdx: nextBlockIdx} + chunk, ok := pending[key] + if !ok && objectDone[nextObjectIdx] { + if blockIdx, found := nextPendingBlock(nextObjectIdx, nextBlockIdx); found { + nextBlockIdx = blockIdx + key.blockIdx = blockIdx + chunk = pending[key] + ok = true + } + } + if ok { + delete(pending, key) + if err := writeChunk(chunk); err != nil { + return err + } + nextBlockIdx++ + continue + } + if objectDone[nextObjectIdx] { + delete(objectDone, nextObjectIdx) + nextObjectIdx++ + nextBlockIdx = 0 + continue + } + return nil + } + } + + for chunk := range chunks { + if err := ctx.Err(); err != nil { + done <- err + return + } + if chunk.objectDone { + objectDone[chunk.objectIdx] = true + } else { + pending[chunkKey{objectIdx: chunk.objectIdx, blockIdx: chunk.blockIdx}] = chunk + } + if err := flush(); err != nil { + done <- err + return + } + } + if err := flush(); err != nil { + done <- err + return + } + if len(pending) > 0 { + done <- moerr.NewInternalErrorNoCtxf("csv pipeline finished with %d unordered chunks still pending", len(pending)) + return } done <- nil } diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go index a66cff4296d14..ddf8de5b06a16 100644 --- a/pkg/tools/checkpointtool/table_dump_test.go +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -31,6 +31,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/sql/util/csvparser" "github.com/matrixorigin/matrixone/pkg/vm/engine" + "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -44,6 +46,62 @@ func TestCSVPipelineErrorKeepsNonCanceledRootCause(t *testing.T) { assert.NoError(t, csvPipelineError(nil, nil)) } +func TestComposeAtUsesLatestUsableGlobalCheckpoint(t *testing.T) { + older := checkpoint.NewCheckpointEntry("", types.BuildTS(1, 0), types.BuildTS(10, 0), checkpoint.ET_Global) + newer := checkpoint.NewCheckpointEntry("", types.BuildTS(11, 0), types.BuildTS(20, 0), checkpoint.ET_Global) + reader := &CheckpointReader{ + entries: []*checkpoint.CheckpointEntry{newer, older}, + getTablesForTest: func(_ *CheckpointReader, entry *checkpoint.CheckpointEntry) ([]*TableInfo, error) { + return []*TableInfo{{TableID: uint64(entry.GetEnd().Physical())}}, nil + }, + } + + view, err := reader.ComposeAt(types.BuildTS(30, 0)) + require.NoError(t, err) + require.NotNil(t, view.BaseEntry) + assert.Equal(t, 0, view.BaseEntry.Index) + assert.Contains(t, view.Tables, uint64(20)) + assert.NotContains(t, view.Tables, uint64(10)) +} + +func TestGetTableEntriesAtReturnsNonMissingObjectEntryError(t *testing.T) { + readErr := errors.New("decode checkpoint entry failed") + entry := checkpoint.NewCheckpointEntry("", types.BuildTS(1, 0), types.BuildTS(10, 0), checkpoint.ET_Global) + reader := &CheckpointReader{ + entries: []*checkpoint.CheckpointEntry{entry}, + getTablesForTest: func(_ *CheckpointReader, _ *checkpoint.CheckpointEntry) ([]*TableInfo, error) { + return []*TableInfo{{TableID: 42, DataRanges: []ckputil.TableRange{{TableID: 42}}}}, nil + }, + getObjectEntriesForTest: func(_ *CheckpointReader, _ *checkpoint.CheckpointEntry, _ uint64) ([]*ObjectEntryInfo, []*ObjectEntryInfo, error) { + return nil, nil, readErr + }, + } + + _, _, err := reader.getTableEntriesAt(context.Background(), 42, types.BuildTS(10, 0)) + require.ErrorIs(t, err, readErr) +} + +func TestWriteCSVChunksPreservesStorageOrder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + chunks := make(chan csvPipelineChunk, 5) + done := make(chan error, 1) + counters := &csvPipelineCounters{} + var buf bytes.Buffer + + go writeCSVChunks(ctx, &buf, chunks, counters, cancel, done) + chunks <- csvPipelineChunk{objectIdx: 1, blockIdx: 0, data: []byte("c\n")} + chunks <- csvPipelineChunk{objectIdx: 0, blockIdx: 1, data: []byte("b\n")} + chunks <- csvPipelineChunk{objectIdx: 0, blockIdx: 0, data: []byte("a\n")} + chunks <- csvPipelineChunk{objectIdx: 0, objectDone: true} + chunks <- csvPipelineChunk{objectIdx: 1, objectDone: true} + close(chunks) + + require.NoError(t, <-done) + assert.Equal(t, "a\nb\nc\n", buf.String()) +} + func encodedSQLType(t *testing.T, typ types.Type) string { t.Helper() data, err := types.Encode(&typ) From d1f48b25be8c24d5e5233b1204727235ba49307c Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Mon, 6 Jul 2026 10:02:13 +0800 Subject: [PATCH 76/76] fix ckp account restore script system tables --- cmd/mo-object-tool/ckp/checkpoint.go | 35 ++++++++++++++++++++++- cmd/mo-object-tool/ckp/checkpoint_test.go | 32 +++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index d206668318b33..a262669bfccfd 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -34,6 +34,7 @@ import ( "text/tabwriter" "time" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -612,10 +613,14 @@ Examples: fmt.Fprintf(cmd.OutOrStdout(), "Dumped %d tables to %s\n", len(tables), outputDir) } if loadScript { - scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, tables, dumpDataByTableID(dumpPlans), snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) + scriptTables, skippedSystemTables := filterAccountRestoreScriptTables(tables, accountIDSet) + scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, scriptTables, dumpDataByTableID(dumpPlans), snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) if err != nil { return err } + if skippedSystemTables > 0 { + fmt.Fprintf(cmd.OutOrStdout(), "Restore script skipped %d system tables that are not restorable by tenant admin; CSV files were still dumped.\n", skippedSystemTables) + } fmt.Fprintf(cmd.OutOrStdout(), "Restore script written to %s\n", scriptPath) } return nil @@ -901,6 +906,34 @@ func writeRestoreScript( return scriptPath, nil } +func filterAccountRestoreScriptTables( + tables []checkpointtool.TableCatalogEntry, + accountDump bool, +) ([]checkpointtool.TableCatalogEntry, int) { + if !accountDump { + return tables, 0 + } + filtered := make([]checkpointtool.TableCatalogEntry, 0, len(tables)) + skipped := 0 + for _, table := range tables { + if isSystemDatabase(table.DatabaseName) { + skipped++ + continue + } + filtered = append(filtered, table) + } + return filtered, skipped +} + +func isSystemDatabase(databaseName string) bool { + for _, systemDatabase := range catalog.SystemDatabases { + if strings.EqualFold(databaseName, systemDatabase) { + return true + } + } + return false +} + func orderTablesForRestore( tables []checkpointtool.TableCatalogEntry, dumpDataByTable map[uint64]*checkpointtool.TableDumpData, diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go index 2f6c4b6b582b3..75d1091e5ba6e 100644 --- a/cmd/mo-object-tool/ckp/checkpoint_test.go +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -222,6 +222,38 @@ func TestShouldWriteLoadDataSkipsExternalAndViewRelations(t *testing.T) { assert.False(t, shouldWriteLoadData(false, checkpointtool.TableCatalogEntry{RelKind: "r"})) } +func TestFilterAccountRestoreScriptTablesSkipsSystemDatabases(t *testing.T) { + tables := []checkpointtool.TableCatalogEntry{ + {DatabaseName: "mo_catalog", TableName: "mo_user"}, + {DatabaseName: "mysql", TableName: "user"}, + {DatabaseName: "system", TableName: "statement_info"}, + {DatabaseName: "system_metrics", TableName: "metric"}, + {DatabaseName: "coverage", TableName: "t1"}, + {DatabaseName: "Coverage_Extra", TableName: "t2"}, + } + + filtered, skipped := filterAccountRestoreScriptTables(tables, true) + + require.Len(t, filtered, 2) + assert.Equal(t, 4, skipped) + assert.Equal(t, "coverage", filtered[0].DatabaseName) + assert.Equal(t, "t1", filtered[0].TableName) + assert.Equal(t, "Coverage_Extra", filtered[1].DatabaseName) + assert.Equal(t, "t2", filtered[1].TableName) +} + +func TestFilterAccountRestoreScriptTablesLeavesNonAccountDumpUnchanged(t *testing.T) { + tables := []checkpointtool.TableCatalogEntry{ + {DatabaseName: "mo_catalog", TableName: "mo_user"}, + {DatabaseName: "coverage", TableName: "t1"}, + } + + filtered, skipped := filterAccountRestoreScriptTables(tables, false) + + assert.Equal(t, tables, filtered) + assert.Zero(t, skipped) +} + func TestPackageExternalTableSourceCopiesAndRewritesFilepath(t *testing.T) { tmpDir := t.TempDir() sourcePath := filepath.Join(tmpDir, "local_ext_people.csv")